From 4ae05d5821aa66214ce543bc1e1d141488e3bec6 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 16:48:50 +0800 Subject: [PATCH 001/343] feat(validation): start canonical case migration --- .github/workflows/s3-script-tests.yml | 4 + .../workflows/validation-catalog-tests.yml | 34 + README.md | 52 + docs/validation-migration-map.md | 50 + rustfs-s3-compat-test.sh | 173 +- scripts/validation_catalog.py | 208 + scripts/validation_report.py | 184 + scripts/validation_result.py | 101 + test_validation_migration.py | 188 + test_validation_s3_basic.sh | 73 + validation-cases.json | 4150 +++++++++++++++++ 11 files changed, 5212 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/validation-catalog-tests.yml create mode 100644 docs/validation-migration-map.md create mode 100644 scripts/validation_catalog.py create mode 100644 scripts/validation_report.py create mode 100644 scripts/validation_result.py create mode 100644 test_validation_migration.py create mode 100644 test_validation_s3_basic.sh create mode 100644 validation-cases.json diff --git a/.github/workflows/s3-script-tests.yml b/.github/workflows/s3-script-tests.yml index 64f74358..45aeec1e 100644 --- a/.github/workflows/s3-script-tests.yml +++ b/.github/workflows/s3-script-tests.yml @@ -6,6 +6,7 @@ on: - rustfs-s3-compat-test.sh - test_rustfs_s3_usage.py - test_rustfs_s3_signing.py + - test_validation_s3_basic.sh - .github/workflows/s3-script-tests.yml push: branches: [main] @@ -13,6 +14,7 @@ on: - rustfs-s3-compat-test.sh - test_rustfs_s3_usage.py - test_rustfs_s3_signing.py + - test_validation_s3_basic.sh - .github/workflows/s3-script-tests.yml permissions: @@ -32,3 +34,5 @@ jobs: run: python3 -m pip install botocore==1.43.19 - name: Test S3 cases and signatures without a cluster run: python3 -m unittest test_rustfs_s3_usage.py test_rustfs_s3_signing.py + - name: Test validation S3 round trip without a cluster + run: bash test_validation_s3_basic.sh diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml new file mode 100644 index 00000000..fbff1ce2 --- /dev/null +++ b/.github/workflows/validation-catalog-tests.yml @@ -0,0 +1,34 @@ +name: Validation Catalog Tests + +on: + pull_request: + paths: + - validation-cases.json + - scripts/validation_catalog.py + - scripts/validation_report.py + - scripts/validation_result.py + - test_validation_migration.py + - .github/workflows/validation-catalog-tests.yml + push: + branches: [main] + paths: + - validation-cases.json + - scripts/validation_catalog.py + - scripts/validation_report.py + - scripts/validation_result.py + - test_validation_migration.py + - .github/workflows/validation-catalog-tests.yml + +permissions: + contents: read + +jobs: + contract-tests: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - run: python3 scripts/validation_catalog.py check + - run: python3 -B -m unittest -v test_validation_migration.py diff --git a/README.md b/README.md index 84ed18c1..00ddd540 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,58 @@ # auto-testing Auto testing scripts +## Release validation migration + +`validation-cases.json` snapshots all 158 canonical cases from +`rustfs-release-validation/scenarios/catalog.yaml` (66 implemented, 84 +procedure-ready, 8 planned). The source status is historical context. A run +result carries `migration_status: verified` only after its assertions, topology, +and live evidence are checked in this repository. `reuse_candidates` names +existing suites to inspect and never grants coverage by itself. + +```bash +python3 scripts/validation_catalog.py check +python3 scripts/validation_catalog.py summary +python3 -B -m unittest -v test_validation_migration.py +``` + +To refresh after an intentional source catalog change: + +```bash +python3 scripts/validation_catalog.py sync \ + --source /absolute/path/to/rustfs-release-validation/scenarios/catalog.yaml +``` + +`scripts/validation_report.py` accepts one JSON result per canonical case ID. +Results must separately report source status, migration status, verdict, +cleanup, candidate/rc provenance, and evidence for each source assertion. +Pass `--gate-status implemented` only after the 66 migrated executors are +ready; missing or unverified cases fail that gate. Without a gate status, the +report shows migration progress and still rejects malformed or false PASS +results. + +The first executable canonical case is `s3.basic-object-round-trip`, using +the existing single-disk S3 suite deployment. It requires a candidate manifest +with `source_sha`, `package_url`, and `package_sha256`. The installed binary +SHA-256 is measured on the test node and included in result provenance: + +```bash +./rustfs-s3-compat-test.sh \ + --validation-case s3.basic-object-round-trip \ + --candidate-manifest /absolute/path/candidate.json \ + --testing-commit "$(git rev-parse HEAD)" \ + --validation-artifacts-dir /absolute/path/new-artifacts-dir \ + --yes + +python3 scripts/validation_report.py \ + --results-dir /absolute/path/new-artifacts-dir/cases \ + --output /absolute/path/new-artifacts-dir/report.md +``` + +The report's full implemented-case gate is intentionally not enabled by this +single-case run. Run it with `--gate-status implemented` only after all 66 +implemented source cases have verified executors and live results. + ## Fault-tolerance suite `rustfs-fault-tolerance-test.sh` writes a machine-readable completion result diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md new file mode 100644 index 00000000..c68127bd --- /dev/null +++ b/docs/validation-migration-map.md @@ -0,0 +1,50 @@ +# Release validation case migration + +Source: `rustfs-release-validation/scenarios/catalog.yaml`. The generated +`validation-cases.json` preserves all 158 canonical IDs, source statuses, +assertions, probes, capabilities, and target environments. Its source commit +and SHA-256 bind the snapshot to the source catalog. `reuse_candidates` lists code to inspect; +it is not a coverage claim. A case's run result remains `unverified` until a +live run proves each source assertion on the required topology. + +| Source module | Cases | Existing auto-testing code to inspect | Current gap | +| --- | ---: | --- | --- | +| Pool lifecycle | 13 | `rustfs_pool_expand.sh` | Current steps combine operations; each validation ID needs its own assertions, object manifest, and result. | +| ILM/scanner | 15 | `rustfs-tier-test.sh`, `rustfs-s3-compat-test.sh` | Existing tier/lifecycle CRUD tests do not prove the accelerated scanner deadlines and all 15 expiration outcomes. | +| Bucket replication | 18 | `rustfs-replication-test.sh` REP-101..109 | Existing cases cover some operations; source filters, exact topology, sync behavior, failure recovery, and result evidence need individual checks. | +| Site replication | 20 | `rustfs-replication-test.sh` SITE-101..106 | Existing cases cover part of setup, IAM, objects, policy, status, and removal; canonical case boundaries and missing assertions remain. | +| Heal | 84 | `rustfs_heal_test.sh`, `rustfs-fault-tolerance-test.sh` | Existing fault flows are useful helpers, but each A-J procedure requires its own fixture, fault, public assertion, and cleanup evidence. | +| Planned | 8 | KMS, Tier/Event/Audit, Security, S3, Storage suites | Existing tests may implement the behavior; compare each source assertion before promoting it. | + +`s3.basic-object-round-trip` now has a dedicated executor on `single-1disk`. +Its Bash function combines bucket creation/removal, object PUT/HEAD/GET/LIST/ +DELETE, payload SHA-256, metadata, ETag, and `NoSuchKey` checks on one fixture. +It has local mock coverage; live evidence against an immutable candidate is +still required before calling the migration verified. + +The eight planned cases have these concrete reuse candidates: + +| Validation ID | Candidate IDs | Specific item to verify | +| --- | --- | --- | +| `scanner.usage-convergence` | STOR-134 | Existing test checks 20 objects appear in Admin usage; add scanner result and fixture agreement. | +| `kms.sse-kms-round-trip` | KMS-104, KMS-301, KMS-214 | Compare key/metadata semantics and failure redaction. | +| `tiering.transition-and-restore` | TIER-105, TIER-107 | Compare remote tier restore and object checksum. | +| `events.delivery` | EVT-103, EVT-104 | Freeze and verify the exact delivery count contract. | +| `audit.request-correlation` | AUD-102, AUD-104, AUD-106 | Verify request correlation and secret redaction in the same event trail. | +| `iam.policy-enforcement` | IAM-105, IAM-109 | Verify explicit deny and allowed action on the same principal/policy fixture. | +| `oidc.login-and-session` | OIDC-103 | The live Keycloak path can be skipped by the old suite; validation requires an actual session-boundary result. | +| `s3.basic-object-round-trip` | S3C-101, 201, 203, 205, 206, 207 | Tie CRUD, payload, metadata, ETag, and error semantics to the same fixture. | + +Migration work on an ID is complete only when its result contains the exact +source assertion text in order, evidence paths for each assertion, the +candidate/source/auto-testing/rc provenance, and a clean finalizer result. +The report rejects a PASS that lacks any of those fields. Existing suite +results cannot be renamed or counted automatically. + +The runner should use the catalog's target environment as a profile, keep +one unique artifact directory per run, write `RUNNING` before invoking a case, +and retain the primary failure when cleanup also fails. Tests that mutate the +shared fleet need the RustFS workflow concurrency group and an environment +lease. Special Heal topologies and source-bound tools are provided by +separate RustFS build/environment jobs; ordinary case execution consumes +their immutable artifacts. diff --git a/rustfs-s3-compat-test.sh b/rustfs-s3-compat-test.sh index 3ff5e5c9..ec2d891e 100755 --- a/rustfs-s3-compat-test.sh +++ b/rustfs-s3-compat-test.sh @@ -20,6 +20,7 @@ # ./rustfs-s3-compat-test.sh --all-topologies --dry-run # set -Eeuo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Per-run temp dir (shared runners may leave stale /tmp files owned by other users) TEST_TMP="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-test.XXXXXX")" @@ -82,6 +83,12 @@ ASSUME_YES=0 KEEP_ENV=0 LOG_FILE="" SELECTED_TOPOLOGIES=() +VALIDATION_CASE="" +VALIDATION_ARTIFACTS_DIR="" +VALIDATION_CANDIDATE_MANIFEST="" +VALIDATION_TESTING_COMMIT="" +VALIDATION_OBJECT_CLEANUP_STATUS=PASS +VALIDATION_BINARY_SHA256="" ENDPOINT="" VOLUMES="" @@ -356,10 +363,11 @@ EOF } env_teardown() { - local topo="$1" nodes_str nodes=() + local topo="$1" nodes_str nodes=() failed=0 node read -r -a nodes <<<"$(topology_nodes "${topo}")" log "environment teardown: ${topo}" - for node in "${NODES[@]}"; do cleanup_node "${node}"; done + for node in "${NODES[@]}"; do cleanup_node "${node}" || failed=1; done + return "${failed}" } # ==================== Server-domain / virtual-hosted-style helpers ==================== @@ -475,6 +483,48 @@ run_test() { fi } +validation_write_evidence() { + local evidence_dir="${VALIDATION_ARTIFACTS_DIR}/evidence/${VALIDATION_CASE}" + local item + mkdir -p "${evidence_dir}" + cp "${TEST_TMP}/test.out" "${evidence_dir}/test.log" + for item in validation-put.json validation-head.json validation-list.json validation-missing.txt; do + if [ -f "${TEST_TMP}/${item}" ]; then cp "${TEST_TMP}/${item}" "${evidence_dir}/${item}"; fi + done +} + +validation_write_result() { + local verdict="$1" reason="$2" cleanup_status="$3" + local result_file="${VALIDATION_ARTIFACTS_DIR}/cases/${VALIDATION_CASE}.json" + local args=( + "${SCRIPT_DIR}/scripts/validation_result.py" + --case-id "${VALIDATION_CASE}" + --profile "single-1disk" + --result-file "${result_file}" + --verdict "${verdict}" + --cleanup-status "${cleanup_status}" + ) + if [ -n "${reason}" ]; then args+=(--reason "${reason}"); fi + if [ "${verdict}" != RUNNING ]; then + [ -n "${VALIDATION_CANDIDATE_MANIFEST}" ] || die "validation result requires --candidate-manifest" + [ -n "${VALIDATION_TESTING_COMMIT}" ] || die "validation result requires --testing-commit" + args+=( + --candidate-manifest "${VALIDATION_CANDIDATE_MANIFEST}" + --rc-bin "$(command -v rc)" + --testing-commit "${VALIDATION_TESTING_COMMIT}" + --binary-sha256 "${VALIDATION_BINARY_SHA256}" + ) + args+=(--evidence "evidence/${VALIDATION_CASE}/test.log") + fi + if [ "${verdict}" = PASS ]; then + local item + for item in validation-put.json validation-head.json validation-list.json validation-missing.txt; do + args+=(--evidence "evidence/${VALIDATION_CASE}/${item}") + done + fi + python3 "${args[@]}" +} + # rc helper: capture output + exit code. rc exit 7 = gated/unsupported. rc_run() { # usage: rc_run ; sets RC_OUT, RC_CODE @@ -824,6 +874,64 @@ run_all_tests() { run_test SNW-114 "ResponseAndChecksum" test_snw114 } +# Keep the validation fixture together so CRUD, ETag, metadata, and errors +# describe the same object rather than separate S3C-* fixtures. +test_validation_s3_basic_object_round_trip() { + local bucket="val-basic-${RANDOM}-$$" key="object" source="${TEST_TMP}/validation-basic-source" + local downloaded="${TEST_TMP}/validation-basic-downloaded" expected_sha expected_etag put_etag + local case_rc=0 cleanup_rc=0 + VALIDATION_OBJECT_CLEANUP_STATUS=PASS + make_test_file "${source}" 4096 + expected_sha="$(sha256sum "${source}" | awk '{print $1}')" + expected_etag="\"$(md5_of_file "${source}")\"" + + new_bucket "${bucket}" || return 1 + ( + aws_run s3api put-object --bucket "${bucket}" --key "${key}" --body "${source}" \ + --metadata validation=roundtrip --content-type application/octet-stream + assert_aws_ok || exit 1 + printf '%s\n' "${AWS_OUT}" > "${TEST_TMP}/validation-put.json" + put_etag="$(printf '%s' "${AWS_OUT}" | jq -r '.ETag // empty')" + [ "${put_etag}" = "${expected_etag}" ] || { echo "PUT ETag mismatch: ${put_etag}"; exit 1; } + + aws_run s3api head-object --bucket "${bucket}" --key "${key}" + assert_aws_ok || exit 1 + printf '%s\n' "${AWS_OUT}" > "${TEST_TMP}/validation-head.json" + printf '%s' "${AWS_OUT}" | jq -e --arg etag "${expected_etag}" \ + '.ContentLength == 4096 and .ETag == $etag and .Metadata.validation == "roundtrip" and .ContentType == "application/octet-stream"' >/dev/null \ + || { echo "HEAD fields mismatch: ${AWS_OUT}"; exit 1; } + + aws_run s3api get-object --bucket "${bucket}" --key "${key}" "${downloaded}" + assert_aws_ok || exit 1 + [ "$(sha256sum "${downloaded}" | awk '{print $1}')" = "${expected_sha}" ] \ + || { echo "GET payload mismatch"; exit 1; } + + aws_run s3api list-objects-v2 --bucket "${bucket}" + assert_aws_ok || exit 1 + printf '%s\n' "${AWS_OUT}" > "${TEST_TMP}/validation-list.json" + printf '%s' "${AWS_OUT}" | jq -e --arg key "${key}" \ + 'any(.Contents[]?; .Key == $key and .Size == 4096)' >/dev/null \ + || { echo "LIST omitted the object: ${AWS_OUT}"; exit 1; } + + aws_run s3api delete-object --bucket "${bucket}" --key "${key}" + assert_aws_ok || exit 1 + aws_run s3api get-object --bucket "${bucket}" --key "${key}" "${downloaded}" + assert_aws_fails_with "NoSuchKey" || exit 1 + printf '%s\n' "${AWS_OUT}" > "${TEST_TMP}/validation-missing.txt" + ) || case_rc=$? + + # Finalization is explicit because the old rm_bucket helper ignores errors. + aws_run s3api delete-object --bucket "${bucket}" --key "${key}" + if [ "${AWS_CODE}" -ne 0 ] && ! printf '%s' "${AWS_OUT}" | grep -qiE 'NoSuchKey|not exist'; then cleanup_rc=1; fi + aws_run s3api delete-bucket --bucket "${bucket}" + assert_aws_ok || cleanup_rc=1 + if [ "${cleanup_rc}" -ne 0 ]; then + VALIDATION_OBJECT_CLEANUP_STATUS=FAIL + echo "validation bucket cleanup failed" + fi + [ "${case_rc}" -eq 0 ] && [ "${cleanup_rc}" -eq 0 ] +} + # ---------- 1. Bucket basics ---------- test_s3c101() { @@ -2689,6 +2797,10 @@ Options: --ssh-user USER SSH user for the nodes (default azureuser) --ssh-port PORT SSH port for the nodes (default 22) --log-file FILE Append all output to FILE + --validation-case ID Run only s3.basic-object-round-trip (single-single) + --validation-artifacts-dir DIR Write canonical validation result and evidence + --candidate-manifest FILE Immutable candidate manifest for validation results + --testing-commit SHA auto-testing commit used for validation results --dry-run Preview commands without executing them -y, --yes Skip all confirmation prompts -h, --help Show this help @@ -2710,6 +2822,10 @@ main() { --ssh-user) SSH_USER="$1"; shift ;; --ssh-port) SSH_PORT="$1"; shift ;; --log-file) LOG_FILE="$1"; shift ;; + --validation-case) VALIDATION_CASE="$1"; shift ;; + --validation-artifacts-dir) VALIDATION_ARTIFACTS_DIR="$1"; shift ;; + --candidate-manifest) VALIDATION_CANDIDATE_MANIFEST="$1"; shift ;; + --testing-commit) VALIDATION_TESTING_COMMIT="$1"; shift ;; --dry-run) DRY_RUN=1 ;; -y|--yes) ASSUME_YES=1 ;; -h|--help) usage; exit 0 ;; @@ -2727,6 +2843,27 @@ main() { if [ "${all}" -eq 1 ]; then SELECTED_TOPOLOGIES=("${TOPO_SINGLE_SINGLE}" "${TOPO_SINGLE_MULTI}" "${TOPO_MULTI_MULTI}") fi + if [ -n "${VALIDATION_CASE}" ]; then + [ "${VALIDATION_CASE}" = "s3.basic-object-round-trip" ] || die "unknown validation case: ${VALIDATION_CASE}" + [ "${all}" -eq 0 ] || die "--validation-case cannot be combined with --all-topologies" + [ "${#SELECTED_TOPOLOGIES[@]}" -eq 0 ] || die "--validation-case chooses its own topology" + SELECTED_TOPOLOGIES=("${TOPO_SINGLE_SINGLE}") + [ "${KEEP_ENV}" -eq 0 ] || die "--validation-case requires environment teardown" + if [ "${DRY_RUN}" -eq 0 ]; then + [ -n "${VALIDATION_ARTIFACTS_DIR}" ] || die "--validation-case requires --validation-artifacts-dir" + [ -n "${VALIDATION_CANDIDATE_MANIFEST}" ] || die "--validation-case requires --candidate-manifest" + [ -n "${VALIDATION_TESTING_COMMIT}" ] || die "--validation-case requires --testing-commit" + [ ! -e "${VALIDATION_ARTIFACTS_DIR}" ] || die "validation artifacts directory already exists" + PACKAGE_URL="$(jq -er '.package_url' "${VALIDATION_CANDIDATE_MANIFEST}")" || die "candidate URL missing" + PACKAGE_SHA256="$(jq -er '.package_sha256' "${VALIDATION_CANDIDATE_MANIFEST}")" || die "candidate package SHA missing" + [[ "${PACKAGE_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "invalid candidate package SHA" + jq -e '.source_sha | test("^[0-9a-f]{40}$")' "${VALIDATION_CANDIDATE_MANIFEST}" >/dev/null \ + || die "candidate source commit missing" + [ "${VALIDATION_TESTING_COMMIT}" = "$(git -C "${SCRIPT_DIR}" rev-parse HEAD)" ] \ + || die "testing commit does not match this checkout" + mkdir -p "${VALIDATION_ARTIFACTS_DIR}" + fi + fi [ "${#SELECTED_TOPOLOGIES[@]}" -gt 0 ] || die "no topology selected (--all-topologies / --topology)" # aws may be installed under a user-local bin not present in the default PATH. local _p @@ -2747,21 +2884,47 @@ main() { for topo in "${SELECTED_TOPOLOGIES[@]}"; do log "== topology: ${topo} ==" CURRENT_TOPOLOGY="${topo}" + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + validation_write_result RUNNING "validation case started" PENDING + fi env_setup "${topo}" + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + VALIDATION_BINARY_SHA256="$(printf 'sha256sum /usr/bin/rustfs\n' | run_remote "${NODES[0]}" | awk '{print $1}')" + [[ "${VALIDATION_BINARY_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "installed RustFS binary SHA-256 unavailable" + fi PASS=0; FAIL=0; UNSUPPORTED=0; FAILED_TESTS=() - run_all_tests + if [ -n "${VALIDATION_CASE}" ]; then + run_test "${VALIDATION_CASE}" "BasicObjectRoundTrip" test_validation_s3_basic_object_round_trip + else + run_all_tests + fi results_summary record_topology_result "${topo}" + local cleanup_status=PASS if [ "${KEEP_ENV}" -eq 0 ] || [ "${topo}" != "${SELECTED_TOPOLOGIES[${#SELECTED_TOPOLOGIES[@]}-1]}" ]; then - env_teardown "${topo}" + env_teardown "${topo}" || cleanup_status=FAIL + fi + if [ "${VALIDATION_OBJECT_CLEANUP_STATUS}" != PASS ]; then cleanup_status=FAIL; fi + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + validation_write_evidence + if [ "${FAIL}" -eq 0 ] && [ "${UNSUPPORTED}" -eq 0 ] && [ "${cleanup_status}" = PASS ]; then + validation_write_result PASS "" PASS + elif [ "${FAIL}" -eq 0 ] && [ "${UNSUPPORTED}" -eq 0 ]; then + validation_write_result INFRA_ERROR "environment teardown failed" FAIL + else + validation_write_result FAIL "case failed or unsupported; inspect evidence" "${cleanup_status}" + fi fi if [ "${FAIL}" -gt 0 ]; then overall_fail=1 fi + if [ "${UNSUPPORTED}" -gt 0 ] || [ "${cleanup_status}" != PASS ]; then overall_fail=1; fi done final_summary [ "${overall_fail}" -eq 0 ] || exit 1 log "all done" } -main "$@" +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/scripts/validation_catalog.py b/scripts/validation_catalog.py new file mode 100644 index 00000000..436760fe --- /dev/null +++ b/scripts/validation_catalog.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Snapshot and validate the release-validation case catalog.""" + +from __future__ import annotations + +import argparse +from collections import Counter +import hashlib +import json +from pathlib import Path +import re +import subprocess +import sys + + +EXPECTED_STATUS_COUNTS = { + "implemented": 66, + "procedure-ready": 84, + "planned": 8, +} +COMMIT = re.compile(r"[0-9a-f]{40}\Z") + +# These are places to inspect, not evidence that a validation assertion passes. +MODULE_CANDIDATES = { + "pool-lifecycle": ["rustfs_pool_expand.sh"], + "heal-scanner": ["rustfs_heal_test.sh", "rustfs-fault-tolerance-test.sh"], + "heal-extended": ["rustfs_heal_test.sh", "rustfs-fault-tolerance-test.sh"], + "ilm-expiration": ["rustfs-tier-test.sh", "rustfs-s3-compat-test.sh"], + "bucket-replication": ["rustfs-replication-test.sh"], + "site-replication": ["rustfs-replication-test.sh"], + "kms": ["rustfs-kms-test.sh"], + "data-services": ["rustfs-tier-test.sh"], + "identity-access": ["rustfs-security-test.sh"], + "s3-compatibility": ["rustfs-s3-compat-test.sh"], +} + +CASE_CANDIDATES = { + "scanner.usage-convergence": ["STOR-134"], + "ilm.configuration-roundtrip": ["S3C-704"], + "ilm.transition-and-restore": ["TIER-105", "TIER-107"], + "bucket-replication.source-versioning-required": ["REP-103"], + "bucket-replication.put-object": ["REP-104"], + "bucket-replication.delete-marker": ["REP-105"], + "bucket-replication.existing-object-enabled": ["REP-107"], + "bucket-replication.multipart-object": ["REP-108"], + "site-replication.add-and-info": ["SITE-101"], + "site-replication.object-bidirectional": ["SITE-103"], + "site-replication.bucket-policy-metadata": ["SITE-104"], + "site-replication.resync": ["SITE-105"], + "site-replication.remove-all": ["SITE-106"], + "kms.sse-kms-round-trip": ["KMS-104", "KMS-301", "KMS-214"], + "tiering.transition-and-restore": ["TIER-105", "TIER-107"], + "events.delivery": ["EVT-103", "EVT-104"], + "audit.request-correlation": ["AUD-102", "AUD-104", "AUD-106"], + "iam.policy-enforcement": ["IAM-105", "IAM-109"], + "oidc.login-and-session": ["OIDC-103"], + "s3.basic-object-round-trip": [ + "S3C-101", "S3C-201", "S3C-203", "S3C-205", "S3C-206", "S3C-207", + ], +} + +CASE_SUITES = { + "scanner.usage-convergence": ["rustfs-storage-test.sh"], +} + + +def validate_snapshot(snapshot: dict) -> None: + if snapshot.get("schema") != 1: + raise ValueError("unsupported validation catalog schema") + if not isinstance(snapshot.get("source_sha256"), str) or len(snapshot["source_sha256"]) != 64: + raise ValueError("missing source catalog SHA-256") + if not isinstance(snapshot.get("source_commit"), str) or not COMMIT.fullmatch(snapshot["source_commit"]): + raise ValueError("missing source commit") + + modules = snapshot.get("modules") + if not isinstance(modules, list): + raise ValueError("modules must be a list") + ids: set[str] = set() + statuses: Counter[str] = Counter() + for module in modules: + if not module.get("id") or not isinstance(module.get("cases"), list): + raise ValueError("invalid module") + for case in module["cases"]: + case_id = case.get("id") + if not isinstance(case_id, str) or not case_id: + raise ValueError("case without an ID") + if case_id in ids: + raise ValueError(f"duplicate case ID: {case_id}") + ids.add(case_id) + status = case.get("source_status") + if status not in EXPECTED_STATUS_COUNTS: + raise ValueError(f"invalid source status for {case_id}: {status}") + statuses[status] += 1 + if not isinstance(case.get("assertions"), list) or not case["assertions"]: + raise ValueError(f"missing assertions: {case_id}") + candidates = case.get("reuse_candidates") + if not isinstance(candidates, dict) or not isinstance(candidates.get("suites"), list): + raise ValueError(f"missing reuse assessment: {case_id}") + if not isinstance(candidates.get("case_ids"), list): + raise ValueError(f"invalid reuse case IDs: {case_id}") + if dict(statuses) != EXPECTED_STATUS_COUNTS: + raise ValueError(f"unexpected case counts: {dict(statuses)}") + + +def validate_reuse_candidates(snapshot: dict, checkout: Path) -> None: + for module in snapshot["modules"]: + for case in module["cases"]: + candidates = case["reuse_candidates"] + suites = candidates["suites"] + contents = [] + for suite in suites: + if Path(suite).name != suite or not (checkout / suite).is_file(): + raise ValueError(f"missing reuse suite for {case['id']}: {suite}") + contents.append((checkout / suite).read_text(encoding="utf-8")) + for old_id in candidates["case_ids"]: + marker = re.compile(r"\brun_test\s+" + re.escape(old_id) + r"\b") + if not any(marker.search(content) for content in contents): + raise ValueError(f"missing reuse case for {case['id']}: {old_id}") + + +def build_snapshot(source: Path) -> dict: + try: + import yaml + except ImportError as exc: + raise RuntimeError("PyYAML is required only to refresh the source snapshot") from exc + + raw = source.read_bytes() + source_commit = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=source.parent.parent, + check=True, capture_output=True, text=True, + ).stdout.strip() + catalog = yaml.safe_load(raw) + if catalog.get("kind") != "ScenarioCatalog": + raise ValueError("source is not a ScenarioCatalog") + modules = [] + for module in catalog["spec"]["modules"]: + cases = [] + for source_case in module["cases"]: + case = { + "id": source_case["id"], + "source_status": source_case["status"], + "priority": source_case["priority"], + "channels": source_case["channels"], + "assertions": source_case["assertions"], + "probes": source_case.get("probes", {}), + "reuse_candidates": { + "suites": CASE_SUITES.get(source_case["id"], MODULE_CANDIDATES[module["id"]]), + "case_ids": CASE_CANDIDATES.get(source_case["id"], []), + }, + } + cases.append(case) + modules.append({ + "id": module["id"], + "target_environments": module.get("targetEnvironments", []), + "required_capabilities": module.get("requiredCapabilities", []), + "cases": cases, + }) + snapshot = { + "schema": 1, + "source": "rustfs-release-validation/scenarios/catalog.yaml", + "source_commit": source_commit, + "source_sha256": hashlib.sha256(raw).hexdigest(), + "modules": modules, + } + validate_snapshot(snapshot) + return snapshot + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("sync", "check", "summary")) + parser.add_argument("--source", type=Path, help="release-validation scenarios/catalog.yaml") + parser.add_argument( + "--catalog", type=Path, + default=Path(__file__).resolve().parent.parent / "validation-cases.json", + ) + args = parser.parse_args() + if args.command == "sync": + if args.source is None: + parser.error("sync requires --source") + snapshot = build_snapshot(args.source) + args.catalog.write_text(json.dumps(snapshot, ensure_ascii=False, indent=2) + "\n") + print(f"wrote {args.catalog}") + return 0 + + snapshot = json.loads(args.catalog.read_text()) + validate_snapshot(snapshot) + validate_reuse_candidates(snapshot, Path(__file__).resolve().parent.parent) + if args.source is not None: + current_sha = hashlib.sha256(args.source.read_bytes()).hexdigest() + if snapshot["source_sha256"] != current_sha: + raise ValueError("source catalog changed; refresh the snapshot") + if args.command == "summary": + for module in snapshot["modules"]: + statuses = Counter(case["source_status"] for case in module["cases"]) + print(f"{module['id']}: {len(module['cases'])} {dict(statuses)}") + else: + source_note = "source hash matched" if args.source is not None else "source checkout not supplied" + print(f"validation catalog: 158 unique cases, statuses 66/84/8, {source_note}") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError, RuntimeError, subprocess.CalledProcessError) as exc: + print(f"validation catalog error: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_report.py b/scripts/validation_report.py new file mode 100644 index 00000000..3678270c --- /dev/null +++ b/scripts/validation_report.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Report migration progress and gate verified validation case results.""" + +from __future__ import annotations + +import argparse +from collections import Counter +import json +from pathlib import Path +import re +import sys + +try: + from .validation_catalog import validate_snapshot +except ImportError: + from validation_catalog import validate_snapshot + + +TERMINAL = {"PASS", "FAIL", "INFRA_ERROR", "BLOCKED", "UNSUPPORTED"} +ALL_VERDICTS = TERMINAL | {"RUNNING"} +GATE_SOURCE_STATUSES = {"implemented", "procedure-ready", "planned"} +SHA256 = re.compile(r"[0-9a-f]{64}\Z") +COMMIT = re.compile(r"[0-9a-f]{40}\Z") + + +def load_results(directory: Path) -> tuple[dict[str, dict], list[str]]: + records: dict[str, dict] = {} + errors: list[str] = [] + if not directory.is_dir(): + return records, [f"missing results directory: {directory}"] + for path in sorted(directory.glob("*.json")): + try: + result = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + errors.append(f"invalid result {path.name}: {exc}") + continue + if not isinstance(result, dict): + errors.append(f"result is not an object: {path.name}") + continue + case_id = result.get("case_id") + if not isinstance(case_id, str) or not case_id: + errors.append(f"result has no case_id: {path.name}") + elif case_id in records: + errors.append(f"duplicate result: {case_id}") + else: + records[case_id] = result + return records, errors + + +def check_result( + case: dict, result: dict, *, gate: bool, artifacts_root: Path, + target_environments: list[str], +) -> list[str]: + case_id = case["id"] + errors: list[str] = [] + if result.get("schema_version") != 1: + errors.append(f"{case_id}: unsupported result schema") + verdict = result.get("verdict") + if verdict not in ALL_VERDICTS: + errors.append(f"{case_id}: invalid verdict {verdict!r}") + elif verdict == "RUNNING": + errors.append(f"{case_id}: result remained RUNNING") + if result.get("source_status") != case["source_status"]: + errors.append(f"{case_id}: source status does not match catalog") + profile = result.get("profile") + if not isinstance(profile, str) or not profile: + errors.append(f"{case_id}: missing environment profile") + elif target_environments and profile not in target_environments: + errors.append(f"{case_id}: profile {profile!r} is outside source environments") + if result.get("migration_status") not in {"unverified", "verified"}: + errors.append(f"{case_id}: invalid migration status") + if verdict == "PASS": + if result.get("migration_status") != "verified": + errors.append(f"{case_id}: PASS without verified migration") + evidence = result.get("assertions") + if not isinstance(evidence, list) or len(evidence) != len(case["assertions"]): + errors.append(f"{case_id}: PASS lacks one result per source assertion") + else: + for index, (source_assertion, item) in enumerate(zip(case["assertions"], evidence), 1): + if ( + not isinstance(item, dict) + or item.get("source_assertion") != source_assertion + or item.get("status") != "PASS" + or not isinstance(item.get("evidence"), list) + or not item["evidence"] + or any(not isinstance(path, str) or not path for path in item["evidence"]) + ): + errors.append(f"{case_id}: assertion {index} lacks matching source text or evidence") + elif any( + Path(path).is_absolute() + or ".." in Path(path).parts + or not (artifacts_root / path).is_file() + or not (artifacts_root / path).resolve().is_relative_to(artifacts_root.resolve()) + for path in item["evidence"] + ): + errors.append(f"{case_id}: assertion {index} has missing or unsafe evidence paths") + if result.get("cleanup_status") != "PASS": + errors.append(f"{case_id}: PASS requires clean environment") + provenance = result.get("provenance") + if not isinstance(provenance, dict) or not ( + isinstance(provenance.get("candidate_sha256"), str) + and SHA256.fullmatch(provenance["candidate_sha256"]) + and isinstance(provenance.get("binary_sha256"), str) + and SHA256.fullmatch(provenance["binary_sha256"]) + and isinstance(provenance.get("candidate_url"), str) + and bool(provenance["candidate_url"]) + and isinstance(provenance.get("source_commit"), str) + and COMMIT.fullmatch(provenance["source_commit"]) + and isinstance(provenance.get("testing_commit"), str) + and COMMIT.fullmatch(provenance["testing_commit"]) + and isinstance(provenance.get("rc_path"), str) + and Path(provenance["rc_path"]).is_absolute() + and isinstance(provenance.get("rc_sha256"), str) + and SHA256.fullmatch(provenance["rc_sha256"]) + ): + errors.append(f"{case_id}: PASS lacks candidate and client provenance") + if gate and verdict != "PASS": + errors.append(f"{case_id}: gate verdict is {verdict!r}") + if verdict in {"BLOCKED", "UNSUPPORTED", "INFRA_ERROR", "FAIL"} and not result.get("reason"): + errors.append(f"{case_id}: {verdict} requires a reason") + if verdict in {"PASS", "FAIL", "INFRA_ERROR"} and result.get("cleanup_status") not in {"PASS", "FAIL"}: + errors.append(f"{case_id}: terminal result requires known cleanup status") + return errors + + +def validate_results(catalog: dict, directory: Path, *, gate_statuses: set[str]) -> tuple[list[dict], list[str]]: + validate_snapshot(catalog) + cases = { + case["id"]: (case, module["target_environments"]) + for module in catalog["modules"] for case in module["cases"] + } + records, errors = load_results(directory) + for case_id in sorted(set(records) - set(cases)): + errors.append(f"unexpected case ID: {case_id}") + rows = [] + for case_id, (case, target_environments) in cases.items(): + result = records.get(case_id) + gated = case["source_status"] in gate_statuses + if result is None: + if gated: + errors.append(f"missing gate result: {case_id}") + rows.append({"case_id": case_id, "source_status": case["source_status"], "verdict": "MISSING"}) + continue + errors.extend(check_result( + case, result, gate=gated, artifacts_root=directory.parent, + target_environments=target_environments, + )) + rows.append({"case_id": case_id, "source_status": case["source_status"], "verdict": result.get("verdict")}) + return rows, errors + + +def render_report(rows: list[dict], errors: list[str]) -> str: + counts = Counter(row["verdict"] for row in rows) + lines = ["# Validation migration report", "", f"Cases: {len(rows)}", ""] + lines.extend(f"- {status}: {counts.get(status, 0)}" for status in ("PASS", "FAIL", "INFRA_ERROR", "BLOCKED", "UNSUPPORTED", "RUNNING", "MISSING")) + lines.extend(["", "| Case | Source status | Verdict |", "| --- | --- | --- |"]) + for row in rows: + lines.append(f"| {row['case_id']} | {row['source_status']} | {row['verdict']} |") + if errors: + lines.extend(["", "## Validation errors", ""]) + lines.extend(f"- {error}" for error in errors) + return "\n".join(lines) + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--catalog", type=Path, default=Path(__file__).resolve().parent.parent / "validation-cases.json") + parser.add_argument("--results-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--gate-status", action="append", choices=sorted(GATE_SOURCE_STATUSES), default=[]) + args = parser.parse_args() + try: + catalog = json.loads(args.catalog.read_text(encoding="utf-8")) + rows, errors = validate_results(catalog, args.results_dir, gate_statuses=set(args.gate_status)) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(render_report(rows, errors), encoding="utf-8") + except (OSError, ValueError, json.JSONDecodeError) as exc: + print(f"validation report error: {exc}", file=sys.stderr) + return 1 + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validation_result.py b/scripts/validation_result.py new file mode 100644 index 00000000..2fb24f12 --- /dev/null +++ b/scripts/validation_result.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Atomically write one canonical release-validation case result.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import sys +import tempfile + +from validation_catalog import validate_snapshot + + +def write_result(path: Path, value: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as stream: + temporary = Path(stream.name) + json.dump(value, stream, ensure_ascii=False, indent=2) + stream.write("\n") + os.replace(temporary, path) + + +def case_from_catalog(catalog: dict, case_id: str) -> dict: + validate_snapshot(catalog) + for module in catalog["modules"]: + for case in module["cases"]: + if case["id"] == case_id: + return case + raise ValueError(f"unknown validation case: {case_id}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--catalog", type=Path, default=Path(__file__).resolve().parent.parent / "validation-cases.json") + parser.add_argument("--case-id", required=True) + parser.add_argument("--profile", required=True) + parser.add_argument("--result-file", type=Path, required=True) + parser.add_argument("--verdict", choices=("RUNNING", "PASS", "FAIL", "INFRA_ERROR"), required=True) + parser.add_argument("--reason", default="") + parser.add_argument("--cleanup-status", default="PENDING") + parser.add_argument("--evidence", action="append", default=[]) + parser.add_argument("--candidate-manifest", type=Path) + parser.add_argument("--rc-bin", type=Path) + parser.add_argument("--testing-commit") + parser.add_argument("--binary-sha256") + args = parser.parse_args() + catalog = json.loads(args.catalog.read_text(encoding="utf-8")) + case = case_from_catalog(catalog, args.case_id) + result = { + "schema_version": 1, + "case_id": args.case_id, + "source_status": case["source_status"], + "migration_status": "unverified" if args.verdict == "RUNNING" else "verified", + "profile": args.profile, + "verdict": args.verdict, + "cleanup_status": args.cleanup_status, + } + if args.verdict != "RUNNING": + if not args.candidate_manifest or not args.rc_bin or not args.testing_commit: + parser.error("terminal results require candidate manifest, rc binary, and testing commit") + candidate = json.loads(args.candidate_manifest.read_text(encoding="utf-8")) + binary_sha256 = args.binary_sha256 or candidate.get("binary_sha256") + if not isinstance(binary_sha256, str) or len(binary_sha256) != 64 or any( + character not in "0123456789abcdef" for character in binary_sha256 + ): + parser.error("terminal results require a valid installed binary SHA-256") + if candidate.get("binary_sha256") and candidate["binary_sha256"] != binary_sha256: + parser.error("installed binary SHA-256 differs from candidate manifest") + rc_path = args.rc_bin.resolve(strict=True) + result["provenance"] = { + "candidate_url": candidate["package_url"], + "candidate_sha256": candidate["package_sha256"], + "binary_sha256": binary_sha256, + "source_commit": candidate["source_sha"], + "testing_commit": args.testing_commit, + "rc_path": str(rc_path), + "rc_sha256": hashlib.sha256(rc_path.read_bytes()).hexdigest(), + } + result["evidence"] = args.evidence + if args.verdict == "PASS": + if not args.evidence or args.cleanup_status != "PASS": + parser.error("PASS requires evidence and clean finalization") + result["assertions"] = [ + {"source_assertion": text, "status": "PASS", "evidence": args.evidence} + for text in case["assertions"] + ] + else: + result["reason"] = args.reason or "case or environment failed; inspect evidence" + write_result(args.result_file, result) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(f"validation result error: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/test_validation_migration.py b/test_validation_migration.py new file mode 100644 index 00000000..dbfcf002 --- /dev/null +++ b/test_validation_migration.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from scripts.validation_catalog import build_snapshot, validate_reuse_candidates, validate_snapshot +from scripts.validation_report import validate_results + + +ROOT = Path(__file__).resolve().parent +SOURCE = Path("/Volumes/Developer/KAI/rustfs-release-validation/scenarios/catalog.yaml") + + +class CatalogTest(unittest.TestCase): + def test_snapshot_contains_all_source_cases(self) -> None: + snapshot = json.loads((ROOT / "validation-cases.json").read_text(encoding="utf-8")) + validate_snapshot(snapshot) + validate_reuse_candidates(snapshot, ROOT) + self.assertEqual(158, sum(len(module["cases"]) for module in snapshot["modules"])) + if SOURCE.is_file(): + self.assertEqual(build_snapshot(SOURCE), snapshot) + + def test_duplicate_case_is_rejected(self) -> None: + snapshot = json.loads((ROOT / "validation-cases.json").read_text(encoding="utf-8")) + snapshot["modules"][0]["cases"].append(snapshot["modules"][0]["cases"][0]) + with self.assertRaisesRegex(ValueError, "duplicate case ID"): + validate_snapshot(snapshot) + + +class ReportTest(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.results = Path(self.tmp.name) / "cases" + self.results.mkdir() + self.catalog = json.loads((ROOT / "validation-cases.json").read_text(encoding="utf-8")) + self.case = self.catalog["modules"][0]["cases"][0] + + def tearDown(self) -> None: + self.tmp.cleanup() + + def write_result(self, name: str, **changes: object) -> None: + value = { + "schema_version": 1, + "case_id": self.case["id"], + "source_status": self.case["source_status"], + "migration_status": "unverified", + "verdict": "BLOCKED", + "reason": "executor not migrated", + "profile": "lifecycle-a", + } + value.update(changes) + (self.results / name).write_text(json.dumps(value), encoding="utf-8") + + def test_missing_implemented_case_fails_gate(self) -> None: + _, errors = validate_results(self.catalog, self.results, gate_statuses={"implemented"}) + self.assertIn(f"missing gate result: {self.case['id']}", errors) + + def test_duplicate_and_running_fail(self) -> None: + self.write_result("one.json", verdict="RUNNING") + self.write_result("two.json", verdict="RUNNING") + _, errors = validate_results(self.catalog, self.results, gate_statuses=set()) + self.assertIn(f"duplicate result: {self.case['id']}", errors) + self.assertIn(f"{self.case['id']}: result remained RUNNING", errors) + + def test_pass_needs_verified_assertions_cleanup_and_provenance(self) -> None: + self.write_result("one.json", verdict="PASS") + _, errors = validate_results(self.catalog, self.results, gate_statuses=set()) + self.assertIn(f"{self.case['id']}: PASS without verified migration", errors) + self.assertIn(f"{self.case['id']}: PASS lacks one result per source assertion", errors) + self.assertIn(f"{self.case['id']}: PASS requires clean environment", errors) + self.assertIn(f"{self.case['id']}: PASS lacks candidate and client provenance", errors) + + def test_complete_case_contract(self) -> None: + evidence = self.results.parent / "case-evidence.json" + evidence.write_text("{}", encoding="utf-8") + self.write_result( + "one.json", + verdict="PASS", + migration_status="verified", + assertions=[ + {"source_assertion": text, "status": "PASS", "evidence": ["case-evidence.json"]} + for text in self.case["assertions"] + ], + cleanup_status="PASS", + provenance={ + "candidate_sha256": "a" * 64, + "binary_sha256": "e" * 64, + "candidate_url": "https://example.invalid/rustfs.deb", + "source_commit": "b" * 40, + "testing_commit": "c" * 40, + "rc_path": "/opt/bin/rc", + "rc_sha256": "d" * 64, + }, + ) + _, errors = validate_results(self.catalog, self.results, gate_statuses=set()) + self.assertEqual([], errors) + + def test_pass_cannot_relabel_source_assertions(self) -> None: + (self.results.parent / "evidence.json").write_text("{}", encoding="utf-8") + self.write_result( + "one.json", + verdict="PASS", + migration_status="verified", + assertions=[ + {"source_assertion": "weaker assertion", "status": "PASS", "evidence": ["evidence.json"]} + for _ in self.case["assertions"] + ], + cleanup_status="PASS", + provenance={ + "candidate_sha256": "a" * 64, + "binary_sha256": "e" * 64, + "candidate_url": "https://example.invalid/rustfs.deb", + "source_commit": "b" * 40, + "testing_commit": "c" * 40, + "rc_path": "/opt/bin/rc", + "rc_sha256": "d" * 64, + }, + ) + _, errors = validate_results(self.catalog, self.results, gate_statuses=set()) + self.assertIn(f"{self.case['id']}: assertion 1 lacks matching source text or evidence", errors) + + def test_missing_evidence_file_fails(self) -> None: + self.write_result( + "one.json", + verdict="PASS", + migration_status="verified", + assertions=[ + {"source_assertion": text, "status": "PASS", "evidence": ["missing.json"]} + for text in self.case["assertions"] + ], + cleanup_status="PASS", + provenance={ + "candidate_sha256": "a" * 64, + "binary_sha256": "e" * 64, + "candidate_url": "https://example.invalid/rustfs.deb", + "source_commit": "b" * 40, + "testing_commit": "c" * 40, + "rc_path": "/opt/bin/rc", + "rc_sha256": "d" * 64, + }, + ) + _, errors = validate_results(self.catalog, self.results, gate_statuses=set()) + self.assertIn(f"{self.case['id']}: assertion 1 has missing or unsafe evidence paths", errors) + + +class ResultWriterTest(unittest.TestCase): + def test_writer_and_report_contract(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + results = root / "cases" + results.mkdir() + rc = root / "rc" + rc.write_bytes(b"test-rc") + candidate = root / "candidate.json" + candidate.write_text(json.dumps({ + "package_url": "https://example.invalid/rustfs.deb", + "package_sha256": "a" * 64, + "source_sha": "c" * 40, + }), encoding="utf-8") + evidence = root / "evidence.json" + evidence.write_text("{}", encoding="utf-8") + command = [ + sys.executable, str(ROOT / "scripts/validation_result.py"), + "--case-id", "s3.basic-object-round-trip", + "--profile", "single-1disk", + "--result-file", str(results / "s3.basic-object-round-trip.json"), + ] + subprocess.run(command + ["--verdict", "RUNNING"], check=True, capture_output=True) + self.assertEqual("RUNNING", json.loads((results / "s3.basic-object-round-trip.json").read_text())["verdict"]) + subprocess.run(command + [ + "--verdict", "PASS", "--cleanup-status", "PASS", + "--candidate-manifest", str(candidate), + "--rc-bin", str(rc), + "--testing-commit", "d" * 40, + "--binary-sha256", "b" * 64, + "--evidence", "evidence.json", + ], check=True, capture_output=True) + catalog = json.loads((ROOT / "validation-cases.json").read_text(encoding="utf-8")) + _, errors = validate_results(catalog, results, gate_statuses=set()) + self.assertEqual([], errors) + + +if __name__ == "__main__": + unittest.main() diff --git a/test_validation_s3_basic.sh b/test_validation_s3_basic.sh new file mode 100644 index 00000000..a20c0e31 --- /dev/null +++ b/test_validation_s3_basic.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/rustfs-s3-compat-test.sh" + +MOCK_DELETED=0 +MOCK_BUCKET_DELETED=0 +MOCK_BAD_METADATA=0 +MOCK_SOURCE="" + +aws_run() { + local method="${2:-}" argument + AWS_CODE=0 + AWS_OUT='{}' + case "${method}" in + create-bucket) ;; + put-object) + for argument in "$@"; do + if [ "${MOCK_EXPECT_BODY:-0}" -eq 1 ]; then MOCK_SOURCE="${argument}"; MOCK_EXPECT_BODY=0; break; fi + if [ "${argument}" = "--body" ]; then MOCK_EXPECT_BODY=1; fi + done + AWS_OUT="$(printf '{\"ETag\":\"\\\"%s\\\"\"}' "$(md5_of_file "${MOCK_SOURCE}")")" + ;; + head-object) + if [ "${MOCK_BAD_METADATA}" -eq 1 ]; then + AWS_OUT='{"ContentLength":4096,"ETag":"wrong","Metadata":{"validation":"wrong"},"ContentType":"application/octet-stream"}' + else + AWS_OUT="$(printf '{\"ContentLength\":4096,\"ETag\":\"\\\"%s\\\"\",\"Metadata\":{\"validation\":\"roundtrip\"},\"ContentType\":\"application/octet-stream\"}' "$(md5_of_file "${MOCK_SOURCE}")")" + fi + ;; + get-object) + if [ "${MOCK_DELETED}" -eq 1 ]; then + AWS_CODE=255 + AWS_OUT='NoSuchKey' + else + cp "${MOCK_SOURCE}" "${@: -1}" + fi + ;; + list-objects-v2) + AWS_OUT='{"Contents":[{"Key":"object","Size":4096}]}' + ;; + delete-object) MOCK_DELETED=1 ;; + delete-bucket) MOCK_BUCKET_DELETED=1 ;; + *) printf 'unexpected AWS call: %s\n' "${method}" >&2; return 1 ;; + esac +} + +test_validation_s3_basic_object_round_trip +[ "${MOCK_BUCKET_DELETED}" -eq 1 ] + +MOCK_DELETED=0 +MOCK_BUCKET_DELETED=0 +MOCK_BAD_METADATA=1 +if test_validation_s3_basic_object_round_trip; then + printf 'validation case accepted bad HEAD metadata\n' >&2 + exit 1 +fi +[ "${MOCK_BUCKET_DELETED}" -eq 1 ] || { printf 'failure did not clean bucket\n' >&2; exit 1; } + +"${SCRIPT_DIR}/rustfs-s3-compat-test.sh" --validation-case s3.basic-object-round-trip \ + --dry-run --yes --package-url https://example.invalid/rustfs.deb \ + > "${TEST_TMP}/validation-dry-run.log" +grep -q '\[DRY-RUN\] s3.basic-object-round-trip BasicObjectRoundTrip' "${TEST_TMP}/validation-dry-run.log" +if grep -q '\[DRY-RUN\] S3C-' "${TEST_TMP}/validation-dry-run.log"; then + printf 'validation mode scheduled legacy S3C cases\n' >&2 + exit 1 +fi +if "${SCRIPT_DIR}/rustfs-s3-compat-test.sh" --validation-case unknown \ + --dry-run --yes > "${TEST_TMP}/validation-bad-case.log" 2>&1; then + printf 'unknown validation case was accepted\n' >&2 + exit 1 +fi diff --git a/validation-cases.json b/validation-cases.json new file mode 100644 index 00000000..07b20057 --- /dev/null +++ b/validation-cases.json @@ -0,0 +1,4150 @@ +{ + "schema": 1, + "source": "rustfs-release-validation/scenarios/catalog.yaml", + "source_commit": "524eb5b32c3236497fdd8a9271df0494fa3d9974", + "source_sha256": "463fdd02dee43e42fef4f3aacf7506709ce87af535e1a53d91c521bc2e361cad", + "modules": [ + { + "id": "pool-lifecycle", + "target_environments": [ + "lifecycle-a" + ], + "required_capabilities": [ + "expansion", + "rebalance", + "decommission", + "restart-recovery" + ], + "cases": [ + { + "id": "pool.status", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "admin-api" + ], + "assertions": [ + "A clean seed topology reports exactly one active pool with a stable ID, non-empty command line, and internally valid capacity counters.", + "After expansion, rc and the Admin API report the same two unique pools, lifecycle states, operation states, and capacity values.", + "Pool-specific status selected by either numeric ID or exact command-line expression agrees with the corresponding list entry and no response exposes credentials." + ], + "probes": { + "rc": "rc --json admin pool list rustfs; rc --json admin pool status rustfs POOL_ID --by-id", + "adminApi": "GET /rustfs/admin/v3/pools/list" + }, + "reuse_candidates": { + "suites": [ + "rustfs_pool_expand.sh" + ], + "case_ids": [] + } + }, + { + "id": "pool.expand-and-rebalance", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "Existing objects are written and checksum-verified while only the seed pool is configured.", + "Appending the second pool makes every declared node ready and preserves pool order, IDs, and the complete logical object manifest.", + "rc expansion start is acknowledged with a non-empty operation ID that remains stable in Admin API and rc status.", + "Rebalance is observed active, completes with non-zero object, version, or byte movement, and reports no failed state, last error, cleanup warning, or incomplete stop propagation.", + "Every object remains readable with identical size, SHA-256, metadata, and ETag after rebalance." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs_pool_expand.sh" + ], + "case_ids": [] + } + }, + { + "id": "pool.expansion-restart-persistence", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "The expanded two-pool map survives a graceful full-cluster restart on the same candidate.", + "A rebalance already showing non-zero progress resumes after the restart under the same operation ID, never regresses its counters, and completes cleanly.", + "Every active node returns readiness and the same pool membership after restart.", + "Seed-pool objects remain checksum-correct when read through both expansion-pool nodes." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs_pool_expand.sh" + ], + "case_ids": [] + } + }, + { + "id": "pool.rebalance-stop-resume", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "A rebalance with real data is observed in Started state with non-zero progress before stop is issued.", + "Stop converges on all participating pools, records stoppedAt, and has no failed peers, pending terminal reload, last error, or cleanup warning.", + "The stopped operation remains stopped under the same ID after a full-cluster restart and is not auto-resumed.", + "A new start uses a different operation ID, resumes remaining work, and reaches a clean non-zero terminal completion without data loss." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs_pool_expand.sh" + ], + "case_ids": [] + } + }, + { + "id": "pool.rebalance-live-io", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "PUT, GET, HEAD, LIST, COPY, and DELETE remain available while rebalance is actively moving data.", + "The movement status remains active with non-zero progress across the online-I/O observation window.", + "Baseline and newly committed objects remain checksum-correct after terminal completion." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs_pool_expand.sh" + ], + "case_ids": [] + } + }, + { + "id": "pool.decommission", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "Seed data is created before expansion so the selected source pool provably owns customer objects.", + "Decommission start is acknowledged, becomes active, and reaches complete/decommissioned with non-zero moved objects and bytes.", + "Failed object and byte counters are zero, queued work is empty, and no unresolved entries remain.", + "The complete logical manifest is readable and checksum-correct from surviving-pool nodes." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs_pool_expand.sh" + ], + "case_ids": [] + } + }, + { + "id": "pool.decommission-cancel-clear-resume", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "Cancel is issued only after non-zero movement and its canceled/blocked state survives a full-cluster restart without corrupting committed objects or auto-resuming.", + "Clear removes only terminal decommission metadata, restores the pool to an eligible active state, and remains cleared after a second restart.", + "A subsequent start is accepted and completes cleanly with the original manifest intact." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs_pool_expand.sh" + ], + "case_ids": [] + } + }, + { + "id": "pool.decommission-restart-recovery", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "A decommission with non-zero progress survives a graceful restart of every configured node.", + "Recovery never reports failed, canceled, unresolved, or regressed movement counters and eventually reaches terminal success.", + "Objects written before the restart remain checksum-correct on the surviving pool." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs_pool_expand.sh" + ], + "case_ids": [] + } + }, + { + "id": "pool.decommission-versioned-integrity", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "Multiple immutable versions and a delete marker created on the seed pool retain their exact version IDs and payload hashes after decommission.", + "Historical versions remain directly addressable and the sole delete marker remains latest; an unversioned GET returns that marker's exact identity without resurrecting the object." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs_pool_expand.sh" + ], + "case_ids": [] + } + }, + { + "id": "pool.decommission-multipart-integrity", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "A completed multipart object retains its length, ETag, metadata, and SHA-256 after decommission.", + "An in-progress multipart upload and uploaded part remain addressable and can be completed after the source pool is retired.", + "The resumed upload disappears from the pending list and its final length, ETag, metadata, and payload are exact." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs_pool_expand.sh" + ], + "case_ids": [] + } + }, + { + "id": "pool.decommission-live-io", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "Concurrent PUT operations through multiple active endpoints succeed or retry only documented transient responses while decommission has non-zero progress.", + "Concurrent GET, HEAD, LIST, COPY, and DELETE operations remain customer-visible during movement.", + "Baseline and concurrently committed objects remain complete and checksum-correct after terminal success." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs_pool_expand.sh" + ], + "case_ids": [] + } + }, + { + "id": "pool.retired-config-removal", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "Only after successful decommission, the retired pool expression can be removed and the surviving-pool nodes can restart cleanly.", + "The resulting topology reports exactly one active pool and no reference to the retired pool command line.", + "Migrated objects and a fresh post-removal object remain readable through the stable service endpoint." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs_pool_expand.sh" + ], + "case_ids": [] + } + }, + { + "id": "pool.operation-guards", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "Status/start/cancel/clear with unknown pool IDs, idle cancel/clear, single-pool rebalance, and attempts to retire the only pool fail closed without partial lifecycle metadata.", + "Duplicate starts are rejected, decommission is rejected while rebalance is active, and rebalance is rejected while decommission is active.", + "A canceled decommission cannot restart until its terminal metadata is explicitly cleared.", + "Cancel, stop, and clear return the cluster to a clean state while preserving all committed objects." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs_pool_expand.sh" + ], + "case_ids": [] + } + } + ] + }, + { + "id": "heal-scanner", + "target_environments": [ + "single-4disk", + "cluster-1pool" + ], + "required_capabilities": [ + "heal", + "scanner" + ], + "cases": [ + { + "id": "heal.admin-healthy-noop", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "对健康对象分别执行 normal/deep，再重复查询", + "不修改用户数据或版本；Normal 只接受 `metadata_healthy`，Deep 才要求 `verified_healthy`，healthy 与 repaired 分开计数，不能把检查盘数当修复盘数。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-a", + "designCode": "A01", + "procedure": "docs/heal-cases.md#A01", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F0" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.admin-scope-isolation", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "分别以 root、bucket、prefix、object、pool/set 选择器启动", + "目标范围实际遍历；相邻前缀、其他桶及未选 set 的故障保持原状；按对象清单而非目录数量验收。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-a", + "designCode": "A02", + "procedure": "docs/heal-cases.md#A02", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DS", + "fault": "F1" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.admin-token-cross-node-status", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "从节点甲启动，经乙及稳定入口按原 token 查询", + "任务身份、scope、终态及核心计数有界收敛；状态缺失/未知不能解释为完成。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-a", + "designCode": "A03", + "procedure": "docs/heal-cases.md#A03", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DB", + "fault": "F1" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.admin-dry-run", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "对缺片与损坏对象执行 dryRun，参数化 remove/recreate", + "有明确 dry_run_observed 诊断；用户对象分片、xl.meta、版本与删除标记不变；后台修复干扰须排除。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-a", + "designCode": "A04", + "procedure": "docs/heal-cases.md#A04", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DV", + "fault": "F1,F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.admin-overlap-and-force-start", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "活动中重发同范围、父子前缀和不相交请求，再 forceStart", + "按已固定 overlap policy 验证合并返回原 token 或明确冲突；forceStart 不能泄漏旧任务责任或并发破坏对象。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-a", + "designCode": "A05", + "procedure": "docs/heal-cases.md#A05", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DS", + "fault": "F1" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.admin-stop-and-stale-token", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "非零修复进度时 stop，稳定后新建任务，再发旧 token 的查询/停止", + "停止需等在途写结束并稳定；新任务身份独立，旧请求不得终止新任务；重复 stop 按公开契约处理。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-a", + "designCode": "A06", + "procedure": "docs/heal-cases.md#A06", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F1" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.admin-invalid-request", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "发送非法 selector、重复 query、非法布尔/scanMode、冲突 flags、超长 body", + "服务端返回已核实的结构化错误；无误启动、范围扩大或持久化副作用；CLI 参数错误不能替代服务端负例。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-a", + "designCode": "A07", + "procedure": "docs/heal-cases.md#A07", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F0" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.admin-authorization", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "匿名、坏签名、过期签名、无 HealAdminAction 用户与最小授权用户分别请求", + "未授权 start/status/stop 均不能越权或泄漏对象信息；授权用户可完成规定操作，凭据不得写入证据。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-a", + "designCode": "A08", + "procedure": "docs/heal-cases.md#A08", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F0" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.admin-nolock-safety", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "通过签名 API 传 nolock=true，并与同对象提交制造确定性争用", + "兼容字段不得跳过存储层对象锁；最后成功写入和已确认删除保持正确，原生屏障与真机竞争证据分开。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-a", + "designCode": "A09", + "procedure": "docs/heal-cases.md#A09", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DL,DV", + "fault": "F1,F9" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.admin-option-contract", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "参数化 recursive、recreate、remove、updateParity、readRepair 与合法/非法 pool/set 组合", + "仅测试已核实的候选语义；normal/deep 与危险 remove 的副作用明确；API 独有选项不得伪称 rc 已覆盖。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-a", + "designCode": "A10", + "procedure": "docs/heal-cases.md#A10", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DV", + "fault": "F1,F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.disk-loss-and-repair", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "让一个已确认 set 成员盘对 heal 服务不可用，持续读取，恢复原盘", + "记录真实故障窗口和源/目标盘身份；在可恢复条件下最终补齐目标分片及完整 manifest，不只验证降级 GET。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-b", + "designCode": "B01", + "procedure": "docs/heal-cases.md#B01", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F4" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.disk-readonly-and-permission", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "仅对独占测试卷制造只读/拒绝访问，恢复后重试", + "不可写目标不得 reported repaired；错误及待修责任可见；恢复后真实写入目标且不落到系统盘或其他目录。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-b", + "designCode": "B02", + "procedure": "docs/heal-cases.md#B02", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F4,F5" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.disk-replacement-formatted", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "换入空的新逻辑盘，确认新 disk UUID/正确格式后触发重建", + "新盘与旧盘身份可区分且属于正确 pool/set/slot；逐版本重建完成，不能从隔离备份复制回来冒充 heal。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-b", + "designCode": "B03", + "procedure": "docs/heal-cases.md#B03", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DV", + "fault": "F8" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.disk-replacement-unformatted", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "引入空未格式化测试盘，并与带异集群格式的盘分别比较", + "按候选支持的初始化契约安全接纳或明确拒绝；不得覆盖外来身份或将未就绪盘标为可写健康。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-b", + "designCode": "B04", + "procedure": "docs/heal-cases.md#B04", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F8" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.multiple-shard-loss", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "按实际同版本 EC 布局依次缺失 1..M 个 shard，覆盖 data/parity 角色组合", + "每个参数实例单独满足读取与写入 quorum；满足修复条件时全部目标重建，不把跨版本或跨 set 的 shard 合并。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-b", + "designCode": "B05", + "procedure": "docs/heal-cases.md#B05", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DV", + "fault": "F1" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.read-quorum-boundary", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "保留恰好 R 个同版本权威 shard，测试读取及修复提交，再恢复写 quorum", + "读取边界和修复写入边界分别判定；不能因还能 GET 就要求在写 quorum 不足时提交成功。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-b", + "designCode": "B06", + "procedure": "docs/heal-cases.md#B06", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F1" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.below-read-quorum", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "使权威来源小于 R，执行 GET 与 deep heal，再恢复被隔离源", + "故障期间不得返回伪造的成功内容、空对象或误删记录；明确 unresolved/失败；恢复后原对象和版本完整。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-b", + "designCode": "B07", + "procedure": "docs/heal-cases.md#B07", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DV", + "fault": "F1" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.below-write-quorum", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "构造读 quorum 尚存而写/锁 quorum 不足，执行 heal 与新 PUT", + "不能虚报持久化修复或成功写入；分别记录对象、元数据、分布式锁 quorum；恢复后核对所有已确认写入。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-b", + "designCode": "B08", + "procedure": "docs/heal-cases.md#B08", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DL", + "fault": "F4" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.node-outage-rejoin", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "停止同一跨节点 set 的一个节点，执行允许的降级 I/O，再重入", + "按实际丢失盘数判定可用性；缺失期间成功提交的对象补齐；节点重新 ready 不等于所有对象已重建。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-b", + "designCode": "B09", + "procedure": "docs/heal-cases.md#B09", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DB,DV,DL", + "fault": "F6" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.disk-and-node-flapping", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "重复短暂离线/在线并在重建途中再次失联", + "重试与队列有界、身份稳定，无重复任务风暴；最终故障解除后收敛，所有已确认对象保持正确。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-b", + "designCode": "B10", + "procedure": "docs/heal-cases.md#B10", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DB,DL", + "fault": "F4,F6" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.missing-part-file", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "仅隔离指定对象某个 part/shard，保留 format 与其他元数据，执行 deep heal", + "对应数据或校验分片重新生成，part/版本身份准确；不得只看到 xl.meta 重现就通过。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-c", + "designCode": "C01", + "procedure": "docs/heal-cases.md#C01", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DM", + "fault": "F1" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.data-shard-bitrot", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "在指定 data shard 的已知偏移翻转字节，包含首/中/尾位置", + "故障前后摘要证明注入生效；deep 或 read-repair 发现损坏并修复；独立 bitrot/EC 及 S3 全内容验证通过。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-c", + "designCode": "C02", + "procedure": "docs/heal-cases.md#C02", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DM", + "fault": "F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.parity-shard-bitrot", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "损坏 parity shard，使普通 GET 仍可能成功，再执行 deep heal", + "必须识别并重建损坏 parity；只有 GET 正确、无目标分片修复不能通过。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-c", + "designCode": "C03", + "procedure": "docs/heal-cases.md#C03", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DM", + "fault": "F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.truncated-and-zeroed-shard", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "分别截断 shard、置零和破坏校验头，保留正确元数据仲裁来源", + "短读、全零和校验损坏被准确识别；修复不能接受截断内容或错误长度。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-c", + "designCode": "C04", + "procedure": "docs/heal-cases.md#C04", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DM", + "fault": "F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.missing-or-corrupt-xlmeta", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "在少数成员上移走或损坏 xl.meta/inline 内容，保留足够权威来源", + "恢复同一对象的全部应存版本、数据引用与删除标记；不以新建空元数据替代真正修复。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-c", + "designCode": "C05", + "procedure": "docs/heal-cases.md#C05", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DV", + "fault": "F3" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.stale-version-rejoin", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "盘离线后执行覆盖/版本删除，再让携带旧元数据的盘重入", + "以已确认的新代次/版本为准；旧当前值与已删版本不复活，旧盘被纠正而非成为新的错误权威。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-c", + "designCode": "C06", + "procedure": "docs/heal-cases.md#C06", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DV,DL", + "fault": "F4" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.conflicting-subquorum-authority", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "用受控原生夹具构造互不构成 quorum 的版本/代次集合，再做对应真机反例", + "禁止把不同版本的 shard 数量并集当成权威；无法证明时安全拒绝且保留可恢复证据，不猜测较新时间戳。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-c", + "designCode": "C07", + "procedure": "docs/heal-cases.md#C07", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DV", + "fault": "F3,F9" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.metadata-payload-mismatch", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "构造元数据声称某长度/摘要但实际 part 不匹配的少数副本", + "元数据与内容联合判断；不把可解析的 xl.meta 当健康；权威不足时拒绝覆盖正确副本。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-c", + "designCode": "C08", + "procedure": "docs/heal-cases.md#C08", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DV", + "fault": "F2,F3" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.foreign-format-and-unknown-schema", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "提供外来 disk/set 身份、未来 schema 或不支持的记录格式", + "明确隔离/拒绝且不破坏原记录，不把未知当 missing 后直接重建；解除故障后原数据可用。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-c", + "designCode": "C09", + "procedure": "docs/heal-cases.md#C09", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F3,F8" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.dangling-and-orphan-safety", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "构造 dangling part、旧 data_dir 与共享历史引用，分别 remove=false/true、宽限期前后检查", + "默认不得破坏仍可恢复数据；显式 remove 只清除具有充分删除证明且过期的本次孤儿，不删共享版本或 MPU。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-c", + "designCode": "C10", + "procedure": "docs/heal-cases.md#C10", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DV,DM", + "fault": "F1,F3" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.bucket-incarnation-recreation", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "盘离线时完成旧桶合法清空/删除，再同名建桶写入新对象，恢复旧盘与旧任务", + "旧 bucket incarnation 的 checkpoint/MRF/元数据不得污染新桶；旧任务不得删除或恢复新 incarnation 的对象。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-c", + "designCode": "C11", + "procedure": "docs/heal-cases.md#C11", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DV", + "fault": "F4,F9" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.object-size-boundaries", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "覆盖 0/1 字节、页/inline/EC block 边界±1以及大对象", + "每个实际存储形态均确认后注入；长度、SHA-256、ETag、HEAD/LIST 与边界 Range 正确，无假设阈值。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-d", + "designCode": "D01", + "procedure": "docs/heal-cases.md#D01", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F1,F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.completed-multipart-object", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "修复已完成的不等长多 part 对象，分别损坏首/中/末 part", + "完整内容、part 顺序、原版本/ETag/元数据和跨 part Range 不变；multipart ETag 不能当整对象 MD5。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-d", + "designCode": "D02", + "procedure": "docs/heal-cases.md#D02", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DM", + "fault": "F1,F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.historical-versions", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "不同版本分布不同损坏位置，按对象和递归范围修复", + "所有原 versionId 可寻址且内容正确，最新版本身份不漂移，无多余版本或遗漏历史版本。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-d", + "designCode": "D03", + "procedure": "docs/heal-cases.md#D03", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DV", + "fault": "F1,F3" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.delete-marker-semantics", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "缺失/陈旧 marker 副本后 heal，并按当前值与精确 marker versionId 查询", + "当前 GET/HEAD 与指定 marker 的状态码及身份头符合契约；marker 不复活数据，LIST versions 无重复。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-d", + "designCode": "D04", + "procedure": "docs/heal-cases.md#D04", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DV", + "fault": "F3,F4" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.versioning-suspended-null", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "在版本开启、暂停、null 版本覆盖之间建夹具再修复", + "null 和非 null 版本不混淆，不新增假版本；暂停前历史对象仍准确可读。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-d", + "designCode": "D05", + "procedure": "docs/heal-cases.md#D05", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DV", + "fault": "F1,F3" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.object-metadata-and-tags", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "包含用户 metadata、Content-Type/Encoding/Disposition、tags 的对象修复", + "对外 metadata、tags 和相关响应头逐项相同；不要求无关内部元数据字节完全一致。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-d", + "designCode": "D06", + "procedure": "docs/heal-cases.md#D06", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DS", + "fault": "F1,F3" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.range-head-and-conditions", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "修复前后验证 HEAD、合法/越界 Range、If-Match/If-None-Match 及时间条件", + "200/206/304/412/416 等按逐请求契约判断，绝不返回旧内容、错误范围或错误 ETag。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-d", + "designCode": "D07", + "procedure": "docs/heal-cases.md#D07", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DM", + "fault": "F1,F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.encrypted-objects", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "按候选支持的 SSE-S3/SSE-KMS/SSE-C 建加密对象并修复", + "授权解密后原内容及版本一致，物理验证使用正确编码/密文层；密钥不可用时不误删、不降级明文，证据脱敏。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-d", + "designCode": "D08", + "procedure": "docs/heal-cases.md#D08", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DE", + "fault": "F1,F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.object-lock-retention", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "对开启 retention/legal hold 的受保护版本修复，期间测试非法删除", + "heal 不绕过保留/hold，也不丢失其元数据；拒绝删除与合法修复均有明确证据。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-d", + "designCode": "D09", + "procedure": "docs/heal-cases.md#D09", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DO,DV", + "fault": "F1,F3" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.special-keys-and-pagination", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "覆盖中文、空格、百分号、编码斜线、多层前缀和多页对象列表", + "逻辑 key 与物理路径映射准确，不跨桶/前缀；分页无遗漏重复，不能把用户 key 当可执行 shell 或裸文件路径。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-d", + "designCode": "D10", + "procedure": "docs/heal-cases.md#D10", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DS", + "fault": "F1,F3" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.bucket-metadata-integrity", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "在隔离桶白名单内测试可支持的桶元数据恢复，并核对外部 bucket 配置", + "版本设置、策略/标签/保留配置按契约保持；不得遍历凭据配置或把未知内部记录当可重建数据。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-d", + "designCode": "D11", + "procedure": "docs/heal-cases.md#D11", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DS,DV", + "fault": "F3" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.manual-normal-attribution", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "禁止预读目标，保留 format，隔离背景干扰后显式 normal heal", + "证明本次 Admin token 实际覆盖目标并修复可由 normal 检出的缺失；不要求 normal 发现所有静默 bitrot。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-e", + "designCode": "E01", + "procedure": "docs/heal-cases.md#E01", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F1" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.manual-deep-attribution", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "在普通读取仍可成功的 parity 损坏上显式 deep heal", + "按 token/对象/版本及物理变更证明 deep 检查和修复，不把后台提前修复算给手工任务。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-e", + "designCode": "E02", + "procedure": "docs/heal-cases.md#E02", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.read-repair-attribution", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "不发 Admin heal，对已确认的特定损坏对象发一次可追踪 GET/Range", + "读取正确且后续真实补片；若走异步队列需证实 source=read-repair，不能只有 GET 成功。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-e", + "designCode": "E03", + "procedure": "docs/heal-cases.md#E03", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DV", + "fault": "F1,F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.scanner-cold-object-repair", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "不 GET/HEAD 损坏目标、不发手工 heal，仅等待已启用的 scanner 路径", + "冷对象被发现并修复，来源和遍历窗口明确；scanner 周期结束但目标没检查不能通过。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-e", + "designCode": "E04", + "procedure": "docs/heal-cases.md#E04", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F1,F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.mrf-offline-write-replay", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "成员离线时完成有 quorum 的 PUT/DELETE，确认待修责任后重入", + "durable MRF 责任持久且身份精确；只在 repaired/verified healthy/authoritatively absent 等已证明结果后释放，成功提交不丢失。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-e", + "designCode": "E05", + "procedure": "docs/heal-cases.md#E05", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DL,DV", + "fault": "F4" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.replacement-auto-rebuild", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "替换盘接纳后不发 Admin/目标 GET，观察自动 replacement heal", + "替换代次、ready/completion 与盘上全部目标版本一致；不能用服务 ready、队列空或计数重置代替完成。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-e", + "designCode": "E06", + "procedure": "docs/heal-cases.md#E06", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DV", + "fault": "F8" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.root-graceful-coordinator-restart", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "root heal 非零进度时优雅重启原协调者，保留原拓扑与记录盘", + "在明确支持的边界内恢复原 task ID 和剩余责任/预算，完成后无遗留活动记录；不扩张为协调盘丢失后的故障转移。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-f", + "designCode": "F01", + "procedure": "docs/heal-cases.md#F01", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DX", + "fault": "F6" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.target-graceful-restart", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "替换目标仅完成部分重建时优雅重启目标节点", + "故障确实发生于部分完成窗口；重启后续建同一目标盘/代次，最终 manifest 与全部目标分片正确。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-f", + "designCode": "F02", + "procedure": "docs/heal-cases.md#F02", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DB,DV,DX", + "fault": "F6,F8" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.target-process-crash", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "部分重建时 SIGKILL 测试进程，再正常启动", + "unclean shutdown 与修复进度有真实证据；已提交对象不丢失，未完成目标不会被当完整；明确不是断电测试。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-f", + "designCode": "F03", + "procedure": "docs/heal-cases.md#F03", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DB,DV,DX", + "fault": "F6,F8" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.coordinator-process-crash", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "协调者非优雅退出并恢复，分别观察 root、MRF、replacement", + "按各机制已核实的持久化边界判断；不得把丢失的易失 token 当成功，需显式记录剩余责任并可安全重扫恢复。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-f", + "designCode": "F04", + "procedure": "docs/heal-cases.md#F04", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DB,DX", + "fault": "F6" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.full-cluster-restart", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "有未完成修复责任时重启全部测试节点，固定原拓扑/磁盘身份", + "有持久化保证的责任不遗失；全部端点恢复一致，旧版本/marker/MPU 与目标分片完整，重复重启不扩散错误。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-f", + "designCode": "F05", + "procedure": "docs/heal-cases.md#F05", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DB,DV,DM", + "fault": "F6" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.cancelled-task-restart", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "完成取消及在途退出后重启，随后 forceStart/新任务正常修复", + "已取消的任务不自动复活；旧 checkpoint/token 不能取消或覆盖新任务，旧责任不会误标为修复完成。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-f", + "designCode": "F06", + "procedure": "docs/heal-cases.md#F06", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DX", + "fault": "F6" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.checkpoint-and-mrf-corruption", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "受控构造 checkpoint/MRF 截断、坏校验、未来 schema、陈旧代次及写入失败", + "已确认持久化责任不得静默删除或伪 ACK；未知记录保留且可诊断；精确崩溃窗口由原生屏障证明，不伪称真机命中。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-f", + "designCode": "F07", + "procedure": "docs/heal-cases.md#F07", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DV", + "fault": "F9" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.race-unversioned-overwrite", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "同 key 连续覆盖与 heal 同时推进，包含提交尾部清理窗口", + "按可证明的提交顺序验证最终最后一次成功写入；不能回滚到旧 data_dir，未确认超时写入单独解析。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-g", + "designCode": "G01", + "procedure": "docs/heal-cases.md#G01", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DL", + "fault": "F1,F2,F9" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.race-versioned-put", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "修复历史版本时创建新版本并重复读取各 versionId", + "新旧版本不混片、不串内容；已返回成功的每个版本都保留，最新值按确定的先后关系验收。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-g", + "designCode": "G02", + "procedure": "docs/heal-cases.md#G02", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DV,DL", + "fault": "F1,F9" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.race-version-delete", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "对损坏的精确历史 versionId 发 DELETE，与 deep heal 竞争", + "已确认删除的版本不能被复活，其他历史版本不受影响；原生临界点测试与真机竞争分别留证。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-g", + "designCode": "G03", + "procedure": "docs/heal-cases.md#G03", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DV", + "fault": "F1,F2,F9" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.race-markers-and-batch-delete", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "与创建/删除 marker、批量版本删除和无版本删除分别交错", + "按逐项 DELETE 回执核对版本/marker 集合，无复活、误删或遗漏批量错误。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-g", + "designCode": "G04", + "procedure": "docs/heal-cases.md#G04", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DV", + "fault": "F3,F9" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.race-multipart-complete-abort", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "heal 期间对不同已知 UploadId 分别续传、Complete、Abort", + "成功 Complete 内容只出现一次；Abort 不破坏已提交对象或别的 upload；未完成 part 不被伪造为完整对象。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-g", + "designCode": "G05", + "procedure": "docs/heal-cases.md#G05", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DM,DL", + "fault": "F1,F9" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.race-copy-and-metadata-update", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "COPY 源/目标和标签/元数据更新与修复交错", + "最终对象内容与对应提交的 metadata 一致，读修复发现新代次应重验/放弃陈旧提交而非覆盖。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-g", + "designCode": "G06", + "procedure": "docs/heal-cases.md#G06", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DS,DL", + "fault": "F1,F9" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.race-ilm-expiry-and-transition", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "分别与当前/历史版本过期、远端 transition/restore 同窗口执行", + "已授权过期不被恢复；迁移状态与远端版本引用正确；只有实际观察到交叠才计并发覆盖，tier 未配置需明确阻断对应实例。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-g", + "designCode": "G07", + "procedure": "docs/heal-cases.md#G07", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DV,DE", + "fault": "F1,F9,F11" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.race-replication-resync", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "在隔离复制两端与 heal 同时运行新增/删除/重同步", + "对端版本、删除语义及源数据最终一致，无无限回环或重复传播；不得依赖仅源端 GET 宣称复制正确。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-g", + "designCode": "G08", + "procedure": "docs/heal-cases.md#G08", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DV,DL", + "fault": "F1,F9,F11" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.slow-io-and-eio", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "在独占故障层注入有界延迟、短读、EIO，解除后观察恢复", + "暂态与永久错误分开，重试有界且无忙循环；解除故障后实际修复，期间不得提前标健康。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-i", + "designCode": "I01", + "procedure": "docs/heal-cases.md#I01", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DX", + "fault": "F5" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.enospc-inodes-and-quota", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "限制专用测试卷空间/inode，并参数化本次桶配额", + "仅填专用受限卷、不耗尽共享真实盘；失败不破坏好副本或释放尚需容量，释放空间后正确续修。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-i", + "designCode": "I02", + "procedure": "docs/heal-cases.md#I02", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DM", + "fault": "F5,F11" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.network-partition-and-lock-quorum", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "仅隔离测试服务节点间流量，覆盖少数派、读/写/锁 quorum 边界及恢复", + "少数派不得承诺无安全性的修复或写入；cluster coverage 不完整不能显示全局健康；恢复无分叉、跨代覆盖。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-i", + "designCode": "I03", + "procedure": "docs/heal-cases.md#I03", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DB,DV,DL", + "fault": "F7" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.queue-pressure-and-admission", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "多来源请求冲击有限队列，混合重复任务、不同优先级和多个桶", + "Full/Dropped/Merged 语义与实际任务一致；丢弃不能丢 durable 责任；在预先声明负载窗口内无无限堆积或健康误报。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-i", + "designCode": "I04", + "procedure": "docs/heal-cases.md#I04", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DX", + "fault": "F11" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.lock-timeout-and-caller-abort", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "真实对象锁争用超过预算及调用方断开，再释放锁恢复", + "取消/超时不得提前释放在途所有权或重复提交；陈旧 read-repair 必须重验代次；恢复后无泄漏和错误覆盖。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-i", + "designCode": "I05", + "procedure": "docs/heal-cases.md#I05", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DL,DV", + "fault": "F5,F9" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.kms-and-tier-outage", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "仅让已分配测试 KMS/远端 tier 暂不可用，持续查询与修复", + "依赖故障不得解释为对象不存在/可删/已修复，不暴露密钥；恢复后原版本可解密或恢复，责任未丢失。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-i", + "designCode": "I06", + "procedure": "docs/heal-cases.md#I06", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DE,DV", + "fault": "F11" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.status-partial-disabled-unknown", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "分别检查 enabled/disabled/uninitialized、全体可达及缺失 peer 的状态", + "idle 只在覆盖完整且真实无活动时成立；partial/unknown/degraded/disabled 不被客户端转换为全部完成。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-j", + "designCode": "J01", + "procedure": "docs/heal-cases.md#J01", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB", + "fault": "F0,F6" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.outcome-counters-and-error-truth", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "同批次混合健康、可修、不可修、deferred、dry-run 对象", + "execution/coverage/disposition/计数和逐对象物理结果相符；unknown、skipped 或 completed_with_errors 不能算修复成功。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-j", + "designCode": "J02", + "procedure": "docs/heal-cases.md#J02", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DV", + "fault": "F1,F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.incremental-cursor-and-truncation", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "生成超过结果保留上限的任务，分页轮询 sinceSeq，并跨节点/终态读取", + "nextSeq/minSeq 单调且 gap/truncation 明示；有界 items 不是完整责任清单，最终须用独立 manifest/计数补证，不静默丢错误。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-j", + "designCode": "J03", + "procedure": "docs/heal-cases.md#J03", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DS,DX", + "fault": "F1" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.repeat-idempotency", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "第一次修复后重复 normal/deep、重启再检查", + "versioned 首次允许所有 identity repaired 或至少一个 repaired 加其余正健康态，marker 仅 repaired/metadata_healthy，计数必须与 receipt 集合一致；normal repeat 接受 `metadata_healthy`，deep repeat 要求 `verified_healthy`，repeat marker 为 `metadata_healthy` 且 healed=0；unknown/skipped/failed 不得成为 PASS。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-j", + "designCode": "J04", + "procedure": "docs/heal-cases.md#J04", + "executionMode": "operator-assisted", + "topology": "H1,H2", + "fixture": "DB,DV", + "fault": "F1" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.performance-abba", + "source_status": "procedure-ready", + "priority": "P2", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "同硬件同候选按 ABBA 运行无 heal 基线和有 heal 负载，重复并固定限速", + "分别报告修复吞吐、GET/PUT P95/P99、错误率和 CPU/IO；阈值事先冻结，不能事后挑轮次或把功能 PASS 充性能验收。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-j", + "designCode": "J06", + "procedure": "docs/heal-cases.md#J06", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DX,DL", + "fault": "F1,F11" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-convergence", + "source_status": "planned", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "Scanner and data-usage results converge to the uploaded fixture." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-storage-test.sh" + ], + "case_ids": [ + "STOR-134" + ] + } + } + ] + }, + { + "id": "heal-extended", + "target_environments": [], + "required_capabilities": [ + "heal", + "scanner" + ], + "cases": [ + { + "id": "heal.mixed-version-upgrade-rollback", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "固定前后两个明确兼容候选，覆盖滚动升级、混合 peer、回滚与重启", + "能力不足必须显示 partial/unsupported 而非完整成功；按格式兼容契约保留责任，旧版本不能误退休新版本记录。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-f", + "designCode": "F08", + "procedure": "docs/heal-cases.md#F08", + "executionMode": "operator-assisted", + "topology": "H2", + "fixture": "DB,DV", + "fault": "F6,F9" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.dedicated-host-forced-restart", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "经专项授权在部分修复/持续写入时强制重启独占测试 VM/主机", + "保留平台事件与 ACK ledger；验证落盘和恢复，不把 SIGKILL 当断电；VM 强制停止也不等同于物理控制器断电。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-f", + "designCode": "F09", + "procedure": "docs/heal-cases.md#F09", + "executionMode": "operator-assisted", + "topology": "H6", + "fixture": "DB,DV,DX", + "fault": "F10" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.with-expansion-rebalance", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "先证明源 pool 持有夹具，再扩容并与修复同时 rebalance", + "修复目标不得指向错误 pool/set；新旧数据完整、移动终态正确，保留 scanner backlog 提交权威检查。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-h", + "designCode": "H01", + "procedure": "docs/heal-cases.md#H01", + "executionMode": "operator-assisted", + "topology": "H5", + "fixture": "DB,DV", + "fault": "F1,F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.with-rebalance-stop-resume", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "heal 活动期间 stop/restart/resume rebalance", + "两类任务状态和身份独立；停止/恢复移动不误终止修复或丢失对象，恢复后两者均满足各自契约。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-h", + "designCode": "H02", + "procedure": "docs/heal-cases.md#H02", + "executionMode": "operator-assisted", + "topology": "H5", + "fixture": "DB,DL", + "fault": "F1,F6" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.with-decommission", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "分别在待退役源和存活目标制造可恢复缺片，再执行退役", + "不得提前删源或以临时副本冒充迁移成功；按 Pool 既定写入限制验收；只剩 survivor 后仍完整可读且有物理证明。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-h", + "designCode": "H03", + "procedure": "docs/heal-cases.md#H03", + "executionMode": "operator-assisted", + "topology": "H5", + "fixture": "DB,DV,DM", + "fault": "F1,F2" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.with-decommission-cancel-clear", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "heal/移动争用中取消退役、Clear、恢复，并覆盖容量锁超时恢复", + "保持持久化后通知及在途退出顺序；不误取消新代次；对 #2423 的重试窗口单独注明真机是否实际触发，不只看最终绿色。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-h", + "designCode": "H04", + "procedure": "docs/heal-cases.md#H04", + "executionMode": "operator-assisted", + "topology": "H5", + "fixture": "DB,DL", + "fault": "F1,F9" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.after-retired-pool-removal", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "源 pool 完全退役移除并停机后，只在 survivor 上触发修复", + "不再向已移除源写入或依赖其兜底；来源权威和版本正确；#2422 路径不能出现无权威覆盖或错误回退。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-h", + "designCode": "H05", + "procedure": "docs/heal-cases.md#H05", + "executionMode": "operator-assisted", + "topology": "H5", + "fixture": "DB,DV", + "fault": "F1,F3" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.multi-pool-target-isolation", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "仅破坏选定 pool/set，并在另一 pool 保留不同对象与控制故障", + "选择器、任务、容量/配额和目标 drive 一致；未选 pool 不受错误修复，跨 pool 存在不等于该目标已修复。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-h", + "designCode": "H06", + "procedure": "docs/heal-cases.md#H06", + "executionMode": "operator-assisted", + "topology": "H5", + "fixture": "DB,DV", + "fault": "F1,F4" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.ec84-multi-set-restart", + "source_status": "procedure-ready", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "固定 EC8+4 单 set 与至少两 set，部分重建时重启/崩溃目标", + "每个 set 的故障覆盖和重建分别证明，不跨 set 拼 shard；必须满足真实盘数及映射，不能用 4 盘用例替代 EC8+4。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-h", + "designCode": "H07", + "procedure": "docs/heal-cases.md#H07", + "executionMode": "operator-assisted", + "topology": "H3,H4", + "fixture": "DB,DV,DX", + "fault": "F6,F8" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "heal.long-soak", + "source_status": "procedure-ready", + "priority": "P2", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "独占环境运行预注册 24h/72h 窗口，混合修复、掉线、写入和版本删除", + "每次 ACK 可追踪；最终全量 manifest 正确，队列/内存/FD/临时空间无无界增长，无后台 panic 或未解释持久责任。" + ], + "probes": { + "design": "docs/heal-validation.md#cases-j", + "designCode": "J05", + "procedure": "docs/heal-cases.md#J05", + "executionMode": "operator-assisted", + "topology": "H2,H5", + "fixture": "DX,DL,DV", + "fault": "F4,F6,F11" + }, + "reuse_candidates": { + "suites": [ + "rustfs_heal_test.sh", + "rustfs-fault-tolerance-test.sh" + ], + "case_ids": [] + } + } + ] + }, + { + "id": "ilm-expiration", + "target_environments": [ + "ilm-2x4" + ], + "required_capabilities": [ + "lifecycle", + "scanner", + "versioning", + "multipart", + "restart-recovery" + ], + "cases": [ + { + "id": "scanner.status-readiness", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "rc", + "admin-api" + ], + "assertions": [ + "Scanner runtime settings match the accelerated test contract.", + "The leader completes a successful scanner cycle within the deadline." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "ilm.configuration-roundtrip", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "Lifecycle configuration create, read, list, and delete agree across clients." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [ + "S3C-704" + ] + } + }, + { + "id": "ilm.invalid-configuration", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "s3-api" + ], + "assertions": [ + "Invalid days and status values fail with a 4xx response and do not persist partial configuration." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "ilm.current-expiration", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "Due current objects are deleted while a nonmatching control object survives." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "ilm.tag-filter-expiration", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "A tag-filtered rule expires only the matching object and preserves the control object." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "ilm.disabled-rule", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "s3-api" + ], + "assertions": [ + "A disabled rule does not mutate its matching object after the accelerated due time." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "ilm.rule-removal", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "s3-api" + ], + "assertions": [ + "Removing a rule before it becomes due prevents the queued lifecycle action." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "ilm.versioned-current-expiration", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "Current-version expiration creates a delete marker and retains the addressable original version." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "ilm.noncurrent-expiration", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "A due noncurrent version is deleted without changing the latest version." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "ilm.newer-noncurrent-retention", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "s3-api" + ], + "assertions": [ + "NewerNoncurrentVersions retains the requested history and expires only older versions." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "ilm.delete-all-versions", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "ExpiredObjectAllVersions removes every version of the matching key." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "ilm.delete-marker-expiration", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "The noncurrent version and its resulting expired delete marker are removed." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "ilm.multipart-expiration", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "A due incomplete multipart upload is automatically aborted." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "ilm.transition-and-restore", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "An object transitions to a real remote RustFS tier without changing its bytes.", + "A restore request completes and the restored object remains checksum-correct." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [ + "TIER-105", + "TIER-107" + ] + } + }, + { + "id": "ilm.restart-persistence", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "A lifecycle rule survives a full cluster restart and still expires its object." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh", + "rustfs-s3-compat-test.sh" + ], + "case_ids": [] + } + } + ] + }, + { + "id": "bucket-replication", + "target_environments": [ + "replication-2site" + ], + "required_capabilities": [ + "bucket-replication", + "versioning", + "multipart", + "scanner", + "restart-recovery", + "outage-recovery" + ], + "cases": [ + { + "id": "bucket-replication.source-versioning-required", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "An unversioned source is rejected without persisting a replication configuration." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [ + "REP-103" + ] + } + }, + { + "id": "bucket-replication.target-versioning-required", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "An unversioned target is rejected without creating a usable rule." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "bucket-replication.configuration-roundtrip", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "rc and S3 agree on the rule, destination, flags, and removal result." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "bucket-replication.put-object", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "Payload, metadata, and source version identity converge on the target." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [ + "REP-104" + ] + } + }, + { + "id": "bucket-replication.sync-put-object", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "A synchronous PUT returns only after the matching target version is readable." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "bucket-replication.prefix-filter", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "Prefix matching replicates the included key and excludes the control key." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "bucket-replication.tag-filter", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "Tag matching replicates the included version and excludes the control version." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "bucket-replication.zero-byte-special-key", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "s3-api" + ], + "assertions": [ + "Zero-byte and encoded keys retain their exact key, length, bytes, and version identity." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "bucket-replication.multipart-object", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "A completed multipart object converges with the same size, checksum, and version identity." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [ + "REP-108" + ] + } + }, + { + "id": "bucket-replication.copy-object", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "s3-api" + ], + "assertions": [ + "CopyObject output is replicated with its replacement metadata and payload." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "bucket-replication.delete-marker", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "A source delete marker converges by version ID while the preceding object version remains." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [ + "REP-105" + ] + } + }, + { + "id": "bucket-replication.version-delete", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "Permanent deletion removes only the addressed target version." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "bucket-replication.existing-object-enabled", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "Existing-object replication backfills an object created before the rule." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [ + "REP-107" + ] + } + }, + { + "id": "bucket-replication.existing-object-disabled", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "An object created before a rule stays absent when existing-object replication is disabled while a new control object replicates." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "bucket-replication.target-check", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "The active target check validates write, version, multipart, delete, and cleanup paths without leaving probe artifacts." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "bucket-replication.status-observability", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "rc list and status identify the configured target and expose no credentials." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "bucket-replication.outage-recovery", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "A target-outage write converges automatically after the target returns." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "bucket-replication.source-restart-recovery", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "A queued write survives source restart and converges after target recovery." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + } + ] + }, + { + "id": "site-replication", + "target_environments": [ + "replication-2site" + ], + "required_capabilities": [ + "site-replication", + "versioning", + "iam", + "restart-recovery", + "outage-recovery" + ], + "cases": [ + { + "id": "site-replication.unconfigured-info", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "admin-api" + ], + "assertions": [ + "Fresh sites report disabled state without stale credentials or pending operations." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.invalid-credentials", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "admin-api" + ], + "assertions": [ + "Invalid peer credentials fail closed without partially enabling either site." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.add-and-info", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "admin-api" + ], + "assertions": [ + "Both sites converge on the same two unique deployments and a non-secret replication identity." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [ + "SITE-101" + ] + } + }, + { + "id": "site-replication.status-observability", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "admin-api" + ], + "assertions": [ + "Status exposes bucket, IAM, ILM, peer-state, and metrics scopes without secrets." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.bucket-create", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "Bucket creation on site A becomes visible on site B." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.object-bidirectional", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "Objects flow in both directions with matching bytes, metadata, and version IDs and without loops." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [ + "SITE-103" + ] + } + }, + { + "id": "site-replication.versioning-metadata", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "Enable converges, suspension fails closed while replication is active, and both sites remain enabled." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.lifecycle-expiry-toggle", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "Expiry metadata is gated while disabled, converges after enable, and the global flag can be disabled again." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.bucket-policy-metadata", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "Bucket policy create and delete operations converge exactly." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [ + "SITE-104" + ] + } + }, + { + "id": "site-replication.object-tags", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "s3-api" + ], + "assertions": [ + "Successive object tag sets converge on the peer." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.delete-convergence", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "s3-api" + ], + "assertions": [ + "Delete markers and permanent version deletes converge without deleting unrelated versions." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.iam-user-policy", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "User, policy, and mapping converge and enforce the same allow/deny boundary on the peer." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.iam-group-policy", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "Group membership and group policy mapping converge and authorize the member on the peer." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.service-account", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "A service account converges and enforces its session policy without leaking its secret." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.peer-edit", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "admin-api" + ], + "assertions": [ + "A deployment-ID-targeted rename converges without changing endpoint or deployment identity." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.resync", + "source_status": "implemented", + "priority": "P1", + "channels": [ + "admin-api" + ], + "assertions": [ + "Resync start and status expose a durable operation ID and cancellation or terminal state is explicit." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [ + "SITE-105" + ] + } + }, + { + "id": "site-replication.outage-recovery", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "Peer outage is observable and queued bucket/object operations converge after recovery." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.restart-persistence", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "Peer membership and bidirectional replication survive both service restarts." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "site-replication.remove-all", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "Remove-all disables both sites and subsequent buckets no longer propagate." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [ + "SITE-106" + ] + } + }, + { + "id": "site-replication.service-account-rotation", + "source_status": "implemented", + "priority": "P0", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "A fresh post-removal group is created, then rotation settles in one request on both sites and bidirectional replication remains live." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-replication-test.sh" + ], + "case_ids": [] + } + } + ] + }, + { + "id": "kms", + "target_environments": [], + "required_capabilities": [ + "kms" + ], + "cases": [ + { + "id": "kms.sse-kms-round-trip", + "source_status": "planned", + "priority": "P0", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "Encrypted objects round-trip with the expected key and metadata semantics.", + "Expected failures do not return plaintext or leak key material." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-kms-test.sh" + ], + "case_ids": [ + "KMS-104", + "KMS-301", + "KMS-214" + ] + } + } + ] + }, + { + "id": "data-services", + "target_environments": [ + "lifecycle-b" + ], + "required_capabilities": [ + "lifecycle" + ], + "cases": [ + { + "id": "tiering.transition-and-restore", + "source_status": "planned", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "Transitioned objects remain readable and can be restored." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh" + ], + "case_ids": [ + "TIER-105", + "TIER-107" + ] + } + }, + { + "id": "events.delivery", + "source_status": "planned", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "Configured object events are delivered once according to the case contract." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh" + ], + "case_ids": [ + "EVT-103", + "EVT-104" + ] + } + }, + { + "id": "audit.request-correlation", + "source_status": "planned", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "Customer operations produce correlated audit records without secrets." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-tier-test.sh" + ], + "case_ids": [ + "AUD-102", + "AUD-104", + "AUD-106" + ] + } + } + ] + }, + { + "id": "identity-access", + "target_environments": [], + "required_capabilities": [ + "oidc", + "iam" + ], + "cases": [ + { + "id": "iam.policy-enforcement", + "source_status": "planned", + "priority": "P0", + "channels": [ + "rc", + "admin-api", + "s3-api" + ], + "assertions": [ + "Allowed actions succeed and explicitly denied actions fail closed." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-security-test.sh" + ], + "case_ids": [ + "IAM-105", + "IAM-109" + ] + } + }, + { + "id": "oidc.login-and-session", + "source_status": "planned", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api" + ], + "assertions": [ + "OIDC identity and session claims enforce the configured access boundary." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-security-test.sh" + ], + "case_ids": [ + "OIDC-103" + ] + } + } + ] + }, + { + "id": "s3-compatibility", + "target_environments": [ + "single-1disk" + ], + "required_capabilities": [ + "s3-basic" + ], + "cases": [ + { + "id": "s3.basic-object-round-trip", + "source_status": "planned", + "priority": "P0", + "channels": [ + "rc", + "s3-api" + ], + "assertions": [ + "Bucket and object CRUD preserve payload, metadata, ETag, and error semantics." + ], + "probes": {}, + "reuse_candidates": { + "suites": [ + "rustfs-s3-compat-test.sh" + ], + "case_ids": [ + "S3C-101", + "S3C-201", + "S3C-203", + "S3C-205", + "S3C-206", + "S3C-207" + ] + } + } + ] + } + ] +} From 9116663dc659fae39fc1ec298b35d00043576239 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 17:00:08 +0800 Subject: [PATCH 002/343] feat(validation): verify scanner usage convergence --- .../workflows/validation-catalog-tests.yml | 6 + README.md | 21 +- docs/validation-migration-map.md | 9 +- rustfs-storage-test.sh | 179 +++++++++++++++++- test_validation_scanner_usage.sh | 107 +++++++++++ 5 files changed, 309 insertions(+), 13 deletions(-) create mode 100644 test_validation_scanner_usage.sh diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index fbff1ce2..b572a32c 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -8,6 +8,8 @@ on: - scripts/validation_report.py - scripts/validation_result.py - test_validation_migration.py + - rustfs-storage-test.sh + - test_validation_scanner_usage.sh - .github/workflows/validation-catalog-tests.yml push: branches: [main] @@ -17,6 +19,8 @@ on: - scripts/validation_report.py - scripts/validation_result.py - test_validation_migration.py + - rustfs-storage-test.sh + - test_validation_scanner_usage.sh - .github/workflows/validation-catalog-tests.yml permissions: @@ -32,3 +36,5 @@ jobs: persist-credentials: false - run: python3 scripts/validation_catalog.py check - run: python3 -B -m unittest -v test_validation_migration.py + - run: bash -n rustfs-storage-test.sh test_validation_scanner_usage.sh + - run: bash test_validation_scanner_usage.sh diff --git a/README.md b/README.md index 00ddd540..34dda425 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,13 @@ ready; missing or unverified cases fail that gate. Without a gate status, the report shows migration progress and still rejects malformed or false PASS results. -The first executable canonical case is `s3.basic-object-round-trip`, using -the existing single-disk S3 suite deployment. It requires a candidate manifest -with `source_sha`, `package_url`, and `package_sha256`. The installed binary +The executable canonical cases currently include `s3.basic-object-round-trip` +and `scanner.usage-convergence`. The S3 case uses the existing single-disk +deployment. The scanner case uses the storage suite's single-node four-disk +deployment, waits for a post-upload successful scanner cycle, and requires a +complete usage snapshot with exactly 20 objects and 20,690 logical bytes. +Both require a candidate manifest with `source_sha`, `package_url`, and +`package_sha256`. The installed binary SHA-256 is measured on the test node and included in result provenance: ```bash @@ -44,13 +48,20 @@ SHA-256 is measured on the test node and included in result provenance: --validation-artifacts-dir /absolute/path/new-artifacts-dir \ --yes +./rustfs-storage-test.sh \ + --validation-case scanner.usage-convergence \ + --candidate-manifest /absolute/path/candidate.json \ + --testing-commit "$(git rev-parse HEAD)" \ + --validation-artifacts-dir /absolute/path/another-new-artifacts-dir \ + --yes + python3 scripts/validation_report.py \ --results-dir /absolute/path/new-artifacts-dir/cases \ --output /absolute/path/new-artifacts-dir/report.md ``` -The report's full implemented-case gate is intentionally not enabled by this -single-case run. Run it with `--gate-status implemented` only after all 66 +The report's full implemented-case gate is intentionally not enabled by these +runs. Run it with `--gate-status implemented` only after all 66 implemented source cases have verified executors and live results. ## Fault-tolerance suite diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index c68127bd..4c183659 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -22,11 +22,18 @@ DELETE, payload SHA-256, metadata, ETag, and `NoSuchKey` checks on one fixture. It has local mock coverage; live evidence against an immutable candidate is still required before calling the migration verified. +`scanner.usage-convergence` has a dedicated executor on `single-4disk`. It +uses the storage suite's deployment and SigV4 Admin client, accelerates the +scanner cycle, records the 20-object fixture, and checks a successful leader +cycle after upload plus a newer complete usage snapshot with exact object and +logical-byte counts. The old STOR-134 threshold remains a separate regression +test. This case also awaits a live candidate run. + The eight planned cases have these concrete reuse candidates: | Validation ID | Candidate IDs | Specific item to verify | | --- | --- | --- | -| `scanner.usage-convergence` | STOR-134 | Existing test checks 20 objects appear in Admin usage; add scanner result and fixture agreement. | +| `scanner.usage-convergence` | STOR-134 | Dedicated executor added; live candidate evidence is pending. | | `kms.sse-kms-round-trip` | KMS-104, KMS-301, KMS-214 | Compare key/metadata semantics and failure redaction. | | `tiering.transition-and-restore` | TIER-105, TIER-107 | Compare remote tier restore and object checksum. | | `events.delivery` | EVT-103, EVT-104 | Freeze and verify the exact delivery count contract. | diff --git a/rustfs-storage-test.sh b/rustfs-storage-test.sh index 057e6cd1..6c0d42fb 100755 --- a/rustfs-storage-test.sh +++ b/rustfs-storage-test.sh @@ -1,7 +1,3 @@ - # Both keys carry a compressible extension: disk compression is decided - # from the request key/Content-Type, and aws s3api sends no Content-Type on - # put-object or copy-object, so an extension-less key is never compressed - # (same include filter as MinIO) and the assertion below would be vacuous. #!/usr/bin/env bash # # rustfs-storage-test.sh @@ -32,6 +28,7 @@ # reported as UNSUPPORTED, not FAIL. # set -Eeuo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # ==================== Configuration ==================== @@ -96,6 +93,12 @@ SELECTED_TOPOLOGIES=() CURRENT_TOPOLOGY="" ENDPOINT="" VOLUMES="" +VALIDATION_CASE="" +VALIDATION_ARTIFACTS_DIR="" +VALIDATION_CANDIDATE_MANIFEST="" +VALIDATION_TESTING_COMMIT="" +VALIDATION_BINARY_SHA256="" +VALIDATION_OBJECT_CLEANUP_STATUS=PASS TEST_TMP="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-storage.XXXXXX")" trap 'rm -rf "${TEST_TMP}"; rm -f "${ADMIN_API_CODE_FILE}"' EXIT @@ -187,6 +190,9 @@ RUSTFS_CONSOLE_ENABLE=${RUSTFS_CONSOLE_ENABLE} RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL} RUSTFS_OBS_LOG_DIRECTORY="${RUSTFS_OBS_LOG_DIRECTORY}" EOF + if [ "${VALIDATION_CASE}" = "scanner.usage-convergence" ]; then + printf 'RUSTFS_SCANNER_ENABLED=true\nRUSTFS_SCANNER_CYCLE=5\nRUSTFS_SCANNER_START_DELAY_SECS=0\n' + fi } write_rustfs_config() { @@ -498,6 +504,36 @@ run_test() { fi } +validation_write_evidence() { + local evidence_dir="${VALIDATION_ARTIFACTS_DIR}/evidence/${VALIDATION_CASE}" item + mkdir -p "${evidence_dir}" + cp "${TEST_TMP}/test.out" "${evidence_dir}/test.log" + for item in validation-fixture.json validation-list.json validation-baseline-usage.json validation-scanner.json validation-usage.json; do + if [ -f "${TEST_TMP}/${item}" ]; then cp "${TEST_TMP}/${item}" "${evidence_dir}/${item}"; fi + done +} + +validation_write_result() { + local verdict="$1" reason="$2" cleanup_status="$3" item + local args=("${SCRIPT_DIR}/scripts/validation_result.py" + --case-id "${VALIDATION_CASE}" --profile single-4disk + --result-file "${VALIDATION_ARTIFACTS_DIR}/cases/${VALIDATION_CASE}.json" + --verdict "${verdict}" --cleanup-status "${cleanup_status}") + if [ -n "${reason}" ]; then args+=(--reason "${reason}"); fi + if [ "${verdict}" != RUNNING ]; then + args+=(--candidate-manifest "${VALIDATION_CANDIDATE_MANIFEST}" + --rc-bin "$(command -v rc)" --testing-commit "${VALIDATION_TESTING_COMMIT}" + --binary-sha256 "${VALIDATION_BINARY_SHA256}" + --evidence "evidence/${VALIDATION_CASE}/test.log") + fi + if [ "${verdict}" = PASS ]; then + for item in validation-fixture.json validation-list.json validation-baseline-usage.json validation-scanner.json validation-usage.json; do + args+=(--evidence "evidence/${VALIDATION_CASE}/${item}") + done + fi + python3 "${args[@]}" +} + rc_run() { RC_OUT="$(rc "$@" 2>&1)"; RC_CODE=$? } @@ -686,6 +722,79 @@ test_stor134() { # admin usage APIs must respond promptly and converge to actual return 1 } +validation_scanner_usage_matches() { + local scanner_file="$1" usage_file="$2" bucket="$3" baseline="$4" bytes="$5" uploaded="$6" + jq -e --argjson uploaded "${uploaded}" \ + '.enabled == true and .freshness.state == "fresh" and .metrics.last_cycle_result == "success" and .metrics.leader_lock_held_by_this_process == true and .cycle_schedule.execution_role == "leader" and .freshness.last_cycle_end_unix_secs >= $uploaded' \ + "${scanner_file}" >/dev/null 2>&1 && \ + jq -e --arg bucket "${bucket}" --argjson baseline "${baseline}" --argjson bytes "${bytes}" \ + '.usage_snapshot_complete == true and .usage_snapshot_converged != false and .scanner_cycle > $baseline and .buckets_usage[$bucket].objects_count == 20 and .buckets_usage[$bucket].size == $bytes' \ + "${usage_file}" >/dev/null 2>&1 +} + +test_validation_scanner_usage_convergence() { + local bucket="val-usage-${RANDOM}-$$" i key size sha expected_size=0 baseline_cycle=0 + local upload_end deadline case_rc=0 cleanup_rc=0 + local fixture_tsv="${TEST_TMP}/validation-fixture.tsv" + VALIDATION_OBJECT_CLEANUP_STATUS=PASS + : > "${fixture_tsv}" + new_bucket "${bucket}" || return 1 + ( + admin_api GET "/rustfs/admin/v3/datausageinfo" "" "" > "${TEST_TMP}/validation-baseline-usage.json" + [ "$(admin_api_code)" = 200 ] || { echo "baseline usage API failed"; exit 1; } + baseline_cycle="$(jq -er '.scanner_cycle // 0 | numbers' "${TEST_TMP}/validation-baseline-usage.json")" || exit 1 + for i in $(seq 1 20); do + key="obj-${i}" + size=$((1024 + i)) + make_test_file "${TEST_TMP}/${key}" "${size}" + sha="$(sha256sum "${TEST_TMP}/${key}" | awk '{print $1}')" + printf '%s\t%s\t%s\n' "${key}" "${size}" "${sha}" >> "${fixture_tsv}" + expected_size=$((expected_size + size)) + aws_run s3api put-object --bucket "${bucket}" --key "${key}" --body "${TEST_TMP}/${key}" + assert_aws_ok || exit 1 + done + jq -Rn --arg bucket "${bucket}" '[inputs | split("\t") | {key:.[0], size:(.[1]|tonumber), sha256:.[2]}] | {bucket:$bucket, objects:., expected_count:length, expected_size:(map(.size)|add)}' \ + < "${fixture_tsv}" > "${TEST_TMP}/validation-fixture.json" + upload_end="$(date -u +%s)" + aws_run s3api list-objects-v2 --bucket "${bucket}" + assert_aws_ok || exit 1 + printf '%s\n' "${AWS_OUT}" > "${TEST_TMP}/validation-list.json" + jq -e --slurpfile fixture "${TEST_TMP}/validation-fixture.json" \ + '(.Contents // [] | map({key:.Key,size:.Size}) | sort_by(.key)) == ($fixture[0].objects | map({key,size}) | sort_by(.key))' \ + "${TEST_TMP}/validation-list.json" >/dev/null || { echo "S3 fixture list differs from uploaded manifest"; exit 1; } + + deadline=$((SECONDS + 300)) + while [ "${SECONDS}" -lt "${deadline}" ]; do + admin_api GET "/rustfs/admin/v3/scanner/status" "" "" > "${TEST_TMP}/validation-scanner.json" + local scanner_code usage_code + scanner_code="$(admin_api_code)" + admin_api GET "/rustfs/admin/v3/datausageinfo" "" "" > "${TEST_TMP}/validation-usage.json" + usage_code="$(admin_api_code)" + if [ "${scanner_code}" = 200 ] && [ "${usage_code}" = 200 ] && \ + validation_scanner_usage_matches "${TEST_TMP}/validation-scanner.json" "${TEST_TMP}/validation-usage.json" \ + "${bucket}" "${baseline_cycle}" "${expected_size}" "${upload_end}"; then + echo "scanner completed after upload; usage converged to 20 objects and ${expected_size} bytes" + exit 0 + fi + sleep 5 + done + echo "scanner or usage did not converge within 300 seconds" + exit 1 + ) || case_rc=$? + + for i in $(seq 1 20); do + aws_run s3api delete-object --bucket "${bucket}" --key "obj-${i}" + assert_aws_ok || cleanup_rc=1 + done + aws_run s3api delete-bucket --bucket "${bucket}" + assert_aws_ok || cleanup_rc=1 + if [ "${cleanup_rc}" -ne 0 ]; then + VALIDATION_OBJECT_CLEANUP_STATUS=FAIL + echo "validation bucket cleanup failed" + fi + [ "${case_rc}" -eq 0 ] && [ "${cleanup_rc}" -eq 0 ] +} + test_stor101() { new_bucket "stor101-b1" || return 1 make_compressible_file "${TEST_TMP}/stor101-v" 1048576 @@ -1080,6 +1189,10 @@ Options: --ssh-user SSH user for the test nodes --ssh-port SSH port for the test nodes --log-file append all output to a log file + --validation-case ID run only scanner.usage-convergence (single-multi) + --validation-artifacts-dir DIR write canonical result and evidence + --candidate-manifest FILE immutable candidate manifest + --testing-commit SHA auto-testing commit used for validation results --dry-run preview only -y, --yes skip confirmation -h, --help show this help @@ -1101,6 +1214,10 @@ main() { --ssh-user) SSH_USER="$1"; shift ;; --ssh-port) SSH_PORT="$1"; shift ;; --log-file) LOG_FILE="$1"; shift ;; + --validation-case) VALIDATION_CASE="$1"; shift ;; + --validation-artifacts-dir) VALIDATION_ARTIFACTS_DIR="$1"; shift ;; + --candidate-manifest) VALIDATION_CANDIDATE_MANIFEST="$1"; shift ;; + --testing-commit) VALIDATION_TESTING_COMMIT="$1"; shift ;; --dry-run) DRY_RUN=1 ;; -y|--yes) ASSUME_YES=1 ;; -h|--help) usage; exit 0 ;; @@ -1118,6 +1235,27 @@ main() { if [ "${all}" -eq 1 ]; then SELECTED_TOPOLOGIES=("${TOPO_SINGLE_SINGLE}" "${TOPO_SINGLE_MULTI}" "${TOPO_MULTI_MULTI}") fi + if [ -n "${VALIDATION_CASE}" ]; then + [ "${VALIDATION_CASE}" = "scanner.usage-convergence" ] || die "unknown validation case: ${VALIDATION_CASE}" + [ "${all}" -eq 0 ] || die "--validation-case cannot be combined with --all-topologies" + [ "${#SELECTED_TOPOLOGIES[@]}" -eq 0 ] || die "--validation-case chooses its own topology" + [ "${KEEP_ENV}" -eq 0 ] || die "--validation-case requires environment teardown" + SELECTED_TOPOLOGIES=("${TOPO_SINGLE_MULTI}") + if [ "${DRY_RUN}" -eq 0 ]; then + [ -n "${VALIDATION_ARTIFACTS_DIR}" ] || die "--validation-case requires --validation-artifacts-dir" + [ -n "${VALIDATION_CANDIDATE_MANIFEST}" ] || die "--validation-case requires --candidate-manifest" + [ -n "${VALIDATION_TESTING_COMMIT}" ] || die "--validation-case requires --testing-commit" + [ ! -e "${VALIDATION_ARTIFACTS_DIR}" ] || die "validation artifacts directory already exists" + PACKAGE_URL="$(jq -er '.package_url' "${VALIDATION_CANDIDATE_MANIFEST}")" || die "candidate URL missing" + PACKAGE_SHA256="$(jq -er '.package_sha256' "${VALIDATION_CANDIDATE_MANIFEST}")" || die "candidate package SHA missing" + [[ "${PACKAGE_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "invalid candidate package SHA" + jq -e '.source_sha | test("^[0-9a-f]{40}$")' "${VALIDATION_CANDIDATE_MANIFEST}" >/dev/null \ + || die "candidate source commit missing" + [ "${VALIDATION_TESTING_COMMIT}" = "$(git -C "${SCRIPT_DIR}" rev-parse HEAD)" ] \ + || die "testing commit does not match this checkout" + mkdir -p "${VALIDATION_ARTIFACTS_DIR}" + fi + fi [ "${#SELECTED_TOPOLOGIES[@]}" -gt 0 ] || die "no topology selected (--all-topologies / --topology)" local _p for _p in /root/.local/bin /home/*/.local/bin /usr/local/bin; do @@ -1130,26 +1268,53 @@ main() { need_cmd jq "JSON processor" need_cmd curl "HTTP client" need_cmd openssl "OpenSSL (SigV4 signing)" + if [ -n "${VALIDATION_CASE}" ]; then need_cmd python3 "validation result writer"; fi log "nodes: ${NODES[*]} package: $(resolve_package_url)" local overall_fail=0 topo for topo in "${SELECTED_TOPOLOGIES[@]}"; do log "== topology: ${topo} ==" + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + validation_write_result RUNNING "validation case started" PENDING + fi env_setup "${topo}" CURRENT_TOPOLOGY="${topo}" + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + VALIDATION_BINARY_SHA256="$(printf 'sha256sum /usr/bin/rustfs\n' | run_remote "${NODES[0]}" | awk '{print $1}')" + [[ "${VALIDATION_BINARY_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "installed RustFS binary SHA-256 unavailable" + fi PASS=0; FAIL=0; UNSUPPORTED=0; FAILED_TESTS=() - run_all_tests + if [ -n "${VALIDATION_CASE}" ]; then + run_test "${VALIDATION_CASE}" "UsageConvergence" test_validation_scanner_usage_convergence + else + run_all_tests + fi results_summary record_topology_result "${topo}" + local cleanup_status=PASS if [ "${KEEP_ENV}" -eq 0 ] || [ "${topo}" != "${SELECTED_TOPOLOGIES[${#SELECTED_TOPOLOGIES[@]}-1]}" ]; then - env_teardown "${topo}" + env_teardown "${topo}" || cleanup_status=FAIL + fi + if [ "${VALIDATION_OBJECT_CLEANUP_STATUS}" != PASS ]; then cleanup_status=FAIL; fi + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + validation_write_evidence + if [ "${FAIL}" -eq 0 ] && [ "${UNSUPPORTED}" -eq 0 ] && [ "${cleanup_status}" = PASS ]; then + validation_write_result PASS "" PASS + elif [ "${FAIL}" -eq 0 ] && [ "${UNSUPPORTED}" -eq 0 ]; then + validation_write_result INFRA_ERROR "environment teardown failed" FAIL + else + validation_write_result FAIL "case failed or unsupported; inspect evidence" "${cleanup_status}" + fi fi if [ "${FAIL}" -gt 0 ]; then overall_fail=1 fi + if [ "${UNSUPPORTED}" -gt 0 ] || [ "${cleanup_status}" != PASS ]; then overall_fail=1; fi done final_summary [ "${overall_fail}" -eq 0 ] || exit 1 log "all done" } -main "$@" +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/test_validation_scanner_usage.sh b/test_validation_scanner_usage.sh new file mode 100644 index 00000000..78ffc338 --- /dev/null +++ b/test_validation_scanner_usage.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/rustfs-storage-test.sh" + +MOCK_OBJECTS_FILE="${TEST_TMP}/mock-objects.jsonl" +MOCK_BUCKET="" +MOCK_BUCKET_DELETED=0 +MOCK_DELETE_FAIL=0 +: > "${MOCK_OBJECTS_FILE}" + +aws_run() { + local operation="${2:-}" key="" body="" previous="" argument + AWS_CODE=0 + AWS_OUT='{}' + case "${operation}" in + create-bucket) + MOCK_BUCKET="$4" + ;; + put-object) + for argument in "$@"; do + case "${previous}" in + --key) key="${argument}" ;; + --body) body="${argument}" ;; + esac + previous="${argument}" + done + jq -nc --arg key "${key}" --argjson size "$(wc -c < "${body}")" '{Key:$key,Size:$size}' >> "${MOCK_OBJECTS_FILE}" + ;; + list-objects-v2) + AWS_OUT="$(jq -s '{Contents:.}' "${MOCK_OBJECTS_FILE}")" + ;; + delete-object) ;; + delete-bucket) + MOCK_BUCKET_DELETED=$((MOCK_BUCKET_DELETED + 1)) + if [ "${MOCK_DELETE_FAIL}" -eq 1 ]; then AWS_CODE=1; AWS_OUT='mock cleanup failure'; fi + ;; + *) printf 'unexpected AWS operation: %s\n' "${operation}" >&2; return 1 ;; + esac +} + +admin_api() { + local path="$2" count + printf '200' > "${ADMIN_API_CODE_FILE}" + count="$(wc -l < "${MOCK_OBJECTS_FILE}" | tr -d ' ')" + case "${path}" in + /rustfs/admin/v3/scanner/status) + jq -nc --argjson now "$(date -u +%s)" \ + '{enabled:true,freshness:{state:"fresh",last_cycle_end_unix_secs:$now},metrics:{last_cycle_result:"success",leader_lock_held_by_this_process:true},cycle_schedule:{execution_role:"leader"}}' + ;; + /rustfs/admin/v3/datausageinfo) + if [ "${count}" -eq 0 ]; then + printf '{"scanner_cycle":0,"usage_snapshot_complete":true,"buckets_usage":{}}\n' + else + jq -nc --arg bucket "${MOCK_BUCKET}" --argjson bytes "$(jq -s 'map(.Size) | add' "${MOCK_OBJECTS_FILE}")" \ + '{scanner_cycle:1,usage_snapshot_complete:true,usage_snapshot_converged:true,buckets_usage:{($bucket):{objects_count:20,size:$bytes}}}' + fi + ;; + *) printf 'unexpected Admin path: %s\n' "${path}" >&2; return 1 ;; + esac +} + +test_validation_scanner_usage_convergence +[ "${VALIDATION_OBJECT_CLEANUP_STATUS}" = PASS ] +[ "${MOCK_BUCKET_DELETED}" -eq 1 ] +jq -e '.expected_count == 20 and .expected_size == 20690 and (.objects | length) == 20' \ + "${TEST_TMP}/validation-fixture.json" >/dev/null +validation_scanner_usage_matches "${TEST_TMP}/validation-scanner.json" "${TEST_TMP}/validation-usage.json" \ + "${MOCK_BUCKET}" 0 20690 0 + +jq '.metrics.last_cycle_result = "partial"' "${TEST_TMP}/validation-scanner.json" > "${TEST_TMP}/bad-scanner.json" +if validation_scanner_usage_matches "${TEST_TMP}/bad-scanner.json" "${TEST_TMP}/validation-usage.json" "${MOCK_BUCKET}" 0 20690 0; then + printf 'partial scanner cycle passed\n' >&2; exit 1 +fi +jq '.scanner_cycle = 0' "${TEST_TMP}/validation-usage.json" > "${TEST_TMP}/bad-usage.json" +if validation_scanner_usage_matches "${TEST_TMP}/validation-scanner.json" "${TEST_TMP}/bad-usage.json" "${MOCK_BUCKET}" 0 20690 0; then + printf 'stale usage snapshot passed\n' >&2; exit 1 +fi +jq '.buckets_usage[].size = 1' "${TEST_TMP}/validation-usage.json" > "${TEST_TMP}/bad-usage.json" +if validation_scanner_usage_matches "${TEST_TMP}/validation-scanner.json" "${TEST_TMP}/bad-usage.json" "${MOCK_BUCKET}" 0 20690 0; then + printf 'wrong fixture size passed\n' >&2; exit 1 +fi +jq '.buckets_usage[].objects_count = 19' "${TEST_TMP}/validation-usage.json" > "${TEST_TMP}/bad-usage.json" +if validation_scanner_usage_matches "${TEST_TMP}/validation-scanner.json" "${TEST_TMP}/bad-usage.json" "${MOCK_BUCKET}" 0 20690 0; then + printf 'wrong object count passed\n' >&2; exit 1 +fi +jq '.usage_snapshot_complete = false' "${TEST_TMP}/validation-usage.json" > "${TEST_TMP}/bad-usage.json" +if validation_scanner_usage_matches "${TEST_TMP}/validation-scanner.json" "${TEST_TMP}/bad-usage.json" "${MOCK_BUCKET}" 0 20690 0; then + printf 'incomplete usage snapshot passed\n' >&2; exit 1 +fi + +MOCK_DELETE_FAIL=1 +: > "${MOCK_OBJECTS_FILE}" +if test_validation_scanner_usage_convergence; then + printf 'failed cleanup passed\n' >&2; exit 1 +fi +[ "${VALIDATION_OBJECT_CLEANUP_STATUS}" = FAIL ] + +"${SCRIPT_DIR}/rustfs-storage-test.sh" --validation-case scanner.usage-convergence \ + --dry-run --yes --package-url https://example.invalid/rustfs.deb \ + > "${TEST_TMP}/validation-dry-run.log" +grep -q '\[DRY-RUN\] scanner.usage-convergence UsageConvergence' "${TEST_TMP}/validation-dry-run.log" +grep -q 'RUSTFS_SCANNER_CYCLE=5' "${TEST_TMP}/validation-dry-run.log" +if grep -q '\[DRY-RUN\] STOR-' "${TEST_TMP}/validation-dry-run.log"; then + printf 'validation mode scheduled legacy storage cases\n' >&2; exit 1 +fi From 130fcdf1571bcac33e8e62f298a90de355a365f1 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 17:08:07 +0800 Subject: [PATCH 003/343] feat(validation): add canonical SSE-KMS case --- .github/workflows/kms-script-tests.yml | 4 + README.md | 11 +- docs/validation-migration-map.md | 7 +- rustfs-kms-test.sh | 166 ++++++++++++++++++++++++- test_validation_kms.sh | 75 +++++++++++ 5 files changed, 257 insertions(+), 6 deletions(-) create mode 100644 test_validation_kms.sh diff --git a/.github/workflows/kms-script-tests.yml b/.github/workflows/kms-script-tests.yml index e53a32ea..60b58201 100644 --- a/.github/workflows/kms-script-tests.yml +++ b/.github/workflows/kms-script-tests.yml @@ -5,12 +5,14 @@ on: paths: - rustfs-kms-test.sh - test_rustfs_kms_presign.py + - test_validation_kms.sh - .github/workflows/kms-script-tests.yml push: branches: [main] paths: - rustfs-kms-test.sh - test_rustfs_kms_presign.py + - test_validation_kms.sh - .github/workflows/kms-script-tests.yml permissions: @@ -28,3 +30,5 @@ jobs: run: bash -n rustfs-kms-test.sh - name: Test presigned KMS case without a server run: python3 -m unittest test_rustfs_kms_presign.py + - name: Test canonical KMS round trip without a server + run: bash test_validation_kms.sh diff --git a/README.md b/README.md index 34dda425..601cf686 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,8 @@ ready; missing or unverified cases fail that gate. Without a gate status, the report shows migration progress and still rejects malformed or false PASS results. -The executable canonical cases currently include `s3.basic-object-round-trip` -and `scanner.usage-convergence`. The S3 case uses the existing single-disk +The executable canonical cases currently include `s3.basic-object-round-trip`, +`scanner.usage-convergence`, and `kms.sse-kms-round-trip`. The S3 and KMS cases use single-disk deployment. The scanner case uses the storage suite's single-node four-disk deployment, waits for a post-upload successful scanner cycle, and requires a complete usage snapshot with exactly 20 objects and 20,690 logical bytes. @@ -55,6 +55,13 @@ SHA-256 is measured on the test node and included in result provenance: --validation-artifacts-dir /absolute/path/another-new-artifacts-dir \ --yes +./rustfs-kms-test.sh \ + --validation-case kms.sse-kms-round-trip \ + --candidate-manifest /absolute/path/candidate.json \ + --testing-commit "$(git rev-parse HEAD)" \ + --validation-artifacts-dir /absolute/path/kms-new-artifacts-dir \ + --yes + python3 scripts/validation_report.py \ --results-dir /absolute/path/new-artifacts-dir/cases \ --output /absolute/path/new-artifacts-dir/report.md diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 4c183659..160be8c7 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -29,12 +29,17 @@ cycle after upload plus a newer complete usage snapshot with exact object and logical-byte counts. The old STOR-134 threshold remains a separate regression test. This case also awaits a live candidate run. +`kms.sse-kms-round-trip` has a dedicated executor using the existing local +KMS single-disk deployment. It checks the same object's encryption key, +metadata, and payload, then requires an invalid-key PUT to fail without +exposing plaintext or key material. Its live candidate run is pending. + The eight planned cases have these concrete reuse candidates: | Validation ID | Candidate IDs | Specific item to verify | | --- | --- | --- | | `scanner.usage-convergence` | STOR-134 | Dedicated executor added; live candidate evidence is pending. | -| `kms.sse-kms-round-trip` | KMS-104, KMS-301, KMS-214 | Compare key/metadata semantics and failure redaction. | +| `kms.sse-kms-round-trip` | KMS-104, KMS-301, KMS-214 | Dedicated executor added; live candidate evidence is pending. | | `tiering.transition-and-restore` | TIER-105, TIER-107 | Compare remote tier restore and object checksum. | | `events.delivery` | EVT-103, EVT-104 | Freeze and verify the exact delivery count contract. | | `audit.request-correlation` | AUD-102, AUD-104, AUD-106 | Verify request correlation and secret redaction in the same event trail. | diff --git a/rustfs-kms-test.sh b/rustfs-kms-test.sh index 59b20825..7391bf7e 100755 --- a/rustfs-kms-test.sh +++ b/rustfs-kms-test.sh @@ -18,6 +18,7 @@ # ./rustfs-kms-test.sh --topology single-single --backends local -y # set -Eeuo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Per-run temp dir (shared runners may leave stale /tmp files owned by other users) TEST_TMP="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-test.XXXXXX")" @@ -125,6 +126,12 @@ ENDPOINT="" VOLUMES="" KMS_BACKEND="local" KMS_EXTRA_ENV="${KMS_EXTRA_ENV:-}" +VALIDATION_CASE="" +VALIDATION_ARTIFACTS_DIR="" +VALIDATION_CANDIDATE_MANIFEST="" +VALIDATION_TESTING_COMMIT="" +VALIDATION_BINARY_SHA256="" +VALIDATION_OBJECT_CLEANUP_STATUS=PASS # ==================== Helpers ==================== @@ -710,6 +717,36 @@ run_test() { fi } +validation_write_evidence() { + local evidence_dir="${VALIDATION_ARTIFACTS_DIR}/evidence/${VALIDATION_CASE}" item + mkdir -p "${evidence_dir}" + cp "${TEST_TMP}/test.out" "${evidence_dir}/test.log" + for item in validation-put.json validation-head.json validation-get.json validation-failure.txt validation-missing.txt validation-fixture.json; do + if [ -f "${TEST_TMP}/${item}" ]; then cp "${TEST_TMP}/${item}" "${evidence_dir}/${item}"; fi + done +} + +validation_write_result() { + local verdict="$1" reason="$2" cleanup_status="$3" item + local args=("${SCRIPT_DIR}/scripts/validation_result.py" + --case-id "${VALIDATION_CASE}" --profile single-1disk + --result-file "${VALIDATION_ARTIFACTS_DIR}/cases/${VALIDATION_CASE}.json" + --verdict "${verdict}" --cleanup-status "${cleanup_status}") + if [ -n "${reason}" ]; then args+=(--reason "${reason}"); fi + if [ "${verdict}" != RUNNING ]; then + args+=(--candidate-manifest "${VALIDATION_CANDIDATE_MANIFEST}" + --rc-bin "$(command -v rc)" --testing-commit "${VALIDATION_TESTING_COMMIT}" + --binary-sha256 "${VALIDATION_BINARY_SHA256}" + --evidence "evidence/${VALIDATION_CASE}/test.log") + fi + if [ "${verdict}" = PASS ]; then + for item in validation-put.json validation-head.json validation-get.json validation-failure.txt validation-missing.txt validation-fixture.json; do + args+=(--evidence "evidence/${VALIDATION_CASE}/${item}") + done + fi + python3 "${args[@]}" +} + rc_run() { RC_OUT="$(rc "$@" 2>&1)"; RC_CODE=$? return "${RC_CODE}" @@ -1218,6 +1255,72 @@ test_kms301() { rm_bucket kms301-b1 } +test_validation_kms_sse_kms_round_trip() { + local bucket="val-kms-${RANDOM}-$$" key="encrypted" rejected="rejected" + local source="${TEST_TMP}/validation-kms-source" downloaded="${TEST_TMP}/validation-kms-downloaded" + local marker="validation-kms-plaintext-${RANDOM}-$$" kid="" expected_sha case_rc=0 cleanup_rc=0 + VALIDATION_OBJECT_CLEANUP_STATUS=PASS + kms_create_key "val-kms-${RANDOM}-$$" "validation=roundtrip" || return 1 + kid="${KMS_KEY_ID}" + printf '%s' "${marker}" > "${source}" + head -c 4096 /dev/urandom >> "${source}" + expected_sha="$(sha256sum "${source}" | awk '{print $1}')" + jq -n --arg bucket "${bucket}" --arg key "${key}" --arg key_id "${kid}" \ + --arg sha256 "${expected_sha}" --argjson bytes "$(wc -c < "${source}")" \ + '{bucket:$bucket,key:$key,key_id:$key_id,sha256:$sha256,bytes:$bytes,metadata:{validation:"kms-roundtrip"}}' \ + > "${TEST_TMP}/validation-fixture.json" + new_bucket "${bucket}" || return 1 + ( + aws_run s3api put-object --bucket "${bucket}" --key "${key}" --body "${source}" \ + --server-side-encryption aws:kms --ssekms-key-id "${kid}" --metadata validation=kms-roundtrip + assert_aws_ok || exit 1 + printf '%s\n' "${AWS_OUT}" > "${TEST_TMP}/validation-put.json" + printf '%s' "${AWS_OUT}" | jq -e --arg kid "${kid}" \ + '.ServerSideEncryption == "aws:kms" and (.SSEKMSKeyId | strings | contains($kid))' >/dev/null \ + || { echo "PUT lacks expected SSE-KMS key"; exit 1; } + + aws_run s3api head-object --bucket "${bucket}" --key "${key}" + assert_aws_ok || exit 1 + printf '%s\n' "${AWS_OUT}" > "${TEST_TMP}/validation-head.json" + jq -e --arg kid "${kid}" --argjson bytes "$(wc -c < "${source}")" \ + '.ServerSideEncryption == "aws:kms" and (.SSEKMSKeyId | strings | contains($kid)) and .ContentLength == $bytes and .Metadata.validation == "kms-roundtrip"' \ + "${TEST_TMP}/validation-head.json" >/dev/null || { echo "HEAD encryption or metadata mismatch"; exit 1; } + + aws_run s3api get-object --bucket "${bucket}" --key "${key}" "${downloaded}" + assert_aws_ok || exit 1 + printf '%s\n' "${AWS_OUT}" > "${TEST_TMP}/validation-get.json" + [ "$(sha256sum "${downloaded}" | awk '{print $1}')" = "${expected_sha}" ] \ + || { echo "SSE-KMS payload mismatch"; exit 1; } + + aws_run s3api put-object --bucket "${bucket}" --key "${rejected}" --body "${source}" \ + --server-side-encryption aws:kms --ssekms-key-id "missing-${RANDOM}-$$" + assert_aws_fails || { echo "invalid KMS key unexpectedly accepted"; exit 1; } + printf '%s\n' "${AWS_OUT}" > "${TEST_TMP}/validation-failure.txt" + for secret in "${marker}" "${KMS_LOCAL_MASTER_KEY}" "${KMS_BACKUP_KEK}" "${VAULT_TOKEN}"; do + [ -n "${secret}" ] || continue + if [[ "${AWS_OUT}" == *"${secret}"* ]]; then echo "expected failure leaked plaintext or key material"; exit 1; fi + done + aws_run s3api head-object --bucket "${bucket}" --key "${rejected}" + assert_aws_fails || { echo "failed encrypted PUT persisted an object"; exit 1; } + printf '%s\n' "${AWS_OUT}" > "${TEST_TMP}/validation-missing.txt" + if [[ "${AWS_OUT}" == *"${marker}"* ]] || [[ "${AWS_OUT}" == *"${KMS_LOCAL_MASTER_KEY}"* ]]; then + echo "missing-object failure leaked plaintext or key material"; exit 1 + fi + ) || case_rc=$? + + for key in "${key}" "${rejected}"; do + aws_run s3api delete-object --bucket "${bucket}" --key "${key}" + assert_aws_ok || cleanup_rc=1 + done + aws_run s3api delete-bucket --bucket "${bucket}" + assert_aws_ok || cleanup_rc=1 + if [ "${cleanup_rc}" -ne 0 ]; then + VALIDATION_OBJECT_CLEANUP_STATUS=FAIL + echo "validation bucket cleanup failed" + fi + [ "${case_rc}" -eq 0 ] && [ "${cleanup_rc}" -eq 0 ] +} + test_kms302() { kms_create_key "k302" "env=ci" || return 1 local kid="${KMS_KEY_ID}" @@ -2139,6 +2242,10 @@ Usage: --ssh-user SSH user for the test nodes (default: azureuser) --log-file Tee output to a log file --dry-run Preview commands without executing them + --validation-case ID Run only kms.sse-kms-round-trip (local, single-single) + --validation-artifacts-dir DIR Write canonical result and evidence + --candidate-manifest FILE Immutable candidate manifest + --testing-commit SHA auto-testing commit used for validation results -y, --yes Skip confirmation prompts -h, --help Show this help USAGE @@ -2161,6 +2268,10 @@ main() { --ssh-user) SSH_USER="$1"; shift ;; --log-file) LOG_FILE="$1"; shift ;; --dry-run) DRY_RUN=1 ;; + --validation-case) VALIDATION_CASE="$1"; shift ;; + --validation-artifacts-dir) VALIDATION_ARTIFACTS_DIR="$1"; shift ;; + --candidate-manifest) VALIDATION_CANDIDATE_MANIFEST="$1"; shift ;; + --testing-commit) VALIDATION_TESTING_COMMIT="$1"; shift ;; -y|--yes) ASSUME_YES=1 ;; -h|--help) usage; exit 0 ;; *) die "unknown option: ${opt} (see --help)" ;; @@ -2177,6 +2288,28 @@ main() { if [ "${all}" -eq 1 ]; then SELECTED_TOPOLOGIES=("${TOPO_SINGLE_SINGLE}" "${TOPO_SINGLE_MULTI}" "${TOPO_MULTI_MULTI}") fi + if [ -n "${VALIDATION_CASE}" ]; then + [ "${VALIDATION_CASE}" = "kms.sse-kms-round-trip" ] || die "unknown validation case: ${VALIDATION_CASE}" + [ "${all}" -eq 0 ] || die "--validation-case cannot be combined with --all-topologies" + [ "${#SELECTED_TOPOLOGIES[@]}" -eq 0 ] || die "--validation-case chooses its own topology" + [ "${KEEP_ENV}" -eq 0 ] || die "--validation-case requires environment teardown" + SELECTED_TOPOLOGIES=("${TOPO_SINGLE_SINGLE}") + backends_arg=local + if [ "${DRY_RUN}" -eq 0 ]; then + [ -n "${VALIDATION_ARTIFACTS_DIR}" ] || die "--validation-case requires --validation-artifacts-dir" + [ -n "${VALIDATION_CANDIDATE_MANIFEST}" ] || die "--validation-case requires --candidate-manifest" + [ -n "${VALIDATION_TESTING_COMMIT}" ] || die "--validation-case requires --testing-commit" + [ ! -e "${VALIDATION_ARTIFACTS_DIR}" ] || die "validation artifacts directory already exists" + PACKAGE_URL="$(jq -er '.package_url' "${VALIDATION_CANDIDATE_MANIFEST}")" || die "candidate URL missing" + PACKAGE_SHA256="$(jq -er '.package_sha256' "${VALIDATION_CANDIDATE_MANIFEST}")" || die "candidate package SHA missing" + [[ "${PACKAGE_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "invalid candidate package SHA" + jq -e '.source_sha | test("^[0-9a-f]{40}$")' "${VALIDATION_CANDIDATE_MANIFEST}" >/dev/null \ + || die "candidate source commit missing" + [ "${VALIDATION_TESTING_COMMIT}" = "$(git -C "${SCRIPT_DIR}" rev-parse HEAD)" ] \ + || die "testing commit does not match this checkout" + mkdir -p "${VALIDATION_ARTIFACTS_DIR}" + fi + fi [ "${#SELECTED_TOPOLOGIES[@]}" -gt 0 ] || die "no topology selected (--all-topologies / --topology)" IFS=',' read -r -a BACKENDS <<<"${backends_arg}" [ "${#BACKENDS[@]}" -gt 0 ] || die "no backends selected (--backends local,vault-kv2)" @@ -2192,6 +2325,7 @@ main() { need_cmd jq "JSON processor" need_cmd curl "HTTP client" need_cmd openssl "OpenSSL (SigV4)" + if [ -n "${VALIDATION_CASE}" ]; then need_cmd python3 "validation result writer"; fi log "nodes: ${NODES[*]} package: $(resolve_package_url)" log "backends: ${BACKENDS[*]} topologies: ${SELECTED_TOPOLOGIES[*]}" @@ -2205,17 +2339,41 @@ main() { vault_stop vault_start fi + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + validation_write_result RUNNING "validation case started" PENDING + fi env_setup "${topo}" + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + VALIDATION_BINARY_SHA256="$(printf 'sha256sum /usr/bin/rustfs\n' | run_remote "${NODES[0]}" | awk '{print $1}')" + [[ "${VALIDATION_BINARY_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "installed RustFS binary SHA-256 unavailable" + fi PASS=0; FAIL=0; UNSUPPORTED=0; FAILED_TESTS=() KMS_KEY_ID="" - run_all_tests + if [ -n "${VALIDATION_CASE}" ]; then + run_test "${VALIDATION_CASE}" "SseKmsRoundTrip" test_validation_kms_sse_kms_round_trip + else + run_all_tests + fi results_summary record_topology_result "${topo}/${KMS_BACKEND}" + local cleanup_status=PASS if [ "${KEEP_ENV}" -eq 0 ]; then - env_teardown "${topo}" + env_teardown "${topo}" || cleanup_status=FAIL if [ "${KMS_BACKEND}" = "vault-kv2" ]; then vault_stop; fi fi + if [ "${VALIDATION_OBJECT_CLEANUP_STATUS}" != PASS ]; then cleanup_status=FAIL; fi + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + validation_write_evidence + if [ "${FAIL}" -eq 0 ] && [ "${UNSUPPORTED}" -eq 0 ] && [ "${cleanup_status}" = PASS ]; then + validation_write_result PASS "" PASS + elif [ "${FAIL}" -eq 0 ] && [ "${UNSUPPORTED}" -eq 0 ]; then + validation_write_result INFRA_ERROR "environment teardown failed" FAIL + else + validation_write_result FAIL "case failed or unsupported; inspect evidence" "${cleanup_status}" + fi + fi if [ "${FAIL}" -gt 0 ]; then overall_fail=1; fi + if [ "${UNSUPPORTED}" -gt 0 ] || [ "${cleanup_status}" != PASS ]; then overall_fail=1; fi done done final_summary @@ -2223,4 +2381,6 @@ main() { log "all done" } -main "$@" +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/test_validation_kms.sh b/test_validation_kms.sh new file mode 100644 index 00000000..b7375d3a --- /dev/null +++ b/test_validation_kms.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/rustfs-kms-test.sh" + +MOCK_SOURCE="" +MOCK_BUCKET_DELETED=0 +MOCK_LEAK=0 + +kms_create_key() { + KMS_KEY_ID="key-validation-1" +} + +aws_run() { + local operation="${2:-}" argument previous="" key="" body="" kid="" + AWS_CODE=0 + AWS_OUT='{}' + for argument in "$@"; do + case "${previous}" in + --key) key="${argument}" ;; + --body) body="${argument}" ;; + --ssekms-key-id) kid="${argument}" ;; + esac + previous="${argument}" + done + case "${operation}" in + create-bucket) ;; + put-object) + if [[ "${kid}" == missing-* ]]; then + AWS_CODE=255 + AWS_OUT='InvalidKMSKey' + if [ "${MOCK_LEAK}" -eq 1 ]; then AWS_OUT="${AWS_OUT} ${marker}"; fi + else + MOCK_SOURCE="${body}" + AWS_OUT="$(jq -nc --arg kid "${kid}" '{ServerSideEncryption:"aws:kms",SSEKMSKeyId:$kid}')" + fi + ;; + head-object) + if [ "${key}" = rejected ]; then + AWS_CODE=255 + AWS_OUT='NotFound' + else + AWS_OUT="$(jq -nc --arg kid "${KMS_KEY_ID}" --argjson bytes "$(wc -c < "${MOCK_SOURCE}")" \ + '{ServerSideEncryption:"aws:kms",SSEKMSKeyId:$kid,ContentLength:$bytes,Metadata:{validation:"kms-roundtrip"}}')" + fi + ;; + get-object) + cp "${MOCK_SOURCE}" "${@: -1}" + ;; + delete-object) ;; + delete-bucket) MOCK_BUCKET_DELETED=$((MOCK_BUCKET_DELETED + 1)) ;; + *) printf 'unexpected AWS operation: %s\n' "${operation}" >&2; return 1 ;; + esac +} + +test_validation_kms_sse_kms_round_trip +[ "${VALIDATION_OBJECT_CLEANUP_STATUS}" = PASS ] +[ "${MOCK_BUCKET_DELETED}" -eq 1 ] +jq -e '.key_id == "key-validation-1" and .metadata.validation == "kms-roundtrip" and .bytes > 4096' \ + "${TEST_TMP}/validation-fixture.json" >/dev/null + +MOCK_LEAK=1 +if test_validation_kms_sse_kms_round_trip; then + printf 'plaintext leak passed\n' >&2; exit 1 +fi +[ "${MOCK_BUCKET_DELETED}" -eq 2 ] || { printf 'failed case did not clean bucket\n' >&2; exit 1; } + +"${SCRIPT_DIR}/rustfs-kms-test.sh" --validation-case kms.sse-kms-round-trip \ + --dry-run --yes --package-url https://example.invalid/rustfs.deb \ + > "${TEST_TMP}/validation-dry-run.log" +grep -q '\[DRY-RUN\] kms.sse-kms-round-trip SseKmsRoundTrip' "${TEST_TMP}/validation-dry-run.log" +if grep -q '\[DRY-RUN\] KMS-' "${TEST_TMP}/validation-dry-run.log"; then + printf 'validation mode scheduled legacy KMS cases\n' >&2; exit 1 +fi From 30ee6c4897971a066b9bbcb620365f52abd8c1e6 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 17:33:21 +0800 Subject: [PATCH 004/343] feat(validation): add tier case and pinned runner dispatch --- docs/validation-migration-map.md | 31 ++++++- rustfs-tier-test.sh | 149 ++++++++++++++++++++++++++++++- scripts/run_validation_case.sh | 53 +++++++++++ scripts/validation_report.py | 16 +++- test_validation_migration.py | 10 +++ test_validation_tier.sh | 77 ++++++++++++++++ 6 files changed, 329 insertions(+), 7 deletions(-) create mode 100755 scripts/run_validation_case.sh create mode 100644 test_validation_tier.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 160be8c7..d9c4ef23 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -34,13 +34,42 @@ KMS single-disk deployment. It checks the same object's encryption key, metadata, and payload, then requires an invalid-key PUT to fail without exposing plaintext or key material. Its live candidate run is pending. +`tiering.transition-and-restore` has a dedicated executor using the Tier +suite's existing `single-single` source and warm-tier nodes. It verifies the +same object's transition state, readable bytes after transition, restore +state, and SHA-256 after restore. The case is recorded under the source +`lifecycle-b` environment profile. Its live candidate run is pending. + +"Existing validation environment" means the fleet used by `auto-testing` +and the RustFS `smoke-testing` workflows, with `RUSTFS_NODES` and +`RUSTFS_SSH_USER` supplied there. For Heal, reuse the nodes deployed by +`rustfs_heal_test.sh`; the source H1-H6 names describe required topology and +fault capabilities, not a separate SSH fleet. The ten `heal-extended` cases +have no source target-environment field, so their runtime binding must record +the actual auto-testing Heal nodes and topology before any result is scored. +The source's standalone ZIP, port 9100/9200, and dedicated systemd profiles +are reference implementations for assertions, not deployment targets here. + +The RustFS Functional Chain workflow has a manual validation lane on the same +`smoke-testing` runner and `rustfs-shared-functional-tests-v2` concurrency +group. Set `build_run_id` and `build_run_attempt` to one successful nightly +build with its retained candidate artifact, `auto_testing_sha` to an exact +commit, and `validation_case` to one migrated ID. The lane resolves the +immutable package URL and SHA-256 from that build attempt, then calls +`scripts/run_validation_case.sh`. The dispatcher rejects an ID without an +executor before touching the fleet and runs the result report with +`--gate-case`, so a missing, failed, or incomplete selected case fails the +workflow. Each run uploads its candidate identity, canonical result, and +case evidence. The four currently dispatchable IDs are the S3, scanner, KMS, +and tiering cases above; the other 154 IDs remain unverified. + The eight planned cases have these concrete reuse candidates: | Validation ID | Candidate IDs | Specific item to verify | | --- | --- | --- | | `scanner.usage-convergence` | STOR-134 | Dedicated executor added; live candidate evidence is pending. | | `kms.sse-kms-round-trip` | KMS-104, KMS-301, KMS-214 | Dedicated executor added; live candidate evidence is pending. | -| `tiering.transition-and-restore` | TIER-105, TIER-107 | Compare remote tier restore and object checksum. | +| `tiering.transition-and-restore` | TIER-105, TIER-107 | Dedicated executor added; live candidate evidence is pending. | | `events.delivery` | EVT-103, EVT-104 | Freeze and verify the exact delivery count contract. | | `audit.request-correlation` | AUD-102, AUD-104, AUD-106 | Verify request correlation and secret redaction in the same event trail. | | `iam.policy-enforcement` | IAM-105, IAM-109 | Verify explicit deny and allowed action on the same principal/policy fixture. | diff --git a/rustfs-tier-test.sh b/rustfs-tier-test.sh index 0d17bdbc..302267b0 100755 --- a/rustfs-tier-test.sh +++ b/rustfs-tier-test.sh @@ -365,6 +365,11 @@ RESULTS_DIR="" RESULT_LOG_DIR="" PROVENANCE_FILE="" RESIDUE_FILE="" +VALIDATION_CASE="" +VALIDATION_ARTIFACTS_DIR="" +VALIDATION_CANDIDATE_MANIFEST="" +VALIDATION_TESTING_COMMIT="" +VALIDATION_BINARY_SHA256="" # ==================== Helpers ==================== @@ -1511,6 +1516,39 @@ run_test() { write_case_result "${status}" "${rc}" "$([ "${cleanup_rc}" -eq 0 ] && printf PASS || printf FAIL)" "$((SECONDS-started))" "${cleanup_detail}" } +validation_write_evidence() { + local evidence_dir="${VALIDATION_ARTIFACTS_DIR}/evidence/${VALIDATION_CASE}" item + mkdir -p "${evidence_dir}" + cp "${TEST_TMP}/test.out" "${evidence_dir}/test.log" + cp "${RESULTS_DIR}/${CURRENT_TOPOLOGY}--${VALIDATION_CASE}.json" "${evidence_dir}/suite-result.json" + for item in validation-transition.json validation-head.json validation-restore-head.json validation-object.json; do + if [ -f "${TEST_TMP}/${item}" ]; then cp "${TEST_TMP}/${item}" "${evidence_dir}/${item}"; fi + done +} + +validation_write_result() { + local verdict="$1" reason="$2" cleanup_status="$3" + local args=("${SCRIPT_DIR}/scripts/validation_result.py" + --case-id "${VALIDATION_CASE}" --profile lifecycle-b + --result-file "${VALIDATION_ARTIFACTS_DIR}/cases/${VALIDATION_CASE}.json" + --verdict "${verdict}" --cleanup-status "${cleanup_status}") + if [ -n "${reason}" ]; then args+=(--reason "${reason}"); fi + if [ "${verdict}" != RUNNING ]; then + args+=(--candidate-manifest "${VALIDATION_CANDIDATE_MANIFEST}" + --rc-bin "${RC_BIN}" --testing-commit "${VALIDATION_TESTING_COMMIT}" + --binary-sha256 "${VALIDATION_BINARY_SHA256}" + --evidence "evidence/${VALIDATION_CASE}/test.log" + --evidence "evidence/${VALIDATION_CASE}/suite-result.json") + fi + if [ "${verdict}" = PASS ]; then + local item + for item in validation-transition.json validation-head.json validation-restore-head.json validation-object.json; do + args+=(--evidence "evidence/${VALIDATION_CASE}/${item}") + done + fi + python3 "${args[@]}" +} + rc_run() { if RC_OUT="$(run_bounded "${COMMAND_TIMEOUT}" "${RC_BIN}" "$@" 2>&1)"; then RC_CODE=0; else RC_CODE=$?; fi return "${RC_CODE}" @@ -1956,6 +1994,61 @@ final_summary() { # ==================== Tier / Event / Audit test cases ==================== +test_validation_tiering_transition_and_restore() { + local bucket="val-tier-${RUN_TOKEN}" source="${TEST_TMP}/validation-source.bin" + local downloaded="${TEST_TMP}/validation-downloaded.bin" key=round-trip.bin + local transition job waited=0 source_sha downloaded_sha transitioned_sha + new_bucket "${bucket}" || return 1 + add_tier rustfs "${TIER_NAME}" "${TARGET_ENDPOINT}" "${TIER_BUCKET}" "${ACCESS_KEY}" "${SECRET_KEY}" + assert_rc_ok || return 1 + rc_run ilm rule add "local/${bucket}" --storage-class "${TIER_NAME}" --transition-days 0 + assert_rc_ok || return 1 + make_test_file "${source}" 1048576 + source_sha="$(sha256_file "${source}")" + [[ "${source_sha}" =~ ^[0-9a-f]{64}$ ]] || { echo "source checksum unavailable"; return 1; } + aws_run s3api put-object --bucket "${bucket}" --key "${key}" --body "${source}" + assert_aws_ok || return 1 + transition="$(run_transition "${bucket}")" || return 1 + printf '%s\n' "${transition}" > "${TEST_TMP}/validation-transition.json" + job="$(printf '%s' "${transition}" | jq -r '.job_id // empty')" + if [ -n "${job}" ]; then wait_transition_job "${job}" >/dev/null || return 1; fi + while [ "${waited}" -lt 120 ]; do + aws_run s3api head-object --bucket "${bucket}" --key "${key}" + if [ "${AWS_CODE}" -eq 0 ] && printf '%s' "${AWS_OUT}" | jq -e --arg tier "${TIER_NAME}" \ + '.StorageClass == $tier' >/dev/null 2>&1; then break; fi + sleep 3; waited=$((waited+3)) + done + [ "${waited}" -lt 120 ] || { echo "object did not reach warm tier: ${AWS_OUT}"; return 1; } + printf '%s\n' "${AWS_OUT}" > "${TEST_TMP}/validation-head.json" + aws_run s3api get-object --bucket "${bucket}" --key "${key}" "${downloaded}" + assert_aws_ok || return 1 + downloaded_sha="$(sha256_file "${downloaded}")" + [ "${downloaded_sha}" = "${source_sha}" ] || { echo "transitioned object checksum mismatch"; return 1; } + transitioned_sha="${downloaded_sha}" + aws_run s3api restore-object --bucket "${bucket}" --key "${key}" --restore-request '{"Days":1}' + assert_aws_ok || return 1 + waited=0 + while [ "${waited}" -lt 90 ]; do + aws_run s3api head-object --bucket "${bucket}" --key "${key}" + if [ "${AWS_CODE}" -eq 0 ] && printf '%s' "${AWS_OUT}" | jq -e \ + '(.Restore // ."x-amz-restore" // "") | startswith("ongoing-request=\"false\"")' \ + >/dev/null 2>&1; then break; fi + sleep 3; waited=$((waited+3)) + done + [ "${waited}" -lt 90 ] || { echo "restore state did not appear: ${AWS_OUT}"; return 1; } + printf '%s\n' "${AWS_OUT}" > "${TEST_TMP}/validation-restore-head.json" + aws_run s3api get-object --bucket "${bucket}" --key "${key}" "${downloaded}" + assert_aws_ok || return 1 + downloaded_sha="$(sha256_file "${downloaded}")" + [ "${downloaded_sha}" = "${source_sha}" ] || { echo "restored object checksum mismatch"; return 1; } + jq -nc --arg bucket "${bucket}" --arg key "${key}" --arg source_sha256 "${source_sha}" \ + --arg transitioned_sha256 "${transitioned_sha}" --arg restored_sha256 "${downloaded_sha}" \ + --arg tier "${TIER_NAME}" \ + '{bucket:$bucket,key:$key,source_sha256:$source_sha256,transitioned_sha256:$transitioned_sha256, + restored_sha256:$restored_sha256,tier:$tier,restore_complete:true}' \ + > "${TEST_TMP}/validation-object.json" +} + run_all_tests() { run_test TIER-101 "TierAddValidation" test_tier101 run_test TIER-102 "TierListInfo" test_tier102 @@ -2786,6 +2879,10 @@ Usage: --rc-bin Exact rc binary to execute (default: resolve rc from PATH once) --expected-rc-sha256 Fail before setup unless the rc binary matches this SHA-256 --artifacts-dir Persist structured case results, provenance, logs, and residue + --validation-case Run one canonical validation case on the existing Tier topology + --validation-artifacts-dir Unique canonical result and evidence directory + --candidate-manifest Immutable candidate URL, package SHA-256 and source SHA + --testing-commit Exact auto-testing checkout commit --keep-env Do not tear down the environment after the last run --ssh-user SSH user for the test nodes (default: azureuser) --log-file Tee output to a log file @@ -2809,6 +2906,10 @@ main() { --rc-bin) RC_BIN_REQUEST="$1"; shift ;; --expected-rc-sha256) RUSTFS_EXPECTED_RC_SHA256="$1"; shift ;; --artifacts-dir) ARTIFACTS_DIR="$1"; shift ;; + --validation-case) VALIDATION_CASE="$1"; shift ;; + --validation-artifacts-dir) VALIDATION_ARTIFACTS_DIR="$1"; shift ;; + --candidate-manifest) VALIDATION_CANDIDATE_MANIFEST="$1"; shift ;; + --testing-commit) VALIDATION_TESTING_COMMIT="$1"; shift ;; --keep-env) KEEP_ENV=1 ;; --ssh-user) SSH_USER="$1"; shift ;; --log-file) LOG_FILE="$1"; shift ;; @@ -2822,6 +2923,27 @@ main() { if [ "${all}" -eq 1 ]; then SELECTED_TOPOLOGIES=("${TOPO_SINGLE_SINGLE}" "${TOPO_SINGLE_MULTI}") fi + if [ -n "${VALIDATION_CASE}" ]; then + [ "${VALIDATION_CASE}" = tiering.transition-and-restore ] || die "unknown validation case: ${VALIDATION_CASE}" + [ "${all}" -eq 0 ] || die "--validation-case cannot run all topologies" + [ "${#SELECTED_TOPOLOGIES[@]}" -eq 1 ] || die "--validation-case requires one topology" + [ "${SELECTED_TOPOLOGIES[0]}" = "${TOPO_SINGLE_SINGLE}" ] || die "validation case requires single-single topology" + [ "${KEEP_ENV}" -eq 0 ] || die "--validation-case requires environment teardown" + if [ "${DRY_RUN}" -eq 0 ]; then + [ -n "${VALIDATION_ARTIFACTS_DIR}" ] || die "--validation-case requires --validation-artifacts-dir" + [ -n "${VALIDATION_CANDIDATE_MANIFEST}" ] || die "--validation-case requires --candidate-manifest" + [ -n "${VALIDATION_TESTING_COMMIT}" ] || die "--validation-case requires --testing-commit" + [ ! -e "${VALIDATION_ARTIFACTS_DIR}" ] || die "validation artifacts directory already exists" + PACKAGE_URL="$(jq -er '.package_url' "${VALIDATION_CANDIDATE_MANIFEST}")" || die "candidate URL missing" + PACKAGE_SHA256="$(jq -er '.package_sha256' "${VALIDATION_CANDIDATE_MANIFEST}")" || die "candidate package SHA missing" + [[ "${PACKAGE_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "invalid candidate package SHA" + jq -e '.source_sha | test("^[0-9a-f]{40}$")' "${VALIDATION_CANDIDATE_MANIFEST}" >/dev/null \ + || die "candidate source commit missing" + [ "${VALIDATION_TESTING_COMMIT}" = "$(git -C "${SCRIPT_DIR}" rev-parse HEAD)" ] \ + || die "testing commit does not match this checkout" + mkdir -p "${VALIDATION_ARTIFACTS_DIR}" + fi + fi [ "${#SELECTED_TOPOLOGIES[@]}" -gt 0 ] || die "no topology selected (--all-topologies / --topology)" local _p @@ -2849,15 +2971,36 @@ main() { for topo in "${SELECTED_TOPOLOGIES[@]}"; do CURRENT_TOPOLOGY="${topo}" log "== topology: ${topo} ==" + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + validation_write_result RUNNING "validation case started" PENDING + fi env_setup "${topo}" + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + VALIDATION_BINARY_SHA256="$(printf 'sha256sum /usr/bin/rustfs\n' | run_remote "${SOURCE_NODE}" | awk '{print $1}')" + [[ "${VALIDATION_BINARY_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "installed RustFS binary SHA-256 unavailable" + fi PASS=0; FAIL=0; UNSUPPORTED=0; FAILED_TESTS=() - run_all_tests + if [ -n "${VALIDATION_CASE}" ]; then + run_test "${VALIDATION_CASE}" "TransitionAndRestore" test_validation_tiering_transition_and_restore + else + run_all_tests + fi results_summary record_topology_result "${topo}" - if [ "${KEEP_ENV}" -eq 0 ]; then - env_teardown "${topo}" + local cleanup_status=PASS + if [ "${KEEP_ENV}" -eq 0 ]; then env_teardown "${topo}" || cleanup_status=FAIL; fi + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + validation_write_evidence + if [ "${FAIL}" -eq 0 ] && [ "${UNSUPPORTED}" -eq 0 ] && [ "${cleanup_status}" = PASS ]; then + validation_write_result PASS "" PASS + elif [ "${FAIL}" -eq 0 ] && [ "${UNSUPPORTED}" -eq 0 ]; then + validation_write_result INFRA_ERROR "environment teardown failed" FAIL + else + validation_write_result FAIL "case failed or unsupported; inspect evidence" "${cleanup_status}" + fi fi if [ "${FAIL}" -gt 0 ]; then overall_fail=1; fi + if [ "${UNSUPPORTED}" -gt 0 ] || [ "${cleanup_status}" != PASS ]; then overall_fail=1; fi done final_summary [ "${overall_fail}" -eq 0 ] || exit 1 diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh new file mode 100755 index 00000000..ad741b29 --- /dev/null +++ b/scripts/run_validation_case.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if [ "$#" -ne 4 ]; then + printf 'usage: %s CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR\n' "$0" >&2 + exit 2 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CASE_ID="$1" +CANDIDATE_MANIFEST="$2" +TESTING_COMMIT="$3" +ARTIFACTS_DIR="$4" + +case "${CASE_ID}" in + s3.basic-object-round-trip) + SUITE=rustfs-s3-compat-test.sh; EXTRA_ARGS=() ;; + scanner.usage-convergence) + SUITE=rustfs-storage-test.sh; EXTRA_ARGS=() ;; + kms.sse-kms-round-trip) + SUITE=rustfs-kms-test.sh; EXTRA_ARGS=() ;; + tiering.transition-and-restore) + SUITE=rustfs-tier-test.sh; EXTRA_ARGS=(--topology single-single) ;; + *) + printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 + exit 2 ;; +esac + +[ -f "${CANDIDATE_MANIFEST}" ] || { printf 'candidate manifest missing\n' >&2; exit 2; } +[[ "${TESTING_COMMIT}" =~ ^[0-9a-f]{40}$ ]] || { printf 'invalid testing commit\n' >&2; exit 2; } +[ "${TESTING_COMMIT}" = "$(git -C "${SCRIPT_DIR}" rev-parse HEAD)" ] || { + printf 'testing commit does not match checkout\n' >&2; exit 2; +} +[ ! -e "${ARTIFACTS_DIR}" ] || { printf 'artifacts directory already exists\n' >&2; exit 2; } + +mkdir -p -- "${ARTIFACTS_DIR}" +case_dir="${ARTIFACTS_DIR}/${CASE_ID}" +suite_rc=0 +"${SCRIPT_DIR}/${SUITE}" --validation-case "${CASE_ID}" "${EXTRA_ARGS[@]}" \ + --candidate-manifest "${CANDIDATE_MANIFEST}" \ + --testing-commit "${TESTING_COMMIT}" \ + --validation-artifacts-dir "${case_dir}" -y \ + || suite_rc=$? + +report_rc=0 +python3 "${SCRIPT_DIR}/scripts/validation_report.py" \ + --catalog "${SCRIPT_DIR}/validation-cases.json" \ + --results-dir "${case_dir}/cases" \ + --output "${ARTIFACTS_DIR}/report.md" \ + --gate-case "${CASE_ID}" || report_rc=$? + +if [ "${suite_rc}" -ne 0 ]; then exit "${suite_rc}"; fi +exit "${report_rc}" diff --git a/scripts/validation_report.py b/scripts/validation_report.py index 3678270c..c8325406 100644 --- a/scripts/validation_report.py +++ b/scripts/validation_report.py @@ -123,19 +123,25 @@ def check_result( return errors -def validate_results(catalog: dict, directory: Path, *, gate_statuses: set[str]) -> tuple[list[dict], list[str]]: +def validate_results( + catalog: dict, directory: Path, *, gate_statuses: set[str], gate_cases: set[str] | None = None, +) -> tuple[list[dict], list[str]]: validate_snapshot(catalog) cases = { case["id"]: (case, module["target_environments"]) for module in catalog["modules"] for case in module["cases"] } + gate_cases = gate_cases or set() + unknown = gate_cases - set(cases) + if unknown: + raise ValueError(f"unknown gate case: {', '.join(sorted(unknown))}") records, errors = load_results(directory) for case_id in sorted(set(records) - set(cases)): errors.append(f"unexpected case ID: {case_id}") rows = [] for case_id, (case, target_environments) in cases.items(): result = records.get(case_id) - gated = case["source_status"] in gate_statuses + gated = case["source_status"] in gate_statuses or case_id in gate_cases if result is None: if gated: errors.append(f"missing gate result: {case_id}") @@ -168,10 +174,14 @@ def main() -> int: parser.add_argument("--results-dir", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--gate-status", action="append", choices=sorted(GATE_SOURCE_STATUSES), default=[]) + parser.add_argument("--gate-case", action="append", default=[]) args = parser.parse_args() try: catalog = json.loads(args.catalog.read_text(encoding="utf-8")) - rows, errors = validate_results(catalog, args.results_dir, gate_statuses=set(args.gate_status)) + rows, errors = validate_results( + catalog, args.results_dir, + gate_statuses=set(args.gate_status), gate_cases=set(args.gate_case), + ) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(render_report(rows, errors), encoding="utf-8") except (OSError, ValueError, json.JSONDecodeError) as exc: diff --git a/test_validation_migration.py b/test_validation_migration.py index dbfcf002..6a999998 100644 --- a/test_validation_migration.py +++ b/test_validation_migration.py @@ -59,6 +59,16 @@ def test_missing_implemented_case_fails_gate(self) -> None: _, errors = validate_results(self.catalog, self.results, gate_statuses={"implemented"}) self.assertIn(f"missing gate result: {self.case['id']}", errors) + def test_selected_case_requires_live_pass(self) -> None: + _, errors = validate_results( + self.catalog, self.results, gate_statuses=set(), gate_cases={self.case["id"]}, + ) + self.assertIn(f"missing gate result: {self.case['id']}", errors) + with self.assertRaisesRegex(ValueError, "unknown gate case"): + validate_results( + self.catalog, self.results, gate_statuses=set(), gate_cases={"missing.case"}, + ) + def test_duplicate_and_running_fail(self) -> None: self.write_result("one.json", verdict="RUNNING") self.write_result("two.json", verdict="RUNNING") diff --git a/test_validation_tier.sh b/test_validation_tier.sh new file mode 100644 index 00000000..0f6d0e17 --- /dev/null +++ b/test_validation_tier.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-validation-tier.XXXXXX")" +export RUSTFS_TIER_TEST_TMP="${TEST_ROOT}/source" +export TIER_SOURCE_ONLY=1 +# shellcheck source=rustfs-tier-test.sh +source "${SCRIPT_DIR}/rustfs-tier-test.sh" +trap 'rm -rf -- "${TEST_ROOT}"' EXIT + +RUN_TOKEN=unit-1 +MOCK_CORRUPT=0 +MOCK_GET_COUNT=0 + +new_bucket() { :; } +add_tier() { RC_CODE=0; } +rc_run() { RC_CODE=0; } +run_transition() { printf '{"job_id":"job-1"}\n'; } +wait_transition_job() { [ "$1" = job-1 ]; } +sleep() { :; } + +aws_run() { + local operation="${2:-}" argument previous="" output="" + AWS_CODE=0 + AWS_OUT='{}' + for argument in "$@"; do + if [ "${previous}" = "--bucket" ]; then + [ "${argument}" = "val-tier-unit-1" ] || return 1 + fi + previous="${argument}" + done + case "${operation}" in + put-object) ;; + head-object) + if [ "${MOCK_GET_COUNT}" -ge 1 ]; then + AWS_OUT='{"StorageClass":"WARM","Restore":"ongoing-request=\"false\""}' + else + AWS_OUT='{"StorageClass":"WARM"}' + fi + ;; + get-object) + MOCK_GET_COUNT=$((MOCK_GET_COUNT + 1)) + output="${@: -1}" + if [ "${MOCK_CORRUPT}" -eq 1 ]; then + printf 'corrupt' > "${output}" + else + cp "${TEST_TMP}/validation-source.bin" "${output}" + fi + ;; + restore-object) ;; + *) printf 'unexpected AWS operation: %s\n' "${operation}" >&2; return 1 ;; + esac +} + +test_validation_tiering_transition_and_restore +[ "${MOCK_GET_COUNT}" -eq 2 ] +jq -e '.tier == "WARM" and .restore_complete == true and + (.source_sha256 | test("^[0-9a-f]{64}$")) and + .source_sha256 == .transitioned_sha256 and .source_sha256 == .restored_sha256' \ + "${TEST_TMP}/validation-object.json" >/dev/null + +MOCK_CORRUPT=1 +MOCK_GET_COUNT=0 +if test_validation_tiering_transition_and_restore; then + printf 'corrupted transitioned object passed\n' >&2 + exit 1 +fi + +TIER_SOURCE_ONLY=0 "${SCRIPT_DIR}/rustfs-tier-test.sh" --validation-case tiering.transition-and-restore \ + --topology single-single --dry-run --yes --package-url https://example.invalid/rustfs.deb \ + > "${TEST_ROOT}/validation-dry-run.log" +grep -q '\[DRY-RUN\] tiering.transition-and-restore TransitionAndRestore' "${TEST_ROOT}/validation-dry-run.log" +if grep -q '\[DRY-RUN\] TIER-' "${TEST_ROOT}/validation-dry-run.log"; then + printf 'validation mode scheduled legacy tier cases\n' >&2 + exit 1 +fi From 9d5b55c162fa48908d947cb9b8365a6fdfa48cca Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 17:43:15 +0800 Subject: [PATCH 005/343] fix: initialize tier validation script directory --- rustfs-tier-test.sh | 1 + test_validation_tier.sh | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/rustfs-tier-test.sh b/rustfs-tier-test.sh index 302267b0..f39968cf 100755 --- a/rustfs-tier-test.sh +++ b/rustfs-tier-test.sh @@ -20,6 +20,7 @@ # ./rustfs-tier-test.sh --topology single-single -y --version 1.0.0-rc.4-preview.1 # set -Eeuo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Per-run temp dir (shared runners may leave stale /tmp files owned by other users). # Tests may inject an isolated directory while sourcing this file. diff --git a/test_validation_tier.sh b/test_validation_tier.sh index 0f6d0e17..27a23ba7 100644 --- a/test_validation_tier.sh +++ b/test_validation_tier.sh @@ -1,12 +1,13 @@ #!/usr/bin/env bash set -Eeuo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-validation-tier.XXXXXX")" export RUSTFS_TIER_TEST_TMP="${TEST_ROOT}/source" export TIER_SOURCE_ONLY=1 # shellcheck source=rustfs-tier-test.sh -source "${SCRIPT_DIR}/rustfs-tier-test.sh" +source "${TEST_SCRIPT_DIR}/rustfs-tier-test.sh" +[ "${SCRIPT_DIR}" = "${TEST_SCRIPT_DIR}" ] trap 'rm -rf -- "${TEST_ROOT}"' EXIT RUN_TOKEN=unit-1 @@ -67,7 +68,7 @@ if test_validation_tiering_transition_and_restore; then exit 1 fi -TIER_SOURCE_ONLY=0 "${SCRIPT_DIR}/rustfs-tier-test.sh" --validation-case tiering.transition-and-restore \ +TIER_SOURCE_ONLY=0 "${TEST_SCRIPT_DIR}/rustfs-tier-test.sh" --validation-case tiering.transition-and-restore \ --topology single-single --dry-run --yes --package-url https://example.invalid/rustfs.deb \ > "${TEST_ROOT}/validation-dry-run.log" grep -q '\[DRY-RUN\] tiering.transition-and-restore TransitionAndRestore' "${TEST_ROOT}/validation-dry-run.log" From 4bdf55fbcd09a54f50803bc99b4306519759a7a8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 17:49:09 +0800 Subject: [PATCH 006/343] fix: isolate tier validation setup from notify modules --- rustfs-tier-test.sh | 28 +++++++++++++++++++--------- test_validation_tier.sh | 2 +- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/rustfs-tier-test.sh b/rustfs-tier-test.sh index f39968cf..ca339204 100755 --- a/rustfs-tier-test.sh +++ b/rustfs-tier-test.sh @@ -238,6 +238,10 @@ suite_exit() { if [ "${CASE_ACTIVE:-0}" -eq 1 ] && declare -F case_cleanup >/dev/null 2>&1; then case_cleanup "signal-or-exit" >> "${TEST_TMP}/test.out" 2>&1 || true fi + if [ "${VALIDATION_ENV_ACTIVE:-0}" -eq 1 ]; then + VALIDATION_ENV_ACTIVE=0 + env_teardown "${CURRENT_TOPOLOGY}" || exit_code=1 + fi if finish_log_capture "${exit_code}"; then exit_code=0 else @@ -371,6 +375,7 @@ VALIDATION_ARTIFACTS_DIR="" VALIDATION_CANDIDATE_MANIFEST="" VALIDATION_TESTING_COMMIT="" VALIDATION_BINARY_SHA256="" +VALIDATION_ENV_ACTIVE=0 # ==================== Helpers ==================== @@ -1120,14 +1125,15 @@ EOF # Create the warm-tier bucket on the target instance. aws_target_run s3api create-bucket --bucket "${TIER_BUCKET}" \ || die "failed to create unique warm-tier bucket ${TIER_BUCKET}: ${AWS_OUT}" - receiver_start - mqtt_start - # Enable the notify + audit modules via the (API-managed) module switch so - # EVT-108 / AUD-107 can toggle them later. Not env-managed. - local mbody mcode - mbody="$(admin_api PUT /rustfs/admin/v3/module-switches "" '{"notify_enabled":true,"audit_enabled":true}')"; mcode="$(admin_api_code)" - [ "${mcode}" = "200" ] || { echo "enable modules -> HTTP ${mcode}: ${mbody}"; die "failed to enable notify/audit modules"; } - wait_targets_ready + if [ "${VALIDATION_CASE}" != tiering.transition-and-restore ]; then + receiver_start + mqtt_start + # EVT and AUD use the API-managed module switches. + local mbody mcode + mbody="$(admin_api PUT /rustfs/admin/v3/module-switches "" '{"notify_enabled":true,"audit_enabled":true}')"; mcode="$(admin_api_code)" + [ "${mcode}" = "200" ] || { echo "enable modules -> HTTP ${mcode}: ${mbody}"; die "failed to enable notify/audit modules"; } + wait_targets_ready + fi log "rc aliases local/target; warm bucket ${TIER_BUCKET} ready" fi } @@ -2974,6 +2980,7 @@ main() { log "== topology: ${topo} ==" if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then validation_write_result RUNNING "validation case started" PENDING + VALIDATION_ENV_ACTIVE=1 fi env_setup "${topo}" if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then @@ -2989,7 +2996,10 @@ main() { results_summary record_topology_result "${topo}" local cleanup_status=PASS - if [ "${KEEP_ENV}" -eq 0 ]; then env_teardown "${topo}" || cleanup_status=FAIL; fi + if [ "${KEEP_ENV}" -eq 0 ]; then + VALIDATION_ENV_ACTIVE=0 + env_teardown "${topo}" || cleanup_status=FAIL + fi if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then validation_write_evidence if [ "${FAIL}" -eq 0 ] && [ "${UNSUPPORTED}" -eq 0 ] && [ "${cleanup_status}" = PASS ]; then diff --git a/test_validation_tier.sh b/test_validation_tier.sh index 27a23ba7..bfc82a1f 100644 --- a/test_validation_tier.sh +++ b/test_validation_tier.sh @@ -42,7 +42,7 @@ aws_run() { ;; get-object) MOCK_GET_COUNT=$((MOCK_GET_COUNT + 1)) - output="${@: -1}" + output="${*: -1}" if [ "${MOCK_CORRUPT}" -eq 1 ]; then printf 'corrupt' > "${output}" else From 154619892e7e5b145d128573b6a5de6839b8cc26 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 17:55:28 +0800 Subject: [PATCH 007/343] feat: add canonical event and audit validation executors --- README.md | 10 +++- docs/validation-migration-map.md | 35 +++++++----- rustfs-tier-test.sh | 59 ++++++++++++++++++-- scripts/run_validation_case.sh | 2 +- scripts/validation_webhooks.py | 96 ++++++++++++++++++++++++++++++++ test_validation_tier.sh | 7 +++ test_validation_webhooks.py | 47 ++++++++++++++++ 7 files changed, 236 insertions(+), 20 deletions(-) create mode 100644 scripts/validation_webhooks.py create mode 100644 test_validation_webhooks.py diff --git a/README.md b/README.md index 601cf686..1cedc3fd 100644 --- a/README.md +++ b/README.md @@ -32,11 +32,13 @@ report shows migration progress and still rejects malformed or false PASS results. The executable canonical cases currently include `s3.basic-object-round-trip`, -`scanner.usage-convergence`, and `kms.sse-kms-round-trip`. The S3 and KMS cases use single-disk +`scanner.usage-convergence`, `kms.sse-kms-round-trip`, +`tiering.transition-and-restore`, `events.delivery`, and +`audit.request-correlation`. The S3 and KMS cases use single-disk deployment. The scanner case uses the storage suite's single-node four-disk deployment, waits for a post-upload successful scanner cycle, and requires a complete usage snapshot with exactly 20 objects and 20,690 logical bytes. -Both require a candidate manifest with `source_sha`, `package_url`, and +Each requires a candidate manifest with `source_sha`, `package_url`, and `package_sha256`. The installed binary SHA-256 is measured on the test node and included in result provenance: @@ -71,6 +73,10 @@ The report's full implemented-case gate is intentionally not enabled by these runs. Run it with `--gate-status implemented` only after all 66 implemented source cases have verified executors and live results. +S3, scanner, KMS, and tiering have live PASS evidence against nightly build +`37958901959` attempt `1` on the current auto-testing fleet. Events and audit +have local executor tests but still need live verification. + ## Fault-tolerance suite `rustfs-fault-tolerance-test.sh` writes a machine-readable completion result diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index d9c4ef23..518fd60b 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -19,26 +19,34 @@ live run proves each source assertion on the required topology. `s3.basic-object-round-trip` now has a dedicated executor on `single-1disk`. Its Bash function combines bucket creation/removal, object PUT/HEAD/GET/LIST/ DELETE, payload SHA-256, metadata, ETag, and `NoSuchKey` checks on one fixture. -It has local mock coverage; live evidence against an immutable candidate is -still required before calling the migration verified. +It has local mock coverage and live PASS evidence in RustFS Actions run +`38042083422`. `scanner.usage-convergence` has a dedicated executor on `single-4disk`. It uses the storage suite's deployment and SigV4 Admin client, accelerates the scanner cycle, records the 20-object fixture, and checks a successful leader cycle after upload plus a newer complete usage snapshot with exact object and logical-byte counts. The old STOR-134 threshold remains a separate regression -test. This case also awaits a live candidate run. +test. Its live candidate run passed in RustFS Actions run `38042200106`. `kms.sse-kms-round-trip` has a dedicated executor using the existing local KMS single-disk deployment. It checks the same object's encryption key, metadata, and payload, then requires an invalid-key PUT to fail without -exposing plaintext or key material. Its live candidate run is pending. +exposing plaintext or key material. Its live candidate run passed in RustFS +Actions run `38042359938`. `tiering.transition-and-restore` has a dedicated executor using the Tier suite's existing `single-single` source and warm-tier nodes. It verifies the same object's transition state, readable bytes after transition, restore state, and SHA-256 after restore. The case is recorded under the source -`lifecycle-b` environment profile. Its live candidate run is pending. +`lifecycle-b` environment profile. Its live candidate run passed in RustFS +Actions run `38042759074`. + +`events.delivery` and `audit.request-correlation` now have dedicated executors +on the Tier suite's `single-single` topology. Event delivery is counted for a +unique object after a quiet window; audit checks matching PUT/GET records, +distinct request IDs, successful status, and absence of known secrets. Both +still require live candidate evidence before being counted as verified. "Existing validation environment" means the fleet used by `auto-testing` and the RustFS `smoke-testing` workflows, with `RUSTFS_NODES` and @@ -60,21 +68,22 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The four currently dispatchable IDs are the S3, scanner, KMS, -and tiering cases above; the other 154 IDs remain unverified. +case evidence. The six currently dispatchable IDs are the S3, scanner, KMS, +tiering, events, and audit cases above. Four have live PASS evidence; the other +154 IDs remain unverified. The eight planned cases have these concrete reuse candidates: | Validation ID | Candidate IDs | Specific item to verify | | --- | --- | --- | -| `scanner.usage-convergence` | STOR-134 | Dedicated executor added; live candidate evidence is pending. | -| `kms.sse-kms-round-trip` | KMS-104, KMS-301, KMS-214 | Dedicated executor added; live candidate evidence is pending. | -| `tiering.transition-and-restore` | TIER-105, TIER-107 | Dedicated executor added; live candidate evidence is pending. | -| `events.delivery` | EVT-103, EVT-104 | Freeze and verify the exact delivery count contract. | -| `audit.request-correlation` | AUD-102, AUD-104, AUD-106 | Verify request correlation and secret redaction in the same event trail. | +| `scanner.usage-convergence` | STOR-134 | Dedicated executor and live PASS evidence. | +| `kms.sse-kms-round-trip` | KMS-104, KMS-301, KMS-214 | Dedicated executor and live PASS evidence. | +| `tiering.transition-and-restore` | TIER-105, TIER-107 | Dedicated executor and live PASS evidence. | +| `events.delivery` | EVT-103, EVT-104 | Dedicated executor; live evidence pending. | +| `audit.request-correlation` | AUD-102, AUD-104, AUD-106 | Dedicated executor; live evidence pending. | | `iam.policy-enforcement` | IAM-105, IAM-109 | Verify explicit deny and allowed action on the same principal/policy fixture. | | `oidc.login-and-session` | OIDC-103 | The live Keycloak path can be skipped by the old suite; validation requires an actual session-boundary result. | -| `s3.basic-object-round-trip` | S3C-101, 201, 203, 205, 206, 207 | Tie CRUD, payload, metadata, ETag, and error semantics to the same fixture. | +| `s3.basic-object-round-trip` | S3C-101, 201, 203, 205, 206, 207 | Dedicated executor and live PASS evidence. | Migration work on an ID is complete only when its result contains the exact source assertion text in order, evidence paths for each assertion, the diff --git a/rustfs-tier-test.sh b/rustfs-tier-test.sh index ca339204..27d4e00d 100755 --- a/rustfs-tier-test.sh +++ b/rustfs-tier-test.sh @@ -1528,11 +1528,20 @@ validation_write_evidence() { mkdir -p "${evidence_dir}" cp "${TEST_TMP}/test.out" "${evidence_dir}/test.log" cp "${RESULTS_DIR}/${CURRENT_TOPOLOGY}--${VALIDATION_CASE}.json" "${evidence_dir}/suite-result.json" - for item in validation-transition.json validation-head.json validation-restore-head.json validation-object.json; do + for item in $(validation_evidence_items); do if [ -f "${TEST_TMP}/${item}" ]; then cp "${TEST_TMP}/${item}" "${evidence_dir}/${item}"; fi done } +validation_evidence_items() { + case "${VALIDATION_CASE}" in + tiering.transition-and-restore) + printf '%s\n' validation-transition.json validation-head.json validation-restore-head.json validation-object.json ;; + events.delivery) printf '%s\n' validation-events.json ;; + audit.request-correlation) printf '%s\n' validation-audit.json ;; + esac +} + validation_write_result() { local verdict="$1" reason="$2" cleanup_status="$3" local args=("${SCRIPT_DIR}/scripts/validation_result.py" @@ -1549,7 +1558,7 @@ validation_write_result() { fi if [ "${verdict}" = PASS ]; then local item - for item in validation-transition.json validation-head.json validation-restore-head.json validation-object.json; do + for item in $(validation_evidence_items); do args+=(--evidence "evidence/${VALIDATION_CASE}/${item}") done fi @@ -2056,6 +2065,41 @@ test_validation_tiering_transition_and_restore() { > "${TEST_TMP}/validation-object.json" } +test_validation_events_delivery() { + local bucket="val-events-${RUN_TOKEN}" key="round-trip.bin" arn + new_bucket "${bucket}" || return 1 + arn="$(get_first_arn)" + [ -n "${arn}" ] || { echo "notification target ARN missing"; return 1; } + rc_run event add "local/${bucket}" "${arn}" --event s3:ObjectCreated:Put + assert_rc_ok || return 1 + events_reset + make_test_file "${TEST_TMP}/validation-event-source.bin" 65536 + aws_run s3api put-object --bucket "${bucket}" --key "${key}" --body "${TEST_TMP}/validation-event-source.bin" + assert_aws_ok || return 1 + wait_notify 1 "${bucket}" || return 1 + sleep 15 + python3 "${SCRIPT_DIR}/scripts/validation_webhooks.py" events \ + --input "${NOTIFY_EVENTS}" --bucket "${bucket}" --key "${key}" \ + --output "${TEST_TMP}/validation-events.json" +} + +test_validation_audit_request_correlation() { + local bucket="val-audit-${RUN_TOKEN}" key="round-trip.bin" + new_bucket "${bucket}" || return 1 + events_reset + make_test_file "${TEST_TMP}/validation-audit-source.bin" 65536 + aws_run s3api put-object --bucket "${bucket}" --key "${key}" --body "${TEST_TMP}/validation-audit-source.bin" + assert_aws_ok || return 1 + aws_run s3api get-object --bucket "${bucket}" --key "${key}" "${TEST_TMP}/validation-audit-read.bin" + assert_aws_ok || return 1 + cmp "${TEST_TMP}/validation-audit-source.bin" "${TEST_TMP}/validation-audit-read.bin" || return 1 + wait_audit 2 "${bucket}" || return 1 + VALIDATION_AUDIT_TOKEN="${AUDIT_TOKEN}" VALIDATION_NOTIFY_TOKEN="${NOTIFY_TOKEN}" \ + python3 "${SCRIPT_DIR}/scripts/validation_webhooks.py" audit \ + --input "${AUDIT_EVENTS}" --bucket "${bucket}" --key "${key}" \ + --output "${TEST_TMP}/validation-audit.json" +} + run_all_tests() { run_test TIER-101 "TierAddValidation" test_tier101 run_test TIER-102 "TierListInfo" test_tier102 @@ -2931,7 +2975,10 @@ main() { SELECTED_TOPOLOGIES=("${TOPO_SINGLE_SINGLE}" "${TOPO_SINGLE_MULTI}") fi if [ -n "${VALIDATION_CASE}" ]; then - [ "${VALIDATION_CASE}" = tiering.transition-and-restore ] || die "unknown validation case: ${VALIDATION_CASE}" + case "${VALIDATION_CASE}" in + tiering.transition-and-restore|events.delivery|audit.request-correlation) ;; + *) die "unknown validation case: ${VALIDATION_CASE}" ;; + esac [ "${all}" -eq 0 ] || die "--validation-case cannot run all topologies" [ "${#SELECTED_TOPOLOGIES[@]}" -eq 1 ] || die "--validation-case requires one topology" [ "${SELECTED_TOPOLOGIES[0]}" = "${TOPO_SINGLE_SINGLE}" ] || die "validation case requires single-single topology" @@ -2989,7 +3036,11 @@ main() { fi PASS=0; FAIL=0; UNSUPPORTED=0; FAILED_TESTS=() if [ -n "${VALIDATION_CASE}" ]; then - run_test "${VALIDATION_CASE}" "TransitionAndRestore" test_validation_tiering_transition_and_restore + case "${VALIDATION_CASE}" in + tiering.transition-and-restore) run_test "${VALIDATION_CASE}" "TransitionAndRestore" test_validation_tiering_transition_and_restore ;; + events.delivery) run_test "${VALIDATION_CASE}" "ObjectEventDelivery" test_validation_events_delivery ;; + audit.request-correlation) run_test "${VALIDATION_CASE}" "AuditRequestCorrelation" test_validation_audit_request_correlation ;; + esac else run_all_tests fi diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index ad741b29..84ee98b9 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -19,7 +19,7 @@ case "${CASE_ID}" in SUITE=rustfs-storage-test.sh; EXTRA_ARGS=() ;; kms.sse-kms-round-trip) SUITE=rustfs-kms-test.sh; EXTRA_ARGS=() ;; - tiering.transition-and-restore) + tiering.transition-and-restore|events.delivery|audit.request-correlation) SUITE=rustfs-tier-test.sh; EXTRA_ARGS=(--topology single-single) ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_webhooks.py b/scripts/validation_webhooks.py new file mode 100644 index 00000000..c23bc8d1 --- /dev/null +++ b/scripts/validation_webhooks.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Check canonical event and audit webhook evidence without exporting secrets.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import sys +from urllib.parse import unquote_plus + + +def records(path: Path) -> list[dict]: + values = [] + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + value = json.loads(line) + if not isinstance(value, dict): + raise ValueError(f"webhook line {number} is not an object") + values.append(value) + return values + + +def event_summary(values: list[dict], bucket: str, key: str) -> dict: + matches = [] + for value in values: + for event in value.get("Records", [value]): + if not isinstance(event, dict): + raise ValueError("event record is not an object") + s3 = event.get("s3") or {} + event_bucket = (s3.get("bucket") or {}).get("name") or event.get("Bucket") + event_key = (s3.get("object") or {}).get("key") or event.get("Key") + if event_bucket == bucket and isinstance(event_key, str) and unquote_plus(event_key) == key: + matches.append(event) + if len(matches) != 1: + raise ValueError(f"expected one matching object event, found {len(matches)}") + event = matches[0] + name = event.get("eventName") or event.get("EventName") + if name != "s3:ObjectCreated:Put": + raise ValueError(f"unexpected event type: {name!r}") + return {"bucket": bucket, "key": key, "delivery_count": 1, "event_name": name} + + +def audit_summary(values: list[dict], bucket: str, key: str) -> dict: + serialized = json.dumps(values, ensure_ascii=False).lower() + for name in ("RUSTFS_SECRET_KEY", "VALIDATION_AUDIT_TOKEN", "VALIDATION_NOTIFY_TOKEN"): + secret = os.environ.get(name, "") + if secret and secret.lower() in serialized: + raise ValueError(f"audit payload exposes {name}") + matches: dict[str, dict] = {} + for value in values: + api = value.get("api") or {} + if api.get("bucket") != bucket or api.get("object") != key: + continue + name = api.get("name") + if name not in {"PutObject", "GetObject"}: + continue + request_id = value.get("requestID") + if not isinstance(request_id, str) or not request_id: + raise ValueError(f"{name} audit record lacks requestID") + if api.get("status_code") != 200: + raise ValueError(f"{name} audit record lacks successful status") + headers = value.get("requestHeader") or {} + for header, content in headers.items(): + if header.lower() == "authorization" and isinstance(content, str) and content.strip("*"): + raise ValueError("audit payload exposes authorization header") + matches[name] = {"request_id": request_id, "status_code": 200} + if set(matches) != {"PutObject", "GetObject"}: + raise ValueError("matching PUT and GET audit records were not both delivered") + if matches["PutObject"]["request_id"] == matches["GetObject"]["request_id"]: + raise ValueError("PUT and GET audit records share a request ID") + return {"bucket": bucket, "key": key, "operations": matches, "secret_check": "PASS"} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("events", "audit")) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--bucket", required=True) + parser.add_argument("--key", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + values = records(args.input) + result = (event_summary if args.mode == "events" else audit_summary)(values, args.bucket, args.key) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError, json.JSONDecodeError) as exc: + print(f"validation webhook error: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/test_validation_tier.sh b/test_validation_tier.sh index bfc82a1f..5155434a 100644 --- a/test_validation_tier.sh +++ b/test_validation_tier.sh @@ -76,3 +76,10 @@ if grep -q '\[DRY-RUN\] TIER-' "${TEST_ROOT}/validation-dry-run.log"; then printf 'validation mode scheduled legacy tier cases\n' >&2 exit 1 fi + +for case_id in events.delivery audit.request-correlation; do + TIER_SOURCE_ONLY=0 "${TEST_SCRIPT_DIR}/rustfs-tier-test.sh" --validation-case "${case_id}" \ + --topology single-single --dry-run --yes --package-url https://example.invalid/rustfs.deb \ + > "${TEST_ROOT}/${case_id}.log" + grep -q "\[DRY-RUN\] ${case_id}" "${TEST_ROOT}/${case_id}.log" +done diff --git a/test_validation_webhooks.py b/test_validation_webhooks.py new file mode 100644 index 00000000..e686d61e --- /dev/null +++ b/test_validation_webhooks.py @@ -0,0 +1,47 @@ +"""Contract checks for canonical webhook evidence parsing.""" + +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from scripts.validation_webhooks import audit_summary, event_summary, records + + +class ValidationWebhooksTest(unittest.TestCase): + def test_event_requires_exactly_one_matching_put(self): + event = {"Records": [{ + "eventName": "s3:ObjectCreated:Put", + "s3": {"bucket": {"name": "bucket"}, "object": {"key": "round+trip.bin"}}, + }]} + self.assertEqual(event_summary([event], "bucket", "round trip.bin")["delivery_count"], 1) + with self.assertRaisesRegex(ValueError, "found 2"): + event_summary([event, event], "bucket", "round trip.bin") + with self.assertRaisesRegex(ValueError, "unexpected event type"): + event_summary([{"Records": [{**event["Records"][0], "eventName": "s3:ObjectRemoved:Delete"}]}], "bucket", "round trip.bin") + + def test_audit_requires_correlated_operations_and_no_secret(self): + put = {"api": {"name": "PutObject", "bucket": "bucket", "object": "key", "status_code": 200}, + "requestID": "request-put"} + get = {"api": {"name": "GetObject", "bucket": "bucket", "object": "key", "status_code": 200}, + "requestID": "request-get"} + self.assertEqual(set(audit_summary([put, get], "bucket", "key")["operations"]), {"PutObject", "GetObject"}) + with self.assertRaisesRegex(ValueError, "share a request ID"): + audit_summary([put, {**get, "requestID": "request-put"}], "bucket", "key") + with self.assertRaisesRegex(ValueError, "lacks requestID"): + audit_summary([put, {**get, "requestID": ""}], "bucket", "key") + with patch.dict(os.environ, {"RUSTFS_SECRET_KEY": "private-value"}): + with self.assertRaisesRegex(ValueError, "exposes RUSTFS_SECRET_KEY"): + audit_summary([put, {**get, "requestHeader": {"x-secret": "private-value"}}], "bucket", "key") + + def test_records_rejects_non_object_jsonl(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "events.jsonl" + path.write_text('{}\n[]\n', encoding="utf-8") + with self.assertRaisesRegex(ValueError, "line 2"): + records(path) + + +if __name__ == "__main__": + unittest.main() From fbebacbf7a989635085eb7703ab2cc3c8d3e0daf Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 18:02:32 +0800 Subject: [PATCH 008/343] feat: add canonical IAM policy enforcement validation --- README.md | 8 +- docs/validation-migration-map.md | 11 ++- rustfs-security-test.sh | 136 ++++++++++++++++++++++++++++++- scripts/run_validation_case.sh | 4 +- 4 files changed, 149 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 1cedc3fd..51833f41 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,9 @@ results. The executable canonical cases currently include `s3.basic-object-round-trip`, `scanner.usage-convergence`, `kms.sse-kms-round-trip`, -`tiering.transition-and-restore`, `events.delivery`, and -`audit.request-correlation`. The S3 and KMS cases use single-disk +`tiering.transition-and-restore`, `events.delivery`, +`audit.request-correlation`, and `iam.policy-enforcement`. The S3, KMS, and IAM +cases use single-disk deployment. The scanner case uses the storage suite's single-node four-disk deployment, waits for a post-upload successful scanner cycle, and requires a complete usage snapshot with exactly 20 objects and 20,690 logical bytes. @@ -75,7 +76,8 @@ implemented source cases have verified executors and live results. S3, scanner, KMS, and tiering have live PASS evidence against nightly build `37958901959` attempt `1` on the current auto-testing fleet. Events and audit -have local executor tests but still need live verification. +have local executor tests but still need live verification. IAM also awaits a +live run. ## Fault-tolerance suite diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 518fd60b..cf897149 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -48,6 +48,11 @@ unique object after a quiet window; audit checks matching PUT/GET records, distinct request IDs, successful status, and absence of known secrets. Both still require live candidate evidence before being counted as verified. +`iam.policy-enforcement` has a dedicated executor on the Security suite's +`single-single` deployment. It uses one principal with a policy that allows +PUT/GET and explicitly denies DELETE, then checks the denied DELETE did not +remove the object. Live candidate evidence is pending. + "Existing validation environment" means the fleet used by `auto-testing` and the RustFS `smoke-testing` workflows, with `RUSTFS_NODES` and `RUSTFS_SSH_USER` supplied there. For Heal, reuse the nodes deployed by @@ -68,8 +73,8 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The six currently dispatchable IDs are the S3, scanner, KMS, -tiering, events, and audit cases above. Four have live PASS evidence; the other +case evidence. The seven currently dispatchable IDs are the S3, scanner, KMS, +tiering, events, audit, and IAM cases above. Four have live PASS evidence; the other 154 IDs remain unverified. The eight planned cases have these concrete reuse candidates: @@ -81,7 +86,7 @@ The eight planned cases have these concrete reuse candidates: | `tiering.transition-and-restore` | TIER-105, TIER-107 | Dedicated executor and live PASS evidence. | | `events.delivery` | EVT-103, EVT-104 | Dedicated executor; live evidence pending. | | `audit.request-correlation` | AUD-102, AUD-104, AUD-106 | Dedicated executor; live evidence pending. | -| `iam.policy-enforcement` | IAM-105, IAM-109 | Verify explicit deny and allowed action on the same principal/policy fixture. | +| `iam.policy-enforcement` | IAM-105, IAM-109 | Dedicated executor; live evidence pending. | | `oidc.login-and-session` | OIDC-103 | The live Keycloak path can be skipped by the old suite; validation requires an actual session-boundary result. | | `s3.basic-object-round-trip` | S3C-101, 201, 203, 205, 206, 207 | Dedicated executor and live PASS evidence. | diff --git a/rustfs-security-test.sh b/rustfs-security-test.sh index 4476e538..7c767f52 100644 --- a/rustfs-security-test.sh +++ b/rustfs-security-test.sh @@ -116,6 +116,13 @@ OIDC_RUN=0 SELECTED_TOPOLOGIES=() ENDPOINT="" VOLUMES="" +VALIDATION_CASE="" +VALIDATION_ARTIFACTS_DIR="" +VALIDATION_CANDIDATE_MANIFEST="" +VALIDATION_TESTING_COMMIT="" +VALIDATION_BINARY_SHA256="" +VALIDATION_OBJECT_CLEANUP_STATUS=PASS +RUN_TOKEN="$(date -u +%y%m%d%H%M%S)-$$" # ==================== Helpers ==================== @@ -582,6 +589,35 @@ run_test() { fi } +validation_write_evidence() { + local evidence_dir="${VALIDATION_ARTIFACTS_DIR}/evidence/${VALIDATION_CASE}" + mkdir -p "${evidence_dir}" + cp "${TEST_TMP}/${VALIDATION_CASE}.out" "${evidence_dir}/test.log" + for item in validation-iam-policy.json validation-iam-observations.json; do + if [ -f "${TEST_TMP}/${item}" ]; then cp "${TEST_TMP}/${item}" "${evidence_dir}/${item}"; fi + done +} + +validation_write_result() { + local verdict="$1" reason="$2" cleanup_status="$3" + local args=("${SCRIPT_DIR}/scripts/validation_result.py" + --case-id "${VALIDATION_CASE}" --profile single-1disk + --result-file "${VALIDATION_ARTIFACTS_DIR}/cases/${VALIDATION_CASE}.json" + --verdict "${verdict}" --cleanup-status "${cleanup_status}") + if [ -n "${reason}" ]; then args+=(--reason "${reason}"); fi + if [ "${verdict}" != RUNNING ]; then + args+=(--candidate-manifest "${VALIDATION_CANDIDATE_MANIFEST}" + --rc-bin "$(command -v rc)" --testing-commit "${VALIDATION_TESTING_COMMIT}" + --binary-sha256 "${VALIDATION_BINARY_SHA256}" + --evidence "evidence/${VALIDATION_CASE}/test.log") + fi + if [ "${verdict}" = PASS ]; then + args+=(--evidence "evidence/${VALIDATION_CASE}/validation-iam-policy.json" + --evidence "evidence/${VALIDATION_CASE}/validation-iam-observations.json") + fi + python3 "${args[@]}" +} + skip_test() { local id="$1" name="$2" SKIP=$((SKIP+1)) @@ -678,6 +714,56 @@ case_iam_109_deny_precedence() { admin_user_remove "${user}" >/dev/null } +validation_iam_policy_body() { + local user="$1" secret="$2" bucket="$3" policy_name="$4" policy="$5" out + out="$(admin_user_add "${user}" "${secret}")"; assert_admin_ok 200 "add validation user" || return 1 + out="$(admin_policy_add "${policy_name}" "${policy}")"; assert_admin_ok 200 "add validation policy" || return 1 + out="$(admin_policy_attach "${user}" "${policy_name}")"; assert_admin_ok 200 "attach validation policy" || return 1 + printf 'validation policy attached to %s\n' "${user}" + printf '%s\n' "${policy}" > "${TEST_TMP}/validation-iam-policy.json" + printf 'validation-payload-%s\n' "${RUN_TOKEN}" > "${TEST_TMP}/validation-iam-source.bin" + aws_run_as "${user}" "${secret}" -- s3api put-object --bucket "${bucket}" --key object.bin --body "${TEST_TMP}/validation-iam-source.bin" + assert_aws_ok "allowed PUT" || return 1 + aws_run_as "${user}" "${secret}" -- s3api get-object --bucket "${bucket}" --key object.bin "${TEST_TMP}/validation-iam-read.bin" + assert_aws_ok "allowed GET" || return 1 + cmp "${TEST_TMP}/validation-iam-source.bin" "${TEST_TMP}/validation-iam-read.bin" || return 1 + aws_run_as "${user}" "${secret}" -- s3api delete-object --bucket "${bucket}" --key object.bin + assert_aws_fails "explicit deny DELETE" || return 1 + [[ "${AWS_OUT}" == *AccessDenied* ]] || { echo "DELETE failure was not AccessDenied"; return 1; } + aws_run s3api get-object --bucket "${bucket}" --key object.bin "${TEST_TMP}/validation-iam-root-read.bin" + assert_aws_ok "denied DELETE preserved object" || return 1 + cmp "${TEST_TMP}/validation-iam-source.bin" "${TEST_TMP}/validation-iam-root-read.bin" || return 1 + jq -n --arg user "${user}" --arg bucket "${bucket}" \ + --arg sha256 "$(sha256sum "${TEST_TMP}/validation-iam-source.bin" | awk '{print $1}')" \ + '{principal:$user,bucket:$bucket,allowed:["PutObject","GetObject"],explicitly_denied:"DeleteObject",object_preserved:true,payload_sha256:$sha256}' \ + > "${TEST_TMP}/validation-iam-observations.json" +} + +test_validation_iam_policy_enforcement() { + local user="valiam-${RUN_TOKEN//-/}" secret + secret="$(openssl rand -hex 24)" || return 1 + local bucket="val-iam-${RUN_TOKEN}" policy_name="valiam-${RUN_TOKEN//-/}" result=0 out + local policy + policy="$(jq -nc --arg bucket "${bucket}" '{Version:"2012-10-17",Statement:[ + {Effect:"Allow",Action:["s3:PutObject","s3:GetObject"],Resource:["arn:aws:s3:::"+$bucket+"/*"]}, + {Effect:"Deny",Action:["s3:DeleteObject"],Resource:["arn:aws:s3:::"+$bucket+"/*"]} + ]}')" + new_bucket "${bucket}" || return 1 + validation_iam_policy_body "${user}" "${secret}" "${bucket}" "${policy_name}" "${policy}" || result=$? + if [ -f "${TEST_TMP}/validation-iam-source.bin" ]; then + aws_run s3api delete-object --bucket "${bucket}" --key object.bin + [ "${AWS_CODE}" -eq 0 ] || VALIDATION_OBJECT_CLEANUP_STATUS=FAIL + fi + out="$(admin_user_remove "${user}")" + assert_admin_ok 200 "remove validation user" || VALIDATION_OBJECT_CLEANUP_STATUS=FAIL + out="$(admin_api DELETE "/rustfs/admin/v3/remove-canned-policy?name=${policy_name}")" + assert_admin_ok 200 "remove validation policy" || VALIDATION_OBJECT_CLEANUP_STATUS=FAIL + aws_run s3api delete-bucket --bucket "${bucket}" + [ "${AWS_CODE}" -eq 0 ] || VALIDATION_OBJECT_CLEANUP_STATUS=FAIL + [ "${VALIDATION_OBJECT_CLEANUP_STATUS}" = PASS ] || result=1 + return "${result}" +} + case_iam_113_access_key_list_hides_secret() { local user="sec113u" secret="sec113secret-xyz" out out="$(admin_user_add "${user}" "${secret}")"; assert_admin_ok 200 "add-user" @@ -944,6 +1030,10 @@ parse_args() { --package-url) [ "$#" -ge 2 ] || die "--package-url requires a value"; PACKAGE_URL="$2"; shift ;; --version) [ "$#" -ge 2 ] || die "--version requires a value"; RUSTFS_VERSION="$2"; shift ;; --oidc-live) OIDC_LIVE=1 ;; + --validation-case) [ "$#" -ge 2 ] || die "--validation-case requires a value"; VALIDATION_CASE="$2"; shift ;; + --validation-artifacts-dir) [ "$#" -ge 2 ] || die "--validation-artifacts-dir requires a value"; VALIDATION_ARTIFACTS_DIR="$2"; shift ;; + --candidate-manifest) [ "$#" -ge 2 ] || die "--candidate-manifest requires a value"; VALIDATION_CANDIDATE_MANIFEST="$2"; shift ;; + --testing-commit) [ "$#" -ge 2 ] || die "--testing-commit requires a value"; VALIDATION_TESTING_COMMIT="$2"; shift ;; --keep-env) KEEP_ENV=1 ;; -y) ASSUME_YES=1 ;; --dry-run) DRY_RUN=1 ;; @@ -951,6 +1041,24 @@ parse_args() { esac shift done + if [ -n "${VALIDATION_CASE}" ]; then + [ "${VALIDATION_CASE}" = iam.policy-enforcement ] || die "unknown validation case: ${VALIDATION_CASE}" + [ "${#SELECTED_TOPOLOGIES[@]}" -eq 0 ] || die "--validation-case chooses its topology" + [ "${KEEP_ENV}" -eq 0 ] || die "--validation-case requires environment teardown" + SELECTED_TOPOLOGIES=("${TOPO_SINGLE_SINGLE}") + if [ "${DRY_RUN}" -eq 0 ]; then + [ -n "${VALIDATION_ARTIFACTS_DIR}" ] || die "--validation-artifacts-dir required" + [ -n "${VALIDATION_CANDIDATE_MANIFEST}" ] || die "--candidate-manifest required" + [ -n "${VALIDATION_TESTING_COMMIT}" ] || die "--testing-commit required" + [ ! -e "${VALIDATION_ARTIFACTS_DIR}" ] || die "validation artifacts directory already exists" + PACKAGE_URL="$(jq -er '.package_url' "${VALIDATION_CANDIDATE_MANIFEST}")" || die "candidate URL missing" + PACKAGE_SHA256="$(jq -er '.package_sha256' "${VALIDATION_CANDIDATE_MANIFEST}")" || die "candidate package SHA missing" + [[ "${PACKAGE_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "invalid candidate package SHA" + jq -e '.source_sha | test("^[0-9a-f]{40}$")' "${VALIDATION_CANDIDATE_MANIFEST}" >/dev/null || die "candidate source commit missing" + [ "${VALIDATION_TESTING_COMMIT}" = "$(git -C "${SCRIPT_DIR}" rev-parse HEAD)" ] || die "testing commit does not match checkout" + mkdir -p "${VALIDATION_ARTIFACTS_DIR}" + fi + fi [ "${#SELECTED_TOPOLOGIES[@]}" -gt 0 ] || die "select a topology (--topology ) or --all-topologies" } @@ -991,6 +1099,7 @@ main() { need_cmd aws "aws cli" need_cmd jq "jq" need_cmd python3 "python3" + if [ -n "${VALIDATION_CASE}" ]; then need_cmd rc "rc client provenance"; fi TEST_SCRIPT_REVISION="$(git -C "${SCRIPT_DIR}" rev-parse HEAD 2>/dev/null || printf unknown)" log "test scripts commit: ${TEST_SCRIPT_REVISION}" if [ "${DRY_RUN}" -eq 0 ]; then @@ -1004,15 +1113,36 @@ main() { for topo in "${SELECTED_TOPOLOGIES[@]}"; do CURRENT_TOPOLOGY="${topo}" log "===== topology ${topo} =====" + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + validation_write_result RUNNING "validation case started" PENDING + fi env_setup "${topo}" + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + VALIDATION_BINARY_SHA256="$(printf 'sha256sum /usr/bin/rustfs\n' | run_remote "${NODES[0]}" | awk '{print $1}')" + [[ "${VALIDATION_BINARY_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "installed RustFS binary SHA-256 unavailable" + fi if [ -z "${RUSTFS_VERSION_INFO}" ]; then RUSTFS_VERSION_INFO="$(detect_rustfs_version)" fi CASE_RESULTS=() - run_security_cases + if [ -n "${VALIDATION_CASE}" ]; then + run_test "${VALIDATION_CASE}" "PolicyEnforcement" test_validation_iam_policy_enforcement + else + run_security_cases + fi record_topology_results "${topo}" - if [ "${KEEP_ENV}" -eq 0 ]; then - env_teardown "${topo}" + local cleanup_status=PASS + if [ "${KEEP_ENV}" -eq 0 ]; then env_teardown "${topo}" || cleanup_status=FAIL; fi + if [ "${VALIDATION_OBJECT_CLEANUP_STATUS}" != PASS ]; then cleanup_status=FAIL; fi + if [ -n "${VALIDATION_CASE}" ] && [ "${DRY_RUN}" -eq 0 ]; then + validation_write_evidence + if [ "${FAIL}" -eq 0 ] && [ "${cleanup_status}" = PASS ]; then + validation_write_result PASS "" PASS + elif [ "${FAIL}" -eq 0 ]; then + validation_write_result INFRA_ERROR "environment teardown failed" FAIL + else + validation_write_result FAIL "IAM assertion failed; inspect evidence" "${cleanup_status}" + fi fi done diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 84ee98b9..9d4ba33f 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -21,6 +21,8 @@ case "${CASE_ID}" in SUITE=rustfs-kms-test.sh; EXTRA_ARGS=() ;; tiering.transition-and-restore|events.delivery|audit.request-correlation) SUITE=rustfs-tier-test.sh; EXTRA_ARGS=(--topology single-single) ;; + iam.policy-enforcement) + SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 exit 2 ;; @@ -36,7 +38,7 @@ esac mkdir -p -- "${ARTIFACTS_DIR}" case_dir="${ARTIFACTS_DIR}/${CASE_ID}" suite_rc=0 -"${SCRIPT_DIR}/${SUITE}" --validation-case "${CASE_ID}" "${EXTRA_ARGS[@]}" \ +bash "${SCRIPT_DIR}/${SUITE}" --validation-case "${CASE_ID}" "${EXTRA_ARGS[@]}" \ --candidate-manifest "${CANDIDATE_MANIFEST}" \ --testing-commit "${TESTING_COMMIT}" \ --validation-artifacts-dir "${case_dir}" -y \ From 3abdc1d3bf36a2e41513644f954284bc6d930f82 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 18:05:29 +0800 Subject: [PATCH 009/343] fix: keep security preview independent of validation runtime --- README.md | 7 +++---- docs/validation-migration-map.md | 9 +++++---- rustfs-security-test.sh | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 51833f41..57181ebb 100644 --- a/README.md +++ b/README.md @@ -74,10 +74,9 @@ The report's full implemented-case gate is intentionally not enabled by these runs. Run it with `--gate-status implemented` only after all 66 implemented source cases have verified executors and live results. -S3, scanner, KMS, and tiering have live PASS evidence against nightly build -`37958901959` attempt `1` on the current auto-testing fleet. Events and audit -have local executor tests but still need live verification. IAM also awaits a -live run. +S3, scanner, KMS, tiering, and IAM have live PASS evidence against nightly +build `37958901959` attempt `1` on the current auto-testing fleet. Events and +audit have local executor tests but still need live verification. ## Fault-tolerance suite diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index cf897149..e3db1fb2 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -51,7 +51,8 @@ still require live candidate evidence before being counted as verified. `iam.policy-enforcement` has a dedicated executor on the Security suite's `single-single` deployment. It uses one principal with a policy that allows PUT/GET and explicitly denies DELETE, then checks the denied DELETE did not -remove the object. Live candidate evidence is pending. +remove the object. Its live candidate run passed in RustFS Actions run +`38043548625`. "Existing validation environment" means the fleet used by `auto-testing` and the RustFS `smoke-testing` workflows, with `RUSTFS_NODES` and @@ -74,8 +75,8 @@ executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and case evidence. The seven currently dispatchable IDs are the S3, scanner, KMS, -tiering, events, audit, and IAM cases above. Four have live PASS evidence; the other -154 IDs remain unverified. +tiering, events, audit, and IAM cases above. Five have live PASS evidence; the +other 153 IDs remain unverified. The eight planned cases have these concrete reuse candidates: @@ -86,7 +87,7 @@ The eight planned cases have these concrete reuse candidates: | `tiering.transition-and-restore` | TIER-105, TIER-107 | Dedicated executor and live PASS evidence. | | `events.delivery` | EVT-103, EVT-104 | Dedicated executor; live evidence pending. | | `audit.request-correlation` | AUD-102, AUD-104, AUD-106 | Dedicated executor; live evidence pending. | -| `iam.policy-enforcement` | IAM-105, IAM-109 | Dedicated executor; live evidence pending. | +| `iam.policy-enforcement` | IAM-105, IAM-109 | Dedicated executor and live PASS evidence. | | `oidc.login-and-session` | OIDC-103 | The live Keycloak path can be skipped by the old suite; validation requires an actual session-boundary result. | | `s3.basic-object-round-trip` | S3C-101, 201, 203, 205, 206, 207 | Dedicated executor and live PASS evidence. | diff --git a/rustfs-security-test.sh b/rustfs-security-test.sh index 7c767f52..ec690803 100644 --- a/rustfs-security-test.sh +++ b/rustfs-security-test.sh @@ -122,7 +122,7 @@ VALIDATION_CANDIDATE_MANIFEST="" VALIDATION_TESTING_COMMIT="" VALIDATION_BINARY_SHA256="" VALIDATION_OBJECT_CLEANUP_STATUS=PASS -RUN_TOKEN="$(date -u +%y%m%d%H%M%S)-$$" +RUN_TOKEN="" # ==================== Helpers ==================== @@ -1043,6 +1043,7 @@ parse_args() { done if [ -n "${VALIDATION_CASE}" ]; then [ "${VALIDATION_CASE}" = iam.policy-enforcement ] || die "unknown validation case: ${VALIDATION_CASE}" + RUN_TOKEN="$(date -u +%y%m%d%H%M%S)-$$" [ "${#SELECTED_TOPOLOGIES[@]}" -eq 0 ] || die "--validation-case chooses its topology" [ "${KEEP_ENV}" -eq 0 ] || die "--validation-case requires environment teardown" SELECTED_TOPOLOGIES=("${TOPO_SINGLE_SINGLE}") From 988cf11fd98812b0218dc40f29c49a0193db5f89 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 18:12:22 +0800 Subject: [PATCH 010/343] feat: validate scanner readiness on two-node topology --- README.md | 3 +- docs/validation-migration-map.md | 9 +++- rustfs-storage-test.sh | 88 +++++++++++++++++++++++++++++--- scripts/run_validation_case.sh | 2 +- test_validation_scanner_usage.sh | 24 +++++++++ 5 files changed, 114 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 57181ebb..3db556bd 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,8 @@ results. The executable canonical cases currently include `s3.basic-object-round-trip`, `scanner.usage-convergence`, `kms.sse-kms-round-trip`, `tiering.transition-and-restore`, `events.delivery`, -`audit.request-correlation`, and `iam.policy-enforcement`. The S3, KMS, and IAM +`audit.request-correlation`, `iam.policy-enforcement`, and +`scanner.status-readiness`. The S3, KMS, and IAM cases use single-disk deployment. The scanner case uses the storage suite's single-node four-disk deployment, waits for a post-upload successful scanner cycle, and requires a diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index e3db1fb2..7d482649 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -29,6 +29,11 @@ cycle after upload plus a newer complete usage snapshot with exact object and logical-byte counts. The old STOR-134 threshold remains a separate regression test. Its live candidate run passed in RustFS Actions run `38042200106`. +`scanner.status-readiness` has a dedicated executor on two of the existing +auto-testing nodes, each with four disks. It checks both nodes' accelerated +scanner runtime settings and requires exactly one leader to finish a new +successful cycle within 120 seconds. Its live candidate run is pending. + `kms.sse-kms-round-trip` has a dedicated executor using the existing local KMS single-disk deployment. It checks the same object's encryption key, metadata, and payload, then requires an invalid-key PUT to fail without @@ -74,8 +79,8 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The seven currently dispatchable IDs are the S3, scanner, KMS, -tiering, events, audit, and IAM cases above. Five have live PASS evidence; the +case evidence. The eight currently dispatchable IDs are the S3, two scanner, +KMS, tiering, events, audit, and IAM cases above. Five have live PASS evidence; the other 153 IDs remain unverified. The eight planned cases have these concrete reuse candidates: diff --git a/rustfs-storage-test.sh b/rustfs-storage-test.sh index 6c0d42fb..cc046862 100755 --- a/rustfs-storage-test.sh +++ b/rustfs-storage-test.sh @@ -73,6 +73,7 @@ RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/" TOPO_SINGLE_SINGLE="single-single" TOPO_SINGLE_MULTI="single-multi" TOPO_MULTI_MULTI="multi-multi" +TOPO_ILM_2X4="ilm-2x4" ENDPOINT_SINGLE_SINGLE="http://rustfs-node1:9000" ENDPOINT_SINGLE_MULTI="http://rustfs-node1:9000" @@ -81,6 +82,7 @@ ENDPOINT_MULTI_MULTI="http://rustfs-node1:9000" VOLUMES_SINGLE_SINGLE="/data/rustfs1/mnmd" VOLUMES_SINGLE_MULTI="/data/rustfs{1...4}/mnmd" VOLUMES_MULTI_MULTI="http://rustfs-node{1...4}:9000/data/rustfs{1...4}/mnmd" +VOLUMES_ILM_2X4="http://rustfs-node{1...2}:9000/data/rustfs{1...4}/mnmd" SERVICE_TIMEOUT=300 @@ -190,7 +192,7 @@ RUSTFS_CONSOLE_ENABLE=${RUSTFS_CONSOLE_ENABLE} RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL} RUSTFS_OBS_LOG_DIRECTORY="${RUSTFS_OBS_LOG_DIRECTORY}" EOF - if [ "${VALIDATION_CASE}" = "scanner.usage-convergence" ]; then + if [ "${VALIDATION_CASE}" = "scanner.usage-convergence" ] || [ "${VALIDATION_CASE}" = "scanner.status-readiness" ]; then printf 'RUSTFS_SCANNER_ENABLED=true\nRUSTFS_SCANNER_CYCLE=5\nRUSTFS_SCANNER_START_DELAY_SECS=0\n' fi } @@ -278,6 +280,7 @@ resolve_topology() { "${TOPO_SINGLE_SINGLE}") ENDPOINT="${ENDPOINT_SINGLE_SINGLE}"; VOLUMES="${VOLUMES_SINGLE_SINGLE}"; return 0 ;; "${TOPO_SINGLE_MULTI}") ENDPOINT="${ENDPOINT_SINGLE_MULTI}"; VOLUMES="${VOLUMES_SINGLE_MULTI}"; return 0 ;; "${TOPO_MULTI_MULTI}") ENDPOINT="${ENDPOINT_MULTI_MULTI}"; VOLUMES="${VOLUMES_MULTI_MULTI}"; return 0 ;; + "${TOPO_ILM_2X4}") ENDPOINT="${ENDPOINT_SINGLE_MULTI}"; VOLUMES="${VOLUMES_ILM_2X4}"; return 0 ;; *) die "unknown topology: $1 (single-single|single-multi|multi-multi)" ;; esac } @@ -285,6 +288,7 @@ resolve_topology() { topology_nodes() { case "$1" in "${TOPO_SINGLE_SINGLE}"|"${TOPO_SINGLE_MULTI}") printf '%s' "${NODES[0]}" ;; + "${TOPO_ILM_2X4}") printf '%s %s' "${NODES[0]}" "${NODES[1]}" ;; *) printf '%s' "${NODES[*]}" ;; esac } @@ -508,15 +512,26 @@ validation_write_evidence() { local evidence_dir="${VALIDATION_ARTIFACTS_DIR}/evidence/${VALIDATION_CASE}" item mkdir -p "${evidence_dir}" cp "${TEST_TMP}/test.out" "${evidence_dir}/test.log" - for item in validation-fixture.json validation-list.json validation-baseline-usage.json validation-scanner.json validation-usage.json; do + for item in $(validation_evidence_items); do if [ -f "${TEST_TMP}/${item}" ]; then cp "${TEST_TMP}/${item}" "${evidence_dir}/${item}"; fi done } +validation_evidence_items() { + case "${VALIDATION_CASE}" in + scanner.usage-convergence) + printf '%s\n' validation-fixture.json validation-list.json validation-baseline-usage.json validation-scanner.json validation-usage.json ;; + scanner.status-readiness) + printf '%s\n' validation-scanner-node1.json validation-scanner-node2.json validation-scanner-contract.json ;; + esac +} + validation_write_result() { local verdict="$1" reason="$2" cleanup_status="$3" item + local profile=single-4disk + [ "${VALIDATION_CASE}" = scanner.status-readiness ] && profile=ilm-2x4 local args=("${SCRIPT_DIR}/scripts/validation_result.py" - --case-id "${VALIDATION_CASE}" --profile single-4disk + --case-id "${VALIDATION_CASE}" --profile "${profile}" --result-file "${VALIDATION_ARTIFACTS_DIR}/cases/${VALIDATION_CASE}.json" --verdict "${verdict}" --cleanup-status "${cleanup_status}") if [ -n "${reason}" ]; then args+=(--reason "${reason}"); fi @@ -527,7 +542,7 @@ validation_write_result() { --evidence "evidence/${VALIDATION_CASE}/test.log") fi if [ "${verdict}" = PASS ]; then - for item in validation-fixture.json validation-list.json validation-baseline-usage.json validation-scanner.json validation-usage.json; do + for item in $(validation_evidence_items); do args+=(--evidence "evidence/${VALIDATION_CASE}/${item}") done fi @@ -795,6 +810,52 @@ test_validation_scanner_usage_convergence() { [ "${case_rc}" -eq 0 ] && [ "${cleanup_rc}" -eq 0 ] } +validation_scanner_status_ready() { + local node1="$1" node2="$2" started_at="$3" + jq -se --argjson started "${started_at}" ' + length == 2 and + all(.[]; .enabled == true and + .runtime_config.cycle_interval_seconds == {value:5,source:"env"} and + .runtime_config.start_delay_seconds == {value:0,source:"env"}) and + ([.[] | select(.metrics.leader_lock_held_by_this_process == true)] | length) == 1 and + any(.[]; .metrics.leader_lock_held_by_this_process == true and + .metrics.leader_lock_state == "acquired" and + .metrics.last_cycle_result == "success" and + .metrics.last_cycle_end_unix_secs >= $started and + .freshness.state == "fresh" and + .cycle_schedule.execution_role == "leader")' \ + "${node1}" "${node2}" >/dev/null +} + +test_validation_scanner_status_readiness() { + local started_at deadline status_code node endpoint index + started_at="$(date -u +%s)" + deadline=$((SECONDS + 120)) + while [ "${SECONDS}" -lt "${deadline}" ]; do + status_code=0 + for index in 1 2; do + node="${NODES[index-1]}" + endpoint="http://rustfs-node${index}:9000" + (API_ENDPOINT="${endpoint}"; admin_api GET /rustfs/admin/v3/scanner/status "" "" > "${TEST_TMP}/validation-scanner-node${index}.json" + [ "$(admin_api_code)" = 200 ]) || status_code=1 + [ "${status_code}" -eq 0 ] || { echo "scanner status unavailable on ${node}"; break; } + done + if [ "${status_code}" -eq 0 ] && validation_scanner_status_ready \ + "${TEST_TMP}/validation-scanner-node1.json" "${TEST_TMP}/validation-scanner-node2.json" "${started_at}"; then + jq -n --argjson started "${started_at}" --slurpfile node1 "${TEST_TMP}/validation-scanner-node1.json" \ + --slurpfile node2 "${TEST_TMP}/validation-scanner-node2.json" \ + '{started_at_unix:$started,cycle_interval_seconds:5,start_delay_seconds:0, + leader:([($node1[0],$node2[0]) | select(.metrics.leader_lock_held_by_this_process == true) | + {last_cycle_end_unix_secs:.metrics.last_cycle_end_unix_secs,last_cycle_result:.metrics.last_cycle_result}] | first)}' \ + > "${TEST_TMP}/validation-scanner-contract.json" + return 0 + fi + sleep 5 + done + echo "two-node scanner runtime or successful new leader cycle did not meet the 120-second deadline" + return 1 +} + test_stor101() { new_bucket "stor101-b1" || return 1 make_compressible_file "${TEST_TMP}/stor101-v" 1048576 @@ -1189,7 +1250,7 @@ Options: --ssh-user SSH user for the test nodes --ssh-port SSH port for the test nodes --log-file append all output to a log file - --validation-case ID run only scanner.usage-convergence (single-multi) + --validation-case ID run a canonical scanner case on its required topology --validation-artifacts-dir DIR write canonical result and evidence --candidate-manifest FILE immutable candidate manifest --testing-commit SHA auto-testing commit used for validation results @@ -1236,11 +1297,19 @@ main() { SELECTED_TOPOLOGIES=("${TOPO_SINGLE_SINGLE}" "${TOPO_SINGLE_MULTI}" "${TOPO_MULTI_MULTI}") fi if [ -n "${VALIDATION_CASE}" ]; then - [ "${VALIDATION_CASE}" = "scanner.usage-convergence" ] || die "unknown validation case: ${VALIDATION_CASE}" + case "${VALIDATION_CASE}" in + scanner.usage-convergence|scanner.status-readiness) ;; + *) die "unknown validation case: ${VALIDATION_CASE}" ;; + esac [ "${all}" -eq 0 ] || die "--validation-case cannot be combined with --all-topologies" [ "${#SELECTED_TOPOLOGIES[@]}" -eq 0 ] || die "--validation-case chooses its own topology" [ "${KEEP_ENV}" -eq 0 ] || die "--validation-case requires environment teardown" - SELECTED_TOPOLOGIES=("${TOPO_SINGLE_MULTI}") + if [ "${VALIDATION_CASE}" = scanner.status-readiness ]; then + SELECTED_TOPOLOGIES=("${TOPO_ILM_2X4}") + [ "${#NODES[@]}" -ge 2 ] || die "scanner.status-readiness needs two test nodes" + else + SELECTED_TOPOLOGIES=("${TOPO_SINGLE_MULTI}") + fi if [ "${DRY_RUN}" -eq 0 ]; then [ -n "${VALIDATION_ARTIFACTS_DIR}" ] || die "--validation-case requires --validation-artifacts-dir" [ -n "${VALIDATION_CANDIDATE_MANIFEST}" ] || die "--validation-case requires --candidate-manifest" @@ -1284,7 +1353,10 @@ main() { fi PASS=0; FAIL=0; UNSUPPORTED=0; FAILED_TESTS=() if [ -n "${VALIDATION_CASE}" ]; then - run_test "${VALIDATION_CASE}" "UsageConvergence" test_validation_scanner_usage_convergence + case "${VALIDATION_CASE}" in + scanner.usage-convergence) run_test "${VALIDATION_CASE}" "UsageConvergence" test_validation_scanner_usage_convergence ;; + scanner.status-readiness) run_test "${VALIDATION_CASE}" "StatusReadiness" test_validation_scanner_status_readiness ;; + esac else run_all_tests fi diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 9d4ba33f..a34f4f22 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -15,7 +15,7 @@ ARTIFACTS_DIR="$4" case "${CASE_ID}" in s3.basic-object-round-trip) SUITE=rustfs-s3-compat-test.sh; EXTRA_ARGS=() ;; - scanner.usage-convergence) + scanner.usage-convergence|scanner.status-readiness) SUITE=rustfs-storage-test.sh; EXTRA_ARGS=() ;; kms.sse-kms-round-trip) SUITE=rustfs-kms-test.sh; EXTRA_ARGS=() ;; diff --git a/test_validation_scanner_usage.sh b/test_validation_scanner_usage.sh index 78ffc338..5d5f75b9 100644 --- a/test_validation_scanner_usage.sh +++ b/test_validation_scanner_usage.sh @@ -105,3 +105,27 @@ grep -q 'RUSTFS_SCANNER_CYCLE=5' "${TEST_TMP}/validation-dry-run.log" if grep -q '\[DRY-RUN\] STOR-' "${TEST_TMP}/validation-dry-run.log"; then printf 'validation mode scheduled legacy storage cases\n' >&2; exit 1 fi + +"${SCRIPT_DIR}/rustfs-storage-test.sh" --validation-case scanner.status-readiness \ + --dry-run --yes --package-url https://example.invalid/rustfs.deb \ + > "${TEST_TMP}/validation-status-dry-run.log" +grep -q '\[DRY-RUN\] scanner.status-readiness StatusReadiness' "${TEST_TMP}/validation-status-dry-run.log" +grep -q 'RUSTFS_VOLUMES="http://rustfs-node{1...2}:9000/data/rustfs{1...4}/mnmd"' "${TEST_TMP}/validation-status-dry-run.log" +[ "$(grep -c 'RUSTFS_SCANNER_CYCLE=5' "${TEST_TMP}/validation-status-dry-run.log")" -eq 2 ] + +jq -nc --argjson now "$(date -u +%s)" '{enabled:true, + runtime_config:{cycle_interval_seconds:{value:5,source:"env"},start_delay_seconds:{value:0,source:"env"}}, + metrics:{leader_lock_held_by_this_process:true,leader_lock_state:"acquired",last_cycle_result:"success",last_cycle_end_unix_secs:$now}, + freshness:{state:"fresh"},cycle_schedule:{execution_role:"leader"}}' > "${TEST_TMP}/ready-leader.json" +jq -n '{enabled:true, + runtime_config:{cycle_interval_seconds:{value:5,source:"env"},start_delay_seconds:{value:0,source:"env"}}, + metrics:{leader_lock_held_by_this_process:false},freshness:{state:"fresh"},cycle_schedule:{execution_role:"follower"}}' \ + > "${TEST_TMP}/ready-follower.json" +validation_scanner_status_ready "${TEST_TMP}/ready-leader.json" "${TEST_TMP}/ready-follower.json" 0 +jq '.runtime_config.cycle_interval_seconds.value = 60' "${TEST_TMP}/ready-follower.json" > "${TEST_TMP}/wrong-config.json" +if validation_scanner_status_ready "${TEST_TMP}/ready-leader.json" "${TEST_TMP}/wrong-config.json" 0; then + printf 'wrong scanner configuration passed\n' >&2; exit 1 +fi +if validation_scanner_status_ready "${TEST_TMP}/ready-leader.json" "${TEST_TMP}/ready-leader.json" 0; then + printf 'two scanner leaders passed\n' >&2; exit 1 +fi From 66e3bf34dc2ed8c38703345613760c18fa456174 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 18:26:43 +0800 Subject: [PATCH 011/343] Migrate ILM release validation cases onto storage test fleet --- .../workflows/validation-catalog-tests.yml | 5 + docs/validation-migration-map.md | 19 +- rustfs-storage-test.sh | 86 +- scripts/run_validation_case.sh | 2 +- scripts/validation_ilm/case_plan.py | 30 + scripts/validation_ilm/ilm_behavior.py | 2677 +++++++++++++++++ scripts/validation_ilm/ilm_timing.py | 304 ++ scripts/validation_ilm/rc_binary.py | 21 + scripts/validation_ilm/requirements.txt | 2 + scripts/validation_ilm/restart_cluster.sh | 30 + scripts/validation_ilm/verify_result.py | 46 + test_validation_ilm.py | 45 + 12 files changed, 3257 insertions(+), 10 deletions(-) create mode 100644 scripts/validation_ilm/case_plan.py create mode 100644 scripts/validation_ilm/ilm_behavior.py create mode 100644 scripts/validation_ilm/ilm_timing.py create mode 100644 scripts/validation_ilm/rc_binary.py create mode 100644 scripts/validation_ilm/requirements.txt create mode 100644 scripts/validation_ilm/restart_cluster.sh create mode 100644 scripts/validation_ilm/verify_result.py create mode 100644 test_validation_ilm.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index b572a32c..6ab0fc7a 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -8,6 +8,8 @@ on: - scripts/validation_report.py - scripts/validation_result.py - test_validation_migration.py + - test_validation_ilm.py + - scripts/validation_ilm/** - rustfs-storage-test.sh - test_validation_scanner_usage.sh - .github/workflows/validation-catalog-tests.yml @@ -19,6 +21,8 @@ on: - scripts/validation_report.py - scripts/validation_result.py - test_validation_migration.py + - test_validation_ilm.py + - scripts/validation_ilm/** - rustfs-storage-test.sh - test_validation_scanner_usage.sh - .github/workflows/validation-catalog-tests.yml @@ -36,5 +40,6 @@ jobs: persist-credentials: false - run: python3 scripts/validation_catalog.py check - run: python3 -B -m unittest -v test_validation_migration.py + - run: python3 -B -m unittest -v test_validation_ilm.py - run: bash -n rustfs-storage-test.sh test_validation_scanner_usage.sh - run: bash test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 7d482649..535f9c15 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -32,7 +32,18 @@ test. Its live candidate run passed in RustFS Actions run `38042200106`. `scanner.status-readiness` has a dedicated executor on two of the existing auto-testing nodes, each with four disks. It checks both nodes' accelerated scanner runtime settings and requires exactly one leader to finish a new -successful cycle within 120 seconds. Its live candidate run is pending. +successful cycle within 120 seconds. Its live candidate run passed in RustFS +Actions run `38044125099`. + +The 14 remaining ILM IDs now dispatch to the source ILM black-box behavior +checks under `scripts/validation_ilm`. The Storage suite deploys the same +two-node/four-disk source topology on current auto-testing nodes with the +source's accelerated scanner/ILM timing, and supplies a cluster restart hook +for `ilm.restart-persistence`. `ilm.transition-and-restore` uses the third +auto-testing node as an independent warm-tier target. Each selected case must +pass its own harness result, evidence, and bucket-cleanup verification before +the canonical result can become PASS. These executors have local contract and +dry-run checks, but no live PASS evidence yet. `kms.sse-kms-round-trip` has a dedicated executor using the existing local KMS single-disk deployment. It checks the same object's encryption key, @@ -79,9 +90,9 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The eight currently dispatchable IDs are the S3, two scanner, -KMS, tiering, events, audit, and IAM cases above. Five have live PASS evidence; the -other 153 IDs remain unverified. +case evidence. The 22 currently dispatchable IDs are the S3, two scanner, +KMS, tiering, events, audit, IAM, and 14 additional ILM cases above. Six have +live PASS evidence; the other 152 IDs remain unverified. The eight planned cases have these concrete reuse candidates: diff --git a/rustfs-storage-test.sh b/rustfs-storage-test.sh index cc046862..e111b7c4 100755 --- a/rustfs-storage-test.sh +++ b/rustfs-storage-test.sh @@ -83,6 +83,7 @@ VOLUMES_SINGLE_SINGLE="/data/rustfs1/mnmd" VOLUMES_SINGLE_MULTI="/data/rustfs{1...4}/mnmd" VOLUMES_MULTI_MULTI="http://rustfs-node{1...4}:9000/data/rustfs{1...4}/mnmd" VOLUMES_ILM_2X4="http://rustfs-node{1...2}:9000/data/rustfs{1...4}/mnmd" +ILM_TIER_ENDPOINT="http://rustfs-node3:9000" SERVICE_TIMEOUT=300 @@ -194,6 +195,11 @@ RUSTFS_OBS_LOG_DIRECTORY="${RUSTFS_OBS_LOG_DIRECTORY}" EOF if [ "${VALIDATION_CASE}" = "scanner.usage-convergence" ] || [ "${VALIDATION_CASE}" = "scanner.status-readiness" ]; then printf 'RUSTFS_SCANNER_ENABLED=true\nRUSTFS_SCANNER_CYCLE=5\nRUSTFS_SCANNER_START_DELAY_SECS=0\n' + elif [[ "${VALIDATION_CASE}" == ilm.* ]]; then + printf 'RUSTFS_SCANNER_ENABLED=true\nRUSTFS_SCANNER_CYCLE=2\nRUSTFS_SCANNER_START_DELAY_SECS=0\n' + printf 'RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=1\nRUSTFS_ILM_DEBUG_DAY_SECS=10\nRUSTFS_ILM_PROCESS_TIME=1\n' + printf 'RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL=2s\n' + printf 'HTTP_PROXY=\nHTTPS_PROXY=\nALL_PROXY=\nhttp_proxy=\nhttps_proxy=\nall_proxy=\n' fi } @@ -288,7 +294,12 @@ resolve_topology() { topology_nodes() { case "$1" in "${TOPO_SINGLE_SINGLE}"|"${TOPO_SINGLE_MULTI}") printf '%s' "${NODES[0]}" ;; - "${TOPO_ILM_2X4}") printf '%s %s' "${NODES[0]}" "${NODES[1]}" ;; + "${TOPO_ILM_2X4}") + if [ "${VALIDATION_CASE}" = ilm.transition-and-restore ]; then + printf '%s %s %s' "${NODES[0]}" "${NODES[1]}" "${NODES[2]}" + else + printf '%s %s' "${NODES[0]}" "${NODES[1]}" + fi ;; *) printf '%s' "${NODES[*]}" ;; esac } @@ -328,8 +339,13 @@ EOF for node in "${nodes[@]}"; do printf '%s\n' "${script}" | run_remote "${node}"; done for node in "${nodes[@]}"; do - write_rustfs_config "${node}" "${VOLUMES}" - ensure_volume_dirs "${node}" "${VOLUMES}" + if [ "${VALIDATION_CASE}" = ilm.transition-and-restore ] && [ "${node}" = "${NODES[2]}" ]; then + VALIDATION_CASE="" write_rustfs_config "${node}" "${VOLUMES_SINGLE_SINGLE}" + ensure_volume_dirs "${node}" "${VOLUMES_SINGLE_SINGLE}" + else + write_rustfs_config "${node}" "${VOLUMES}" + ensure_volume_dirs "${node}" "${VOLUMES}" + fi done start_and_wait_nodes "${nodes[@]}" @@ -343,6 +359,16 @@ EOF sleep 5; waited=$((waited+5)) done [ "${waited}" -lt "${SERVICE_TIMEOUT}" ] || die "S3 endpoint ${ENDPOINT} not ready" + if [ "${VALIDATION_CASE}" = ilm.transition-and-restore ]; then + waited=0 + while [ "${waited}" -lt "${SERVICE_TIMEOUT}" ]; do + if curl --noproxy '*' -fsS --max-time 5 "${ILM_TIER_ENDPOINT}/minio/health/live" >/dev/null 2>&1; then + log "ILM warm tier ready: ${ILM_TIER_ENDPOINT}"; break + fi + sleep 5; waited=$((waited+5)) + done + [ "${waited}" -lt "${SERVICE_TIMEOUT}" ] || die "ILM warm tier ${ILM_TIER_ENDPOINT} not ready" + fi rc alias set local "${ENDPOINT}" "${ACCESS_KEY}" "${SECRET_KEY}" >/dev/null 2>&1 \ || die "rc alias set failed for ${ENDPOINT}" @@ -515,6 +541,9 @@ validation_write_evidence() { for item in $(validation_evidence_items); do if [ -f "${TEST_TMP}/${item}" ]; then cp "${TEST_TMP}/${item}" "${evidence_dir}/${item}"; fi done + if [[ "${VALIDATION_CASE}" == ilm.* ]] && [ -d "${TEST_TMP}/ilm-evidence" ]; then + cp -R "${TEST_TMP}/ilm-evidence" "${evidence_dir}/ilm" + fi } validation_evidence_items() { @@ -529,7 +558,9 @@ validation_evidence_items() { validation_write_result() { local verdict="$1" reason="$2" cleanup_status="$3" item local profile=single-4disk - [ "${VALIDATION_CASE}" = scanner.status-readiness ] && profile=ilm-2x4 + if [ "${VALIDATION_CASE}" = scanner.status-readiness ] || [[ "${VALIDATION_CASE}" == ilm.* ]]; then + profile=ilm-2x4 + fi local args=("${SCRIPT_DIR}/scripts/validation_result.py" --case-id "${VALIDATION_CASE}" --profile "${profile}" --result-file "${VALIDATION_ARTIFACTS_DIR}/cases/${VALIDATION_CASE}.json" @@ -545,10 +576,44 @@ validation_write_result() { for item in $(validation_evidence_items); do args+=(--evidence "evidence/${VALIDATION_CASE}/${item}") done + if [[ "${VALIDATION_CASE}" == ilm.* ]]; then + local evidence_root="${VALIDATION_ARTIFACTS_DIR}/evidence/${VALIDATION_CASE}" + local evidence_file + while IFS= read -r -d '' evidence_file; do + args+=(--evidence "${evidence_file#"${VALIDATION_ARTIFACTS_DIR}/"}") + done < <(find "${evidence_root}/ilm" -type f -print0 | sort -z) + fi fi python3 "${args[@]}" } +test_validation_ilm_case() { + local harness_dir="${TEST_TMP}/ilm-evidence" harness_rc=0 + local -a ilm_args=() + if [ "${VALIDATION_CASE}" = ilm.restart-persistence ]; then + ilm_args+=(--restart-hook "$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1:]))' \ + bash "${SCRIPT_DIR}/scripts/validation_ilm/restart_cluster.sh" "${NODES[0]}" "${NODES[1]}")") + fi + if [ "${VALIDATION_CASE}" = ilm.transition-and-restore ]; then + ilm_args+=(--tier-endpoint "${ILM_TIER_ENDPOINT}" + --tier-access-key "${ACCESS_KEY}" --tier-secret-key "${SECRET_KEY}") + fi + python3 -B "${SCRIPT_DIR}/scripts/validation_ilm/ilm_behavior.py" \ + --case "${VALIDATION_CASE}" \ + --endpoint "${ENDPOINT}" \ + --scanner-endpoint "http://rustfs-node1:9000" \ + --scanner-endpoint "http://rustfs-node2:9000" \ + --access-key "${ACCESS_KEY}" --secret-key "${SECRET_KEY}" \ + --run-id "${VALIDATION_CASE}-$(date +%s)-$$" \ + --evidence-dir "${harness_dir}" \ + --rc-bin "$(command -v rc)" \ + "${ilm_args[@]}" \ + --cleanup-on-failure || harness_rc=$? + python3 -B "${SCRIPT_DIR}/scripts/validation_ilm/verify_result.py" \ + "${VALIDATION_CASE}" "${harness_dir}" || return 1 + [ "${harness_rc}" -eq 0 ] +} + rc_run() { RC_OUT="$(rc "$@" 2>&1)"; RC_CODE=$? } @@ -1299,14 +1364,18 @@ main() { if [ -n "${VALIDATION_CASE}" ]; then case "${VALIDATION_CASE}" in scanner.usage-convergence|scanner.status-readiness) ;; + ilm.configuration-roundtrip|ilm.invalid-configuration|ilm.current-expiration|ilm.tag-filter-expiration|ilm.disabled-rule|ilm.rule-removal|ilm.versioned-current-expiration|ilm.noncurrent-expiration|ilm.newer-noncurrent-retention|ilm.delete-all-versions|ilm.delete-marker-expiration|ilm.multipart-expiration|ilm.restart-persistence|ilm.transition-and-restore) ;; *) die "unknown validation case: ${VALIDATION_CASE}" ;; esac [ "${all}" -eq 0 ] || die "--validation-case cannot be combined with --all-topologies" [ "${#SELECTED_TOPOLOGIES[@]}" -eq 0 ] || die "--validation-case chooses its own topology" [ "${KEEP_ENV}" -eq 0 ] || die "--validation-case requires environment teardown" - if [ "${VALIDATION_CASE}" = scanner.status-readiness ]; then + if [ "${VALIDATION_CASE}" = scanner.status-readiness ] || [[ "${VALIDATION_CASE}" == ilm.* ]]; then SELECTED_TOPOLOGIES=("${TOPO_ILM_2X4}") [ "${#NODES[@]}" -ge 2 ] || die "scanner.status-readiness needs two test nodes" + if [ "${VALIDATION_CASE}" = ilm.transition-and-restore ]; then + [ "${#NODES[@]}" -ge 3 ] || die "ilm.transition-and-restore needs three test nodes" + fi else SELECTED_TOPOLOGIES=("${TOPO_SINGLE_MULTI}") fi @@ -1338,6 +1407,9 @@ main() { need_cmd curl "HTTP client" need_cmd openssl "OpenSSL (SigV4 signing)" if [ -n "${VALIDATION_CASE}" ]; then need_cmd python3 "validation result writer"; fi + if [[ "${VALIDATION_CASE}" == ilm.* ]] && [ "${DRY_RUN}" -eq 0 ]; then + python3 -c 'import boto3, botocore' || die "ILM validation requires boto3 and botocore" + fi log "nodes: ${NODES[*]} package: $(resolve_package_url)" local overall_fail=0 topo for topo in "${SELECTED_TOPOLOGIES[@]}"; do @@ -1356,6 +1428,7 @@ main() { case "${VALIDATION_CASE}" in scanner.usage-convergence) run_test "${VALIDATION_CASE}" "UsageConvergence" test_validation_scanner_usage_convergence ;; scanner.status-readiness) run_test "${VALIDATION_CASE}" "StatusReadiness" test_validation_scanner_status_readiness ;; + ilm.*) run_test "${VALIDATION_CASE}" "ILMBehavior" test_validation_ilm_case ;; esac else run_all_tests @@ -1373,6 +1446,9 @@ main() { validation_write_result PASS "" PASS elif [ "${FAIL}" -eq 0 ] && [ "${UNSUPPORTED}" -eq 0 ]; then validation_write_result INFRA_ERROR "environment teardown failed" FAIL + elif [[ "${VALIDATION_CASE}" == ilm.* ]] && [ -f "${TEST_TMP}/ilm-evidence/result.json" ] && \ + jq -e '.status.state == "infra_error"' "${TEST_TMP}/ilm-evidence/result.json" >/dev/null; then + validation_write_result INFRA_ERROR "ILM harness preflight or execution infrastructure failed; inspect evidence" "${cleanup_status}" else validation_write_result FAIL "case failed or unsupported; inspect evidence" "${cleanup_status}" fi diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index a34f4f22..08bcc52e 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -15,7 +15,7 @@ ARTIFACTS_DIR="$4" case "${CASE_ID}" in s3.basic-object-round-trip) SUITE=rustfs-s3-compat-test.sh; EXTRA_ARGS=() ;; - scanner.usage-convergence|scanner.status-readiness) + scanner.usage-convergence|scanner.status-readiness|ilm.configuration-roundtrip|ilm.invalid-configuration|ilm.current-expiration|ilm.tag-filter-expiration|ilm.disabled-rule|ilm.rule-removal|ilm.versioned-current-expiration|ilm.noncurrent-expiration|ilm.newer-noncurrent-retention|ilm.delete-all-versions|ilm.delete-marker-expiration|ilm.multipart-expiration|ilm.restart-persistence|ilm.transition-and-restore) SUITE=rustfs-storage-test.sh; EXTRA_ARGS=() ;; kms.sse-kms-round-trip) SUITE=rustfs-kms-test.sh; EXTRA_ARGS=() ;; diff --git a/scripts/validation_ilm/case_plan.py b/scripts/validation_ilm/case_plan.py new file mode 100644 index 00000000..f81d22f8 --- /dev/null +++ b/scripts/validation_ilm/case_plan.py @@ -0,0 +1,30 @@ +"""Select ILM cases from the canonical auto-testing catalog.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Iterable + + +CATALOG = Path(__file__).resolve().parents[2] / "validation-cases.json" + + +def ilm_case_ids() -> tuple[str, ...]: + catalog = json.loads(CATALOG.read_text(encoding="utf-8")) + modules = [module for module in catalog["modules"] if module["id"] == "ilm-expiration"] + if len(modules) != 1: + raise ValueError("canonical ILM module is missing or duplicated") + return tuple(case["id"] for case in modules[0]["cases"]) + + +ILM_CASE_IDS = ilm_case_ids() + + +def ilm_harness_scored_cases(requested_cases: Iterable[str] | None) -> tuple[str, ...]: + if requested_cases is None: + return ILM_CASE_IDS + selected = tuple(requested_cases) + if len(selected) != len(set(selected)) or set(selected) - set(ILM_CASE_IDS): + raise ValueError("duplicate or unknown canonical ILM case") + return tuple(case_id for case_id in ILM_CASE_IDS if case_id in selected) diff --git a/scripts/validation_ilm/ilm_behavior.py b/scripts/validation_ilm/ilm_behavior.py new file mode 100644 index 00000000..dc7af7d5 --- /dev/null +++ b/scripts/validation_ilm/ilm_behavior.py @@ -0,0 +1,2677 @@ +#!/usr/bin/env python3 +"""Run destructive-in-a-test-bucket RustFS ILM behavior checks.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import math +import os +import re +import secrets +import shlex +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +import xml.etree.ElementTree as ET +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Callable +from urllib import error as urllib_error +from urllib import request as urllib_request +from urllib.parse import quote, unquote, urlparse + +from ilm_timing import ( + BoundedS3Client, + CycleProgress, + DueWindow, + ServerClock, + expiry_window, + parse_expiry_date, + time_budget, +) + +from case_plan import ( + ILM_CASE_IDS as CASE_IDS, + ilm_harness_scored_cases, +) +from rc_binary import resolve_rc_binary + +try: + import boto3 + import botocore + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.config import Config + from botocore.credentials import Credentials + from botocore.exceptions import ( + ClientError, + ConnectTimeoutError, + EndpointConnectionError, + ReadTimeoutError, + ) +except ImportError as exc: # pragma: no cover - dependency guidance + raise SystemExit( + "boto3 is required for ILM validation" + ) from exc + + +INFRA_EXCEPTIONS = (ConnectTimeoutError, EndpointConnectionError, ReadTimeoutError) +INFRA_ERROR_CODES = { + "AccessDenied", + "InvalidAccessKeyId", + "InvalidToken", + "RequestTimeout", + "ServiceUnavailable", + "SignatureDoesNotMatch", + "SlowDown", +} + +S3_DELETE_RETRYABLE_ERROR_CODES = { + "InternalError", + "RequestTimeout", + "ServiceUnavailable", + "SlowDown", +} + +def utc_now() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + +def json_default(value: Any) -> Any: + if isinstance(value, datetime): + return value.astimezone(UTC).isoformat().replace("+00:00", "Z") + if isinstance(value, bytes): + return { + "bytes": len(value), + "sha256": hashlib.sha256(value).hexdigest(), + } + return str(value) + + +def safe_name(value: str) -> str: + return "".join(char if char.isalnum() or char in "-." else "-" for char in value) + + +@dataclass +class CaseResult: + id: str + state: str + message: str + started_at: str + finished_at: str + duration_seconds: float + evidence: list[str] = field(default_factory=list) + + +class CaseExecutionAndCleanupError(Exception): + def __init__( + self, + primary_phase: str, + primary_error: Exception, + cleanup_phase: str, + cleanup_error: Exception, + ) -> None: + self.primary_phase = primary_phase + self.primary_error = primary_error + self.cleanup_phase = cleanup_phase + self.cleanup_error = cleanup_error + super().__init__( + f"{primary_phase} failed: {type(primary_error).__name__}: " + f"{primary_error}; cleanup {cleanup_phase} failed: " + f"{type(cleanup_error).__name__}: {cleanup_error}" + ) + + +class EventRecorder: + def __init__(self, path: Path) -> None: + self.path = path + self.lock = threading.Lock() + self.path.parent.mkdir(parents=True, exist_ok=True) + + def record(self, event: str, **details: Any) -> None: + item = {"timestamp": utc_now(), "monotonic": time.monotonic(), "event": event, "details": details} + with self.lock: + with self.path.open("a", encoding="utf-8") as handle: + handle.write( + json.dumps(item, default=json_default, sort_keys=True) + "\n" + ) + + +class IlmHarness: + def __init__(self, args: argparse.Namespace) -> None: + self.args = args + self.evidence_dir = args.evidence_dir.resolve() + self.evidence_dir.mkdir(parents=True, exist_ok=True) + self.case_dir = self.evidence_dir / "cases" + self.case_dir.mkdir(parents=True, exist_ok=True) + self.events = EventRecorder(self.evidence_dir / "events.jsonl") + self.results: list[CaseResult] = [] + self.created_buckets: set[str] = set() + self.uncertain_buckets: set[str] = set() + self.state_lock = threading.Lock() + self.rc_home = Path(tempfile.mkdtemp(prefix="rustfs-ilm-rc-")) + alias_base = f"ilm-{safe_name(args.run_id)[-16:]}" + self.rc_aliases = tuple( + (f"{alias_base}-{index}", endpoint) + for index, endpoint in enumerate(args.scanner_endpoint) + ) + self.rc_alias = self.rc_aliases[0][0] + self.rc_env = os.environ.copy() + self.rc_env["RC_CONFIG_DIR"] = str(self.rc_home) + self.rc_bin = args.rc_bin + self.rc_version = "unavailable" + self.rc_binary_sha256 = "unavailable" + self.expected_rc_binary_sha256 = os.environ.get( + "RUSTFS_EXPECTED_RC_SHA256", "" + ).lower() + self.clock: ServerClock | None = None + self.readiness: dict[str, Any] = {"state": "not_started"} + self.blocked_reason: str | None = None + self.case_buckets: set[str] = set() + self.client = boto3.client( + "s3", + endpoint_url=args.endpoint, + aws_access_key_id=args.access_key, + aws_secret_access_key=args.secret_key, + region_name=args.region, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + retries={"total_max_attempts": 1, "mode": "standard"}, + connect_timeout=5, + read_timeout=args.request_timeout, + ), + ) + self.client = BoundedS3Client(self.client, args.request_timeout) + self.node_clients = {args.endpoint: self.client} + for endpoint in args.scanner_endpoint: + if endpoint not in self.node_clients: + self.node_clients[endpoint] = BoundedS3Client(boto3.client( + "s3", endpoint_url=endpoint, region_name=args.region, + aws_access_key_id=args.access_key, aws_secret_access_key=args.secret_key, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, + retries={"total_max_attempts": 1, "mode": "standard"}, + connect_timeout=5, read_timeout=args.request_timeout), + ), args.request_timeout) + self.tier_client = None + if args.tier_endpoint and args.tier_access_key and args.tier_secret_key: + self.tier_client = boto3.client( + "s3", + endpoint_url=args.tier_endpoint, + aws_access_key_id=args.tier_access_key, + aws_secret_access_key=args.tier_secret_key, + region_name=args.region, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + retries={"total_max_attempts": 1, "mode": "standard"}, + connect_timeout=5, + read_timeout=args.request_timeout, + ), + ) + self.tier_client = BoundedS3Client(self.tier_client, args.request_timeout) + + def close(self) -> None: + for client in self.node_clients.values(): + client.close() + if self.tier_client is not None: + self.tier_client.close() + shutil.rmtree(self.rc_home, ignore_errors=True) + + def evidence_path(self, case_id: str, name: str) -> Path: + directory = self.case_dir / safe_name(case_id) + directory.mkdir(parents=True, exist_ok=True) + return directory / name + + def save_json(self, case_id: str, name: str, value: Any) -> str: + path = self.evidence_path(case_id, name) + path.write_text( + json.dumps(value, default=json_default, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return str(path.relative_to(self.evidence_dir)) + + def save_text(self, case_id: str, name: str, value: str) -> str: + path = self.evidence_path(case_id, name) + path.write_text(value, encoding="utf-8") + return str(path.relative_to(self.evidence_dir)) + + def redact(self, value: str) -> str: + redacted = value + for name in ( + "access_key", + "secret_key", + "tier_access_key", + "tier_secret_key", + ): + secret = getattr(self.args, name, None) + if isinstance(secret, str) and secret: + redacted = redacted.replace(secret, "[REDACTED]") + return redacted + + def save_phase_error( + self, + case_id: str, + name: str, + phase: str, + exc: Exception, + ) -> str: + state = self.classify_exception(exc) + message = self.redact(f"{type(exc).__name__}: {exc}") + self.events.record( + name.removesuffix(".json").replace("-", "."), + case=case_id, + phase=phase, + state=state, + errorType=type(exc).__name__, + message=message, + ) + return self.save_json( + case_id, + name, + { + "phase": phase, + "state": state, + "errorType": type(exc).__name__, + "message": message, + }, + ) + + def bucket_name(self, case_id: str) -> str: + digest = hashlib.sha256( + f"{self.args.run_id}:{case_id}".encode("utf-8") + ).hexdigest()[:12] + prefix = safe_name(self.args.bucket_prefix.lower()).strip("-")[:45] + return f"{prefix}-{digest}" + + def create_bucket(self, bucket: str) -> None: + self.events.record("bucket.create.start", bucket=bucket) + try: + self.client.create_bucket(Bucket=bucket) + except Exception: + # A lost response can leave a remote creation in progress. Do not + # claim clean isolation or continue to another fixture in this run. + self.uncertain_buckets.add(bucket) + self.blocked_reason = f"bucket creation was not confirmed for {bucket}; isolation requires inspection" + raise + with self.state_lock: + self.created_buckets.add(bucket) + self.case_buckets.add(bucket) + self.events.record("bucket.create.complete", bucket=bucket) + + def put_lifecycle( + self, case_id: str, bucket: str, configuration: dict[str, Any] + ) -> list[str]: + evidence = [ + self.save_json(case_id, "lifecycle-request.json", configuration) + ] + response = self.client.put_bucket_lifecycle_configuration( + Bucket=bucket, + LifecycleConfiguration=configuration, + ) + evidence.append(self.save_json(case_id, "lifecycle-put-response.json", response)) + self.events.record( + "lifecycle.put", + bucket=bucket, + rule_ids=[rule.get("ID") for rule in configuration["Rules"]], + ) + return evidence + + def get_lifecycle(self, bucket: str) -> dict[str, Any]: + return self.client.get_bucket_lifecycle_configuration(Bucket=bucket) + + def raw_s3_request( + self, + method: str, + bucket: str, + *, + query: str = "", + body: bytes | None = None, + content_type: str | None = None, + allow_error: bool = False, + ) -> tuple[int, dict[str, str], bytes]: + url = f"{self.args.endpoint.rstrip('/')}/{quote(bucket, safe='')}" + if query: + url = f"{url}?{query}" + headers: dict[str, str] = {} + if content_type: + headers["Content-Type"] = content_type + payload = body if body is not None else b"" + headers["x-amz-content-sha256"] = hashlib.sha256(payload).hexdigest() + if body is not None: + digest = hashlib.md5(body, usedforsecurity=False).digest() + headers["Content-MD5"] = base64.b64encode(digest).decode("ascii") + signed = AWSRequest(method=method, url=url, data=body, headers=headers) + SigV4Auth( + Credentials(self.args.access_key, self.args.secret_key), + "s3", + self.args.region, + ).add_auth(signed) + outgoing = urllib_request.Request( + url, + data=body, + headers=dict(signed.headers.items()), + method=method, + ) + opener = urllib_request.build_opener(urllib_request.ProxyHandler({})) + with time_budget(self.args.request_timeout, "raw S3 request", TimeoutError): + try: + with opener.open(outgoing, timeout=self.args.request_timeout) as response: + return response.status, dict(response.headers.items()), response.read() + except urllib_error.HTTPError as exc: + with exc: + response_body = exc.read() + if allow_error: + return exc.code, dict(exc.headers.items()), response_body + raise AssertionError( + f"raw S3 {method} request failed with HTTP {exc.code}: " + f"{response_body.decode('utf-8', errors='replace')[:500]}" + ) from exc + + def configure_rc(self) -> None: + requested_rc_bin = self.rc_bin + try: + self.rc_bin, self.rc_binary_sha256 = resolve_rc_binary(requested_rc_bin) + except FileNotFoundError as exc: + raise RuntimeError( + f"required rc binary not found: {requested_rc_bin}" + ) from exc + version = subprocess.run( + [self.rc_bin, "--version"], + env=self.rc_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + timeout=getattr(self.args, "request_timeout", 10), + ) + if version.returncode != 0: + raise RuntimeError( + f"rc --version failed with exit {version.returncode}: " + f"{version.stderr.strip()}" + ) + self.rc_version = (version.stdout or version.stderr).strip() + if ( + self.expected_rc_binary_sha256 + and self.rc_binary_sha256.lower() != self.expected_rc_binary_sha256 + ): + raise RuntimeError( + "resolved rc binary SHA-256 differs from RUSTFS_EXPECTED_RC_SHA256" + ) + self.events.record( + "rc.client.identified", + binaryPath=self.rc_bin, + version=self.rc_version, + binarySha256=self.rc_binary_sha256, + ) + for alias, endpoint in self.rc_aliases: + command = [ + self.rc_bin, + "--json", + "--no-color", + "alias", + "set", + alias, + endpoint, + self.args.access_key, + self.args.secret_key, + "--bucket-lookup", + "path", + ] + deadline = time.monotonic() + min(self.args.timeout, 60) + attempts = 0 + while True: + attempts += 1 + result = subprocess.run( + command, + env=self.rc_env, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + check=False, + timeout=getattr(self.args, "request_timeout", 10), + ) + if result.returncode == 0: + break + error = result.stderr.strip() + try: + error_document = json.loads(error) + except json.JSONDecodeError: + error_document = {} + retryable = ( + isinstance(error_document, dict) + and isinstance(error_document.get("details"), dict) + and error_document["details"].get("retryable") is True + ) + if not retryable or time.monotonic() >= deadline: + raise RuntimeError( + f"rc alias setup failed for {endpoint}: {error}" + ) + self.events.record( + "rc.alias.retry", + alias=alias, + endpoint=endpoint, + attempt=attempts, + error=error[:500], + ) + time.sleep(self.args.poll_interval) + self.events.record( + "rc.alias.configured", + alias=alias, + endpoint=endpoint, + attempts=attempts, + ) + + def wait_for_s3_connectivity(self) -> None: + deadline = time.monotonic() + min(self.args.timeout, 60) + attempts = 0 + while True: + attempts += 1 + try: + self.client.list_buckets() + self.events.record( + "s3.connectivity.ready", + endpoint=self.args.endpoint, + attempts=attempts, + ) + return + except (*INFRA_EXCEPTIONS, ClientError) as exc: + retryable = isinstance(exc, INFRA_EXCEPTIONS) + if isinstance(exc, ClientError): + response = exc.response + status = response.get("ResponseMetadata", {}).get( + "HTTPStatusCode", 0 + ) + code = response.get("Error", {}).get("Code", "") + retryable = status >= 500 or code in { + "RequestTimeout", + "ServiceUnavailable", + "SlowDown", + } + if not retryable or time.monotonic() >= deadline: + raise + self.events.record( + "s3.connectivity.retry", + endpoint=self.args.endpoint, + attempt=attempts, + error=f"{type(exc).__name__}: {exc}"[:500], + ) + time.sleep(self.args.poll_interval) + + def rc_lifecycle_list(self, case_id: str, bucket: str) -> str: + command = [ + self.rc_bin, + "--json", + "--no-color", + "bucket", + "lifecycle", + "rule", + "list", + "--force", + f"{self.rc_alias}/{bucket}", + ] + result = subprocess.run( + command, + env=self.rc_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + timeout=getattr(self.args, "request_timeout", 10), + ) + output = result.stdout + path = self.evidence_path(case_id, "rc-lifecycle-list.json") + path.write_text(output, encoding="utf-8") + if result.returncode != 0: + raise AssertionError( + f"rc lifecycle list failed with exit {result.returncode}: " + f"{result.stderr.strip()}" + ) + self.events.record("rc.lifecycle.list", bucket=bucket) + return output + + def rc_command( + self, + case_id: str, + name: str, + arguments: list[str], + *, + expect_success: bool = True, + secrets_to_redact: tuple[str, ...] = (), + ) -> subprocess.CompletedProcess[str]: + command = [self.rc_bin, "--json", "--no-color", *arguments] + result = subprocess.run( + command, + env=self.rc_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + timeout=getattr(self.args, "request_timeout", 10), + ) + sanitized = [ + "[REDACTED]" if token in secrets_to_redact else token for token in command + ] + output = { + "command": sanitized, + "exitCode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + } + text = json.dumps(output, indent=2, sort_keys=True) + "\n" + for secret in secrets_to_redact: + text = text.replace(secret, "[REDACTED]") + self.evidence_path(case_id, f"{name}.json").write_text( + text, encoding="utf-8" + ) + if expect_success and result.returncode != 0: + error = result.stderr + for secret in secrets_to_redact: + error = error.replace(secret, "[REDACTED]") + raise AssertionError( + f"rc {name} failed with exit {result.returncode}: {error.strip()}" + ) + if not expect_success and result.returncode == 0: + raise AssertionError(f"rc {name} unexpectedly succeeded") + return result + + @staticmethod + def rc_error_document(stderr: str) -> dict[str, Any]: + try: + parsed = json.loads(stderr.strip()) + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + @classmethod + def rc_error_mentions(cls, stderr: str, code: str) -> bool: + if code in stderr: + return True + document = cls.rc_error_document(stderr) + if document.get("error") == code: + return True + details = document.get("details") + if isinstance(details, dict): + return any( + isinstance(value, str) and code in value + for value in details.values() + ) + return False + + def rc_tier_remove_eventually( + self, + case_id: str, + tier_name: str, + *, + timeout: float | None = None, + ) -> subprocess.CompletedProcess[str]: + command = [ + self.rc_bin, + "--json", + "--no-color", + "ilm", + "tier", + "remove", + tier_name, + self.rc_alias, + "--force", + ] + deadline = time.monotonic() + (self.args.timeout if timeout is None else timeout) + attempts = 0 + attempt_summaries: list[dict[str, Any]] = [] + while True: + attempts += 1 + remaining = deadline - time.monotonic() + with time_budget(remaining, "rc tier removal"): + result = subprocess.run( + command, + env=self.rc_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + timeout=min(getattr(self.args, "request_timeout", 10), remaining), + ) + attempt_name = f"tier-remove-attempt-{attempts}.json" + attempt_summary = { + "attempt": attempts, + "exitCode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + } + attempt_summaries.append(attempt_summary) + self.evidence_path(case_id, attempt_name).write_text( + json.dumps( + {"command": command, **attempt_summary}, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + self.evidence_path(case_id, "tier-remove.json").write_text( + json.dumps( + { + "command": command, + "attempts": attempt_summaries, + "exitCode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + if time.monotonic() >= deadline: + raise AssertionError("rc tier removal completed after its deadline") + if result.returncode == 0: + self.events.record( + "rc.tier_remove.complete", + case=case_id, + tier=tier_name, + attempts=attempts, + ) + return result + + retryable = self.rc_error_mentions( + result.stderr, + "XMinioAdminTierBackendInUse", + ) + if not retryable or time.monotonic() >= deadline: + error = result.stderr.strip() + self.events.record( + "rc.tier_remove.failed", + case=case_id, + tier=tier_name, + attempts=attempts, + retryable=retryable, + error=error[:500], + ) + raise AssertionError( + f"rc tier-remove failed with exit {result.returncode}: {error}" + ) + + self.events.record( + "rc.tier_remove.retry", + case=case_id, + tier=tier_name, + attempt=attempts, + error=result.stderr.strip()[:500], + ) + time.sleep(min(self.args.poll_interval, max(0, deadline - time.monotonic()))) + + @staticmethod + def client_error_code(exc: ClientError) -> str: + response = getattr(exc, "response", {}) + if not isinstance(response, dict): + return "" + error = response.get("Error", {}) + if not isinstance(error, dict): + return "" + code = error.get("Code") + return code if isinstance(code, str) else "" + + def delete_object_eventually( + self, + case_id: str, + bucket: str, + key: str, + *, + evidence_name: str, + timeout: float | None = None, + ) -> dict[str, Any]: + deadline = time.monotonic() + (self.args.timeout if timeout is None else timeout) + attempts = 0 + attempt_summaries: list[dict[str, Any]] = [] + while True: + attempts += 1 + try: + with time_budget(deadline - time.monotonic(), "S3 object deletion"): + response = self.client.delete_object(Bucket=bucket, Key=key) + except ClientError as exc: + code = self.client_error_code(exc) + retryable = code in S3_DELETE_RETRYABLE_ERROR_CODES + attempt_summary = { + "attempt": attempts, + "state": "failed", + "errorCode": code, + "errorMessage": str(exc), + "retryable": retryable, + } + attempt_summaries.append(attempt_summary) + self.save_json( + case_id, + f"{evidence_name}-attempt-{attempts}.json", + attempt_summary, + ) + self.save_json( + case_id, + f"{evidence_name}.json", + { + "bucket": bucket, + "key": key, + "attempts": attempt_summaries, + "state": "failed", + }, + ) + if not retryable or time.monotonic() >= deadline: + self.events.record( + "s3.delete_object.failed", + case=case_id, + bucket=bucket, + key=key, + attempts=attempts, + errorCode=code, + retryable=retryable, + ) + raise + self.events.record( + "s3.delete_object.retry", + case=case_id, + bucket=bucket, + key=key, + attempt=attempts, + errorCode=code, + ) + time.sleep(min(self.args.poll_interval, max(0, deadline - time.monotonic()))) + continue + + attempt_summary = { + "attempt": attempts, + "state": "success", + "response": response, + } + attempt_summaries.append(attempt_summary) + self.save_json( + case_id, + f"{evidence_name}-attempt-{attempts}.json", + attempt_summary, + ) + summary = { + "bucket": bucket, + "key": key, + "attempts": attempt_summaries, + "state": "success", + "response": response, + } + self.save_json(case_id, f"{evidence_name}.json", summary) + if time.monotonic() >= deadline: + raise AssertionError("S3 object deletion completed after its deadline") + self.events.record( + "s3.delete_object.complete", + case=case_id, + bucket=bucket, + key=key, + attempts=attempts, + ) + return summary + + def rc_scanner_status(self, case_id: str, name: str) -> dict[str, Any]: + nodes: list[dict[str, Any]] = [] + errors: list[str] = [] + for alias, endpoint in self.rc_aliases: + command = [ + self.rc_bin, + "--json", + "--no-color", + "admin", + "scanner", + "status", + alias, + ] + result = subprocess.run( + command, + env=self.rc_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + timeout=getattr(self.args, "request_timeout", 10), + ) + if result.returncode != 0: + error = ( + f"{endpoint}: rc scanner status exited {result.returncode}: " + f"{result.stderr.strip()}" + ) + nodes.append({"endpoint": endpoint, "error": error}) + errors.append(error) + continue + try: + status = json.loads(result.stdout) + except json.JSONDecodeError: + error = f"{endpoint}: rc scanner status did not return JSON" + nodes.append({"endpoint": endpoint, "error": error}) + errors.append(error) + continue + nodes.append({"endpoint": endpoint, "status": status}) + cluster_status = {"nodes": nodes} + self.save_json(case_id, name, cluster_status) + leaders = [ + node["endpoint"] + for node in nodes + if node.get("status", {}) + .get("data", {}) + .get("metrics", {}) + .get("leader_lock_held_by_this_process") + is True + ] + observations = [] + for node in nodes: + data = node.get("status", {}).get("data", {}) + metrics = data.get("metrics", {}) + observations.append({ + "endpoint": node["endpoint"], "error": node.get("error"), + "enabled": data.get("enabled"), "health": data.get("health"), + "freshness": data.get("freshness", {}), + "executionRole": data.get("cycle_schedule", {}).get("execution_role"), + "leaderHeldHere": metrics.get("leader_lock_held_by_this_process"), + "leaderLock": metrics.get("leader_lock_state"), + "lastCycleResult": metrics.get("last_cycle_result"), + "lifetimeCycles": metrics.get("life_time_ops", {}).get("scan_cycle"), + "currentCycle": metrics.get("current_cycle"), + "currentStarted": metrics.get("current_started"), + }) + self.events.record( + "rc.scanner.status", + case=case_id, + endpoints=[node["endpoint"] for node in nodes], + leaders=leaders, + observations=observations, + ) + if errors: + raise RuntimeError("; ".join(errors)) + return cluster_status + + def calibrate_clock(self, case_id: str) -> None: + samples = [] + for endpoint, client in self.node_clients.items(): + started = time.time() + response = client.list_buckets() + finished = time.time() + samples.append((endpoint, ServerClock.response_date(response), started, finished)) + self.clock = ServerClock(samples, max_skew=self.args.max_clock_skew) + self.clock_sample_index = getattr(self, "clock_sample_index", 0) + 1 + self.save_json(case_id, f"clock-{self.clock_sample_index:04d}.json", self.clock.as_dict()) + + def prepared_days(self, allowance: float = 0) -> int: + # Independent of how long previous cases took; choose before creating data. + setup = (self.args.setup_budget + allowance + 2 * self.args.max_clock_skew + + 2 * self.args.poll_interval) + return max(1, math.ceil(setup / self.args.debug_day_seconds)) + + def object_window(self, case_id: str, bucket: str, key: str, days: int) -> tuple[DueWindow, dict[str, Any]]: + assert self.clock is not None + started = time.time() + head = self.client.head_object(Bucket=bucket, Key=key) + self.clock.validate_response(head, started, time.time()) + due = expiry_window(head["LastModified"], days, self.args.debug_day_seconds, self.args.process_seconds) + self.save_json(case_id, f"timing-{safe_name(bucket)}-{safe_name(key)}.json", { + "bucket": bucket, "key": key, "days": days, + "lastModified": head["LastModified"], "due": due.as_dict(), + }) + return due, head + + def assert_body(self, bucket: str, key: str, expected: bytes, **kwargs: Any) -> None: + assert self.clock is not None + started = time.time() + response = self.client.get_object(Bucket=bucket, Key=key, **kwargs) + body = response["Body"].read() + self.clock.validate_response(response, started, time.time()) + if body != expected: + raise AssertionError(f"object content changed: {bucket}/{key}") + + def object_absent(self, bucket: str, key: str) -> bool: + try: + self.client.head_object(Bucket=bucket, Key=key) + except ClientError as exc: + if self.client_error_code(exc) in {"NoSuchKey", "404", "NotFound"}: + return True + raise + return False + + def confirm_configuration( + self, case_id: str, bucket: str, expected: dict[str, Any] | None + ) -> None: + def matches() -> tuple[bool, Any]: + views = {} + for endpoint, client in self.node_clients.items(): + try: + response = client.get_bucket_lifecycle_configuration(Bucket=bucket) + views[endpoint] = response.get("Rules", []) + except ClientError as exc: + if self.client_error_code(exc) not in {"NoSuchLifecycleConfiguration", "NoSuchLifecycle"}: + raise + views[endpoint] = None + expected_rules = None if expected is None else expected["Rules"] + return all(view == expected_rules for view in views.values()), views + + final = self.wait_until(case_id, f"lifecycle configuration for {bucket} on every source node", matches, + timeout=self.args.setup_budget) + phase = "absent" if expected is None else "present" + self.save_json(case_id, f"configuration-node-views-{bucket}-{phase}.json", + {"bucket": bucket, "expected": expected, "views": final}) + + def observe_protected_object( + self, case_id: str, bucket: str, key: str, body: bytes, + due: DueWindow, control_bucket: str, control_key: str, + ) -> list[str]: + assert self.clock is not None + progress = CycleProgress() + observation_started: float | None = None + + def preserved() -> tuple[bool, Any]: + nonlocal observation_started + self.assert_body(bucket, key, body) + due_passed = self.clock.passed(due) + if due_passed and observation_started is None: + observation_started = time.monotonic() + cycles = 0 + control_expired = False + if observation_started is not None: + cluster = self.rc_scanner_status(case_id, "negative-scanner-latest.json") + cycles = progress.observe(cluster) + control_expired = self.object_absent(control_bucket, control_key) + observed_for = 0 if observation_started is None else time.monotonic() - observation_started + detail = {"pastDue": due_passed, "newSuccessfulCycles": cycles, + "positiveControlExpired": control_expired, "observedSeconds": observed_for} + complete = (due_passed and cycles >= 2 and control_expired + and observed_for >= self.args.negative_window) + if complete: + self.assert_body(bucket, key, body) + return complete, detail + + final = self.wait_until(case_id, "protected object with active expiry control", preserved, + timeout=self.clock.timeout_after(due, self.args.timeout)) + return [self.save_json(case_id, "negative-observation-final.json", final)] + + def wait_until( + self, + case_id: str, + description: str, + predicate: Callable[[], tuple[bool, Any]], + timeout: float | None = None, + ) -> Any: + budget = self.args.timeout if timeout is None else timeout + deadline = time.monotonic() + budget + attempts = 0 + last_detail: Any = None + while time.monotonic() < deadline: + attempts += 1 + with time_budget(deadline - time.monotonic(), description): + complete, last_detail = predicate() + self.events.record( + "poll.observation", case=case_id, description=description, + attempts=attempts, complete=complete, detail=last_detail, + ) + if time.monotonic() >= deadline: + break + if complete: + self.events.record( + "poll.complete", + case=case_id, + description=description, + attempts=attempts, + detail=last_detail, + ) + return last_detail + time.sleep(min(self.args.poll_interval, max(0, deadline - time.monotonic()))) + self.events.record( + "poll.timeout", + case=case_id, + description=description, + attempts=attempts, + detail=last_detail, + ) + self.save_json( + case_id, + f"poll-timeout-{safe_name(description)}.json", + { + "description": description, + "attempts": attempts, + "lastDetail": last_detail, + }, + ) + raise AssertionError( + f"timed out waiting for {description}; last observation: {last_detail}" + ) + + def cleanup_bucket(self, bucket: str) -> None: + self.events.record("bucket.cleanup.start", bucket=bucket) + try: + self.client.delete_bucket_lifecycle(Bucket=bucket) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") not in { + "NoSuchBucket", + "NoSuchLifecycleConfiguration", + }: + raise + + try: + uploads = self.client.list_multipart_uploads(Bucket=bucket).get( + "Uploads", [] + ) + for upload in uploads: + self.client.abort_multipart_upload( + Bucket=bucket, + Key=upload["Key"], + UploadId=upload["UploadId"], + ) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") != "NoSuchBucket": + raise + + deadline = time.monotonic() + max(30.0, min(self.args.timeout, 60.0)) + attempts = 0 + while True: + attempts += 1 + version_items: list[dict[str, str]] = [] + object_items: list[dict[str, str]] = [] + try: + version_response = self.client.list_object_versions(Bucket=bucket) + version_items = [ + {"Key": item["Key"], "VersionId": item["VersionId"]} + for group in ("Versions", "DeleteMarkers") + for item in version_response.get(group, []) + ] + if version_items: + self.client.delete_objects( + Bucket=bucket, + Delete={"Objects": version_items, "Quiet": True}, + ) + + object_response = self.client.list_objects_v2(Bucket=bucket) + object_items = [ + {"Key": item["Key"]} + for item in object_response.get("Contents", []) + ] + if object_items: + self.client.delete_objects( + Bucket=bucket, + Delete={"Objects": object_items, "Quiet": True}, + ) + self.client.delete_bucket(Bucket=bucket) + break + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + if code == "NoSuchBucket": + break + if code != "BucketNotEmpty" or time.monotonic() >= deadline: + raise + self.events.record( + "bucket.cleanup.retry", + bucket=bucket, + attempt=attempts, + errorCode=code, + visibleVersions=len(version_items), + visibleObjects=len(object_items), + ) + time.sleep(self.args.poll_interval) + with self.state_lock: + self.created_buckets.discard(bucket) + self.events.record( + "bucket.cleanup.complete", + bucket=bucket, + attempts=attempts, + ) + + def case_scanner_status_readiness( + self, case_id: str, bucket: str + ) -> list[str]: + del bucket + progress = CycleProgress() + + def scanner_ready() -> tuple[bool, Any]: + cluster = self.rc_scanner_status( + case_id, "scanner-readiness-latest.json" + ) + details: list[dict[str, Any]] = [] + configured = True + leaders: list[dict[str, Any]] = [] + for node in cluster["nodes"]: + data = node["status"].get("data", {}) + metrics = data.get("metrics", {}) + runtime = data.get("runtime_config", {}) + cycle = runtime.get("cycle_interval_seconds", {}) + start_delay = runtime.get("start_delay_seconds", {}) + schedule = data.get("cycle_schedule", {}) + node_detail = { + "endpoint": node["endpoint"], + "health": data.get("health"), + "enabled": data.get("enabled"), + "freshness": data.get("freshness", {}).get("state"), + "leaderLock": metrics.get("leader_lock_state"), + "leaderHeldHere": metrics.get( + "leader_lock_held_by_this_process" + ), + "lastCycleResult": metrics.get("last_cycle_result"), + "lastCyclePartialReason": metrics.get( + "last_cycle_partial_reason" + ), + "cycleInterval": cycle, + "startDelay": start_delay, + "executionRole": schedule.get("execution_role"), + "effectiveIntervalSeconds": schedule.get( + "effective_interval_seconds" + ), + } + details.append(node_detail) + configured = configured and ( + data.get("enabled") is True + and cycle.get("value") == self.args.scanner_cycle_seconds + and cycle.get("source") == "env" + and start_delay.get("value") == 0 + and start_delay.get("source") == "env" + ) + if metrics.get("leader_lock_held_by_this_process") is True: + leaders.append(node_detail) + healthy_leader = ( + len(leaders) == 1 + and leaders[0]["health"] == "healthy" + and leaders[0]["freshness"] == "fresh" + and leaders[0]["leaderLock"] == "acquired" + and leaders[0]["executionRole"] == "leader" + and leaders[0]["lastCycleResult"] == "success" + ) + cycles = progress.observe(cluster) + detail = {"leaderCount": len(leaders), "nodes": details, "newSuccessfulCycles": cycles} + return configured and healthy_leader and cycles >= 1, detail + + final = self.wait_until( + case_id, + "a successful scanner cycle with the configured runtime", + scanner_ready, + ) + return [self.save_json(case_id, "scanner-readiness-final.json", final)] + + def case_configuration_roundtrip(self, case_id: str, bucket: str) -> list[str]: + configuration = { + "Rules": [ + { + "ID": "roundtrip-expire", + "Status": "Enabled", + "Filter": {"Prefix": "expire/"}, + "Expiration": {"Days": 1}, + }, + { + "ID": "roundtrip-disabled", + "Status": "Disabled", + "Filter": {"Prefix": "disabled/"}, + "Expiration": {"Days": 5}, + }, + ] + } + evidence = self.put_lifecycle(case_id, bucket, configuration) + response = self.get_lifecycle(bucket) + evidence.append(self.save_json(case_id, "lifecycle-get-response.json", response)) + observed = { + (rule.get("ID"), rule.get("Status"), rule.get("Expiration", {}).get("Days")) + for rule in response.get("Rules", []) + } + expected = { + ("roundtrip-expire", "Enabled", 1), + ("roundtrip-disabled", "Disabled", 5), + } + if not expected.issubset(observed): + raise AssertionError( + f"lifecycle API round-trip mismatch: expected {expected}, got {observed}" + ) + rc_output = self.rc_lifecycle_list(case_id, bucket) + for rule_id in ("roundtrip-expire", "roundtrip-disabled"): + if rule_id not in rc_output: + raise AssertionError(f"rc output does not contain rule {rule_id!r}") + evidence.append( + str( + self.evidence_path(case_id, "rc-lifecycle-list.json").relative_to( + self.evidence_dir + ) + ) + ) + self.client.delete_bucket_lifecycle(Bucket=bucket) + try: + self.get_lifecycle(bucket) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + if code not in {"NoSuchLifecycleConfiguration", "NoSuchLifecycle"}: + raise + else: + raise AssertionError("deleted lifecycle configuration is still readable") + return evidence + + def case_invalid_configuration(self, case_id: str, bucket: str) -> list[str]: + fixtures = { + "negative-days": """ + + invalid-daysEnabled + -1 + +""", + "invalid-status": """ + + invalid-statusSometimes + 1 + +""", + } + evidence: list[str] = [] + observations: dict[str, Any] = {} + for name, lifecycle_xml in fixtures.items(): + status, headers, body = self.raw_s3_request( + "PUT", + bucket, + query="lifecycle", + body=lifecycle_xml.encode("utf-8"), + content_type="application/xml", + allow_error=True, + ) + observations[name] = { + "httpStatus": status, + "headers": headers, + "body": body.decode("utf-8", errors="replace"), + } + if status < 400 or status >= 500: + raise AssertionError( + f"invalid lifecycle {name} returned HTTP {status}, expected a 4xx rejection" + ) + try: + self.get_lifecycle(bucket) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code not in {"NoSuchLifecycleConfiguration", "NoSuchLifecycle"}: + raise + observations["storedConfiguration"] = {"error": code} + else: + raise AssertionError("an invalid lifecycle configuration was partially persisted") + evidence.append(self.save_json(case_id, "invalid-configurations.json", observations)) + return evidence + + def case_current_expiration(self, case_id: str, bucket: str) -> list[str]: + assert self.clock is not None + days = self.prepared_days() + expiring = { + "expire/one.txt": b"current-expiration-one", + "expire/two.txt": b"current-expiration-two", + } + control_key = "keep/control.txt" + control_body = b"control-object-must-survive" + due_by_key: dict[str, DueWindow] = {} + for key, body in expiring.items(): + self.client.put_object(Bucket=bucket, Key=key, Body=body) + due_by_key[key], _ = self.object_window(case_id, bucket, key, days) + self.client.put_object(Bucket=bucket, Key=control_key, Body=control_body) + configuration = { + "Rules": [ + { + "ID": "expire-current", + "Status": "Enabled", + "Filter": {"Prefix": "expire/"}, + "Expiration": {"Days": days}, + } + ] + } + evidence = self.put_lifecycle(case_id, bucket, configuration) + self.confirm_configuration(case_id, bucket, configuration) + due = DueWindow(min(item.earliest for item in due_by_key.values()), + max(item.latest for item in due_by_key.values())) + self.clock.require_before(due, margin=self.args.request_timeout) + header_due, head = self.object_window(case_id, bucket, "expire/one.txt", days) + evidence.append(self.save_json(case_id, "head-before-expiry.json", head)) + self.clock.require_before(due, margin=self.args.poll_interval) + expiration = str(head.get("Expiration", "")) + date_match = re.search(r'expiry-date="([^"]+)"', expiration) + id_match = re.search(r'rule-id="([^"]+)"', expiration) + if not date_match or not id_match or unquote(id_match.group(1)) != "expire-current": + raise AssertionError(f"missing or incorrect expiration header: {expiration}") + expires = parse_expiry_date(date_match.group(1)) + if not header_due.earliest <= expires <= header_due.latest: + raise AssertionError("expiration header disagrees with independent Days calculation") + + def expired() -> tuple[bool, Any]: + response = self.client.list_objects_v2(Bucket=bucket, Prefix="expire/") + keys = [item["Key"] for item in response.get("Contents", [])] + for key, object_due in due_by_key.items(): + if key not in keys and self.clock.now().latest < object_due.earliest: + raise AssertionError(f"object expired before its Days deadline: {key}") + return not keys, {"remaining": keys} + + final = self.wait_until(case_id, "current objects to expire", expired, + timeout=self.clock.timeout_after(due, self.args.timeout)) + evidence.append(self.save_json(case_id, "expiration-final.json", final)) + body = self.client.get_object(Bucket=bucket, Key=control_key)["Body"].read() + if body != control_body: + raise AssertionError("control object changed while expiring filtered objects") + return evidence + + def case_tag_filter_expiration(self, case_id: str, bucket: str) -> list[str]: + matching_key = "tag-filter/matching.txt" + control_key = "tag-filter/control.txt" + matching_body = b"tag-filter-matching" + control_body = b"tag-filter-control" + self.client.put_object( + Bucket=bucket, + Key=matching_key, + Body=matching_body, + Tagging="retention=short", + ) + self.client.put_object( + Bucket=bucket, + Key=control_key, + Body=control_body, + Tagging="retention=long", + ) + configuration = { + "Rules": [ + { + "ID": "expire-by-tag", + "Status": "Enabled", + "Filter": {"Tag": {"Key": "retention", "Value": "short"}}, + "Expiration": {"Days": 1}, + } + ] + } + evidence = self.put_lifecycle(case_id, bucket, configuration) + + def expired() -> tuple[bool, Any]: + response = self.client.list_objects_v2(Bucket=bucket, Prefix="tag-filter/") + keys = [item["Key"] for item in response.get("Contents", [])] + return matching_key not in keys and control_key in keys, {"keys": keys} + + final = self.wait_until(case_id, "tag-filtered object to expire", expired) + evidence.append(self.save_json(case_id, "tag-filter-final.json", final)) + body = self.client.get_object(Bucket=bucket, Key=control_key)["Body"].read() + if body != control_body: + raise AssertionError("tag-filter control object changed") + return evidence + + def case_disabled_rule(self, case_id: str, bucket: str) -> list[str]: + assert self.clock is not None + days = self.prepared_days() + key = "disabled/object.txt" + body = b"disabled-rule-must-not-run" + self.client.put_object(Bucket=bucket, Key=key, Body=body) + due, _ = self.object_window(case_id, bucket, key, days) + control_key = "enabled/control.txt" + self.client.put_object(Bucket=bucket, Key=control_key, Body=b"enabled-control") + control_due, _ = self.object_window(case_id, bucket, control_key, days) + configuration = { + "Rules": [ + { + "ID": "disabled-expiration", + "Status": "Disabled", + "Filter": {"Prefix": "disabled/"}, + "Expiration": {"Days": days}, + }, + {"ID": "enabled-positive-control", "Status": "Enabled", + "Filter": {"Prefix": "enabled/"}, "Expiration": {"Days": days}}, + ] + } + evidence = self.put_lifecycle(case_id, bucket, configuration) + self.confirm_configuration(case_id, bucket, configuration) + self.clock.require_before(due, margin=self.args.request_timeout) + self.assert_body(bucket, key, body) + self.clock.require_before(due, margin=self.args.poll_interval) + window = DueWindow(due.earliest, max(due.latest, control_due.latest)) + evidence.extend(self.observe_protected_object(case_id, bucket, key, body, window, bucket, control_key)) + return evidence + + def case_rule_removal(self, case_id: str, bucket: str) -> list[str]: + assert self.clock is not None + days = self.prepared_days() + key = "removed-rule/object.txt" + body = b"removed-rule-must-not-run" + self.client.put_object(Bucket=bucket, Key=key, Body=body) + due, _ = self.object_window(case_id, bucket, key, days) + control_bucket = self.bucket_name(f"{case_id}.control") + self.create_bucket(control_bucket) + self.client.put_object(Bucket=control_bucket, Key=key, Body=b"enabled-control") + control_due, _ = self.object_window(case_id, control_bucket, key, days) + configuration = { + "Rules": [ + { + "ID": "remove-before-due", + "Status": "Enabled", + "Filter": {"Prefix": "removed-rule/"}, + "Expiration": {"Days": days}, + } + ] + } + evidence = self.put_lifecycle(case_id, bucket, configuration) + self.confirm_configuration(case_id, bucket, configuration) + self.client.put_bucket_lifecycle_configuration(Bucket=control_bucket, LifecycleConfiguration=configuration) + self.confirm_configuration(case_id, control_bucket, configuration) + self.clock.require_before(due, margin=self.args.poll_interval) + deleted = self.client.delete_bucket_lifecycle(Bucket=bucket) + evidence.append(self.save_json(case_id, "lifecycle-delete-response.json", deleted)) + self.confirm_configuration(case_id, bucket, None) + self.clock.require_before(due, margin=self.args.request_timeout) + self.assert_body(bucket, key, body) + self.clock.require_before(due, margin=self.args.poll_interval) + window = DueWindow(due.earliest, max(due.latest, control_due.latest)) + evidence.extend(self.observe_protected_object(case_id, bucket, key, body, window, control_bucket, key)) + return evidence + + def case_versioned_current_expiration(self, case_id: str, bucket: str) -> list[str]: + self.enable_versioning(bucket) + key = "versioned-current/object.txt" + put = self.client.put_object(Bucket=bucket, Key=key, Body=b"versioned-current") + configuration = { + "Rules": [ + { + "ID": "expire-versioned-current", + "Status": "Enabled", + "Filter": {"Prefix": "versioned-current/"}, + "Expiration": {"Days": 1}, + } + ] + } + evidence = self.put_lifecycle(case_id, bucket, configuration) + + def marker_created() -> tuple[bool, Any]: + response = self.client.list_object_versions(Bucket=bucket, Prefix=key) + markers = [ + item + for item in response.get("DeleteMarkers", []) + if item["Key"] == key and item["IsLatest"] + ] + versions = [ + item for item in response.get("Versions", []) if item["Key"] == key + ] + return bool(markers) and any( + item["VersionId"] == put.get("VersionId") for item in versions + ), {"deleteMarkers": markers, "versions": versions} + + final = self.wait_until( + case_id, + "versioned current expiration delete marker", + marker_created, + timeout=max(self.args.timeout, self.args.debug_day_seconds * 12), + ) + try: + self.client.get_object(Bucket=bucket, Key=key) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") not in {"NoSuchKey", "404", "NotFound"}: + raise + else: + raise AssertionError("versioned current object remains visible after expiration") + original = self.client.get_object( + Bucket=bucket, Key=key, VersionId=put["VersionId"] + )["Body"].read() + if original != b"versioned-current": + raise AssertionError("expiration changed the retained original version") + evidence.append(self.save_json(case_id, "versioned-current-final.json", final)) + return evidence + + def enable_versioning(self, bucket: str) -> None: + self.client.put_bucket_versioning( + Bucket=bucket, + VersioningConfiguration={"Status": "Enabled"}, + ) + observed = self.client.get_bucket_versioning(Bucket=bucket).get("Status") + if observed != "Enabled": + raise AssertionError(f"versioning did not become enabled: {observed!r}") + + def case_noncurrent_expiration(self, case_id: str, bucket: str) -> list[str]: + self.enable_versioning(bucket) + key = "versioned/object.txt" + v1 = self.client.put_object(Bucket=bucket, Key=key, Body=b"version-one")[ + "VersionId" + ] + v2 = self.client.put_object(Bucket=bucket, Key=key, Body=b"version-two")[ + "VersionId" + ] + configuration = { + "Rules": [ + { + "ID": "expire-noncurrent", + "Status": "Enabled", + "Filter": {"Prefix": "versioned/"}, + "NoncurrentVersionExpiration": {"NoncurrentDays": 1}, + } + ] + } + evidence = self.put_lifecycle(case_id, bucket, configuration) + evidence.append( + self.save_json( + case_id, + "versions-before.json", + self.client.list_object_versions(Bucket=bucket, Prefix=key), + ) + ) + + def old_version_removed() -> tuple[bool, Any]: + response = self.client.list_object_versions(Bucket=bucket, Prefix=key) + versions = [ + { + "versionId": item["VersionId"], + "isLatest": item["IsLatest"], + } + for item in response.get("Versions", []) + if item["Key"] == key + ] + ids = {item["versionId"] for item in versions} + return v1 not in ids and v2 in ids, {"versions": versions} + + final = self.wait_until( + case_id, "noncurrent version to expire", old_version_removed + ) + evidence.append(self.save_json(case_id, "versions-final.json", final)) + body = self.client.get_object(Bucket=bucket, Key=key)["Body"].read() + if body != b"version-two": + raise AssertionError("latest version changed during noncurrent expiration") + return evidence + + def case_newer_noncurrent_retention( + self, case_id: str, bucket: str + ) -> list[str]: + self.enable_versioning(bucket) + key = "newer-noncurrent/object.txt" + version_ids = [ + self.client.put_object( + Bucket=bucket, + Key=key, + Body=f"version-{number}".encode("ascii"), + )["VersionId"] + for number in range(1, 5) + ] + configuration = { + "Rules": [ + { + "ID": "retain-two-newer-noncurrent", + "Status": "Enabled", + "Filter": {"Prefix": "newer-noncurrent/"}, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 1, + "NewerNoncurrentVersions": 2, + }, + } + ] + } + evidence = self.put_lifecycle(case_id, bucket, configuration) + evidence.append( + self.save_json( + case_id, + "newer-noncurrent-before.json", + self.client.list_object_versions(Bucket=bucket, Prefix=key), + ) + ) + expected = set(version_ids[-3:]) + + def retained() -> tuple[bool, Any]: + response = self.client.list_object_versions(Bucket=bucket, Prefix=key) + observed = { + item["VersionId"] + for item in response.get("Versions", []) + if item["Key"] == key + } + return observed == expected, { + "expected": sorted(expected), + "observed": sorted(observed), + "expired": version_ids[0], + } + + final = self.wait_until( + case_id, + "NewerNoncurrentVersions retention", + retained, + timeout=max(self.args.timeout, self.args.debug_day_seconds * 12), + ) + latest = self.client.get_object(Bucket=bucket, Key=key)["Body"].read() + if latest != b"version-4": + raise AssertionError("latest object changed while retaining noncurrent versions") + evidence.append(self.save_json(case_id, "newer-noncurrent-final.json", final)) + return evidence + + def case_delete_all_versions(self, case_id: str, bucket: str) -> list[str]: + self.enable_versioning(bucket) + key = "all-versions/object.txt" + version_ids = [ + self.client.put_object( + Bucket=bucket, + Key=key, + Body=f"version-{number}".encode("ascii"), + )["VersionId"] + for number in range(1, 4) + ] + lifecycle_xml = """ + + + delete-all-versions + Enabled + all-versions/ + + 1 + true + + + +""" + evidence = [ + self.save_text(case_id, "lifecycle-request.xml", lifecycle_xml) + ] + status, headers, _ = self.raw_s3_request( + "PUT", + bucket, + query="lifecycle", + body=lifecycle_xml.encode("utf-8"), + content_type="application/xml", + ) + evidence.append( + self.save_json( + case_id, + "lifecycle-put-response.json", + {"httpStatus": status, "headers": headers}, + ) + ) + _, _, stored_xml = self.raw_s3_request( + "GET", bucket, query="lifecycle" + ) + stored_text = stored_xml.decode("utf-8") + evidence.append( + self.save_text(case_id, "lifecycle-get-response.xml", stored_text) + ) + if "true" not in stored_text: + raise AssertionError( + "stored lifecycle configuration lost ExpiredObjectAllVersions" + ) + evidence.append( + self.save_json( + case_id, + "versions-before.json", + self.client.list_object_versions(Bucket=bucket, Prefix=key), + ) + ) + + def all_versions_removed() -> tuple[bool, Any]: + response = self.client.list_object_versions(Bucket=bucket, Prefix=key) + versions = [ + item["VersionId"] + for item in response.get("Versions", []) + if item["Key"] == key + ] + markers = [ + item["VersionId"] + for item in response.get("DeleteMarkers", []) + if item["Key"] == key + ] + return not versions and not markers, { + "expectedVersionIds": version_ids, + "versions": versions, + "deleteMarkers": markers, + } + + final = self.wait_until( + case_id, + "all versions to expire", + all_versions_removed, + timeout=max(self.args.timeout, self.args.debug_day_seconds * 12), + ) + evidence.append(self.save_json(case_id, "versions-final.json", final)) + return evidence + + def case_delete_marker_expiration(self, case_id: str, bucket: str) -> list[str]: + self.enable_versioning(bucket) + key = "markers/object.txt" + configuration = { + "Rules": [ + { + "ID": "expire-delete-marker", + "Status": "Enabled", + "Filter": {"Prefix": "markers/"}, + "Expiration": {"ExpiredObjectDeleteMarker": True}, + "NoncurrentVersionExpiration": {"NoncurrentDays": 1}, + } + ] + } + evidence = self.put_lifecycle(case_id, bucket, configuration) + self.client.put_object(Bucket=bucket, Key=key, Body=b"marker-source") + self.client.delete_object(Bucket=bucket, Key=key) + evidence.append( + self.save_json( + case_id, + "markers-before.json", + self.client.list_object_versions(Bucket=bucket, Prefix=key), + ) + ) + + def marker_chain_removed() -> tuple[bool, Any]: + response = self.client.list_object_versions(Bucket=bucket, Prefix=key) + versions = [ + item["VersionId"] + for item in response.get("Versions", []) + if item["Key"] == key + ] + markers = [ + item["VersionId"] + for item in response.get("DeleteMarkers", []) + if item["Key"] == key + ] + return not versions and not markers, { + "versions": versions, + "deleteMarkers": markers, + } + + final = self.wait_until( + case_id, + "noncurrent object and expired delete marker to be removed", + marker_chain_removed, + timeout=max(self.args.timeout, self.args.debug_day_seconds * 12), + ) + evidence.append(self.save_json(case_id, "markers-final.json", final)) + return evidence + + def case_multipart_expiration(self, case_id: str, bucket: str) -> list[str]: + key = "multipart/incomplete.bin" + configuration = { + "Rules": [ + { + "ID": "abort-incomplete-multipart", + "Status": "Enabled", + "Filter": {"Prefix": "multipart/"}, + "AbortIncompleteMultipartUpload": {"DaysAfterInitiation": 1}, + } + ] + } + evidence = self.put_lifecycle(case_id, bucket, configuration) + created = self.client.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = created["UploadId"] + self.client.upload_part( + Bucket=bucket, + Key=key, + UploadId=upload_id, + PartNumber=1, + Body=b"x" * (5 * 1024 * 1024), + ) + evidence.append( + self.save_json( + case_id, + "multipart-before.json", + self.client.list_multipart_uploads(Bucket=bucket, Prefix="multipart/"), + ) + ) + + def upload_removed() -> tuple[bool, Any]: + response = self.client.list_multipart_uploads( + Bucket=bucket, Prefix="multipart/" + ) + uploads = [ + {"key": item["Key"], "uploadId": item["UploadId"]} + for item in response.get("Uploads", []) + ] + return not any(item["uploadId"] == upload_id for item in uploads), { + "uploads": uploads + } + + final = self.wait_until( + case_id, + "incomplete multipart upload to be aborted", + upload_removed, + timeout=max(self.args.timeout, self.args.debug_day_seconds * 12), + ) + evidence.append(self.save_json(case_id, "multipart-final.json", final)) + return evidence + + def case_transition_and_restore(self, case_id: str, bucket: str) -> list[str]: + if self.tier_client is None: + raise RuntimeError( + "transition-and-restore requires tier endpoint and credentials" + ) + digest = hashlib.sha256( + f"{self.args.run_id}:{case_id}".encode("utf-8") + ).hexdigest()[:12] + tier_name = f"RV{digest.upper()}" + tier_bucket = f"rv-tier-{digest}" + key = "transition/archive.bin" + body = (b"rustfs-transition-restore-" * 4096) + secrets.token_bytes(32) + tier_created = False + tier_added = False + evidence: list[str] = [] + phase = "remote-tier-bucket-create" + primary_error: Exception | None = None + primary_phase = "" + cleanup_error: Exception | None = None + cleanup_phase = "" + try: + self.tier_client.create_bucket(Bucket=tier_bucket) + tier_created = True + phase = "tier-add" + add = self.rc_command( + case_id, + "tier-add", + [ + "ilm", + "tier", + "add", + "rustfs", + tier_name, + self.rc_alias, + "--endpoint", + self.args.tier_endpoint, + "--access-key", + self.args.tier_access_key, + "--secret-key", + self.args.tier_secret_key, + "--bucket", + tier_bucket, + ], + secrets_to_redact=(self.args.tier_secret_key,), + ) + tier_added = add.returncode == 0 + evidence.append( + str( + self.evidence_path(case_id, "tier-add.json").relative_to( + self.evidence_dir + ) + ) + ) + phase = "source-put" + put = self.client.put_object(Bucket=bucket, Key=key, Body=body) + configuration = { + "Rules": [ + { + "ID": "transition-to-real-tier", + "Status": "Enabled", + "Filter": {"Prefix": "transition/"}, + "Transitions": [{"Days": 1, "StorageClass": tier_name}], + } + ] + } + phase = "lifecycle-put" + evidence.extend(self.put_lifecycle(case_id, bucket, configuration)) + evidence.append( + self.save_json( + case_id, + "transition-source-put.json", + { + "versionId": put.get("VersionId"), + "bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest(), + }, + ) + ) + + def transitioned() -> tuple[bool, Any]: + head = self.client.head_object(Bucket=bucket, Key=key) + observed = head.get("StorageClass") or head.get("storage-class") + return str(observed or "").upper() == tier_name, { + "storageClass": observed, + "restore": head.get("Restore"), + "contentLength": head.get("ContentLength"), + } + + phase = "transition-wait" + final = self.wait_until( + case_id, + "object transition to remote tier", + transitioned, + timeout=max(self.args.timeout, self.args.debug_day_seconds * 18), + ) + evidence.append(self.save_json(case_id, "transition-final.json", final)) + phase = "transparent-read" + transparent = self.client.get_object(Bucket=bucket, Key=key)["Body"].read() + if transparent != body: + raise AssertionError("transparent read changed transitioned object bytes") + evidence.append( + self.save_json( + case_id, + "transition-transparent-read.json", + { + "bytes": len(transparent), + "sha256": hashlib.sha256(transparent).hexdigest(), + }, + ) + ) + phase = "remote-object-list" + remote_objects = self.tier_client.list_objects_v2(Bucket=tier_bucket) + if not remote_objects.get("Contents"): + raise AssertionError("remote tier bucket contains no transitioned object") + evidence.append( + self.save_json(case_id, "tier-remote-objects.json", remote_objects) + ) + phase = "restore-request" + self.rc_command( + case_id, + "tier-restore", + [ + "ilm", + "restore", + f"{self.rc_alias}/{bucket}/{key}", + "--days", + "1", + "--force", + ], + ) + evidence.append( + str( + self.evidence_path(case_id, "tier-restore.json").relative_to( + self.evidence_dir + ) + ) + ) + + def restored() -> tuple[bool, Any]: + head = self.client.head_object(Bucket=bucket, Key=key) + restore = str(head.get("Restore", "")) + return bool(restore) and 'ongoing-request="false"' in restore, { + "restore": restore, + "storageClass": head.get("StorageClass"), + } + + phase = "restore-wait" + restore_final = self.wait_until( + case_id, + "transitioned object restore completion", + restored, + timeout=max(self.args.timeout, self.args.debug_day_seconds * 18), + ) + evidence.append( + self.save_json(case_id, "restore-final.json", restore_final) + ) + phase = "restored-read" + restored_body = self.client.get_object(Bucket=bucket, Key=key)["Body"].read() + if restored_body != body: + raise AssertionError("restored object bytes differ from original") + except Exception as exc: + primary_error = exc + primary_phase = phase + evidence.append( + self.save_phase_error( + case_id, + "transition-primary-error.json", + primary_phase, + exc, + ) + ) + finally: + cleanup_phase = "lifecycle-delete" + try: + try: + self.client.delete_bucket_lifecycle(Bucket=bucket) + except ClientError: + pass + if tier_added: + # A transitioned object is an authoritative reference to its tier. + # Remove the source object and wait for its remote payload cleanup + # before asking RustFS to remove the tier configuration. --force + # bypasses lifecycle-config references, but deliberately does not + # bypass live object references. + cleanup_phase = "source-delete" + self.delete_object_eventually( + case_id, + bucket, + key, + evidence_name="transition-source-delete", + timeout=max( + self.args.timeout, + self.args.debug_day_seconds * 12, + ), + ) + evidence.append( + str( + self.evidence_path( + case_id, + "transition-source-delete.json", + ).relative_to(self.evidence_dir) + ) + ) + + def remote_tier_empty() -> tuple[bool, Any]: + response = self.tier_client.list_objects_v2(Bucket=tier_bucket) + remaining = [ + {"key": item["Key"], "size": item.get("Size")} + for item in response.get("Contents", []) + ] + return not remaining, {"remaining": remaining} + + cleanup_phase = "remote-object-cleanup-wait" + cleanup_final = self.wait_until( + case_id, + "transitioned object cleanup from remote tier", + remote_tier_empty, + timeout=max( + self.args.timeout, + self.args.debug_day_seconds * 12, + ), + ) + evidence.append( + self.save_json( + case_id, + "tier-object-cleanup-final.json", + cleanup_final, + ) + ) + cleanup_phase = "tier-remove" + self.rc_tier_remove_eventually( + case_id, + tier_name, + timeout=max( + self.args.timeout, + self.args.debug_day_seconds * 12, + ), + ) + evidence.append( + str( + self.evidence_path( + case_id, "tier-remove.json" + ).relative_to(self.evidence_dir) + ) + ) + if tier_created: + cleanup_phase = "remote-tier-bucket-delete" + try: + versions = self.tier_client.list_object_versions( + Bucket=tier_bucket + ) + version_items = [ + {"Key": item["Key"], "VersionId": item["VersionId"]} + for group in ("Versions", "DeleteMarkers") + for item in versions.get(group, []) + ] + if version_items: + self.tier_client.delete_objects( + Bucket=tier_bucket, + Delete={"Objects": version_items, "Quiet": True}, + ) + objects = self.tier_client.list_objects_v2( + Bucket=tier_bucket + ).get("Contents", []) + if objects: + self.tier_client.delete_objects( + Bucket=tier_bucket, + Delete={ + "Objects": [ + {"Key": item["Key"]} for item in objects + ], + "Quiet": True, + }, + ) + self.tier_client.delete_bucket(Bucket=tier_bucket) + except ClientError: + pass + except Exception as exc: + cleanup_error = exc + evidence.append( + self.save_phase_error( + case_id, + "transition-cleanup-error.json", + cleanup_phase, + exc, + ) + ) + + if primary_error is not None: + if cleanup_error is not None: + raise CaseExecutionAndCleanupError( + primary_phase, + primary_error, + cleanup_phase, + cleanup_error, + ) from cleanup_error + raise primary_error + if cleanup_error is not None: + raise cleanup_error + return evidence + + def case_restart_persistence(self, case_id: str, bucket: str) -> list[str]: + assert self.clock is not None + if not self.args.restart_hook: + raise RuntimeError("restart-persistence requires --restart-hook") + days = self.prepared_days(self.args.restart_timeout + self.args.setup_budget) + key = "restart/expire.txt" + body = b"survive-restart-then-expire" + configuration = {"Rules": [{ + "ID": "restart-expiry", "Status": "Enabled", + "Filter": {"Prefix": "restart/"}, "Expiration": {"Days": days}, + }]} + evidence = self.put_lifecycle(case_id, bucket, configuration) + self.confirm_configuration(case_id, bucket, configuration) + self.client.put_object(Bucket=bucket, Key=key, Body=body) + due, before = self.object_window(case_id, bucket, key, days) + self.assert_body(bucket, key, body) + self.clock.require_before(due, margin=self.args.restart_timeout + self.args.setup_budget) + evidence.append(self.save_json(case_id, "object-before-restart.json", before)) + command = json.loads(self.args.restart_hook) + if not isinstance(command, list) or not command or not all( + isinstance(item, str) and item for item in command + ): + raise RuntimeError("--restart-hook must be a JSON array of command tokens") + self.events.record("cluster.restart.start", case=case_id) + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, start_new_session=True) + output = "" + try: + with time_budget(self.args.restart_timeout, "cluster restart", TimeoutError): + output, _ = process.communicate(timeout=self.args.restart_timeout) + if process.returncode != 0: + raise RuntimeError(f"cluster restart hook failed with exit {process.returncode}") + except BaseException as primary_error: + # Stop the local driver/SSH process group; never leave it running into + # the next case. Remote service state remains unknown and is preserved. + self.blocked_reason = "restart hook failed; service state requires inspection" + if isinstance(primary_error, subprocess.TimeoutExpired) and primary_error.output: + output = primary_error.output + if isinstance(output, bytes): + output = output.decode("utf-8", errors="replace") + try: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + output, _ = process.communicate(timeout=5) + except Exception as cleanup_error: + evidence.append(self.save_json(case_id, "restart-hook-termination-error.json", { + "errorType": type(cleanup_error).__name__, + "message": self.redact(str(cleanup_error)), + })) + raise + finally: + evidence.append(self.save_text(case_id, "restart-hook.log", self.redact(output))) + self.events.record("cluster.restart.complete", case=case_id) + self.calibrate_clock(case_id) + self.confirm_configuration(case_id, bucket, configuration) + self.clock.require_before(due, margin=self.args.request_timeout) + self.assert_body(bucket, key, body) + self.clock.require_before(due, margin=self.args.request_timeout) + _, after = self.object_window(case_id, bucket, key, days) + if (after["LastModified"], after["ETag"]) != (before["LastModified"], before["ETag"]): + raise AssertionError("restart changed the object identity") + evidence.append(self.save_json(case_id, "object-after-restart-before-expiry.json", after)) + evidence.extend(self.case_scanner_status_readiness(case_id, bucket)) + + def object_expired() -> tuple[bool, Any]: + absent = self.object_absent(bucket, key) + if absent and self.clock.now().latest < due.earliest: + raise AssertionError("object expired before its Days deadline") + return absent, {"absent": absent, "due": due.as_dict()} + + final = self.wait_until(case_id, "object expiry after verified restart", object_expired, + timeout=self.clock.timeout_after(due, self.args.timeout)) + evidence.append(self.save_json(case_id, "restart-expiration-final.json", final)) + return evidence + + def classify_exception(self, exc: BaseException) -> str: + if isinstance(exc, CaseExecutionAndCleanupError): + states = { + self.classify_exception(exc.primary_error), + self.classify_exception(exc.cleanup_error), + } + return "infra_error" if "infra_error" in states else "fail" + if isinstance(exc, AssertionError): + return "fail" + if isinstance(exc, INFRA_EXCEPTIONS): + return "infra_error" + if isinstance(exc, ClientError): + code = exc.response.get("Error", {}).get("Code") + return "infra_error" if code in INFRA_ERROR_CODES else "fail" + return "infra_error" + + def run_case( + self, + case_id: str, + function: Callable[[str, str], list[str]], + *, + create_bucket: bool = True, + ) -> None: + bucket = self.bucket_name(case_id) + self.case_buckets = set() + started_at = utc_now() + started = time.monotonic() + state = "pass" + message = "all assertions passed" + evidence: list[str] = [] + bucket_label = f" bucket={bucket}" if create_bucket else "" + print(f"[RUN ] {case_id}{bucket_label}", flush=True) + self.events.record( + "case.start", + case=case_id, + bucket=bucket if create_bucket else None, + ) + try: + with time_budget(self.args.case_timeout, f"case {case_id}"): + self.rc_scanner_status(case_id, "scanner-before.json") + self.calibrate_clock(case_id) + if create_bucket: + self.create_bucket(bucket) + evidence.extend(function(case_id, bucket)) + except Exception as exc: # retain the failure before inspecting cleanup + state = self.classify_exception(exc) + message = self.redact(f"{type(exc).__name__}: {exc}") + self.events.record( + "case.error", + case=case_id, + state=state, + errorType=type(exc).__name__, + message=message, + ) + self.save_json( + case_id, + "case-error.json", + { + "state": state, + "errorType": type(exc).__name__, + "message": message, + }, + ) + finally: + try: + self.rc_scanner_status(case_id, "scanner-after.json") + except Exception as status_exc: + if state == "pass": + state = "infra_error" + message = ( + "scanner status collection failed: " + f"{type(status_exc).__name__}: {status_exc}" + ) + self.events.record( + "rc.scanner.status.error", + case=case_id, + message=self.redact(str(status_exc)), + ) + should_cleanup = state == "pass" or self.args.cleanup_on_failure + owned_buckets = self.case_buckets & self.created_buckets + if owned_buckets and should_cleanup: + try: + with time_budget(self.args.cleanup_timeout, f"cleanup {case_id}", TimeoutError): + for owned_bucket in sorted(owned_buckets): + self.cleanup_bucket(owned_bucket) + except Exception as cleanup_exc: + cleanup_state = self.classify_exception(cleanup_exc) + if state == "pass": + state = cleanup_state + elif cleanup_state == "infra_error": + state = "infra_error" + cleanup_message = self.redact( + f"{type(cleanup_exc).__name__}: {cleanup_exc}" + ) + message = f"cleanup failed after {message}: {cleanup_message}" + self.events.record( + "case.cleanup.error", + case=case_id, + state=cleanup_state, + message=cleanup_message, + ) + self.save_json( + case_id, + "case-cleanup-error.json", + { + "state": cleanup_state, + "errorType": type(cleanup_exc).__name__, + "message": cleanup_message, + }, + ) + elif owned_buckets: + message += f"; preserved failed buckets {', '.join(sorted(owned_buckets))}" + if self.case_buckets & self.created_buckets: + self.blocked_reason = f"{case_id} left preserved or uncleaned test buckets" + if state != "pass" and not self.args.cleanup_on_failure and self.blocked_reason is None: + self.blocked_reason = f"{case_id} failed; the run is preserved for inspection" + + case_evidence_dir = self.evidence_path(case_id, "evidence-index").parent + evidence = sorted( + str(path.relative_to(self.evidence_dir)) + for path in case_evidence_dir.iterdir() + if path.is_file() and path.name != "evidence-index" + ) + + duration = time.monotonic() - started + result = CaseResult( + id=case_id, + state=state, + message=message, + started_at=started_at, + finished_at=utc_now(), + duration_seconds=round(duration, 3), + evidence=evidence, + ) + with self.state_lock: + self.results.append(result) + self.events.record("case.complete", **asdict(result)) + label = "PASS" if state == "pass" else state.upper() + print(f"[{label:5}] {case_id}: {message}", flush=True) + + def cleanup_outcome(self) -> dict[str, Any]: + """Describe the logical bucket cleanup completed by the ILM harness.""" + + uncertain = sorted(getattr(self, "uncertain_buckets", set())) + if uncertain: + return { + "attempted": False, + "state": "infra_error", + "message": "bucket creation outcomes are unknown; isolation was not verified", + "uncertainBuckets": uncertain, + } + preserved = sorted(self.created_buckets) + if not preserved: + return { + "attempted": bool(self.results), + "state": "pass", + "message": "no ILM test buckets remain after logical cleanup", + } + if self.args.cleanup_on_failure: + return { + "attempted": True, + "state": "infra_error", + "message": "ILM cleanup was requested but test buckets remain", + "preservedBuckets": preserved, + } + return { + "attempted": False, + "state": "preserved", + "message": "failed-case ILM buckets were preserved for investigation", + "preservedBuckets": preserved, + } + + def write_results(self, started_at: str, started: float) -> str: + selected_cases = ilm_harness_scored_cases(self.args.case) + selection_mode = "filtered" if self.args.case is not None else "full" + case_order = {case_id: index for index, case_id in enumerate(CASE_IDS)} + self.results.sort(key=lambda item: case_order.get(item.id, len(case_order))) + states = {result.state for result in self.results} + if "fail" in states: + overall = "fail" + elif "infra_error" in states: + overall = "infra_error" + else: + overall = "pass" + finished_at = utc_now() + result_document = { + "apiVersion": "validation.rustfs.io/v1alpha1", + "kind": "TestRunResult", + "metadata": { + "runId": self.args.run_id, + "environmentId": self.args.environment_id, + }, + "status": { + "state": overall, + "startedAt": started_at, + "finishedAt": finished_at, + "cases": [ + { + "id": item.id, + "state": item.state, + "message": item.message, + "evidence": item.evidence, + } + for item in self.results + ], + "evidence": ["events.jsonl", "run-manifest.json", "junit.xml"], + }, + } + (self.evidence_dir / "result.json").write_text( + json.dumps(result_document, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + manifest = { + "runId": self.args.run_id, + "environmentId": self.args.environment_id, + "endpoint": self.args.endpoint, + "scannerEndpoints": self.args.scanner_endpoint, + "startedAt": started_at, + "finishedAt": finished_at, + "durationSeconds": round(time.monotonic() - started, 3), + "debugDaySeconds": self.args.debug_day_seconds, + "processSeconds": self.args.process_seconds, + "timingBudgets": { + "requestSeconds": self.args.request_timeout, + "caseSeconds": self.args.case_timeout, + "cleanupSeconds": self.args.cleanup_timeout, + "setupSeconds": self.args.setup_budget, + "restartSeconds": self.args.restart_timeout, + "negativeWindowSeconds": self.args.negative_window, + "maxClockSkewSeconds": self.args.max_clock_skew, + }, + "readinessOutcome": self.readiness, + "executionMode": "serial", + "scannerPollIntervalSeconds": self.args.poll_interval, + "boto3Version": boto3.__version__, + "botocoreVersion": botocore.__version__, + "rcBinaryPath": self.rc_bin, + "rcVersion": self.rc_version, + "rcBinarySha256": self.rc_binary_sha256, + "expectedRcBinarySha256": self.expected_rc_binary_sha256 or None, + "executionPlan": { + "selectionMode": selection_mode, + "module": "ilm", + "scoredCases": list(selected_cases), + "scoredCaseCount": len(selected_cases), + "fixtures": [], + }, + "fixtureOutcomes": [], + "cleanupOnFailure": self.args.cleanup_on_failure, + "cleanupOutcome": self.cleanup_outcome(), + "preservedBuckets": sorted(self.created_buckets), + } + (self.evidence_dir / "run-manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + self.write_junit(overall, time.monotonic() - started) + return overall + + def write_junit(self, overall: str, duration: float) -> None: + failures = sum(item.state == "fail" for item in self.results) + errors = sum(item.state == "infra_error" for item in self.results) + suite = ET.Element( + "testsuite", + { + "name": "rustfs-fixed-environment-ilm", + "tests": str(len(self.results)), + "failures": str(failures), + "errors": str(errors), + "skipped": "0", + "time": f"{duration:.3f}", + }, + ) + for item in self.results: + case = ET.SubElement( + suite, + "testcase", + { + "classname": "rustfs.ilm", + "name": item.id, + "time": f"{item.duration_seconds:.3f}", + }, + ) + if item.state == "fail": + ET.SubElement(case, "failure", {"message": item.message}).text = ( + item.message + ) + elif item.state == "infra_error": + ET.SubElement(case, "error", {"message": item.message}).text = ( + item.message + ) + ET.SubElement(case, "system-out").text = "\n".join(item.evidence) + tree = ET.ElementTree(suite) + ET.indent(tree, space=" ") + tree.write( + self.evidence_dir / "junit.xml", + encoding="utf-8", + xml_declaration=True, + ) + self.events.record("run.complete", state=overall) + + def record_blocked(self, case_id: str, reason: str, *, state: str = "infra_error") -> None: + message = f"blocked: {reason}" + evidence = self.save_json(case_id, "blocked.json", {"state": state, "message": message}) + now = utc_now() + self.results.append(CaseResult(case_id, state, message, now, now, 0, [evidence])) + self.events.record("case.blocked", case=case_id, state=state, message=message) + + def run(self) -> int: + started_at = utc_now() + started = time.monotonic() + selected_cases = ilm_harness_scored_cases(self.args.case) + self.events.record( + "run.start", + runId=self.args.run_id, + environmentId=self.args.environment_id, + endpoint=self.args.endpoint, + selectionMode="filtered" if self.args.case is not None else "full", + scoredCases=list(selected_cases), + fixtures=[], + ) + try: + with time_budget(self.args.timeout, "ILM preflight"): + self.wait_for_s3_connectivity() + self.configure_rc() + self.calibrate_clock("fixture.scanner-readiness") + self.case_scanner_status_readiness("fixture.scanner-readiness", "") + self.readiness = {"state": "pass", "message": "clock and new scanner cycle verified"} + except Exception as exc: + message = self.redact(f"precondition_error: {type(exc).__name__}: {exc}") + self.readiness = {"state": self.classify_exception(exc), "message": message} + for index, case_id in enumerate(selected_cases): + self.record_blocked(case_id, message, + state=self.classify_exception(exc) if index == 0 else "infra_error") + overall = self.write_results(started_at, started) + return 2 if overall == "infra_error" else 1 + + cases: dict[str, Callable[[str, str], list[str]]] = { + "scanner.status-readiness": self.case_scanner_status_readiness, + "ilm.configuration-roundtrip": self.case_configuration_roundtrip, + "ilm.invalid-configuration": self.case_invalid_configuration, + "ilm.current-expiration": self.case_current_expiration, + "ilm.tag-filter-expiration": self.case_tag_filter_expiration, + "ilm.disabled-rule": self.case_disabled_rule, + "ilm.rule-removal": self.case_rule_removal, + "ilm.versioned-current-expiration": self.case_versioned_current_expiration, + "ilm.noncurrent-expiration": self.case_noncurrent_expiration, + "ilm.newer-noncurrent-retention": self.case_newer_noncurrent_retention, + "ilm.delete-all-versions": self.case_delete_all_versions, + "ilm.delete-marker-expiration": self.case_delete_marker_expiration, + "ilm.multipart-expiration": self.case_multipart_expiration, + "ilm.transition-and-restore": self.case_transition_and_restore, + "ilm.restart-persistence": self.case_restart_persistence, + } + for case_id in selected_cases: + if self.blocked_reason: + self.record_blocked(case_id, self.blocked_reason) + continue + self.run_case(case_id, cases[case_id], create_bucket=case_id != "scanner.status-readiness") + + overall = self.write_results(started_at, started) + print( + f"ILM validation {overall}: result={self.evidence_dir / 'result.json'}", + flush=True, + ) + return {"pass": 0, "fail": 1, "infra_error": 2}[overall] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--endpoint", default=os.environ.get("RUSTFS_ENDPOINT")) + parser.add_argument( + "--scanner-endpoint", + action="append", + help="node endpoint queried for scanner status; repeat for every node", + ) + parser.add_argument("--access-key", default=os.environ.get("RUSTFS_ACCESS_KEY")) + parser.add_argument("--secret-key", default=os.environ.get("RUSTFS_SECRET_KEY")) + parser.add_argument("--region", default=os.environ.get("RUSTFS_REGION", "us-east-1")) + parser.add_argument("--environment-id", default="ilm-2x4") + parser.add_argument("--run-id", required=True) + parser.add_argument("--evidence-dir", required=True, type=Path) + parser.add_argument("--bucket-prefix", default="rustfs-ilm-validation") + parser.add_argument("--debug-day-seconds", type=int, default=10) + parser.add_argument("--process-seconds", type=int, default=1) + parser.add_argument("--scanner-cycle-seconds", type=int, default=2) + parser.add_argument("--timeout", type=float, default=120) + parser.add_argument("--poll-interval", type=float, default=2) + parser.add_argument("--request-timeout", type=float, default=10) + parser.add_argument("--case-timeout", type=float, default=900) + parser.add_argument("--cleanup-timeout", type=float, default=120) + parser.add_argument("--setup-budget", type=float, default=40) + parser.add_argument("--restart-timeout", type=float, default=120) + parser.add_argument("--negative-window", type=float, default=20) + parser.add_argument("--max-clock-skew", type=float, default=5) + parser.add_argument("--rc-bin", default="rc") + parser.add_argument( + "--tier-endpoint", default=os.environ.get("RUSTFS_ILM_TIER_ENDPOINT") + ) + parser.add_argument( + "--tier-access-key", default=os.environ.get("RUSTFS_ILM_TIER_ACCESS_KEY") + ) + parser.add_argument( + "--tier-secret-key", default=os.environ.get("RUSTFS_ILM_TIER_SECRET_KEY") + ) + parser.add_argument( + "--restart-hook", + help="JSON array of command tokens used to restart the fixed cluster", + ) + parser.add_argument( + "--case", + action="append", + choices=CASE_IDS, + help="run only the named case; repeat to select multiple cases", + ) + parser.add_argument( + "--cleanup-on-failure", + action="store_true", + help="delete failed-case buckets instead of preserving them for diagnosis", + ) + args = parser.parse_args() + if not args.endpoint or not args.access_key or not args.secret_key: + parser.error("endpoint, access key, and secret key are required") + tier_values = (args.tier_endpoint, args.tier_access_key, args.tier_secret_key) + if any(tier_values) and not all(tier_values): + parser.error("tier endpoint, access key, and secret key must be provided together") + parsed = urlparse(args.endpoint) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + parser.error("endpoint must be an absolute HTTP(S) URL") + if ( + args.debug_day_seconds < 1 + or args.process_seconds < 1 + or args.scanner_cycle_seconds < 1 + or args.timeout <= 0 + or args.poll_interval <= 0 + ): + parser.error("timing values must be positive") + for name in ("timeout", "poll_interval", "request_timeout", "case_timeout", + "cleanup_timeout", "setup_budget", "restart_timeout", "negative_window", "max_clock_skew"): + if not math.isfinite(getattr(args, name)) or getattr(args, name) <= 0: + parser.error(f"{name.replace('_', '-')} must be finite and positive") + args.scanner_endpoint = list( + dict.fromkeys(args.scanner_endpoint or [args.endpoint]) + ) + scanner_hosts: list[str] = [] + for endpoint in args.scanner_endpoint: + scanner_parsed = urlparse(endpoint) + if ( + scanner_parsed.scheme not in {"http", "https"} + or not scanner_parsed.netloc + ): + parser.error("scanner endpoints must be absolute HTTP(S) URLs") + scanner_hosts.append(scanner_parsed.hostname or "") + no_proxy = os.environ.get("NO_PROXY", os.environ.get("no_proxy", "")) + additions = [parsed.hostname or "", *scanner_hosts, "127.0.0.1", "localhost"] + if args.tier_endpoint: + tier_parsed = urlparse(args.tier_endpoint) + if tier_parsed.scheme not in {"http", "https"} or not tier_parsed.netloc: + parser.error("tier endpoint must be an absolute HTTP(S) URL") + additions.append(tier_parsed.hostname or "") + os.environ["NO_PROXY"] = ",".join(filter(None, [no_proxy, *additions])) + os.environ["no_proxy"] = os.environ["NO_PROXY"] + return args + + +def main() -> int: + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(line_buffering=True) + args = parse_args() + harness = IlmHarness(args) + try: + return harness.run() + finally: + harness.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation_ilm/ilm_timing.py b/scripts/validation_ilm/ilm_timing.py new file mode 100644 index 00000000..25a870ac --- /dev/null +++ b/scripts/validation_ilm/ilm_timing.py @@ -0,0 +1,304 @@ +"""Independent time and progress oracles for serial, live ILM validation.""" + +from __future__ import annotations + +import math +import signal +import threading +import time +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from email.utils import parsedate_to_datetime +from typing import Any, Callable, Iterator + + +class TimingPreconditionError(RuntimeError): + """The driver cannot establish the required before/after observation window.""" + + +class DeadlineExceeded(AssertionError): + """An operation did not satisfy its contract within the original budget.""" + + +def parse_expiry_date(value: str) -> datetime: + """Read an absolute expiry instant, not certify its wire-format parity. + + AWS examples use HTTP dates; current RustFS also emits RFC 3339 here. + Both retain their timezone and must satisfy the same independent due window. + """ + try: + try: + parsed = parsedate_to_datetime(value) + except (ValueError, TypeError, IndexError): + parsed = datetime.fromisoformat(value) + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError("expiry-date has no timezone") + return parsed.astimezone(UTC) + except (ValueError, TypeError, IndexError) as exc: + raise AssertionError(f"invalid expiry-date: {value!r}") from exc + + +# Real elapsed time is deliberately independent of fake clocks in oracle tests. +_elapsed = time.monotonic +_budgets: list[tuple[float, str, type[Exception]]] = [] + + +def _alarm(_signum: int, _frame: Any) -> None: + _end, label, error = min(_budgets, key=lambda item: item[0]) + raise error(f"deadline exceeded: {label}") + + +def _arm() -> None: + remaining = min(item[0] for item in _budgets) - _elapsed() + # An expired parent is already unwinding. Re-arming it would interrupt its + # exception handling repeatedly, possibly hiding the original failure. + signal.setitimer(signal.ITIMER_REAL, remaining if remaining > 0 else 0) + + +def _check() -> None: + if _budgets: + end, label, error = min(_budgets, key=lambda item: item[0]) + if _elapsed() >= end: + raise error(f"deadline exceeded: {label}") + + +@contextmanager +def time_budget( + seconds: float, label: str, error: type[Exception] = DeadlineExceeded +) -> Iterator[None]: + """Interrupt stalled I/O and body reads; nested budgets cannot extend parents. + + Live ILM cases run serially on the main thread on the Linux driver. A thread + pool timeout would leave the timed-out request mutating the next fixture. + """ + if not math.isfinite(seconds) or seconds <= 0: + raise error(f"deadline exceeded: {label}") + if threading.current_thread() is not threading.main_thread(): + raise RuntimeError("ILM time budgets require serial main-thread execution") + _check() + previous_handler = None + if not _budgets: + if signal.getitimer(signal.ITIMER_REAL)[0] > 0: + raise RuntimeError("ILM cannot replace an existing process alarm") + previous_handler = signal.signal(signal.SIGALRM, _alarm) + end = _elapsed() + seconds + entry = (end, label, error) + try: + _budgets.append(entry) + _arm() + yield + _check() + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + if entry in _budgets: + _budgets.remove(entry) + if _budgets: + _arm() + else: + signal.signal(signal.SIGALRM, previous_handler) + + +@dataclass(frozen=True) +class DueWindow: + earliest: datetime + latest: datetime + + def as_dict(self) -> dict[str, str]: + return {"earliest": self.earliest.isoformat(), "latest": self.latest.isoformat()} + + +def expiry_window( + modified: datetime, days: int, day_seconds: int, process_seconds: int = 1 +) -> DueWindow: + """Round independently, preserving uncertainty in second-resolution S3 time. + + LastModified may hide subsecond precision. The inclusive upper bound is + deliberately conservative, including the next reported second. + """ + if modified.tzinfo is None or days <= 0 or day_seconds <= 0 or process_seconds <= 0: + raise TimingPreconditionError("expiry requires aware time and positive Days/intervals") + epoch = datetime(1970, 1, 1, tzinfo=UTC) + boundary = process_seconds * 1_000_000 + + def rounded(value: datetime) -> datetime: + delta = value.astimezone(UTC) - epoch + timedelta(seconds=days * day_seconds) + micros = (delta.days * 86400 + delta.seconds) * 1_000_000 + delta.microseconds + return epoch + timedelta(microseconds=((micros + boundary - 1) // boundary) * boundary) + + return DueWindow(rounded(modified), rounded(modified + timedelta(seconds=1))) + + +class ServerClock: + """Map server UTC intervals onto one monotonic driver timeline.""" + + def __init__( + self, + samples: list[tuple[str, datetime, float, float]], + *, + max_skew: float = 5, + wall: Callable[[], float] = time.time, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + if not samples: + raise TimingPreconditionError("no server clock samples") + self.wall = wall + self.monotonic = monotonic + self.anchor_wall = wall() + self.anchor_mono = monotonic() + self.offset_lo = min(date.timestamp() - end for _, date, _, end in samples) + self.offset_hi = max(date.timestamp() + 1 - start for _, date, start, _ in samples) + if max(abs(self.offset_lo), abs(self.offset_hi)) > max_skew: + raise TimingPreconditionError("server clock skew or sampling latency exceeds budget") + self.samples = samples + + @staticmethod + def response_date(response: dict[str, Any]) -> datetime: + value = response.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("date") + try: + result = parsedate_to_datetime(value) + if result.tzinfo is None: + raise ValueError("Date has no timezone") + return result.astimezone(UTC) + except (ValueError, TypeError, IndexError) as exc: + raise TimingPreconditionError("missing or invalid S3 response Date") from exc + + def now(self) -> DueWindow: + projected = self.anchor_wall + self.monotonic() - self.anchor_mono + if abs(self.wall() - projected) > 1: + raise TimingPreconditionError("driver wall clock jumped during the case") + return DueWindow( + datetime.fromtimestamp(projected + self.offset_lo, UTC), + datetime.fromtimestamp(projected + self.offset_hi, UTC), + ) + + def validate_response(self, response: dict[str, Any], start: float, end: float) -> None: + date = self.response_date(response).timestamp() + self.now() + if date + 1 < start + self.offset_lo or date > end + self.offset_hi: + raise TimingPreconditionError("server Date moved outside the calibrated clock interval") + + def require_before(self, due: DueWindow, *, margin: float = 0) -> None: + if self.now().latest + timedelta(seconds=margin) >= due.earliest: + raise TimingPreconditionError("missed the required before-expiry observation window") + + def passed(self, due: DueWindow) -> bool: + return self.now().earliest > due.latest + + def timeout_after(self, due: DueWindow, allowance: float) -> float: + return max(0, (due.latest - self.now().earliest).total_seconds()) + allowance + + def as_dict(self) -> dict[str, Any]: + return { + "anchorUtc": datetime.fromtimestamp(self.anchor_wall, UTC).isoformat(), + "anchorMonotonic": self.anchor_mono, + "offsetSeconds": [self.offset_lo, self.offset_hi], + "samples": [ + {"endpoint": endpoint, "date": date.isoformat(), "sentUtc": start, "receivedUtc": end} + for endpoint, date, start, end in self.samples + ], + } + + +def successful_cycle(cluster: dict[str, Any]) -> tuple[str, int, int | None] | None: + """Accept only a single fresh healthy leader and a completed success.""" + leaders = [] + for node in cluster.get("nodes", []): + data = node.get("status", {}).get("data", {}) + metrics = data.get("metrics", {}) + if metrics.get("leader_lock_held_by_this_process") is True: + leaders.append((node.get("endpoint"), data, metrics)) + if len(leaders) != 1: + return None + endpoint, data, metrics = leaders[0] + freshness = data.get("freshness", {}) + ended = freshness.get("last_cycle_end_unix_secs") + if not ( + isinstance(endpoint, str) + and data.get("enabled") is True + and data.get("health") == "healthy" + and freshness.get("state") == "fresh" + and metrics.get("last_cycle_result") == "success" + and metrics.get("leader_lock_state") == "acquired" + and data.get("cycle_schedule", {}).get("execution_role") == "leader" + and isinstance(ended, int) and not isinstance(ended, bool) and ended > 0 + ): + return None + lifetime_cycles = metrics.get("life_time_ops", {}).get("scan_cycle") + if lifetime_cycles is not None and ( + not isinstance(lifetime_cycles, int) or isinstance(lifetime_cycles, bool) or lifetime_cycles < 0 + ): + return None + return endpoint, ended, lifetime_cycles + + +class CycleProgress: + """Count distinct observed cycles after a baseline, never historical success.""" + + def __init__(self) -> None: + self.last: tuple[str, int, int | None] | None = None + self.count = 0 + + def observe(self, cluster: dict[str, Any]) -> int: + current = successful_cycle(cluster) + if current is None: + # A gap/election may conceal a restart: re-establish the baseline. + self.last = None + self.count = 0 + elif (self.last is None or current[0] != self.last[0] or current[1] < self.last[1] + or (current[2] is None) != (self.last[2] is None) + or (current[2] is not None and self.last[2] is not None and current[2] < self.last[2])): + self.last = current + self.count = 0 + elif current[1] > self.last[1]: + self.count += 1 + self.last = current + else: + # Preserve the latest lifetime counter between completions to detect + # an observed same-node reset even when the UTC end keeps advancing. + self.last = current + return self.count + + +class BoundedBody: + def __init__(self, body: Any, seconds: float) -> None: + self.body = body + self.end = _elapsed() + seconds + + def read(self, *args: Any, **kwargs: Any) -> bytes: + try: + with time_budget(self.end - _elapsed(), "S3 response body", TimeoutError): + result = self.body.read(*args, **kwargs) + amount = args[0] if args else kwargs.get("amt") + if amount is None or amount < 0 or not result: + self.body.close() + return result + except BaseException: + self.body.close() + raise + + def close(self) -> None: + self.body.close() + + +class BoundedS3Client: + """Bound complete SDK calls, including hidden retries and streamed bodies.""" + + def __init__(self, client: Any, seconds: float) -> None: + self.client = client + self.seconds = seconds + + def __getattr__(self, name: str) -> Any: + member = getattr(self.client, name) + if not callable(member) or name == "close": + return member + + def call(*args: Any, **kwargs: Any) -> Any: + started = _elapsed() + with time_budget(self.seconds, f"S3 {name}", TimeoutError): + response = member(*args, **kwargs) + if isinstance(response, dict) and "Body" in response: + response["Body"] = BoundedBody(response["Body"], self.seconds - (_elapsed() - started)) + return response + + return call diff --git a/scripts/validation_ilm/rc_binary.py b/scripts/validation_ilm/rc_binary.py new file mode 100644 index 00000000..efc53a47 --- /dev/null +++ b/scripts/validation_ilm/rc_binary.py @@ -0,0 +1,21 @@ +"""Resolve and identify the rc executable used by live validation.""" + +from __future__ import annotations + +import hashlib +import shutil +from pathlib import Path + + +def resolve_rc_binary(selection: str) -> tuple[str, str]: + """Return one canonical executable path and its SHA-256 digest.""" + + located = shutil.which(selection) + if located is None: + raise FileNotFoundError(selection) + resolved = Path(located).resolve(strict=True) + digest = hashlib.sha256() + with resolved.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return str(resolved), digest.hexdigest() diff --git a/scripts/validation_ilm/requirements.txt b/scripts/validation_ilm/requirements.txt new file mode 100644 index 00000000..09c149ee --- /dev/null +++ b/scripts/validation_ilm/requirements.txt @@ -0,0 +1,2 @@ +boto3==1.43.77 +botocore==1.43.77 diff --git a/scripts/validation_ilm/restart_cluster.sh b/scripts/validation_ilm/restart_cluster.sh new file mode 100644 index 00000000..89bd1506 --- /dev/null +++ b/scripts/validation_ilm/restart_cluster.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if [ "$#" -ne 2 ]; then + printf 'usage: %s NODE1 NODE2\n' "$0" >&2 + exit 2 +fi + +ssh_user="${RUSTFS_SSH_USER:-azureuser}" +ssh_port="${RUSTFS_SSH_PORT:-22}" +ssh_opts=(-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new -p "${ssh_port}") +pids=() +for node in "$@"; do + ssh "${ssh_opts[@]}" "${ssh_user}@${node}" \ + 'if [ "$(id -u)" -ne 0 ]; then sudo -n systemctl restart rustfs; else systemctl restart rustfs; fi' & + pids+=("$!") +done +for pid in "${pids[@]}"; do wait "${pid}"; done + +for endpoint in http://rustfs-node1:9000 http://rustfs-node2:9000; do + ready=0 + for ((attempt=0; attempt<24; attempt++)); do + if curl --noproxy '*' -fsS --max-time 3 "${endpoint}/minio/health/live" >/dev/null; then + ready=1 + break + fi + sleep 2 + done + [ "${ready}" -eq 1 ] || { printf 'node not ready after restart: %s\n' "${endpoint}" >&2; exit 1; } +done diff --git a/scripts/validation_ilm/verify_result.py b/scripts/validation_ilm/verify_result.py new file mode 100644 index 00000000..76e63c0a --- /dev/null +++ b/scripts/validation_ilm/verify_result.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Check an ILM harness result before granting a canonical PASS.""" + +from __future__ import annotations + +import json +from pathlib import Path +import sys + + +def verify(case_id: str, evidence_dir: Path) -> None: + result = json.loads((evidence_dir / "result.json").read_text(encoding="utf-8")) + manifest = json.loads((evidence_dir / "run-manifest.json").read_text(encoding="utf-8")) + cases = result["status"]["cases"] + if result["status"]["state"] != "pass" or len(cases) != 1: + raise ValueError("ILM run did not pass exactly one case") + case = cases[0] + if case["id"] != case_id or case["state"] != "pass": + raise ValueError("ILM result does not pass the selected case") + if manifest["executionPlan"]["scoredCases"] != [case_id]: + raise ValueError("ILM execution plan differs from selected case") + if manifest["cleanupOutcome"]["state"] != "pass" or manifest["preservedBuckets"]: + raise ValueError("ILM test bucket cleanup did not pass") + if not case["evidence"]: + raise ValueError("ILM case has no assertion evidence") + root = evidence_dir.resolve(strict=True) + for name in case["evidence"]: + path = (root / name).resolve(strict=True) + if not path.is_relative_to(root) or not path.is_file(): + raise ValueError(f"unsafe ILM evidence path: {name}") + + +def main() -> int: + if len(sys.argv) != 3: + print(f"usage: {sys.argv[0]} CASE_ID EVIDENCE_DIR", file=sys.stderr) + return 2 + try: + verify(sys.argv[1], Path(sys.argv[2])) + except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError) as exc: + print(f"ILM verification failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test_validation_ilm.py b/test_validation_ilm.py new file mode 100644 index 00000000..cac29682 --- /dev/null +++ b/test_validation_ilm.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +import json +import tempfile +import unittest +from pathlib import Path + +from scripts.validation_ilm.case_plan import ILM_CASE_IDS, ilm_harness_scored_cases +from scripts.validation_ilm.verify_result import verify + + +class IlmContractTest(unittest.TestCase): + def test_catalog_drives_case_selection(self) -> None: + self.assertEqual(15, len(ILM_CASE_IDS)) + self.assertEqual(("ilm.configuration-roundtrip",), ilm_harness_scored_cases(["ilm.configuration-roundtrip"])) + with self.assertRaises(ValueError): + ilm_harness_scored_cases(["missing.case"]) + + def test_result_requires_selected_case_evidence_and_cleanup(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + case_id = "ilm.configuration-roundtrip" + (root / "proof.json").write_text("{}", encoding="utf-8") + result = {"status": {"state": "pass", "cases": [{ + "id": case_id, "state": "pass", "evidence": ["proof.json"], + }]}} + manifest = {"executionPlan": {"scoredCases": [case_id]}, + "cleanupOutcome": {"state": "pass"}, "preservedBuckets": []} + (root / "result.json").write_text(json.dumps(result), encoding="utf-8") + (root / "run-manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + verify(case_id, root) + result["status"]["cases"][0]["evidence"] = ["../outside.json"] + (root / "result.json").write_text(json.dumps(result), encoding="utf-8") + with self.assertRaises(OSError): + verify(case_id, root) + result["status"]["cases"][0]["evidence"] = ["proof.json"] + (root / "result.json").write_text(json.dumps(result), encoding="utf-8") + manifest["preservedBuckets"] = ["leftover"] + (root / "run-manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "cleanup"): + verify(case_id, root) + + +if __name__ == "__main__": + unittest.main() From c8b7557547941f4c0177db71e061b14f08d4c86a Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 18:41:21 +0800 Subject: [PATCH 012/343] Migrate pool lifecycle validation onto existing pool fleet --- .../workflows/validation-catalog-tests.yml | 9 +- rustfs_pool_expand.sh | 5 + scripts/run_validation_case.sh | 10 + scripts/validation_pool/case_plan.py | 30 + scripts/validation_pool/pool_behavior.py | 2812 +++++++++++++++++ scripts/validation_pool/pool_disk_probe.py | 197 ++ scripts/validation_pool/pool_helpers.py | 25 + scripts/validation_pool/pool_status.py | 588 ++++ scripts/validation_pool/rc_binary.py | 10 + scripts/validation_pool/run_case.sh | 236 ++ test_validation_pool.py | 24 + 11 files changed, 3945 insertions(+), 1 deletion(-) create mode 100644 scripts/validation_pool/case_plan.py create mode 100644 scripts/validation_pool/pool_behavior.py create mode 100644 scripts/validation_pool/pool_disk_probe.py create mode 100644 scripts/validation_pool/pool_helpers.py create mode 100644 scripts/validation_pool/pool_status.py create mode 100644 scripts/validation_pool/rc_binary.py create mode 100644 scripts/validation_pool/run_case.sh create mode 100644 test_validation_pool.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 6ab0fc7a..28501f32 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -10,6 +10,9 @@ on: - test_validation_migration.py - test_validation_ilm.py - scripts/validation_ilm/** + - test_validation_pool.py + - scripts/validation_pool/** + - rustfs_pool_expand.sh - rustfs-storage-test.sh - test_validation_scanner_usage.sh - .github/workflows/validation-catalog-tests.yml @@ -23,6 +26,9 @@ on: - test_validation_migration.py - test_validation_ilm.py - scripts/validation_ilm/** + - test_validation_pool.py + - scripts/validation_pool/** + - rustfs_pool_expand.sh - rustfs-storage-test.sh - test_validation_scanner_usage.sh - .github/workflows/validation-catalog-tests.yml @@ -41,5 +47,6 @@ jobs: - run: python3 scripts/validation_catalog.py check - run: python3 -B -m unittest -v test_validation_migration.py - run: python3 -B -m unittest -v test_validation_ilm.py - - run: bash -n rustfs-storage-test.sh test_validation_scanner_usage.sh + - run: python3 -B -m unittest -v test_validation_pool.py + - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh scripts/validation_pool/run_case.sh test_validation_scanner_usage.sh - run: bash test_validation_scanner_usage.sh diff --git a/rustfs_pool_expand.sh b/rustfs_pool_expand.sh index e5adf71a..2cad31a8 100755 --- a/rustfs_pool_expand.sh +++ b/rustfs_pool_expand.sh @@ -580,6 +580,11 @@ RUSTFS_CONSOLE_ENABLE=${RUSTFS_CONSOLE_ENABLE} RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL} RUSTFS_OBS_LOG_DIRECTORY="${RUSTFS_OBS_LOG_DIRECTORY}" EOF + if [ "${RUSTFS_POOL_VALIDATION_MODE:-0}" = 1 ]; then + printf 'RUST_LOG=warn,rustfs::ecstore=info,rustfs::rebalance=debug\n' + printf 'RUSTFS_SCANNER_ENABLED=true\nRUSTFS_SCANNER_START_DELAY_SECS=0\nRUSTFS_SCANNER_CYCLE=2\n' + printf 'HTTP_PROXY=\nHTTPS_PROXY=\nALL_PROXY=\nhttp_proxy=\nhttps_proxy=\nall_proxy=\n' + fi } # Expand a volume expression's {N...M} range into concrete local paths, e.g.: diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 08bcc52e..e975876b 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -13,6 +13,8 @@ TESTING_COMMIT="$3" ARTIFACTS_DIR="$4" case "${CASE_ID}" in + pool.*) + SUITE=scripts/validation_pool/run_case.sh; EXTRA_ARGS=() ;; s3.basic-object-round-trip) SUITE=rustfs-s3-compat-test.sh; EXTRA_ARGS=() ;; scanner.usage-convergence|scanner.status-readiness|ilm.configuration-roundtrip|ilm.invalid-configuration|ilm.current-expiration|ilm.tag-filter-expiration|ilm.disabled-rule|ilm.rule-removal|ilm.versioned-current-expiration|ilm.noncurrent-expiration|ilm.newer-noncurrent-retention|ilm.delete-all-versions|ilm.delete-marker-expiration|ilm.multipart-expiration|ilm.restart-persistence|ilm.transition-and-restore) @@ -28,6 +30,14 @@ case "${CASE_ID}" in exit 2 ;; esac +if [[ "${CASE_ID}" == pool.* ]]; then + python3 -B -c 'import json,sys; catalog=json.load(open(sys.argv[1])); valid={case["id"] for module in catalog["modules"] if module["id"] == "pool-lifecycle" for case in module["cases"]}; sys.exit(0 if sys.argv[2] in valid else 2)' \ + "${SCRIPT_DIR}/validation-cases.json" "${CASE_ID}" || { + printf 'unknown canonical pool case: %s\n' "${CASE_ID}" >&2 + exit 2 + } +fi + [ -f "${CANDIDATE_MANIFEST}" ] || { printf 'candidate manifest missing\n' >&2; exit 2; } [[ "${TESTING_COMMIT}" =~ ^[0-9a-f]{40}$ ]] || { printf 'invalid testing commit\n' >&2; exit 2; } [ "${TESTING_COMMIT}" = "$(git -C "${SCRIPT_DIR}" rev-parse HEAD)" ] || { diff --git a/scripts/validation_pool/case_plan.py b/scripts/validation_pool/case_plan.py new file mode 100644 index 00000000..ec793cf9 --- /dev/null +++ b/scripts/validation_pool/case_plan.py @@ -0,0 +1,30 @@ +"""Select pool lifecycle cases from the canonical auto-testing catalog.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Iterable + + +CATALOG = Path(__file__).resolve().parents[2] / "validation-cases.json" + + +def pool_case_ids() -> tuple[str, ...]: + catalog = json.loads(CATALOG.read_text(encoding="utf-8")) + modules = [module for module in catalog["modules"] if module["id"] == "pool-lifecycle"] + if len(modules) != 1: + raise ValueError("canonical pool module is missing or duplicated") + return tuple(case["id"] for case in modules[0]["cases"]) + + +POOL_CASE_IDS = pool_case_ids() + + +def pool_harness_scored_cases(requested_cases: Iterable[str] | None) -> tuple[str, ...]: + if requested_cases is None: + return POOL_CASE_IDS + selected = tuple(requested_cases) + if len(selected) != len(set(selected)) or set(selected) - set(POOL_CASE_IDS): + raise ValueError("duplicate or unknown canonical pool case") + return tuple(case_id for case_id in POOL_CASE_IDS if case_id in selected) diff --git a/scripts/validation_pool/pool_behavior.py b/scripts/validation_pool/pool_behavior.py new file mode 100644 index 00000000..d47e3480 --- /dev/null +++ b/scripts/validation_pool/pool_behavior.py @@ -0,0 +1,2812 @@ +#!/usr/bin/env python3 +"""Run destructive fixed-environment RustFS pool lifecycle validation.""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import hashlib +import json +import os +import secrets +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from email.utils import format_datetime +from pathlib import Path +from typing import Any, Callable + +from case_plan import POOL_CASE_IDS as CASE_IDS, pool_harness_scored_cases +from pool_helpers import active_endpoint_indices, active_fixture_bytes +from pool_status import ( + PoolStatusError, + assert_pool_parity, + canonical_pool, + decommission_active, + decommission_active_or_completed, + decommission_canceled, + decommission_cleared, + decommission_complete, + decommission_entry, + decommission_multipart_blocked, + multipart_drain_failure_records, + pool_entries, + rc_failure_is_retryable, + rebalance_active, + rebalance_active_or_completed, + rebalance_complete, + rebalance_entries, + rebalance_stopped, + validate_pool_inventory, +) +from rc_binary import resolve_rc_binary + +try: + import boto3 + import botocore + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.config import Config + from botocore.credentials import Credentials + from botocore.exceptions import ( + ClientError, + ConnectTimeoutError, + EndpointConnectionError, + ReadTimeoutError, + ) +except ImportError as exc: # pragma: no cover - dependency guidance + raise SystemExit( + "boto3 is required for pool validation" + ) from exc + + +ADMIN_PREFIX = "/rustfs/admin/v3" +INFRA_EXCEPTIONS = (ConnectTimeoutError, EndpointConnectionError, ReadTimeoutError) +INFRA_ERROR_CODES = { + "AccessDenied", + "InvalidAccessKeyId", + "InvalidToken", + "RequestTimeout", + "ServiceUnavailable", + "SignatureDoesNotMatch", + "SlowDown", +} +TRANSIENT_S3_CODES = {"InternalError", "RequestTimeout", "ServiceUnavailable", "SlowDown"} + + +def utc_now() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + +def json_default(value: Any) -> Any: + if isinstance(value, datetime): + return value.astimezone(UTC).isoformat().replace("+00:00", "Z") + if isinstance(value, bytes): + return {"bytes": len(value), "sha256": hashlib.sha256(value).hexdigest()} + return str(value) + + +def safe_name(value: str) -> str: + return "".join(character if character.isalnum() or character in "-." else "-" for character in value) + + +def deterministic_payload(seed: str, size: int) -> bytes: + block = hashlib.sha256(seed.encode("utf-8")).digest() + return (block * ((size + len(block) - 1) // len(block)))[:size] + + +@dataclass +class CaseResult: + id: str + state: str + message: str + started_at: str + finished_at: str + duration_seconds: float + evidence: list[str] = field(default_factory=list) + + +class EventRecorder: + def __init__(self, path: Path) -> None: + self.path = path + self.lock = threading.Lock() + self.path.parent.mkdir(parents=True, exist_ok=True) + + def record(self, event: str, **details: Any) -> None: + item = {"timestamp": utc_now(), "event": event, "details": details} + with self.lock: + with self.path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(item, default=json_default, sort_keys=True) + "\n") + + +class PoolHarness: + def __init__(self, args: argparse.Namespace) -> None: + args.endpoint = args.endpoint.rstrip("/") + args.node_endpoint = [endpoint.rstrip("/") for endpoint in args.node_endpoint] + self.args = args + self.evidence_dir = args.evidence_dir.resolve() + self.case_dir = self.evidence_dir / "cases" + self.case_dir.mkdir(parents=True, exist_ok=True) + self.events = EventRecorder(self.evidence_dir / "events.jsonl") + self.results: list[CaseResult] = [] + self.created_buckets: set[str] = set() + self.current_layout = "initial" + self.isolation_failure: dict[str, str] | None = None + self.cleanup_outcome_value: dict[str, Any] = { + "attempted": False, + "state": "preserved", + "message": "physical cleanup has not run", + } + self.rc_home = Path(tempfile.mkdtemp(prefix="rustfs-pool-rc-")) + self.rc_env = os.environ.copy() + self.rc_env["RC_CONFIG_DIR"] = str(self.rc_home) + self.rc_bin = args.rc_bin + self.rc_version = "unavailable" + self.rc_binary_sha256 = "unavailable" + self.expected_rc_binary_sha256 = os.environ.get( + "RUSTFS_EXPECTED_RC_SHA256", "" + ).lower() + self.rc_alias = f"pool-{safe_name(args.run_id)[-16:]}" + self.initial_endpoint = args.endpoint + self.rc_configured = False + self.secrets = {args.access_key, args.secret_key} + self.client = self.s3_client(args.endpoint) + self.node_clients = { + endpoint: self.s3_client(endpoint) for endpoint in args.node_endpoint + } + + def close(self) -> None: + shutil.rmtree(self.rc_home, ignore_errors=True) + + def redact(self, value: str) -> str: + result = value + for secret in sorted((item for item in self.secrets if item), key=len, reverse=True): + result = result.replace(secret, "[REDACTED]") + return result + + def redact_document(self, value: Any) -> Any: + serialized = json.dumps(value, default=json_default, sort_keys=True) + return json.loads(self.redact(serialized)) + + def evidence_path(self, case_id: str, name: str) -> Path: + directory = self.case_dir / safe_name(case_id) + directory.mkdir(parents=True, exist_ok=True) + return directory / name + + def save_json(self, case_id: str, name: str, value: Any) -> str: + path = self.evidence_path(case_id, name) + text = json.dumps(value, default=json_default, indent=2, sort_keys=True) + "\n" + path.write_text(self.redact(text), encoding="utf-8") + return str(path.relative_to(self.evidence_dir)) + + def save_text(self, case_id: str, name: str, value: str) -> str: + path = self.evidence_path(case_id, name) + path.write_text(self.redact(value), encoding="utf-8") + return str(path.relative_to(self.evidence_dir)) + + def s3_client(self, endpoint: str) -> Any: + return boto3.client( + "s3", + endpoint_url=endpoint, + aws_access_key_id=self.args.access_key, + aws_secret_access_key=self.args.secret_key, + region_name=self.args.region, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + retries={"max_attempts": 3, "mode": "standard"}, + connect_timeout=5, + read_timeout=60, + ), + ) + + def configure_rc(self) -> None: + requested = self.rc_bin + try: + self.rc_bin, self.rc_binary_sha256 = resolve_rc_binary(requested) + except FileNotFoundError as exc: + raise RuntimeError(f"required rc binary not found: {requested}") from exc + version = subprocess.run( + [self.rc_bin, "--version"], + env=self.rc_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + if version.returncode != 0: + raise RuntimeError( + f"rc --version failed with exit {version.returncode}: {version.stderr.strip()}" + ) + self.rc_version = self.redact((version.stdout or version.stderr).strip()) + if ( + self.expected_rc_binary_sha256 + and self.rc_binary_sha256.lower() != self.expected_rc_binary_sha256 + ): + raise RuntimeError( + "resolved rc binary SHA-256 differs from RUSTFS_EXPECTED_RC_SHA256" + ) + self.set_rc_alias(self.args.endpoint) + self.rc_configured = True + self.events.record( + "rc.client.identified", + binaryPath=self.rc_bin, + version=self.rc_version, + binarySha256=self.rc_binary_sha256, + ) + + def set_rc_alias(self, endpoint: str) -> None: + command = [ + self.rc_bin, + "--json", + "--no-color", + "alias", + "set", + self.rc_alias, + endpoint, + self.args.access_key, + self.args.secret_key, + "--bucket-lookup", + "path", + ] + result = subprocess.run( + command, + env=self.rc_env, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"rc alias setup failed: {self.redact(result.stderr.strip())}") + self.events.record( + "rc.alias.configured", + alias=self.rc_alias, + endpoint=endpoint, + ) + + def control_endpoint(self, layout: str) -> str: + if self.initial_endpoint not in self.args.node_endpoint: + return self.initial_endpoint + active = active_endpoint_indices(len(self.args.node_endpoint), layout) + active_endpoints = tuple(self.args.node_endpoint[index] for index in active) + if self.initial_endpoint in active_endpoints: + return self.initial_endpoint + return active_endpoints[0] + + def retained_pool_endpoints(self) -> tuple[str, ...]: + indices = active_endpoint_indices(len(self.args.node_endpoint), "retired") + return tuple(self.args.node_endpoint[index] for index in indices) + + def retained_pool_client(self) -> Any: + return self.node_clients[self.retained_pool_endpoints()[0]] + + def retire_decommissioned_pool(self, case_id: str) -> Any: + if self.current_layout != "expanded": + raise AssertionError("pool retirement requires the expanded topology") + self.wait_decommission(case_id, 0, "complete", label="before-retirement") + expanded, _ = self.pool_snapshot( + case_id, "before-retired-config-removal", 2, require_active=False + ) + pools = {item["id"]: item for item in validate_pool_inventory(expanded, 2, require_active=False)} + if set(pools) != {0, 1}: + raise AssertionError("pool retirement requires the original pool identities") + if (pools[0]["status"].lower() != "decommissioned" + or pools[0]["decommissionStatus"].lower() != "complete" + or pools[1]["status"].lower() != "active"): + raise AssertionError("pool retirement requires a completed source and an active retained pool") + retained_cmdline = pools[1]["cmdline"] + self.census(case_id, "before-retired-config-removal") + self.apply_topology(case_id, "retired") + remaining, _ = self.pool_snapshot(case_id, "after-retired-config-removal", 1) + if canonical_pool(pool_entries(remaining)[0])["cmdline"] != retained_cmdline: + raise AssertionError("retired topology does not contain exactly the retained pool") + self.census(case_id, "after-retired-config-removal") + return self.retained_pool_client() + + def select_control_endpoint(self, layout: str) -> None: + endpoint = self.control_endpoint(layout) + self.args.endpoint = endpoint + self.client = self.s3_client(endpoint) + if self.rc_configured: + self.set_rc_alias(endpoint) + + def run_rc( + self, + case_id: str, + name: str, + arguments: list[str], + *, + expect_success: bool = True, + expected_error_code: str | None = None, + ) -> subprocess.CompletedProcess[str]: + if not expect_success and expected_error_code not in { + "InvalidArgument", "InvalidRequest", "OperationAborted", "NotImplemented" + }: + raise AssertionError(f"rc {name} requires an expected server error for a negative test") + if expect_success and expected_error_code is not None: + raise AssertionError(f"rc {name} cannot expect both success and a server error") + command = [self.rc_bin, "--json", "--no-color", *arguments] + result = subprocess.run( + command, + env=self.rc_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + self.save_json( + case_id, + f"{name}.json", + { + "command": [ + "[REDACTED]" if token in self.secrets else token + for token in command + ], + "exitCode": result.returncode, + "stdout": self.redact(result.stdout), + "stderr": self.redact(result.stderr), + }, + ) + if expect_success and result.returncode != 0: + raise AssertionError( + f"rc {name} exited {result.returncode}: {self.redact(result.stderr.strip())}" + ) + if not expect_success and result.returncode == 0: + raise AssertionError(f"rc {name} unexpectedly succeeded") + if not expect_success: + try: + document = json.loads(result.stderr) + except json.JSONDecodeError as exc: + raise AssertionError(f"rc {name} did not return a structured server rejection") from exc + details = document.get("details") if isinstance(document, dict) else None + message = document.get("error") if isinstance(document, dict) else None + if not isinstance(details, dict) or details.get("retryable") is not False or not isinstance(message, str): + raise AssertionError(f"rc {name} did not return a non-retryable server rejection") + xml_start = message.find("") + if xml_start >= 0: + error_xml = message[xml_start:] + elif ( + expected_error_code == "NotImplemented" + and arguments == ["admin", "rebalance", "start", self.rc_alias] + and details.get("type") == "unsupported_feature" + and "Unsupported feature:" in message + ): + # rc strips the XML code from NotImplemented. Confirm the + # single-pool server guard directly, not merely CLI capability. + status, headers, body = self.signed_request( + self.args.endpoint, "POST", f"{ADMIN_PREFIX}/rebalance/start" + ) + error_xml = body.decode("utf-8", errors="replace") + self.save_json(case_id, f"{name}-server-rejection.json", { + "status": status, "requestId": headers.get("x-amz-request-id"), + "body": self.redact(error_xml), + }) + if status != 501: + raise AssertionError(f"rc {name} server proof returned HTTP {status}, expected 501") + else: + raise AssertionError(f"rc {name} did not retain its expected server error code") + try: + root = ET.fromstring(error_xml) + except ET.ParseError as exc: + raise AssertionError(f"rc {name} returned malformed server error XML") from exc + codes = root.findall("Code") if root.tag == "Error" else [] + if len(codes) != 1 or codes[0].text != expected_error_code: + observed = [element.text for element in codes] + raise AssertionError(f"rc {name} returned server errors {observed!r}, expected {expected_error_code}") + return result + + @staticmethod + def parse_json_output(result: subprocess.CompletedProcess[str], operation: str) -> Any: + try: + return json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise AssertionError(f"{operation} did not return JSON") from exc + + def assert_no_secrets(self, document: Any, label: str) -> None: + serialized = json.dumps(document, default=json_default, sort_keys=True) + exposed = [secret for secret in self.secrets if secret and secret in serialized] + if exposed: + raise AssertionError(f"{label} exposed configured credentials") + + def signed_request( + self, + endpoint: str, + method: str, + path: str, + *, + query: str = "", + body: bytes | None = None, + timeout: float = 60, + ) -> tuple[int, dict[str, str], bytes]: + url = endpoint.rstrip("/") + urllib.parse.quote(path, safe="/-._~") + if query: + url += f"?{query}" + payload = body or b"" + headers = {"x-amz-content-sha256": hashlib.sha256(payload).hexdigest()} + request = AWSRequest(method=method, url=url, data=body, headers=headers) + SigV4Auth( + Credentials(self.args.access_key, self.args.secret_key), + "s3", + self.args.region, + ).add_auth(request) + outgoing = urllib.request.Request( + url, + data=body, + headers=dict(request.headers.items()), + method=method, + ) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + try: + with opener.open(outgoing, timeout=timeout) as response: + return response.status, dict(response.headers.items()), response.read() + except urllib.error.HTTPError as exc: + return exc.code, dict(exc.headers.items()), exc.read() + + def admin_json( + self, + case_id: str, + name: str, + subpath: str, + *, + endpoint: str | None = None, + query: str = "", + expected_status: int = 200, + ) -> Any: + status, headers, body = self.signed_request( + endpoint or self.args.endpoint, + "GET", + f"{ADMIN_PREFIX}/{subpath}", + query=query, + ) + try: + parsed = json.loads(body) if body.strip() else {} + except json.JSONDecodeError: + parsed = {"raw": body.decode("utf-8", errors="replace")} + self.save_json( + case_id, + f"{name}.json", + { + "status": status, + "requestId": headers.get("x-amz-request-id"), + "body": parsed, + }, + ) + if status != expected_status: + raise AssertionError( + f"GET {subpath} returned HTTP {status}, expected {expected_status}: {parsed!r}" + ) + return parsed + + def hook( + self, + case_id: str, + name: str, + raw: str | None, + extra: tuple[str, ...] = (), + *, + timeout: float | None = None, + ) -> str: + if not raw: + raise RuntimeError(f"{name} requires a driver hook") + try: + command = json.loads(raw) + except json.JSONDecodeError as exc: + raise RuntimeError(f"{name} hook is not valid JSON") from exc + if not isinstance(command, list) or not command or not all( + isinstance(token, str) and token for token in command + ): + raise RuntimeError(f"{name} hook must be a JSON command array") + result = subprocess.run( + [*command, *extra], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=False, + timeout=timeout, + ) + reference = self.save_text(case_id, f"{name}.log", result.stdout) + self.events.record( + "driver.hook", + case=case_id, + hook=name, + exitCode=result.returncode, + arguments=list(extra), + ) + if result.returncode != 0: + raise RuntimeError(f"{name} hook exited {result.returncode}") + return reference + + def reset_case_environment(self, case_id: str) -> None: + self.hook(case_id, "physical-reset-initial", self.args.reset_hook) + self.current_layout = "initial" + self.select_control_endpoint("initial") + self.wait_s3(case_id, self.client, "stable initial endpoint") + + def apply_topology(self, case_id: str, layout: str) -> None: + self.hook( + case_id, + f"topology-{layout}", + self.args.topology_hook, + ("--layout", layout), + ) + self.current_layout = layout + self.select_control_endpoint(layout) + self.wait_s3(case_id, self.client, f"stable {layout} endpoint") + indices = active_endpoint_indices(len(self.args.node_endpoint), layout) + for index in indices: + endpoint = self.args.node_endpoint[index] + client = self.node_clients[endpoint] + self.wait_s3(case_id, client, f"node {index} {layout} endpoint") + + def restart_topology(self, case_id: str, layout: str) -> None: + self.hook( + case_id, + f"restart-{layout}", + self.args.restart_hook, + ("--layout", layout), + ) + self.current_layout = layout + self.select_control_endpoint(layout) + self.wait_s3(case_id, self.client, f"stable endpoint after {layout} restart") + + def census(self, case_id: str, phase: str) -> str: + return self.hook( + case_id, + f"physical-census-{safe_name(phase)}", + self.args.census_hook, + ("--phase", phase), + ) + + def capture_diagnostics(self, case_id: str, phase: str) -> None: + raw = getattr(self.args, "diagnostic_hook", None) + if not raw: + return + errors = [] + for index, endpoint in enumerate(self.args.node_endpoint): + try: + self.admin_json( + case_id, + f"diagnostic-{safe_name(phase)}-node-{index}-decommission", + "decommission/status", + endpoint=endpoint, + ) + except Exception as exc: + errors.append(self.redact(f"node {index}: {type(exc).__name__}: {exc}")) + try: + self.hook( + case_id, + f"diagnostic-{safe_name(phase)}-metadata", + raw, + ("--phase", phase), + timeout=60, + ) + except Exception as exc: + errors.append(self.redact(f"metadata: {type(exc).__name__}: {exc}")) + self.save_json( + case_id, + f"diagnostic-{safe_name(phase)}-collection.json", + {"phase": phase, "errors": errors, "complete": not errors}, + ) + + def wait_canceled_quiescence(self, case_id: str, pool_id: int, *, label: str) -> None: + stable_since: float | None = None + prior: tuple[int, int] | None = None + history: list[dict[str, Any]] = [] + + def probe() -> tuple[bool, Any]: + nonlocal stable_since, prior + counters = [] + for endpoint in self.args.node_endpoint: + try: + status, _, body = self.signed_request( + endpoint, "GET", f"{ADMIN_PREFIX}/decommission/status" + ) + except INFRA_EXCEPTIONS as exc: + # A gap in observations is not evidence of quiescence. + # Preserve wait_until's infrastructure classification. + stable_since = None + history.append({ + "capturedAt": utc_now(), "endpoint": endpoint, + "error": self.redact(f"{type(exc).__name__}: {exc}"), + }) + raise + if status != 200: + stable_since = None + return False, {"endpoint": endpoint, "httpStatus": status} + document = json.loads(body) + if not decommission_canceled(document, pool_id): + stable_since = None + return False, document + counters.append(self.decommission_counters(document, pool_id)) + now = time.monotonic() + sample = {"capturedAt": utc_now(), "counters": counters} + history.append(sample) + if len(set(counters)) != 1: + stable_since = None + return False, sample + current = counters[0] + if current != prior or stable_since is None: + prior, stable_since = current, now + return now - stable_since >= 3.0, sample + + try: + self.wait_until( + case_id, + f"decommission pool {pool_id} canceled and quiescent on all nodes", + probe, + timeout=min(self.args.timeout, 30), + ) + finally: + self.save_json(case_id, f"{safe_name(label)}-quiescence-samples.json", history) + + def wait_until( + self, + case_id: str, + description: str, + probe: Callable[[], tuple[bool, Any]], + *, + timeout: float | None = None, + ) -> Any: + deadline = time.monotonic() + (timeout or self.args.timeout) + attempts = 0 + last: Any = None + last_was_infrastructure = False + while time.monotonic() < deadline: + attempts += 1 + try: + complete, last = probe() + last_was_infrastructure = False + except INFRA_EXCEPTIONS as exc: + complete, last = False, {"error": f"{type(exc).__name__}: {exc}"} + last_was_infrastructure = True + if complete: + self.events.record( + "poll.complete", + case=case_id, + description=description, + attempts=attempts, + detail=self.redact_document(last), + ) + return last + time.sleep(self.args.poll_interval) + self.save_json( + case_id, + f"poll-timeout-{safe_name(description)}.json", + {"description": description, "attempts": attempts, "last": last}, + ) + error_type = RuntimeError if last_was_infrastructure else AssertionError + raise error_type(f"timed out waiting for {description}; last={last!r}") + + def wait_s3(self, case_id: str, client: Any, description: str) -> None: + def probe() -> tuple[bool, Any]: + try: + response = client.list_buckets() + return True, { + "requestId": response.get("ResponseMetadata", {}).get("RequestId") + } + except INFRA_EXCEPTIONS: + raise + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code not in TRANSIENT_S3_CODES: + raise + return False, {"error": f"ClientError[{code}]: {exc}"[:500]} + + self.wait_until(case_id, description, probe, timeout=min(self.args.timeout, 180)) + + def bucket_name(self, case_id: str) -> str: + digest = hashlib.sha256(f"{self.args.run_id}:{case_id}".encode()).hexdigest()[:14] + return f"rv-pool-{digest}" + + def create_bucket(self, case_id: str) -> str: + bucket = self.bucket_name(case_id) + self.client.create_bucket(Bucket=bucket) + self.created_buckets.add(bucket) + self.events.record("bucket.created", case=case_id, bucket=bucket) + return bucket + + @staticmethod + def delete_batches(client: Any, bucket: str, items: list[dict[str, str]]) -> None: + for offset in range(0, len(items), 1000): + batch = items[offset : offset + 1000] + try: + response = client.delete_objects( + Bucket=bucket, + Delete={"Objects": batch, "Quiet": True}, + ) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code not in {"MethodNotAllowed", "NotImplemented", "XNotImplemented"}: + raise + # Some S3-compatible candidates do not implement multi-delete + # for explicit version IDs. Individual deletes are the portable + # cleanup fallback and retain the same isolation guarantee. + for item in batch: + client.delete_object(Bucket=bucket, **item) + continue + if response.get("Errors"): + raise RuntimeError(f"delete_objects failed: {response['Errors'][:5]!r}") + + def cleanup_bucket(self, bucket: str) -> None: + deadline = time.monotonic() + min(self.args.timeout, 120) + while True: + try: + uploads: list[dict[str, Any]] = [] + for page in self.client.get_paginator("list_multipart_uploads").paginate(Bucket=bucket): + uploads.extend(page.get("Uploads", [])) + for upload in uploads: + self.client.abort_multipart_upload( + Bucket=bucket, + Key=upload["Key"], + UploadId=upload["UploadId"], + ) + versions: list[dict[str, str]] = [] + for page in self.client.get_paginator("list_object_versions").paginate(Bucket=bucket): + versions.extend( + {"Key": item["Key"], "VersionId": item["VersionId"]} + for group in ("Versions", "DeleteMarkers") + for item in page.get(group, []) + ) + if versions: + self.delete_batches(self.client, bucket, versions) + objects: list[dict[str, str]] = [] + for page in self.client.get_paginator("list_objects_v2").paginate(Bucket=bucket): + objects.extend({"Key": item["Key"]} for item in page.get("Contents", [])) + if objects: + self.delete_batches(self.client, bucket, objects) + self.client.delete_bucket(Bucket=bucket) + self.created_buckets.discard(bucket) + self.events.record("bucket.cleanup.complete", bucket=bucket) + return + except INFRA_EXCEPTIONS as exc: + if time.monotonic() >= deadline: + raise + self.events.record( + "bucket.cleanup.retry", + bucket=bucket, + errorType=type(exc).__name__, + ) + time.sleep(self.args.poll_interval) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code == "NoSuchBucket": + self.created_buckets.discard(bucket) + return + retryable = code == "BucketNotEmpty" or code in TRANSIENT_S3_CODES + if not retryable or time.monotonic() >= deadline: + raise + self.events.record( + "bucket.cleanup.retry", + bucket=bucket, + errorCode=code, + ) + time.sleep(self.args.poll_interval) + + def s3_retrying( + self, + operation: str, + function: Callable[[], Any], + *, + timeout: float | None = None, + ) -> Any: + deadline = time.monotonic() + min(timeout or self.args.timeout, 120) + attempt = 0 + while True: + attempt += 1 + try: + return function() + except (*INFRA_EXCEPTIONS, ClientError) as exc: + retryable = isinstance(exc, INFRA_EXCEPTIONS) + error_code: str | None = None + if isinstance(exc, ClientError): + error_code = exc.response.get("Error", {}).get("Code") + retryable = error_code in TRANSIENT_S3_CODES + if not retryable or time.monotonic() >= deadline: + response_metadata = ( + exc.response.get("ResponseMetadata", {}) + if isinstance(exc, ClientError) else {} + ) + self.events.record( + "s3.operation.exhausted" if retryable else "s3.operation.failed", + operation=operation, + attempt=attempt, + errorType=type(exc).__name__, + errorCode=error_code, + httpStatus=response_metadata.get("HTTPStatusCode"), + requestId=response_metadata.get("RequestId"), + sdkRetryAttempts=response_metadata.get("RetryAttempts"), + ) + raise + self.events.record( + "s3.operation.retry", + operation=operation, + attempt=attempt, + errorType=type(exc).__name__, + errorCode=error_code, + ) + time.sleep(self.args.poll_interval) + + def put_retrying( + self, + client: Any, + bucket: str, + key: str, + body: bytes, + metadata: dict[str, str], + ) -> dict[str, Any]: + return self.s3_retrying( + f"PUT s3://{bucket}/{key}", + lambda: client.put_object( + Bucket=bucket, + Key=key, + Body=body, + Metadata=metadata, + ), + ) + + def put_inventory( + self, + case_id: str, + bucket: str, + *, + count: int | None = None, + size: int | None = None, + prefix: str = "baseline", + ) -> list[dict[str, Any]]: + count = count or self.args.fixture_objects + size = size or self.args.fixture_bytes + + def put(index: int) -> dict[str, Any]: + key = f"{prefix}/{index:05d}.bin" + actual_size = size if index >= 3 else (0, 1, 4097)[index] + body = deterministic_payload(f"{case_id}:{key}", actual_size) + metadata = {"case": safe_name(case_id), "ordinal": str(index)} + response = self.put_retrying(self.client, bucket, key, body, metadata) + return { + "key": key, + "size": actual_size, + "sha256": hashlib.sha256(body).hexdigest(), + "etag": str(response.get("ETag", "")).strip('"'), + "metadata": metadata, + "versionId": response.get("VersionId"), + } + + with concurrent.futures.ThreadPoolExecutor(max_workers=self.args.workers) as pool: + inventory = list(pool.map(put, range(count))) + inventory.sort(key=lambda item: item["key"]) + self.save_json(case_id, f"{prefix}-manifest.json", inventory) + self.events.record( + "fixture.inventory.created", + case=case_id, + bucket=bucket, + objects=len(inventory), + bytes=sum(item["size"] for item in inventory), + ) + return inventory + + def assert_inventory( + self, + case_id: str, + client: Any, + bucket: str, + inventory: list[dict[str, Any]], + label: str, + ) -> None: + def check(item: dict[str, Any]) -> dict[str, Any]: + def fetch() -> tuple[dict[str, Any], bytes]: + response = client.get_object(Bucket=bucket, Key=item["key"]) + return response, response["Body"].read() + + response, body = self.s3_retrying( + f"GET integrity object for {label}", fetch + ) + observed = { + "key": item["key"], + "size": len(body), + "sha256": hashlib.sha256(body).hexdigest(), + "etag": str(response.get("ETag", "")).strip('"'), + "metadata": response.get("Metadata", {}), + "versionId": response.get("VersionId"), + } + for field in ("size", "sha256", "etag", "metadata"): + if observed[field] != item[field]: + raise AssertionError( + f"{label} {item['key']} {field} mismatch: expected " + f"{item[field]!r}, got {observed[field]!r}" + ) + if item.get("versionId") is not None and observed["versionId"] != item["versionId"]: + raise AssertionError( + f"{label} {item['key']} version ID changed: " + f"{item['versionId']!r} -> {observed['versionId']!r}" + ) + return observed + + with concurrent.futures.ThreadPoolExecutor(max_workers=self.args.workers) as pool: + observed = list(pool.map(check, inventory)) + + def list_keys() -> set[str]: + return { + item["Key"] + for page in client.get_paginator("list_objects_v2").paginate(Bucket=bucket) + for item in page.get("Contents", []) + } + + listed = self.s3_retrying(f"LIST integrity objects for {label}", list_keys) + missing = sorted(item["key"] for item in inventory if item["key"] not in listed) + if missing: + raise AssertionError(f"{label} listing omitted keys: {missing[:10]!r}") + self.save_json(case_id, f"integrity-{safe_name(label)}.json", observed) + + def pool_snapshot( + self, + case_id: str, + label: str, + expected_count: int, + *, + require_active: bool = True, + ) -> tuple[Any, Any]: + deadline = time.monotonic() + min(self.args.active_timeout, 10) + attempt = 0 + while True: + attempt += 1 + suffix = "" if require_active else f"-{attempt}" + api = self.admin_json( + case_id, f"{label}{suffix}-admin-pool-list", "pools/list" + ) + rc_result = self.run_rc( + case_id, + f"{label}{suffix}-rc-pool-list", + ["admin", "pool", "list", self.rc_alias], + ) + rc = self.parse_json_output(rc_result, "rc admin pool list") + self.assert_no_secrets(api, f"{label} Admin pool response") + self.assert_no_secrets(rc, f"{label} rc pool response") + try: + assert_pool_parity( + rc, api, expected_count, require_active=require_active + ) + except PoolStatusError: + if time.monotonic() >= deadline: + raise + time.sleep(self.args.poll_interval) + continue + return api, rc + + def assert_pool_specific_status(self, case_id: str, expected_count: int) -> None: + api_list, _ = self.pool_snapshot(case_id, "specific-source", expected_count) + initial = validate_pool_inventory(api_list, expected_count) + stable_fields = ( + "id", "cmdline", "status", "decommissionStatus", "rebalanceStatus", "totalSize", + ) + + def stable_values(entry: dict[str, Any]) -> tuple[Any, ...]: + return tuple(entry[field] for field in stable_fields) + + for expected in initial: + pool_id = expected["id"] + deadline = time.monotonic() + min(self.args.active_timeout, 10) + attempt = 0 + while True: + attempt += 1 + suffix = "" if attempt == 1 else f"-{attempt}" + observed = [] + for selector, by_id, label in ( + (str(pool_id), True, f"pool-{pool_id}-status"), + (expected["cmdline"], False, f"pool-{pool_id}-expression-status"), + ): + query = urllib.parse.urlencode({"pool": selector, "by-id": str(by_id).lower()}) + api = self.admin_json( + case_id, f"admin-{label}{suffix}", "pools/status", query=query, + ) + self.assert_no_secrets(api, f"{label} Admin status") + observed.extend(validate_pool_inventory(api, 1)) + arguments = ["admin", "pool", "status", self.rc_alias, selector] + if by_id: + arguments.append("--by-id") + rc_result = self.run_rc(case_id, f"rc-{label}{suffix}", arguments) + rc = self.parse_json_output(rc_result, "rc admin pool status") + self.assert_no_secrets(rc, f"{label} rc status") + observed.extend(validate_pool_inventory(rc, 1)) + + if any(stable_values(item) != stable_values(expected) for item in observed): + raise AssertionError( + f"pool {pool_id} selector identity/state/total capacity disagrees with pool list: " + f"list={expected!r} observed={observed!r}" + ) + if all(item == expected for item in observed): + break + + # Disk usage can change between the sequential requests even + # without client writes. Reacquire the list and all selectors; + # never ignore capacity fields or accept a lasting mismatch. + remaining = deadline - time.monotonic() + if remaining <= 0: + raise AssertionError( + f"pool {pool_id} selector capacity snapshots did not converge: " + f"list={expected!r} observed={observed!r}" + ) + self.events.record( + "pool.selector.capacity.resample", case=case_id, poolId=pool_id, + attempt=attempt, expected=expected, observed=observed, + ) + time.sleep(min(self.args.poll_interval, remaining)) + if time.monotonic() >= deadline: + raise AssertionError(f"pool {pool_id} selector capacity snapshots did not converge before deadline") + refreshed_api = self.admin_json( + case_id, f"specific-source-pool-{pool_id}-resample-{attempt + 1}-admin-pool-list", + "pools/list", + ) + self.assert_no_secrets(refreshed_api, f"pool {pool_id} refreshed Admin pool list") + refreshed = validate_pool_inventory(refreshed_api, expected_count) + if tuple(map(stable_values, initial)) != tuple(map(stable_values, refreshed)): + raise AssertionError("pool inventory identity/state/total capacity changed during selector sampling") + expected = next(item for item in refreshed if item["id"] == pool_id) + + def start_rebalance(self, case_id: str, *, command: str = "rebalance") -> str: + deadline = time.monotonic() + min(self.args.active_timeout, 60) + attempt = 0 + while True: + attempt += 1 + arguments = ["admin", command, "start", self.rc_alias] + result = subprocess.run( + [self.rc_bin, "--json", "--no-color", *arguments], + env=self.rc_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + self.save_json( + case_id, + f"{command}-start-{attempt}.json", + { + "command": arguments, + "exitCode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + }, + ) + if result.returncode == 0: + document = self.parse_json_output( + result, f"rc admin {command} start" + ) + operation_id = ( + document.get("id") if isinstance(document, dict) else None + ) + if not isinstance(operation_id, str) or not operation_id: + raise AssertionError( + f"{command} start omitted operation ID: {document!r}" + ) + return operation_id + error_text = f"{result.stdout}\n{result.stderr}" + if ( + rc_failure_is_retryable(result.stdout, result.stderr) + and time.monotonic() < deadline + ): + time.sleep(self.args.poll_interval) + continue + raise AssertionError( + f"{command} start failed: {self.redact(error_text.strip())}" + ) + + def rebalance_status( + self, + case_id: str, + operation_id: str, + label: str, + *, + allow_stopped: bool = False, + ) -> Any: + deadline = time.monotonic() + min(self.args.active_timeout, 10) + attempt = 0 + last_states: tuple[Any, Any] | None = None + while True: + attempt += 1 + api = self.admin_json( + case_id, f"{label}-{attempt}-admin", "rebalance/status" + ) + rc_result = self.run_rc( + case_id, + f"{label}-{attempt}-rc", + ["admin", "rebalance", "status", self.rc_alias], + ) + rc = self.parse_json_output(rc_result, "rc admin rebalance status") + self.assert_no_secrets(api, f"{label} Admin rebalance response") + self.assert_no_secrets(rc, f"{label} rc rebalance response") + api_pools = rebalance_entries( + api, + operation_id, + allow_stopped=allow_stopped, + ) + rc_pools = rebalance_entries( + rc, + operation_id, + allow_stopped=allow_stopped, + ) + api_states = tuple( + (item.get("id"), item.get("status")) for item in api_pools + ) + rc_states = tuple( + (item.get("id"), item.get("status")) for item in rc_pools + ) + if api_states == rc_states: + return api + last_states = (rc_states, api_states) + if time.monotonic() >= deadline: + raise AssertionError( + f"rc/Admin rebalance state did not converge: {last_states!r}" + ) + time.sleep(self.args.poll_interval) + + def wait_rebalance( + self, + case_id: str, + operation_id: str, + state: str, + *, + label: str | None = None, + allow_fast_completion: bool = False, + ) -> Any: + predicates: dict[str, Callable[[Any, str], bool]] = { + "active": ( + rebalance_active_or_completed + if allow_fast_completion + else lambda document, expected: rebalance_active( + document, expected, require_progress=True + ) + ), + "complete": rebalance_complete, + "stopped": rebalance_stopped, + } + artifact_label = label or state + + def probe() -> tuple[bool, Any]: + status, _, body = self.signed_request( + self.args.endpoint, "GET", f"{ADMIN_PREFIX}/rebalance/status" + ) + if status != 200: + return False, {"httpStatus": status, "body": body.decode(errors="replace")[:500]} + document = json.loads(body) + try: + matched = predicates[state](document, operation_id) + except PoolStatusError: + self.save_json( + case_id, + f"rebalance-{artifact_label}-failed-status.json", + document, + ) + raise + return matched, document + + result = self.wait_until( + case_id, + f"rebalance {operation_id} {state}", + probe, + timeout=self.args.active_timeout if state == "active" else self.args.timeout, + ) + self.save_json(case_id, f"rebalance-{artifact_label}-final.json", result) + if ( + state == "active" + and allow_fast_completion + and rebalance_complete(result, operation_id) + ): + self.events.record( + "rebalance.fast-completion", + case=case_id, + operationId=operation_id, + evidence="terminal status carried nonzero movement progress", + ) + self.rebalance_status( + case_id, + operation_id, + f"rebalance-{artifact_label}-parity", + allow_stopped=state == "stopped", + ) + if state in {"complete", "stopped"}: + inventory, _ = self.pool_snapshot( + case_id, + f"rebalance-{artifact_label}-pool-inventory", + 2, + ) + observed = { + str(item.get("rebalanceStatus", "")).lower() + for item in pool_entries(inventory) + } + allowed = ( + {"completed", "none"} + if state == "complete" + else {"stopped", "completed", "none"} + ) + required = "completed" if state == "complete" else "stopped" + if not observed.issubset(allowed) or required not in observed: + raise AssertionError( + f"pool inventory disagrees with rebalance {state}: {observed}" + ) + return result + + def stop_rebalance(self, case_id: str, *, command: str = "rebalance") -> None: + self.run_rc( + case_id, + f"{command}-stop", + ["admin", command, "stop", self.rc_alias], + ) + + def start_decommission(self, case_id: str, pool_id: int = 0, *, label: str = "decommission-start") -> None: + self.capture_diagnostics(case_id, f"before-{label}") + deadline = time.monotonic() + min(self.args.active_timeout, 60) + attempt = 0 + while True: + attempt += 1 + command = [ + "admin", + "decommission", + "start", + self.rc_alias, + str(pool_id), + "--by-id", + ] + result = subprocess.run( + [self.rc_bin, "--json", "--no-color", *command], + env=self.rc_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + self.save_json( + case_id, + f"{label}-{attempt}.json", + { + "command": command, + "exitCode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + }, + ) + if result.returncode == 0: + return + error_text = f"{result.stdout}\n{result.stderr}" + if ( + rc_failure_is_retryable(result.stdout, result.stderr) + and time.monotonic() < deadline + ): + time.sleep(self.args.poll_interval) + continue + raise AssertionError( + f"decommission start failed: {self.redact(error_text.strip())}" + ) + + def decommission_status( + self, case_id: str, pool_id: int, label: str, *, + predicate: Callable[[Any, int], bool] | None = None, + ) -> Any: + deadline = time.monotonic() + min(self.args.active_timeout, 10) + attempt = 0 + fields = ("id", "cmdline", "status", "poolStatus") + last_states: tuple[Any, Any] | None = None + while True: + attempt += 1 + api = self.admin_json( + case_id, f"{label}-{attempt}-admin", "decommission/status" + ) + rc_result = self.run_rc( + case_id, + f"{label}-{attempt}-rc", + ["admin", "decommission", "status", self.rc_alias], + ) + rc = self.parse_json_output(rc_result, "rc admin decommission status") + self.assert_no_secrets(api, f"{label} Admin decommission response") + self.assert_no_secrets(rc, f"{label} rc decommission response") + api_entry = decommission_entry(api, pool_id) + rc_entry = decommission_entry(rc, pool_id) + api_state = tuple(api_entry.get(field) for field in fields) + rc_state = tuple(rc_entry.get(field) for field in fields) + if api_state == rc_state: + if predicate is not None and not all(predicate(document, pool_id) for document in (api, rc)): + raise PoolStatusError("rc/Admin decommission details violate the expected state") + return api + last_states = (rc_state, api_state) + if time.monotonic() >= deadline: + raise AssertionError( + f"rc/Admin decommission state did not converge: {last_states!r}" + ) + time.sleep(self.args.poll_interval) + + def wait_decommission( + self, + case_id: str, + pool_id: int, + state: str, + *, + label: str | None = None, + allow_fast_completion: bool = False, + ) -> Any: + predicates: dict[str, Callable[[Any, int], bool]] = { + "active": ( + decommission_active_or_completed + if allow_fast_completion + else lambda document, expected: decommission_active( + document, expected, require_progress=True + ) + ), + "complete": decommission_complete, + "canceled": decommission_canceled, + "cleared": decommission_cleared, + } + artifact_label = label or state + + def probe() -> tuple[bool, Any]: + status, _, body = self.signed_request( + self.args.endpoint, "GET", f"{ADMIN_PREFIX}/decommission/status" + ) + if status != 200: + return False, {"httpStatus": status, "body": body.decode(errors="replace")[:500]} + document = json.loads(body) + try: + matched = predicates[state](document, pool_id) + except PoolStatusError: + self.save_json( + case_id, + f"decommission-{artifact_label}-failed-status.json", + document, + ) + raise + return matched, document + + result = self.wait_until( + case_id, + f"decommission pool {pool_id} {state}", + probe, + timeout=self.args.active_timeout if state == "active" else self.args.timeout, + ) + self.save_json( + case_id, f"decommission-{artifact_label}-final.json", result + ) + if ( + state == "active" + and allow_fast_completion + and decommission_complete(result, pool_id) + ): + self.events.record( + "decommission.fast-completion", + case=case_id, + poolId=pool_id, + evidence="terminal status carried nonzero movement progress", + ) + self.decommission_status( + case_id, pool_id, f"decommission-{artifact_label}-parity" + ) + if state == "canceled": + self.wait_canceled_quiescence(case_id, pool_id, label=artifact_label) + result = self.decommission_status( + case_id, pool_id, f"decommission-{artifact_label}-quiescent" + ) + if not decommission_canceled(result, pool_id): + raise AssertionError("decommission left canceled state after quiescence") + self.save_json(case_id, f"decommission-{artifact_label}-final.json", result) + if state == "active": + self.capture_diagnostics(case_id, f"decommission-{artifact_label}") + if state in {"complete", "canceled", "cleared"}: + expected_count = len(pool_entries(result)) + require_active = state == "cleared" + inventory, _ = self.pool_snapshot( + case_id, + f"decommission-{artifact_label}-pool-inventory", + expected_count, + require_active=require_active, + ) + matches = [ + item for item in pool_entries(inventory) if item.get("id") == pool_id + ] + if len(matches) != 1: + raise AssertionError( + f"pool inventory omitted decommission pool {pool_id}" + ) + entry = matches[0] + expected_states = { + "complete": ("decommissioned", "complete", True), + "canceled": ("blocked", "canceled", True), + "cleared": ("active", "none", False), + } + pool_state, operation_state, has_info = expected_states[state] + if ( + str(entry.get("status", "")).lower() != pool_state + or str(entry.get("decommissionStatus", "")).lower() + != operation_state + or (entry.get("decommissionInfo") is not None) is not has_info + ): + raise AssertionError( + f"pool inventory disagrees with decommission {state}: {entry!r}" + ) + return result + + def cancel_decommission(self, case_id: str, pool_id: int = 0) -> None: + self.run_rc( + case_id, + "decommission-cancel", + ["admin", "decommission", "cancel", self.rc_alias, str(pool_id), "--by-id"], + ) + + def clear_decommission(self, case_id: str, pool_id: int = 0) -> None: + self.run_rc( + case_id, + "decommission-clear", + ["admin", "decommission", "clear", self.rc_alias, str(pool_id), "--by-id"], + ) + + def seed_and_expand( + self, + case_id: str, + *, + count: int | None = None, + size: int | None = None, + ) -> tuple[str, list[dict[str, Any]]]: + bucket = self.create_bucket(case_id) + inventory = self.put_inventory(case_id, bucket, count=count, size=size) + self.assert_inventory(case_id, self.client, bucket, inventory, "before-expansion") + self.census(case_id, "before-expansion") + self.apply_topology(case_id, "expanded") + self.pool_snapshot(case_id, "after-expansion", 2) + self.assert_inventory( + case_id, + self.retained_pool_client(), + bucket, + inventory, + "after-expansion-pool1-node0", + ) + return bucket, inventory + + def case_status(self, case_id: str) -> None: + bucket = self.create_bucket(case_id) + body = deterministic_payload(case_id, 8193) + self.client.put_object(Bucket=bucket, Key="status-probe.bin", Body=body) + initial, _ = self.pool_snapshot(case_id, "initial", 1) + self.assert_pool_specific_status(case_id, 1) + self.census(case_id, "initial") + self.apply_topology(case_id, "expanded") + expanded, _ = self.pool_snapshot(case_id, "expanded", 2) + self.assert_pool_specific_status(case_id, 2) + self.census(case_id, "expanded") + if pool_entries(initial)[0]["cmdline"] == pool_entries(expanded)[1]["cmdline"]: + raise AssertionError("expansion pool reused the seed pool command line") + initial_identity = canonical_pool(pool_entries(initial)[0]) + expanded_seed = canonical_pool(pool_entries(expanded)[0]) + if ( + initial_identity["id"], + initial_identity["cmdline"], + ) != ( + expanded_seed["id"], + expanded_seed["cmdline"], + ): + raise AssertionError( + "seed pool identity or ordering changed during expansion: " + f"{initial_identity!r} -> {expanded_seed!r}" + ) + for endpoint in self.retained_pool_endpoints(): + def read_probe() -> bytes: + response = self.node_clients[endpoint].get_object( + Bucket=bucket, Key="status-probe.bin" + ) + return response["Body"].read() + + observed = self.s3_retrying( + f"GET status probe through {endpoint}", read_probe + ) + if observed != body: + raise AssertionError(f"status probe changed through {endpoint}") + + def case_expand_and_rebalance(self, case_id: str) -> None: + bucket, inventory = self.seed_and_expand(case_id) + operation_id = self.start_rebalance(case_id, command="expand") + self.wait_rebalance( + case_id, + operation_id, + "active", + allow_fast_completion=True, + ) + self.wait_rebalance(case_id, operation_id, "complete") + self.census(case_id, "after-rebalance") + self.assert_inventory(case_id, self.client, bucket, inventory, "after-rebalance") + + def case_expansion_restart_persistence(self, case_id: str) -> None: + bucket, inventory = self.seed_and_expand( + case_id, size=active_fixture_bytes(self.args.fixture_bytes) + ) + before, _ = self.pool_snapshot(case_id, "before-restart", 2) + operation_id = self.start_rebalance(case_id) + active = self.wait_rebalance(case_id, operation_id, "active") + active_counters = self.rebalance_counters(active) + self.restart_topology(case_id, "expanded") + after, _ = self.pool_snapshot( + case_id, "after-restart", 2, require_active=False + ) + unexpected_states = { + str(item.get("status", "")).lower() + for item in pool_entries(after) + }.difference({"active", "rebalancing"}) + if unexpected_states: + raise AssertionError( + f"unexpected pool states after rebalance restart: {unexpected_states}" + ) + before_identity = tuple((item["id"], item["cmdline"]) for item in pool_entries(before)) + after_identity = tuple((item["id"], item["cmdline"]) for item in pool_entries(after)) + if before_identity != after_identity: + raise AssertionError( + f"pool identity changed across restart: {before_identity!r} -> {after_identity!r}" + ) + completed = self.wait_rebalance(case_id, operation_id, "complete") + self.pool_snapshot(case_id, "after-rebalance-restart-complete", 2) + completed_counters = self.rebalance_counters(completed) + if any( + final < initial + for initial, final in zip( + active_counters, completed_counters, strict=True + ) + ): + raise AssertionError( + "rebalance counters regressed across restart: " + f"{active_counters} -> {completed_counters}" + ) + for index in range(len(self.args.node_endpoint)): + self.assert_inventory( + case_id, + self.node_clients[self.args.node_endpoint[index]], + bucket, + inventory, + f"restart-node-{index}", + ) + + @staticmethod + def rebalance_counters(document: Any) -> tuple[int, int, int]: + counters = [0, 0, 0] + for pool in document.get("pools", []): + progress = pool.get("progress") or {} + for index, field in enumerate(("objects", "versions", "bytes")): + counters[index] += int(progress.get(field, 0)) + return tuple(counters) + + def case_rebalance_stop_resume(self, case_id: str) -> None: + bucket, inventory = self.seed_and_expand( + case_id, size=active_fixture_bytes(self.args.fixture_bytes) + ) + first_id = self.start_rebalance(case_id) + self.wait_rebalance(case_id, first_id, "active") + self.stop_rebalance(case_id) + self.wait_rebalance( + case_id, + first_id, + "stopped", + label="stopped-before-restart", + ) + self.assert_inventory(case_id, self.client, bucket, inventory, "after-rebalance-stop") + self.restart_topology(case_id, "expanded") + self.wait_rebalance( + case_id, + first_id, + "stopped", + label="stopped-after-restart", + ) + second_id = self.start_rebalance(case_id) + if second_id == first_id: + raise AssertionError("rebalance resume reused the stopped operation ID") + self.wait_rebalance(case_id, second_id, "complete") + self.assert_inventory(case_id, self.client, bucket, inventory, "after-rebalance-resume") + + def exercise_live_io( + self, + case_id: str, + bucket: str, + *, + prefix: str, + client: Any | None = None, + ) -> list[dict[str, Any]]: + client = client or self.client + + def live_put(index: int) -> dict[str, Any]: + key = f"{prefix}/put-{index:02d}.bin" + body = deterministic_payload(f"{case_id}:{key}", 64 * 1024) + response = self.put_retrying(client, bucket, key, body, {"phase": prefix}) + return { + "key": key, + "size": len(body), + "sha256": hashlib.sha256(body).hexdigest(), + "etag": str(response.get("ETag", "")).strip('"'), + "metadata": {"phase": prefix}, + "versionId": response.get("VersionId"), + } + + with concurrent.futures.ThreadPoolExecutor( + max_workers=self.args.workers + ) as pool: + committed = list(pool.map(live_put, range(self.args.live_objects))) + committed.sort(key=lambda item: item["key"]) + first = committed[0] + head = self.s3_retrying( + f"HEAD live object for {prefix}", + lambda: client.head_object(Bucket=bucket, Key=first["key"]), + ) + if head.get("ContentLength") != first["size"]: + raise AssertionError("HEAD returned the wrong live-object length") + + def read_live_object() -> bytes: + response = client.get_object(Bucket=bucket, Key=first["key"]) + return response["Body"].read() + + got = self.s3_retrying(f"GET live object for {prefix}", read_live_object) + if hashlib.sha256(got).hexdigest() != first["sha256"]: + raise AssertionError("GET returned corrupt live-object bytes") + listed = self.s3_retrying( + f"LIST live objects for {prefix}", + lambda: client.list_objects_v2(Bucket=bucket, Prefix=f"{prefix}/"), + ).get("Contents", []) + if not any(item.get("Key") == first["key"] for item in listed): + raise AssertionError("LIST omitted a committed live object") + copy_key = f"{prefix}/copy.bin" + copy_response = self.s3_retrying( + f"COPY live object for {prefix}", + lambda: client.copy_object( + Bucket=bucket, + Key=copy_key, + CopySource={"Bucket": bucket, "Key": first["key"]}, + Metadata={"phase": prefix}, + MetadataDirective="REPLACE", + ), + ) + committed.append( + { + "key": copy_key, + "size": first["size"], + "sha256": first["sha256"], + "etag": str(copy_response.get("CopyObjectResult", {}).get("ETag", "")).strip('"'), + "metadata": {"phase": prefix}, + "versionId": copy_response.get("VersionId"), + } + ) + delete_key = f"{prefix}/delete-control.bin" + self.s3_retrying( + f"PUT delete control for {prefix}", + lambda: client.put_object(Bucket=bucket, Key=delete_key, Body=b"delete-me"), + ) + self.s3_retrying( + f"DELETE control for {prefix}", + lambda: client.delete_object(Bucket=bucket, Key=delete_key), + ) + try: + self.s3_retrying( + f"HEAD deleted control for {prefix}", + lambda: client.head_object(Bucket=bucket, Key=delete_key), + ) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") not in { + "404", + "NoSuchKey", + "NotFound", + }: + raise + else: + raise AssertionError("DELETE control object remains visible") + self.save_json(case_id, f"{prefix}-live-manifest.json", committed) + return committed + + def case_rebalance_live_io(self, case_id: str) -> None: + bucket, inventory = self.seed_and_expand( + case_id, size=active_fixture_bytes(self.args.fixture_bytes) + ) + operation_id = self.start_rebalance(case_id) + self.wait_rebalance(case_id, operation_id, "active") + live = self.exercise_live_io(case_id, bucket, prefix="during-rebalance") + status, _, body = self.signed_request( + self.args.endpoint, "GET", f"{ADMIN_PREFIX}/rebalance/status" + ) + if status != 200 or not rebalance_active(json.loads(body), operation_id, require_progress=True): + raise AssertionError("rebalance did not remain active across the live-I/O window") + self.wait_rebalance(case_id, operation_id, "complete") + self.assert_inventory(case_id, self.client, bucket, inventory + live, "rebalance-live-final") + + def case_decommission(self, case_id: str) -> None: + bucket, inventory = self.seed_and_expand(case_id) + self.start_decommission(case_id) + self.wait_decommission( + case_id, 0, "active", allow_fast_completion=True + ) + self.wait_decommission(case_id, 0, "complete") + self.census(case_id, "after-decommission") + self.assert_inventory( + case_id, + self.retire_decommissioned_pool(case_id), + bucket, + inventory, + "decommission-survivor", + ) + + def case_decommission_cancel_clear_resume(self, case_id: str) -> None: + bucket, inventory = self.seed_and_expand( + case_id, size=active_fixture_bytes(self.args.fixture_bytes) + ) + self.start_decommission(case_id) + self.wait_decommission(case_id, 0, "active") + self.cancel_decommission(case_id) + canceled_before = self.wait_decommission( + case_id, + 0, + "canceled", + label="canceled-before-restart", + ) + self.assert_inventory(case_id, self.client, bucket, inventory, "after-decommission-cancel") + self.restart_topology(case_id, "expanded") + canceled_after = self.wait_decommission( + case_id, + 0, + "canceled", + label="canceled-after-restart", + ) + if self.decommission_counters(canceled_before, 0) != self.decommission_counters(canceled_after, 0): + raise AssertionError("canceled decommission progress changed across restart") + self.clear_decommission(case_id) + self.wait_decommission( + case_id, + 0, + "cleared", + label="cleared-before-restart", + ) + self.restart_topology(case_id, "expanded") + self.wait_decommission( + case_id, + 0, + "cleared", + label="cleared-after-restart", + ) + self.start_decommission(case_id) + self.wait_decommission(case_id, 0, "complete") + self.assert_inventory( + case_id, self.retire_decommissioned_pool(case_id), bucket, inventory, "after-decommission-resume" + ) + + @staticmethod + def decommission_counters(document: Any, pool_id: int) -> tuple[int, int]: + info = decommission_entry(document, pool_id).get("decommissionInfo", {}) + return int(info.get("objectsDecommissioned", 0)), int(info.get("bytesDecommissioned", 0)) + + def case_decommission_restart_recovery(self, case_id: str) -> None: + bucket, inventory = self.seed_and_expand( + case_id, size=active_fixture_bytes(self.args.fixture_bytes) + ) + self.start_decommission(case_id) + before = self.wait_decommission(case_id, 0, "active") + before_counters = self.decommission_counters(before, 0) + self.restart_topology(case_id, "expanded") + final = self.wait_decommission(case_id, 0, "complete") + final_counters = self.decommission_counters(final, 0) + if any(after < prior for prior, after in zip(before_counters, final_counters, strict=True)): + raise AssertionError( + f"decommission counters regressed across restart: {before_counters} -> {final_counters}" + ) + self.assert_inventory( + case_id, + self.retire_decommissioned_pool(case_id), + bucket, + inventory, + "decommission-restart-final", + ) + + def assert_expanded_delete_marker_removal(self, case_id: str, bucket: str) -> None: + if self.current_layout != "expanded": + raise AssertionError("delete-marker API probes require the expanded topology") + for operation in ("single", "batch"): + key = f"delete-api-probe/{operation}.bin" + payload = b"historical data behind a delete marker" + original = self.client.put_object(Bucket=bucket, Key=key, Body=payload) + marker = self.client.delete_object(Bucket=bucket, Key=key) + marker_id = marker.get("VersionId") + if not original.get("VersionId") or not marker_id or marker.get("DeleteMarker") is not True: + raise AssertionError("delete-marker API probe did not create versioned history") + self.save_json(case_id, f"delete-api-{operation}-fixture.json", {"original": original, "marker": marker}) + for method in (self.client.get_object, self.client.head_object): + try: + method(Bucket=bucket, Key=key, VersionId=marker_id) + except ClientError as exc: + if (exc.response.get("Error", {}).get("Code") not in {"405", "MethodNotAllowed"} + or exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") != 405): + raise + else: + raise AssertionError("GET/HEAD of an explicit delete marker must remain 405") + if operation == "single": + response = self.client.delete_object(Bucket=bucket, Key=key, VersionId=marker_id) + self.save_json(case_id, "delete-api-single-response.json", response) + if response.get("VersionId") != marker_id or response.get("DeleteMarker") is not True: + raise AssertionError("single delete did not acknowledge the exact marker version") + else: + response = self.client.delete_objects( + Bucket=bucket, Delete={"Objects": [{"Key": key, "VersionId": marker_id}], "Quiet": False} + ) + self.save_json(case_id, "delete-api-batch-response.json", response) + deleted = response.get("Deleted", []) + if (response.get("Errors") or len(deleted) != 1 or deleted[0].get("Key") != key + or deleted[0].get("VersionId") != marker_id or deleted[0].get("DeleteMarker") is not True): + raise AssertionError("batch delete did not acknowledge the exact marker version") + versions = self.client.list_object_versions(Bucket=bucket, Prefix=key) + self.save_json(case_id, f"delete-api-{operation}-remaining.json", versions) + if any(item.get("Key") == key for item in versions.get("DeleteMarkers", [])): + raise AssertionError("marker version remains after successful delete acknowledgement") + remaining = [item for item in versions.get("Versions", []) if item.get("Key") == key] + if (len(remaining) != 1 or remaining[0].get("VersionId") != original["VersionId"] + or remaining[0].get("IsLatest") is not True): + raise AssertionError("marker removal changed the historical version identity") + if self.client.get_object(Bucket=bucket, Key=key)["Body"].read() != payload: + raise AssertionError("marker removal damaged the historical payload") + + def case_decommission_versioned_integrity(self, case_id: str) -> None: + bucket = self.create_bucket(case_id) + self.client.put_bucket_versioning( + Bucket=bucket, VersioningConfiguration={"Status": "Enabled"} + ) + key = "history/versioned.bin" + payloads = [ + deterministic_payload(f"{case_id}:v1", 37 * 1024), + deterministic_payload(f"{case_id}:v2", 71 * 1024), + ] + version_ids: list[str] = [] + for payload in payloads: + response = self.client.put_object(Bucket=bucket, Key=key, Body=payload) + version_id = response.get("VersionId") + if not isinstance(version_id, str) or not version_id: + raise AssertionError("versioned PUT omitted VersionId") + version_ids.append(version_id) + deleted = self.client.delete_object(Bucket=bucket, Key=key) + marker_id = deleted.get("VersionId") + if deleted.get("DeleteMarker") is not True or not marker_id: + raise AssertionError("versioned DELETE did not create a delete marker") + self.save_json(case_id, "version-fixture.json", { + "versionIds": version_ids, "deleteMarkerVersionId": marker_id, + "payloads": [{"size": len(payload), "sha256": hashlib.sha256(payload).hexdigest()} for payload in payloads], + }) + inventory = self.put_inventory(case_id, bucket, count=max(32, self.args.fixture_objects // 2)) + self.apply_topology(case_id, "expanded") + self.assert_expanded_delete_marker_removal(case_id, bucket) + self.start_decommission(case_id) + self.wait_decommission(case_id, 0, "complete") + checker = self.retire_decommissioned_pool(case_id) + versions = checker.list_object_versions(Bucket=bucket, Prefix=key) + self.save_json( + case_id, "version-timeline.json", + {"versionIds": version_ids, "deleteMarkerVersionId": marker_id, "observed": versions}, + ) + observed_versions = {item["VersionId"] for item in versions.get("Versions", []) if item.get("Key") == key} + observed_markers = { + item["VersionId"] + for item in versions.get("DeleteMarkers", []) + if item.get("Key") == key + } + latest_markers = { + item["VersionId"] + for item in versions.get("DeleteMarkers", []) + if item.get("Key") == key and item.get("IsLatest") is True + } + if ( + set(version_ids) != observed_versions + or observed_markers != {marker_id} + or latest_markers != {marker_id} + ): + raise AssertionError( + "version timeline changed: " + f"versions={observed_versions}, markers={observed_markers}, " + f"latestMarkers={latest_markers}" + ) + for version_id, expected in zip(version_ids, payloads, strict=True): + body = checker.get_object( + Bucket=bucket, Key=key, VersionId=version_id + )["Body"].read() + if body != expected: + raise AssertionError(f"historical version {version_id} changed") + marker_modified = next( + item.get("LastModified") for item in versions["DeleteMarkers"] + if item.get("Key") == key and item.get("VersionId") == marker_id + ) + self.assert_delete_marker_reads(case_id, checker, bucket, key, marker_id, marker_modified) + self.assert_inventory(case_id, checker, bucket, inventory, "versioned-decommission-final") + + def assert_delete_marker_reads( + self, case_id: str, checker: Any, bucket: str, key: str, + marker_id: str, marker_modified: datetime | None, + ) -> None: + for operation, explicit, filename in ( + ("get_object", False, "latest-delete-marker-response.json"), + ("head_object", False, "latest-delete-marker-head-response.json"), + ("get_object", True, "explicit-delete-marker-get-response.json"), + ("head_object", True, "explicit-delete-marker-head-response.json"), + ): + request = {"Bucket": bucket, "Key": key} + if explicit: + request["VersionId"] = marker_id + try: + response = getattr(checker, operation)(**request) + except ClientError as exc: + self.save_json(case_id, filename, exc.response) + expected_status = 405 if explicit else 404 + expected_codes = {"405", "MethodNotAllowed"} if explicit else {"404", "NoSuchKey", "NotFound"} + if exc.response.get("Error", {}).get("Code") not in expected_codes: + raise + metadata = exc.response.get("ResponseMetadata", {}) + if metadata.get("HTTPStatusCode") != expected_status: + raise AssertionError(f"{operation} delete marker returned the wrong HTTP status") + headers = metadata.get("HTTPHeaders", {}) + if str(headers.get("x-amz-delete-marker", "")).lower() != "true": + raise AssertionError( + f"{'explicit' if explicit else 'latest'} version returned {expected_status} " + "without delete-marker identity" + ) + if headers.get("x-amz-version-id") != marker_id: + raise AssertionError("delete-marker response returned the wrong VersionId") + if explicit: + if not isinstance(marker_modified, datetime) or marker_modified.utcoffset() is None: + raise AssertionError("version listing omitted the marker LastModified timestamp") + expected_date = format_datetime(marker_modified.astimezone(UTC).replace(microsecond=0), usegmt=True) + if headers.get("last-modified") != expected_date: + raise AssertionError("explicit delete-marker response returned the wrong Last-Modified") + else: + body = response.get("Body") + if body is not None: + body.close() + raise AssertionError("latest or explicit delete marker no longer hides the versioned object") + + def multipart_request( + self, case_id: str, label: str, client: Any, operation: str, **params: Any, + ) -> Any: + request = dict(params) + if "Body" in request: + body = request.pop("Body") + request.update(bodyBytes=len(body), bodySha256=hashlib.sha256(body).hexdigest()) + evidence = {"operation": operation, "request": request, "startedAt": utc_now()} + try: + response = getattr(client, operation)(**params) + except Exception as exc: + evidence.update(error=f"{type(exc).__name__}: {exc}", response=getattr(exc, "response", None)) + self.save_json(case_id, f"mpu-{label}.json", evidence) + raise + evidence["response"] = {key: value for key, value in response.items() if key != "Body"} + try: + self.save_json(case_id, f"mpu-{label}.json", evidence) + except Exception: + if "Body" in response: + response["Body"].close() + raise + return response + + def multipart_entries( + self, case_id: str, label: str, client: Any, operation: str, **params: Any, + ) -> list[dict[str, Any]]: + field = "Uploads" if operation == "list_multipart_uploads" else "Parts" + entries: list[dict[str, Any]] = [] + identities: set[Any] = set() + deadline = time.monotonic() + self.args.active_timeout + last_truncated: bool | None = None + try: + pages = client.get_paginator(operation).paginate(**params, PaginationConfig={"PageSize": 1}) + for index, page in enumerate(pages, 1): + self.save_json(case_id, f"mpu-{label}-page-{index}.json", {"request": params, "response": page}) + if index > 64 or time.monotonic() >= deadline: + raise AssertionError("MPU fixture pagination exceeded its bounded budget") + if (not isinstance(page, dict) or type(page.get("IsTruncated")) is not bool + or not isinstance(page.get(field, []), list)): + raise AssertionError("malformed MPU listing page") + last_truncated = page["IsTruncated"] + for name in ("Bucket", "Key", "UploadId"): + if name in params and name in page and page[name] != params[name]: + raise AssertionError("MPU listing response has the wrong identity") + for item in page.get(field, []): + if not isinstance(item, dict): + raise AssertionError("malformed MPU listing entry") + if field == "Uploads": + identity = (item.get("Key"), item.get("UploadId")) + valid = all(isinstance(value, str) and value for value in identity) + else: + identity = item.get("PartNumber") + valid = ( + type(identity) is int and 1 <= identity <= 10000 + and type(item.get("Size")) is int and item["Size"] >= 0 + and isinstance(item.get("ETag"), str) and bool(item["ETag"]) + ) + if not valid or identity in identities: + raise AssertionError("invalid or duplicate MPU listing identity") + identities.add(identity) + entries.append(item) + except Exception as exc: + self.save_json(case_id, f"mpu-{label}-error.json", { + "operation": operation, "request": params, "error": f"{type(exc).__name__}: {exc}", + "response": getattr(exc, "response", None), + }) + raise + if last_truncated is None: + raise AssertionError("MPU paginator returned no response") + if last_truncated: + raise AssertionError("MPU paginator stopped before the final page") + return entries + + def assert_multipart_uploads( + self, case_id: str, bucket: str, uploads: list[dict[str, Any]], label: str, + ) -> None: + expected = {(item["key"], item["uploadId"]) for item in uploads} + for index, endpoint in enumerate(self.args.node_endpoint): + client = self.node_clients[endpoint] + prefix = f"{label}-node-{index}" + listed = self.multipart_entries(case_id, prefix, client, "list_multipart_uploads", Bucket=bucket) + if {(item["Key"], item["UploadId"]) for item in listed} != expected: + raise AssertionError("pending MPU inventory changed or contains an unexpected upload") + for upload in uploads: + parts = self.multipart_entries( + case_id, f"{prefix}-{safe_name(upload['key'])}-parts", client, "list_parts", + Bucket=bucket, Key=upload["key"], UploadId=upload["uploadId"], + ) + fields = ("PartNumber", "ETag", "Size") + observed_parts = [tuple(item[name] for name in fields) for item in parts] + expected_parts = [tuple(item[name] for name in fields) for item in upload["parts"]] + if observed_parts != expected_parts: + raise AssertionError("pending MPU part number, ETag or length changed") + + def wait_multipart_drain_blocked(self, case_id: str, bucket: str, since: str) -> None: + attempt = 0 + + def probe() -> tuple[bool, Any]: + nonlocal attempt + attempt += 1 + status, _, body = self.signed_request(self.args.endpoint, "GET", f"{ADMIN_PREFIX}/decommission/status") + if status != 200: + raise PoolStatusError(f"MPU drain status returned HTTP {status}") + document = json.loads(body) + try: + return decommission_multipart_blocked(document, 0), document + finally: + self.save_json(case_id, f"mpu-drain-status-{attempt}.json", document) + result = self.wait_until(case_id, "owned MPU to block final decommission", probe) + self.save_json(case_id, "mpu-drain-blocked.json", result) + node_attempt = 0 + + def nodes_probe() -> tuple[bool, Any]: + nonlocal node_attempt + node_attempt += 1 + documents = [ + self.admin_json(case_id, f"mpu-drain-blocked-{node_attempt}-node-{index}", "decommission/status", endpoint=endpoint) + for index, endpoint in enumerate(self.args.node_endpoint) + ] + matched = [decommission_multipart_blocked(document, 0) for document in documents] + return all(matched), documents + + self.wait_until(case_id, "MPU drain failure on every node", nodes_probe, timeout=min(self.args.active_timeout, 10)) + parity = self.decommission_status(case_id, 0, "mpu-drain-blocked-parity", predicate=decommission_multipart_blocked) + if not decommission_multipart_blocked(parity, 0): + raise PoolStatusError("MPU drain failure disagrees with rc/Admin parity") + reference = self.hook( + case_id, "mpu-drain-journal", self.args.diagnostic_hook, + ("--phase", "mpu-drain-failure", "--since", since), timeout=60, + ) + journal = json.loads((self.evidence_dir / reference).read_text(encoding="utf-8")) + records = multipart_drain_failure_records(journal, bucket=bucket, since=since, node_count=len(self.args.node_endpoint)) + self.save_json(case_id, "mpu-drain-reason.json", records) + + def case_decommission_multipart_integrity(self, case_id: str) -> None: + if not self.args.diagnostic_hook: + raise RuntimeError("MPU drain verification requires the read-only journal hook") + bucket = self.create_bucket(case_id) + inventory = self.put_inventory( + case_id, bucket, count=max(32, self.args.fixture_objects // 2), + size=active_fixture_bytes(self.args.fixture_bytes), + ) + first = deterministic_payload(f"{case_id}:multipart:first", 5 * 1024 * 1024) + second = deterministic_payload(f"{case_id}:multipart:second", 1024 * 1024) + + def part(upload: dict[str, Any], number: int, data: bytes, client: Any) -> None: + response = self.multipart_request( + case_id, f"{upload['kind']}-part-{number}", client, "upload_part", + Bucket=bucket, Key=upload["key"], UploadId=upload["uploadId"], PartNumber=number, Body=data, + ) + if not isinstance(response.get("ETag"), str) or not response["ETag"]: + raise AssertionError("MPU UploadPart omitted its ETag") + upload["parts"].append({ + "PartNumber": number, "ETag": response["ETag"], "Size": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + }) + + def complete(upload: dict[str, Any], client: Any) -> None: + parts = [{name: item[name] for name in ("ETag", "PartNumber")} for item in upload["parts"]] + response = self.multipart_request( + case_id, f"{upload['kind']}-complete", client, "complete_multipart_upload", + Bucket=bucket, Key=upload["key"], UploadId=upload["uploadId"], MultipartUpload={"Parts": parts}, + ) + if not isinstance(response.get("ETag"), str) or not response["ETag"]: + raise AssertionError("MPU Complete omitted its ETag") + upload["etag"] = response["ETag"] + + uploads = [] + for kind in ("completed", "pending", "abort-active", "abort-failed", "resume-cleared"): + key = f"multipart/{kind}.bin" + created = self.multipart_request( + case_id, f"{kind}-create", self.client, "create_multipart_upload", + Bucket=bucket, Key=key, Metadata={"kind": kind}, + ) + if not isinstance(created.get("UploadId"), str) or not created["UploadId"]: + raise AssertionError("MPU creation omitted its UploadId") + upload = {"kind": kind, "key": key, "uploadId": created["UploadId"], "parts": []} + uploads.append(upload) + part(upload, 1, first, self.client) + if kind == "completed": + part(upload, 2, second, self.client) + complete(upload, self.client) + completed, pending, abort_active, abort_failed, resume_cleared = uploads + self.save_json(case_id, "multipart-fixture.json", {"bucket": bucket, "uploads": uploads}) + self.apply_topology(case_id, "expanded") + self.assert_multipart_uploads(case_id, bucket, uploads[1:], "before-start") + since = utc_now() + self.start_decommission(case_id) + self.wait_decommission(case_id, 0, "active", label="mpu-active") + # Both ends of the S3 window must be observed active. A fast terminal + # transition is a coverage failure, never permission to resume afterward. + self.assert_multipart_uploads(case_id, bucket, uploads[1:], "active") + client = self.retained_pool_client() + part(pending, 2, second, client) + self.assert_multipart_uploads(case_id, bucket, uploads[1:], "active-two-parts") + complete(pending, client) + self.multipart_request( + case_id, "abort-active", client, "abort_multipart_upload", + Bucket=bucket, Key=abort_active["key"], UploadId=abort_active["uploadId"], + ) + active = self.decommission_status(case_id, 0, "mpu-after-active-io") + if not decommission_active(active, 0, require_progress=True): + raise AssertionError("decommission did not remain active across MPU continuation") + self.save_json(case_id, "multipart-after-active-io.json", {"bucket": bucket, "uploads": uploads}) + + # Retain owned uploads to exercise the gate and both recovery paths. + self.wait_multipart_drain_blocked(case_id, bucket, since) + self.assert_multipart_uploads(case_id, bucket, [abort_failed, resume_cleared], "blocked") + self.multipart_request( + case_id, "abort-failed", client, "abort_multipart_upload", + Bucket=bucket, Key=abort_failed["key"], UploadId=abort_failed["uploadId"], + ) + self.assert_multipart_uploads(case_id, bucket, [resume_cleared], "before-clear") + self.clear_decommission(case_id) + self.wait_decommission(case_id, 0, "cleared", label="mpu-drained-clear") + self.assert_multipart_uploads(case_id, bucket, [resume_cleared], "after-clear") + # Clear does not abort MPU data. Completing this source-owned UploadId + # before retry also supplies real movement work for its strict counters. + part(resume_cleared, 2, second, client) + self.assert_multipart_uploads(case_id, bucket, [resume_cleared], "cleared-two-parts") + complete(resume_cleared, client) + self.assert_multipart_uploads(case_id, bucket, [], "drained-before-retry") + self.save_json(case_id, "multipart-after-clear.json", {"bucket": bucket, "uploads": uploads}) + self.start_decommission(case_id, label="mpu-drained-retry") + self.wait_decommission(case_id, 0, "complete", label="mpu-drained-complete") + checker = self.retire_decommissioned_pool(case_id) + expected_body = first + second + for upload in (completed, pending, resume_cleared): + head = self.multipart_request( + case_id, f"{upload['kind']}-final-head", checker, "head_object", Bucket=bucket, Key=upload["key"], + ) + if ( + head.get("ContentLength") != len(expected_body) + or str(head.get("ETag", "")).strip('"') != upload["etag"].strip('"') + or head.get("Metadata") != {"kind": upload["kind"]} + ): + raise AssertionError("multipart length, ETag or metadata changed after decommission") + response = self.multipart_request( + case_id, f"{upload['kind']}-final-get", checker, "get_object", Bucket=bucket, Key=upload["key"], + ) + stream = response["Body"] + try: + body = stream.read() + finally: + stream.close() + self.save_json(case_id, f"mpu-{upload['kind']}-final-content.json", { + "bytes": len(body), "sha256": hashlib.sha256(body).hexdigest(), + }) + if body != expected_body: + raise AssertionError("multipart payload changed after decommission") + if self.multipart_entries(case_id, "retired-pending", checker, "list_multipart_uploads", Bucket=bucket): + raise AssertionError("drained multipart uploads reappeared after retirement") + self.assert_inventory(case_id, checker, bucket, inventory, "multipart-decommission-final") + self.save_json(case_id, "multipart-summary.json", { + "bucket": bucket, "uploads": uploads, "sha256": hashlib.sha256(expected_body).hexdigest(), + "drainRecoveryVerified": True, + }) + + def case_decommission_live_io(self, case_id: str) -> None: + bucket, inventory = self.seed_and_expand( + case_id, size=active_fixture_bytes(self.args.fixture_bytes) + ) + self.start_decommission(case_id) + self.wait_decommission(case_id, 0, "active") + clients = [self.node_clients[endpoint] for endpoint in self.args.node_endpoint] + + def live_put(index: int) -> dict[str, Any]: + client = clients[index % len(clients)] + key = f"during-decommission/concurrent-{index:02d}.bin" + body = deterministic_payload(f"{case_id}:{key}", 64 * 1024) + response = self.put_retrying(client, bucket, key, body, {"phase": "decommission"}) + return { + "key": key, + "size": len(body), + "sha256": hashlib.sha256(body).hexdigest(), + "etag": str(response.get("ETag", "")).strip('"'), + "metadata": {"phase": "decommission"}, + "versionId": response.get("VersionId"), + } + + with concurrent.futures.ThreadPoolExecutor(max_workers=self.args.workers) as pool: + live = list(pool.map(live_put, range(self.args.live_objects))) + live.extend( + self.exercise_live_io( + case_id, + bucket, + prefix="during-decommission-serial", + client=self.retained_pool_client(), + ) + ) + status, _, body = self.signed_request( + self.args.endpoint, "GET", f"{ADMIN_PREFIX}/decommission/status" + ) + if status != 200 or not decommission_active(json.loads(body), 0, require_progress=True): + raise AssertionError("decommission did not remain active across concurrent I/O") + self.wait_decommission(case_id, 0, "complete") + self.assert_inventory( + case_id, + self.retire_decommissioned_pool(case_id), + bucket, + inventory + live, + "decommission-live-final", + ) + + def case_retired_config_removal(self, case_id: str) -> None: + bucket, inventory = self.seed_and_expand(case_id) + self.start_decommission(case_id) + checker = self.retire_decommissioned_pool(case_id) + self.assert_inventory(case_id, checker, bucket, inventory, "retired-config-final") + payload = b"written after retired pool removal" + self.client.put_object(Bucket=bucket, Key="after-removal.bin", Body=payload) + observed = self.client.get_object(Bucket=bucket, Key="after-removal.bin")["Body"].read() + if observed != payload: + raise AssertionError("post-removal object round-trip failed") + self.census(case_id, "after-retired-config-removal") + + def case_operation_guards(self, case_id: str) -> None: + bucket = self.create_bucket(case_id) + inventory = self.put_inventory( + case_id, + bucket, + size=active_fixture_bytes(self.args.fixture_bytes), + ) + self.run_rc( + case_id, + "unknown-pool-status", + ["admin", "pool", "status", self.rc_alias, "999999", "--by-id"], + expect_success=False, + expected_error_code="InvalidArgument", + ) + self.run_rc( + case_id, + "unknown-pool-decommission", + ["admin", "decommission", "start", self.rc_alias, "999999", "--by-id"], + expect_success=False, + expected_error_code="InvalidArgument", + ) + self.run_rc( + case_id, + "unknown-pool-decommission-status", + ["admin", "decommission", "status", self.rc_alias, "999999", "--by-id"], + expect_success=False, + expected_error_code="InvalidArgument", + ) + for operation in ("cancel", "clear"): + self.run_rc( + case_id, + f"unknown-pool-decommission-{operation}", + [ + "admin", + "decommission", + operation, + self.rc_alias, + "999999", + "--by-id", + ], + expect_success=False, + expected_error_code="InvalidArgument", + ) + self.run_rc( + case_id, + "only-pool-decommission", + ["admin", "decommission", "start", self.rc_alias, "0", "--by-id"], + expect_success=False, + expected_error_code="InvalidArgument", + ) + self.run_rc( + case_id, + "single-pool-rebalance", + ["admin", "rebalance", "start", self.rc_alias], + expect_success=False, + expected_error_code="NotImplemented", + ) + for operation in ("cancel", "clear"): + self.run_rc( + case_id, + f"idle-pool-decommission-{operation}", + [ + "admin", + "decommission", + operation, + self.rc_alias, + "0", + "--by-id", + ], + expect_success=False, + expected_error_code="InvalidArgument", + ) + guarded, _ = self.pool_snapshot(case_id, "after-single-pool-guards", 1) + guarded_pool = pool_entries(guarded)[0] + if str(guarded_pool.get("decommissionStatus", "")).lower() not in { + "", + "none", + } or guarded_pool.get("decommissionInfo") is not None: + raise AssertionError( + "rejected single-pool operations left partial decommission metadata" + ) + if str(guarded_pool.get("rebalanceStatus", "")).lower() not in { + "", + "none", + }: + raise AssertionError( + "rejected single-pool rebalance left partial operation metadata" + ) + self.apply_topology(case_id, "expanded") + rebalance_id = self.start_rebalance(case_id) + self.wait_rebalance(case_id, rebalance_id, "active") + self.run_rc( + case_id, + "duplicate-rebalance-start", + ["admin", "rebalance", "start", self.rc_alias], + expect_success=False, + expected_error_code="OperationAborted", + ) + self.run_rc( + case_id, + "decommission-during-rebalance", + ["admin", "decommission", "start", self.rc_alias, "0", "--by-id"], + expect_success=False, + expected_error_code="OperationAborted", + ) + self.wait_rebalance(case_id, rebalance_id, "active") + self.stop_rebalance(case_id) + self.wait_rebalance(case_id, rebalance_id, "stopped") + self.start_decommission(case_id) + self.wait_decommission(case_id, 0, "active") + self.run_rc( + case_id, + "duplicate-decommission-start", + ["admin", "decommission", "start", self.rc_alias, "0", "--by-id"], + expect_success=False, + expected_error_code="InvalidRequest", + ) + self.run_rc( + case_id, + "rebalance-during-decommission", + ["admin", "rebalance", "start", self.rc_alias], + expect_success=False, + expected_error_code="InvalidRequest", + ) + self.wait_decommission(case_id, 0, "active") + self.cancel_decommission(case_id) + self.wait_decommission(case_id, 0, "canceled") + self.run_rc( + case_id, + "restart-canceled-decommission-before-clear", + ["admin", "decommission", "start", self.rc_alias, "0", "--by-id"], + expect_success=False, + expected_error_code="InvalidArgument", + ) + self.clear_decommission(case_id) + self.wait_decommission(case_id, 0, "cleared") + self.assert_inventory(case_id, self.client, bucket, inventory, "operation-guards-final") + + def classify_exception(self, exc: BaseException, *, case_id: str | None = None) -> str: + if isinstance(exc, (AssertionError, PoolStatusError)): + return "fail" + if isinstance(exc, ClientError): + code = exc.response.get("Error", {}).get("Code", "") + if code == "SlowDown" and case_id in { + "pool.decommission-live-io", "pool.rebalance-live-io" + }: + return "fail" + return "infra_error" if code in INFRA_ERROR_CODES else "fail" + return "infra_error" + + def run_case(self, case_id: str, function: Callable[[str], None]) -> None: + started_at = utc_now() + started = time.monotonic() + state = "pass" + message = "all assertions passed" + print(f"[RUN ] {case_id}", flush=True) + self.events.record("case.start", case=case_id) + try: + self.reset_case_environment(case_id) + function(case_id) + except Exception as exc: + state = self.classify_exception(exc, case_id=case_id) + message = self.redact(f"{type(exc).__name__}: {exc}") + self.events.record( + "case.error", + case=case_id, + state=state, + errorType=type(exc).__name__, + message=self.redact(str(exc)), + ) + try: + self.capture_diagnostics(case_id, "case-failure-before-cleanup") + except Exception as diagnostic_exc: + diagnostic_message = self.redact( + f"{type(diagnostic_exc).__name__}: {diagnostic_exc}" + ) + message += f"; diagnostic collection failed: {diagnostic_message}" + self.events.record( + "diagnostic.collection.error", case=case_id, message=diagnostic_message + ) + if not self.args.cleanup_on_failure: + self.isolation_failure = {"caseId": case_id, "message": message} + finally: + bucket = self.bucket_name(case_id) + if bucket in self.created_buckets and (state == "pass" or self.args.cleanup_on_failure): + try: + self.cleanup_bucket(bucket) + except Exception as exc: + cleanup_state = self.classify_exception(exc) + cleanup_message = self.redact(f"{type(exc).__name__}: {exc}") + self.save_json( + case_id, "cleanup-error.json", {"state": cleanup_state, "message": cleanup_message} + ) + self.events.record( + "case.cleanup.error", case=case_id, state=cleanup_state, message=cleanup_message + ) + if state == "pass": + state = cleanup_state + message = self.redact( + f"{message}; cleanup failed: {cleanup_message}" + ) + if not self.args.cleanup_on_failure: + self.isolation_failure = {"caseId": case_id, "message": message} + elif bucket in self.created_buckets: + message += f"; preserved failed bucket {bucket} in {self.current_layout} topology" + directory = self.evidence_path(case_id, "evidence-index").parent + evidence = sorted( + str(path.relative_to(self.evidence_dir)) + for path in directory.iterdir() + if path.is_file() and path.name != "evidence-index" + ) + if not evidence: + evidence = [self.save_json(case_id, "case-summary.json", {"state": state, "message": message})] + result = CaseResult( + id=case_id, + state=state, + message=message, + started_at=started_at, + finished_at=utc_now(), + duration_seconds=round(time.monotonic() - started, 3), + evidence=evidence, + ) + self.results.append(result) + self.events.record("case.complete", **asdict(result)) + label = "PASS" if state == "pass" else state.upper() + print(f"[{label:5}] {case_id}: {message}", flush=True) + + def block_case(self, case_id: str) -> None: + assert self.isolation_failure is not None + message = ( + "blocked to preserve the failed pool topology from " + f"{self.isolation_failure['caseId']}: {self.isolation_failure['message']}" + ) + evidence = [ + self.save_json( + case_id, + "blocked-by-isolation.json", + {"state": "infra_error", "reason": self.isolation_failure}, + ) + ] + now = utc_now() + self.results.append( + CaseResult( + id=case_id, + state="infra_error", + message=message, + started_at=now, + finished_at=now, + duration_seconds=0.0, + evidence=evidence, + ) + ) + self.events.record("case.blocked", case=case_id, reason=self.isolation_failure) + print(f"[INFRA] {case_id}: {message}", flush=True) + + def finalize_physical_cleanup(self) -> None: + states = {item.state for item in self.results} + should_cleanup = not states.difference({"pass"}) or self.args.cleanup_on_failure + if not should_cleanup: + self.cleanup_outcome_value = { + "attempted": False, + "state": "preserved", + "message": "non-pass pool topology and data were preserved for investigation", + "preservedBuckets": sorted(self.created_buckets), + "layout": self.current_layout, + } + return + try: + reference = self.hook("run-cleanup", "physical-cleanup", self.args.cleanup_hook) + except Exception as exc: + self.cleanup_outcome_value = { + "attempted": True, + "state": "infra_error", + "message": self.redact( + f"physical cleanup failed: {type(exc).__name__}: {exc}" + ), + } + for result in self.results: + if result.state == "pass": + result.state = "infra_error" + result.message = self.cleanup_outcome_value["message"] + result.evidence.append(reference if "reference" in locals() else self.save_json( + result.id, "physical-cleanup-failure.json", self.cleanup_outcome_value + )) + break + return + self.created_buckets.clear() + self.cleanup_outcome_value = { + "attempted": True, + "state": "pass", + "message": "marker-guarded physical pool data cleanup completed", + "evidence": reference, + } + + def write_junit(self, overall: str, duration: float) -> None: + suite = ET.Element( + "testsuite", + { + "name": "rustfs-fixed-environment-pool-lifecycle", + "tests": str(len(self.results)), + "failures": str(sum(item.state == "fail" for item in self.results)), + "errors": str(sum(item.state == "infra_error" for item in self.results)), + "skipped": "0", + "time": f"{duration:.3f}", + }, + ) + for item in self.results: + case = ET.SubElement( + suite, + "testcase", + { + "classname": "rustfs.pool_lifecycle", + "name": item.id, + "time": f"{item.duration_seconds:.3f}", + }, + ) + if item.state == "fail": + ET.SubElement(case, "failure", {"message": item.message}).text = item.message + elif item.state == "infra_error": + ET.SubElement(case, "error", {"message": item.message}).text = item.message + ET.SubElement(case, "system-out").text = "\n".join(item.evidence) + tree = ET.ElementTree(suite) + ET.indent(tree, space=" ") + tree.write(self.evidence_dir / "junit.xml", encoding="utf-8", xml_declaration=True) + self.events.record("run.complete", state=overall) + + def write_results(self, started_at: str, started: float) -> str: + selected_cases = pool_harness_scored_cases(self.args.case) + order = {case_id: index for index, case_id in enumerate(CASE_IDS)} + self.results.sort(key=lambda item: order[item.id]) + states = {item.state for item in self.results} + overall = "fail" if "fail" in states else "infra_error" if "infra_error" in states else "pass" + finished_at = utc_now() + manifest = { + "runId": self.args.run_id, + "environmentId": self.args.environment_id, + "endpoint": self.initial_endpoint, + "finalControlEndpoint": self.args.endpoint, + "nodeEndpoints": self.args.node_endpoint, + "startedAt": started_at, + "finishedAt": finished_at, + "durationSeconds": round(time.monotonic() - started, 3), + "fixtureObjects": self.args.fixture_objects, + "fixtureBytes": self.args.fixture_bytes, + "activeFixtureBytes": active_fixture_bytes(self.args.fixture_bytes), + "liveObjects": self.args.live_objects, + "boto3Version": boto3.__version__, + "botocoreVersion": botocore.__version__, + "rcBinaryPath": self.rc_bin, + "rcVersion": self.rc_version, + "rcBinarySha256": self.rc_binary_sha256, + "expectedRcBinarySha256": self.expected_rc_binary_sha256 or None, + "executionPlan": { + "selectionMode": "filtered" if self.args.case is not None else "full", + "module": "pool", + "scoredCases": list(selected_cases), + "scoredCaseCount": len(selected_cases), + "fixtures": [], + }, + "fixtureOutcomes": [], + "cleanupOnFailure": self.args.cleanup_on_failure, + "cleanupOutcome": self.cleanup_outcome_value, + "preservedBuckets": sorted(self.created_buckets), + "finalLayout": self.current_layout, + } + (self.evidence_dir / "run-manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + result = { + "apiVersion": "validation.rustfs.io/v1alpha1", + "kind": "TestRunResult", + "metadata": { + "runId": self.args.run_id, + "environmentId": self.args.environment_id, + }, + "status": { + "state": overall, + "startedAt": started_at, + "finishedAt": finished_at, + "cases": [ + { + "id": item.id, + "state": item.state, + "message": item.message, + "evidence": item.evidence, + } + for item in self.results + ], + "evidence": ["events.jsonl", "run-manifest.json", "junit.xml"], + }, + } + (self.evidence_dir / "result.json").write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + self.write_junit(overall, time.monotonic() - started) + return overall + + def preflight_failure(self, selected_cases: tuple[str, ...], exc: BaseException) -> None: + message = self.redact( + f"pool harness preflight failed: {type(exc).__name__}: {exc}" + ) + for case_id in selected_cases: + evidence = [self.save_json(case_id, "preflight-failure.json", {"message": message})] + self.results.append( + CaseResult( + id=case_id, + state="infra_error", + message=message, + started_at=utc_now(), + finished_at=utc_now(), + duration_seconds=0.0, + evidence=evidence, + ) + ) + + def run(self) -> int: + started_at = utc_now() + started = time.monotonic() + selected_cases = pool_harness_scored_cases(self.args.case) + self.events.record( + "run.start", + runId=self.args.run_id, + environmentId=self.args.environment_id, + selectionMode="filtered" if self.args.case is not None else "full", + scoredCases=list(selected_cases), + ) + try: + self.wait_s3("run-preflight", self.client, "initial S3 connectivity") + self.configure_rc() + except Exception as exc: + self.preflight_failure(selected_cases, exc) + self.finalize_physical_cleanup() + overall = self.write_results(started_at, started) + return 2 if overall == "infra_error" else 1 + + cases: dict[str, Callable[[str], None]] = { + "pool.status": self.case_status, + "pool.expand-and-rebalance": self.case_expand_and_rebalance, + "pool.expansion-restart-persistence": self.case_expansion_restart_persistence, + "pool.rebalance-stop-resume": self.case_rebalance_stop_resume, + "pool.rebalance-live-io": self.case_rebalance_live_io, + "pool.decommission": self.case_decommission, + "pool.decommission-cancel-clear-resume": self.case_decommission_cancel_clear_resume, + "pool.decommission-restart-recovery": self.case_decommission_restart_recovery, + "pool.decommission-versioned-integrity": self.case_decommission_versioned_integrity, + "pool.decommission-multipart-integrity": self.case_decommission_multipart_integrity, + "pool.decommission-live-io": self.case_decommission_live_io, + "pool.retired-config-removal": self.case_retired_config_removal, + "pool.operation-guards": self.case_operation_guards, + } + for case_id in selected_cases: + if self.isolation_failure is not None: + self.block_case(case_id) + else: + self.run_case(case_id, cases[case_id]) + self.finalize_physical_cleanup() + overall = self.write_results(started_at, started) + print( + f"Pool lifecycle validation {overall}: result={self.evidence_dir / 'result.json'}", + flush=True, + ) + return {"pass": 0, "fail": 1, "infra_error": 2}[overall] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--endpoint", default=os.environ.get("RUSTFS_POOL_ENDPOINT")) + parser.add_argument("--node-endpoint", action="append", required=True) + parser.add_argument("--access-key", default=os.environ.get("RUSTFS_ACCESS_KEY")) + parser.add_argument("--secret-key", default=os.environ.get("RUSTFS_SECRET_KEY")) + parser.add_argument("--region", default=os.environ.get("RUSTFS_REGION", "us-east-1")) + parser.add_argument("--environment-id", default="lifecycle-a") + parser.add_argument("--run-id", required=True) + parser.add_argument("--evidence-dir", required=True, type=Path) + parser.add_argument("--timeout", type=float, default=600) + parser.add_argument("--active-timeout", type=float, default=90) + parser.add_argument("--poll-interval", type=float, default=0.5) + parser.add_argument("--fixture-objects", type=int, default=128) + parser.add_argument("--fixture-bytes", type=int, default=1024 * 1024) + parser.add_argument("--live-objects", type=int, default=16) + parser.add_argument("--workers", type=int, default=8) + parser.add_argument("--rc-bin", default="rc") + parser.add_argument("--reset-hook", required=True) + parser.add_argument("--topology-hook", required=True) + parser.add_argument("--restart-hook", required=True) + parser.add_argument("--census-hook", required=True) + parser.add_argument("--diagnostic-hook") + parser.add_argument("--cleanup-hook", required=True) + parser.add_argument("--case", action="append", choices=CASE_IDS) + parser.add_argument("--cleanup-on-failure", action="store_true") + args = parser.parse_args() + if not args.endpoint or not args.access_key or not args.secret_key: + parser.error("endpoint, access key, and secret key are required") + if len(args.node_endpoint) not in {2, 4}: + parser.error("--node-endpoint must be supplied exactly two or four times") + for value in (args.endpoint, *args.node_endpoint): + parsed = urllib.parse.urlparse(value) + if ( + parsed.scheme not in {"http", "https"} + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + ): + parser.error(f"invalid absolute HTTP(S) endpoint: {value!r}") + if any(character in args.access_key + args.secret_key for character in "\r\n"): + parser.error("access key and secret key must not contain line breaks") + if ( + args.timeout <= 0 + or args.active_timeout <= 0 + or args.poll_interval <= 0 + or args.fixture_objects < 4 + or args.fixture_bytes < 1 + or args.live_objects < 1 + or args.workers < 1 + ): + parser.error("timeouts and fixture/worker sizes must be positive") + return args + + +def main() -> int: + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(line_buffering=True) + args = parse_args() + harness = PoolHarness(args) + try: + return harness.run() + finally: + harness.close() + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except KeyboardInterrupt: + raise SystemExit(130) from None diff --git a/scripts/validation_pool/pool_disk_probe.py b/scripts/validation_pool/pool_disk_probe.py new file mode 100644 index 00000000..6c242186 --- /dev/null +++ b/scripts/validation_pool/pool_disk_probe.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Read-only, descriptor-scoped census and allowlisted pool metadata evidence.""" + +from __future__ import annotations + +import argparse +import base64 +import bisect +import errno +import hashlib +import json +import os +import stat +import time +from pathlib import Path, PurePosixPath +from typing import Any + + +DIRECTORY_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW +CACHE_OBJECTS = {".usage-cache.bin", ".usage-cache.bin.bkp"} +DIAGNOSTIC_OBJECTS = ( + ".rustfs.sys/buckets/.scanner-pause-backlog.json", + ".rustfs.sys/pool.bin", +) +MAX_FILE_BYTES = 4 * 1024 * 1024 +MAX_OBJECT_BYTES = 16 * 1024 * 1024 + + +class CensusEntryMissing(FileNotFoundError): + """A specific child disappeared while its anchored parent was being read.""" + + +def _cache_path(relative: str) -> bool: + parts = PurePosixPath(relative).parts + return ( + len(parts) >= 3 + and parts[:2] == (".rustfs.sys", "buckets") + and parts[2] in CACHE_OBJECTS + ) + + +def _check_root(root: Path, descriptor: int) -> os.stat_result: + expected = os.fstat(descriptor) + observed = root.lstat() + if not stat.S_ISDIR(observed.st_mode) or ( + observed.st_dev, observed.st_ino + ) != (expected.st_dev, expected.st_ino): + raise RuntimeError("census root identity changed") + return expected + + +def _census_once(descriptor: int) -> dict[str, Any]: + device = os.fstat(descriptor).st_dev + result: dict[str, Any] = { + "files": 0, "directories": 1, "xlMeta": 0, "bytes": 0, "samples": [], + } + + def walk(parent: int, prefix: str, depth: int) -> None: + if depth > 128: + raise RuntimeError("census directory depth exceeds 128") + with os.scandir(parent) as entries: + for entry in entries: + relative = f"{prefix}/{entry.name}" if prefix else entry.name + try: + info = os.stat(entry.name, dir_fd=parent, follow_symlinks=False) + if stat.S_ISDIR(info.st_mode): + if info.st_dev != device: + raise RuntimeError(f"unexpected nested mount: {relative}") + child = os.open(entry.name, DIRECTORY_FLAGS, dir_fd=parent) + try: + opened = os.fstat(child) + if (opened.st_dev, opened.st_ino) != (info.st_dev, info.st_ino): + raise RuntimeError(f"directory identity changed: {relative}") + result["directories"] += 1 + walk(child, relative, depth + 1) + finally: + os.close(child) + elif stat.S_ISREG(info.st_mode): + result["files"] += 1 + result["bytes"] += info.st_size + result["xlMeta"] += int(entry.name == "xl.meta") + bisect.insort(result["samples"], (relative, info.st_size)) + del result["samples"][20:] + except CensusEntryMissing: + raise + except FileNotFoundError as exc: + raise CensusEntryMissing(errno.ENOENT, "entry disappeared", relative) from exc + + walk(descriptor, "", 0) + return result + + +def census(root: Path, *, attempts: int = 3) -> dict[str, Any]: + if attempts < 1: + raise ValueError("census attempts must be positive") + descriptor = os.open(root, DIRECTORY_FLAGS) + races: list[str] = [] + try: + for attempt in range(1, attempts + 1): + _check_root(root, descriptor) + try: + result = _census_once(descriptor) + except CensusEntryMissing as exc: + _check_root(root, descriptor) + if not _cache_path(str(exc.filename)) or attempt == attempts: + raise + races.append(str(exc.filename)) + time.sleep(0.05) + continue + _check_root(root, descriptor) + result.update({ + "sampling": "non-atomic-online", + "attempts": attempt, + "retriedCachePaths": races, + }) + return result + raise AssertionError("unreachable census retry state") + finally: + os.close(descriptor) + + +def metadata_snapshot(root: Path) -> dict[str, Any]: + """Read only the two non-credential objects needed for migration diagnosis.""" + descriptor = os.open(root, DIRECTORY_FLAGS) + records: dict[str, Any] = {} + try: + _check_root(root, descriptor) + for object_name in DIAGNOSTIC_OBJECTS: + parent = os.dup(descriptor) + record: dict[str, Any] = {"files": []} + records[object_name] = record + try: + for component in PurePosixPath(object_name).parts: + child = os.open(component, DIRECTORY_FLAGS, dir_fd=parent) + os.close(parent) + parent = child + total = 0 + for name in sorted(os.listdir(parent)): + # Object metadata and inline/single-part payloads only. + file_descriptor = os.open( + name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent + ) + try: + before = os.fstat(file_descriptor) + if not stat.S_ISREG(before.st_mode): + continue + if before.st_size > MAX_FILE_BYTES or total + before.st_size > MAX_OBJECT_BYTES: + record["files"].append({"name": name, "state": "size-limit"}) + continue + chunks = [] + remaining = MAX_FILE_BYTES + 1 + while remaining: + block = os.read(file_descriptor, min(65536, remaining)) + if not block: + break + chunks.append(block) + remaining -= len(block) + data = b"".join(chunks) + if len(data) > MAX_FILE_BYTES or total + len(data) > MAX_OBJECT_BYTES: + raise RuntimeError("diagnostic file grew past size limit") + after = os.fstat(file_descriptor) + total += len(data) + record["files"].append({ + "name": name, + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + "mtimeNs": before.st_mtime_ns, + "stableDuringRead": ( + before.st_size, before.st_mtime_ns, before.st_ctime_ns + ) == (after.st_size, after.st_mtime_ns, after.st_ctime_ns), + "base64": base64.b64encode(data).decode("ascii"), + }) + finally: + os.close(file_descriptor) + record["state"] = "captured" + except OSError as exc: + record.update({"state": "unavailable", "errno": exc.errno}) + finally: + os.close(parent) + _check_root(root, descriptor) + return {"sampling": "non-atomic-online", "objects": records} + finally: + os.close(descriptor) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("census", "metadata")) + parser.add_argument("root", type=Path) + args = parser.parse_args() + result = census(args.root) if args.mode == "census" else metadata_snapshot(args.root) + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation_pool/pool_helpers.py b/scripts/validation_pool/pool_helpers.py new file mode 100644 index 00000000..cd96be15 --- /dev/null +++ b/scripts/validation_pool/pool_helpers.py @@ -0,0 +1,25 @@ +"""Pool topology and fixture rules used by the source behavior checks.""" + +MIN_ACTIVE_FIXTURE_BYTES = 32 * 1024 * 1024 + + +def active_endpoint_indices(endpoint_count: int, layout: str) -> tuple[int, ...]: + if layout not in {"initial", "expanded", "retired"}: + raise ValueError(f"unknown pool topology layout: {layout}") + if endpoint_count == 2: + return { + "initial": (0,), + "expanded": (0, 1), + "retired": (1,), + }[layout] + if endpoint_count == 4: + return { + "initial": (0, 1), + "expanded": (0, 1, 2, 3), + "retired": (2, 3), + }[layout] + raise ValueError("pool lifecycle requires exactly two or four node endpoints") + + +def active_fixture_bytes(configured_bytes: int) -> int: + return max(configured_bytes, MIN_ACTIVE_FIXTURE_BYTES) diff --git a/scripts/validation_pool/pool_status.py b/scripts/validation_pool/pool_status.py new file mode 100644 index 00000000..02963247 --- /dev/null +++ b/scripts/validation_pool/pool_status.py @@ -0,0 +1,588 @@ +"""Strict, side-effect-free validators for pool lifecycle Admin API output.""" + +from __future__ import annotations + +import json +from datetime import datetime +from typing import Any + + +class PoolStatusError(ValueError): + """A pool lifecycle response does not satisfy the release-gate contract.""" + + +def _integer(value: Any, field: str, *, minimum: int = 0) -> int: + if type(value) is not int or value < minimum: + raise PoolStatusError(f"{field} must be an integer >= {minimum}, got {value!r}") + return value + + +def _number(value: Any, field: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise PoolStatusError(f"{field} must be numeric, got {value!r}") + result = float(value) + if not 0.0 <= result <= 1.0: + raise PoolStatusError(f"{field} must be in 0.0..=1.0, got {value!r}") + return result + + +def _text(value: Any, field: str, *, allow_empty: bool = False) -> str: + if not isinstance(value, str) or (not allow_empty and not value): + qualifier = "a string" if allow_empty else "a non-empty string" + raise PoolStatusError(f"{field} must be {qualifier}, got {value!r}") + return value + + +def _timestamp(value: Any, field: str, *, allow_none: bool = False) -> str | None: + if value is None and allow_none: + return None + text = _text(value, field) + try: + parsed = datetime.fromisoformat(text[:-1] + "+00:00" if text.endswith("Z") else text) + except ValueError as exc: + raise PoolStatusError(f"{field} must be an RFC 3339 timestamp") from exc + if parsed.tzinfo is None: + raise PoolStatusError(f"{field} must include a timezone") + return text + + +def pool_entries(document: Any) -> list[dict[str, Any]]: + """Return pool entries from raw API, rc list, or rc/API status shapes.""" + + raw: Any + if isinstance(document, list): + raw = document + elif isinstance(document, dict) and isinstance(document.get("pools"), list): + raw = document["pools"] + elif isinstance(document, dict) and isinstance(document.get("pool"), dict): + raw = [document["pool"]] + elif isinstance(document, dict) and "id" in document: + raw = [document] + else: + raise PoolStatusError("pool response must be an array or contain pools/pool") + if not raw: + raise PoolStatusError("pool response must contain at least one pool") + if not all(isinstance(item, dict) for item in raw): + raise PoolStatusError("every pool response entry must be an object") + return list(raw) + + +def canonical_pool(entry: dict[str, Any], *, require_capacity: bool = True) -> dict[str, Any]: + """Validate and normalize fields shared by pool list and status output.""" + + result: dict[str, Any] = { + "id": _integer(entry.get("id"), "pool.id"), + "cmdline": _text(entry.get("cmdline"), "pool.cmdline"), + "status": _text(entry.get("status"), "pool.status"), + "decommissionStatus": _text( + entry.get("decommissionStatus"), + "pool.decommissionStatus", + allow_empty=True, + ), + "rebalanceStatus": _text( + entry.get("rebalanceStatus"), + "pool.rebalanceStatus", + allow_empty=True, + ), + } + _timestamp(entry.get("lastUpdate"), "pool.lastUpdate") + if require_capacity: + total = _integer(entry.get("totalSize"), "pool.totalSize") + current = _integer(entry.get("currentSize"), "pool.currentSize") + used_size = _integer(entry.get("usedSize"), "pool.usedSize") + used = _number(entry.get("used"), "pool.used") + if current > total or used_size > total: + raise PoolStatusError( + "pool currentSize/usedSize cannot exceed totalSize: " + f"total={total} current={current} used={used_size}" + ) + expected_used_size = total - current + if used_size != expected_used_size: + raise PoolStatusError( + "pool capacity counters are inconsistent: " + f"total={total} current={current} used={used_size}" + ) + expected_ratio = used_size / total if total else 0.0 + if abs(used - expected_ratio) > 1e-9: + raise PoolStatusError( + "pool used ratio is inconsistent with byte counters: " + f"used={used} expected={expected_ratio}" + ) + result.update( + { + "totalSize": total, + "currentSize": current, + "usedSize": used_size, + # ``used`` is derived from the integer counters. Independent + # JSON encoders may emit adjacent decimal representations, so + # normalize it after validating the supplied ratio above. + "used": expected_ratio, + } + ) + return result + + +def rc_failure_is_retryable(*streams: str) -> bool: + """Recognize an rc JSON failure that explicitly permits a bounded retry.""" + + if any("requires a live fleet capability proof" in stream for stream in streams): + return True + for stream in streams: + try: + document = json.loads(stream) + except (json.JSONDecodeError, TypeError): + continue + if not isinstance(document, dict): + continue + details = document.get("details") + if isinstance(details, dict) and details.get("retryable") is True: + return True + return False + + +def validate_pool_inventory( + document: Any, + expected_count: int, + *, + require_active: bool = True, +) -> tuple[dict[str, Any], ...]: + entries = tuple(canonical_pool(item) for item in pool_entries(document)) + if len(entries) != expected_count: + raise PoolStatusError( + f"expected exactly {expected_count} pools, observed {len(entries)}" + ) + ids = tuple(item["id"] for item in entries) + if len(ids) != len(set(ids)): + raise PoolStatusError(f"pool IDs are not unique: {ids}") + if ids != tuple(sorted(ids)): + raise PoolStatusError(f"pool IDs are not in stable ascending order: {ids}") + cmdlines = tuple(item["cmdline"] for item in entries) + if len(cmdlines) != len(set(cmdlines)): + raise PoolStatusError("pool command lines are not unique") + if require_active: + inactive = [ + item["id"] + for item in entries + if item["status"].lower() != "active" + ] + if inactive: + raise PoolStatusError(f"expected active pools, inactive IDs: {inactive}") + return entries + + +def assert_pool_parity( + left: Any, + right: Any, + expected_count: int, + *, + require_active: bool = True, +) -> None: + left_entries = validate_pool_inventory( + left, expected_count, require_active=require_active + ) + right_entries = validate_pool_inventory( + right, expected_count, require_active=require_active + ) + if left_entries != right_entries: + raise PoolStatusError( + f"rc/Admin pool status mismatch: rc={left_entries!r} admin={right_entries!r}" + ) + + +def decommission_entry(document: Any, pool_id: int) -> dict[str, Any]: + matches = [ + entry + for entry in pool_entries(document) + if entry.get("id") == pool_id + ] + if len(matches) != 1: + raise PoolStatusError( + f"decommission status must contain pool {pool_id} exactly once" + ) + entry = matches[0] + _integer(entry.get("id"), "decommission.id") + _text(entry.get("cmdline"), "decommission.cmdline") + _text(entry.get("status"), "decommission.status") + _text(entry.get("poolStatus"), "decommission.poolStatus") + return entry + + +def _decommission_info(entry: dict[str, Any]) -> dict[str, Any]: + info = entry.get("decommissionInfo") + if not isinstance(info, dict): + raise PoolStatusError("decommissionInfo must be an object while an operation exists") + for field in ("complete", "failed", "canceled", "queued"): + if type(info.get(field)) is not bool: + raise PoolStatusError(f"decommissionInfo.{field} must be a boolean") + for field in ("startSize", "totalSize", "currentSize"): + _optional_counter(info, field) + for field in ( + "objectsDecommissioned", + "objectsDecommissionedFailed", + "bytesDecommissioned", + "bytesDecommissionedFailed", + ): + _optional_counter(info, field) + for field in ("queuedBuckets", "decommissionedBuckets"): + value = info.get(field, []) + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise PoolStatusError(f"decommissionInfo.{field} must be a string array") + for field in ("bucket", "prefix", "object", "stage"): + _text(info.get(field, ""), f"decommissionInfo.{field}", allow_empty=True) + _timestamp( + info.get("startTime"), "decommissionInfo.startTime", allow_none=True + ) + waiting_reason = info.get("waitingReason") + if waiting_reason is not None and not isinstance(waiting_reason, str): + raise PoolStatusError( + "decommissionInfo.waitingReason must be a string or null" + ) + unresolved = info.get("unresolvedEntries", []) + if not isinstance(unresolved, list): + raise PoolStatusError("decommissionInfo.unresolvedEntries must be an array") + return info + + +def _decommission_failure(entry: dict[str, Any]) -> str | None: + info = entry.get("decommissionInfo") + if not isinstance(info, dict): + return None + _decommission_info(entry) + status = str(entry.get("status", "")).lower() + if status in {"failed", "canceled", "cancelled"} or info.get("failed") is True: + return "decommission reported failed or canceled" + if _optional_counter(info, "objectsDecommissionedFailed") > 0: + return "decommission reported failed objects" + if _optional_counter(info, "bytesDecommissionedFailed") > 0: + return "decommission reported failed bytes" + unresolved = info.get("unresolvedEntries", []) + if unresolved is not None and not isinstance(unresolved, list): + return "decommission unresolvedEntries is malformed" + if unresolved: + return "decommission reported unresolved entries" + return None + + +def _optional_counter(document: dict[str, Any], field: str) -> int: + value = document.get(field, 0) + return _integer(value, field) + + +def decommission_active(document: Any, pool_id: int, *, require_progress: bool) -> bool: + entry = decommission_entry(document, pool_id) + reason = _decommission_failure(entry) + if reason: + raise PoolStatusError(reason) + info = _decommission_info(entry) + running = str(entry["status"]).lower() == "running" and str( + entry["poolStatus"] + ).lower() == "decommissioning" + queued = info.get("queued") is True + if not running or queued or info.get("complete") is True or info.get("canceled") is True: + return False + if not require_progress: + return True + return ( + _optional_counter(info, "objectsDecommissioned") > 0 + and _optional_counter(info, "bytesDecommissioned") > 0 + ) + + +def decommission_active_or_completed(document: Any, pool_id: int) -> bool: + """Accept durable movement counters when decommission finishes before polling.""" + + return decommission_active( + document, pool_id, require_progress=True + ) or decommission_complete(document, pool_id) + + +def decommission_complete(document: Any, pool_id: int) -> bool: + entry = decommission_entry(document, pool_id) + reason = _decommission_failure(entry) + if reason: + raise PoolStatusError(reason) + info = _decommission_info(entry) + queued_buckets = info.get("queuedBuckets", []) + if not isinstance(queued_buckets, list): + raise PoolStatusError("queuedBuckets must be an array") + terminal = ( + info.get("complete") is True + and str(entry["status"]).lower() == "complete" + and str(entry["poolStatus"]).lower() == "decommissioned" + ) + clean = ( + info.get("failed") is False + and info.get("canceled") is False + and info.get("queued") is False + and not queued_buckets + and _optional_counter(info, "objectsDecommissionedFailed") == 0 + and _optional_counter(info, "bytesDecommissionedFailed") == 0 + ) + moved = ( + _optional_counter(info, "objectsDecommissioned") > 0 + and _optional_counter(info, "bytesDecommissioned") > 0 + ) + return terminal and clean and moved + + +def decommission_canceled(document: Any, pool_id: int) -> bool: + entry = decommission_entry(document, pool_id) + info = _decommission_info(entry) + status_canceled = str(entry["status"]).lower() in { + "canceled", + "cancelled", + } + if info.get("failed") is True: + raise PoolStatusError("canceled decommission also reports failed=true") + if info.get("complete") is True: + raise PoolStatusError("canceled decommission also reports complete=true") + if ( + _optional_counter(info, "objectsDecommissionedFailed") > 0 + or _optional_counter(info, "bytesDecommissionedFailed") > 0 + or info.get("unresolvedEntries", []) + ): + raise PoolStatusError("canceled decommission reports unresolved failures") + return ( + info.get("canceled") is True + and status_canceled + and info.get("queued") is False + # Cancel retains unfinished buckets for diagnostics; queued is the + # scheduling flag. Completion, unlike cancellation, drains the list. + and info.get("startTime") is None + and str(entry["poolStatus"]).lower() == "blocked" + ) + + +def decommission_cleared(document: Any, pool_id: int) -> bool: + entry = decommission_entry(document, pool_id) + return ( + entry.get("decommissionInfo") is None + and str(entry["status"]).lower() in {"", "none"} + and str(entry["poolStatus"]).lower() == "active" + ) + + +def decommission_multipart_blocked(document: Any, pool_id: int) -> bool: + """Recognize a clean post-check failure; the caller must prove its MPU cause.""" + entry = decommission_entry(document, pool_id) + info = _decommission_info(entry) + for field in ("objectsDecommissioned", "bytesDecommissioned", "objectsDecommissionedFailed", "bytesDecommissionedFailed"): + _integer(info.get(field), f"MPU drain.{field}") + if (info["complete"] or info["canceled"] or entry["status"].lower() in {"complete", "canceled", "cancelled"} + or entry["poolStatus"].lower() == "decommissioned"): + raise PoolStatusError("MPU drain protection completed or canceled unexpectedly") + if any(_optional_counter(info, field) for field in ("objectsDecommissionedFailed", "bytesDecommissionedFailed")) or info.get("unresolvedEntries"): + raise PoolStatusError("MPU drain observation contains unrelated migration failures") + if info["failed"] or entry["status"].lower() == "failed": + if not ( + info["failed"] and entry["status"].lower() == "failed" + and entry["poolStatus"].lower() == "blocked" and not info["queued"] + and info.get("startTime") is None and info.get("queuedBuckets") == [] + and _optional_counter(info, "objectsDecommissioned") > 0 + and _optional_counter(info, "bytesDecommissioned") > 0 + ): + raise PoolStatusError("MPU drain failure has inconsistent terminal state") + return True + return False + + +def multipart_drain_failure_records( + document: Any, *, bucket: str, since: str, node_count: int, +) -> list[dict[str, Any]]: + """Require fresh, scoped journal evidence before accepting an expected failure.""" + if not isinstance(document, dict) or document.get("mode") != "decommission-journal" or document.get("since") != since: + raise PoolStatusError("MPU drain journal has the wrong collection identity") + nodes = document.get("nodes") + if not isinstance(nodes, dict) or len(nodes) != node_count or not all(isinstance(node, str) and node for node in nodes): + raise PoolStatusError("MPU drain journal omitted a node") + _timestamp(since, "MPU journal.since") + lower_bound = datetime.fromisoformat(since.replace("Z", "+00:00")) + matches = [] + for node, output in nodes.items(): + if not isinstance(output, str): + raise PoolStatusError("MPU drain journal node output must be text") + for line in output.splitlines(): + if not line.startswith("{"): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise PoolStatusError("MPU drain journal contains malformed JSON") from exc + if not isinstance(record, dict) or record.get("event") != "decommission_state" or record.get("state") != "routine_failed": + continue + _timestamp(record.get("timestamp"), "MPU failure.timestamp") + if datetime.fromisoformat(record["timestamp"].replace("Z", "+00:00")) < lower_bound: + continue + error = record.get("error") + if not ( + type(record.get("pool_index")) is int and record["pool_index"] == 0 + and isinstance(error, str) and "post-check failed:" in error + and "pool 0 still contains multipart upload `" in error + and f"` for bucket `{bucket}`; resolve it before retrying decommission" in error + ): + raise PoolStatusError("unexpected failure instead of the owned MPU drain protection") + matches.append({**record, "node": node}) + if not matches: + raise PoolStatusError("missing fresh MPU drain failure evidence for the owned bucket") + return matches + + +def rebalance_entries( + document: Any, + expected_id: str, + *, + allow_stopping: bool = False, + allow_stopped: bool = False, +) -> tuple[dict[str, Any], ...]: + if not isinstance(document, dict): + raise PoolStatusError("rebalance status must be an object") + operation_id = _text(document.get("id"), "rebalance.id") + if operation_id != expected_id: + raise PoolStatusError( + f"rebalance operation ID changed: expected {expected_id!r}, got {operation_id!r}" + ) + pools = document.get("pools") + if not isinstance(pools, list) or not pools or not all( + isinstance(pool, dict) for pool in pools + ): + raise PoolStatusError("rebalance status must contain non-empty object pools") + ids: list[int] = [] + for pool in pools: + ids.append(_integer(pool.get("id"), "rebalance.pool.id")) + state = _text(pool.get("status"), "rebalance.pool.status") + _number(pool.get("used"), "rebalance.pool.used") + stopping = pool.get("stopping") + if stopping is not None and type(stopping) is not bool: + raise PoolStatusError("rebalance.pool.stopping must be a boolean") + if state.lower() in {"failed", "canceled", "cancelled"}: + raise PoolStatusError(f"rebalance entered unsuccessful state {state!r}") + if state.lower() == "stopped" and not allow_stopped: + raise PoolStatusError("rebalance stopped before successful completion") + if pool.get("stopping") is True and not allow_stopping: + raise PoolStatusError("rebalance still reports stopping=true") + last_error = pool.get("lastError") + if last_error not in {None, ""}: + raise PoolStatusError(f"rebalance reported lastError={last_error!r}") + warnings = pool.get("cleanupWarnings") + if not isinstance(warnings, dict): + raise PoolStatusError("rebalance cleanupWarnings must be an object") + warning_count = _optional_counter(warnings, "count") + warning_details: list[str] = [] + for field in ("lastMsg", "lastBucket", "lastObject", "lastAt"): + value = warnings.get(field) + if value is not None and not isinstance(value, str): + raise PoolStatusError( + f"rebalance cleanupWarnings.{field} must be a string or null" + ) + if isinstance(value, str) and value: + warning_details.append(value) + _timestamp( + warnings.get("lastAt"), + "rebalance cleanupWarnings.lastAt", + allow_none=True, + ) + warning_entries = warnings.get("entries", []) + if not isinstance(warning_entries, list) or not all( + isinstance(item, dict) for item in warning_entries + ): + raise PoolStatusError( + "rebalance cleanupWarnings.entries must be an object array" + ) + if warning_count != 0: + raise PoolStatusError("rebalance reported cleanup warnings") + if warning_details or warning_entries: + raise PoolStatusError( + "rebalance reported cleanup warning details with count=0" + ) + if len(ids) != len(set(ids)): + raise PoolStatusError(f"rebalance pool IDs are duplicated: {ids}") + propagation = document.get("stopPropagation") + if propagation is not None: + if not isinstance(propagation, dict): + raise PoolStatusError("stopPropagation must be an object") + for field in ("failedPeers", "terminalReloadFailedPeers"): + peers = propagation.get(field, []) + if not isinstance(peers, list) or not all( + isinstance(peer, str) for peer in peers + ): + raise PoolStatusError( + f"stopPropagation.{field} must be a string array" + ) + if peers: + raise PoolStatusError(f"stopPropagation.{field} is not empty") + pending_reload = propagation.get("pendingTerminalReload", False) + if type(pending_reload) is not bool: + raise PoolStatusError( + "stopPropagation.pendingTerminalReload must be a boolean" + ) + if pending_reload: + raise PoolStatusError("rebalance has a pending terminal reload") + for field in ("lastAttemptAt", "terminalReloadAttemptAt"): + _timestamp( + propagation.get(field), + f"stopPropagation.{field}", + allow_none=True, + ) + stopped_at = document.get("stoppedAt") + _timestamp(stopped_at, "rebalance.stoppedAt", allow_none=True) + return tuple(pools) + + +def _rebalance_progress(pool: dict[str, Any]) -> int: + progress = pool.get("progress") + if progress is None: + return 0 + if not isinstance(progress, dict): + raise PoolStatusError("rebalance progress must be an object or null") + moved = sum( + _optional_counter(progress, field) + for field in ("objects", "versions", "bytes") + ) + for field in ("remainingBuckets", "elapsed", "eta"): + _optional_counter(progress, field) + for field in ("bucket", "object"): + _text(progress.get(field, ""), f"rebalance.progress.{field}", allow_empty=True) + return moved + + +def rebalance_active(document: Any, expected_id: str, *, require_progress: bool) -> bool: + pools = rebalance_entries(document, expected_id) + active = [pool for pool in pools if str(pool["status"]).lower() == "started"] + if not active: + return False + return not require_progress or any(_rebalance_progress(pool) > 0 for pool in active) + + +def rebalance_active_or_completed(document: Any, expected_id: str) -> bool: + """Accept observable progress even when a small rebalance finishes before polling.""" + + return rebalance_active( + document, expected_id, require_progress=True + ) or rebalance_complete(document, expected_id) + + +def rebalance_complete(document: Any, expected_id: str) -> bool: + pools = rebalance_entries(document, expected_id) + completed = [pool for pool in pools if str(pool["status"]).lower() == "completed"] + all_terminal = all( + str(pool["status"]).lower() in {"completed", "none"} for pool in pools + ) + return all_terminal and bool(completed) and any( + _rebalance_progress(pool) > 0 for pool in completed + ) + + +def rebalance_stopped(document: Any, expected_id: str) -> bool: + pools = rebalance_entries( + document, + expected_id, + allow_stopping=True, + allow_stopped=True, + ) + stopped_at = document.get("stoppedAt") + if not isinstance(stopped_at, str) or not stopped_at: + return False + return not any(pool.get("stopping") is True for pool in pools) and all( + str(pool["status"]).lower() in {"stopped", "completed", "none"} + for pool in pools + ) and any(str(pool["status"]).lower() == "stopped" for pool in pools) diff --git a/scripts/validation_pool/rc_binary.py b/scripts/validation_pool/rc_binary.py new file mode 100644 index 00000000..6bc1068e --- /dev/null +++ b/scripts/validation_pool/rc_binary.py @@ -0,0 +1,10 @@ +"""Share the pinned rc executable resolver with ILM validation.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from scripts.validation_ilm.rc_binary import resolve_rc_binary # noqa: E402 diff --git a/scripts/validation_pool/run_case.sh b/scripts/validation_pool/run_case.sh new file mode 100644 index 00000000..89a7d7c7 --- /dev/null +++ b/scripts/validation_pool/run_case.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUSTFS_POOL_SCRIPT_SOURCE_ONLY=1 source "${ROOT}/rustfs_pool_expand.sh" +export RUSTFS_POOL_VALIDATION_MODE=1 +RUSTFS_OBS_LOGGER_LEVEL=info + +[ "${#NODES[@]}" -ge 2 ] || die "pool validation needs two auto-testing nodes" +NODES=("${NODES[0]}" "${NODES[1]}") +NODE_ENDPOINTS=(http://rustfs-node1:9000 http://rustfs-node2:9000) +LEGACY_ENDPOINT="${NODE_ENDPOINTS[0]}" +ADMIN_ENDPOINT="${NODE_ENDPOINTS[0]}" +WARP_ENDPOINT="${NODE_ENDPOINTS[0]}" +ASSUME_YES=1 +resolve_endpoints + +POOL1_VOLUMES="http://rustfs-node2:9000/data/rustfs{1...4}/mnmd" +MARKER=/etc/rustfs-pool-validation-id + +check_marker() { + local node + [ -n "${POOL_VALIDATION_RUN_ID:-}" ] || die "pool run identity missing" + for node in "${NODES[@]}"; do + ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ + "test \"\$(cat ${MARKER})\" = \"${POOL_VALIDATION_RUN_ID}\"" \ + || die "pool environment marker does not match on ${node}" + done +} + +mark_nodes() { + local node + for node in "${NODES[@]}"; do + printf 'printf %%s %q | sudo -n tee %s >/dev/null\n' \ + "${POOL_VALIDATION_RUN_ID}" "${MARKER}" | run_remote "${node}" + done +} + +reset_initial() { + local node + for node in "${NODES[@]}"; do service_action stop "${node}"; done + for node in "${NODES[@]}"; do + run_remote "${node}" <<'REMOTE' +set -euo pipefail +SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" +for i in 1 2 3 4; do + ${SUDO} rm -rf "/data/rustfs${i}/mnmd" + ${SUDO} install -d -o rustfs -g rustfs "/data/rustfs${i}/mnmd" +done +REMOTE + done + write_rustfs_config "${NODES[0]}" "${VOLUMES_1}" + start_and_wait_nodes "${NODES[0]}" + wait_node_ready 0 +} + +retire_pool_config() { + service_action stop "${NODES[0]}" + service_action stop "${NODES[1]}" + write_rustfs_config "${NODES[1]}" "${POOL1_VOLUMES}" + ensure_volume_dirs "${NODES[1]}" "${POOL1_VOLUMES}" + start_and_wait_nodes "${NODES[1]}" + wait_node_ready 1 +} + +disk_probe() { + local mode="$1" phase="$2" node index path + printf 'phase=%s mode=%s\n' "${phase}" "${mode}" + for node in "${NODES[@]}"; do + for index in 1 2 3 4; do + path="/data/rustfs${index}/mnmd" + printf 'node=%s path=%s\n' "${node}" "${path}" + { + printf 'set -euo pipefail\n' + printf 'findmnt -rn -T %q -o TARGET,SOURCE,FSTYPE,UUID\n' "${path}" + printf "python3 - %q %q <<'POOL_DISK_PROBE'\n" "${mode}" "${path}" + cat "${ROOT}/scripts/validation_pool/pool_disk_probe.py" + printf '\nPOOL_DISK_PROBE\n' + } | run_remote "${node}" + done + done +} + +pool_hook() { + local action="${1:-}" layout="" phase="" since="" node output0 output1 journal_since + shift || true + while [ "$#" -gt 0 ]; do + case "$1" in + --layout) layout="$2"; shift 2 ;; + --phase) phase="$2"; shift 2 ;; + --since) since="$2"; shift 2 ;; + *) die "unknown pool hook option: $1" ;; + esac + done + check_marker + case "${action}" in + reset) reset_initial ;; + topology) + case "${layout}" in + initial) reset_initial ;; + expanded) step5_expand_pool2 ;; + retired) retire_pool_config ;; + *) die "unknown pool layout: ${layout}" ;; + esac ;; + restart) + case "${layout}" in + expanded) + service_action_all restart + wait_node_ready 0 + wait_node_ready 1 ;; + retired) + service_action restart "${NODES[1]}" + wait_node_ready 1 ;; + *) die "unknown pool restart layout: ${layout}" ;; + esac ;; + census) disk_probe census "${phase}" ;; + diagnostic) + if [ -n "${since}" ]; then + journal_since="$(python3 -c 'from datetime import datetime,timezone; import sys; value=datetime.fromisoformat(sys.argv[1].replace("Z","+00:00")); assert value.tzinfo is not None; print(value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f UTC"))' "${since}")" + output0="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${NODES[0]}" \ + "journalctl -u rustfs --since $(printf '%q' "${journal_since}") --no-pager --output=cat -n 2000")" + output1="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${NODES[1]}" \ + "journalctl -u rustfs --since $(printf '%q' "${journal_since}") --no-pager --output=cat -n 2000")" + jq -n --arg phase "${phase}" --arg since "${since}" \ + --arg node0 "${NODES[0]}" --arg node1 "${NODES[1]}" \ + --arg output0 "${output0}" --arg output1 "${output1}" \ + '{environmentId:"lifecycle-a",phase:$phase,mode:"decommission-journal",since:$since,nodes:{($node0):$output0,($node1):$output1}}' + else + disk_probe metadata "${phase}" + fi ;; + cleanup) step0_reset ;; + *) die "unknown pool hook: ${action}" ;; + esac +} + +hook_json() { + python3 -c 'import json,sys; print(json.dumps(sys.argv[1:]))' \ + bash "${ROOT}/scripts/validation_pool/run_case.sh" hook "$1" +} + +setup_environment() { + step0_reset + step1_download + step2_install + step3_start_first_pool + mark_nodes +} + +run_validation() { + local case_id="" candidate="" testing_commit="" artifacts="" setup_rc=0 harness_rc=0 cleanup_rc=0 + while [ "$#" -gt 0 ]; do + case "$1" in + --validation-case) case_id="$2"; shift 2 ;; + --candidate-manifest) candidate="$2"; shift 2 ;; + --testing-commit) testing_commit="$2"; shift 2 ;; + --validation-artifacts-dir) artifacts="$2"; shift 2 ;; + --dry-run) DRY_RUN=1; shift ;; + -y|--yes) shift ;; + *) die "unknown pool validation option: $1" ;; + esac + done + [[ "${case_id}" == pool.* ]] || die "pool validation case required" + python3 -B -c 'import sys; sys.path.insert(0,sys.argv[1]); from scripts.validation_pool.case_plan import POOL_CASE_IDS; sys.exit(0 if sys.argv[2] in POOL_CASE_IDS else 2)' \ + "${ROOT}" "${case_id}" || die "unknown canonical pool case: ${case_id}" + if [ "${DRY_RUN}" -eq 1 ]; then + POOL_VALIDATION_RUN_ID="${case_id}-dry-run" + setup_environment + return 0 + fi + [ -f "${candidate}" ] || die "candidate manifest missing" + [ -n "${artifacts}" ] && [ ! -e "${artifacts}" ] || die "fresh artifact directory required" + [ "${testing_commit}" = "$(git -C "${ROOT}" rev-parse HEAD)" ] || die "testing commit differs from checkout" + PACKAGE_URL="$(jq -er '.package_url' "${candidate}")" + PACKAGE_SHA256="$(jq -er '.package_sha256' "${candidate}")" + [[ "${PACKAGE_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "invalid candidate SHA-256" + export RUSTFS_NODES="${NODES[*]}" RUSTFS_ADMIN_ENDPOINT="${NODE_ENDPOINTS[0]}" + export POOL_VALIDATION_RUN_ID="${case_id}-$(date +%s)-$$" + mkdir -p "${artifacts}/cases" "${artifacts}/evidence/${case_id}/pool" + local evidence="${artifacts}/evidence/${case_id}" harness="${artifacts}/evidence/${case_id}/pool" + python3 "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" \ + --profile lifecycle-a --result-file "${artifacts}/cases/${case_id}.json" \ + --verdict RUNNING --cleanup-status PENDING + (setup_environment) > "${evidence}/setup.log" 2>&1 || setup_rc=$? + if [ "${setup_rc}" -ne 0 ]; then + cat "${evidence}/setup.log" + (step0_reset) > "${evidence}/cleanup.log" 2>&1 || true + return 1 + fi + local binary_sha + binary_sha="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${NODES[0]}" sha256sum /usr/bin/rustfs | awk '{print $1}')" + [[ "${binary_sha}" =~ ^[0-9a-f]{64}$ ]] || die "installed RustFS binary SHA-256 unavailable" + python3 -B "${ROOT}/scripts/validation_pool/pool_behavior.py" \ + --case "${case_id}" --endpoint "${NODE_ENDPOINTS[0]}" \ + --node-endpoint "${NODE_ENDPOINTS[0]}" --node-endpoint "${NODE_ENDPOINTS[1]}" \ + --access-key "${ACCESS_KEY}" --secret-key "${SECRET_KEY}" \ + --run-id "${POOL_VALIDATION_RUN_ID}" --evidence-dir "${harness}" \ + --rc-bin "$(command -v rc)" --cleanup-on-failure \ + --reset-hook "$(hook_json reset)" --topology-hook "$(hook_json topology)" \ + --restart-hook "$(hook_json restart)" --census-hook "$(hook_json census)" \ + --diagnostic-hook "$(hook_json diagnostic)" --cleanup-hook "$(hook_json cleanup)" \ + > "${evidence}/test.log" 2>&1 || harness_rc=$? + cat "${evidence}/test.log" + if ! jq -e '.cleanupOutcome.state == "pass"' "${harness}/run-manifest.json" >/dev/null 2>&1; then + (pool_hook cleanup) > "${evidence}/fallback-cleanup.log" 2>&1 || cleanup_rc=$? + fi + local verdict=FAIL reason="pool behavior assertions failed; inspect evidence" cleanup_status=FAIL + if jq -e '.cleanupOutcome.state == "pass"' "${harness}/run-manifest.json" >/dev/null 2>&1; then + cleanup_status=PASS + fi + if [ "${harness_rc}" -eq 0 ] && [ "${cleanup_rc}" -eq 0 ] && \ + python3 -B "${ROOT}/scripts/validation_ilm/verify_result.py" "${case_id}" "${harness}"; then + verdict=PASS; reason="" + elif jq -e '.status.state == "infra_error"' "${harness}/result.json" >/dev/null 2>&1; then + verdict=INFRA_ERROR; reason="pool environment or execution infrastructure failed; inspect evidence" + fi + local args=("${ROOT}/scripts/validation_result.py" --case-id "${case_id}" + --profile lifecycle-a --result-file "${artifacts}/cases/${case_id}.json" + --verdict "${verdict}" --cleanup-status "${cleanup_status}" + --candidate-manifest "${candidate}" --rc-bin "$(command -v rc)" + --testing-commit "${testing_commit}" --binary-sha256 "${binary_sha}" + --evidence "evidence/${case_id}/setup.log" --evidence "evidence/${case_id}/test.log") + if [ -n "${reason}" ]; then args+=(--reason "${reason}"); fi + if [ "${verdict}" = PASS ]; then + local file + while IFS= read -r -d '' file; do + args+=(--evidence "${file#"${artifacts}/"}") + done < <(find "${harness}" -type f -print0 | sort -z) + fi + python3 "${args[@]}" + [ "${verdict}" = PASS ] +} + +case "${1:-}" in + hook) shift; pool_hook "$@" ;; + *) run_validation "$@" ;; +esac diff --git a/test_validation_pool.py b/test_validation_pool.py new file mode 100644 index 00000000..a7d1e7bf --- /dev/null +++ b/test_validation_pool.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +import unittest + +from scripts.validation_pool.case_plan import POOL_CASE_IDS, pool_harness_scored_cases +from scripts.validation_pool.pool_helpers import active_endpoint_indices, active_fixture_bytes + + +class PoolContractTest(unittest.TestCase): + def test_catalog_drives_case_selection(self) -> None: + self.assertEqual(13, len(POOL_CASE_IDS)) + self.assertEqual(("pool.status",), pool_harness_scored_cases(["pool.status"])) + with self.assertRaises(ValueError): + pool_harness_scored_cases(["pool.missing"]) + + def test_two_node_layout_and_active_fixture_size(self) -> None: + self.assertEqual((0,), active_endpoint_indices(2, "initial")) + self.assertEqual((0, 1), active_endpoint_indices(2, "expanded")) + self.assertEqual((1,), active_endpoint_indices(2, "retired")) + self.assertEqual(32 * 1024 * 1024, active_fixture_bytes(1024)) + + +if __name__ == "__main__": + unittest.main() From 75ad3aa072396affdcf9879f73f1b5888a431375 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 18:52:25 +0800 Subject: [PATCH 013/343] Migrate replication validation and isolate webhook cases --- .../workflows/validation-catalog-tests.yml | 9 +- docs/validation-migration-map.md | 37 +- rustfs-replication-test.sh | 4 +- rustfs-tier-test.sh | 17 +- scripts/run_validation_case.sh | 12 +- scripts/validation_replication/case_plan.py | 28 + scripts/validation_replication/rc_binary.py | 10 + .../replication_behavior.py | 3699 +++++++++++++++++ .../replication_metrics.py | 91 + scripts/validation_replication/run_case.sh | 136 + .../validation_replication/verify_result.py | 48 + test_validation_replication.py | 43 + 12 files changed, 4118 insertions(+), 16 deletions(-) create mode 100644 scripts/validation_replication/case_plan.py create mode 100644 scripts/validation_replication/rc_binary.py create mode 100644 scripts/validation_replication/replication_behavior.py create mode 100644 scripts/validation_replication/replication_metrics.py create mode 100644 scripts/validation_replication/run_case.sh create mode 100644 scripts/validation_replication/verify_result.py create mode 100644 test_validation_replication.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 28501f32..3c28b7d5 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -13,6 +13,9 @@ on: - test_validation_pool.py - scripts/validation_pool/** - rustfs_pool_expand.sh + - test_validation_replication.py + - scripts/validation_replication/** + - rustfs-replication-test.sh - rustfs-storage-test.sh - test_validation_scanner_usage.sh - .github/workflows/validation-catalog-tests.yml @@ -29,6 +32,9 @@ on: - test_validation_pool.py - scripts/validation_pool/** - rustfs_pool_expand.sh + - test_validation_replication.py + - scripts/validation_replication/** + - rustfs-replication-test.sh - rustfs-storage-test.sh - test_validation_scanner_usage.sh - .github/workflows/validation-catalog-tests.yml @@ -48,5 +54,6 @@ jobs: - run: python3 -B -m unittest -v test_validation_migration.py - run: python3 -B -m unittest -v test_validation_ilm.py - run: python3 -B -m unittest -v test_validation_pool.py - - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh scripts/validation_pool/run_case.sh test_validation_scanner_usage.sh + - run: python3 -B -m unittest -v test_validation_replication.py + - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh - run: bash test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 535f9c15..474fa4f9 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -42,8 +42,26 @@ source's accelerated scanner/ILM timing, and supplies a cluster restart hook for `ilm.restart-persistence`. `ilm.transition-and-restore` uses the third auto-testing node as an independent warm-tier target. Each selected case must pass its own harness result, evidence, and bucket-cleanup verification before -the canonical result can become PASS. These executors have local contract and -dry-run checks, but no live PASS evidence yet. +the canonical result can become PASS. `ilm.configuration-roundtrip`, +`ilm.invalid-configuration`, `ilm.current-expiration`, +`ilm.tag-filter-expiration`, `ilm.disabled-rule`, and `ilm.rule-removal` have +live PASS evidence in RustFS Actions runs `38044956285`, `38045047518`, +`38045154019`, `38045412209`, `38045496042`, and `38045714246` respectively. +The other eight ILM executors have local contract and dry-run checks but no +live PASS evidence yet. + +All 13 Pool lifecycle IDs now dispatch to the source black-box pool checks. +The driver reuses the existing auto-testing Pool suite's two-node seed and +expansion operations, supplies marker-guarded reset/topology/restart/physical +census hooks, and requires source case evidence plus physical cleanup before +canonical PASS. These cases have not yet completed live validation. + +The 18 bucket-replication and 20 site-replication IDs now dispatch to the +source black-box checks. The driver uses the existing auto-testing four-node +main cluster and second RustFS instance on port 19000, including the source's +site-group fixture and site outage hooks. Logical bucket isolation and +physical teardown are required for PASS. These cases have not yet completed +live validation. `kms.sse-kms-round-trip` has a dedicated executor using the existing local KMS single-disk deployment. It checks the same object's encryption key, @@ -61,8 +79,12 @@ Actions run `38042759074`. `events.delivery` and `audit.request-correlation` now have dedicated executors on the Tier suite's `single-single` topology. Event delivery is counted for a unique object after a quiet window; audit checks matching PUT/GET records, -distinct request IDs, successful status, and absence of known secrets. Both -still require live candidate evidence before being counted as verified. +distinct request IDs, successful status, and absence of known secrets. The +first two event attempts stopped before case execution because the candidate +could not converge local notification runtime while both notification and +audit modules were enabled. The validation setup now enables only the required +webhook channel and retains its full setup log. Both cases still require live +candidate evidence before being counted as verified. `iam.policy-enforcement` has a dedicated executor on the Security suite's `single-single` deployment. It uses one principal with a policy that allows @@ -90,9 +112,10 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 22 currently dispatchable IDs are the S3, two scanner, -KMS, tiering, events, audit, IAM, and 14 additional ILM cases above. Six have -live PASS evidence; the other 152 IDs remain unverified. +case evidence. The 73 currently dispatchable IDs are the S3, two scanner, +KMS, tiering, events, audit, IAM, 14 additional ILM, 13 Pool, and 38 +replication cases above. Twelve have live PASS evidence; the other 146 IDs +remain unverified. The eight planned cases have these concrete reuse candidates: diff --git a/rustfs-replication-test.sh b/rustfs-replication-test.sh index 98bc70b8..99a124a5 100755 --- a/rustfs-replication-test.sh +++ b/rustfs-replication-test.sh @@ -1290,4 +1290,6 @@ main() { log "all done" } -main "$@" +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/rustfs-tier-test.sh b/rustfs-tier-test.sh index 27d4e00d..bea1f190 100755 --- a/rustfs-tier-test.sh +++ b/rustfs-tier-test.sh @@ -684,6 +684,10 @@ wait_service_active() { rustfs_config_body() { local volumes="$1" + local mqtt_enabled=true + if [ "${VALIDATION_CASE}" = events.delivery ] || [ "${VALIDATION_CASE}" = audit.request-correlation ]; then + mqtt_enabled=false + fi cat < HTTP ${mcode}: ${mbody}"; die "failed to enable notify/audit modules"; } - wait_targets_ready + [ "${VALIDATION_CASE}" = audit.request-correlation ] || wait_targets_ready fi log "rc aliases local/target; warm bucket ${TIER_BUCKET} ready" fi diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index e975876b..e307ac27 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -15,6 +15,8 @@ ARTIFACTS_DIR="$4" case "${CASE_ID}" in pool.*) SUITE=scripts/validation_pool/run_case.sh; EXTRA_ARGS=() ;; + bucket-replication.*|site-replication.*) + SUITE=scripts/validation_replication/run_case.sh; EXTRA_ARGS=() ;; s3.basic-object-round-trip) SUITE=rustfs-s3-compat-test.sh; EXTRA_ARGS=() ;; scanner.usage-convergence|scanner.status-readiness|ilm.configuration-roundtrip|ilm.invalid-configuration|ilm.current-expiration|ilm.tag-filter-expiration|ilm.disabled-rule|ilm.rule-removal|ilm.versioned-current-expiration|ilm.noncurrent-expiration|ilm.newer-noncurrent-retention|ilm.delete-all-versions|ilm.delete-marker-expiration|ilm.multipart-expiration|ilm.restart-persistence|ilm.transition-and-restore) @@ -30,14 +32,18 @@ case "${CASE_ID}" in exit 2 ;; esac -if [[ "${CASE_ID}" == pool.* ]]; then - python3 -B -c 'import json,sys; catalog=json.load(open(sys.argv[1])); valid={case["id"] for module in catalog["modules"] if module["id"] == "pool-lifecycle" for case in module["cases"]}; sys.exit(0 if sys.argv[2] in valid else 2)' \ +if [[ "${CASE_ID}" == pool.* || "${CASE_ID}" == bucket-replication.* || "${CASE_ID}" == site-replication.* ]]; then + python3 -B -c 'import json,sys; catalog=json.load(open(sys.argv[1])); valid={case["id"] for module in catalog["modules"] for case in module["cases"]}; sys.exit(0 if sys.argv[2] in valid else 2)' \ "${SCRIPT_DIR}/validation-cases.json" "${CASE_ID}" || { - printf 'unknown canonical pool case: %s\n' "${CASE_ID}" >&2 + printf 'unknown canonical validation case: %s\n' "${CASE_ID}" >&2 exit 2 } fi +if [ "${SUITE}" = rustfs-tier-test.sh ]; then + EXTRA_ARGS+=(--log-file "${ARTIFACTS_DIR}/logs/${CASE_ID}.log") +fi + [ -f "${CANDIDATE_MANIFEST}" ] || { printf 'candidate manifest missing\n' >&2; exit 2; } [[ "${TESTING_COMMIT}" =~ ^[0-9a-f]{40}$ ]] || { printf 'invalid testing commit\n' >&2; exit 2; } [ "${TESTING_COMMIT}" = "$(git -C "${SCRIPT_DIR}" rev-parse HEAD)" ] || { diff --git a/scripts/validation_replication/case_plan.py b/scripts/validation_replication/case_plan.py new file mode 100644 index 00000000..061e5993 --- /dev/null +++ b/scripts/validation_replication/case_plan.py @@ -0,0 +1,28 @@ +"""Select replication cases and their source setup fixture.""" + +from __future__ import annotations + +import json +from pathlib import Path + + +CATALOG = Path(__file__).resolve().parents[2] / "validation-cases.json" + + +def module_cases(module_id: str) -> tuple[str, ...]: + catalog = json.loads(CATALOG.read_text(encoding="utf-8")) + modules = [module for module in catalog["modules"] if module["id"] == module_id] + if len(modules) != 1: + raise ValueError(f"canonical replication module missing or duplicated: {module_id}") + return tuple(case["id"] for case in modules[0]["cases"]) + + +BUCKET_CASE_IDS = module_cases("bucket-replication") +SITE_CASE_IDS = module_cases("site-replication") +SITE_GROUP_ACTIVE_FIXTURE = "site-group-active" +SITE_GROUP_ACTIVE_CASE_IDS = frozenset(SITE_CASE_IDS) - { + "site-replication.unconfigured-info", + "site-replication.invalid-credentials", + "site-replication.add-and-info", + "site-replication.service-account-rotation", +} diff --git a/scripts/validation_replication/rc_binary.py b/scripts/validation_replication/rc_binary.py new file mode 100644 index 00000000..6bc1068e --- /dev/null +++ b/scripts/validation_replication/rc_binary.py @@ -0,0 +1,10 @@ +"""Share the pinned rc executable resolver with ILM validation.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from scripts.validation_ilm.rc_binary import resolve_rc_binary # noqa: E402 diff --git a/scripts/validation_replication/replication_behavior.py b/scripts/validation_replication/replication_behavior.py new file mode 100644 index 00000000..58d009f8 --- /dev/null +++ b/scripts/validation_replication/replication_behavior.py @@ -0,0 +1,3699 @@ +#!/usr/bin/env python3 +"""Run bucket- and site-replication black-box checks against two RustFS sites.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import secrets +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Callable + +from case_plan import ( + BUCKET_CASE_IDS, + SITE_CASE_IDS, + SITE_GROUP_ACTIVE_CASE_IDS, + SITE_GROUP_ACTIVE_FIXTURE, +) +from rc_binary import resolve_rc_binary +from replication_metrics import replication_metrics_ready + +try: + import boto3 + import botocore + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.config import Config + from botocore.credentials import Credentials + from botocore.exceptions import ( + ClientError, + ConnectTimeoutError, + EndpointConnectionError, + ReadTimeoutError, + ) +except ImportError as exc: # pragma: no cover - dependency guidance + raise SystemExit( + "boto3 is required for replication validation" + ) from exc + + +ADMIN_PREFIX = "/rustfs/admin/v3" +INFRA_EXCEPTIONS = (ConnectTimeoutError, EndpointConnectionError, ReadTimeoutError) +INFRA_ERROR_CODES = { + "InvalidAccessKeyId", + "InvalidToken", + "RequestTimeout", + "ServiceUnavailable", + "SignatureDoesNotMatch", + "SlowDown", +} + +def utc_now() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + +def safe_name(value: str) -> str: + return "".join(char if char.isalnum() or char in "-." else "-" for char in value) + + +def json_default(value: Any) -> Any: + if isinstance(value, datetime): + return value.astimezone(UTC).isoformat().replace("+00:00", "Z") + if isinstance(value, bytes): + return {"bytes": len(value), "sha256": hashlib.sha256(value).hexdigest()} + return str(value) + + +def without_response_metadata(value: dict[str, Any]) -> dict[str, Any]: + return {key: item for key, item in value.items() if key != "ResponseMetadata"} + + +@dataclass(frozen=True) +class Site: + id: str + endpoint: str + access_key: str + secret_key: str + + +@dataclass +class CaseResult: + id: str + state: str + message: str + started_at: str + finished_at: str + duration_seconds: float + evidence: list[str] = field(default_factory=list) + + +class BucketCleanupResidual(RuntimeError): + """A logical test bucket survived best-effort cleanup before phase reset.""" + + def __init__( + self, + bucket: str, + last_error: BaseException | None, + inventory: dict[str, Any], + ): + self.bucket = bucket + self.inventory = inventory + super().__init__(f"{bucket} remained non-empty after cleanup retries: {last_error}") + + +class LogicalCleanupFailure(RuntimeError): + """One case's logical cleanup failed, with isolation evidence persisted.""" + + def __init__( + self, + *, + residuals: list[dict[str, Any]], + errors: list[dict[str, Any]], + evidence: str, + tracked_buckets: list[dict[str, str]], + isolation_restored: bool, + ) -> None: + self.residuals = residuals + self.errors = errors + self.evidence = evidence + self.tracked_buckets = tracked_buckets + self.isolation_restored = isolation_restored + super().__init__( + "logical bucket cleanup failed: " + f"residuals={len(residuals)} errors={len(errors)} " + f"isolation_restored={isolation_restored} evidence={evidence}" + ) + + +class PreconditionError(RuntimeError): + """A diagnostic fixture could not establish a trustworthy product state.""" + + +def _site_endpoint_identity(endpoint: str) -> tuple[str, int] | None: + """Return RustFS' product identity for one site endpoint. + + RustFS intentionally keys a site by lower-case host plus effective port; + transport scheme and URL path are not part of the persisted identity. + Keeping the test harness on that exact contract prevents it from trying to + add two URLs that the product will collapse into one deployment. + """ + + if not isinstance(endpoint, str): + return None + try: + parsed = urllib.parse.urlsplit(endpoint) + scheme = parsed.scheme.lower() + host = (parsed.hostname or "").lower() + port = parsed.port + except ValueError: + return None + if scheme not in {"http", "https"} or not host: + return None + if port is None: + port = 80 if scheme == "http" else 443 + return host, port + + +def classify_site_group_state( + info_by_site: dict[str, Any], + expected_endpoints: tuple[str, str], +) -> tuple[str, str]: + """Classify the two-site control-plane snapshot without mutating it.""" + + expected_identities = tuple( + _site_endpoint_identity(endpoint) for endpoint in expected_endpoints + ) + if any(identity is None for identity in expected_identities): + return "malformed", "a configured validation endpoint is invalid" + if len(set(expected_identities)) != 2: + return "malformed", "the configured validation endpoints are not distinct" + + expected_sites = ("site-a", "site-b") + if any(not isinstance(info_by_site.get(site_id), dict) for site_id in expected_sites): + return "malformed", "site info is missing or is not an object" + + snapshots = [info_by_site[site_id] for site_id in expected_sites] + enabled = [snapshot.get("enabled") for snapshot in snapshots] + if all(value is False for value in enabled): + stale = [ + expected_sites[index] + for index, snapshot in enumerate(snapshots) + if snapshot.get("serviceAccountAccessKey") not in (None, "") + or snapshot.get("pendingOperation") is not None + or snapshot.get("sites") not in (None, []) + or _site_retry_state_is_degraded(snapshot.get("retryStats")) + ] + if stale: + return "malformed", f"disabled site(s) retain replication state: {', '.join(stale)}" + return "disabled", "both sites are cleanly unconfigured" + + if not all(value is True for value in enabled): + return "partial", f"site enabled flags differ or are invalid: {enabled!r}" + + if any(snapshot.get("pendingOperation") is not None for snapshot in snapshots): + return "pending", "a site-replication operation is still pending" + + degraded = [ + expected_sites[index] + for index, snapshot in enumerate(snapshots) + if _site_retry_state_is_degraded(snapshot.get("retryStats")) + ] + if degraded: + return "degraded", f"site retry state is not clear: {', '.join(degraded)}" + + expected_identity_set = set(expected_identities) + + peer_maps: list[dict[tuple[str, int], str]] = [] + service_accounts: list[str] = [] + for site_id, snapshot in zip(expected_sites, snapshots, strict=True): + service_account = snapshot.get("serviceAccountAccessKey") + if not isinstance(service_account, str) or not service_account: + return "malformed", f"{site_id} has no site-replication service account" + service_accounts.append(service_account) + peers = snapshot.get("sites") + if not isinstance(peers, list) or len(peers) != 2: + return "malformed", f"{site_id} peer membership does not contain exactly two entries" + if not all(isinstance(peer, dict) for peer in peers): + return "malformed", f"{site_id} peer membership contains a non-object entry" + peer_map: dict[tuple[str, int], str] = {} + for peer in peers: + deployment_id = peer.get("deploymentID") + endpoint = peer.get("endpoint") + sync = peer.get("sync") + if not isinstance(deployment_id, str) or not deployment_id: + return "malformed", f"{site_id} has an invalid deployment ID" + identity = _site_endpoint_identity(endpoint) + if identity is None: + return "malformed", f"{site_id} has an invalid peer endpoint" + if not isinstance(sync, str) or sync.lower() != "enable": + return "malformed", f"{site_id} has a peer without sync=enable" + if identity in peer_map: + return "malformed", f"{site_id} reports a duplicate peer endpoint" + peer_map[identity] = deployment_id + if len(set(peer_map.values())) != 2: + return "malformed", f"{site_id} does not report two unique deployment IDs" + if set(peer_map) != expected_identity_set: + return "malformed", ( + f"{site_id} site-group endpoints do not match the configured " + "validation sites" + ) + peer_maps.append(peer_map) + + if peer_maps[0] != peer_maps[1]: + return "malformed", "the two sites report different endpoint/deployment membership" + if len(set(service_accounts)) != 1: + return "malformed", "the two sites report different service-account access keys" + return "active", "both sites report one stable two-deployment group" + + +def site_group_has_ilm_expiry_value( + info_by_site: dict[str, Any], + expected: bool, +) -> tuple[bool, str]: + """Require every peer view to expose one exact ILM-expiry flag value.""" + + for site_id in ("site-a", "site-b"): + snapshot = info_by_site.get(site_id) + if not isinstance(snapshot, dict): + return False, f"{site_id} site info is missing or malformed" + peers = snapshot.get("sites") + if not isinstance(peers, list) or len(peers) != 2: + return False, f"{site_id} does not expose exactly two site peers" + for peer in peers: + if not isinstance(peer, dict): + return False, f"{site_id} contains a non-object site peer" + if peer.get("replicate-ilm-expiry") is not expected: + return False, ( + f"{site_id} peer {peer.get('deploymentID')!r} does not expose " + f"replicate-ilm-expiry={str(expected).lower()}" + ) + return True, ( + "all site peers expose " + f"replicate-ilm-expiry={str(expected).lower()}" + ) + + +def _site_retry_state_is_degraded(value: Any) -> bool: + if value is None: + return False + if not isinstance(value, dict): + return True + pending = value.get("pending", 0) + failed = value.get("failed", 0) + last_error = value.get("lastError", "") + return ( + not isinstance(pending, int) + or isinstance(pending, bool) + or pending < 0 + or not isinstance(failed, int) + or isinstance(failed, bool) + or failed < 0 + or pending != 0 + or failed != 0 + or not isinstance(last_error, str) + or bool(last_error) + ) + + +class EventRecorder: + def __init__(self, path: Path) -> None: + self.path = path + self.lock = threading.Lock() + path.parent.mkdir(parents=True, exist_ok=True) + + def record(self, event: str, **details: Any) -> None: + item = {"timestamp": utc_now(), "event": event, "details": details} + with self.lock, self.path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(item, default=json_default, sort_keys=True) + "\n") + + +class ReplicationHarness: + def __init__(self, args: argparse.Namespace) -> None: + self.args = args + self.evidence_dir = args.evidence_dir.resolve() + self.case_dir = self.evidence_dir / "cases" + self.case_dir.mkdir(parents=True, exist_ok=True) + self.events = EventRecorder(self.evidence_dir / "events.jsonl") + self.results: list[CaseResult] = [] + self.rc_home = Path(tempfile.mkdtemp(prefix="rustfs-replication-rc-")) + self.rc_env = os.environ.copy() + self.rc_env["RC_CONFIG_DIR"] = str(self.rc_home) + self.rc_bin = args.rc_bin + self.rc_version = "unavailable" + self.rc_binary_sha256 = "unavailable" + self.expected_rc_binary_sha256 = os.environ.get( + "RUSTFS_EXPECTED_RC_SHA256", "" + ).lower() + self.site_a = Site("site-a", args.endpoint_a.rstrip("/"), args.access_key, args.secret_key) + self.site_b = Site("site-b", args.endpoint_b.rstrip("/"), args.access_key, args.secret_key) + self.alias_a = f"rv-a-{safe_name(args.run_id)[-12:]}" + self.alias_b = f"rv-b-{safe_name(args.run_id)[-12:]}" + self.client_a = self.s3_client(self.site_a) + self.client_b = self.s3_client(self.site_b) + self.created_buckets: list[tuple[Any, str]] = [] + self.secrets: set[str] = {args.access_key, args.secret_key} + self.isolation_failure: dict[str, Any] | None = None + self.requested_fixtures = frozenset(getattr(args, "fixture", None) or ()) + self.site_group_tracker_state = "fresh" + self.site_group_mutation_attempted = False + self.fixture_outcomes: list[dict[str, Any]] = [] + + def close(self) -> None: + shutil.rmtree(self.rc_home, ignore_errors=True) + + def s3_client(self, site: Site, access_key: str | None = None, secret_key: str | None = None): + return boto3.client( + "s3", + endpoint_url=site.endpoint, + aws_access_key_id=access_key or site.access_key, + aws_secret_access_key=secret_key or site.secret_key, + region_name=self.args.region, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + retries={"max_attempts": 3, "mode": "standard"}, + connect_timeout=5, + read_timeout=30, + ), + ) + + def evidence_path(self, case_id: str, name: str) -> Path: + directory = self.case_dir / safe_name(case_id) + directory.mkdir(parents=True, exist_ok=True) + return directory / name + + def save_json(self, case_id: str, name: str, value: Any) -> str: + path = self.evidence_path(case_id, name) + text = json.dumps(value, default=json_default, indent=2, sort_keys=True) + "\n" + path.write_text(self.redact(text), encoding="utf-8") + return str(path.relative_to(self.evidence_dir)) + + def save_text(self, case_id: str, name: str, value: str) -> str: + path = self.evidence_path(case_id, name) + path.write_text(self.redact(value), encoding="utf-8") + return str(path.relative_to(self.evidence_dir)) + + def redact(self, value: str) -> str: + result = value + for secret in sorted((item for item in self.secrets if item), key=len, reverse=True): + result = result.replace(secret, "[REDACTED]") + return result + + def bucket_name(self, case_id: str, role: str = "src") -> str: + digest = hashlib.sha256(f"{self.args.run_id}:{case_id}:{role}".encode()).hexdigest()[:12] + return f"rv-{role}-{digest}" + + def identity_name(self, case_id: str, role: str) -> str: + digest = hashlib.sha256(f"{self.args.run_id}:{case_id}:{role}".encode()).hexdigest()[:10] + return f"rv-{role}-{digest}" + + def create_bucket(self, client: Any, bucket: str) -> None: + client.create_bucket(Bucket=bucket) + self.created_buckets.append((client, bucket)) + self.events.record("bucket.created", bucket=bucket) + + @staticmethod + def enable_versioning(client: Any, bucket: str) -> None: + client.put_bucket_versioning( + Bucket=bucket, VersioningConfiguration={"Status": "Enabled"} + ) + status = client.get_bucket_versioning(Bucket=bucket).get("Status") + if status != "Enabled": + raise AssertionError(f"versioning did not become enabled: {status!r}") + + def wait_until( + self, + case_id: str, + description: str, + probe: Callable[[], tuple[bool, Any]], + timeout: float | None = None, + ) -> Any: + deadline = time.monotonic() + (timeout or self.args.timeout) + attempts = 0 + last: Any = None + while time.monotonic() < deadline: + attempts += 1 + try: + complete, last = probe() + except INFRA_EXCEPTIONS as exc: + complete, last = False, {"error": f"{type(exc).__name__}: {exc}"} + if complete: + self.events.record( + "poll.complete", + case=case_id, + description=description, + attempts=attempts, + detail=last, + ) + return last + time.sleep(self.args.poll_interval) + self.save_json( + case_id, + f"poll-timeout-{safe_name(description)}.json", + {"description": description, "attempts": attempts, "last": last}, + ) + raise AssertionError(f"timed out waiting for {description}; last={last}") + + def wait_api(self, case_id: str, client: Any, label: str) -> None: + def probe() -> tuple[bool, Any]: + try: + response = client.list_buckets() + return True, {"requestId": response.get("ResponseMetadata", {}).get("RequestId")} + except (*INFRA_EXCEPTIONS, ClientError) as exc: + return False, {"error": f"{type(exc).__name__}: {exc}"[:400]} + + self.wait_until(case_id, f"{label} S3 API readiness", probe, timeout=120) + + def configure_rc(self) -> None: + requested_rc_bin = self.rc_bin + try: + self.rc_bin, self.rc_binary_sha256 = resolve_rc_binary(requested_rc_bin) + except FileNotFoundError as exc: + raise RuntimeError( + f"required rc binary not found: {requested_rc_bin}" + ) from exc + version = subprocess.run( + [self.rc_bin, "--version"], + env=self.rc_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + if version.returncode != 0: + raise RuntimeError(f"rc --version failed: {version.stderr.strip()}") + self.rc_version = (version.stdout or version.stderr).strip() + if ( + self.expected_rc_binary_sha256 + and self.rc_binary_sha256.lower() != self.expected_rc_binary_sha256 + ): + raise RuntimeError( + "resolved rc binary SHA-256 differs from RUSTFS_EXPECTED_RC_SHA256" + ) + for alias, site in ((self.alias_a, self.site_a), (self.alias_b, self.site_b)): + result = subprocess.run( + [ + self.rc_bin, + "--no-color", + "alias", + "set", + alias, + site.endpoint, + site.access_key, + site.secret_key, + "--bucket-lookup", + "path", + ], + env=self.rc_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"rc alias set failed for {site.id}: {result.stderr.strip()}") + self.events.record( + "rc.configured", + binaryPath=self.rc_bin, + version=self.rc_version, + binarySha256=self.rc_binary_sha256, + ) + + def run_rc( + self, + case_id: str, + name: str, + arguments: list[str], + *, + expect_success: bool = True, + ) -> subprocess.CompletedProcess[str]: + command = [self.rc_bin, "--json", "--no-color", *arguments] + result = subprocess.run( + command, + env=self.rc_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + sanitized = ["[REDACTED]" if token in self.secrets else token for token in command] + self.save_json( + case_id, + f"{name}.json", + { + "command": sanitized, + "exitCode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + }, + ) + if expect_success and result.returncode != 0: + raise AssertionError( + f"rc {name} exited {result.returncode}: {self.redact(result.stderr.strip())}" + ) + if not expect_success and result.returncode == 0: + raise AssertionError(f"rc {name} unexpectedly succeeded") + return result + + def signed_request( + self, + site: Site, + method: str, + path: str, + *, + query: str = "", + body: bytes | None = None, + content_type: str | None = None, + timeout: float = 30, + ) -> tuple[int, dict[str, str], bytes]: + url = f"{site.endpoint}{urllib.parse.quote(path, safe='/-._~')}" + if query: + url += f"?{query}" + payload = body or b"" + headers = {"x-amz-content-sha256": hashlib.sha256(payload).hexdigest()} + if content_type: + headers["content-type"] = content_type + request = AWSRequest(method=method, url=url, data=body, headers=headers) + SigV4Auth(Credentials(site.access_key, site.secret_key), "s3", self.args.region).add_auth(request) + outgoing = urllib.request.Request( + url, + data=body, + headers=dict(request.headers.items()), + method=method, + ) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + try: + with opener.open(outgoing, timeout=timeout) as response: + return response.status, dict(response.headers.items()), response.read() + except urllib.error.HTTPError as exc: + return exc.code, dict(exc.headers.items()), exc.read() + + def admin_request( + self, + site: Site, + method: str, + subpath: str, + *, + query: str = "", + payload: Any | None = None, + ) -> tuple[int, dict[str, str], bytes]: + body = None if payload is None else json.dumps(payload).encode("utf-8") + return self.signed_request( + site, + method, + f"{ADMIN_PREFIX}/{subpath}", + query=query, + body=body, + content_type="application/json" if body is not None else None, + ) + + @staticmethod + def json_body(body: bytes) -> Any: + if not body.strip(): + return {} + return json.loads(body) + + @staticmethod + def required_json_object(body: bytes, operation: str) -> dict[str, Any]: + try: + parsed = ReplicationHarness.json_body(body) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise AssertionError(f"{operation} returned malformed JSON") from exc + if not isinstance(parsed, dict): + raise AssertionError( + f"{operation} returned a non-object JSON value: {parsed!r}" + ) + return parsed + + def hook(self, case_id: str, name: str, raw: str | None) -> str: + if not raw: + raise RuntimeError(f"{name} requires its driver hook") + command = json.loads(raw) + if not isinstance(command, list) or not command or not all(isinstance(x, str) and x for x in command): + raise RuntimeError(f"{name} hook must be a JSON command array") + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + path = self.save_text(case_id, f"{name}.log", result.stdout) + if result.returncode != 0: + raise RuntimeError(f"{name} hook exited {result.returncode}") + return path + + def wait_object( + self, + case_id: str, + client: Any, + bucket: str, + key: str, + expected: bytes, + *, + expected_version: str | None = None, + timeout: float | None = None, + ) -> dict[str, Any]: + expected_sha = hashlib.sha256(expected).hexdigest() + + def probe() -> tuple[bool, Any]: + try: + response = client.get_object(Bucket=bucket, Key=key) + body = response["Body"].read() + version = response.get("VersionId") + detail = { + "bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest(), + "versionId": version, + "metadata": response.get("Metadata", {}), + } + matches = detail["sha256"] == expected_sha + if expected_version is not None: + matches = matches and version == expected_version + return matches, detail + except ClientError as exc: + return False, {"error": exc.response.get("Error", {}).get("Code", "unknown")} + + return self.wait_until(case_id, f"{bucket}/{key} replication", probe, timeout=timeout) + + @staticmethod + def object_exists(client: Any, bucket: str, key: str) -> tuple[bool, Any]: + try: + response = client.head_object(Bucket=bucket, Key=key) + return True, {"versionId": response.get("VersionId"), "contentLength": response.get("ContentLength")} + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "unknown") + if code in {"404", "NoSuchKey", "NoSuchBucket", "NotFound"}: + return False, {"error": code} + raise + + def assert_absent_for( + self, case_id: str, client: Any, bucket: str, key: str, duration: float + ) -> dict[str, Any]: + deadline = time.monotonic() + duration + observations = 0 + last: Any = None + while time.monotonic() < deadline: + observations += 1 + exists, last = self.object_exists(client, bucket, key) + if exists: + raise AssertionError(f"unexpected object appeared at {bucket}/{key}: {last}") + time.sleep(self.args.poll_interval) + detail = {"durationSeconds": duration, "observations": observations, "last": last} + self.events.record("negative-window.complete", case=case_id, **detail) + return detail + + def setup_bucket_replication( + self, + case_id: str, + *, + flags: str | None = None, + prefix: str | None = None, + synchronous: bool = False, + preexisting: tuple[str, bytes] | None = None, + ) -> tuple[str, str, dict[str, Any]]: + source = self.bucket_name(case_id, "src") + target = self.bucket_name(case_id, "dst") + self.create_bucket(self.client_a, source) + self.create_bucket(self.client_b, target) + self.enable_versioning(self.client_a, source) + self.enable_versioning(self.client_b, target) + if preexisting: + key, body = preexisting + self.client_a.put_object(Bucket=source, Key=key, Body=body) + arguments = [ + "bucket", + "replication", + "add", + f"{self.alias_a}/{source}", + "--remote-bucket", + f"{self.alias_b}/{target}", + "--id", + f"rule-{hashlib.sha256(case_id.encode()).hexdigest()[:8]}", + "--healthcheck-seconds", + "1", + "--force", + ] + if flags: + arguments.extend(("--replicate", flags)) + if prefix: + arguments.extend(("--prefix", prefix)) + if synchronous: + arguments.append("--sync") + self.run_rc(case_id, "replication-add", arguments) + config = self.client_a.get_bucket_replication(Bucket=source).get("ReplicationConfiguration", {}) + self.save_json(case_id, "replication-configuration.json", config) + if not config.get("Rules"): + raise AssertionError("replication configuration has no rules") + return source, target, config + + @staticmethod + def delete_replication_configuration(client: Any, bucket: str) -> None: + try: + client.delete_bucket_replication(Bucket=bucket) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") not in { + "NoSuchBucket", + "ReplicationConfigurationNotFoundError", + "NoSuchReplicationConfiguration", + }: + raise + + @staticmethod + def delete_object_batch(client: Any, bucket: str, items: list[dict[str, str]]) -> None: + for offset in range(0, len(items), 1000): + response = client.delete_objects( + Bucket=bucket, + Delete={"Objects": items[offset : offset + 1000], "Quiet": True}, + ) + errors = response.get("Errors", []) + if errors: + raise RuntimeError(f"delete_objects reported errors: {errors[:5]}") + + def inspect_bucket_cleanup_inventory( + self, + client: Any, + bucket: str, + ) -> dict[str, Any]: + inventory: dict[str, Any] = {"bucket": bucket} + try: + uploads = client.list_multipart_uploads(Bucket=bucket) + inventory["multipartUploads"] = [ + {"key": item.get("Key"), "uploadId": item.get("UploadId")} + for item in uploads.get("Uploads", [])[:20] + ] + inventory["multipartUploadsTruncated"] = bool( + uploads.get("IsTruncated") + ) + except Exception as exc: + inventory["multipartInspectionError"] = f"{type(exc).__name__}: {exc}" + try: + version_count = 0 + marker_count = 0 + version_samples: list[dict[str, Any]] = [] + for page in client.get_paginator("list_object_versions").paginate( + Bucket=bucket + ): + versions = page.get("Versions", []) + markers = page.get("DeleteMarkers", []) + version_count += len(versions) + marker_count += len(markers) + for kind, items in (("version", versions), ("delete-marker", markers)): + for item in items: + if len(version_samples) < 20: + version_samples.append( + { + "kind": kind, + "key": item.get("Key"), + "versionId": item.get("VersionId"), + } + ) + inventory["versions"] = version_count + inventory["deleteMarkers"] = marker_count + inventory["versionSamples"] = version_samples + except Exception as exc: + inventory["versionInspectionError"] = f"{type(exc).__name__}: {exc}" + try: + object_count = 0 + object_samples: list[str] = [] + for page in client.get_paginator("list_objects_v2").paginate( + Bucket=bucket + ): + objects = page.get("Contents", []) + object_count += len(objects) + for item in objects: + if len(object_samples) < 20: + object_samples.append(str(item.get("Key", ""))) + inventory["objects"] = object_count + inventory["objectSamples"] = object_samples + except Exception as exc: + inventory["objectInspectionError"] = f"{type(exc).__name__}: {exc}" + return inventory + + def cleanup_bucket_contents(self, client: Any, bucket: str) -> bool: + """Empty one test bucket without deleting its namespace. + + Site-replicated peers with the same bucket name must all cross this + barrier before either namespace is removed. Otherwise the first + DeleteBucket hook can observe live versions on its peer and persist a + destructive retry liability that contaminates later cases. + """ + + deadline = time.monotonic() + 30 + last_error: BaseException | None = None + changed = False + while time.monotonic() < deadline: + try: + uploads = client.list_multipart_uploads(Bucket=bucket).get( + "Uploads", + [], + ) + for upload in uploads: + changed = True + client.abort_multipart_upload( + Bucket=bucket, + Key=upload["Key"], + UploadId=upload["UploadId"], + ) + + version_items: list[dict[str, str]] = [] + for page in client.get_paginator("list_object_versions").paginate( + Bucket=bucket + ): + version_items.extend( + {"Key": item["Key"], "VersionId": item["VersionId"]} + for group in ("Versions", "DeleteMarkers") + for item in page.get(group, []) + ) + if version_items: + changed = True + self.delete_object_batch(client, bucket, version_items) + + object_items: list[dict[str, str]] = [] + for page in client.get_paginator("list_objects_v2").paginate( + Bucket=bucket + ): + object_items.extend( + {"Key": item["Key"]} for item in page.get("Contents", []) + ) + if object_items: + changed = True + self.delete_object_batch(client, bucket, object_items) + + if not uploads and not version_items and not object_items: + return changed + last_error = RuntimeError( + "bucket contents did not converge to empty" + ) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code == "NoSuchBucket": + return changed + if code != "BucketNotEmpty": + raise + last_error = exc + remaining = deadline - time.monotonic() + if remaining > 0: + time.sleep(min(self.args.poll_interval, remaining)) + raise BucketCleanupResidual( + bucket, + last_error, + self.inspect_bucket_cleanup_inventory(client, bucket), + ) + + def delete_empty_bucket(self, client: Any, bucket: str) -> None: + deadline = time.monotonic() + 30 + last_error: BaseException | None = None + while time.monotonic() < deadline: + try: + client.delete_bucket(Bucket=bucket) + return + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code == "NoSuchBucket": + return + if code != "BucketNotEmpty": + raise + last_error = exc + remaining = deadline - time.monotonic() + if remaining > 0: + time.sleep(min(self.args.poll_interval, remaining)) + raise BucketCleanupResidual( + bucket, + last_error, + self.inspect_bucket_cleanup_inventory(client, bucket), + ) + + def cleanup_bucket(self, client: Any, bucket: str) -> None: + """Compatibility wrapper for callers cleaning one independent bucket.""" + + self.delete_replication_configuration(client, bucket) + self.cleanup_bucket_contents(client, bucket) + self.delete_empty_bucket(client, bucket) + + def site_retry_stats_snapshot(self) -> dict[str, Any]: + return { + site.id: self.site_info(site).get("retryStats") + for site in (self.site_a, self.site_b) + } + + @staticmethod + def normalized_retry_stats(value: Any) -> tuple[int, int, str] | None: + if value is None or value == {}: + return 0, 0, "" + if not isinstance(value, dict): + return None + pending = value.get("pending", 0) + failed = value.get("failed", 0) + last_error = value.get("lastError", "") + if ( + not isinstance(pending, int) + or isinstance(pending, bool) + or pending < 0 + or not isinstance(failed, int) + or isinstance(failed, bool) + or failed < 0 + or not isinstance(last_error, str) + ): + return None + return pending, failed, last_error + + @classmethod + def site_retry_state_regressions( + cls, + before: dict[str, Any], + after: dict[str, Any], + ) -> list[dict[str, Any]]: + regressions: list[dict[str, Any]] = [] + for site_id in ("site-a", "site-b"): + before_raw = before.get(site_id) + after_raw = after.get(site_id) + before_stats = cls.normalized_retry_stats(before_raw) + after_stats = cls.normalized_retry_stats(after_raw) + if after_stats is None: + regressions.append( + { + "site": site_id, + "reason": "post-cleanup retryStats is malformed", + "before": before_raw, + "after": after_raw, + } + ) + continue + if before_raw == after_raw: + continue + after_pending, after_failed, after_error = after_stats + if after_pending == 0 and after_failed == 0 and not after_error: + continue + if before_stats is None: + regressions.append( + { + "site": site_id, + "reason": "retry state changed from an unverifiable baseline", + "before": before_raw, + "after": after_raw, + } + ) + continue + before_pending, before_failed, before_error = before_stats + improved = ( + after_pending <= before_pending + and after_failed <= before_failed + and (after_pending < before_pending or after_failed < before_failed) + ) + unchanged = ( + after_pending == before_pending + and after_failed == before_failed + and after_error == before_error + ) + if not improved and not unchanged: + regressions.append( + { + "site": site_id, + "reason": "cleanup introduced or worsened retry debt", + "before": before_raw, + "after": after_raw, + } + ) + return regressions + + def cleanup_target(self, client: Any, bucket: str) -> dict[str, str]: + if client is self.client_a: + site = "site-a" + elif client is self.client_b: + site = "site-b" + else: + site = "unknown" + return {"site": site, "bucket": bucket} + + def tracked_bucket_summaries(self) -> list[dict[str, str]]: + summaries: list[dict[str, str]] = [] + seen: set[tuple[int, str]] = set() + for client, bucket in self.created_buckets: + identity = (id(client), bucket) + if identity in seen: + continue + seen.add(identity) + summaries.append(self.cleanup_target(client, bucket)) + return summaries + + def cleanup_created_buckets(self, case_id: str) -> None: + errors: list[dict[str, Any]] = [] + residuals: list[dict[str, Any]] = [] + unique: list[tuple[Any, str]] = [] + seen: set[tuple[int, str]] = set() + for client, bucket in self.created_buckets: + identity = (id(client), bucket) + if identity not in seen: + seen.add(identity) + unique.append((client, bucket)) + if self.args.module == "site": + # A site bucket may reach its peer immediately before a failed + # wait returns, so the peer is not necessarily in created_buckets. + # Both deterministic test namespaces remain in cleanup scope. + bucket_names = list(dict.fromkeys(bucket for _, bucket in unique)) + unique = [ + (client, bucket) + for bucket in bucket_names + for client in (self.client_a, self.client_b) + ] + remaining = {(id(client), bucket) for client, bucket in unique} + retry_check_required = self.args.module == "site" and bool(unique) + retry_before: dict[str, Any] | None = None + retry_after: dict[str, Any] | None = None + retry_regressions: list[dict[str, Any]] = [] + retry_isolation_unverified = False + if retry_check_required: + try: + retry_before = self.site_retry_stats_snapshot() + except Exception as exc: + retry_isolation_unverified = True + errors.append( + { + "site": "site-group", + "phase": "retry-state-before-cleanup", + "errorType": type(exc).__name__, + "message": str(exc), + } + ) + + # Remove operator-authored replication configuration while every + # tracked peer bucket still exists. RustFS deliberately retains the + # derived site-replication rules, so this step alone is not an + # all-clear for namespace deletion. + for client, bucket in unique: + try: + self.delete_replication_configuration(client, bucket) + except Exception as exc: + errors.append( + { + **self.cleanup_target(client, bucket), + "phase": "disable-replication", + "errorType": type(exc).__name__, + "message": str(exc), + } + ) + + bucket_sites: dict[str, set[int]] = {} + for client, bucket in unique: + bucket_sites.setdefault(bucket, set()).add(id(client)) + shared_site_bucket = any(len(clients) > 1 for clients in bucket_sites.values()) + contents_clean = True + stable_empty_rounds = 0 + content_barrier_deadline = time.monotonic() + 30 + while contents_clean: + round_changed = False + for client, bucket in unique: + try: + round_changed = ( + self.cleanup_bucket_contents(client, bucket) + or round_changed + ) + except BucketCleanupResidual as exc: + contents_clean = False + residuals.append( + { + "phase": "empty-contents", + **exc.inventory, + **self.cleanup_target(client, bucket), + } + ) + except Exception as exc: + contents_clean = False + errors.append( + { + **self.cleanup_target(client, bucket), + "phase": "empty-contents", + "errorType": type(exc).__name__, + "message": str(exc), + } + ) + if not contents_clean or not shared_site_bucket: + break + stable_empty_rounds = 0 if round_changed else stable_empty_rounds + 1 + if stable_empty_rounds >= 2: + break + remaining_time = content_barrier_deadline - time.monotonic() + if remaining_time <= 0: + contents_clean = False + errors.append( + { + "site": "site-group", + "phase": "empty-contents-stability", + "errorType": "BucketCleanupStabilityTimeout", + "message": ( + "shared site bucket contents did not remain empty " + "across two consecutive cleanup rounds" + ), + "inventories": [ + { + **self.inspect_bucket_cleanup_inventory( + client, + bucket, + ), + **self.cleanup_target(client, bucket), + } + for client, bucket in unique + ], + } + ) + break + time.sleep(min(self.args.poll_interval, remaining_time)) + + # An all-content barrier prevents a peer DeleteBucket hook from seeing + # live versions on the other side. If any side is uncertain, retain + # every namespace and let the existing isolation failure path preserve + # evidence instead of creating destructive retry debt. + if contents_clean: + for client, bucket in reversed(unique): + identity = (id(client), bucket) + try: + self.delete_empty_bucket(client, bucket) + except BucketCleanupResidual as exc: + residuals.append( + { + "phase": "delete-bucket", + **exc.inventory, + **self.cleanup_target(client, bucket), + } + ) + except Exception as exc: + errors.append( + { + **self.cleanup_target(client, bucket), + "phase": "delete-bucket", + "errorType": type(exc).__name__, + "message": str(exc), + } + ) + else: + remaining.discard(identity) + + if retry_check_required: + try: + retry_after = self.site_retry_stats_snapshot() + except Exception as exc: + retry_isolation_unverified = True + errors.append( + { + "site": "site-group", + "phase": "retry-state-after-cleanup", + "errorType": type(exc).__name__, + "message": str(exc), + } + ) + if retry_before is not None and retry_after is not None: + retry_regressions = self.site_retry_state_regressions( + retry_before, + retry_after, + ) + if retry_regressions: + errors.append( + { + "site": "site-group", + "phase": "retry-state-isolation", + "errorType": "CleanupRetryStateRegression", + "message": ( + "logical cleanup introduced or worsened site " + "replication retry debt" + ), + } + ) + + self.created_buckets = [ + (client, bucket) + for client, bucket in unique + if (id(client), bucket) in remaining + ] + if residuals or errors: + tracked_buckets = self.tracked_bucket_summaries() + retry_isolation_failed = bool(retry_regressions) or ( + retry_check_required and retry_isolation_unverified + ) + isolation_restored = not tracked_buckets and not retry_isolation_failed + if isolation_restored: + self.isolation_failure = None + evidence = self.save_json( + case_id, + "cleanup-failure.json", + { + "classification": "logical-cleanup-failure", + "authoritativeCleanup": "replication phase data reset", + "isolationRestored": isolation_restored, + "trackedBuckets": tracked_buckets, + "residuals": residuals, + "errors": errors, + "retryState": ( + { + "before": retry_before, + "after": retry_after, + "regressions": retry_regressions, + "verified": not retry_isolation_unverified, + } + if retry_check_required + else None + ), + }, + ) + self.events.record( + "case.cleanup.failure", + case=case_id, + evidence=evidence, + isolationRestored=isolation_restored, + residualBuckets=[ + {"site": item["site"], "bucket": item["bucket"]} + for item in residuals + ], + errorCount=len(errors), + trackedBuckets=tracked_buckets, + ) + raise LogicalCleanupFailure( + residuals=residuals, + errors=errors, + evidence=evidence, + tracked_buckets=tracked_buckets, + isolation_restored=isolation_restored, + ) + self.isolation_failure = None + + # ------------------------------------------------------------------ + # Bucket replication cases + # ------------------------------------------------------------------ + + def case_bucket_source_versioning_required(self, case_id: str) -> list[str]: + source = self.bucket_name(case_id, "src") + target = self.bucket_name(case_id, "dst") + self.create_bucket(self.client_a, source) + self.create_bucket(self.client_b, target) + self.enable_versioning(self.client_b, target) + result = self.run_rc( + case_id, + "source-versioning-required", + [ + "bucket", + "replication", + "add", + f"{self.alias_a}/{source}", + "--remote-bucket", + f"{self.alias_b}/{target}", + "--force", + ], + expect_success=False, + ) + error = (result.stdout + result.stderr).lower() + if "version" not in error and "replication" not in error: + raise AssertionError(f"unexpected source-versioning error: {error[:500]}") + try: + self.client_a.get_bucket_replication(Bucket=source) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code not in { + "ReplicationConfigurationNotFoundError", + "NoSuchReplicationConfiguration", + "NoSuchConfiguration", + }: + raise + else: + raise AssertionError("rejected source acquired a replication configuration") + return ["cases/%s/source-versioning-required.json" % safe_name(case_id)] + + def case_bucket_target_versioning_required(self, case_id: str) -> list[str]: + source = self.bucket_name(case_id, "src") + target = self.bucket_name(case_id, "dst") + self.create_bucket(self.client_a, source) + self.create_bucket(self.client_b, target) + self.enable_versioning(self.client_a, source) + result = self.run_rc( + case_id, + "target-versioning-required", + [ + "bucket", + "replication", + "add", + f"{self.alias_a}/{source}", + "--remote-bucket", + f"{self.alias_b}/{target}", + "--force", + ], + expect_success=False, + ) + error = (result.stdout + result.stderr).lower() + if "version" not in error and "remote" not in error: + raise AssertionError(f"unexpected target-versioning error: {error[:500]}") + return ["cases/%s/target-versioning-required.json" % safe_name(case_id)] + + def case_bucket_configuration_roundtrip(self, case_id: str) -> list[str]: + source, target, config = self.setup_bucket_replication( + case_id, flags="delete,delete-marker,existing-objects", prefix="roundtrip/" + ) + list_result = self.run_rc( + case_id, + "replication-list", + ["bucket", "replication", "list", f"{self.alias_a}/{source}", "--force"], + ) + combined = list_result.stdout + rule = config["Rules"][0] + rule_id = rule.get("ID", "") + if source not in combined and rule_id not in combined: + raise AssertionError("rc list did not identify the configured rule") + destination = rule.get("Destination", {}).get("Bucket", "") + if not destination or target not in destination: + raise AssertionError(f"destination does not identify target bucket: {destination!r}") + if rule.get("Priority") != 1 or rule.get("Status") != "Enabled": + raise AssertionError(f"replication rule priority/status drifted: {rule}") + if rule.get("Filter", {}).get("Prefix") != "roundtrip/": + raise AssertionError(f"replication prefix did not round-trip: {rule}") + if rule.get("DeleteMarkerReplication", {}).get("Status") != "Enabled": + raise AssertionError("delete-marker flag did not round-trip") + if rule.get("ExistingObjectReplication", {}).get("Status") != "Enabled": + raise AssertionError("existing-object flag did not round-trip") + listed = self.json_body(list_result.stdout.encode("utf-8")) + listed_rules = listed.get("rules", []) if isinstance(listed, dict) else [] + listed_rule = next( + (item for item in listed_rules if item.get("id") == rule_id), None + ) + if not listed_rule: + raise AssertionError(f"rc list omitted configured rule {rule_id!r}") + expected_flags = { + "deleteReplication": True, + "deleteMarkerReplication": True, + "existingObjectReplication": True, + } + if any(listed_rule.get(name) is not value for name, value in expected_flags.items()): + raise AssertionError(f"rc list flag round-trip mismatch: {listed_rule}") + self.client_a.delete_bucket_replication(Bucket=source) + try: + self.client_a.get_bucket_replication(Bucket=source) + except ClientError: + pass + else: + raise AssertionError("replication configuration remained after delete") + return [ + self.save_json(case_id, "roundtrip-final.json", {"ruleId": rule_id, "destination": destination}) + ] + + def case_bucket_put_object(self, case_id: str) -> list[str]: + source, target, _ = self.setup_bucket_replication(case_id) + key = "objects/basic.bin" + body = secrets.token_bytes(64 * 1024) + response = self.client_a.put_object( + Bucket=source, Key=key, Body=body, Metadata={"validation": "bucket-put"} + ) + version = response.get("VersionId") + final = self.wait_object( + case_id, self.client_b, target, key, body, expected_version=version + ) + if final.get("metadata", {}).get("validation") != "bucket-put": + raise AssertionError("object metadata did not replicate") + return [self.save_json(case_id, "object-final.json", final)] + + def case_bucket_sync_put_object(self, case_id: str) -> list[str]: + source, target, _ = self.setup_bucket_replication(case_id, synchronous=True) + key = "sync/immediate.txt" + body = b"synchronous-replication-check" + started = time.monotonic() + response = self.client_a.put_object(Bucket=source, Key=key, Body=body) + elapsed = time.monotonic() - started + try: + target_response = self.client_b.get_object(Bucket=target, Key=key) + except ClientError as exc: + raise AssertionError("synchronous PUT returned before target was readable") from exc + target_body = target_response["Body"].read() + if target_body != body or target_response.get("VersionId") != response.get("VersionId"): + raise AssertionError("synchronous target object differs from source response") + return [ + self.save_json( + case_id, + "sync-put.json", + { + "elapsedSeconds": elapsed, + "sourceVersionId": response.get("VersionId"), + "targetVersionId": target_response.get("VersionId"), + "sha256": hashlib.sha256(target_body).hexdigest(), + }, + ) + ] + + def case_bucket_prefix_filter(self, case_id: str) -> list[str]: + source, target, _ = self.setup_bucket_replication(case_id, prefix="included/") + included_key = "included/object.txt" + excluded_key = "excluded/control.txt" + included = b"prefix-match" + self.client_a.put_object(Bucket=source, Key=included_key, Body=included) + self.client_a.put_object(Bucket=source, Key=excluded_key, Body=b"prefix-control") + final = self.wait_object(case_id, self.client_b, target, included_key, included) + negative = self.assert_absent_for( + case_id, self.client_b, target, excluded_key, self.args.negative_window + ) + return [ + self.save_json(case_id, "prefix-match.json", final), + self.save_json(case_id, "prefix-control.json", negative), + ] + + def case_bucket_tag_filter(self, case_id: str) -> list[str]: + source, target, config = self.setup_bucket_replication(case_id) + rule = config["Rules"][0] + destination = rule["Destination"]["Bucket"] + filtered = { + "Role": config.get("Role", destination), + "Rules": [ + { + "ID": "tag-filter", + "Priority": 1, + "Filter": {"Tag": {"Key": "class", "Value": "gold"}}, + "Status": "Enabled", + "Destination": {"Bucket": destination}, + "DeleteMarkerReplication": {"Status": "Disabled"}, + "ExistingObjectReplication": {"Status": "Disabled"}, + } + ], + } + self.client_a.put_bucket_replication(Bucket=source, ReplicationConfiguration=filtered) + self.save_json(case_id, "tag-replication-configuration.json", filtered) + matching_key = "tags/gold.txt" + control_key = "tags/silver.txt" + matching = b"gold-object" + self.client_a.put_object(Bucket=source, Key=matching_key, Body=matching, Tagging="class=gold") + self.client_a.put_object(Bucket=source, Key=control_key, Body=b"silver-object", Tagging="class=silver") + final = self.wait_object(case_id, self.client_b, target, matching_key, matching) + negative = self.assert_absent_for( + case_id, self.client_b, target, control_key, self.args.negative_window + ) + return [ + self.save_json(case_id, "tag-match.json", final), + self.save_json(case_id, "tag-control.json", negative), + ] + + def case_bucket_zero_byte_special_key(self, case_id: str) -> list[str]: + source, target, _ = self.setup_bucket_replication(case_id) + fixtures = { + "zero/empty.bin": b"", + "special/space key.txt": b"space", + "special/中文-对象.txt": "你好 RustFS".encode("utf-8"), + "special/plus+percent%question?.txt": b"reserved", + } + manifest: list[dict[str, Any]] = [] + for key, body in fixtures.items(): + response = self.client_a.put_object(Bucket=source, Key=key, Body=body) + final = self.wait_object( + case_id, + self.client_b, + target, + key, + body, + expected_version=response.get("VersionId"), + ) + manifest.append({"key": key, **final}) + return [self.save_json(case_id, "special-key-manifest.json", manifest)] + + def case_bucket_multipart_object(self, case_id: str) -> list[str]: + source, target, _ = self.setup_bucket_replication(case_id) + key = "multipart/complete.bin" + parts = [b"a" * (5 * 1024 * 1024), b"b" * (1024 * 1024)] + created = self.client_a.create_multipart_upload(Bucket=source, Key=key) + uploaded = [] + for number, body in enumerate(parts, 1): + response = self.client_a.upload_part( + Bucket=source, + Key=key, + UploadId=created["UploadId"], + PartNumber=number, + Body=body, + ) + uploaded.append({"PartNumber": number, "ETag": response["ETag"]}) + complete = self.client_a.complete_multipart_upload( + Bucket=source, + Key=key, + UploadId=created["UploadId"], + MultipartUpload={"Parts": uploaded}, + ) + body = b"".join(parts) + final = self.wait_object( + case_id, + self.client_b, + target, + key, + body, + expected_version=complete.get("VersionId"), + ) + return [ + self.save_json( + case_id, + "multipart-final.json", + {"uploadId": created["UploadId"], "parts": uploaded, "object": final}, + ) + ] + + def case_bucket_copy_object(self, case_id: str) -> list[str]: + source, target, _ = self.setup_bucket_replication(case_id) + original_key = "copy/original.txt" + copy_key = "copy/clone.txt" + body = b"copy-object-replication" + self.client_a.put_object(Bucket=source, Key=original_key, Body=body) + copied = self.client_a.copy_object( + Bucket=source, + Key=copy_key, + CopySource={"Bucket": source, "Key": original_key}, + Metadata={"copied": "true"}, + MetadataDirective="REPLACE", + ) + final = self.wait_object( + case_id, + self.client_b, + target, + copy_key, + body, + expected_version=copied.get("VersionId"), + ) + if final.get("metadata", {}).get("copied") != "true": + raise AssertionError("CopyObject metadata did not replicate") + return [self.save_json(case_id, "copy-final.json", final)] + + def case_bucket_delete_marker(self, case_id: str) -> list[str]: + source, target, _ = self.setup_bucket_replication(case_id, flags="delete-marker") + key = "delete-marker/object.txt" + put = self.client_a.put_object(Bucket=source, Key=key, Body=b"delete-marker") + self.wait_object( + case_id, self.client_b, target, key, b"delete-marker", expected_version=put.get("VersionId") + ) + deleted = self.client_a.delete_object(Bucket=source, Key=key) + marker_id = deleted.get("VersionId") + + def probe() -> tuple[bool, Any]: + response = self.client_b.list_object_versions(Bucket=target, Prefix=key) + markers = [ + item["VersionId"] + for item in response.get("DeleteMarkers", []) + if item["Key"] == key + ] + versions = [ + item["VersionId"] for item in response.get("Versions", []) if item["Key"] == key + ] + return marker_id in markers and put.get("VersionId") in versions, { + "deleteMarkers": markers, + "versions": versions, + "expectedMarker": marker_id, + } + + final = self.wait_until(case_id, "delete marker replication", probe) + return [self.save_json(case_id, "delete-marker-final.json", final)] + + def case_bucket_version_delete(self, case_id: str) -> list[str]: + source, target, _ = self.setup_bucket_replication(case_id, flags="delete") + key = "version-delete/object.txt" + first = self.client_a.put_object(Bucket=source, Key=key, Body=b"version-one") + second = self.client_a.put_object(Bucket=source, Key=key, Body=b"version-two") + self.wait_object( + case_id, self.client_b, target, key, b"version-two", expected_version=second.get("VersionId") + ) + self.client_a.delete_object(Bucket=source, Key=key, VersionId=first["VersionId"]) + + def probe() -> tuple[bool, Any]: + response = self.client_b.list_object_versions(Bucket=target, Prefix=key) + versions = [ + item["VersionId"] for item in response.get("Versions", []) if item["Key"] == key + ] + return first["VersionId"] not in versions and second["VersionId"] in versions, { + "versions": versions, + "deleted": first["VersionId"], + "retained": second["VersionId"], + } + + final = self.wait_until(case_id, "permanent version delete replication", probe) + return [self.save_json(case_id, "version-delete-final.json", final)] + + def case_bucket_existing_object_enabled(self, case_id: str) -> list[str]: + key = "existing/before-rule.txt" + body = b"existing-object-enabled" + source, target, _ = self.setup_bucket_replication( + case_id, flags="existing-objects", preexisting=(key, body) + ) + source_version = self.client_a.head_object(Bucket=source, Key=key).get("VersionId") + final = self.wait_object( + case_id, + self.client_b, + target, + key, + body, + expected_version=source_version, + timeout=max(self.args.timeout, 240), + ) + return [self.save_json(case_id, "existing-object-final.json", final)] + + def case_bucket_existing_object_disabled(self, case_id: str) -> list[str]: + key = "existing/before-rule.txt" + body = b"existing-object-disabled" + source, target, _ = self.setup_bucket_replication(case_id, preexisting=(key, body)) + new_key = "existing/after-rule.txt" + new_body = b"new-object-positive-control" + self.client_a.put_object(Bucket=source, Key=new_key, Body=new_body) + positive = self.wait_object(case_id, self.client_b, target, new_key, new_body) + negative = self.assert_absent_for( + case_id, self.client_b, target, key, max(self.args.negative_window, 20) + ) + return [ + self.save_json(case_id, "new-object-control.json", positive), + self.save_json(case_id, "existing-object-control.json", negative), + ] + + def case_bucket_target_check(self, case_id: str) -> list[str]: + source, target, _ = self.setup_bucket_replication( + case_id, flags="delete,delete-marker" + ) + status, headers, body = self.signed_request( + self.site_a, "GET", f"/{source}", query="replication-check" + ) + evidence = { + "httpStatus": status, + "headers": headers, + "body": body.decode("utf-8", errors="replace"), + } + if status != 200: + raise AssertionError(f"replication-check returned HTTP {status}: {evidence['body'][:500]}") + if not body.strip(): + raise AssertionError("replication-check returned an empty response") + parsed = self.json_body(body) + evidence["parsed"] = parsed + overall = str(parsed.get("Status", parsed.get("status", ""))).upper() + if overall != "OK": + raise AssertionError(f"replication-check reported {overall}: {parsed}") + if parsed.get("ActiveMutation") is not True: + raise AssertionError("replication-check did not identify its active mutation") + if parsed.get("ProbeNamespace") != ".rustfs.sys/replication-check/": + raise AssertionError(f"unexpected probe namespace: {parsed.get('ProbeNamespace')!r}") + targets = parsed.get("Targets", []) + if not targets: + raise AssertionError("replication-check returned no target results") + required_phases = { + "Bucket", + "Versioning", + "ObjectLock", + "Put", + "VersionFidelity", + "DeleteMarker", + "VersionDelete", + "Cleanup", + } + for item in targets: + if str(item.get("Status", "")).upper() != "OK": + raise AssertionError(f"replication-check target failed: {item}") + phases = item.get("Phases", {}) + missing = required_phases - set(phases) + failed = { + name: value + for name, value in phases.items() + if name in required_phases and str(value.get("Status", "")).upper() != "OK" + } + if missing or failed: + raise AssertionError( + f"replication-check phase mismatch: missing={sorted(missing)} failed={failed}" + ) + leftovers = self.client_b.list_object_versions( + Bucket=target, Prefix=".rustfs.sys/replication-check/" + ) + if leftovers.get("Versions") or leftovers.get("DeleteMarkers"): + raise AssertionError("replication-check left probe artifacts on the target") + return [ + self.save_json(case_id, "replication-check.json", evidence), + self.save_json(case_id, "replication-check-leftovers.json", leftovers), + ] + + def case_bucket_status_observability(self, case_id: str) -> list[str]: + source, target, config = self.setup_bucket_replication(case_id) + body = b"status-observability" + put = self.client_a.put_object(Bucket=source, Key="status/object.txt", Body=body) + self.wait_object( + case_id, + self.client_b, + target, + "status/object.txt", + body, + expected_version=put.get("VersionId"), + ) + listed = self.run_rc( + case_id, + "status-list", + ["bucket", "replication", "list", f"{self.alias_a}/{source}", "--force"], + ) + raw_status, raw_headers, raw_body = self.admin_request( + self.site_a, + "GET", + "replicationmetrics", + query=urllib.parse.urlencode({"bucket": source}), + ) + raw_document = { + "httpStatus": raw_status, + "headers": raw_headers, + "body": raw_body.decode("utf-8", errors="replace"), + } + if raw_status != 200: + raise AssertionError(f"raw replication metrics returned HTTP {raw_status}") + raw_document["parsed"] = self.json_body(raw_body) + self.save_json(case_id, "raw-replication-metrics.json", raw_document) + status_arguments = [ + "bucket", + "replication", + "status", + f"{self.alias_a}/{source}", + "--force", + ] + + def metrics_probe() -> tuple[bool, Any]: + observed = self.rc_probe(status_arguments) + if observed.returncode != 0: + try: + error_document = json.loads(observed.stderr) + except json.JSONDecodeError: + error_document = {} + details = error_document.get("error", {}) if isinstance(error_document, dict) else {} + if details and details.get("retryable") is False: + raise AssertionError( + "rc could not decode the server replication metrics response: " + f"{details.get('message', observed.stderr[:1000])}" + ) + return False, { + "exitCode": observed.returncode, + "stderr": observed.stderr[:1000], + } + try: + document = json.loads(observed.stdout) + except json.JSONDecodeError: + return False, {"stdout": observed.stdout[:1000]} + return replication_metrics_ready(document), document + + metrics = self.wait_until(case_id, "non-zero replication status metrics", metrics_probe) + self.save_json(case_id, "status-metrics.json", metrics) + combined = listed.stdout + json.dumps(metrics) + identifiers = [source, config["Rules"][0].get("ID", ""), target] + if not any(identifier and identifier in combined for identifier in identifiers): + raise AssertionError("rc replication output contains no configured identifier") + if self.args.secret_key in combined: + raise AssertionError("rc replication status leaked the administrator secret") + return [ + self.save_json( + case_id, + "observability-summary.json", + { + "identifiers": identifiers, + "outputBytes": len(combined), + "metrics": metrics, + }, + ) + ] + + def case_bucket_outage_recovery(self, case_id: str) -> list[str]: + source, target, _ = self.setup_bucket_replication(case_id) + key = "outage/recover.txt" + body = b"target-outage-recovery" + evidence = [self.hook(case_id, "site-b-stop", self.args.site_b_stop_hook)] + try: + response = self.client_a.put_object(Bucket=source, Key=key, Body=body) + finally: + evidence.append(self.hook(case_id, "site-b-start", self.args.site_b_start_hook)) + self.wait_api(case_id, self.client_b, "site-b") + final = self.wait_object( + case_id, + self.client_b, + target, + key, + body, + expected_version=response.get("VersionId"), + timeout=max(self.args.timeout, 180), + ) + evidence.append(self.save_json(case_id, "outage-recovery-final.json", final)) + return evidence + + def case_bucket_source_restart_recovery(self, case_id: str) -> list[str]: + source, target, _ = self.setup_bucket_replication(case_id) + key = "mrf/restart.txt" + body = b"source-restart-mrf-recovery" + evidence = [self.hook(case_id, "site-b-stop", self.args.site_b_stop_hook)] + b_started = False + try: + response = self.client_a.put_object(Bucket=source, Key=key, Body=body) + evidence.append(self.hook(case_id, "site-a-restart", self.args.site_a_restart_hook)) + self.wait_api(case_id, self.client_a, "site-a") + evidence.append(self.hook(case_id, "site-b-start", self.args.site_b_start_hook)) + b_started = True + self.wait_api(case_id, self.client_b, "site-b") + finally: + if not b_started: + self.hook(case_id, "site-b-start-recovery", self.args.site_b_start_hook) + self.wait_api(case_id, self.client_b, "site-b") + final = self.wait_object( + case_id, + self.client_b, + target, + key, + body, + expected_version=response.get("VersionId"), + timeout=max(self.args.timeout, 240), + ) + evidence.append(self.save_json(case_id, "source-restart-final.json", final)) + return evidence + + # ------------------------------------------------------------------ + # Site replication cases + # ------------------------------------------------------------------ + + def site_info(self, site: Site) -> dict[str, Any]: + status, _, body = self.admin_request(site, "GET", "site-replication/info") + if status != 200: + raise AssertionError( + f"{site.id} site-replication info returned HTTP {status}: " + f"{body.decode(errors='replace')[:500]}" + ) + parsed = self.json_body(body) + if not isinstance(parsed, dict): + raise AssertionError(f"{site.id} info is not an object") + return parsed + + def site_group_snapshot(self) -> tuple[str, str, dict[str, Any]]: + info = { + site.id: self.site_info(site) + for site in (self.site_a, self.site_b) + } + state, reason = classify_site_group_state( + info, + (self.site_a.endpoint, self.site_b.endpoint), + ) + return state, reason, info + + def wait_site_group_active(self, case_id: str) -> dict[str, Any]: + def probe() -> tuple[bool, Any]: + try: + state, reason, info = self.site_group_snapshot() + except Exception as exc: + return False, { + "state": "unavailable", + "reason": f"{type(exc).__name__}: {exc}", + } + detail = {"state": state, "reason": reason, "sites": info} + return state == "active", detail + + return self.wait_until( + case_id, + "stable two-site replication group", + probe, + timeout=max(self.args.timeout, 120), + ) + + def wait_site_group_disabled(self, case_id: str) -> dict[str, Any]: + def probe() -> tuple[bool, Any]: + try: + state, reason, info = self.site_group_snapshot() + except Exception as exc: + return False, { + "state": "unavailable", + "reason": f"{type(exc).__name__}: {exc}", + } + detail = {"state": state, "reason": reason, "sites": info} + return state == "disabled", detail + + return self.wait_until( + case_id, + "cleanly disabled two-site replication group", + probe, + timeout=max(self.args.timeout, 120), + ) + + def wait_site_group_ilm_expiry_value( + self, + case_id: str, + expected: bool, + ) -> dict[str, Any]: + def probe() -> tuple[bool, Any]: + try: + state, state_reason, info = self.site_group_snapshot() + except Exception as exc: + return False, { + "state": "unavailable", + "stateReason": f"{type(exc).__name__}: {exc}", + } + valid, flag_reason = site_group_has_ilm_expiry_value(info, expected) + detail = { + "state": state, + "stateReason": state_reason, + "flagReason": flag_reason, + "sites": info, + } + return state == "active" and valid, detail + + return self.wait_until( + case_id, + f"site replicate-ilm-expiry={str(expected).lower()}", + probe, + timeout=max(self.args.timeout, 120), + ) + + def ensure_site_group_active( + self, + case_id: str, + *, + evidence_prefix: str, + refuse_recreate: bool, + ) -> tuple[str, dict[str, Any]]: + self.events.record( + "setup.start", + case=case_id, + fixture=SITE_GROUP_ACTIVE_FIXTURE, + ) + try: + state, reason, initial = self.site_group_snapshot() + self.save_json( + case_id, + f"{evidence_prefix}-initial.json", + {"state": state, "reason": reason, "sites": initial}, + ) + outcome = "reused" + if state == "disabled": + if refuse_recreate: + raise PreconditionError( + "site group became disabled after the diagnostic fixture was established" + ) + self.add_site_replication_group( + case_id, + f"{evidence_prefix}-add-response.json", + require_clean_initial_sync=True, + ) + outcome = "created" + elif state != "active": + raise PreconditionError( + f"site-group fixture refuses {state} control-plane state: {reason}" + ) + + final = ( + {"state": state, "reason": reason, "sites": initial} + if state == "active" + else self.wait_site_group_active(case_id) + ) + self.save_json( + case_id, + f"{evidence_prefix}-final.json", + {"outcome": outcome, **final}, + ) + self.events.record( + "setup.group.ready", + case=case_id, + fixture=SITE_GROUP_ACTIVE_FIXTURE, + outcome=outcome, + ) + return outcome, final + except Exception as exc: + self.events.record( + "setup.error", + case=case_id, + fixture=SITE_GROUP_ACTIVE_FIXTURE, + errorType=type(exc).__name__, + message=self.redact(str(exc)), + ) + if isinstance(exc, PreconditionError): + raise + raise PreconditionError( + f"site-group fixture could not establish an active group: " + f"{type(exc).__name__}: {exc}" + ) from exc + + def ensure_planned_fixture(self, case_id: str) -> None: + if ( + SITE_GROUP_ACTIVE_FIXTURE not in self.requested_fixtures + or case_id not in SITE_GROUP_ACTIVE_CASE_IDS + ): + return + try: + if self.site_group_tracker_state in {"tainted", "removed"}: + raise PreconditionError( + "site-group fixture refuses to repair a group after an earlier " + f"{self.site_group_tracker_state} transition" + ) + outcome, final = self.ensure_site_group_active( + case_id, + evidence_prefix="setup-site-group", + refuse_recreate=( + self.site_group_tracker_state != "fresh" + or self.site_group_mutation_attempted + ), + ) + self.site_group_tracker_state = "active" + if case_id == "site-replication.lifecycle-expiry-toggle": + valid, reason = site_group_has_ilm_expiry_value( + final["sites"], + False, + ) + self.save_json( + case_id, + "setup-site-group-ilm-expiry.json", + {"valid": valid, "reason": reason, "sites": final["sites"]}, + ) + if not valid: + self.events.record( + "setup.constraint.error", + case=case_id, + fixture=SITE_GROUP_ACTIVE_FIXTURE, + message=reason, + ) + raise PreconditionError(reason) + self.events.record( + "setup.complete", + case=case_id, + fixture=SITE_GROUP_ACTIVE_FIXTURE, + outcome=outcome, + ) + self.fixture_outcomes.append( + { + "caseId": case_id, + "fixture": SITE_GROUP_ACTIVE_FIXTURE, + "state": "pass", + "outcome": outcome, + } + ) + except Exception as exc: + message = ( + str(exc) + if isinstance(exc, PreconditionError) + else f"{type(exc).__name__}: {exc}" + ) + redacted_message = self.redact(f"precondition_error: {message}") + self.save_json( + case_id, + "setup-site-group-error.json", + { + "fixture": SITE_GROUP_ACTIVE_FIXTURE, + "state": "infra_error", + "message": redacted_message, + "trackerState": self.site_group_tracker_state, + }, + ) + self.events.record( + "setup.blocked", + case=case_id, + fixture=SITE_GROUP_ACTIVE_FIXTURE, + message=redacted_message, + ) + self.fixture_outcomes.append( + { + "caseId": case_id, + "fixture": SITE_GROUP_ACTIVE_FIXTURE, + "state": "infra_error", + "message": redacted_message, + } + ) + if isinstance(exc, PreconditionError): + raise + raise PreconditionError(message) from exc + + def site_peer(self, info: dict[str, Any], endpoint: str) -> dict[str, Any]: + identity = _site_endpoint_identity(endpoint) + if identity is None: + raise AssertionError(f"invalid configured site endpoint: {endpoint}") + matches = [ + peer + for peer in info.get("sites", []) + if _site_endpoint_identity(peer.get("endpoint")) == identity + ] + if len(matches) != 1: + raise AssertionError(f"expected one peer for {endpoint}, got {len(matches)}") + return matches[0] + + def wait_bucket(self, case_id: str, client: Any, bucket: str, present: bool = True) -> Any: + def probe() -> tuple[bool, Any]: + try: + response = client.get_bucket_location(Bucket=bucket) + exists = True + detail: Any = without_response_metadata(response) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "unknown") + exists = code not in {"404", "NoSuchBucket", "NotFound"} + detail = {"error": code} + return exists is present, detail + + return self.wait_until(case_id, f"bucket {bucket} present={present}", probe) + + def create_site_bucket(self, case_id: str, role: str = "site") -> str: + bucket = self.bucket_name(case_id, role) + self.create_bucket(self.client_a, bucket) + self.wait_bucket(case_id, self.client_b, bucket) + self.created_buckets.append((self.client_b, bucket)) + return bucket + + def wait_versioning(self, case_id: str, bucket: str, status: str) -> dict[str, Any]: + def probe() -> tuple[bool, Any]: + response = self.client_b.get_bucket_versioning(Bucket=bucket) + observed = response.get("Status") + return observed == status, {"expected": status, "observed": observed} + + return self.wait_until(case_id, f"versioning={status} on site-b", probe) + + def case_site_unconfigured_info(self, case_id: str) -> list[str]: + info = {site.id: self.site_info(site) for site in (self.site_a, self.site_b)} + state, reason = classify_site_group_state( + info, + (self.site_a.endpoint, self.site_b.endpoint), + ) + if state != "disabled": + raise AssertionError( + f"fresh sites are not cleanly unconfigured: {state}: {reason}" + ) + return [self.save_json(case_id, "unconfigured-info.json", info)] + + def case_site_invalid_credentials(self, case_id: str) -> list[str]: + payload = [ + { + "name": "site-b-invalid", + "endpoints": self.site_b.endpoint, + "accessKey": self.site_b.access_key, + "secretKey": "definitely-invalid-site-secret", + } + ] + status, headers, body = self.admin_request( + self.site_a, + "PUT", + "site-replication/add", + query="replicateILMExpiry=false", + payload=payload, + ) + self.secrets.add("definitely-invalid-site-secret") + document = { + "httpStatus": status, + "headers": headers, + "body": body.decode("utf-8", errors="replace"), + } + if status < 400: + parsed = self.required_json_object(body, "invalid-credential site add") + if parsed.get("success") is not False: + raise AssertionError(f"site add with invalid credentials succeeded: {parsed}") + info = {site.id: self.site_info(site) for site in (self.site_a, self.site_b)} + state, reason = classify_site_group_state( + info, + (self.site_a.endpoint, self.site_b.endpoint), + ) + if state != "disabled": + raise AssertionError( + "invalid site add did not preserve a clean disabled state: " + f"{state}: {reason}" + ) + return [ + self.save_json(case_id, "invalid-add-response.json", document), + self.save_json(case_id, "invalid-add-info.json", info), + ] + + def case_site_add_and_info(self, case_id: str) -> list[str]: + result, response_evidence = self.add_site_replication_group( + case_id, "add-response.json", require_clean_initial_sync=True + ) + settled = self.wait_site_group_active(case_id) + info = settled["sites"] + serialized = json.dumps({"result": result, "info": info}) + if self.args.secret_key in serialized: + raise AssertionError("site replication response leaked administrator secret") + return [ + response_evidence, + self.save_json(case_id, "paired-info.json", settled), + ] + + def add_site_replication_group( + self, + case_id: str, + evidence_name: str, + *, + require_clean_initial_sync: bool, + ) -> tuple[dict[str, Any], str]: + self.site_group_mutation_attempted = True + peers = [ + { + "name": site.id, + "endpoints": site.endpoint, + "accessKey": site.access_key, + "secretKey": site.secret_key, + } + for site in (self.site_a, self.site_b) + ] + status, headers, body = self.admin_request( + self.site_a, + "PUT", + "site-replication/add", + query="replicateILMExpiry=false", + payload=peers, + ) + if status != 200: + raise AssertionError( + f"site-replication add HTTP {status}: {body.decode(errors='replace')[:500]}" + ) + result = self.required_json_object(body, "site-replication add") + response_evidence = self.save_json( + case_id, + evidence_name, + {"httpStatus": status, "headers": headers, "result": result}, + ) + if result.get("success") is not True: + raise AssertionError(f"site-replication add rejected: {result}") + if require_clean_initial_sync and result.get("initialSyncErrorMessage"): + raise AssertionError(f"site-replication initial sync was partial: {result}") + return result, response_evidence + + def case_site_status_observability(self, case_id: str) -> list[str]: + query = ( + "buckets=true&users=true&groups=true&policies=true&metrics=true&" + "peer-state=true&ilm-expiry-rules=true" + ) + status_documents: dict[str, Any] = {} + for site in (self.site_a, self.site_b): + status, _, body = self.admin_request( + site, "GET", "site-replication/status", query=query + ) + if status != 200: + raise AssertionError(f"{site.id} status returned HTTP {status}") + parsed = self.json_body(body) + status_documents[site.id] = parsed + if parsed.get("enabled") is not True: + raise AssertionError(f"{site.id} status is not enabled") + sites = parsed.get("Sites") or parsed.get("sites") + if not isinstance(sites, dict) or len(sites) != 2: + raise AssertionError(f"{site.id} status omitted peer sites") + for bound in ( + "MaxBuckets", + "MaxUsers", + "MaxGroups", + "MaxPolicies", + "MaxILMExpiryRules", + ): + if not isinstance(parsed.get(bound), int): + raise AssertionError(f"{site.id} status omitted numeric {bound}") + if not isinstance(parsed.get("Metrics"), dict): + raise AssertionError(f"{site.id} status omitted Metrics scope") + serialized = json.dumps(status_documents) + if self.args.secret_key in serialized or "secretKey" in serialized: + raise AssertionError("status response leaked credentials") + return [self.save_json(case_id, "site-status.json", status_documents)] + + def case_site_bucket_create(self, case_id: str) -> list[str]: + bucket = self.create_site_bucket(case_id, "bucket") + return [ + self.save_json( + case_id, + "bucket-create.json", + { + "bucket": bucket, + "siteA": [item["Name"] for item in self.client_a.list_buckets().get("Buckets", [])], + "siteB": [item["Name"] for item in self.client_b.list_buckets().get("Buckets", [])], + }, + ) + ] + + def case_site_object_bidirectional(self, case_id: str) -> list[str]: + bucket = self.create_site_bucket(case_id, "bidir") + self.enable_versioning(self.client_a, bucket) + self.wait_versioning(case_id, bucket, "Enabled") + fixtures = [ + (self.client_a, self.client_b, "from-a.txt", b"active-active-a", "a-to-b"), + (self.client_b, self.client_a, "from-b.txt", b"active-active-b", "b-to-a"), + ] + observations = [] + for source_client, target_client, key, body, direction in fixtures: + response = source_client.put_object( + Bucket=bucket, Key=key, Body=body, Metadata={"direction": direction} + ) + final = self.wait_object( + case_id, + target_client, + bucket, + key, + body, + expected_version=response.get("VersionId"), + ) + if final.get("metadata", {}).get("direction") != direction: + raise AssertionError(f"metadata mismatch for {direction}") + observations.append({"direction": direction, **final}) + time.sleep(max(2, self.args.poll_interval * 2)) + for client in (self.client_a, self.client_b): + versions = client.list_object_versions(Bucket=bucket) + relevant = [item for item in versions.get("Versions", []) if item["Key"] in {"from-a.txt", "from-b.txt"}] + if len(relevant) != 2: + raise AssertionError(f"replication loop or missing version detected: {relevant}") + return [self.save_json(case_id, "bidirectional-final.json", observations)] + + def case_site_versioning_metadata(self, case_id: str) -> list[str]: + bucket = self.create_site_bucket(case_id, "versioning") + self.client_a.put_bucket_versioning( + Bucket=bucket, VersioningConfiguration={"Status": "Enabled"} + ) + observations: dict[str, Any] = { + "enabled": self.wait_versioning(case_id, bucket, "Enabled") + } + try: + self.client_a.put_bucket_versioning( + Bucket=bucket, VersioningConfiguration={"Status": "Suspended"} + ) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code != "InvalidBucketState": + raise + observations["suspendRejected"] = { + "error": code, + "message": exc.response.get("Error", {}).get("Message", ""), + } + else: + raise AssertionError( + "site-replicated bucket allowed versioning suspension while a replication rule was active" + ) + source_status = self.client_a.get_bucket_versioning(Bucket=bucket).get("Status") + if source_status != "Enabled": + raise AssertionError(f"source versioning changed after rejected suspension: {source_status!r}") + observations["remainedEnabled"] = self.wait_versioning( + case_id, bucket, "Enabled" + ) + self.client_a.put_bucket_versioning( + Bucket=bucket, VersioningConfiguration={"Status": "Enabled"} + ) + observations["idempotentEnable"] = self.wait_versioning( + case_id, bucket, "Enabled" + ) + return [self.save_json(case_id, "versioning-propagation.json", observations)] + + def case_site_lifecycle_expiry_toggle(self, case_id: str) -> list[str]: + state, state_reason, initial_info = self.site_group_snapshot() + initial_evidence = self.save_json( + case_id, + "lifecycle-initial-site-info.json", + {"state": state, "reason": state_reason, "sites": initial_info}, + ) + if state != "active": + raise PreconditionError( + f"lifecycle case requires an active site group: {state}: {state_reason}" + ) + flags_valid, flags_reason = site_group_has_ilm_expiry_value( + initial_info, + False, + ) + if not flags_valid: + raise PreconditionError(flags_reason) + + bucket = self.create_site_bucket(case_id, "ilm") + initial = { + "Rules": [ + { + "ID": "expiry-disabled-at-site-level", + "Status": "Enabled", + "Filter": {"Prefix": "expiry/"}, + "Expiration": {"Days": 30}, + } + ] + } + self.client_a.put_bucket_lifecycle_configuration( + Bucket=bucket, LifecycleConfiguration=initial + ) + deadline = time.monotonic() + self.args.negative_window + while time.monotonic() < deadline: + try: + remote = self.client_b.get_bucket_lifecycle_configuration(Bucket=bucket) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") not in { + "NoSuchLifecycleConfiguration", + "NoSuchLifecycle", + }: + raise + else: + if remote.get("Rules"): + raise AssertionError("expiry rules replicated while global ILM expiry was disabled") + time.sleep(self.args.poll_interval) + enable_attempted = False + primary_error: Exception | None = None + rollback_error: Exception | None = None + replicated: dict[str, Any] = {} + enabled_info: dict[str, Any] = {} + final_info: dict[str, Any] = {} + enable_evidence = "" + disable_evidence = "" + try: + enable_attempted = True + status, headers, body = self.admin_request( + self.site_a, + "PUT", + "site-replication/edit", + query="enableILMExpiryReplication=true", + payload={}, + ) + enable_result = self.required_json_object( + body, + "enable ILM expiry replication", + ) + enable_evidence = self.save_json( + case_id, + "lifecycle-enable-response.json", + { + "httpStatus": status, + "headers": headers, + "result": enable_result, + }, + ) + if ( + status != 200 + or enable_result.get("success") is not True + ): + raise AssertionError( + f"enable ILM expiry replication failed: HTTP {status} {body!r}" + ) + enabled_info = self.wait_site_group_ilm_expiry_value(case_id, True) + updated = { + "Rules": [ + { + "ID": "expiry-enabled-at-site-level", + "Status": "Enabled", + "Filter": {"Prefix": "expiry/"}, + "Expiration": {"Days": 29}, + } + ] + } + self.client_a.put_bucket_lifecycle_configuration( + Bucket=bucket, LifecycleConfiguration=updated + ) + + def lifecycle_probe() -> tuple[bool, Any]: + try: + response = self.client_b.get_bucket_lifecycle_configuration( + Bucket=bucket + ) + except ClientError as exc: + return False, { + "error": exc.response.get("Error", {}).get("Code") + } + rules = response.get("Rules", []) + return any( + rule.get("ID") == "expiry-enabled-at-site-level" + for rule in rules + ), response + + replicated = self.wait_until( + case_id, + "ILM expiry metadata replication", + lifecycle_probe, + ) + except Exception as exc: + primary_error = exc + finally: + if enable_attempted: + try: + status, headers, body = self.admin_request( + self.site_a, + "PUT", + "site-replication/edit", + query="disableILMExpiryReplication=true", + payload={}, + ) + disable_result = self.required_json_object( + body, + "disable ILM expiry replication", + ) + disable_evidence = self.save_json( + case_id, + "lifecycle-disable-response.json", + { + "httpStatus": status, + "headers": headers, + "result": disable_result, + }, + ) + if ( + status != 200 + or disable_result.get("success") is not True + ): + raise AssertionError( + "disable ILM expiry replication failed: " + f"HTTP {status} {body!r}" + ) + final_info = self.wait_site_group_ilm_expiry_value( + case_id, + False, + ) + except Exception as exc: + rollback_error = exc + + if rollback_error is not None: + self.site_group_tracker_state = "tainted" + if primary_error is not None: + raise RuntimeError( + f"{type(primary_error).__name__}: {primary_error}; " + "lifecycle rollback failed: " + f"{type(rollback_error).__name__}: {rollback_error}" + ) from rollback_error + raise rollback_error + if primary_error is not None: + raise primary_error + return [ + initial_evidence, + enable_evidence, + self.save_json(case_id, "lifecycle-enabled-site-info.json", enabled_info), + self.save_json(case_id, "lifecycle-replicated.json", replicated), + disable_evidence, + self.save_json(case_id, "lifecycle-toggle-final-info.json", final_info), + ] + + def case_site_bucket_policy_metadata(self, case_id: str) -> list[str]: + bucket = self.create_site_bucket(case_id, "policy") + policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ReadObjects", + "Effect": "Allow", + "Principal": {"AWS": ["*"]}, + "Action": ["s3:GetObject"], + "Resource": [f"arn:aws:s3:::{bucket}/*"], + } + ], + } + self.client_a.put_bucket_policy(Bucket=bucket, Policy=json.dumps(policy)) + source_policy = json.loads( + self.client_a.get_bucket_policy(Bucket=bucket)["Policy"] + ) + + def policy_probe() -> tuple[bool, Any]: + try: + response = self.client_b.get_bucket_policy(Bucket=bucket) + except ClientError as exc: + return False, {"error": exc.response.get("Error", {}).get("Code")} + observed = json.loads(response["Policy"]) + return observed == source_policy, observed + + replicated = self.wait_until(case_id, "bucket policy replication", policy_probe) + self.client_a.delete_bucket_policy(Bucket=bucket) + + def delete_probe() -> tuple[bool, Any]: + try: + self.client_b.get_bucket_policy(Bucket=bucket) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + return code in {"NoSuchBucketPolicy", "NoSuchPolicy", "NoSuchBucket"}, {"error": code} + return False, {"policy": "still present"} + + deleted = self.wait_until(case_id, "bucket policy deletion replication", delete_probe) + return [ + self.save_json(case_id, "policy-replicated.json", replicated), + self.save_json(case_id, "policy-deleted.json", deleted), + ] + + def case_site_object_tags(self, case_id: str) -> list[str]: + bucket = self.create_site_bucket(case_id, "tags") + key = "tagged/object.txt" + body = b"site-replication-tags" + self.client_a.put_object(Bucket=bucket, Key=key, Body=body) + self.wait_object(case_id, self.client_b, bucket, key, body) + expected_sets = [ + [{"Key": "stage", "Value": "one"}], + [{"Key": "stage", "Value": "two"}, {"Key": "owner", "Value": "qa"}], + ] + observations = [] + for tags in expected_sets: + self.client_a.put_object_tagging(Bucket=bucket, Key=key, Tagging={"TagSet": tags}) + + def probe(tags=tags) -> tuple[bool, Any]: + response = self.client_b.get_object_tagging(Bucket=bucket, Key=key) + observed = sorted(response.get("TagSet", []), key=lambda item: item["Key"]) + expected = sorted(tags, key=lambda item: item["Key"]) + return observed == expected, observed + + observations.append(self.wait_until(case_id, "object tag replication", probe)) + return [self.save_json(case_id, "object-tags.json", observations)] + + def case_site_delete_convergence(self, case_id: str) -> list[str]: + bucket = self.create_site_bucket(case_id, "delete") + self.enable_versioning(self.client_a, bucket) + self.wait_versioning(case_id, bucket, "Enabled") + key = "delete/object.txt" + first = self.client_a.put_object(Bucket=bucket, Key=key, Body=b"site-version-one") + second = self.client_a.put_object(Bucket=bucket, Key=key, Body=b"site-version-two") + self.wait_object( + case_id, + self.client_b, + bucket, + key, + b"site-version-two", + expected_version=second.get("VersionId"), + ) + marker = self.client_a.delete_object(Bucket=bucket, Key=key).get("VersionId") + + def marker_probe() -> tuple[bool, Any]: + response = self.client_b.list_object_versions(Bucket=bucket, Prefix=key) + markers = [item["VersionId"] for item in response.get("DeleteMarkers", []) if item["Key"] == key] + return marker in markers, {"markers": markers, "expected": marker} + + marker_final = self.wait_until(case_id, "site delete marker convergence", marker_probe) + self.client_a.delete_object(Bucket=bucket, Key=key, VersionId=first["VersionId"]) + + def purge_probe() -> tuple[bool, Any]: + response = self.client_b.list_object_versions(Bucket=bucket, Prefix=key) + versions = [item["VersionId"] for item in response.get("Versions", []) if item["Key"] == key] + return first["VersionId"] not in versions and second["VersionId"] in versions, versions + + purge_final = self.wait_until(case_id, "site version purge convergence", purge_probe) + return [ + self.save_json(case_id, "delete-marker.json", marker_final), + self.save_json(case_id, "version-purge.json", purge_final), + ] + + def rc_probe(self, arguments: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [self.rc_bin, "--json", "--no-color", *arguments], + env=self.rc_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + + def policy_file(self, case_id: str, name: str, actions: list[str]) -> Path: + policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": actions, + "Resource": ["arn:aws:s3:::*"], + } + ], + } + path = self.evidence_path(case_id, name) + path.write_text(json.dumps(policy, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return path + + def wait_rc_success( + self, case_id: str, description: str, arguments: list[str] + ) -> subprocess.CompletedProcess[str]: + last: subprocess.CompletedProcess[str] | None = None + + def probe() -> tuple[bool, Any]: + nonlocal last + last = self.rc_probe(arguments) + return last.returncode == 0, { + "exitCode": last.returncode, + "stdout": last.stdout[:1000], + "stderr": last.stderr[:1000], + } + + self.wait_until(case_id, description, probe) + assert last is not None + self.save_json( + case_id, + f"{safe_name(description)}.json", + {"exitCode": last.returncode, "stdout": last.stdout, "stderr": last.stderr}, + ) + return last + + def assert_iam_credentials( + self, + case_id: str, + access_key: str, + secret_key: str, + label: str, + ) -> dict[str, Any]: + client = self.s3_client(self.site_b, access_key, secret_key) + + def allowed_probe() -> tuple[bool, Any]: + try: + response = client.list_buckets() + return True, {"bucketCount": len(response.get("Buckets", []))} + except ClientError as exc: + return False, {"error": exc.response.get("Error", {}).get("Code")} + + allowed = self.wait_until(case_id, f"{label} authorization on site-b", allowed_probe) + denied_bucket = self.bucket_name(case_id, f"deny-{label}") + try: + client.create_bucket(Bucket=denied_bucket) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code not in {"AccessDenied", "Unauthorized", "InvalidAccessKeyId"}: + raise + denied = {"operation": "CreateBucket", "error": code} + else: + raise AssertionError(f"{label} credentials exceeded the policy boundary") + return {"allowed": allowed, "denied": denied} + + def case_site_iam_user_policy(self, case_id: str) -> list[str]: + user = self.identity_name(case_id, "user") + policy = self.identity_name(case_id, "policy") + secret = f"Rv-{secrets.token_hex(16)}" + self.secrets.add(secret) + policy_path = self.policy_file(case_id, "user-policy.json", ["s3:ListAllMyBuckets"]) + self.run_rc( + case_id, + "iam-policy-create", + ["admin", "policy", "create", self.alias_a, policy, str(policy_path)], + ) + self.run_rc( + case_id, + "iam-user-add", + ["admin", "user", "add", self.alias_a, user, secret], + ) + self.run_rc( + case_id, + "iam-user-policy-attach", + ["admin", "policy", "attach", self.alias_a, policy, "--user", user], + ) + self.wait_rc_success( + case_id, "remote-user-info", ["admin", "user", "info", self.alias_b, user] + ) + self.wait_rc_success( + case_id, "remote-user-policy", ["admin", "policy", "info", self.alias_b, policy] + ) + auth = self.assert_iam_credentials(case_id, user, secret, "user") + return [self.save_json(case_id, "iam-user-authorization.json", auth)] + + def case_site_iam_group_policy(self, case_id: str) -> list[str]: + user = self.identity_name(case_id, "group-user") + group = self.identity_name(case_id, "group") + policy = self.identity_name(case_id, "group-policy") + secret = f"Rv-{secrets.token_hex(16)}" + self.secrets.add(secret) + policy_path = self.policy_file(case_id, "group-policy.json", ["s3:ListAllMyBuckets"]) + self.run_rc( + case_id, + "group-policy-create", + ["admin", "policy", "create", self.alias_a, policy, str(policy_path)], + ) + self.run_rc( + case_id, + "group-user-add", + ["admin", "user", "add", self.alias_a, user, secret], + ) + self.run_rc( + case_id, + "group-add", + ["admin", "group", "add", self.alias_a, group, "--members", user], + ) + self.run_rc( + case_id, + "group-policy-attach", + ["admin", "policy", "attach", self.alias_a, policy, "--group", group], + ) + self.wait_rc_success( + case_id, "remote-group-info", ["admin", "group", "info", self.alias_b, group] + ) + self.wait_rc_success( + case_id, "remote-group-policy", ["admin", "policy", "info", self.alias_b, policy] + ) + auth = self.assert_iam_credentials(case_id, user, secret, "group-user") + return [self.save_json(case_id, "iam-group-authorization.json", auth)] + + def case_site_service_account(self, case_id: str) -> list[str]: + access_key = self.identity_name(case_id, "svc") + secret = f"Svc-{secrets.token_hex(18)}" + self.secrets.add(secret) + policy_path = self.policy_file(case_id, "service-account-policy.json", ["s3:ListAllMyBuckets"]) + self.run_rc( + case_id, + "service-account-create", + [ + "admin", + "service-account", + "create", + self.alias_a, + access_key, + secret, + "--name", + "release-validation", + "--policy", + str(policy_path), + ], + ) + self.wait_rc_success( + case_id, + "remote-service-account-info", + ["admin", "service-account", "info", self.alias_b, access_key], + ) + auth = self.assert_iam_credentials(case_id, access_key, secret, "service-account") + return [self.save_json(case_id, "service-account-authorization.json", auth)] + + def case_site_peer_edit(self, case_id: str) -> list[str]: + info_a = self.site_info(self.site_a) + peer = dict(self.site_peer(info_a, self.site_b.endpoint)) + deployment_id = str(peer.get("deploymentID", "")) + if not deployment_id: + raise AssertionError("remote peer has no deployment ID") + new_name = self.identity_name(case_id, "renamed-site") + peer["name"] = new_name + status, headers, body = self.admin_request( + self.site_a, + "PUT", + "site-replication/edit", + payload=peer, + ) + if status != 200: + raise AssertionError(f"site peer edit returned HTTP {status}: {body!r}") + parsed = self.json_body(body) + if parsed and parsed.get("success") is False: + raise AssertionError(f"site peer edit was rejected: {parsed}") + + def probe() -> tuple[bool, Any]: + infos = {site.id: self.site_info(site) for site in (self.site_a, self.site_b)} + observed: dict[str, Any] = {} + complete = True + for site_id, info in infos.items(): + matches = [ + item + for item in info.get("sites", []) + if str(item.get("deploymentID", "")) == deployment_id + ] + observed[site_id] = matches + complete = complete and len(matches) == 1 and matches[0].get("name") == new_name + if matches and str(matches[0].get("endpoint", "")).rstrip("/") != self.site_b.endpoint: + raise AssertionError("peer rename changed its endpoint") + return complete, observed + + final = self.wait_until(case_id, "peer rename convergence", probe) + return [ + self.save_json( + case_id, + "peer-edit.json", + {"httpStatus": status, "headers": headers, "response": parsed, "final": final}, + ) + ] + + def case_site_resync(self, case_id: str) -> list[str]: + info = self.site_info(self.site_a) + peer = self.site_peer(info, self.site_b.endpoint) + responses: dict[str, Any] = {} + status, _, body = self.admin_request( + self.site_a, + "PUT", + "site-replication/resync/op", + query="operation=start", + payload=peer, + ) + if status != 200: + raise AssertionError(f"site resync start returned HTTP {status}: {body!r}") + responses["start"] = self.json_body(body) + status, _, body = self.admin_request( + self.site_a, + "PUT", + "site-replication/resync/op", + query="operation=status", + payload=peer, + ) + if status != 200: + raise AssertionError(f"site resync status returned HTTP {status}: {body!r}") + responses["status"] = self.json_body(body) + resync_id = responses["status"].get("id") or responses["start"].get("id") + if not resync_id: + raise AssertionError(f"site resync did not return a durable ID: {responses}") + state = str(responses["status"].get("state", responses["status"].get("status", ""))).lower() + if state not in {"completed", "complete", "success", "failed", "cancelled", "canceled"}: + status, _, body = self.admin_request( + self.site_a, + "PUT", + "site-replication/resync/op", + query="operation=cancel", + payload=peer, + ) + if status != 200: + raise AssertionError(f"site resync cancel returned HTTP {status}: {body!r}") + responses["cancel"] = self.json_body(body) + else: + responses["cancel"] = {"notRequired": True, "terminalState": state} + return [self.save_json(case_id, "site-resync.json", responses)] + + def case_site_outage_recovery(self, case_id: str) -> list[str]: + bucket = self.bucket_name(case_id, "outage") + key = "outage/site-retry.txt" + body = b"site-replication-outage" + evidence = [self.hook(case_id, "site-b-stop", self.args.site_b_stop_hook)] + b_started = False + try: + self.create_bucket(self.client_a, bucket) + self.client_a.put_object(Bucket=bucket, Key=key, Body=body) + + def retry_probe() -> tuple[bool, Any]: + info = self.site_info(self.site_a) + stats = info.get("retryStats") + pending = int((stats or {}).get("pending", 0) or 0) + failed = int((stats or {}).get("failed", 0) or 0) + return bool(stats) and (pending > 0 or failed > 0 or bool(stats.get("lastError"))), info + + retry = self.wait_until( + case_id, "source retry observability while peer is offline", retry_probe, timeout=60 + ) + evidence.append(self.save_json(case_id, "retry-visible.json", retry)) + evidence.append(self.hook(case_id, "site-b-start", self.args.site_b_start_hook)) + b_started = True + self.wait_api(case_id, self.client_b, "site-b") + finally: + if not b_started: + self.hook(case_id, "site-b-start-recovery", self.args.site_b_start_hook) + self.wait_api(case_id, self.client_b, "site-b") + self.wait_bucket(case_id, self.client_b, bucket) + self.created_buckets.append((self.client_b, bucket)) + final = self.wait_object( + case_id, self.client_b, bucket, key, body, timeout=max(self.args.timeout, 240) + ) + evidence.append(self.save_json(case_id, "outage-final.json", final)) + + def retry_cleared_probe() -> tuple[bool, Any]: + info = self.site_info(self.site_a) + stats = info.get("retryStats") + if not stats: + return True, info + pending = int(stats.get("pending", 0) or 0) + failed = int(stats.get("failed", 0) or 0) + return pending == 0 and failed == 0 and not stats.get("lastError"), info + + cleared = self.wait_until( + case_id, + "source retry queue to clear after peer recovery", + retry_cleared_probe, + timeout=max(self.args.timeout, 240), + ) + evidence.append(self.save_json(case_id, "retry-cleared.json", cleared)) + return evidence + + def site_smoke(self, case_id: str, prefix: str) -> dict[str, Any]: + bucket = self.create_site_bucket(case_id, prefix) + fixtures = [ + (self.client_a, self.client_b, "from-a.txt", f"{prefix}-a".encode()), + (self.client_b, self.client_a, "from-b.txt", f"{prefix}-b".encode()), + ] + final = [] + for source, target, key, body in fixtures: + response = source.put_object(Bucket=bucket, Key=key, Body=body) + final.append( + self.wait_object( + case_id, + target, + bucket, + key, + body, + expected_version=response.get("VersionId"), + ) + ) + return {"bucket": bucket, "objects": final} + + def case_site_restart_persistence(self, case_id: str) -> list[str]: + evidence = [ + self.hook(case_id, "site-a-restart", self.args.site_a_restart_hook), + self.hook(case_id, "site-b-restart", self.args.site_b_restart_hook), + ] + self.wait_api(case_id, self.client_a, "site-a") + self.wait_api(case_id, self.client_b, "site-b") + info = self.wait_site_group_active(case_id) + smoke = self.site_smoke(case_id, "restart") + evidence.extend( + ( + self.save_json(case_id, "restart-info.json", info), + self.save_json(case_id, "restart-smoke.json", smoke), + ) + ) + return evidence + + def case_site_service_account_rotation(self, case_id: str) -> list[str]: + evidence: list[str] = [] + if ( + self.args.selection_mode == "filtered" + and self.site_group_tracker_state == "tainted" + ): + raise PreconditionError( + "service-account rotation refuses a site group tainted by an " + "earlier selected case" + ) + try: + state, reason, current = self.site_group_snapshot() + except Exception as exc: + raise PreconditionError( + "service-account rotation could not classify the initial site group: " + f"{type(exc).__name__}: {exc}" + ) from exc + evidence.append( + self.save_json( + case_id, + "rotation-initial-site-info.json", + {"state": state, "reason": reason, "sites": current}, + ) + ) + if state == "disabled": + if ( + self.args.selection_mode == "filtered" + and self.site_group_tracker_state not in {"fresh", "removed"} + ): + raise PreconditionError( + "service-account rotation refuses to recreate a site group " + "lost after an earlier selected case" + ) + _, setup_evidence = self.add_site_replication_group( + case_id, + "rotation-setup-add-response.json", + require_clean_initial_sync=True, + ) + evidence.append(setup_evidence) + setup_final = self.wait_site_group_active(case_id) + evidence.append( + self.save_json(case_id, "rotation-setup-final.json", setup_final) + ) + elif state != "active": + raise PreconditionError( + "service-account rotation refuses initial " + f"{state} control-plane state: {reason}" + ) + status, headers, body = self.admin_request( + self.site_a, "POST", "site-replication/rotate-svc-acct" + ) + result = self.required_json_object(body, "service account rotation") + response_evidence = self.save_json( + case_id, + "rotation-response.json", + {"httpStatus": status, "headers": headers, "result": result}, + ) + if status != 200: + raise AssertionError(f"service account rotation returned HTTP {status}: {body!r}") + if result.get("success") is not True: + raise AssertionError(f"service account rotation was incomplete: {result}") + + settled = self.wait_site_group_active(case_id) + smoke = self.site_smoke(case_id, "rotation") + evidence.extend( + [ + response_evidence, + self.save_json(case_id, "rotation-settled.json", settled), + self.save_json(case_id, "rotation-smoke.json", smoke), + ] + ) + return evidence + + def case_site_remove_all(self, case_id: str) -> list[str]: + status, headers, body = self.admin_request( + self.site_a, + "PUT", + "site-replication/remove", + payload={"all": True}, + ) + response_text = body.decode("utf-8", errors="replace") + response_evidence = self.save_json( + case_id, + "remove-response.json", + {"httpStatus": status, "headers": headers, "body": response_text}, + ) + if status != 200: + raise AssertionError(f"site remove-all returned HTTP {status}: {body!r}") + result = self.required_json_object(body, "site remove-all") + if result.get("errorDetail") or str(result.get("status", "")).lower() == "partial": + raise AssertionError(f"site remove-all was partial: {result}") + info = self.wait_site_group_disabled(case_id) + bucket = self.bucket_name(case_id, "removed") + self.create_bucket(self.client_a, bucket) + self.client_a.put_object(Bucket=bucket, Key="control.txt", Body=b"must-not-replicate") + deadline = time.monotonic() + max(self.args.negative_window, 15) + observations = 0 + while time.monotonic() < deadline: + observations += 1 + try: + self.client_b.get_bucket_location(Bucket=bucket) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") not in { + "NoSuchBucket", + "404", + "NotFound", + }: + raise + else: + raise AssertionError("bucket propagated after site replication was removed") + time.sleep(self.args.poll_interval) + return [ + response_evidence, + self.save_json( + case_id, + "remove-all.json", + { + "httpStatus": status, + "headers": headers, + "response": result, + "info": info, + "negativeObservations": observations, + }, + ) + ] + + # ------------------------------------------------------------------ + # Execution and result contracts + # ------------------------------------------------------------------ + + def classify_exception(self, exc: BaseException) -> str: + if isinstance(exc, AssertionError): + return "fail" + if isinstance(exc, INFRA_EXCEPTIONS): + return "infra_error" + if isinstance(exc, ClientError): + code = exc.response.get("Error", {}).get("Code", "") + return "infra_error" if code in INFRA_ERROR_CODES else "fail" + return "infra_error" + + def update_site_group_tracker(self, case_id: str, result: CaseResult) -> None: + if not case_id.startswith("site-replication."): + return + if case_id in { + "site-replication.unconfigured-info", + "site-replication.invalid-credentials", + }: + self.site_group_tracker_state = ( + "fresh" if result.state == "pass" else "tainted" + ) + return + if case_id == "site-replication.add-and-info": + self.site_group_tracker_state = ( + "active" if result.state == "pass" else "tainted" + ) + return + if case_id == "site-replication.remove-all": + self.site_group_tracker_state = ( + "removed" if result.state == "pass" else "tainted" + ) + return + if case_id == "site-replication.service-account-rotation": + self.site_group_tracker_state = ( + "active" if result.state == "pass" else "tainted" + ) + return + if result.message.startswith("precondition_error:"): + self.site_group_tracker_state = "tainted" + return + if result.state == "pass": + self.site_group_tracker_state = "active" + return + if case_id == "site-replication.lifecycle-expiry-toggle": + if self.site_group_tracker_state == "tainted": + return + try: + state, reason, info = self.site_group_snapshot() + flags_valid, flags_reason = site_group_has_ilm_expiry_value( + info, + False, + ) + self.site_group_tracker_state = ( + "active" if state == "active" and flags_valid else "tainted" + ) + self.events.record( + "setup.tracker.probe", + case=case_id, + state=state, + reason=reason, + flagReason=flags_reason, + ) + except Exception as exc: + self.site_group_tracker_state = "tainted" + self.events.record( + "setup.tracker.error", + case=case_id, + message=self.redact(f"{type(exc).__name__}: {exc}"), + ) + return + try: + state, reason, _ = self.site_group_snapshot() + self.site_group_tracker_state = ( + "active" if state == "active" else "tainted" + ) + self.events.record( + "setup.tracker.probe", + case=case_id, + state=state, + reason=reason, + ) + except Exception as exc: + self.site_group_tracker_state = "tainted" + self.events.record( + "setup.tracker.error", + case=case_id, + message=self.redact(f"{type(exc).__name__}: {exc}"), + ) + + def record_isolation_blocked(self, case_id: str) -> CaseResult: + failure = self.isolation_failure + if failure is None: + raise RuntimeError("module isolation is not blocked") + started_at = utc_now() + message = self.redact( + "precondition_error: module isolation is blocked by logical cleanup " + f"from {failure['sourceCase']}; authoritative reset is required" + ) + blocked_evidence = self.save_json( + case_id, + "case-isolation-blocked.json", + { + "state": "infra_error", + "message": message, + "sourceCase": failure["sourceCase"], + "sourceEvidence": failure["evidence"], + "trackedBuckets": failure["trackedBuckets"], + }, + ) + evidence = list( + dict.fromkeys([blocked_evidence, *failure["evidence"]]) + ) + if ( + SITE_GROUP_ACTIVE_FIXTURE in self.requested_fixtures + and case_id in SITE_GROUP_ACTIVE_CASE_IDS + ): + self.fixture_outcomes.append( + { + "caseId": case_id, + "fixture": SITE_GROUP_ACTIVE_FIXTURE, + "state": "infra_error", + "message": message, + } + ) + result = CaseResult( + id=case_id, + state="infra_error", + message=message, + started_at=started_at, + finished_at=utc_now(), + duration_seconds=0.0, + evidence=evidence, + ) + self.results.append(result) + if case_id.startswith("site-replication."): + self.site_group_tracker_state = "tainted" + self.events.record( + "case.isolation.blocked", + case=case_id, + sourceCase=failure["sourceCase"], + evidence=evidence, + trackedBuckets=failure["trackedBuckets"], + ) + self.events.record("case.complete", **asdict(result)) + print(f"[INFRA_ERROR] {case_id}: {message}", flush=True) + return result + + def run_case( + self, + case_id: str, + function: Callable[[str], list[str]], + ) -> CaseResult: + if self.isolation_failure is not None: + return self.record_isolation_blocked(case_id) + started_at = utc_now() + started = time.monotonic() + state = "pass" + message = "all assertions passed" + print(f"[RUN ] {case_id}", flush=True) + self.events.record("case.start", case=case_id) + try: + self.ensure_planned_fixture(case_id) + function(case_id) + except Exception as exc: # keep the remaining cases independent + if isinstance(exc, PreconditionError): + state = "infra_error" + message = f"precondition_error: {exc}" + else: + state = self.classify_exception(exc) + message = f"{type(exc).__name__}: {exc}" + self.events.record( + "case.error", + case=case_id, + state=state, + errorType=type(exc).__name__, + message=self.redact(message), + ) + self.save_json( + case_id, + "case-error.json", + { + "state": state, + "errorType": type(exc).__name__, + "message": self.redact(message), + }, + ) + finally: + try: + # Replication phases intentionally continue after a failed case; + # therefore client-visible evidence is retained while logical test + # buckets are removed. The driver captures node logs before its + # authoritative phase reset. + self.cleanup_created_buckets(case_id) + except Exception as cleanup_exc: + cleanup_message = ( + f"cleanup failed: {type(cleanup_exc).__name__}: {cleanup_exc}" + ) + if state == "pass": + state = "infra_error" + message = cleanup_message + else: + message = f"{message}; {cleanup_message}" + self.events.record( + "case.cleanup.error", case=case_id, message=self.redact(str(cleanup_exc)) + ) + tracked_buckets = self.tracked_bucket_summaries() + isolation_restored = ( + cleanup_exc.isolation_restored + if isinstance(cleanup_exc, LogicalCleanupFailure) + else not tracked_buckets + ) + cleanup_error_evidence = self.save_json( + case_id, + "case-cleanup-error.json", + { + "state": "infra_error", + "errorType": type(cleanup_exc).__name__, + "message": self.redact(cleanup_message), + "isolationRestored": isolation_restored, + "trackedBuckets": tracked_buckets, + "cleanupEvidence": ( + cleanup_exc.evidence + if isinstance(cleanup_exc, LogicalCleanupFailure) + else None + ), + }, + ) + if not isolation_restored: + source_evidence = [cleanup_error_evidence] + if isinstance(cleanup_exc, LogicalCleanupFailure): + source_evidence.insert(0, cleanup_exc.evidence) + self.isolation_failure = { + "state": "blocked", + "sourceCase": case_id, + "evidence": list(dict.fromkeys(source_evidence)), + "trackedBuckets": tracked_buckets, + } + self.events.record( + "module.isolation.blocked", + case=case_id, + evidence=self.isolation_failure["evidence"], + trackedBuckets=tracked_buckets, + ) + + directory = self.evidence_path(case_id, "evidence-index").parent + evidence = sorted( + str(path.relative_to(self.evidence_dir)) + for path in directory.iterdir() + if path.is_file() and path.name != "evidence-index" + ) + result = CaseResult( + id=case_id, + state=state, + message=self.redact(message), + started_at=started_at, + finished_at=utc_now(), + duration_seconds=round(time.monotonic() - started, 3), + evidence=evidence, + ) + self.results.append(result) + if self.isolation_failure is not None and case_id.startswith( + "site-replication." + ): + self.site_group_tracker_state = "tainted" + else: + self.update_site_group_tracker(case_id, result) + self.events.record("case.complete", **asdict(result)) + label = "PASS" if state == "pass" else state.upper() + print(f"[{label:5}] {case_id}: {result.message}", flush=True) + return result + + def case_functions(self) -> dict[str, Callable[[str], list[str]]]: + return { + "bucket-replication.source-versioning-required": self.case_bucket_source_versioning_required, + "bucket-replication.target-versioning-required": self.case_bucket_target_versioning_required, + "bucket-replication.configuration-roundtrip": self.case_bucket_configuration_roundtrip, + "bucket-replication.put-object": self.case_bucket_put_object, + "bucket-replication.sync-put-object": self.case_bucket_sync_put_object, + "bucket-replication.prefix-filter": self.case_bucket_prefix_filter, + "bucket-replication.tag-filter": self.case_bucket_tag_filter, + "bucket-replication.zero-byte-special-key": self.case_bucket_zero_byte_special_key, + "bucket-replication.multipart-object": self.case_bucket_multipart_object, + "bucket-replication.copy-object": self.case_bucket_copy_object, + "bucket-replication.delete-marker": self.case_bucket_delete_marker, + "bucket-replication.version-delete": self.case_bucket_version_delete, + "bucket-replication.existing-object-enabled": self.case_bucket_existing_object_enabled, + "bucket-replication.existing-object-disabled": self.case_bucket_existing_object_disabled, + "bucket-replication.target-check": self.case_bucket_target_check, + "bucket-replication.status-observability": self.case_bucket_status_observability, + "bucket-replication.outage-recovery": self.case_bucket_outage_recovery, + "bucket-replication.source-restart-recovery": self.case_bucket_source_restart_recovery, + "site-replication.unconfigured-info": self.case_site_unconfigured_info, + "site-replication.invalid-credentials": self.case_site_invalid_credentials, + "site-replication.add-and-info": self.case_site_add_and_info, + "site-replication.status-observability": self.case_site_status_observability, + "site-replication.bucket-create": self.case_site_bucket_create, + "site-replication.object-bidirectional": self.case_site_object_bidirectional, + "site-replication.versioning-metadata": self.case_site_versioning_metadata, + "site-replication.lifecycle-expiry-toggle": self.case_site_lifecycle_expiry_toggle, + "site-replication.bucket-policy-metadata": self.case_site_bucket_policy_metadata, + "site-replication.object-tags": self.case_site_object_tags, + "site-replication.delete-convergence": self.case_site_delete_convergence, + "site-replication.iam-user-policy": self.case_site_iam_user_policy, + "site-replication.iam-group-policy": self.case_site_iam_group_policy, + "site-replication.service-account": self.case_site_service_account, + "site-replication.peer-edit": self.case_site_peer_edit, + "site-replication.resync": self.case_site_resync, + "site-replication.outage-recovery": self.case_site_outage_recovery, + "site-replication.restart-persistence": self.case_site_restart_persistence, + "site-replication.remove-all": self.case_site_remove_all, + "site-replication.service-account-rotation": self.case_site_service_account_rotation, + } + + def write_results(self, started_at: str, started: float) -> str: + ordered = BUCKET_CASE_IDS if self.args.module == "bucket" else SITE_CASE_IDS + selected_cases = tuple( + case_id for case_id in ordered if case_id in set(self.args.case or ordered) + ) + case_order = {case_id: index for index, case_id in enumerate(ordered)} + self.results.sort(key=lambda item: case_order.get(item.id, len(case_order))) + states = {result.state for result in self.results} + if "fail" in states: + overall = "fail" + elif "infra_error" in states: + overall = "infra_error" + else: + overall = "pass" + finished_at = utc_now() + document = { + "apiVersion": "validation.rustfs.io/v1alpha1", + "kind": "TestRunResult", + "metadata": { + "runId": self.args.run_id, + "environmentId": self.args.environment_id, + }, + "status": { + "state": overall, + "startedAt": started_at, + "finishedAt": finished_at, + "cases": [ + { + "id": item.id, + "state": item.state, + "message": item.message, + "evidence": item.evidence, + } + for item in self.results + ], + "evidence": ["events.jsonl", "run-manifest.json", "junit.xml"], + }, + } + (self.evidence_dir / "result.json").write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + manifest = { + "runId": self.args.run_id, + "environmentId": self.args.environment_id, + "module": self.args.module, + "endpointA": self.site_a.endpoint, + "endpointB": self.site_b.endpoint, + "startedAt": started_at, + "finishedAt": finished_at, + "durationSeconds": round(time.monotonic() - started, 3), + "pollIntervalSeconds": self.args.poll_interval, + "negativeWindowSeconds": self.args.negative_window, + "boto3Version": boto3.__version__, + "botocoreVersion": botocore.__version__, + "rcBinaryPath": self.rc_bin, + "rcVersion": self.rc_version, + "rcBinarySha256": self.rc_binary_sha256, + "expectedRcBinarySha256": self.expected_rc_binary_sha256 or None, + "executionPlan": { + "selectionMode": self.args.selection_mode, + "module": self.args.module, + "scoredCases": list(selected_cases), + "scoredCaseCount": len(selected_cases), + "fixtures": sorted(self.requested_fixtures), + }, + "fixtureOutcomes": self.fixture_outcomes, + "siteGroupTrackerState": self.site_group_tracker_state, + "isolationOutcome": self.isolation_failure + or { + "state": "pass", + "message": "all tracked logical buckets were removed", + "trackedBuckets": [], + }, + } + (self.evidence_dir / "run-manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + self.write_junit(overall, time.monotonic() - started) + return overall + + def write_junit(self, overall: str, duration: float) -> None: + suite = ET.Element( + "testsuite", + { + "name": f"rustfs-fixed-environment-{self.args.module}-replication", + "tests": str(len(self.results)), + "failures": str(sum(item.state == "fail" for item in self.results)), + "errors": str(sum(item.state == "infra_error" for item in self.results)), + "skipped": "0", + "time": f"{duration:.3f}", + }, + ) + for item in self.results: + case = ET.SubElement( + suite, + "testcase", + { + "classname": f"rustfs.{self.args.module}_replication", + "name": item.id, + "time": f"{item.duration_seconds:.3f}", + }, + ) + if item.state == "fail": + ET.SubElement(case, "failure", {"message": item.message}).text = item.message + elif item.state == "infra_error": + ET.SubElement(case, "error", {"message": item.message}).text = item.message + ET.SubElement(case, "system-out").text = "\n".join(item.evidence) + tree = ET.ElementTree(suite) + ET.indent(tree, space=" ") + tree.write(self.evidence_dir / "junit.xml", encoding="utf-8", xml_declaration=True) + self.events.record("run.complete", state=overall) + + def run(self) -> int: + started_at = utc_now() + started = time.monotonic() + self.events.record( + "run.start", + runId=self.args.run_id, + environmentId=self.args.environment_id, + module=self.args.module, + selectionMode=self.args.selection_mode, + scoredCases=list(self.args.case or ( + BUCKET_CASE_IDS if self.args.module == "bucket" else SITE_CASE_IDS + )), + fixtures=sorted(self.requested_fixtures), + ) + try: + self.wait_api("preflight.connectivity", self.client_a, "site-a") + self.wait_api("preflight.connectivity", self.client_b, "site-b") + self.configure_rc() + except Exception as exc: + self.results.append( + CaseResult( + id="preflight.connectivity", + state="infra_error", + message=self.redact(f"{type(exc).__name__}: {exc}"), + started_at=started_at, + finished_at=utc_now(), + duration_seconds=round(time.monotonic() - started, 3), + ) + ) + overall = self.write_results(started_at, started) + return 2 if overall == "infra_error" else 1 + + functions = self.case_functions() + ordered = BUCKET_CASE_IDS if self.args.module == "bucket" else SITE_CASE_IDS + selected = set(self.args.case or ordered) + for case_id in ordered: + if case_id in selected: + self.run_case(case_id, functions[case_id]) + overall = self.write_results(started_at, started) + print( + f"{self.args.module} replication validation {overall}: " + f"result={self.evidence_dir / 'result.json'}", + flush=True, + ) + return {"pass": 0, "fail": 1, "infra_error": 2}[overall] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--module", choices=("bucket", "site"), required=True) + parser.add_argument("--endpoint-a", default=os.environ.get("RUSTFS_REPLICATION_ENDPOINT_A")) + parser.add_argument("--endpoint-b", default=os.environ.get("RUSTFS_REPLICATION_ENDPOINT_B")) + parser.add_argument("--access-key", default=os.environ.get("RUSTFS_ACCESS_KEY")) + parser.add_argument("--secret-key", default=os.environ.get("RUSTFS_SECRET_KEY")) + parser.add_argument("--region", default=os.environ.get("RUSTFS_REGION", "us-east-1")) + parser.add_argument("--environment-id", default="replication-2site") + parser.add_argument("--run-id", required=True) + parser.add_argument("--selection-mode", choices=("full", "filtered")) + parser.add_argument("--evidence-dir", required=True, type=Path) + parser.add_argument("--timeout", type=float, default=180) + parser.add_argument("--poll-interval", type=float, default=2) + parser.add_argument("--negative-window", type=float, default=12) + parser.add_argument("--rc-bin", default="rc") + parser.add_argument("--site-a-stop-hook") + parser.add_argument("--site-a-start-hook") + parser.add_argument("--site-a-restart-hook") + parser.add_argument("--site-b-stop-hook") + parser.add_argument("--site-b-start-hook") + parser.add_argument("--site-b-restart-hook") + parser.add_argument( + "--case", + action="append", + choices=BUCKET_CASE_IDS + SITE_CASE_IDS, + help="run only the named case; repeat to select multiple cases", + ) + parser.add_argument( + "--fixture", + action="append", + choices=(SITE_GROUP_ACTIVE_FIXTURE,), + help="non-scored setup fixture selected by the case planner", + ) + parser.add_argument("--cleanup-on-failure", action="store_true") + args = parser.parse_args() + selection_mode_was_explicit = args.selection_mode is not None + if not args.endpoint_a or not args.endpoint_b or not args.access_key or not args.secret_key: + parser.error("both endpoints and the administrator access/secret keys are required") + endpoint_identities: list[tuple[str, int]] = [] + for endpoint in (args.endpoint_a, args.endpoint_b): + identity = _site_endpoint_identity(endpoint) + if identity is None: + parser.error("replication endpoints must be absolute HTTP(S) URLs") + endpoint_identities.append(identity) + if len(set(endpoint_identities)) != 2: + parser.error("replication endpoints must be distinct") + if args.timeout <= 0 or args.poll_interval <= 0 or args.negative_window <= 0: + parser.error("timing values must be positive") + selected = set(args.case or ()) + allowed = set(BUCKET_CASE_IDS if args.module == "bucket" else SITE_CASE_IDS) + if selected - allowed: + parser.error(f"selected case does not belong to module {args.module}") + if args.selection_mode is None: + args.selection_mode = "filtered" if args.case else "full" + fixtures = set(args.fixture or ()) + if args.selection_mode == "full": + if selected and selected != allowed: + parser.error("full selection mode requires the complete module case set") + if fixtures: + parser.error("full selection mode must not enable diagnostic fixtures") + else: + if not selected: + parser.error("filtered selection mode requires at least one case") + expected_fixtures = ( + {SITE_GROUP_ACTIVE_FIXTURE} + if args.module == "site" + and SITE_GROUP_ACTIVE_CASE_IDS.intersection(selected) + else set() + ) + if not selection_mode_was_explicit and args.fixture is None: + args.fixture = sorted(expected_fixtures) + fixtures = set(args.fixture) + if fixtures != expected_fixtures: + parser.error( + "filtered fixture set does not match the selected site-case " + f"requirements: expected={sorted(expected_fixtures)} " + f"actual={sorted(fixtures)}" + ) + no_proxy = os.environ.get("NO_PROXY", os.environ.get("no_proxy", "")) + hosts = [identity[0] for identity in endpoint_identities] + os.environ["NO_PROXY"] = ",".join(filter(None, [no_proxy, *hosts, "127.0.0.1", "localhost"])) + os.environ["no_proxy"] = os.environ["NO_PROXY"] + return args + + +def main() -> int: + if hasattr(os.sys.stdout, "reconfigure"): + os.sys.stdout.reconfigure(line_buffering=True) + args = parse_args() + harness = ReplicationHarness(args) + try: + return harness.run() + finally: + harness.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation_replication/replication_metrics.py b/scripts/validation_replication/replication_metrics.py new file mode 100644 index 00000000..4d3eceeb --- /dev/null +++ b/scripts/validation_replication/replication_metrics.py @@ -0,0 +1,91 @@ +"""Strict helpers for consuming the stable rc replication output contract.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + + +U64_MAX = 2**64 - 1 + + +class ReplicationMetricsContractError(ValueError): + """The rc output is incompatible with the expected output-v3 contract.""" + + +def _mapping(value: Any, path: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ReplicationMetricsContractError(f"{path} must be a JSON object") + return value + + +def _root(document: Any) -> Mapping[str, Any]: + root = _mapping(document, "document") + schema_version = root.get("schema_version") + if type(schema_version) is not int or schema_version != 3: + raise ReplicationMetricsContractError( + "document.schema_version must be the JSON integer 3" + ) + if root.get("type") != "replication": + raise ReplicationMetricsContractError( + 'document.type must be the JSON string "replication"' + ) + return root + + +def _counter(container: Mapping[str, Any], name: str, path: str) -> int: + if name not in container: + return 0 + value = container[name] + if type(value) is not int or not 0 <= value <= U64_MAX: + raise ReplicationMetricsContractError( + f"{path}.{name} must be an unsigned 64-bit JSON integer" + ) + return value + + +def _data(document: Any) -> Mapping[str, Any]: + root = _root(document) + if "data" not in root: + raise ReplicationMetricsContractError("document.data is required") + return _mapping(root["data"], "document.data") + + +def replicated_object_count(document: Any) -> int: + """Return the greatest aggregate or per-target replicated object count.""" + + data = _data(document) + totals_value = data.get("totals", {}) + totals = _mapping(totals_value, "document.data.totals") + observed = _counter(totals, "replicated_count", "document.data.totals") + + targets_value = data.get("targets", []) + if not isinstance(targets_value, list): + raise ReplicationMetricsContractError("document.data.targets must be a JSON array") + for index, value in enumerate(targets_value): + target = _mapping(value, f"document.data.targets[{index}]") + observed = max( + observed, + _counter( + target, + "replicated_count", + f"document.data.targets[{index}]", + ), + ) + return observed + + +def replication_metrics_ready(document: Any) -> bool: + """Return whether a successful output-v3 document proves replication progress.""" + + root = _root(document) + if root.get("status") != "success": + return False + observed = replicated_object_count(root) + data = _data(root) + targets = data.get("targets", []) + return ( + data.get("availability") == "available" + and bool(targets) + and observed >= 1 + ) diff --git a/scripts/validation_replication/run_case.sh b/scripts/validation_replication/run_case.sh new file mode 100644 index 00000000..371b1537 --- /dev/null +++ b/scripts/validation_replication/run_case.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "${ROOT}/rustfs-replication-test.sh" +endpoint_host_prefix() { printf 'rustfs-node'; } +SITE2_ENDPOINT="http://rustfs-node1:19000" + +module_for_case() { + case "$1" in + bucket-replication.*) printf bucket ;; + site-replication.*) printf site ;; + *) return 2 ;; + esac +} + +site_action() { + local site="$1" action="$2" node pid fail=0 + case "${site}" in + a) + local pids=() + for node in "${NODES[@]}"; do + service_action "${action}" "${node}" & pids+=("$!") + done + for pid in "${pids[@]}"; do wait "${pid}" || fail=1; done + [ "${fail}" -eq 0 ] || die "site A ${action} failed" + if [ "${action}" != stop ]; then + for node in "${NODES[@]}"; do wait_service_active "${node}" "${RUSTFS_SERVICE}"; done + wait_http_live "${API_ENDPOINT}" + fi ;; + b) + service_action "${action}" "${NODES[0]}" "${SITE2_SERVICE}" + if [ "${action}" != stop ]; then + wait_service_active "${NODES[0]}" "${SITE2_SERVICE}" + wait_http_live "${SITE2_ENDPOINT}" + fi ;; + *) die "unknown replication site: ${site}" ;; + esac +} + +hook_json() { + python3 -c 'import json,sys; print(json.dumps(sys.argv[1:]))' \ + bash "${ROOT}/scripts/validation_replication/run_case.sh" hook "$1" "$2" +} + +setup_environment() { + env_setup_main_cluster + env_setup_site2 +} + +run_validation() { + local case_id="" candidate="" testing_commit="" artifacts="" setup_rc=0 harness_rc=0 teardown_rc=0 + while [ "$#" -gt 0 ]; do + case "$1" in + --validation-case) case_id="$2"; shift 2 ;; + --candidate-manifest) candidate="$2"; shift 2 ;; + --testing-commit) testing_commit="$2"; shift 2 ;; + --validation-artifacts-dir) artifacts="$2"; shift 2 ;; + --dry-run) DRY_RUN=1; shift ;; + -y|--yes) shift ;; + *) die "unknown replication validation option: $1" ;; + esac + done + local module + module="$(module_for_case "${case_id}")" || die "replication case required" + python3 -B -c 'import sys; sys.path.insert(0,sys.argv[1]); from scripts.validation_replication.case_plan import BUCKET_CASE_IDS,SITE_CASE_IDS; valid=BUCKET_CASE_IDS if sys.argv[2]=="bucket" else SITE_CASE_IDS; sys.exit(0 if sys.argv[3] in valid else 2)' \ + "${ROOT}" "${module}" "${case_id}" || die "unknown canonical replication case: ${case_id}" + if [ "${DRY_RUN}" -eq 1 ]; then + setup_environment + return 0 + fi + [ -f "${candidate}" ] || die "candidate manifest missing" + [ -n "${artifacts}" ] && [ ! -e "${artifacts}" ] || die "fresh artifact directory required" + [ "${testing_commit}" = "$(git -C "${ROOT}" rev-parse HEAD)" ] || die "testing commit differs from checkout" + PACKAGE_URL="$(jq -er '.package_url' "${candidate}")" + PACKAGE_SHA256="$(jq -er '.package_sha256' "${candidate}")" + [[ "${PACKAGE_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "invalid candidate SHA-256" + mkdir -p "${artifacts}/cases" "${artifacts}/evidence/${case_id}/replication" + local evidence="${artifacts}/evidence/${case_id}" harness="${artifacts}/evidence/${case_id}/replication" + python3 "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" \ + --profile replication-2site --result-file "${artifacts}/cases/${case_id}.json" \ + --verdict RUNNING --cleanup-status PENDING + (setup_environment) > "${evidence}/setup.log" 2>&1 || setup_rc=$? + if [ "${setup_rc}" -ne 0 ]; then + cat "${evidence}/setup.log" + (env_teardown) > "${evidence}/teardown.log" 2>&1 || true + return 1 + fi + local binary_sha + binary_sha="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${NODES[0]}" sha256sum /usr/bin/rustfs | awk '{print $1}')" + [[ "${binary_sha}" =~ ^[0-9a-f]{64}$ ]] || die "installed RustFS binary SHA-256 unavailable" + python3 -B "${ROOT}/scripts/validation_replication/replication_behavior.py" \ + --module "${module}" --case "${case_id}" --cleanup-on-failure \ + --endpoint-a "${API_ENDPOINT}" --endpoint-b "${SITE2_ENDPOINT}" \ + --access-key "${ACCESS_KEY}" --secret-key "${SECRET_KEY}" \ + --run-id "${case_id}-$(date +%s)-$$" --evidence-dir "${harness}" \ + --rc-bin "$(command -v rc)" \ + --site-a-stop-hook "$(hook_json a stop)" \ + --site-a-start-hook "$(hook_json a start)" \ + --site-a-restart-hook "$(hook_json a restart)" \ + --site-b-stop-hook "$(hook_json b stop)" \ + --site-b-start-hook "$(hook_json b start)" \ + --site-b-restart-hook "$(hook_json b restart)" \ + > "${evidence}/test.log" 2>&1 || harness_rc=$? + cat "${evidence}/test.log" + (env_teardown) > "${evidence}/teardown.log" 2>&1 || teardown_rc=$? + local verdict=FAIL reason="replication behavior assertions failed; inspect evidence" cleanup_status=FAIL + if [ "${teardown_rc}" -eq 0 ]; then cleanup_status=PASS; fi + if [ "${harness_rc}" -eq 0 ] && [ "${teardown_rc}" -eq 0 ] && \ + python3 -B "${ROOT}/scripts/validation_replication/verify_result.py" "${case_id}" "${harness}"; then + verdict=PASS; reason="" + elif jq -e '.status.state == "infra_error"' "${harness}/result.json" >/dev/null 2>&1; then + verdict=INFRA_ERROR; reason="replication environment or execution infrastructure failed; inspect evidence" + fi + local args=("${ROOT}/scripts/validation_result.py" --case-id "${case_id}" + --profile replication-2site --result-file "${artifacts}/cases/${case_id}.json" + --verdict "${verdict}" --cleanup-status "${cleanup_status}" + --candidate-manifest "${candidate}" --rc-bin "$(command -v rc)" + --testing-commit "${testing_commit}" --binary-sha256 "${binary_sha}" + --evidence "evidence/${case_id}/setup.log" --evidence "evidence/${case_id}/test.log" + --evidence "evidence/${case_id}/teardown.log") + if [ -n "${reason}" ]; then args+=(--reason "${reason}"); fi + if [ "${verdict}" = PASS ]; then + local file + while IFS= read -r -d '' file; do + args+=(--evidence "${file#"${artifacts}/"}") + done < <(find "${harness}" -type f -print0 | sort -z) + fi + python3 "${args[@]}" + [ "${verdict}" = PASS ] +} + +case "${1:-}" in + hook) shift; site_action "$@" ;; + *) run_validation "$@" ;; +esac diff --git a/scripts/validation_replication/verify_result.py b/scripts/validation_replication/verify_result.py new file mode 100644 index 00000000..46568c05 --- /dev/null +++ b/scripts/validation_replication/verify_result.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Check one replication case, fixture, and logical isolation outcome.""" + +from __future__ import annotations + +import json +from pathlib import Path +import sys + + +def verify(case_id: str, evidence_dir: Path) -> None: + result = json.loads((evidence_dir / "result.json").read_text(encoding="utf-8")) + manifest = json.loads((evidence_dir / "run-manifest.json").read_text(encoding="utf-8")) + cases = result["status"]["cases"] + if result["status"]["state"] != "pass" or len(cases) != 1: + raise ValueError("replication run did not pass exactly one case") + case = cases[0] + if case["id"] != case_id or case["state"] != "pass": + raise ValueError("replication result does not pass selected case") + if manifest["executionPlan"]["scoredCases"] != [case_id]: + raise ValueError("replication execution plan differs from selected case") + if manifest["isolationOutcome"]["state"] != "pass" or manifest["isolationOutcome"].get("trackedBuckets"): + raise ValueError("replication bucket isolation did not pass") + if any(outcome["state"] != "pass" for outcome in manifest["fixtureOutcomes"]): + raise ValueError("replication fixture did not pass") + if not case["evidence"]: + raise ValueError("replication case has no assertion evidence") + root = evidence_dir.resolve(strict=True) + for name in case["evidence"]: + path = (root / name).resolve(strict=True) + if not path.is_relative_to(root) or not path.is_file(): + raise ValueError(f"unsafe replication evidence path: {name}") + + +def main() -> int: + if len(sys.argv) != 3: + print(f"usage: {sys.argv[0]} CASE_ID EVIDENCE_DIR", file=sys.stderr) + return 2 + try: + verify(sys.argv[1], Path(sys.argv[2])) + except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError) as exc: + print(f"replication verification failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test_validation_replication.py b/test_validation_replication.py new file mode 100644 index 00000000..4407d99a --- /dev/null +++ b/test_validation_replication.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import json +import tempfile +import unittest +from pathlib import Path + +from scripts.validation_replication.case_plan import ( + BUCKET_CASE_IDS, SITE_CASE_IDS, SITE_GROUP_ACTIVE_CASE_IDS, +) +from scripts.validation_replication.verify_result import verify + + +class ReplicationContractTest(unittest.TestCase): + def test_catalog_and_fixture_selection(self) -> None: + self.assertEqual(18, len(BUCKET_CASE_IDS)) + self.assertEqual(20, len(SITE_CASE_IDS)) + self.assertEqual(16, len(SITE_GROUP_ACTIVE_CASE_IDS)) + self.assertNotIn("site-replication.unconfigured-info", SITE_GROUP_ACTIVE_CASE_IDS) + self.assertIn("site-replication.bucket-create", SITE_GROUP_ACTIVE_CASE_IDS) + + def test_pass_requires_case_fixture_and_isolation(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + case_id = "site-replication.bucket-create" + (root / "proof.json").write_text("{}", encoding="utf-8") + result = {"status": {"state": "pass", "cases": [{ + "id": case_id, "state": "pass", "evidence": ["proof.json"], + }]}} + manifest = {"executionPlan": {"scoredCases": [case_id]}, + "isolationOutcome": {"state": "pass", "trackedBuckets": []}, + "fixtureOutcomes": [{"state": "pass"}]} + (root / "result.json").write_text(json.dumps(result), encoding="utf-8") + (root / "run-manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + verify(case_id, root) + manifest["fixtureOutcomes"][0]["state"] = "infra_error" + (root / "run-manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "fixture"): + verify(case_id, root) + + +if __name__ == "__main__": + unittest.main() From 6e1df9d4bf936f40ae7ba787e7bc3cc2efdf0425 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 18:59:52 +0800 Subject: [PATCH 014/343] Use distinct audit secret for canonical validation --- docs/validation-migration-map.md | 16 ++++++++-------- rustfs-tier-test.sh | 8 +++++++- test_validation_webhooks.py | 5 +++++ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 474fa4f9..daa0c36c 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -79,12 +79,12 @@ Actions run `38042759074`. `events.delivery` and `audit.request-correlation` now have dedicated executors on the Tier suite's `single-single` topology. Event delivery is counted for a unique object after a quiet window; audit checks matching PUT/GET records, -distinct request IDs, successful status, and absence of known secrets. The -first two event attempts stopped before case execution because the candidate -could not converge local notification runtime while both notification and -audit modules were enabled. The validation setup now enables only the required -webhook channel and retains its full setup log. Both cases still require live -candidate evidence before being counted as verified. +distinct request IDs, successful status, and absence of known secrets. +`events.delivery` passed live in RustFS Actions run `38046470999`. The audit +attempt in run `38046574073` failed its secret check because the environment +used the same value for the public access key and secret key. The audit case +now generates a distinct secret key before deployment; it still requires a +live rerun before it can be counted as verified. `iam.policy-enforcement` has a dedicated executor on the Security suite's `single-single` deployment. It uses one principal with a policy that allows @@ -114,7 +114,7 @@ executor before touching the fleet and runs the result report with workflow. Each run uploads its candidate identity, canonical result, and case evidence. The 73 currently dispatchable IDs are the S3, two scanner, KMS, tiering, events, audit, IAM, 14 additional ILM, 13 Pool, and 38 -replication cases above. Twelve have live PASS evidence; the other 146 IDs +replication cases above. Thirteen have live PASS evidence; the other 145 IDs remain unverified. The eight planned cases have these concrete reuse candidates: @@ -124,7 +124,7 @@ The eight planned cases have these concrete reuse candidates: | `scanner.usage-convergence` | STOR-134 | Dedicated executor and live PASS evidence. | | `kms.sse-kms-round-trip` | KMS-104, KMS-301, KMS-214 | Dedicated executor and live PASS evidence. | | `tiering.transition-and-restore` | TIER-105, TIER-107 | Dedicated executor and live PASS evidence. | -| `events.delivery` | EVT-103, EVT-104 | Dedicated executor; live evidence pending. | +| `events.delivery` | EVT-103, EVT-104 | Dedicated executor and live PASS evidence. | | `audit.request-correlation` | AUD-102, AUD-104, AUD-106 | Dedicated executor; live evidence pending. | | `iam.policy-enforcement` | IAM-105, IAM-109 | Dedicated executor and live PASS evidence. | | `oidc.login-and-session` | OIDC-103 | The live Keycloak path can be skipped by the old suite; validation requires an actual session-boundary result. | diff --git a/rustfs-tier-test.sh b/rustfs-tier-test.sh index bea1f190..9398bf58 100755 --- a/rustfs-tier-test.sh +++ b/rustfs-tier-test.sh @@ -3020,8 +3020,14 @@ main() { need_cmd python3 "Python 3 (path canonicalization and receiver)" need_cmd curl "HTTP client" need_cmd openssl "OpenSSL (SigV4)" - need_cmd mosquitto_sub "mosquitto clients" + if [ "${VALIDATION_CASE}" != events.delivery ] && [ "${VALIDATION_CASE}" != audit.request-correlation ]; then + need_cmd mosquitto_sub "mosquitto clients" + fi need_cmd timeout "GNU timeout (bounded CLI/network operations)" + if [ "${VALIDATION_CASE}" = audit.request-correlation ] && [ "${DRY_RUN}" -eq 0 ]; then + SECRET_KEY="$(openssl rand -hex 32)" || die "cannot generate distinct audit validation secret" + export RUSTFS_SECRET_KEY="${SECRET_KEY}" + fi init_artifacts resolve_rc_binary write_provenance diff --git a/test_validation_webhooks.py b/test_validation_webhooks.py index e686d61e..6ca00a06 100644 --- a/test_validation_webhooks.py +++ b/test_validation_webhooks.py @@ -34,6 +34,11 @@ def test_audit_requires_correlated_operations_and_no_secret(self): with patch.dict(os.environ, {"RUSTFS_SECRET_KEY": "private-value"}): with self.assertRaisesRegex(ValueError, "exposes RUSTFS_SECRET_KEY"): audit_summary([put, {**get, "requestHeader": {"x-secret": "private-value"}}], "bucket", "key") + public_identity = {**put, "accessKey": "public-access-key"} + self.assertEqual( + audit_summary([public_identity, get], "bucket", "key")["secret_check"], + "PASS", + ) def test_records_rejects_non_object_jsonl(self): with tempfile.TemporaryDirectory() as directory: From ee676e3511f493efeafcb547d47cdd55e0b60542 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 19:42:39 +0800 Subject: [PATCH 015/343] Parse candidate S3 audit operations and await correlation --- docs/validation-migration-map.md | 11 +++++++---- rustfs-tier-test.sh | 3 +-- scripts/validation_webhooks.py | 34 ++++++++++++++++++++++++++------ test_validation_webhooks.py | 16 +++++++++++++++ 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index daa0c36c..7dd77732 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -54,7 +54,8 @@ All 13 Pool lifecycle IDs now dispatch to the source black-box pool checks. The driver reuses the existing auto-testing Pool suite's two-node seed and expansion operations, supplies marker-guarded reset/topology/restart/physical census hooks, and requires source case evidence plus physical cleanup before -canonical PASS. These cases have not yet completed live validation. +canonical PASS. `pool.status` passed live in RustFS Actions run `38047001285`; +the other twelve Pool cases still require live evidence. The 18 bucket-replication and 20 site-replication IDs now dispatch to the source black-box checks. The driver uses the existing auto-testing four-node @@ -83,8 +84,10 @@ distinct request IDs, successful status, and absence of known secrets. `events.delivery` passed live in RustFS Actions run `38046470999`. The audit attempt in run `38046574073` failed its secret check because the environment used the same value for the public access key and secret key. The audit case -now generates a distinct secret key before deployment; it still requires a -live rerun before it can be counted as verified. +now generates a distinct secret key before deployment. Run `38046860333` +exposed an executor parsing mismatch: the candidate emits `s3:PutObject` and +`s3:GetObject`. The parser now recognizes those exact names and waits for both +operations on the selected object; the case still requires a live rerun. `iam.policy-enforcement` has a dedicated executor on the Security suite's `single-single` deployment. It uses one principal with a policy that allows @@ -114,7 +117,7 @@ executor before touching the fleet and runs the result report with workflow. Each run uploads its candidate identity, canonical result, and case evidence. The 73 currently dispatchable IDs are the S3, two scanner, KMS, tiering, events, audit, IAM, 14 additional ILM, 13 Pool, and 38 -replication cases above. Thirteen have live PASS evidence; the other 145 IDs +replication cases above. Fourteen have live PASS evidence; the other 144 IDs remain unverified. The eight planned cases have these concrete reuse candidates: diff --git a/rustfs-tier-test.sh b/rustfs-tier-test.sh index 9398bf58..a7769b24 100755 --- a/rustfs-tier-test.sh +++ b/rustfs-tier-test.sh @@ -2102,11 +2102,10 @@ test_validation_audit_request_correlation() { aws_run s3api get-object --bucket "${bucket}" --key "${key}" "${TEST_TMP}/validation-audit-read.bin" assert_aws_ok || return 1 cmp "${TEST_TMP}/validation-audit-source.bin" "${TEST_TMP}/validation-audit-read.bin" || return 1 - wait_audit 2 "${bucket}" || return 1 VALIDATION_AUDIT_TOKEN="${AUDIT_TOKEN}" VALIDATION_NOTIFY_TOKEN="${NOTIFY_TOKEN}" \ python3 "${SCRIPT_DIR}/scripts/validation_webhooks.py" audit \ --input "${AUDIT_EVENTS}" --bucket "${bucket}" --key "${key}" \ - --output "${TEST_TMP}/validation-audit.json" + --output "${TEST_TMP}/validation-audit.json" --timeout 300 } run_all_tests() { diff --git a/scripts/validation_webhooks.py b/scripts/validation_webhooks.py index c23bc8d1..14c25938 100644 --- a/scripts/validation_webhooks.py +++ b/scripts/validation_webhooks.py @@ -8,9 +8,14 @@ import os from pathlib import Path import sys +import time from urllib.parse import unquote_plus +class IncompleteAuditRecords(ValueError): + """The requested operations have not both arrived yet.""" + + def records(path: Path) -> list[dict]: values = [] for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): @@ -54,8 +59,10 @@ def audit_summary(values: list[dict], bucket: str, key: str) -> dict: api = value.get("api") or {} if api.get("bucket") != bucket or api.get("object") != key: continue - name = api.get("name") - if name not in {"PutObject", "GetObject"}: + api_name = api.get("name") + name = {"PutObject": "PutObject", "GetObject": "GetObject", + "s3:PutObject": "PutObject", "s3:GetObject": "GetObject"}.get(api_name) + if name is None: continue request_id = value.get("requestID") if not isinstance(request_id, str) or not request_id: @@ -66,9 +73,12 @@ def audit_summary(values: list[dict], bucket: str, key: str) -> dict: for header, content in headers.items(): if header.lower() == "authorization" and isinstance(content, str) and content.strip("*"): raise ValueError("audit payload exposes authorization header") - matches[name] = {"request_id": request_id, "status_code": 200} + matches[name] = {"api_name": api_name, "request_id": request_id, "status_code": 200} if set(matches) != {"PutObject", "GetObject"}: - raise ValueError("matching PUT and GET audit records were not both delivered") + raise IncompleteAuditRecords( + "matching PUT and GET audit records were not both delivered; " + f"observed operations: {sorted(matches)}" + ) if matches["PutObject"]["request_id"] == matches["GetObject"]["request_id"]: raise ValueError("PUT and GET audit records share a request ID") return {"bucket": bucket, "key": key, "operations": matches, "secret_check": "PASS"} @@ -81,9 +91,21 @@ def main() -> int: parser.add_argument("--bucket", required=True) parser.add_argument("--key", required=True) parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--timeout", type=float, default=0, + help="Wait this many seconds for both audit operations; invalid records fail immediately.") args = parser.parse_args() - values = records(args.input) - result = (event_summary if args.mode == "events" else audit_summary)(values, args.bucket, args.key) + if args.timeout < 0 or args.timeout > 300: + parser.error("--timeout must be between 0 and 300 seconds") + deadline = time.monotonic() + args.timeout + while True: + values = records(args.input) + try: + result = (event_summary if args.mode == "events" else audit_summary)(values, args.bucket, args.key) + break + except IncompleteAuditRecords: + if time.monotonic() >= deadline: + raise + time.sleep(min(2, max(0, deadline - time.monotonic()))) args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") return 0 diff --git a/test_validation_webhooks.py b/test_validation_webhooks.py index 6ca00a06..c3e84016 100644 --- a/test_validation_webhooks.py +++ b/test_validation_webhooks.py @@ -47,6 +47,22 @@ def test_records_rejects_non_object_jsonl(self): with self.assertRaisesRegex(ValueError, "line 2"): records(path) + def test_audit_accepts_s3_operation_names_and_keeps_boundaries(self): + def entry(name, request_id, key="key"): + return {"api": {"name": name, "bucket": "bucket", "object": key, "status_code": 200}, + "requestID": request_id} + put = entry("s3:PutObject", "put-id") + get = entry("s3:GetObject", "get-id") + result = audit_summary([put, get], "bucket", "key") + self.assertEqual(result["operations"]["GetObject"]["api_name"], "s3:GetObject") + with self.assertRaisesRegex(ValueError, "not both delivered"): + audit_summary([put, entry("s3:GetObject", "other-id", "other-key")], "bucket", "key") + with self.assertRaisesRegex(ValueError, "not both delivered"): + audit_summary([put, entry("admin:GetObject", "admin-id")], "bucket", "key") + with patch.dict(os.environ, {"RUSTFS_SECRET_KEY": "private-value"}): + with self.assertRaisesRegex(ValueError, "exposes RUSTFS_SECRET_KEY"): + audit_summary([put, {**get, "accessKey": "private-value"}], "bucket", "key") + if __name__ == "__main__": unittest.main() From 4ccfbd2be18d30049e136434fe2f5dd1a4c923be Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 19:55:46 +0800 Subject: [PATCH 016/343] test: migrate OIDC session validation and decode audit batches --- docs/validation-migration-map.md | 27 ++- rustfs-security-test.sh | 16 +- scripts/run_validation_case.sh | 2 + scripts/validation_oidc.py | 265 ++++++++++++++++++++++++++++ scripts/validation_oidc/run_case.sh | 122 +++++++++++++ scripts/validation_webhooks.py | 6 + test_validation_oidc.py | 48 +++++ test_validation_webhooks.py | 6 + 8 files changed, 484 insertions(+), 8 deletions(-) create mode 100644 scripts/validation_oidc.py create mode 100644 scripts/validation_oidc/run_case.sh create mode 100644 test_validation_oidc.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 7dd77732..a19ad4cd 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -47,8 +47,10 @@ the canonical result can become PASS. `ilm.configuration-roundtrip`, `ilm.tag-filter-expiration`, `ilm.disabled-rule`, and `ilm.rule-removal` have live PASS evidence in RustFS Actions runs `38044956285`, `38045047518`, `38045154019`, `38045412209`, `38045496042`, and `38045714246` respectively. -The other eight ILM executors have local contract and dry-run checks but no -live PASS evidence yet. +`ilm.delete-all-versions`, `ilm.delete-marker-expiration`, +`ilm.multipart-expiration`, and `ilm.newer-noncurrent-retention` also passed +live in runs `38049740183`, `38049817172`, `38049890130`, and `38049965929`. +The remaining four ILM executors still require live evidence. All 13 Pool lifecycle IDs now dispatch to the source black-box pool checks. The driver reuses the existing auto-testing Pool suite's two-node seed and @@ -61,8 +63,9 @@ The 18 bucket-replication and 20 site-replication IDs now dispatch to the source black-box checks. The driver uses the existing auto-testing four-node main cluster and second RustFS instance on port 19000, including the source's site-group fixture and site outage hooks. Logical bucket isolation and -physical teardown are required for PASS. These cases have not yet completed -live validation. +physical teardown are required for PASS. +`bucket-replication.source-versioning-required` passed live in run +`38049256196`; the other 37 replication cases still require live evidence. `kms.sse-kms-round-trip` has a dedicated executor using the existing local KMS single-disk deployment. It checks the same object's encryption key, @@ -89,6 +92,16 @@ exposed an executor parsing mismatch: the candidate emits `s3:PutObject` and `s3:GetObject`. The parser now recognizes those exact names and waits for both operations on the selected object; the case still requires a live rerun. +`oidc.login-and-session` uses the Security suite's current single-node +deployment and a run-owned Keycloak container with a pinned image. It performs +password login, verifies issuer/audience/subject, exchanges the identity token +for STS credentials, and checks the readonly identity policy plus a narrower +session policy on existing objects. Missing/invalid session tokens, tampered +identity tokens, and wrong audiences must fail. Fixtures, containers, and the +RustFS deployment must all be cleaned before PASS. Runtime credentials are +excluded from evidence. This executor does not require a RustFS checkout and +still requires live validation. + `iam.policy-enforcement` has a dedicated executor on the Security suite's `single-single` deployment. It uses one principal with a policy that allows PUT/GET and explicitly denies DELETE, then checks the denied DELETE did not @@ -115,9 +128,9 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 73 currently dispatchable IDs are the S3, two scanner, +case evidence. The 74 currently dispatchable IDs are the S3, two scanner, KMS, tiering, events, audit, IAM, 14 additional ILM, 13 Pool, and 38 -replication cases above. Fourteen have live PASS evidence; the other 144 IDs +replication cases and OIDC above. Nineteen have live PASS evidence; the other 139 IDs remain unverified. The eight planned cases have these concrete reuse candidates: @@ -130,7 +143,7 @@ The eight planned cases have these concrete reuse candidates: | `events.delivery` | EVT-103, EVT-104 | Dedicated executor and live PASS evidence. | | `audit.request-correlation` | AUD-102, AUD-104, AUD-106 | Dedicated executor; live evidence pending. | | `iam.policy-enforcement` | IAM-105, IAM-109 | Dedicated executor and live PASS evidence. | -| `oidc.login-and-session` | OIDC-103 | The live Keycloak path can be skipped by the old suite; validation requires an actual session-boundary result. | +| `oidc.login-and-session` | OIDC-103 | Dedicated executor using real Keycloak login and STS session boundaries; live evidence pending. | | `s3.basic-object-round-trip` | S3C-101, 201, 203, 205, 206, 207 | Dedicated executor and live PASS evidence. | Migration work on an ID is complete only when its result contains the exact diff --git a/rustfs-security-test.sh b/rustfs-security-test.sh index ec690803..217dc89e 100644 --- a/rustfs-security-test.sh +++ b/rustfs-security-test.sh @@ -251,6 +251,18 @@ RUSTFS_CONSOLE_ENABLE=${RUSTFS_CONSOLE_ENABLE} RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL} RUSTFS_OBS_LOG_DIRECTORY="${RUSTFS_OBS_LOG_DIRECTORY}" EOF + if [ "${VALIDATION_CASE}" = oidc.login-and-session ]; then + cat <&2 exit 2 ;; diff --git a/scripts/validation_oidc.py b/scripts/validation_oidc.py new file mode 100644 index 00000000..f585b919 --- /dev/null +++ b/scripts/validation_oidc.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +"""Validate a real Keycloak identity and STS session on the Security fleet.""" + +from __future__ import annotations + +import argparse +import base64 +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import socket +import subprocess +import sys +import time +from urllib import error, parse, request +import uuid +import xml.etree.ElementTree as ET + + +KEYCLOAK_IMAGE = "quay.io/keycloak/keycloak@sha256:6a7217a100bd3e5de4063a27a538ef999a3c5a88c4b4ec0ffc0a642aee7b2597" +HTTP = request.build_opener(request.ProxyHandler({})) + + +def save(path: Path, value: dict) -> None: + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def docker(*arguments: str) -> str: + result = subprocess.run(["docker", *arguments], capture_output=True, text=True, timeout=300) + if result.returncode: + raise RuntimeError(f"Keycloak container operation {arguments[0]} failed ({result.returncode})") + return result.stdout + + +def http(url: str, data: dict | None = None) -> tuple[int, bytes]: + body = parse.urlencode(data).encode() if data is not None else None + headers = {"Content-Type": "application/x-www-form-urlencoded"} if body is not None else {} + try: + with HTTP.open(request.Request(url, data=body, headers=headers), timeout=15) as response: + return response.status, response.read(2 * 1024 * 1024) + except error.HTTPError as response: + return response.code, response.read(2 * 1024 * 1024) + + +def setup(directory: Path) -> None: + with socket.socket() as sock: + sock.bind(("0.0.0.0", 0)) + port = sock.getsockname()[1] + origin = f"http://{os.environ.get('RUSTFS_HEAL_IP', '10.0.0.4')}:{port}" + issuer = origin + "/realms/rustfs-validation" + state = {"container": "rustfs-validation-oidc-" + secrets.token_hex(8), + "issuer": issuer, "config_url": issuer + "/.well-known/openid-configuration", + "client_secret": secrets.token_hex(24), "wrong_client_secret": secrets.token_hex(24), + "password": secrets.token_hex(24), "subject": str(uuid.uuid4())} + state_path = directory / "oidc-state.json" + save(state_path, state) + state_path.chmod(0o600) + fixture = {"realm": "rustfs-validation", "enabled": True, "sslRequired": "none", + "accessTokenLifespan": 300, "clients": [], + "users": [{"id": state["subject"], "username": "validation-user", "enabled": True, + "firstName": "Validation", "lastName": "User", "emailVerified": True, + "email": "validation@example.test", "credentials": [ + {"type": "password", "value": state["password"], "temporary": False}]}]} + for client_id, secret_name in (("rustfs-validation", "client_secret"), ("wrong-audience", "wrong_client_secret")): + fixture["clients"].append({"clientId": client_id, "secret": state[secret_name], "enabled": True, + "protocol": "openid-connect", "publicClient": False, + "standardFlowEnabled": False, "directAccessGrantsEnabled": True}) + fixture_path = directory / "oidc-realm.json" + save(fixture_path, fixture) + fixture_path.chmod(0o644) + docker("run", "--detach", "--rm", "--name", state["container"], "--memory", "1g", + "--publish", f"0.0.0.0:{port}:8080", "--env", f"KC_HOSTNAME={origin}", + "--volume", f"{fixture_path}:/opt/keycloak/data/import/realm.json:ro", + KEYCLOAK_IMAGE, "start-dev", "--import-realm") + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + try: + status, body = http(state["config_url"]) + discovery = json.loads(body) if status == 200 else {} + if discovery.get("issuer") == issuer and discovery.get("token_endpoint") and discovery.get("jwks_uri"): + save(directory / "validation-oidc-provider.json", + {"image": KEYCLOAK_IMAGE, "issuer": issuer, "discovery_sha256": hashlib.sha256(body).hexdigest(), + "token_endpoint": discovery["token_endpoint"], "jwks_uri": discovery["jwks_uri"]}) + return + except (OSError, ValueError): + pass + if docker("inspect", "--format", "{{.State.Running}}", state["container"]).strip() != "true": + raise RuntimeError("Keycloak exited before discovery was ready") + time.sleep(2) + raise RuntimeError("Keycloak discovery did not become ready within 180 seconds") + + +def xml_values(body: bytes) -> dict[str, str]: + return {element.tag.rsplit("}", 1)[-1]: element.text or "" for element in ET.fromstring(body).iter()} + + +def token(state: dict, client_id: str, secret_name: str) -> tuple[str, dict]: + status, body = http(state["issuer"] + "/protocol/openid-connect/token", + {"grant_type": "password", "client_id": client_id, + "client_secret": state[secret_name], "username": "validation-user", + "password": state["password"], "scope": "openid"}) + if status != 200: + raise RuntimeError(f"Keycloak password login failed (HTTP {status})") + encoded = json.loads(body)["id_token"] + payload = encoded.split(".")[1] + claims = json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) + audience = claims["aud"] if isinstance(claims["aud"], list) else [claims["aud"]] + if claims["iss"] != state["issuer"] or client_id not in audience or claims["sub"] != state["subject"]: + raise RuntimeError("Keycloak identity claims differ from the configured fixture") + return encoded, {name: claims[name] for name in ("iss", "sub", "aud", "exp")} + + +def sts(endpoint: str, identity_token: str, subject: str, policy: dict | None = None) -> tuple[dict, dict]: + data = {"Action": "AssumeRoleWithWebIdentity", "Version": "2011-06-15", + "DurationSeconds": "900", "WebIdentityToken": identity_token} + if policy is not None: + data["Policy"] = json.dumps(policy, separators=(",", ":")) + status, body = http(endpoint.rstrip("/") + "/", data) + values = xml_values(body) + if status != 200: + raise RuntimeError(f"valid OIDC identity failed STS (HTTP {status}, code {values.get('Code', 'missing')})") + required = ("AccessKeyId", "SecretAccessKey", "SessionToken", "Expiration", "SubjectFromWebIdentityToken") + if not all(values.get(name) for name in required) or values["SubjectFromWebIdentityToken"] != subject: + raise RuntimeError("STS omitted credentials or returned a different identity subject") + expiration = datetime.fromisoformat(values["Expiration"].replace("Z", "+00:00")) + remaining = (expiration - datetime.now(timezone.utc)).total_seconds() + if not 0 < remaining <= 960: + raise RuntimeError("STS session expiration is outside the requested 900 second boundary") + credentials = {"AWS_ACCESS_KEY_ID": values["AccessKeyId"], "AWS_SECRET_ACCESS_KEY": values["SecretAccessKey"], + "AWS_SESSION_TOKEN": values["SessionToken"]} + return credentials, {"subject": subject, "expiration": values["Expiration"]} + + +def aws(endpoint: str, credentials: dict, *arguments: str, denied: bool = False) -> str: + environment = os.environ.copy() + for name in ("AWS_SESSION_TOKEN", "AWS_SECURITY_TOKEN", "AWS_PROFILE"): + environment.pop(name, None) + environment.update(credentials) + environment.update(AWS_EC2_METADATA_DISABLED="true", AWS_CONFIG_FILE="/dev/null", + AWS_SHARED_CREDENTIALS_FILE="/dev/null", AWS_DEFAULT_REGION="us-east-1") + result = subprocess.run(["aws", "--endpoint-url", endpoint, "--no-cli-pager", "s3api", *arguments], + env=environment, capture_output=True, text=True, timeout=60) + if denied: + match = re.search(r"An error occurred \(([A-Za-z0-9]+)\)", result.stderr) + code = match.group(1) if match else "missing" + if not result.returncode or code not in {"AccessDenied", "InvalidToken", "InvalidAccessKeyId"}: + raise RuntimeError(f"expected access boundary rejection, received {code} (exit {result.returncode})") + return code + if result.returncode: + raise RuntimeError(f"S3 operation {arguments[0]} failed (exit {result.returncode})") + return result.stdout + + +def test(directory: Path, endpoint: str) -> None: + state = json.loads((directory / "oidc-state.json").read_text()) + root = {"AWS_ACCESS_KEY_ID": os.environ["RUSTFS_ACCESS_KEY"], "AWS_SECRET_ACCESS_KEY": os.environ["RUSTFS_SECRET_KEY"]} + prefix = "val-oidc-" + secrets.token_hex(8) + buckets = [prefix + "-allowed", prefix + "-outside"] + source = directory / "oidc-source.bin" + body = secrets.token_bytes(65536) + source.write_bytes(body) + observations = {"identity": {}, "checks": {}, "cleanup": "PENDING"} + created = [] + primary = None + try: + for bucket in buckets: + aws(endpoint, root, "create-bucket", "--bucket", bucket) + created.append(bucket) + aws(endpoint, root, "put-object", "--bucket", bucket, "--key", "object.bin", "--body", str(source)) + encoded, claims = token(state, "rustfs-validation", "client_secret") + observations["identity"] = claims + identity_credentials, identity_session = sts(endpoint, encoded, state["subject"]) + observations["identity_session"] = identity_session + identity_read = directory / "oidc-identity-read.bin" + aws(endpoint, identity_credentials, "get-object", "--bucket", buckets[0], "--key", "object.bin", str(identity_read)) + if identity_read.read_bytes() != body: + raise RuntimeError("OIDC identity GET returned different object bytes") + identity_denial = aws(endpoint, identity_credentials, "put-object", "--bucket", buckets[0], + "--key", "forbidden.bin", "--body", str(source), denied=True) + if identity_denial != "AccessDenied": + raise RuntimeError("readonly identity boundary did not return AccessDenied") + observations["checks"]["identity_write_denied"] = identity_denial + policy = {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Action": "s3:GetObject", + "Resource": f"arn:aws:s3:::{buckets[0]}/*"}]} + session, session_observation = sts(endpoint, encoded, state["subject"], policy) + observations["session"] = session_observation + received = directory / "oidc-read.bin" + aws(endpoint, session, "get-object", "--bucket", buckets[0], "--key", "object.bin", str(received)) + if received.read_bytes() != body: + raise RuntimeError("OIDC session GET returned different object bytes") + observations["checks"]["allowed_get_sha256"] = hashlib.sha256(body).hexdigest() + outside_code = aws(endpoint, session, "get-object", "--bucket", buckets[1], "--key", "object.bin", + str(directory / "oidc-denied.bin"), denied=True) + if outside_code != "AccessDenied": + raise RuntimeError("session policy boundary did not return AccessDenied") + observations["checks"]["session_outside_denied"] = outside_code + for name, altered in (("missing_session_token", {key: value for key, value in session.items() if key != "AWS_SESSION_TOKEN"}), + ("invalid_session_token", {**session, "AWS_SESSION_TOKEN": "invalid-session-token"})): + observations["checks"][name] = aws(endpoint, altered, "get-object", "--bucket", buckets[0], "--key", "object.bin", + str(directory / "oidc-invalid.bin"), denied=True) + parts = encoded.split(".") + parts[2] = ("A" if parts[2][0] != "A" else "B") + parts[2][1:] + wrong_token, _ = token(state, "wrong-audience", "wrong_client_secret") + for name, invalid in (("tampered_identity", ".".join(parts)), ("wrong_audience", wrong_token)): + status, response = http(endpoint.rstrip("/") + "/", {"Action": "AssumeRoleWithWebIdentity", "Version": "2011-06-15", + "DurationSeconds": "900", "WebIdentityToken": invalid}) + code = xml_values(response).get("Code") + if status != 403 or code != "AccessDenied": + raise RuntimeError(f"invalid OIDC identity accepted or rejected incorrectly (HTTP {status}, code {code})") + observations["checks"][name] = {"http_status": status, "code": code} + for bucket in buckets: + aws(endpoint, root, "get-object", "--bucket", bucket, "--key", "object.bin", str(directory / "oidc-root-read.bin")) + if (directory / "oidc-root-read.bin").read_bytes() != body: + raise RuntimeError("rejected session requests changed the root fixture") + observations["checks"]["fixtures_preserved"] = True + except Exception as exc: + primary = exc + finally: + cleanup_errors = [] + for bucket in created: + try: + aws(endpoint, root, "delete-object", "--bucket", bucket, "--key", "object.bin") + aws(endpoint, root, "delete-bucket", "--bucket", bucket) + except Exception: + cleanup_errors.append(bucket) + observations["cleanup"] = "FAIL" if cleanup_errors else "PASS" + save(directory / "validation-oidc-observations.json", observations) + if primary is not None: + raise primary + if cleanup_errors: + raise RuntimeError("OIDC fixture cleanup failed") + + +def teardown(directory: Path) -> None: + state_path = directory / "oidc-state.json" + if not state_path.exists(): + return + state = json.loads(state_path.read_text()) + containers = docker("ps", "--all", "--format", "{{.Names}}").splitlines() + if state["container"] in containers: + docker("rm", "--force", state["container"]) + if state["container"] in docker("ps", "--all", "--format", "{{.Names}}").splitlines(): + raise RuntimeError("run-owned Keycloak container remains after cleanup") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("setup", "test", "teardown")) + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--endpoint") + args = parser.parse_args() + {"setup": lambda: setup(args.directory), "test": lambda: test(args.directory, args.endpoint), + "teardown": lambda: teardown(args.directory)}[args.mode]() + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"OIDC validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_oidc/run_case.sh b/scripts/validation_oidc/run_case.sh new file mode 100644 index 00000000..be92f33a --- /dev/null +++ b/scripts/validation_oidc/run_case.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUSTFS_SECURITY_SCRIPT_SOURCE_ONLY=1 source "${ROOT}/rustfs-security-test.sh" + +candidate="" testing_commit="" artifacts="" +environment_active=0 provider_active=0 finalized=0 +binary_sha="" evidence="" cleanup_status=PASS + +finalize_environment() { + local cleanup_rc=0 + if [ "${environment_active}" -eq 1 ]; then + environment_active=0 + (env_teardown single-single) >> "${evidence}/cleanup.log" 2>&1 || cleanup_rc=1 + fi + if [ "${provider_active}" -eq 1 ]; then + provider_active=0 + python3 -B "${ROOT}/scripts/validation_oidc.py" teardown --directory "${TEST_TMP}" \ + >> "${evidence}/cleanup.log" 2>&1 || cleanup_rc=1 + fi + [ "${cleanup_rc}" -eq 0 ] || cleanup_status=FAIL + return "${cleanup_rc}" +} + +on_exit() { + local status=$? + trap - EXIT + if [ "${DRY_RUN}" -eq 0 ] && [ "${finalized}" -eq 0 ] && [ -n "${evidence}" ]; then + finalize_environment || true + fi + cleanup_tmp + exit "${status}" +} +trap on_exit EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +while [ "$#" -gt 0 ]; do + case "$1" in + --validation-case) VALIDATION_CASE="$2"; shift 2 ;; + --candidate-manifest) candidate="$2"; shift 2 ;; + --testing-commit) testing_commit="$2"; shift 2 ;; + --validation-artifacts-dir) artifacts="$2"; shift 2 ;; + --dry-run) DRY_RUN=1; shift ;; + -y|--yes) ASSUME_YES=1; shift ;; + *) die "unknown OIDC validation option: $1" ;; + esac +done +[ "${VALIDATION_CASE}" = oidc.login-and-session ] || die "OIDC validation case required" +if [ "${DRY_RUN}" -eq 1 ]; then + VALIDATION_OIDC_ORIGIN="http://${RUSTFS_HEAL_IP:-10.0.0.4}:18090" + VALIDATION_OIDC_ISSUER="${VALIDATION_OIDC_ORIGIN}/realms/rustfs-validation" + VALIDATION_OIDC_CONFIG_URL="${VALIDATION_OIDC_ISSUER}/.well-known/openid-configuration" + VALIDATION_OIDC_CLIENT_SECRET=dry-run + env_setup single-single + env_teardown single-single + exit 0 +fi +for command in docker ssh curl openssl aws jq python3 rc; do need_cmd "${command}" "OIDC validation"; done +[ -f "${candidate}" ] || die "candidate manifest missing" +[ -n "${artifacts}" ] && [ ! -e "${artifacts}" ] || die "fresh artifact directory required" +[ "${testing_commit}" = "$(git -C "${ROOT}" rev-parse HEAD)" ] || die "testing commit differs from checkout" +PACKAGE_URL="$(jq -er '.package_url' "${candidate}")" +PACKAGE_SHA256="$(jq -er '.package_sha256' "${candidate}")" +[[ "${PACKAGE_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "invalid candidate SHA-256" +export RUSTFS_ACCESS_KEY="${ACCESS_KEY}" RUSTFS_SECRET_KEY="${SECRET_KEY}" +evidence="${artifacts}/evidence/${VALIDATION_CASE}" +mkdir -p "${artifacts}/cases" "${evidence}" +python3 "${ROOT}/scripts/validation_result.py" --case-id "${VALIDATION_CASE}" \ + --profile single-1disk --result-file "${artifacts}/cases/${VALIDATION_CASE}.json" \ + --verdict RUNNING --cleanup-status PENDING + +provider_active=1 +python3 -B "${ROOT}/scripts/validation_oidc.py" setup --directory "${TEST_TMP}" \ + > "${evidence}/provider-setup.log" 2>&1 || { + cat "${evidence}/provider-setup.log"; exit 1; + } +VALIDATION_OIDC_ISSUER="$(jq -er '.issuer' "${TEST_TMP}/oidc-state.json")" +VALIDATION_OIDC_ORIGIN="${VALIDATION_OIDC_ISSUER%/realms/*}" +VALIDATION_OIDC_CONFIG_URL="$(jq -er '.config_url' "${TEST_TMP}/oidc-state.json")" +VALIDATION_OIDC_CLIENT_SECRET="$(jq -er '.client_secret' "${TEST_TMP}/oidc-state.json")" +environment_active=1 +ENDPOINT="${RUSTFS_API_ENDPOINT:-${ENDPOINT_SINGLE_SINGLE}}" +(env_setup single-single) > "${evidence}/setup.log" 2>&1 || { + cat "${evidence}/setup.log"; exit 1; +} +binary_sha="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${NODES[0]}" sha256sum /usr/bin/rustfs | awk '{print $1}')" +[[ "${binary_sha}" =~ ^[0-9a-f]{64}$ ]] || die "installed RustFS binary SHA-256 unavailable" +case_rc=0 +python3 -B "${ROOT}/scripts/validation_oidc.py" test --directory "${TEST_TMP}" --endpoint "${ENDPOINT}" \ + > "${evidence}/test.log" 2>&1 || case_rc=$? +cat "${evidence}/test.log" +if ! jq -e '.cleanup == "PASS"' "${TEST_TMP}/validation-oidc-observations.json" >/dev/null 2>&1; then + cleanup_status=FAIL +fi +finalize_environment || true +finalized=1 +for item in validation-oidc-provider.json validation-oidc-observations.json; do + if [ -f "${TEST_TMP}/${item}" ]; then cp "${TEST_TMP}/${item}" "${evidence}/${item}"; fi +done +verdict=FAIL reason="OIDC identity or session assertions failed; inspect evidence" +if [ "${case_rc}" -eq 0 ] && [ "${cleanup_status}" = PASS ]; then + verdict=PASS; reason="" +elif [ "${case_rc}" -eq 0 ]; then + verdict=INFRA_ERROR; reason="OIDC environment finalization failed" +fi +args=("${ROOT}/scripts/validation_result.py" --case-id "${VALIDATION_CASE}" + --profile single-1disk --result-file "${artifacts}/cases/${VALIDATION_CASE}.json" + --verdict "${verdict}" --cleanup-status "${cleanup_status}" + --candidate-manifest "${candidate}" --rc-bin "$(command -v rc)" + --testing-commit "${testing_commit}" --binary-sha256 "${binary_sha}" + --evidence "evidence/${VALIDATION_CASE}/test.log" + --evidence "evidence/${VALIDATION_CASE}/setup.log" + --evidence "evidence/${VALIDATION_CASE}/cleanup.log") +if [ -n "${reason}" ]; then args+=(--reason "${reason}"); fi +if [ "${verdict}" = PASS ]; then + args+=(--evidence "evidence/${VALIDATION_CASE}/validation-oidc-provider.json" + --evidence "evidence/${VALIDATION_CASE}/validation-oidc-observations.json") +fi +python3 "${args[@]}" +[ "${verdict}" = PASS ] diff --git a/scripts/validation_webhooks.py b/scripts/validation_webhooks.py index 14c25938..cbf65d7a 100644 --- a/scripts/validation_webhooks.py +++ b/scripts/validation_webhooks.py @@ -55,7 +55,13 @@ def audit_summary(values: list[dict], bucket: str, key: str) -> dict: if secret and secret.lower() in serialized: raise ValueError(f"audit payload exposes {name}") matches: dict[str, dict] = {} + entries = [] for value in values: + batch = value.get("Records", [value]) + if not isinstance(batch, list) or any(not isinstance(entry, dict) for entry in batch): + raise ValueError("audit Records must be an array of objects") + entries.extend(batch) + for value in entries: api = value.get("api") or {} if api.get("bucket") != bucket or api.get("object") != key: continue diff --git a/test_validation_oidc.py b/test_validation_oidc.py new file mode 100644 index 00000000..0234dc2a --- /dev/null +++ b/test_validation_oidc.py @@ -0,0 +1,48 @@ +"""Reject false OIDC validation success at authentication boundaries.""" + +from datetime import datetime, timedelta, timezone +import os +import subprocess +import unittest +from unittest.mock import patch + +from scripts.validation_oidc import aws, sts + + +class OidcBoundaryTest(unittest.TestCase): + def test_sts_requires_subject_and_bounded_expiration(self): + def response(subject, seconds): + expiration = (datetime.now(timezone.utc) + timedelta(seconds=seconds)).isoformat() + return ("" + "test-keytest-secret" + "test-token" + f"{expiration}" + f"{subject}" + "").encode() + with patch("scripts.validation_oidc.http", return_value=(200, response("expected", 900))): + credentials, observation = sts("http://fixture", "identity-token", "expected") + self.assertEqual(credentials["AWS_SESSION_TOKEN"], "test-token") + self.assertNotIn("test-secret", str(observation)) + for subject, seconds in (("different", 900), ("expected", -1), ("expected", 3600)): + with self.subTest(subject=subject, seconds=seconds): + with patch("scripts.validation_oidc.http", return_value=(200, response(subject, seconds))): + with self.assertRaises(RuntimeError): + sts("http://fixture", "identity-token", "expected") + + def test_transport_error_cannot_prove_access_denial(self): + credentials = {"AWS_ACCESS_KEY_ID": "temporary-key", "AWS_SECRET_ACCESS_KEY": "temporary-secret"} + network_error = subprocess.CompletedProcess([], 255, "", "Could not connect to endpoint") + with patch("scripts.validation_oidc.subprocess.run", return_value=network_error): + with self.assertRaisesRegex(RuntimeError, "expected access boundary rejection"): + aws("http://fixture", credentials, "get-object", denied=True) + access_denied = subprocess.CompletedProcess([], 255, "", "An error occurred (AccessDenied) when calling GetObject") + with patch.dict(os.environ, {"AWS_SESSION_TOKEN": "ambient-token", "AWS_PROFILE": "ambient-profile"}): + with patch("scripts.validation_oidc.subprocess.run", return_value=access_denied) as run: + self.assertEqual(aws("http://fixture", credentials, "get-object", denied=True), "AccessDenied") + environment = run.call_args.kwargs["env"] + self.assertNotIn("AWS_SESSION_TOKEN", environment) + self.assertNotIn("AWS_PROFILE", environment) + + +if __name__ == "__main__": + unittest.main() diff --git a/test_validation_webhooks.py b/test_validation_webhooks.py index c3e84016..940e5965 100644 --- a/test_validation_webhooks.py +++ b/test_validation_webhooks.py @@ -55,6 +55,10 @@ def entry(name, request_id, key="key"): get = entry("s3:GetObject", "get-id") result = audit_summary([put, get], "bucket", "key") self.assertEqual(result["operations"]["GetObject"]["api_name"], "s3:GetObject") + wrapped = [{"Records": [put]}, {"Records": [get]}] + self.assertEqual(audit_summary(wrapped, "bucket", "key"), result) + with self.assertRaisesRegex(ValueError, "array of objects"): + audit_summary([{"Records": [put, "invalid"]}], "bucket", "key") with self.assertRaisesRegex(ValueError, "not both delivered"): audit_summary([put, entry("s3:GetObject", "other-id", "other-key")], "bucket", "key") with self.assertRaisesRegex(ValueError, "not both delivered"): @@ -62,6 +66,8 @@ def entry(name, request_id, key="key"): with patch.dict(os.environ, {"RUSTFS_SECRET_KEY": "private-value"}): with self.assertRaisesRegex(ValueError, "exposes RUSTFS_SECRET_KEY"): audit_summary([put, {**get, "accessKey": "private-value"}], "bucket", "key") + with self.assertRaisesRegex(ValueError, "exposes RUSTFS_SECRET_KEY"): + audit_summary([{"Records": [put, {**get, "accessKey": "private-value"}]}], "bucket", "key") if __name__ == "__main__": From 0b5100c946ecc0d671a3f86199bcaa071461f6e2 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 19:58:54 +0800 Subject: [PATCH 017/343] test: diagnose Docker access for OIDC validation --- scripts/validation_oidc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/validation_oidc.py b/scripts/validation_oidc.py index f585b919..1cbe2cc1 100644 --- a/scripts/validation_oidc.py +++ b/scripts/validation_oidc.py @@ -31,8 +31,10 @@ def save(path: Path, value: dict) -> None: def docker(*arguments: str) -> str: result = subprocess.run(["docker", *arguments], capture_output=True, text=True, timeout=300) + if result.returncode and "permission denied while trying to connect" in result.stderr.lower(): + result = subprocess.run(["sudo", "-n", "docker", *arguments], capture_output=True, text=True, timeout=300) if result.returncode: - raise RuntimeError(f"Keycloak container operation {arguments[0]} failed ({result.returncode})") + raise RuntimeError(f"Keycloak container operation {arguments[0]} failed ({result.returncode}): {result.stderr.strip()[:600]}") return result.stdout From f19a5e32923f306420c3a329c21b6cc4f5d8a0f8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 20:06:47 +0800 Subject: [PATCH 018/343] test: verify configured OIDC session identity claim --- scripts/validation_oidc.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/validation_oidc.py b/scripts/validation_oidc.py index 1cbe2cc1..948623fe 100644 --- a/scripts/validation_oidc.py +++ b/scripts/validation_oidc.py @@ -111,9 +111,10 @@ def token(state: dict, client_id: str, secret_name: str) -> tuple[str, dict]: payload = encoded.split(".")[1] claims = json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) audience = claims["aud"] if isinstance(claims["aud"], list) else [claims["aud"]] - if claims["iss"] != state["issuer"] or client_id not in audience or claims["sub"] != state["subject"]: + if (claims["iss"] != state["issuer"] or client_id not in audience or claims["sub"] != state["subject"] + or claims.get("preferred_username") != "validation-user"): raise RuntimeError("Keycloak identity claims differ from the configured fixture") - return encoded, {name: claims[name] for name in ("iss", "sub", "aud", "exp")} + return encoded, {name: claims[name] for name in ("iss", "sub", "aud", "exp", "preferred_username")} def sts(endpoint: str, identity_token: str, subject: str, policy: dict | None = None) -> tuple[dict, dict]: @@ -175,7 +176,9 @@ def test(directory: Path, endpoint: str) -> None: aws(endpoint, root, "put-object", "--bucket", bucket, "--key", "object.bin", "--body", str(source)) encoded, claims = token(state, "rustfs-validation", "client_secret") observations["identity"] = claims - identity_credentials, identity_session = sts(endpoint, encoded, state["subject"]) + # RustFS exposes the configured username claim as the STS session identity. + # The JWT's issuer-scoped subject was independently checked above. + identity_credentials, identity_session = sts(endpoint, encoded, claims["preferred_username"]) observations["identity_session"] = identity_session identity_read = directory / "oidc-identity-read.bin" aws(endpoint, identity_credentials, "get-object", "--bucket", buckets[0], "--key", "object.bin", str(identity_read)) @@ -188,7 +191,7 @@ def test(directory: Path, endpoint: str) -> None: observations["checks"]["identity_write_denied"] = identity_denial policy = {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Action": "s3:GetObject", "Resource": f"arn:aws:s3:::{buckets[0]}/*"}]} - session, session_observation = sts(endpoint, encoded, state["subject"], policy) + session, session_observation = sts(endpoint, encoded, claims["preferred_username"], policy) observations["session"] = session_observation received = directory / "oidc-read.bin" aws(endpoint, session, "get-object", "--bucket", buckets[0], "--key", "object.bin", str(received)) From 0415119252f9007d8fe4a1a5a693715ec4d938a0 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 20:14:24 +0800 Subject: [PATCH 019/343] fix(validation): distinguish invalid OIDC sessions from policy denials --- scripts/validation_oidc.py | 12 +++++++++--- test_validation_oidc.py | 9 +++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/validation_oidc.py b/scripts/validation_oidc.py index 948623fe..a0ac36eb 100644 --- a/scripts/validation_oidc.py +++ b/scripts/validation_oidc.py @@ -138,7 +138,8 @@ def sts(endpoint: str, identity_token: str, subject: str, policy: dict | None = return credentials, {"subject": subject, "expiration": values["Expiration"]} -def aws(endpoint: str, credentials: dict, *arguments: str, denied: bool = False) -> str: +def aws(endpoint: str, credentials: dict, *arguments: str, denied: bool = False, + rejection_codes: frozenset[str] = frozenset({"AccessDenied", "InvalidToken", "InvalidAccessKeyId"})) -> str: environment = os.environ.copy() for name in ("AWS_SESSION_TOKEN", "AWS_SECURITY_TOKEN", "AWS_PROFILE"): environment.pop(name, None) @@ -150,7 +151,7 @@ def aws(endpoint: str, credentials: dict, *arguments: str, denied: bool = False) if denied: match = re.search(r"An error occurred \(([A-Za-z0-9]+)\)", result.stderr) code = match.group(1) if match else "missing" - if not result.returncode or code not in {"AccessDenied", "InvalidToken", "InvalidAccessKeyId"}: + if not result.returncode or code not in rejection_codes: raise RuntimeError(f"expected access boundary rejection, received {code} (exit {result.returncode})") return code if result.returncode: @@ -203,10 +204,14 @@ def test(directory: Path, endpoint: str) -> None: if outside_code != "AccessDenied": raise RuntimeError("session policy boundary did not return AccessDenied") observations["checks"]["session_outside_denied"] = outside_code + # RustFS auth::check_session_token returns InvalidRequest for absent or + # mismatched temporary tokens. Authorization denials above must still + # return AccessDenied; a generic request error cannot prove that policy. for name, altered in (("missing_session_token", {key: value for key, value in session.items() if key != "AWS_SESSION_TOKEN"}), ("invalid_session_token", {**session, "AWS_SESSION_TOKEN": "invalid-session-token"})): observations["checks"][name] = aws(endpoint, altered, "get-object", "--bucket", buckets[0], "--key", "object.bin", - str(directory / "oidc-invalid.bin"), denied=True) + str(directory / "oidc-invalid.bin"), denied=True, + rejection_codes=frozenset({"InvalidRequest"})) parts = encoded.split(".") parts[2] = ("A" if parts[2][0] != "A" else "B") + parts[2][1:] wrong_token, _ = token(state, "wrong-audience", "wrong_client_secret") @@ -224,6 +229,7 @@ def test(directory: Path, endpoint: str) -> None: observations["checks"]["fixtures_preserved"] = True except Exception as exc: primary = exc + observations["failure"] = {"type": type(exc).__name__, "message": str(exc)} finally: cleanup_errors = [] for bucket in created: diff --git a/test_validation_oidc.py b/test_validation_oidc.py index 0234dc2a..ee97c2d1 100644 --- a/test_validation_oidc.py +++ b/test_validation_oidc.py @@ -43,6 +43,15 @@ def test_transport_error_cannot_prove_access_denial(self): self.assertNotIn("AWS_SESSION_TOKEN", environment) self.assertNotIn("AWS_PROFILE", environment) + def test_invalid_request_only_proves_the_session_token_boundary(self): + rejected = subprocess.CompletedProcess([], 254, "", "An error occurred (InvalidRequest) when calling GetObject") + credentials = {"AWS_ACCESS_KEY_ID": "temporary-key", "AWS_SECRET_ACCESS_KEY": "temporary-secret"} + with patch("scripts.validation_oidc.subprocess.run", return_value=rejected): + with self.assertRaisesRegex(RuntimeError, "expected access boundary rejection"): + aws("http://fixture", credentials, "get-object", denied=True) + self.assertEqual(aws("http://fixture", credentials, "get-object", denied=True, + rejection_codes=frozenset({"InvalidRequest"})), "InvalidRequest") + if __name__ == "__main__": unittest.main() From 1621a20475577b2dff6d25870815223fd5a61770 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 20:28:34 +0800 Subject: [PATCH 020/343] feat(validation): migrate Heal procedures and add current-fleet healthy executor --- docs/validation-heal-cases.md | 1851 +++++++++ docs/validation-heal-contract.md | 304 ++ docs/validation-heal-operations.md | 630 ++++ docs/validation-migration-map.md | 52 +- scripts/run_validation_case.sh | 2 + scripts/validation_heal/fleet_node.py | 234 ++ scripts/validation_heal/heal_fault.py | 318 ++ scripts/validation_heal/heal_probe.py | 453 +++ scripts/validation_heal/heal_signing.py | 19 + scripts/validation_heal/heal_workload.py | 294 ++ .../validation_heal/heal_workload_journal.py | 84 + scripts/validation_heal/healthy.py | 274 ++ scripts/validation_heal/matrix.py | 260 ++ .../prepare_heal_meta_inspector.py | 52 + scripts/validation_heal/recipes.json | 3303 +++++++++++++++++ scripts/validation_heal/run_case.sh | 114 + scripts/validation_heal/source.json | 55 + .../tools/heal_meta_inspect.rs | 82 + test_validation_heal.py | 137 + 19 files changed, 8501 insertions(+), 17 deletions(-) create mode 100644 docs/validation-heal-cases.md create mode 100644 docs/validation-heal-contract.md create mode 100644 docs/validation-heal-operations.md create mode 100644 scripts/validation_heal/fleet_node.py create mode 100644 scripts/validation_heal/heal_fault.py create mode 100644 scripts/validation_heal/heal_probe.py create mode 100644 scripts/validation_heal/heal_signing.py create mode 100644 scripts/validation_heal/heal_workload.py create mode 100644 scripts/validation_heal/heal_workload_journal.py create mode 100644 scripts/validation_heal/healthy.py create mode 100644 scripts/validation_heal/matrix.py create mode 100644 scripts/validation_heal/prepare_heal_meta_inspector.py create mode 100644 scripts/validation_heal/recipes.json create mode 100755 scripts/validation_heal/run_case.sh create mode 100644 scripts/validation_heal/source.json create mode 100644 scripts/validation_heal/tools/heal_meta_inspect.rs create mode 100644 test_validation_heal.py diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md new file mode 100644 index 00000000..9b1c6628 --- /dev/null +++ b/docs/validation-heal-cases.md @@ -0,0 +1,1851 @@ +# Heal validation procedures + +These source procedures require live execution and branch evidence on the current auto-testing Heal fleet. +Use [the operations reference](validation-heal-operations.md) before injecting a fault. A procedure card or complete archive is not a PASS. + + + +## A01: heal.admin-healthy-noop + +Topologies: H1,H2. Branches: normal, deep. + +1. 用 DB 夹具生成完整 manifest；先对每个对象保存 S3 基线和四盘分片摘要，关闭 scanner/MRF/auto-heal。 +2. 依次执行 START object（scanMode=1、2），WAIT 原 token 直至终态；再 STATUS 同 token 两次。 +3. 对每个对象的 outcome 区分 Normal 的 metadata_healthy 与 Deep 的 verified_healthy；两者均要求 healed=0、unchanged>0，Normal 不得冒充 payload 完整性证明。比较目标/控制对象物理摘要和 S3 清单，重复请求不得新建版本。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [BASELINE](validation-heal-operations.md#BASELINE) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [STATUS](validation-heal-operations.md#STATUS) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) + +Source assertions: + +- 对健康对象分别执行 normal/deep，再重复查询 +- 不修改用户数据或版本；Normal 只接受 `metadata_healthy`，Deep 才要求 `verified_healthy`，healthy 与 repaired 分开计数，不能把检查盘数当修复盘数。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A02: heal.admin-scope-isolation + +Topologies: H1,H2. Branches: root, bucket, prefix, object, pool-set. + +1. 用 DS 在两个桶的 a/、ab/、b/ 写入不同内容；另在未选 set 放控制对象；逐对象预先记录实际 pool/set。 +2. 分别隔离选中对象、相邻前缀及另一桶/未选 set 的一个 part。每个 selector 使用新夹具和故障 ID。 +3. 对象请求路径到完整 key；prefix 路径到 a/ 且 recursive=true；bucket 路径只含桶；root 为 /；pool/set 写入 body。START 后 WAIT。 +4. 先 PHYSICAL：选择范围内全部重建、范围外控制故障仍缺失；然后 GET 验证，避免验证请求提前修好控制故障。root 必须包含整个专用服务清单。 + +Operations: [SEED_DS](validation-heal-operations.md#SEED_DS) → [BASELINE](validation-heal-operations.md#BASELINE) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 分别以 root、bucket、prefix、object、pool/set 选择器启动 +- 目标范围实际遍历；相邻前缀、其他桶及未选 set 的故障保持原状；按对象清单而非目录数量验收。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A03: heal.admin-token-cross-node-status + +Topologies: H2. Branches: coordinator-to-peer, coordinator-to-stable-endpoint. + +1. 在 H2 同一跨节点 set 写入 DB；保存协调者、目标、稳定入口分别对应的端点，不把同一节点的两个别名当跨节点。 +2. 隔离目标节点 part；从 n0 START，记录原始 clientToken；通过 n1 和 stable 对同一路径、同 token 执行 STATUS。 +3. 按同一任务身份对齐三端状态，最多等候冻结的状态收敛预算；终态及核心计数必须一致，404/unknown 不能视为结束。 +4. WAIT 后先物理证明目标重建，再从所有 S3 入口 VERIFY；保存每个请求实际连接的节点日志。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [STATUS](validation-heal-operations.md#STATUS) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 从节点甲启动，经乙及稳定入口按原 token 查询 +- 任务身份、scope、终态及核心计数有界收敛；状态缺失/未知不能解释为完成。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A04: heal.admin-dry-run + +Topologies: H1,H2. Branches: missing-remove0-recreate0, missing-remove1-recreate0, bitflip-remove0-recreate1, bitflip-remove1-recreate1, versioned-missing, versioned-bitflip. + +1. 分别用 DB 与 DV 建夹具，冻结全部 versionId/marker；关闭后台来源，不在注入后 GET。 +2. 在原始 part 上做 missing/bitflip，每组独立新目录；保存故障后的 PHYSICAL 快照作为 dry-run 前基线。 +3. START body 同时传 dryRun=true、scanMode=2 和该实例的 remove/recreate；WAIT 仅接受 dry_run_observed，不能要求 dry-run 持有 incarnation。 +4. 前后所有分片/元数据/marker 字节与版本清单相同；没有 repaired ACK。完成非变更证明后恢复故障，再 VERIFY。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [BASELINE](validation-heal-operations.md#BASELINE) → [FAULT](validation-heal-operations.md#FAULT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 对缺片与损坏对象执行 dryRun，参数化 remove/recreate +- 有明确 dry_run_observed 诊断；用户对象分片、xl.meta、版本与删除标记不变；后台修复干扰须排除。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A05: heal.admin-overlap-and-force-start + +Topologies: H1,H2. Branches: same-scope, parent-child, disjoint, force-start. + +1. 用 DX 超过 128 项的外部分片夹具注入多对象缺片；预先从固定候选记录 overlap policy（复用 token 或冲突错误），禁止接受任意结果。 +2. START 递归范围，WINDOW 证明 0 + +## A06: heal.admin-stop-and-stale-token + +Topologies: H1,H2. Branches: stop-active, repeat-stop, old-status, old-stop-new-task. + +1. DX 批量缺片后 START；WINDOW 观察非零且未完成进度，再 STOP 原 token。 +2. 状态显示 aborted/cancelled 后仍观察物理写入静默窗口；停止受理成功不能替代在途退出。 +3. 新 START 得到不同 token，再对旧 token STATUS/STOP；新任务不应被旧请求终止，重复 stop 按冻结的 HTTP/错误码验收。 +4. 新任务完成后完整 VERIFY；记录取消项不能算 repaired，所有暂态错误保留原始 body。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [STOP](validation-heal-operations.md#STOP) → [STATUS](validation-heal-operations.md#STATUS) → [WAIT](validation-heal-operations.md#WAIT) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) + +Source assertions: + +- 非零修复进度时 stop，稳定后新建任务，再发旧 token 的查询/停止 +- 停止需等在途写结束并稳定；新任务身份独立，旧请求不得终止新任务；重复 stop 按公开契约处理。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A07: heal.admin-invalid-request + +Topologies: H1,H2. Branches: duplicate-query, invalid-bool, invalid-scan, conflicting-flags, invalid-selector, oversize-body. + +1. 建立健康 DB 和 PHYSICAL 基线；保存后台/任务状态及对象版本集合，用于检查负请求未意外启动任务。 +2. 逐个发送服务端负例：重复 clientToken；forceStart=wat；scanMode=99；forceStart=true&forceStop=true；pool=-1/set 越界；超过候选限制的 body。每个例固定预期 HTTP 及结构化错误码。 +3. 使用 ADMIN_RAW 的重复 --query 和原始 --body-file，不能只调用 rc 的参数校验；不使用超出专用 Heal 路由的请求。 +4. 全部响应无有效新任务受理，服务仍可用；PHYSICAL、版本、后台责任无改变；若返回 2xx 或错误无明确原因保留 FAIL。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [BASELINE](validation-heal-operations.md#BASELINE) → [ADMIN_RAW](validation-heal-operations.md#ADMIN_RAW) → [STATUS](validation-heal-operations.md#STATUS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 发送非法 selector、重复 query、非法布尔/scanMode、冲突 flags、超长 body +- 服务端返回已核实的结构化错误；无误启动、范围扩大或持久化副作用；CLI 参数错误不能替代服务端负例。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A08: heal.admin-authorization + +Topologies: H1,H2. Branches: anonymous, bad-signature, expired-signature, denied-user, least-privilege-user. + +1. 通过 IAM 手册在专用服务创建两个临时用户，分别没有和仅有 HealAdminAction；用受控凭据环境运行，先证明限制用户不能读未授权对象。 +2. 管理员创建待查询/停止任务；分别对 start/status/stop 发匿名、坏签名、有效签名但时间超期、无授权用户请求；过期请求由隔离客户端签名时间设置产生，不改服务器时间。 +3. 检查冻结的 AccessDenied/签名/时间错误；响应不暴露对象结果。最小授权用户完成受理/查询/停止，证明接口本身可工作。 +4. 移除临时用户和策略并验证凭据失效；所有原始证据先脱敏再归档，不把管理员 token 暴露给无关主体。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [IAM](validation-heal-operations.md#IAM) → [START](validation-heal-operations.md#START) → [ADMIN_RAW](validation-heal-operations.md#ADMIN_RAW) → [STATUS](validation-heal-operations.md#STATUS) → [STOP](validation-heal-operations.md#STOP) → [VERIFY](validation-heal-operations.md#VERIFY) → [RECOVER](validation-heal-operations.md#RECOVER) + +Source assertions: + +- 匿名、坏签名、过期签名、无 HealAdminAction 用户与最小授权用户分别请求 +- 未授权 start/status/stop 均不能越权或泄漏对象信息；授权用户可完成规定操作，凭据不得写入证据。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A09: heal.admin-nolock-safety + +Topologies: H1,H2. Branches: overwrite-nolock, delete-nolock, native-lock-fence. + +1. DB/DV 准备同一目标并缺片；两客户端使用同 key，冻结每个 PUT/DELETE 的唯一 nonce 和 ACK ledger。 +2. 一端 START body nolock=true，另一端按 WORKLOAD 提交覆盖或精确 versionId 删除；记录确实交叠的运行区间。 +3. 按 ACK 因果顺序重建期望清单并 VERIFY；最后成功覆盖不回退、已删除版本不复活。 +4. 另外运行 NATIVE 的锁/提交屏障回归并保存精确测试名、源码 SHA、命中日志；黑盒竞态与 native-lock-fence 单独结果，不以偶然重叠证明强制锁时点。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 通过签名 API 传 nolock=true，并与同对象提交制造确定性争用 +- 兼容字段不得跳过存储层对象锁；最后成功写入和已确认删除保持正确，原生屏障与真机竞争证据分开。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## A10: heal.admin-option-contract + +Topologies: H1,H2. Branches: recursive-false, recursive-true, recreate, remove-false, remove-true, update-parity, read-repair, selector-valid, selector-invalid. + +1. 将候选 HealOpts 支持表和危险参数副作用写入本批 contract.json；不根据 CLI help 推导 API 未暴露字段。 +2. 为每个 variant 使用新 DB/DV：固定范围、足够权威和控制对象；remove=true 只对经 C10 证明无活引用的本次孤儿执行。 +3. 逐一 START 并对比只改变一个字段的基线；recursive 检查子前缀，pool/set 检查实际目标身份，非法组合要求结构化拒绝且无副作用。 +4. WAIT、PHYSICAL、VERIFY 三者验收；updateParity/readRepair 若候选不支持必须显示明确拒绝，不可悄悄按默认修复算覆盖。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 参数化 recursive、recreate、remove、updateParity、readRepair 与合法/非法 pool/set 组合 +- 仅测试已核实的候选语义；normal/deep 与危险 remove 的副作用明确；API 独有选项不得伪称 rc 已覆盖。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B01: heal.disk-loss-and-repair + +Topologies: H1,H2. Branches: original-drive-return, offline-ack-repair. + +1. 由 layout.json 选一成员盘，记录同版本 N/R/W；使用 DRIVE_GATE 仅让本服务看不到该逻辑盘，不卸载共享 XFS。 +2. 故障窗口经测试进程 mount namespace 的读失败与 Admin offline 状态双重见证；在 quorum 允许范围执行 GET 和唯一 key PUT，逐次记 ACK。 +3. DRIVE_GATE 恢复原盘身份后，等待所选手工/自动来源处理离线期间责任；不得复制备份补盘。 +4. 验证所有故障前对象及成功 ACK 对象、目标盘实际新增分片；节点 ready/GET 成功不是补片证据。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [BASELINE](validation-heal-operations.md#BASELINE) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 让一个已确认 set 成员盘对 heal 服务不可用，持续读取，恢复原盘 +- 记录真实故障窗口和源/目标盘身份；在可恢复条件下最终补齐目标分片及完整 manifest，不只验证降级 GET。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B02: heal.disk-readonly-and-permission + +Topologies: H1,H2. Branches: read-only, permission-denied. + +1. 在专用测试服务的 mount namespace 设置目标根只读；permission 实例用专用非 root 服务 UID 和独立目录 ACL，不能对 root 仅 chmod 后假称生效。 +2. 在服务相同身份/namespace 执行有界写探针，要求 EROFS/EACCES；先保存并隔离目标 part，再触发 deep heal。 +3. 目标不可写期间必须保留失败/待修责任，不能 reported repaired；PHYSICAL 确認没有写到父目录/系统盘。 +4. 解除只读或恢复原 ACL，重新修复并 VERIFY；保留故障期间错误和恢复后物理补片证据。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FAULT](validation-heal-operations.md#FAULT) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 仅对独占测试卷制造只读/拒绝访问，恢复后重试 +- 不可写目标不得 reported repaired；错误及待修责任可见；恢复后真实写入目标且不落到系统盘或其他目录。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B03: heal.disk-replacement-formatted + +Topologies: H1,H2. Branches: new-disk-data, new-disk-versioned. + +1. 完整基线后停止本例服务，将目标逻辑盘 data 整体隔离到本 run 的 quarantine；使用 REPLACE 建空的新 data 根。 +2. 按候选初始化契约启动，读取 format.json 证明新 this UUID 属于原 pool/set/slot；其他盘格式不得变成新集群。 +3. 显式 START root/deep 或本实例指定重建来源，旧盘备份不可读给测试进程；WAIT 全部责任。 +4. 逐对象/版本验证目标盘分片与同角色基线摘要；确认不是从 quarantine 拷贝回去；恢复操作只在取证后做并标为 rescue。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [BASELINE](validation-heal-operations.md#BASELINE) → [REPLACE](validation-heal-operations.md#REPLACE) → [FORMAT](validation-heal-operations.md#FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 换入空的新逻辑盘，确认新 disk UUID/正确格式后触发重建 +- 新盘与旧盘身份可区分且属于正确 pool/set/slot；逐版本重建完成，不能从隔离备份复制回来冒充 heal。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B04: heal.disk-replacement-unformatted + +Topologies: H1,H2. Branches: blank-unformatted, foreign-format. + +1. REPLACE 建没有 format 的空逻辑盘；记录空目录证据，启动并观察候选支持的安全初始化或明确拒绝。 +2. 第二实例使用同 run 的第二个隔离小集群生成外来 format，将其按 FORMAT_BAD 手册装入替换根，保留原始副本和所有 UUID。 +3. 外来身份不能被静默覆盖或挂到错误 set；若拒绝，等待有界 deadline 并确认其他正常盘数据不变。 +4. 停止本服务、撤回替换根、恢复原身份后 VERIFY；空盘正常接纳实例还须逐对象重建证明。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [REPLACE](validation-heal-operations.md#REPLACE) → [FORMAT_BAD](validation-heal-operations.md#FORMAT_BAD) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 引入空未格式化测试盘，并与带异集群格式的盘分别比较 +- 按候选支持的初始化契约安全接纳或明确拒绝；不得覆盖外来身份或将未就绪盘标为可写健康。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B05: heal.multiple-shard-loss + +Topologies: H1,H2. Branches: loss-1-data, loss-1-parity, loss-2-through-m-data-parity. + +1. 从候选/实际元数据确定 M=N-R，枚举 1..M 的合法缺片数及 data/parity 组合，写入本批展开参数表；不得只运行 loss=1。 +2. 每实例用相同版本的新对象，将指定成员上的 part 分别 FAULT missing；多份元数据也要证明指向同 version/data_dir。 +3. 在满足修复提交条件时 START deep/WAIT；先检查全部目标片与角色基线相同，再 VERIFY。 +4. 每次恢复隔离备份后换新对象再做下一组合；低于 R 的组合另由 B07 计负例，不合并成绩。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [BASELINE](validation-heal-operations.md#BASELINE) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 按实际同版本 EC 布局依次缺失 1..M 个 shard，覆盖 data/parity 角色组合 +- 每个参数实例单独满足读取与写入 quorum；满足修复条件时全部目标重建，不把跨版本或跨 set 的 shard 合并。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B06: heal.read-quorum-boundary + +Topologies: H1,H2. Branches: exact-r-read, exact-r-heal-write-limited, restored-w. + +1. 记录读取 R、修复写 W、锁 quorum 的来源；选择同一版本保留恰好 R 个权威片，其他片隔离。 +2. 第一次请求 GET 验证读取边界并记录 read-repair 干扰；用第二个相同形态新对象执行只手工 deep heal。 +3. 如果只有 R 但写/锁 quorum 不足，要求拒绝持久化成功；若拓扑中该组合不可实现，在运行前声明精确原因并选满足条件的拓扑，不伪造离线数。 +4. 恢复到 W 后重新启动修复并保存独立 token，比较精确目标片与全部内容；成功读不能充修复成功。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [GET](validation-heal-operations.md#GET) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [RECOVER](validation-heal-operations.md#RECOVER) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 保留恰好 R 个同版本权威 shard，测试读取及修复提交，再恢复写 quorum +- 读取边界和修复写入边界分别判定；不能因还能 GET 就要求在写 quorum 不足时提交成功。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B07: heal.below-read-quorum + +Topologies: H1,H2. Branches: r-minus-1-unversioned, r-minus-1-historical. + +1. 选定精确 versionId 保留 R-1 份同版本权威；隔离其余 part，确保没有其他 pool 或旧版本兜底被当作同版本来源。 +2. GET 必须返回冻结的不可恢复错误，不能返回 200 错内容或空对象；deep heal 的 execution/disposition 必须如实失败/未解决。 +3. 在恢复前快照所有好片和 xl.meta，证实负例没有扩大损坏或执行 remove；保留失败任务原 token。 +4. 仅在负例取证完成后 RECOVER 原始片，再显式 heal 和 VERIFY；恢复后的绿不能覆盖故障期间的错误。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [GET](validation-heal-operations.md#GET) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 使权威来源小于 R，执行 GET 与 deep heal，再恢复被隔离源 +- 故障期间不得返回伪造的成功内容、空对象或误删记录；明确 unresolved/失败；恢复后原对象和版本完整。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B08: heal.below-write-quorum + +Topologies: H1,H2. Branches: read-present-write-absent, lock-quorum-absent. + +1. 用 layout.json 分别计算对象写和锁 quorum；DRIVE_GATE/NETWORK 构造仍可读但不能安全提交的测试服务可见性。 +2. 执行唯一 key PUT 与已有缺片对象 heal；保存每个 ACK/HTTP/body，超时写必须查询唯一 key 解析结果，禁止直接重试换 key。 +3. 期间不能 reported repaired 或返回无法兑现的成功写 ACK；从各 peer 收集 quorum 与锁失败日志。 +4. 解除故障后按全部已确认/已解析写入重建 manifest，实际补齐目标并 VERIFY；普通低 quorum 错误不是基础设施错误。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FORMAT](validation-heal-operations.md#FORMAT) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [NETWORK](validation-heal-operations.md#NETWORK) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 构造读 quorum 尚存而写/锁 quorum 不足，执行 heal 与新 PUT +- 不能虚报持久化修复或成功写入；分别记录对象、元数据、分布式锁 quorum；恢复后核对所有已确认写入。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B09: heal.node-outage-rejoin + +Topologies: H2. Branches: one-node-stop, rejoin-with-new-versions. + +1. 确认 H2 目标 set 跨节点，统计停一节点实际损失的盘数，不能使用每台独立 pool 的 H5 替代。 +2. PROCESS stop 精确测试 unit，见证 MainPID 消失及端点不可达；保持其他节点服务。 +3. 按计算允许的 quorum 执行降级 PUT/DELETE/GET，保存版本化 ACK；重入同一节点同一数据根。 +4. 观察 MRF/修复责任直到补齐离线期间成功对象；从所有节点 VERIFY 和目标 PHYSICAL，不能只看进程 ready。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [PROCESS](validation-heal-operations.md#PROCESS) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 停止同一跨节点 set 的一个节点，执行允许的降级 I/O，再重入 +- 按实际丢失盘数判定可用性；缺失期间成功提交的对象补齐；节点重新 ready 不等于所有对象已重建。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## B10: heal.disk-and-node-flapping + +Topologies: H2. Branches: drive-flap-3, node-flap-3, interrupt-rebuild-again. + +1. 预注册三个故障周期、每次最长窗口和总 deadline；准备带 nonce 的 ACK ledger，设置只属于本服务的 drive gate 或 unit 操作。 +2. 每轮离线/在线都先见证实际状态；第三轮在 WINDOW 证明部分重建时再次离线，不以固定 sleep 替代。 +3. 记录每轮责任/队列、任务身份、错误率及 CPU/FD；超过预注册重试/资源预算即失败，不继续循环等绿。 +4. 彻底解除故障后验证全部已确认对象、目标分片和持久队列收敛；活动窗口没命中记 NOT_OBSERVED。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [PROCESS](validation-heal-operations.md#PROCESS) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [WINDOW](validation-heal-operations.md#WINDOW) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [METRICS](validation-heal-operations.md#METRICS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 重复短暂离线/在线并在重建途中再次失联 +- 重试与队列有界、身份稳定，无重复任务风暴；最终故障解除后收敛，所有已确认对象保持正确。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C01: heal.missing-part-file + +Topologies: H1,H2. Branches: data-single-part, parity-single-part, multipart-first, multipart-middle, multipart-last. + +1. DB/DM 写入后用 FORMAT/PHYSICAL 将每个 version/part 映射到真实盘角色；每个 variant 选不同目标，不按盘序号猜 data/parity。 +2. 停住注入目标写入后 FAULT missing 只移除已冻结摘要的 part，xl.meta 和 format 保持；恢复运行后禁止目标 GET。 +3. START deep 并 WAIT；首先验证该角色目标 part 恢复到故障前完整 SHA-256/长度，再 GET/Range 和版本清单。 +4. 任何仅 xl.meta 恢复、重建错误 part 或 control 变更都失败；清理时才允许使用原始备份。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [FORMAT](validation-heal-operations.md#FORMAT) → [BASELINE](validation-heal-operations.md#BASELINE) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 仅隔离指定对象某个 part/shard，保留 format 与其他元数据，执行 deep heal +- 对应数据或校验分片重新生成，part/版本身份准确；不得只看到 xl.meta 重现就通过。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C02: heal.data-shard-bitrot + +Topologies: H1,H2. Branches: data-first-byte, data-middle-byte, data-last-byte, multipart-data. + +1. 记录候选解析出的 data shard；用不重复 SHAKE 内容避免数据片/校验片摘要相同，验证健康基线。 +2. FAULT bitflip 在冻结的首/中/尾字节偏移各翻转一位；保存偏移、注入前后 hash 及目标物理角色。 +3. 保持足够正确来源，分别执行本批预注册 deep/read-repair 触发；手工实例不得预读目标。 +4. 目标恢复完整原始片摘要，解码后全文和 Range 正确；只返回正确 GET 但坏片仍在必须失败。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 在指定 data shard 的已知偏移翻转字节，包含首/中/尾位置 +- 故障前后摘要证明注入生效；deep 或 read-repair 发现损坏并修复；独立 bitrot/EC 及 S3 全内容验证通过。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C03: heal.parity-shard-bitrot + +Topologies: H1,H2. Branches: parity-first-byte, parity-middle-byte, parity-last-byte. + +1. 选择经 FORMAT 证明的 parity 片，保留足够 data；普通读取是否需要该片另作控制实例，不能触发后再声称纯 deep。 +2. FAULT bitflip 覆盖首/中/尾位置；注入后仅 PHYSICAL，不 GET/HEAD。 +3. START deep/WAIT，要求 repaired 责任对应正确对象/版本/pool/set；与修复前同角色 SHA-256 比较。 +4. 全文 GET 及版本/metadata 对照通过且错误 parity 已消失才通过；后台提前补片则该触发实例未观测。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 损坏 parity shard，使普通 GET 仍可能成功，再执行 deep heal +- 必须识别并重建损坏 parity；只有 GET 正确、无目标分片修复不能通过。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C04: heal.truncated-and-zeroed-shard + +Topologies: H1,H2. Branches: truncate-half, truncate-near-tail, zero-all, checksum-header. + +1. 保存外部 part 完整备份及正确 metadata quorum；每种损坏使用新的独立对象。 +2. FAULT truncate 默认保留一半；near-tail 按 FAULT_FORMAT 手册只截去最后一字节并记录原始长度；zero 清零全部；header 翻转前 32 字节。 +3. 见证实际长度/摘要差异后 START deep，捕获短读/校验诊断与责任结果。 +4. 目标原始长度和完整片摘要恢复，GET 无截断/补零，控制对象不变；拒绝把一份可解析 metadata 当内容健康。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [BASELINE](validation-heal-operations.md#BASELINE) → [FAULT](validation-heal-operations.md#FAULT) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 分别截断 shard、置零和破坏校验头，保留正确元数据仲裁来源 +- 短读、全零和校验损坏被准确识别；修复不能接受截断内容或错误长度。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C05: heal.missing-or-corrupt-xlmeta + +Topologies: H1,H2. Branches: missing-xlmeta, truncated-xlmeta, corrupt-xlmeta, inline-payload, versioned-xlmeta. + +1. DB 小对象与 DV 历史版本先全量读取，FORMAT 解析每盘 xl.meta 的版本/inline/data_dir 关联并保存解析器源码 SHA。 +2. 在少数成员上对精确对象 xl.meta 做 missing/truncate/header；inline 实例仅由匹配候选的解析器定位 payload 后翻位，不能随便偏移后称 inline。 +3. 保留同版本正确来源，START deep/WAIT；恢复后重新解析 xl.meta，核对所有版本、marker 与 data 引用。 +4. 每个 versionId 全 GET 和版本集合一致；可以有合法重编码，但不能新建空 metadata、遗漏历史版本或移动删除语义。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 在少数成员上移走或损坏 xl.meta/inline 内容，保留足够权威来源 +- 恢复同一对象的全部应存版本、数据引用与删除标记；不以新建空元数据替代真正修复。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C06: heal.stale-version-rejoin + +Topologies: H1,H2. Branches: unversioned-overwrite, delete-historical-version, delete-marker. + +1. 保存 DV/旧代次，DRIVE_GATE 让一个成员离线且其旧数据原样保留。 +2. 在线 quorum 执行新内容覆盖、精确旧 versionId 删除或创建 marker；串行 ACK 确定因果顺序，并在在线端确认提交。 +3. 恢复携带旧数据的同一盘，核对原 disk UUID 后观察手工/自动修复，并保存旧/新代次权威选择日志。 +4. 最终最新值与全部历史/marker 严格匹配 ACK 清单；已删除版本不复活；目标旧盘同一 key 的 metadata 与内容纠正。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [START](validation-heal-operations.md#START) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 盘离线后执行覆盖/版本删除，再让携带旧元数据的盘重入 +- 以已确认的新代次/版本为准；旧当前值与已删版本不复活，旧盘被纠正而非成为新的错误权威。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C07: heal.conflicting-subquorum-authority + +Topologies: H1,H2. Branches: disjoint-version-quorums, disjoint-incarnations, native-authority-fence. + +1. 同一个本例 key 串行写两个不同版本/代次，保存各自成员副本；按 FORMAT_BAD 在静止服务下组合为两个各小于 R 的集合，原片全部保留。 +2. 解析每个成员版本身份，生成按 version/incarnation 分组的计数表；即使总数达到 R，也证明每一组都没有 quorum。 +3. GET/deep heal 必须安全拒绝，不以较新 mtime 或跨版本片并集提交；检查原本仍可恢复的证据没有被 remove。 +4. 运行匹配源代码的 NATIVE 权威仲裁回归；在真机解除混合现场后恢复原夹具并 VERIFY，两类证据分开计数。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [FORMAT_BAD](validation-heal-operations.md#FORMAT_BAD) → [GET](validation-heal-operations.md#GET) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 用受控原生夹具构造互不构成 quorum 的版本/代次集合，再做对应真机反例 +- 禁止把不同版本的 shard 数量并集当成权威；无法证明时安全拒绝且保留可恢复证据，不猜测较新时间戳。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C08: heal.metadata-payload-mismatch + +Topologies: H1,H2. Branches: length-mismatch, payload-mismatch, insufficient-authority. + +1. 先保存版本化外部 part 的全套 metadata 与 hash；从本例另一种长度/内容的版本取得替换 payload。 +2. 停服务按 FAULT_FORMAT 只更换少数 part 的内容或截断，保留原 xl.meta 声称的长度/摘要，不改健康成员。 +3. deep heal 对比 metadata 与实际内容；充分权威实例应重建，不充分权威实例应拒绝且保留好的副本。 +4. 取证后恢复，验证精确版本无跨代内容；保留实际解析结果，不把任意无效 header 注入当作长度字段覆盖。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 构造元数据声称某长度/摘要但实际 part 不匹配的少数副本 +- 元数据与内容联合判断；不把可解析的 xl.meta 当健康；权威不足时拒绝覆盖正确副本。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C09: heal.foreign-format-and-unknown-schema + +Topologies: H1,H2. Branches: foreign-disk, foreign-set, future-format, future-object-schema. + +1. 用第二个隔离测试集群生成真实外来 disk/set format；当前实例只有一个替换逻辑根可被修改。 +2. FORMAT_BAD 在服务停止时分别装入外来 format 或按候选解析器支持的版本头写入未来 schema；记录原始 bytes/digest，不碰共享盘 format。 +3. 启动、GET/deep heal，核对明确隔离/unsupported，不得把未知格式当 missing 初始化或写成新集群。 +4. 停止服务收集原记录仍存在的物理证据；撤回坏根、恢复原身份后完整 VERIFY；无法确认格式字段不能冒充已注入。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FORMAT](validation-heal-operations.md#FORMAT) → [REPLACE](validation-heal-operations.md#REPLACE) → [FORMAT_BAD](validation-heal-operations.md#FORMAT_BAD) → [PROCESS](validation-heal-operations.md#PROCESS) → [START](validation-heal-operations.md#START) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 提供外来 disk/set 身份、未来 schema 或不支持的记录格式 +- 明确隔离/拒绝且不破坏原记录，不把未知当 missing 后直接重建；解除故障后原数据可用。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C10: heal.dangling-and-orphan-safety + +Topologies: H1,H2. Branches: remove-false, remove-true-before-grace, remove-true-after-grace, shared-version-reference, pending-mpu. + +1. 创建 DV/DM 并合法覆盖/删除形成候选孤儿；以匹配格式解析器列出所有 version→data_dir 引用和 pending UploadId。 +2. 只在没有活引用的本次测试目录形成 dangling；保存 orphan 成因与真实年龄，宽限期实例等待配置窗口，不改全局时间。 +3. 分别 START remove=false/true；宽限期前、共享历史引用、pending MPU 必须保留，过期且有充分删除证明的孤儿才允许清理。 +4. 比较物理引用集合，所有应存版本及未完成 upload list_parts 完整；remove 误删任何共享引用即失败，救援只在取证后。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 构造 dangling part、旧 data_dir 与共享历史引用，分别 remove=false/true、宽限期前后检查 +- 默认不得破坏仍可恢复数据；显式 remove 只清除具有充分删除证明且过期的本次孤儿，不删共享版本或 MPU。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## C11: heal.bucket-incarnation-recreation + +Topologies: H1,H2. Branches: old-metadata-rejoin, old-checkpoint-replay, old-mrf-replay, native-incarnation-fence. + +1. 建旧 DV 桶并让一盘离线；按 S3 正常列举/删除全部版本和 marker，确认 delete_bucket 成功，禁止直接内部清空绕过删除。 +2. 同名 create_bucket 后写新内容，保存新 incarnation；恢复旧盘并让旧任务/持久 MRF 重放。 +3. 对比任务身份和代次守卫，旧责任不得删除/覆盖新对象，不得把旧 marker 或数据复活到新桶。 +4. 按 NATIVE 验证精确 incarnation 提交屏障；真机清单及新桶配置始终匹配成功 ACK，旧/新物理范围独立留证。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [S3](validation-heal-operations.md#S3) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [START](validation-heal-operations.md#START) → [FORMAT](validation-heal-operations.md#FORMAT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 盘离线时完成旧桶合法清空/删除，再同名建桶写入新对象，恢复旧盘与旧任务 +- 旧 bucket incarnation 的 checkpoint/MRF/元数据不得污染新桶；旧任务不得删除或恢复新 incarnation 的对象。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D01: heal.object-size-boundaries + +Topologies: H1,H2. Branches: zero, one, 4kib-minus-exact-plus, inline-minus-exact-plus, stripe-minus-exact-plus, large-external. + +1. SEED_DB 显式包含 0、1、4095、4096、4097；从固定候选配置/FORMAT 获得 inline 与 EC block 阈值，新增阈值±1 和一个多 block 外部对象。 +2. 每个大小先按物理记录确认实际形态；零字节/inline 没有外部 part 时改用合法 metadata/inline 故障，不能虚构 part。 +3. data/parity 上依次 missing 与 bitflip，START deep/WAIT；每个形态独立 variant 计数。 +4. 核对全体长度/摘要/ETag/HEAD/LIST；跨边界及末字节 Range 为正确 206，空对象范围按冻结契约拒绝。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [RANGE](validation-heal-operations.md#RANGE) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 覆盖 0/1 字节、页/inline/EC block 边界±1以及大对象 +- 每个实际存储形态均确认后注入；长度、SHA-256、ETag、HEAD/LIST 与边界 Range 正确，无假设阈值。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D02: heal.completed-multipart-object + +Topologies: H1,H2. Branches: first-part, middle-part, last-part, cross-part-range. + +1. SEED_DM 建至少三 part 且非末 part≥5 MiB，非等长内容和独立 nonce；保存 UploadId、每个 part ETag、Complete ACK/最终版本。 +2. 每实例损坏第一、中间、末尾 part 对应的一个物理片；按 FORMAT 映射实际 part，不依赖文件名猜逻辑位置。 +3. START deep/WAIT 后先验证对应目标片，再完整 GET；最终 ETag 应与基线一致，但不用 multipart ETag 计算整对象 MD5。 +4. RANGE 覆盖两个 part 边界及尾部，比较逐字节/摘要、metadata 和版本；其他未完成 MPU 仍可 list_parts。 + +Operations: [SEED_DM](validation-heal-operations.md#SEED_DM) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RANGE](validation-heal-operations.md#RANGE) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 修复已完成的不等长多 part 对象，分别损坏首/中/末 part +- 完整内容、part 顺序、原版本/ETag/元数据和跨 part Range 不变；multipart ETag 不能当整对象 MD5。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D03: heal.historical-versions + +Topologies: H1,H2. Branches: old-version-part, current-version-part, recursive-all-versions, multiple-version-targets. + +1. SEED_DV 每 key 三个不同 payload 版本，逐一保存 versionId/ETag；可额外写 marker，清单不按当前 GET 代替历史读取。 +2. 为不同历史/当前版本选不同盘和 part，metadata 故障实例保留同版本 quorum。 +3. 对 object 和 recursive bucket 分别 START/WAIT；物理解析必须显示所有应存 version/data_dir 而不是只最新对象。 +4. 对每个原 versionId 全 GET，LIST versions 无新增/遗漏，isLatest 身份与 marker 顺序保持。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 不同版本分布不同损坏位置，按对象和递归范围修复 +- 所有原 versionId 可寻址且内容正确，最新版本身份不漂移，无多余版本或遗漏历史版本。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D04: heal.delete-marker-semantics + +Topologies: H1,H2. Branches: current-marker, historical-marker, missing-marker-replica, stale-disk-marker. + +1. 创建三个数据版本与当前/历史 marker，记录 delete_object 返回的 VersionId/DeleteMarker。 +2. 缺少 marker 的实例使用 xl.meta 少数副本缺失；陈旧实例通过成员离线后合法创建/删除 marker 形成，不能直接改任意 metadata。 +3. heal 后 current GET/HEAD 对当前 marker 应为 404 且正确 marker 身份头；指定 marker versionId 的 GET 按 S3 契约为 405 并保留版本头。 +4. LIST versions 精确集合与最新标记一致；显式删除当前 marker 后露出的应是预期历史对象，不能产生额外数据版本。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [S3](validation-heal-operations.md#S3) → [FAULT](validation-heal-operations.md#FAULT) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 缺失/陈旧 marker 副本后 heal，并按当前值与精确 marker versionId 查询 +- 当前 GET/HEAD 与指定 marker 的状态码及身份头符合契约；marker 不复活数据，LIST versions 无重复。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D05: heal.versioning-suspended-null + +Topologies: H1,H2. Branches: enabled-to-suspended, null-overwrite, null-delete-marker. + +1. 启用版本写三代，再 put_bucket_versioning Status=Suspended，连续写两次 null 版本并创建/删除 null marker。 +2. 基线同时保存非 null 历史和当前 null 身份；对目标外部 part 或 xl.meta 制造少数缺失。 +3. START deep/WAIT；记录 null 不能被转换成随机 versionId，也不能与启用期历史版本混片。 +4. VERIFY 全部原 versionId、null 内容和 marker；再次开启版本后写新对象，验证旧历史仍可寻址。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [S3](validation-heal-operations.md#S3) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 在版本开启、暂停、null 版本覆盖之间建夹具再修复 +- null 和非 null 版本不混淆，不新增假版本；暂停前历史对象仍准确可读。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D06: heal.object-metadata-and-tags + +Topologies: H1,H2. Branches: user-metadata, content-headers, versioned-tags. + +1. 用 DS 设置中文值可编码的 metadata、Content-Type、Content-Encoding、Content-Disposition、Cache-Control 与两组 tags；内容本身使用已声明编码。 +2. 先 GET/HEAD/get_object_tagging 精确记录；对外部 part 和少数 xl.meta 分别故障。 +3. START deep/WAIT，比较所有对外属性和逐版本 tags，修复不新增版本、不丢用户 key 大小写规范化结果。 +4. 允许内部 metadata 合法重写，但必须通过 FORMAT 引用和物理内容证明；不把 Content-Encoding 当服务端压缩已启用。 + +Operations: [SEED_DS](validation-heal-operations.md#SEED_DS) → [BASELINE](validation-heal-operations.md#BASELINE) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 包含用户 metadata、Content-Type/Encoding/Disposition、tags 的对象修复 +- 对外 metadata、tags 和相关响应头逐项相同；不要求无关内部元数据字节完全一致。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D07: heal.range-head-and-conditions + +Topologies: H1,H2. Branches: head, range-valid, range-outside, if-match, if-none-match, if-modified-since, if-unmodified-since. + +1. DB/DM 基线保存 ETag/LastModified 和跨 part 内容；先执行所有条件请求，冻结预期 200/206/304/412/416。 +2. 对精确 part 制造缺失/坏校验并 repair；同一组 HEAD、Range、IfMatch/IfNoneMatch、时间条件再次请求。 +3. 304/412/416 验证响应无错误成功内容；合法范围比较准确长度、ContentRange 和原片段摘要。 +4. 完整 VERIFY 版本和 metadata；条件请求必须用 API 参数传递，不由本地短路模拟状态码。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [RANGE](validation-heal-operations.md#RANGE) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [RANGE](validation-heal-operations.md#RANGE) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 修复前后验证 HEAD、合法/越界 Range、If-Match/If-None-Match 及时间条件 +- 200/206/304/412/416 等按逐请求契约判断，绝不返回旧内容、错误范围或错误 ETag。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D08: heal.encrypted-objects + +Topologies: H1,H2. Branches: sse-s3, sse-kms, sse-c, encrypted-versioned. + +1. DEPENDENCY 先验证候选支持的每种 SSE 与专用 KMS/key 就绪；不支持的产品能力按明确错误契约另记，不能删除该参数实例。 +2. 用 S3 ServerSideEncryption/SSEKMSKeyId 或 SSECustomerAlgorithm/SSECustomerKey 创建对象；密钥从环境注入到内存，不放命令行/清单。 +3. 在密文物理层选择 part 做 missing/bitflip；deep heal 后比较同角色原密文片或匹配候选的重编码验证。 +4. 用正确授权逐版本解密 GET 校验原文，用缺少/错误密钥请求验证拒绝；不得写出明文替代或把密钥错误判成不存在。 + +Operations: [DEPENDENCY](validation-heal-operations.md#DEPENDENCY) → [SEED_DE](validation-heal-operations.md#SEED_DE) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [S3](validation-heal-operations.md#S3) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 按候选支持的 SSE-S3/SSE-KMS/SSE-C 建加密对象并修复 +- 授权解密后原内容及版本一致，物理验证使用正确编码/密文层；密钥不可用时不误删、不降级明文，证据脱敏。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D09: heal.object-lock-retention + +Topologies: H1,H2. Branches: governance-retention, compliance-retention, legal-hold, historical-protected-version. + +1. 创建启用 ObjectLock 的专用桶，SEED_DO 使用短且真实未到期的 retention 或 ON legal hold；记录 UTC 截止和独占资源到期安排。 +2. 在无 bypass 权限角色下对受保护 versionId DELETE 必须 AccessDenied，确认保护本来有效。 +3. 对受保护版本缺片/xl.meta 缺失后 repair，核对内容及 get_object_retention/legal_hold；保留期内再次删除仍拒绝。 +4. legal hold 仅正常 API 解除；Compliance 等到真实到期后清理，禁止改时钟或内部删文件；未到期现场标 RETAINED，不能交还空环境。 + +Operations: [SEED_DO](validation-heal-operations.md#SEED_DO) → [S3](validation-heal-operations.md#S3) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [RECOVER](validation-heal-operations.md#RECOVER) + +Source assertions: + +- 对开启 retention/legal hold 的受保护版本修复，期间测试非法删除 +- heal 不绕过保留/hold，也不丢失其元数据；拒绝删除与合法修复均有明确证据。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D10: heal.special-keys-and-pagination + +Topologies: H1,H2. Branches: unicode-space-percent, encoded-slash, adjacent-prefixes, multi-page-list. + +1. DS 创建中文、空格、%2F 字面量、真实 / 和多层前缀键；至少 257 个对象并设置 List MaxKeys=17，确保跨页。 +2. FORMAT 建逻辑 key→物理路径映射，确认 %2F 与 / 不混淆；文件故障只用受限相对路径工具，不拼 shell。 +3. 对选定 key 分别 missing part/xl.meta，使用 URL percent-encode 后的 Admin scope，deep heal。 +4. 分页 LIST/versions 与 manifest 无遗漏/重复；控制前缀不变；对每个特殊 key 精确 GET 和 metadata 校验。 + +Operations: [SEED_DS](validation-heal-operations.md#SEED_DS) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [S3](validation-heal-operations.md#S3) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 覆盖中文、空格、百分号、编码斜线、多层前缀和多页对象列表 +- 逻辑 key 与物理路径映射准确，不跨桶/前缀；分页无遗漏重复，不能把用户 key 当可执行 shell 或裸文件路径。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## D11: heal.bucket-metadata-integrity + +Topologies: H1,H2. Branches: versioning-config, bucket-policy-tags, lock-config, unknown-record-negative. + +1. DS/DV 创建隔离桶配置：版本、只授权本桶的策略、bucket tags、可到期锁配置；通过 S3 GET 配置存基线。 +2. FORMAT_BAD 白名单仅包含该桶对应的系统 bucket metadata 文件；在匹配格式/可恢复契约已核实后少数副本缺失/截断。 +3. 不能使用通用 heal_fault 修改 .rustfs.sys；按专用系统记录手册在停止服务、精确备份和审批清单下操作，未知格式实例只要求安全拒绝。 +4. repair 后逐 API 读取配置比对，业务对象版本完整；确认 IAM/凭据配置及其他桶系统记录完全未变。 + +Operations: [SEED_DS](validation-heal-operations.md#SEED_DS) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [S3](validation-heal-operations.md#S3) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) + +Source assertions: + +- 在隔离桶白名单内测试可支持的桶元数据恢复，并核对外部 bucket 配置 +- 版本设置、策略/标签/保留配置按契约保持；不得遍历凭据配置或把未知内部记录当可重建数据。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## E01: heal.manual-normal-attribution + +Topologies: H1,H2. Branches: normal-missing-data, normal-missing-parity. + +1. 部署 manual 配置：关闭 scanner、auto-heal、MRF；保留 format，记录服务实际环境与 PID，不只保存拟用 env 文件。 +2. FAULT missing 后不能 GET/HEAD 目标；先确认缺失仍在，再 START scanMode=1。 +3. WAIT 精确 token/outcome，先 PHYSICAL 证明目标新生成，再 VERIFY；源归属需结合配置、时间线和任务对象。 +4. 正常扫描未发现静默 bitrot 不能因此判 E01 失败；那是 E02/C02/C03 的 deep 实例。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [CONFIG](validation-heal-operations.md#CONFIG) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 禁止预读目标，保留 format，隔离背景干扰后显式 normal heal +- 证明本次 Admin token 实际覆盖目标并修复可由 normal 检出的缺失；不要求 normal 发现所有静默 bitrot。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## E02: heal.manual-deep-attribution + +Topologies: H1,H2. Branches: deep-parity-bitrot, deep-data-bitrot. + +1. manual 配置排除其他后台来源；FORMAT 确认分片角色，注入 bitflip 并保存坏片仍存在证据。 +2. 不先读目标，START scanMode=2；保留请求 body、clientToken、incarnation/pool/set 与最终对象 disposition。 +3. 先验证坏片完整恢复，再读取原文；如果任务开始前已有恢复，记录 ATTRIBUTION_NOT_OBSERVED。 +4. 检查健康 control 没被意外修改，source 不能仅由最后一次发出的命令推测。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [CONFIG](validation-heal-operations.md#CONFIG) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 在普通读取仍可成功的 parity 损坏上显式 deep heal +- 按 token/对象/版本及物理变更证明 deep 检查和修复，不把后台提前修复算给手工任务。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## E03: heal.read-repair-attribution + +Topologies: H1,H2. Branches: get-missing, range-bitrot, historical-version-read. + +1. 关闭 scanner/auto-heal，按候选 read-repair 实现配置其所需 MRF 队列，记录为何开启；不发送 Admin heal。 +2. FAULT 后物理证明损坏尚存；只对一个精确 key/version 发一次带请求 ID 的 GET 或 Range。 +3. GET 内容先验正确还不够：只观察 PHYSICAL/后台状态/日志，直到该片重建；异步路径需关联 read-repair 入队和责任完成。 +4. 完成补片后才全量 VERIFY；如果只能看到读取成功但找不到 source 证据，该路径仍 NOT_OBSERVED，不能计 PASS。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [CONFIG](validation-heal-operations.md#CONFIG) → [FAULT](validation-heal-operations.md#FAULT) → [GET](validation-heal-operations.md#GET) → [RANGE](validation-heal-operations.md#RANGE) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [LOGS](validation-heal-operations.md#LOGS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 不发 Admin heal，对已确认的特定损坏对象发一次可追踪 GET/Range +- 读取正确且后续真实补片；若走异步队列需证实 source=read-repair，不能只有 GET 成功。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## E04: heal.scanner-cold-object-repair + +Topologies: H1,H2. Branches: cold-missing, cold-bitrot, scanner-cycle-boundary. + +1. 启用 scanner 和所需自动修复，预注册 cycle/start delay/bitrot cycle 与预算；先用健康冷对象证明扫描器实际巡检。 +2. 注入时用 PROCESS 短停以保持静止窗口；重启后不发目标 GET/HEAD、不发 Admin heal。 +3. 只用磁盘快照、scanner 遍历日志和后台责任观察，证明当前周期检查目标，记录发现→入队→补片。 +4. PHYSICAL 完成后再全量 VERIFY；单纯周期计数增长、全局 idle 或其他自动 replacement 先修不能记 scanner 通过。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [CONFIG](validation-heal-operations.md#CONFIG) → [PROCESS](validation-heal-operations.md#PROCESS) → [FAULT](validation-heal-operations.md#FAULT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [LOGS](validation-heal-operations.md#LOGS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 不 GET/HEAD 损坏目标、不发手工 heal，仅等待已启用的 scanner 路径 +- 冷对象被发现并修复，来源和遍历窗口明确；scanner 周期结束但目标没检查不能通过。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## E05: heal.mrf-offline-write-replay + +Topologies: H2. Branches: offline-put, offline-version-delete, offline-marker. + +1. 在 H2 开启 durable MRF，先验证目标节点离线时剩余同 set 满足提交条件；DRIVE_GATE 或 PROCESS 只影响目标成员。 +2. 在线节点执行唯一 key PUT、精确版本 DELETE、marker 操作；逐个保存 ACK 和责任 identity。 +3. 在目标仍离线时采集 journal/状态，证明责任已持久化；恢复同一盘身份，不发手工 heal。 +4. 观察 MRF 重放和退休理由；只有实际 repaired/verified_healthy/authoritatively_absent 后退休；VERIFY 全部 ACK 和目标片，不能以队列空独自证明。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [CONFIG](validation-heal-operations.md#CONFIG) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 成员离线时完成有 quorum 的 PUT/DELETE，确认待修责任后重入 +- durable MRF 责任持久且身份精确；只在 repaired/verified healthy/authoritatively absent 等已证明结果后释放，成功提交不丢失。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## E06: heal.replacement-auto-rebuild + +Topologies: H1,H2. Branches: replacement-unversioned, replacement-versioned-marker. + +1. 启用 replacement auto-heal；REPLACE 注入空逻辑盘，旧盘备份对服务不可达；保存新 UUID/slot。 +2. 重启/接纳后不发 Admin heal、不读目标；仅查询 BACKGROUND/REPLACEMENT、日志及 PHYSICAL。 +3. 从 replacement generation、ready 与责任完成状态关联所有 manifest 对象/版本的补片时间线。 +4. 全部目标分片实际重建后 VERIFY；服务 ready、计数重置或队列空都不能提前结束。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [CONFIG](validation-heal-operations.md#CONFIG) → [REPLACE](validation-heal-operations.md#REPLACE) → [FORMAT](validation-heal-operations.md#FORMAT) → [REPLACEMENT](validation-heal-operations.md#REPLACEMENT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 替换盘接纳后不发 Admin/目标 GET，观察自动 replacement heal +- 替换代次、ready/completion 与盘上全部目标版本一致；不能用服务 ready、队列空或计数重置代替完成。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F01: heal.root-graceful-coordinator-restart + +Topologies: H1,H2. Branches: root-partial-graceful, same-record-disk-resume. + +1. DX root heal 启动前保存原协调者、拓扑、记录盘和预算；所有 root checkpoint 通过 CHECKPOINT 只读白名单采集。 +2. WINDOW 证明 0<已修目标数<总数 且任务 running，再 PROCESS graceful-stop 原协调者；记录退出信号和持久记录。 +3. 用原 unit/env/数据根/候选重启原协调者，不换记录盘；观察原任务 ID/剩余责任恢复，不能将新扫描当原任务续跑。 +4. 最终全部对象/片正确，root 活动记录退休，重复重启无重复任务；未观察部分进度保留 NOT_OBSERVED。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PROCESS](validation-heal-operations.md#PROCESS) → [STATUS](validation-heal-operations.md#STATUS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- root heal 非零进度时优雅重启原协调者，保留原拓扑与记录盘 +- 在明确支持的边界内恢复原 task ID 和剩余责任/预算，完成后无遗留活动记录；不扩张为协调盘丢失后的故障转移。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F02: heal.target-graceful-restart + +Topologies: H2. Branches: target-partial-graceful, versioned-target-partial. + +1. REPLACE 目标逻辑盘并启用自动重建，使用 DX/DV 使目标总量可观测；协调者不是重启目标。 +2. PHYSICAL 与 REPLACEMENT 双证实目标只完成部分，再 PROCESS 对目标 unit 优雅停止；保存 MainPID、退出码、前后 boot_id。 +3. 保留同一新 disk UUID/代次，正常启动原目标服务；不能换新目录重测。 +4. 观察余下目标继续构建且任务责任一致，全部版本/marker/片完整；备份不得参与重建。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [REPLACE](validation-heal-operations.md#REPLACE) → [REPLACEMENT](validation-heal-operations.md#REPLACEMENT) → [WINDOW](validation-heal-operations.md#WINDOW) → [PROCESS](validation-heal-operations.md#PROCESS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 替换目标仅完成部分重建时优雅重启目标节点 +- 故障确实发生于部分完成窗口；重启后续建同一目标盘/代次，最终 manifest 与全部目标分片正确。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F03: heal.target-process-crash + +Topologies: H2. Branches: target-partial-sigkill, target-partial-sigkill-versioned. + +1. 与 F02 同样准备替换盘和可观察部分进度，先冻结将被杀掉的 unit MainPID/进程启动时间。 +2. WINDOW 内 PROCESS kill，精确 systemctl kill --signal=SIGKILL --kill-whom=main；记录 signal=9 与非优雅退出，明确不是物理断电。 +3. 正常启动同 unit/根/新 disk UUID；记录恢复 responsibility/代次，不允许丢弃尚未完成对象。 +4. 全量 VERIFY/PHYSICAL，检查临时文件和持久责任无泄漏；最终通过不抹掉这次真实中断的证据。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [REPLACE](validation-heal-operations.md#REPLACE) → [WINDOW](validation-heal-operations.md#WINDOW) → [PROCESS](validation-heal-operations.md#PROCESS) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 部分重建时 SIGKILL 测试进程，再正常启动 +- unclean shutdown 与修复进度有真实证据；已提交对象不丢失，未完成目标不会被当完整；明确不是断电测试。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F04: heal.coordinator-process-crash + +Topologies: H2. Branches: root-coordinator-kill, mrf-coordinator-kill, replacement-coordinator-kill. + +1. 三个机制使用独立场景和冻结的持久化支持表；在协调者非零且未完成进度时采集 CHECKPOINT。 +2. PROCESS SIGKILL 协调者，目标节点保持；恢复原根/原拓扑。 +3. 分别核对机制承诺：持久任务必须续责；易失 token 丢失不能算已完成，应显示未完成并允许明确的新扫描补偿。 +4. 原任务与补偿任务分别保存；最终 VERIFY 所有 ACK、版本和片，不能把安全重扫成功写成原 token 自动恢复。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [SEED_DB](validation-heal-operations.md#SEED_DB) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PROCESS](validation-heal-operations.md#PROCESS) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 协调者非优雅退出并恢复，分别观察 root、MRF、replacement +- 按各机制已核实的持久化边界判断；不得把丢失的易失 token 当成功，需显式记录剩余责任并可安全重扫恢复。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F05: heal.full-cluster-restart + +Topologies: H2. Branches: all-graceful-restart, all-restart-with-mpu, repeat-full-restart. + +1. 准备 DB/DV/DM、pending UploadId 与未完成持久修复责任；冻结全部节点/盘/slot。 +2. WINDOW 命中后按固定顺序只停止本例全部 units；保存每节点退出码、checkpoint/MRF 与完整时序。 +3. 按原拓扑同时或预注册顺序重启，等待全部 peer，不能用自动重新格式化或新数据根恢复。 +4. 完成后全版本/marker/MPU 和物理片校验，再重复一次重启验证幂等，任何丢失的已承诺责任计失败。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [PROCESS](validation-heal-operations.md#PROCESS) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 有未完成修复责任时重启全部测试节点，固定原拓扑/磁盘身份 +- 有持久化保证的责任不遗失；全部端点恢复一致，旧版本/marker/MPU 与目标分片完整，重复重启不扩散错误。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F06: heal.cancelled-task-restart + +Topologies: H1,H2. Branches: cancel-restart, old-token-after-restart, force-new-after-cancel. + +1. DX 缺片任务在 WINDOW 中 STOP；STATUS 与磁盘静默窗口证明取消和在途退出完成。 +2. 保存旧 token/checkpoint，PROCESS 优雅重启；确认已取消任务不会自行复活或报 repaired。 +3. 新 START/forceStart 获取独立 token，向旧 token 发 stop/status；不得影响新任务。 +4. 新任务完成后 VERIFY/PHYSICAL；取消记录/新任务责任不串代。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [STOP](validation-heal-operations.md#STOP) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PROCESS](validation-heal-operations.md#PROCESS) → [STATUS](validation-heal-operations.md#STATUS) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) + +Source assertions: + +- 完成取消及在途退出后重启，随后 forceStart/新任务正常修复 +- 已取消的任务不自动复活；旧 checkpoint/token 不能取消或覆盖新任务，旧责任不会误标为修复完成。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F07: heal.checkpoint-and-mrf-corruption + +Topologies: H1,H2. Branches: checkpoint-truncate, mrf-checksum, future-schema, stale-generation, journal-write-failure, native-persist-fence. + +1. 先制造并证明真实 durable root/MRF 责任，CHECKPOINT 定位精确本 run task/record，其他系统记录禁止触碰。 +2. 停止测试服务，按 FAULT_FORMAT 对这些已审核记录分别截断、坏校验、未来 schema、旧代次；写失败实例用独占限额故障层，不塞满真实共享盘。 +3. 重启后核对解析/错误状态：未知或损坏记录保留、可诊断，不能静默删除并 ACK；责任仍能安全重扫处理。 +4. 精确 fsync/commit 崩溃窗口执行 NATIVE 回归，保存屏障命中证据；真机记录损坏与原生临界点不要混算。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [START](validation-heal-operations.md#START) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PROCESS](validation-heal-operations.md#PROCESS) → [FAULT_FORMAT](validation-heal-operations.md#FAULT_FORMAT) → [IO_FAULT](validation-heal-operations.md#IO_FAULT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 受控构造 checkpoint/MRF 截断、坏校验、未来 schema、陈旧代次及写入失败 +- 已确认持久化责任不得静默删除或伪 ACK；未知记录保留且可诊断；精确崩溃窗口由原生屏障证明，不伪称真机命中。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F08: heal.mixed-version-upgrade-rollback + +Topologies: H2. Branches: rolling-old-new, mixed-peer-status, rollback-new-old, mixed-restart. + +1. 固定两个 release optimized 包的 commit/tree/binary/格式兼容说明；UPGRADE 先验证支持回滚的记录版本，不能使用任意旧包。 +2. 有持久修复责任时逐节点升级，保留数据根和 unit；混合阶段通过各 endpoint STATUS/BACKGROUND 并持续唯一版本写入。 +3. 缺少能力的 peer 必须显示 partial/unsupported，不能 aggregate 成 complete；新记录不被老节点误退休。 +4. 按兼容方案回滚并重启，验证所有 ACK 和责任；不兼容格式只测明确拒绝/保护，禁止冒险降级覆盖。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [START](validation-heal-operations.md#START) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [UPGRADE](validation-heal-operations.md#UPGRADE) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [STATUS](validation-heal-operations.md#STATUS) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 固定前后两个明确兼容候选，覆盖滚动升级、混合 peer、回滚与重启 +- 能力不足必须显示 partial/unsupported 而非完整成功；按格式兼容契约保留责任，旧版本不能误退休新版本记录。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## F09: heal.dedicated-host-forced-restart + +Topologies: H6. Branches: dedicated-vm-forced-stop, dedicated-host-power-cycle. + +1. 先取得 H6 专项资源 ID、独占证明和带外恢复批准；普通 fixed fleet 禁止执行本例。 +2. 持续限量写入并在 replacement 部分重建窗口，记录已成功 ACK/结果未知写集合与最近持久化证据。 +3. HOST_POWER 使用批准的平台命令强制停启，保留平台活动 ID、时间线和 boot_id；VM stop 与真实电源断电分别命名实例。 +4. 恢复后按原盘/根重启并解析未知写；全版本/片和持久责任验收，不把 SIGKILL 或软件 reboot 当作此项证据。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [REPLACE](validation-heal-operations.md#REPLACE) → [WINDOW](validation-heal-operations.md#WINDOW) → [HOST_POWER](validation-heal-operations.md#HOST_POWER) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 经专项授权在部分修复/持续写入时强制重启独占测试 VM/主机 +- 保留平台事件与 ACK ledger；验证落盘和恢复，不把 SIGKILL 当断电；VM 强制停止也不等同于物理控制器断电。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G01: heal.race-unversioned-overwrite + +Topologies: H1,H2. Branches: overwrite-vs-deep, overwrite-vs-read-repair, native-commit-cleanup. + +1. 同 key 串行 nonce 的覆盖客户端与 heal 客户端独立；WORKLOAD 每次记录 request-start/end、ACK、payload hash，不凭跨机墙钟判最后版本。 +2. 旧对象缺片/bitflip 后两端同时运行，WINDOW 证明实际交叠；每个 worker 的同 key PUT 必须串行，避免期望值本身无序。 +3. 停止写入后解析全部未知 ACK，按最后可证明提交核对最新 GET 与 data_dir；不能回滚旧内容。 +4. 运行 NATIVE commit/cleanup 屏障实例；真实竞争没命中精确窗口只算黑盒实例，记录 no-lock/锁重验日志。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FAULT](validation-heal-operations.md#FAULT) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 同 key 连续覆盖与 heal 同时推进，包含提交尾部清理窗口 +- 按可证明的提交顺序验证最终最后一次成功写入；不能回滚到旧 data_dir，未确认超时写入单独解析。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G02: heal.race-versioned-put + +Topologies: H1,H2. Branches: new-version-vs-historical-heal, read-all-versions-during-heal, native-version-fence. + +1. DV 历史版本缺片时，前台串行写入独立 payload 的新版本，保存每个成功 versionId。 +2. START deep 与 WORKLOAD 交叠；读取端始终指定 versionId 验证新旧内容，禁止用 current GET 代替。 +3. 静默后按完整 ACK ledger 重建期望版本集合，最新值根据串行提交顺序判断；全量 PHYSICAL 检查不同 data_dir 无混片。 +4. 精确版本提交屏障运行 NATIVE；任何未解析超时写/缺失 ACK 阻止最终通过。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 修复历史版本时创建新版本并重复读取各 versionId +- 新旧版本不混片、不串内容；已返回成功的每个版本都保留，最新值按确定的先后关系验收。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G03: heal.race-version-delete + +Topologies: H1,H2. Branches: delete-old-version-vs-deep, delete-current-version-vs-deep, native-delete-fence. + +1. 选 DV 的精确 versionId 制造可恢复缺片；前台删除请求明确带该 ID，不是只创建 marker。 +2. WINDOW 内并发 delete_object 与 deep heal，保存 DELETE 成功回执和 heal scope/identity。 +3. 终态按精确 version GET 应不存在，LIST versions 不得复活；其他所有版本内容保持。 +4. 运行 NATIVE 删除/修复提交交错点；清理旧片仅作为最终物理证据，不能先用备份复活已删版本。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [WINDOW](validation-heal-operations.md#WINDOW) → [S3](validation-heal-operations.md#S3) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 对损坏的精确历史 versionId 发 DELETE，与 deep heal 竞争 +- 已确认删除的版本不能被复活，其他历史版本不受影响；原生临界点测试与真机竞争分别留证。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G04: heal.race-markers-and-batch-delete + +Topologies: H1,H2. Branches: create-marker, delete-marker, batch-version-delete, unversioned-delete, native-marker-fence. + +1. 分别建立版本化和非版本化桶；marker创建/删除、批量带versionId删除和无版本DELETE分开运行，不混淆接口语义。 +2. 在 xl.meta 少数缺失下启动 heal，同时提交冻结的删除列表；delete_objects 必须逐项检查 Deleted/Errors，HTTP200不代表全成功。 +3. 按逐项 ACK 计算最终数据版本、marker 和最新值集合；精确 GET验证不存在项和存留项。 +4. native-marker-fence 独立取证；恢复故障不得把已确认删除的旧 metadata 覆盖回来。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [SEED_DB](validation-heal-operations.md#SEED_DB) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [S3](validation-heal-operations.md#S3) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 与创建/删除 marker、批量版本删除和无版本删除分别交错 +- 按逐项 DELETE 回执核对版本/marker 集合，无复活、误删或遗漏批量错误。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G05: heal.race-multipart-complete-abort + +Topologies: H1,H2. Branches: upload-continue, complete, abort, native-mpu-fence. + +1. DM 同时准备已提交对象与两个可识别的 pending UploadId，各自 part/ETag 独立。 +2. 已提交对象缺片并 heal；并发给 upload-A 续传并 Complete，upload-B Abort，WINDOW 保存实际交叠。 +3. Complete 返回版本只出现一次，完整内容为按PartNumber排序的ACK part；Abort 的 UploadId 不再可list_parts，不能误伤另一个upload或已提交对象。 +4. 核对所有 MPU 列表和物理目录；原生提交/取消屏障另用 NATIVE，不允许仅发出并发请求即声称命中。 + +Operations: [SEED_DM](validation-heal-operations.md#SEED_DM) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [WINDOW](validation-heal-operations.md#WINDOW) → [S3](validation-heal-operations.md#S3) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- heal 期间对不同已知 UploadId 分别续传、Complete、Abort +- 成功 Complete 内容只出现一次；Abort 不破坏已提交对象或别的 upload；未完成 part 不被伪造为完整对象。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G06: heal.race-copy-and-metadata-update + +Topologies: H1,H2. Branches: copy-source, copy-destination, replace-metadata, update-tags, native-copy-fence. + +1. DS 源/目标使用不同内容和 metadata，copy 请求 CopySource 使用结构化 Bucket/Key/VersionId，不拼路径字符串。 +2. 源或目标缺片并 heal，同窗口 COPY、MetadataDirective=REPLACE 的自拷贝、put_object_tagging 分别运行。 +3. 保存对应提交ETag/versionId及更新顺序，验证最终内容与该提交 metadata/tags 组合一致；旧 read-repair 必须重验代次。 +4. 检查源目标和control均无交叉覆盖；NATIVE 记录锁升级/陈旧提交拒绝的精确窗口。 + +Operations: [SEED_DS](validation-heal-operations.md#SEED_DS) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [WINDOW](validation-heal-operations.md#WINDOW) → [S3](validation-heal-operations.md#S3) → [VERIFY](validation-heal-operations.md#VERIFY) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- COPY 源/目标和标签/元数据更新与修复交错 +- 最终对象内容与对应提交的 metadata 一致，读修复发现新代次应重验/放弃陈旧提交而非覆盖。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G07: heal.race-ilm-expiry-and-transition + +Topologies: H2. Branches: current-expiry, noncurrent-expiry, transition, restore, native-ilm-fence. + +1. 用现有 ILM 专用远端 tier 能力建立本例桶和规则；记录真实到期窗口/远端version映射，不在共享桶上改规则。 +2. 在实际出现 Expire/Transition/Restore 进度时启动缺片 heal，WINDOW 必须从日志/状态证明交叠；提前完成只能记 NOT_OBSERVED。 +3. 过期 ACK/ILM 删除证明决定哪些版本应不存在，禁止 heal 复活；迁移对象通过真实远端 GET/restore 验证原版本内容。 +4. 本地物理与远端引用双重校验；源端 GET 成功不足，清理只删除本例规则/远端数据。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [SEED_DE](validation-heal-operations.md#SEED_DE) → [DEPENDENCY](validation-heal-operations.md#DEPENDENCY) → [ILM](validation-heal-operations.md#ILM) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [S3](validation-heal-operations.md#S3) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 分别与当前/历史版本过期、远端 transition/restore 同窗口执行 +- 已授权过期不被恢复；迁移状态与远端版本引用正确；只有实际观察到交叠才计并发覆盖，tier 未配置需明确阻断对应实例。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## G08: heal.race-replication-resync + +Topologies: H2. Branches: replicate-put, replicate-delete, resync, native-replication-fence. + +1. 准备两个隔离复制端点/本例桶、明确源目标版本映射和同步规则；先跑一份健康复制控制。 +2. 源端缺片 heal 同时提交 PUT/DELETE 并启动指定 resync；保存两侧日志与跨端版本映射，证明操作时间交叠。 +3. 分别从两端全量列举版本、marker、内容并比对复制契约；源端正确但远端缺失/无限重试必须失败。 +4. 恢复临时规则/凭据，确认无无限循环/重复传播；精确交错仅由匹配 NATIVE 证据补充。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [DEPENDENCY](validation-heal-operations.md#DEPENDENCY) → [REPLICATION](validation-heal-operations.md#REPLICATION) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [WINDOW](validation-heal-operations.md#WINDOW) → [VERIFY](validation-heal-operations.md#VERIFY) → [LOGS](validation-heal-operations.md#LOGS) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 在隔离复制两端与 heal 同时运行新增/删除/重同步 +- 对端版本、删除语义及源数据最终一致，无无限回环或重复传播；不得依赖仅源端 GET 宣称复制正确。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H01: heal.with-expansion-rebalance + +Topologies: H5. Branches: expand-while-data-heal, expand-while-parity-heal, versioned-rebalance. + +1. 使用已批准 H5，先在源 pool 写 DB/DV 并以 FORMAT/PHYSICAL 证明位置；新 pool 为空且未纳入旧选择器。 +2. 目标 data/parity 缺失或bitflip后按 POOL 手册扩容重启，确认新拓扑；启动 rebalance 与 heal。 +3. WINDOW 保存两任务活动证据，逐对象跟踪源/目标 pool/set/版本身份，不能只看全局数量。 +4. 两任务均正常完成后物理证明目标权威副本，VERIFY全版本，保留scanner backlog的持久提交/退休证据。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [POOL](validation-heal-operations.md#POOL) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 先证明源 pool 持有夹具，再扩容并与修复同时 rebalance +- 修复目标不得指向错误 pool/set；新旧数据完整、移动终态正确，保留 scanner backlog 提交权威检查。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H02: heal.with-rebalance-stop-resume + +Topologies: H5. Branches: rebalance-stop, rebalance-resume, rebalance-service-restart. + +1. H5 中使用足够对象使移动可观测，缺片 heal 与 rebalance 同时活动；记录两类独立 task ID。 +2. 只停止 rebalance，观察 heal 不被误取消；然后恢复 rebalance，重启实例仅重启本例 unit。 +3. 持续唯一 key 写入并保存 ACK；停止/恢复前后物理和任务状态要连续，不用新数据根替代恢复。 +4. 最终两类任务各自满足完成契约、对象/版本及移动目标正确，重复停止不影响新代次。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [POOL](validation-heal-operations.md#POOL) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [PROCESS](validation-heal-operations.md#PROCESS) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- heal 活动期间 stop/restart/resume rebalance +- 两类任务状态和身份独立；停止/恢复移动不误终止修复或丢失对象，恢复后两者均满足各自契约。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H03: heal.with-decommission + +Topologies: H5. Branches: heal-decommission-source, heal-survivor-target, versioned-multipart. + +1. 先证实夹具位于待退役源，另写 survivor control；分别在源和目标制造在quorum内可恢复故障。 +2. 按既有 Pool 行为启动退役与 heal；退役期间按当前与 MinIO 一致的写入限制检查拒绝，不能把拒绝写当数据丢失。 +3. 源数据只能在目标提交权威后删除；记录移动 ACK 与 target physical，而不是临时副本数量。 +4. 退役完成后停止/移除源，只访问 survivor 全版本/MPU/内容；若还依赖源兜底必失败。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [POOL](validation-heal-operations.md#POOL) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 分别在待退役源和存活目标制造可恢复缺片，再执行退役 +- 不得提前删源或以临时副本冒充迁移成功；按 Pool 既定写入限制验收；只剩 survivor 后仍完整可读且有物理证明。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H04: heal.with-decommission-cancel-clear + +Topologies: H5. Branches: cancel-during-heal, clear-after-inflight-exit, resume-new-generation, capacity-lock-retry-window. + +1. 在移动和 heal 都有部分进度时取消退役；保存取消持久化记录、通知和在途退出顺序。 +2. 确认已退出后 Clear，再恢复/新建退役；旧 token/代次不得误取消新任务，持续 ACK ledger 不能丢。 +3. 容量锁超时实例需要 LOCK_WINDOW 或匹配 NATIVE 屏障，实际见证 #2423 所涉重试路径；普通取消成功不能代替。 +4. 两流程收敛后 VERIFY/PHYSICAL；没有命中锁窗口单列 NOT_OBSERVED，不声称覆盖 #2423 的临界点。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [POOL](validation-heal-operations.md#POOL) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [LOCK_WINDOW](validation-heal-operations.md#LOCK_WINDOW) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- heal/移动争用中取消退役、Clear、恢复，并覆盖容量锁超时恢复 +- 保持持久化后通知及在途退出顺序；不误取消新代次；对 #2423 的重试窗口单独注明真机是否实际触发，不只看最终绿色。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H05: heal.after-retired-pool-removal + +Topologies: H5. Branches: survivor-missing-part, survivor-missing-metadata, retired-source-offline. + +1. 依既有 Pool 全量迁移并完成退休配置移除，停止源服务，证明入口仅绑定 survivor；保留源不可达证据。 +2. 只在 survivor 的选定对象/版本制造缺片或metadata缺失；START root或正确pool/set deep。 +3. 记录 source authority 选择，不能向已退休 pool 写入，也不能无权威覆盖/错误回退（#2422）。 +4. 检查survivor实际补片、全部版本/marker和源根未变化；在源持续停机情况下全量 VERIFY。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [POOL](validation-heal-operations.md#POOL) → [PROCESS](validation-heal-operations.md#PROCESS) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [LOGS](validation-heal-operations.md#LOGS) + +Source assertions: + +- 源 pool 完全退役移除并停机后，只在 survivor 上触发修复 +- 不再向已移除源写入或依赖其兜底；来源权威和版本正确；#2422 路径不能出现无权威覆盖或错误回退。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H06: heal.multi-pool-target-isolation + +Topologies: H5. Branches: selected-pool, selected-set, unselected-control-fault. + +1. 在两个以上 pool/set分别放不同对象，用实际layout确认归属；健康control与控制故障分开登记。 +2. 只选一个pool/set START；其他pool保留缺片控制，不先GET它们。 +3. PHYSICAL先确认选中目标补齐，未选故障原样保留；任务、容量、配额与drive身份都指向正确目标。 +4. 取证后才恢复控制故障和VERIFY；对象在其他pool可读不能证明所选盘完成修复。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 仅破坏选定 pool/set，并在另一 pool 保留不同对象与控制故障 +- 选择器、任务、容量/配额和目标 drive 一致；未选 pool 不受错误修复，跨 pool 存在不等于该目标已修复。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## H07: heal.ec84-multi-set-restart + +Topologies: H3,H4. Branches: ec84-one-set-restart, ec84-one-set-crash, ec84-two-sets-restart, ec84-two-sets-crash. + +1. 分别绑定 H3 12盘和 H4 至少24盘，FORMAT必须证实每set为8data+4parity；同一物理设备多个目录不能算独立故障盘。 +2. 每个set都准备DB/DV/DX并证明对象归属，REPLACE选定目标，在目标仅部分重建时优雅重启或SIGKILL。 +3. 保存实际中断窗口、盘代次、每set已修/未修对象，不能跨set合并片数。 +4. 每个set单独完成物理与全版本校验；四盘pilot不进入该项分子，native EC8+4回归作为独立补证。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [REPLACE](validation-heal-operations.md#REPLACE) → [WINDOW](validation-heal-operations.md#WINDOW) → [PROCESS](validation-heal-operations.md#PROCESS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 固定 EC8+4 单 set 与至少两 set，部分重建时重启/崩溃目标 +- 每个 set 的故障覆盖和重建分别证明，不跨 set 拼 shard；必须满足真实盘数及映射，不能用 4 盘用例替代 EC8+4。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## I01: heal.slow-io-and-eio + +Topologies: H1,H2. Branches: bounded-latency, short-read, eio. + +1. IO_FAULT 只在预分配专用故障设备或受限faultfs，冻结延迟/错误率、IO范围、最长窗口和原状恢复命令。 +2. 在服务相同namespace/UID先证明迟延/短读/EIO真实生效，再启动目标缺片heal；不得让dd填满共享盘。 +3. 错误窗口记录重试、CPU、队列和物理片，不能提前reported healthy；暂态与永久错误分开记录。 +4. 撤销故障后重新观察责任完成并VERIFY；故障层证据缺失算INFRA_ERROR，不把模拟EIO说成真实介质损坏。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DX](validation-heal-operations.md#SEED_DX) → [IO_FAULT](validation-heal-operations.md#IO_FAULT) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [METRICS](validation-heal-operations.md#METRICS) → [RECOVER](validation-heal-operations.md#RECOVER) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 在独占故障层注入有界延迟、短读、EIO，解除后观察恢复 +- 暂态与永久错误分开，重试有界且无忙循环；解除故障后实际修复，期间不得提前标健康。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## I02: heal.enospc-inodes-and-quota + +Topologies: H1,H2. Branches: bounded-volume-enospc, bounded-inode-exhaustion, bucket-quota. + +1. IO_FAULT使用有容量上限的独立loop/fault卷，预注册字节/inode限制及本例桶配额，不改变共享XFS全局配额。 +2. 先证明服务实际写返回ENOSPC或quota错误；准备缺片目标和DM写入，再触发heal/PUT，逐次保留ACK和错误。 +3. 不可写期间不得损坏健康来源、虚报补片或释放仍需容量；元数据/part双重快照。 +4. 释放本例占用或提高已批准限额后续修，VERIFY全内容和目标片；共享盘可用空间保持安全余量。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DM](validation-heal-operations.md#SEED_DM) → [IO_FAULT](validation-heal-operations.md#IO_FAULT) → [QUOTA](validation-heal-operations.md#QUOTA) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 限制专用测试卷空间/inode，并参数化本次桶配额 +- 仅填专用受限卷、不耗尽共享真实盘；失败不破坏好副本或释放尚需容量，释放空间后正确续修。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## I03: heal.network-partition-and-lock-quorum + +Topologies: H2. Branches: minority-partition, read-boundary, write-boundary, lock-boundary, rejoin. + +1. H2 先冻结实际节点/slot/对象R/W和锁quorum，NETWORK规则只匹配本例服务端口并带run ID、超时恢复。 +2. 构造少数派及边界分区，双端探针证实隔离；SSH、跳板机和其他服务端口保持连通。 +3. 双方执行带nonce的PUT/GET/heal，少数派不能承诺不安全提交；BACKGROUND必须暴露missing peer/partial coverage。 +4. 解除规则并检查规则计数归零，按ACK因果对账全版本/内容及目标片，无分叉、旧代覆盖或丢失责任。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [FORMAT](validation-heal-operations.md#FORMAT) → [NETWORK](validation-heal-operations.md#NETWORK) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [METRICS](validation-heal-operations.md#METRICS) → [RECOVER](validation-heal-operations.md#RECOVER) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 仅隔离测试服务节点间流量，覆盖少数派、读/写/锁 quorum 边界及恢复 +- 少数派不得承诺无安全性的修复或写入；cluster coverage 不完整不能显示全局健康；恢复无分叉、跨代覆盖。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## I04: heal.queue-pressure-and-admission + +Topologies: H1,H2. Branches: full-queue, dropped-admission, merged-duplicates, cross-bucket-fairness. + +1. 专用服务冻结队列容量、并发、最大请求数和时限；DX与多桶夹具通过WORKLOAD施加有上限的Admin/read-repair请求。 +2. 分别记录Full/Dropped/Merged的真实返回及任务ID，重复token/合并不按独立修复数计分。 +3. 持久责任不能因入队失败被退休；记录journal、队列深度、等待时延和资源曲线，超过预算立即停注入。 +4. 停止负载后在deadline内收敛，所有可修目标真实补齐；不能只以CLI exit0或全局idle通过。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [CONFIG](validation-heal-operations.md#CONFIG) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [METRICS](validation-heal-operations.md#METRICS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 多来源请求冲击有限队列，混合重复任务、不同优先级和多个桶 +- Full/Dropped/Merged 语义与实际任务一致；丢弃不能丢 durable 责任；在预先声明负载窗口内无无限堆积或健康误报。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## I05: heal.lock-timeout-and-caller-abort + +Topologies: H1,H2. Branches: object-lock-timeout, caller-disconnect, stale-read-repair, native-lock-owner-fence. + +1. LOCK_WINDOW用本例慢IO/受控原生锁屏障持有目标对象锁，先证明真实争用超过预注册预算。 +2. 同时heal/覆盖或删除；调用方断开实例仅断本次TCP连接，不杀服务器或关闭跳板机；记录请求开始/断开/服务在途结束。 +3. 锁释放后观察取消/超时不提前释放所有权、不重复提交，陈旧read-repair必须重验版本。 +4. 核对所有ACK和最终代次，检查锁/FD无泄漏；黑盒不能证明锁所有权时以NATIVE窗口独立补证，不能猜命中。 + +Operations: [SEED_DV](validation-heal-operations.md#SEED_DV) → [IO_FAULT](validation-heal-operations.md#IO_FAULT) → [LOCK_WINDOW](validation-heal-operations.md#LOCK_WINDOW) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [START](validation-heal-operations.md#START) → [WINDOW](validation-heal-operations.md#WINDOW) → [METRICS](validation-heal-operations.md#METRICS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [NATIVE](validation-heal-operations.md#NATIVE) + +Source assertions: + +- 真实对象锁争用超过预算及调用方断开，再释放锁恢复 +- 取消/超时不得提前释放在途所有权或重复提交；陈旧 read-repair 必须重验代次；恢复后无泄漏和错误覆盖。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## I06: heal.kms-and-tier-outage + +Topologies: H2. Branches: kms-unavailable, kms-denied, tier-unavailable, tier-version-unavailable. + +1. 先在专用KMS/tier租户验证DE/DV夹具可正常解密/restore，保存非敏感key/tier版本标识。 +2. DEPENDENCY仅断本测试客户端/服务到该依赖的专用namespace路由，或撤销本例租户权限；不全局停共享KMS。 +3. 发GET/restore/deep heal并观察责任，依赖错误不能判对象不存在、可remove、已修好；密钥不进入错误日志。 +4. 恢复依赖和权限后原版本可解密/恢复，物理/责任完整；远端对象不存在与远端暂不可达分别判定。 + +Operations: [SEED_DE](validation-heal-operations.md#SEED_DE) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [DEPENDENCY](validation-heal-operations.md#DEPENDENCY) → [NETWORK](validation-heal-operations.md#NETWORK) → [S3](validation-heal-operations.md#S3) → [START](validation-heal-operations.md#START) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 仅让已分配测试 KMS/远端 tier 暂不可用，持续查询与修复 +- 依赖故障不得解释为对象不存在/可删/已修复，不暴露密钥；恢复后原版本可解密或恢复，责任未丢失。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## J01: heal.status-partial-disabled-unknown + +Topologies: H1,H2. Branches: enabled-idle, disabled, uninitialized, missing-peer, recovered-complete. + +1. 在独立服务配置分别观察 enabled无任务、HEAL禁用、初始化未完成，保留实际启动配置及时间线。 +2. H2 PROCESS停一个peer并跨节点查询BACKGROUND/REPLACEMENT；完整coverage必须含expected/responded/unknown明细。 +3. disabled/uninitialized/partial/degraded不能被rc或聚合器解释成全局完成；收集rc原输出与raw Admin逐字段对照。 +4. 恢复所有peer后才允许完整idle，依然用PHYSICAL/manifest核对无遗留责任；未观测初始化窗口单独记NOT_OBSERVED。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [CONFIG](validation-heal-operations.md#CONFIG) → [PROCESS](validation-heal-operations.md#PROCESS) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [REPLACEMENT](validation-heal-operations.md#REPLACEMENT) → [RC_STATUS](validation-heal-operations.md#RC_STATUS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 分别检查 enabled/disabled/uninitialized、全体可达及缺失 peer 的状态 +- idle 只在覆盖完整且真实无活动时成立；partial/unknown/degraded/disabled 不被客户端转换为全部完成。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## J02: heal.outcome-counters-and-error-truth + +Topologies: H1,H2. Branches: mixed-healthy-repaired-failed, dry-run, deferred, unknown-negative. + +1. 同批独立key建立健康、可恢复缺片、低于R的不可恢复对象；deferred由冻结的锁/依赖故障产生，dry-run另建任务。 +2. 保存逐对象物理基线和期望disposition，对结果的execution/coverage/counters/object identities做逐项对账。 +3. unknown/skipped/completed_with_errors不算修复成功，legacy objectsHealed与canonical healed分开解释；attemptFailures不得被隐藏。 +4. 使用独立完整manifest弥补有界outcome窗口，保持不可修对象的正确失败记录；恢复后另任务修复不覆盖原结果。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [LOCK_WINDOW](validation-heal-operations.md#LOCK_WINDOW) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 同批次混合健康、可修、不可修、deferred、dry-run 对象 +- execution/coverage/disposition/计数和逐对象物理结果相符；unknown、skipped 或 completed_with_errors 不能算修复成功。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## J03: heal.incremental-cursor-and-truncation + +Topologies: H1,H2. Branches: within-window-cursor, past-retention-gap, cross-node-cursor, terminal-truncation. + +1. DX至少257个目标，超过128诊断窗口；边启动recursive heal边以sinceSeq查询，每次请求/nextSeq/minSeq/items全部保存。 +2. 单端连续读取应无重复倒退；故意使用过旧cursor应明示truncated/gap；跨节点查询按任务/epoch而非全局数字比较。 +3. 任务终态也读取旧cursor，验证有界列表不被当完整清单；所有错误与缺口原样进入报告。 +4. 最终用全量独立S3/physical manifest与canonical counters验收覆盖；若任务太快仍可做终态截断，但不能声称观察了运行中游标。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [CURSOR](validation-heal-operations.md#CURSOR) → [STATUS](validation-heal-operations.md#STATUS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 生成超过结果保留上限的任务，分页轮询 sinceSeq，并跨节点/终态读取 +- nextSeq/minSeq 单调且 gap/truncation 明示；有界 items 不是完整责任清单，最终须用独立 manifest/计数补证，不静默丢错误。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## J04: heal.repeat-idempotency + +Topologies: H1,H2. Branches: normal-repeat, deep-repeat, repeat-after-restart, versioned-repeat. + +1. DB/DV缺片先修复；versioned 的整份 xl.meta 缺失允许所有 identity repaired，或至少一个 repaired 加其余正健康态；marker 只能为 repaired/metadata_healthy，counters.healed 与 repaired 数量一致、counters.unchanged 与其余数量一致，unknown/skipped/failed=0。完整保存 post-heal S3/physical 作为第二轮基线，不使用最初故障前 xl.meta 要求内部字节完全相同。 +2. 再次 normal/deep：normal 只接受 metadata_healthy，deep 才要求 verified_healthy；两者均 healed=0、unknown/skipped/failed=0，不得新增 repaired，所有原版本/marker 身份与数量一致。 +3. PROCESS原unit重启后再重复；检查新任务有独立token但不产生新对象版本、无限责任或重复修复计数。 +4. 允许经格式证据证明的内部维护记录变化，用户内容/引用不变，目标片摘要稳定。 + +Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) → [PROCESS](validation-heal-operations.md#PROCESS) + +Source assertions: + +- 第一次修复后重复 normal/deep、重启再检查 +- versioned 首次允许所有 identity repaired 或至少一个 repaired 加其余正健康态，marker 仅 repaired/metadata_healthy，计数必须与 receipt 集合一致；normal repeat 接受 `metadata_healthy`，deep repeat 要求 `verified_healthy`，repeat marker 为 `metadata_healthy` 且 healed=0；unknown/skipped/failed 不得成为 PASS。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## J05: heal.long-soak + +Topologies: H2,H5. Branches: 24h, 72h. + +1. 只在专门分配的H2/H5独占lane，冻结完整24h或72h墙钟窗口、循环故障日程、并发/限速/存储上限和到期回收安排。 +2. WORKLOAD持续记录逐请求ACK，周期性故障仅使用本run DRIVE_GATE/PROCESS/依赖规则；每轮见证与恢复，不无限加数据。 +3. 持续METRICS、日志、队列/MRF/checkpoint取样；资源增长、panic、未解释责任达到预注册阈值立即保留失败证据。 +4. 到期停止写入/注入，解除全部故障，解析未知ACK后全量VERIFY和PHYSICAL；短跑不能充24h/72h结果。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [PROCESS](validation-heal-operations.md#PROCESS) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [CHECKPOINT](validation-heal-operations.md#CHECKPOINT) → [METRICS](validation-heal-operations.md#METRICS) → [LOGS](validation-heal-operations.md#LOGS) → [RECOVER](validation-heal-operations.md#RECOVER) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 独占环境运行预注册 24h/72h 窗口，混合修复、掉线、写入和版本删除 +- 每次 ACK 可追踪；最终全量 manifest 正确，队列/内存/FD/临时空间无无界增长，无后台 panic 或未解释持久责任。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 + + + +## J06: heal.performance-abba + +Topologies: H2. Branches: abba-repeat-1, abba-repeat-2, abba-repeat-3. + +1. 同硬件/候选/rc预注册预热、每轮时长、并发、读写比例、数据量/缓存策略、P95/P99与错误率阈值；禁止结果出来再定门槛。 +2. 按A无heal、B有heal、B有heal、A无heal顺序，每轮从相同逻辑内容和队列基线开始；完成至少三组独立ABBA。 +3. WORKLOAD记录每请求延迟/成功/错误/超时，METRICS记录CPU/IO/FD及实际修复字节；错误不从总请求分母删除。 +4. 报告每轮及配对差异、修复吞吐和GET/PUT各自P95/P99；功能VERIFY必须通过，性能阈值也必须独立满足。 + +Operations: [SEED_DX](validation-heal-operations.md#SEED_DX) → [PERFORMANCE](validation-heal-operations.md#PERFORMANCE) → [WORKLOAD](validation-heal-operations.md#WORKLOAD) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [METRICS](validation-heal-operations.md#METRICS) → [LOGS](validation-heal-operations.md#LOGS) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) + +Source assertions: + +- 同硬件同候选按 ABBA 运行无 heal 基线和有 heal 负载，重复并固定限速 +- 分别报告修复吞吐、GET/PUT P95/P99、错误率和 CPU/IO；阈值事先冻结，不能事后挑轮次或把功能 PASS 充性能验收。 + +先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。 + +逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。 diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md new file mode 100644 index 00000000..df96c542 --- /dev/null +++ b/docs/validation-heal-contract.md @@ -0,0 +1,304 @@ +# Heal 真机验证设计与验收契约 + +## 状态与范围 + +本规范保留源仓库的 84 个 Heal canonical 用例、301 个操作分支及 532 个适用拓扑分支模板。资源要求、公共通过判据和逐项附加断言必须一起满足。 + +入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 + +`heal.admin-healthy-noop` 已接入自动分发，覆盖 H1/H2 的 normal/deep；其余操作卡保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 + +H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 + +目标不是只证明“坏盘后仍能 GET”，而是分别证明： + +1. 故障真实发生在指定对象、版本、set 和目标盘。 +2. 客户可见控制接口及期望触发源处理了这份修复责任。 +3. 可恢复时确实修好了目标分片；不可恢复时明确失败且不扩大损坏。 +4. 对象、历史版本、删除语义、并发成功写入和持久化责任正确。 +5. 原始证据完整，临时故障被解除，环境可安全交还。 + +## 版本与已核实的产品边界 + +接口参考来自源设计记录的 RustFS 与 rc 提交，保留下面的源码链接用于追溯。每批执行仍以当前 candidate manifest 的精确源码、包、已安装二进制摘要和 auto-testing/rc 摘要为准，不由历史版本的验证结果推断本次候选已通过。 + +| 已核实的接口/行为 | 对设计的影响 | +| --- | --- | +| 签名 Admin POST `/rustfs/admin/v3/heal/`、`/{bucket}`、`/{bucket}/{prefix}`，token 查询与 forceStop/forceStart | 启动回执仅表示受理；必须查询任务并结合内容与物理证据验收。 | +| POST `/rustfs/admin/v3/background-heal/status`，GET `/rustfs/admin/v4/heal/replacement-recovery` | 后台状态、完整 peer coverage、replacement readiness 与具体任务终态分开判断。 | +| outcome 含 execution、coverage、counters、逐对象 disposition；objects/items 是有界诊断窗口 | finished、队列为 0、HTTP 200 都不是“所有对象已修好”的充分条件。 | +| query 支持 sinceSeq；结果含 nextSeq/minSeq/truncated | 需要游标、截断和缺口负例；被截断的 items 不能用作完整清单。 | +| Admin 保留 nolock wire 字段，但实际强制走正常对象锁路径 | A09 验证其不能绕过锁；不沿用历史架构文档中允许 Admin 绕锁的旧描述。 | +| root heal 优雅重启恢复限于原协调者、原拓扑且记录盘仍可读 | 不预设跨协调者故障转移、记录盘丢失恢复或任意降级兼容；MRF/replacement/root 各按各自责任契约验收。 | +| 参考 rc 提供 heal start/status/stop 及 bucket/prefix/token、normal/deep、dry-run、remove/recreate | pool/set、forceStart、sinceSeq 等未在该 CLI 版本暴露的功能通过签名 Admin 测，不虚构 rc 参数；新批次先检查实际 help/schema。 | + +源码：[Admin heal](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/rustfs/src/admin/handlers/heal.rs)、[HealOpts](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/crates/heal-contracts/src/heal_channel.rs)、[outcome/责任回执](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/crates/heal/src/heal/outcome.rs)、[rc 命令](https://github.com/rustfs/cli/blob/204eb5ad85b46a007afd4da538cbeb1cadd867b6/crates/cli/src/commands/admin/heal.rs)。 + +## 拓扑与故障域 + +下表是**需求模型，不是已经分配的可用机器清单**。所有真机动作必须经跳板机和固定 fleet lease，不能私自创建/删除 Azure VM、磁盘或网络。已有 example profile 不是运行许可；实现时必须绑定节点、端口、服务、mount UUID、盘数、pool/set/slot 与独占范围，读回实际 EC 元数据确认布局。 + +| 编号 | 最低形状/用途 | 当前边界 | +| --- | --- | --- | +| H1 | 单节点四独立盘、一个 EC set；基础盘修复 | 可参考 single-4disk；使用独立 heal 服务和每盘 heal-validation 根。需要先绑定空闲资源，不复用现有 Pool 数据目录。 | +| H2 | 四节点各四盘、同一分布式 pool，明确固定 set 划分；跨节点故障/锁/状态 | 可参考 cluster-1pool；必须证明目标 set 跨节点，不能只看全局节点总数。 | +| H3 | 精确 EC8+4 单 set，参考主仓库的 3 节点×4 盘测试形状 | 是测试参数，不是生产部署推荐；旧两节点 Pool 环境不满足此拓扑。 | +| H4 | 精确 EC8+4 至少两个 set，参考 3 节点×8 盘、同 pool | 至少 24 个独立盘端点及正确 set 映射；不得用 4 盘或一个 set 替代。固定资源未分配时为阻塞，不是通过。 | +| H5 | 两个以上 pool；分开登记两节点各自四盘的生命周期形状和跨节点 pool 形状 | 现 lifecycle-a 每台一个独立 pool，无节点级冗余；可验证盘级修复及移动交互，不能证明节点故障容错。对齐主仓库多 pool EC8+4 基准时，另登记其三单机×12 盘/三个 pool 布局，不混用计数。 | +| H6 | 独占 VM/主机与专用故障盘，可做强制重启 | 需要额外运维授权、带外恢复和事件证据；普通故障 lane 不允许自动进入。 | + +生产拓扑取舍可参考 [RustFS Linux 部署说明](https://docs.rustfs.com/en/installation/linux)：单机多盘的盘冗余不等于整机故障冗余。本设计的结论只对实际验证的拓扑成立。 + +每个对象/版本预先记录 N、data/parity、分片索引与 set/盘映射，再从候选实现与元数据确定读取 quorum R、修复写 quorum W 和锁 quorum。测试 R-1/R/R+1 与 W-1/W/W+1 的**合法组合**，不以全局在线盘数或固定“坏一半仍能写”代替计算。跨版本、跨 set 的片不能相加；R 与 W 不一定相同。N=1 无 EC 的能力/不可恢复负例属于额外参数实例，不得宣传可自动找回完全丢失的数据。 + +## 故障后端与安全门禁 + +所有注入必须有 machine-readable fault-plan 和 fault-receipt；不存在“命令成功即注入成功”。默认每次只隔离本用例一个已解析目标，再按预注册组合扩展。使用以下分类，报告必须记录实际 faultBackend，不能互相冒充。 + +| 编号 | 故障类别 | 注入有效性与限制 | +| --- | --- | --- | +| F0 | 无故障、纯接口请求 | 记录稳定基线；仅用于无副作用/负请求验证。 | +| F1 | 指定 part/shard 缺失 | 先校验精确路径和摘要，以同盘隔离备份/原子移位保存，再证明目标缺失；不是物理掉盘。 | +| F2 | 指定字节翻转、截断、校验头损坏 | 在隔离测试对象副本上操作，保存偏移、长度、前后摘要及原始备份；必须有足够未损坏权威。 | +| F3 | 指定 xl.meta/inline/身份/桶元数据缺失或损坏 | 需要与候选格式匹配的只读解析器和允许列表；不得操作 IAM/凭据配置、共享 format 或无关对象。 | +| F4 | 原逻辑盘/服务卷短暂不可达、只读 | 采用仅影响测试服务的受限挂载命名空间或故障层，并实际验证访问失败。不得卸载含其他服务数据的共享 XFS mount；chmod 不能未经验证当作有效故障。 | +| F5 | 慢 I/O、EIO、短读、ENOSPC/inode 故障 | 只在预分配独占测试设备或有容量上限的故障层；不得塞满共享数据盘。明确模拟故障与真实设备故障的区别。 | +| F6 | 测试进程优雅停止、SIGKILL、重启 | 限定已验证服务和 MainPID，证明在部分进度窗口发生；记录 boot/process 身份。SIGKILL 不是断电。 | +| F7 | 节点间测试端口网络故障 | 仅对本服务地址/端口、受控 namespace 或独占环境，保护 SSH/跳板机/管理平面；规则带唯一 ID、超时和幂等恢复。 | +| F8 | 空替换逻辑盘、新 disk UUID | 验证原/新盘和 pool/set/slot；不把移动回备份当重建。真实设备 detach/attach 另需独占资源授权，不修改共享盘。 | +| F9 | checkpoint/commit/锁窗口的确定性故障 | 优先原生 runtime-fence/E2E 屏障；Release 黑盒无法证明精确窗口时，单独列原生证据与真机覆盖缺口，禁止给优化包塞入隐藏测试 hook 冒充同一候选。 | +| F10 | 独占 VM/主机强制重启 | 单独审批的破坏性 lane；有带外恢复和平台时间线。VM forced stop 不证明物理控制器缓存断电语义。 | +| F11 | 队列/请求压力、测试 KMS/tier/配额依赖故障 | 只作用于已分配测试租户/进程，限量限时，禁止全局断开共享 KMS 或远端服务。 | + +fault-plan 至少包含 run/case/variant ID、拓扑指纹、对象/版本/incarnation、pool/set/slot、盘 UUID、精确文件允许列表、设备/根目录身份、注入前条件、变更预算、备份摘要、恢复动作和自动解除故障截止时间。路径按目录描述符解析，拒绝链接逃逸、设备漂移、未知文件和自由 shell 插值。 + +故障解除与数据清理是两件事：网络/只读/进程等临时环境损伤要安全解除；失败数据、分片、checkpoint 和证据应保留，环境标记 DIRTY/QUARANTINED，不能继续下一例。捕获证据后可按已批准流程恢复业务可达性，但不覆盖失败记录或把恢复重跑算首次通过。备份只可用于失败后的救援/重置，不参与修复成功判定。 + +## 数据夹具与参数维度 + +| 编号 | 内容 | +| --- | --- | +| DB | 确定性基础 manifest：默认 128 个对象，包含 0/1、4095/4096/4097 字节以及实际 inline、EC stripe/block、multipart 边界±1；按磁盘证据确认形态，不硬编码内部阈值。 | +| DM | 已完成的多 part 对象、非等长 part、跨 part Range；另外创建可追踪 UploadId 的 pending/Complete/Abort 夹具，遵守 S3 最小非末 part 大小限制。 | +| DV | 每 key 至少三个不同内容版本、当前/历史 delete marker、versioning suspended/null、明确删除回执；记录 versionId、Last-Modified、ETag 和版本列表。 | +| DS | 多桶、相邻前缀、多页 key、中文/空格/百分号等编码，metadata/tags/content headers 与隔离桶配置。 | +| DE | 候选明确支持且依赖就绪的 SSE-S3/KMS/C、远端 tier 版本；密钥只由受控凭据渠道提供，禁止进入日志/manifest。 | +| DO | object-lock、retention 和 legal hold 的受保护版本；使用可到期测试策略，防止成功用例因无法清理永久占用环境。 | +| DL | 并发 PUT/GET/HEAD/LIST/COPY/DELETE、版本及 MPU 操作的逐请求 ACK ledger；每 key 用递增 nonce 与确定性内容，保存请求/响应和不确定结果。 | +| DX | 保证可观察部分进度、大于状态窗口（>128 项）的夹具；默认上限在运行计划中冻结，长稳/性能单独配额，不靠无限加数据等到偶然命中。 | + +按 case/variant 参数化：normal/deep、data/parity shard、inline/外置 part、单片/多片、原盘/新盘、对象/前缀/root/set、版本/加密、协调者/目标节点、单节点/多节点/多 set/pool。使用预先冻结的 pairwise 组合控制规模；损坏+quorum、旧代次+并发删除、目标重建+进程中断等高风险组合必须显式三维覆盖，不由 pairwise 替代。 + +候选声明支持的非默认 parity/storage class、服务端压缩 on/off 也应加入 DB/DM/DV 参数；Content-Encoding 响应头不是服务端压缩已被执行的证据。每个实例都从实际元数据确认编码形态。Object Lock 的拒绝删除必须发生于真实保留期内；清理不能绕过 Compliance、修改主机时间或以内部删文件规避限制。Legal hold 仅由本次授权测试角色正常解除；保留期超出 lane 窗口的实例必须预先分配独占 stateful 环境及到期处理方案，未清理环境不能交还 READY。 + +重试不能隐藏 ACK：PUT/Complete 超时属于结果未知，必须按唯一 key/version/UploadId 查询解析；不能把未收到成功的一次写当丢失，也不能把重试新版本当同一次成功。并发时只根据可证明的 happens-before/提交回执判断“最后一次”，不凭跨节点墙钟排序。 + +## 公共通过判据（每例均适用） + +每行用例的完整规范 = 本节 + 对应拓扑/夹具/故障契约 + 行内操作与附加断言。参数实例独立留证、计分。 + +1. **预检和基线。** 全部预期 peer/磁盘、candidate/rc/harness 身份一致；预先 GET/HEAD/LIST 完整基线并保存必要的盘上记录。记录扫描/自动 heal 配置与任务归属。 +2. **注入和见证。** 以只读物理证据确认故障确实发生，未影响其他对象/服务；注入后不要先 GET 目标而意外触发 read-repair。受理过快或已被其他来源修好，不能冒充本例触发成功。 +3. **控制面真相。** 收集每次 rc exitCode、签名 Admin HTTP/body、token、source、scope、outcome 和 peer coverage。区分 repaired、verified_healthy、metadata_healthy、authoritatively_absent、deferred、failed、unknown、cancelled、dry_run_observed；deferred、failed、unknown、cancelled、dry_run_observed 不能作为“修复完成”。`metadata_healthy` 只证明 metadata/identity 健康，不证明 payload integrity；Normal scan 不得冒充 `verified_healthy`，deep payload 证明继续要求 `verified_healthy`。实际 wire 枚举以固定候选为准。 +4. **对象正确。** 所有必须存在的对象/版本全量 GET 验证长度/SHA-256/ETag/metadata/tags；HEAD、分页 LIST、历史版本与 marker 语义、已确认删除和并发 ACK 全部核对。允许修复产生合法内部元数据变化，不要求整个 xl.meta 字节不变。 +5. **实际补片。** 在静默校验窗口解析目标物理数据与元数据，核对版本/代次/data_dir、part 长度、bitrot 校验与 EC 一致性。对 data/parity 各有独立重建或重新编码对照，或用包含目标片的固定合法解码集合佐证；不能只统计文件、看 mtime 或依赖 GET 自动纠删码兜底。密文/压缩存储层不能直接拿明文摘要比较。格式不支持/HighwayHash 未验证时必须标明缺口，不能声称已完成深度物理验收。 +6. **持久化与幂等。** 需重启的用例验证重启后责任和数据；重复 heal 不引入新版本、复活删除或无限增加任务。状态计数按进程/任务/epoch 区分，不能把重置后的全局计数简单判为回退。 +7. **完成条件。** 正向修复要有相关责任完成、范围 coverage 完整、对象与物理证据一致。无关的全局 idle、列表项被截断、缺失 peer、unknown/残留待修工作都不能代替证明。预期不可恢复/权限/取消/dry-run 用例按预注册负向契约通过，明确它不是修复成功。 +8. **恢复和证据。** 解除临时故障，保存 before/fault/after/restart 四阶段证据，校验全局结果/JUnit/计划/对象数量/证据引用及清理。任何取证缺失、目标误命中、恢复失败或结果结构损坏均阻断该 lane。 + +针对精确时间窗口，必须观察活动/部分进度或受控屏障，再注入故障；普通重建允许快速完成但必须有实际补片证据。禁止用固定 sleep 或“重跑直到绿”代替见证。建议起始测试预算：接口就绪 90 秒、普通修复 15 分钟、重启恢复 30 分钟、同 epoch 静默窗口 5 秒；这些是待校准的**测试预算，不是产品 SLO**，所有最终预算须在运行前冻结。预期暂态错误只按明确 code 有界重试，每次都留证。 + +判定类型：产品安全/内容/终态错误为 fail；环境注入/取证工具失败为 infra_error；缺少尚未分配拓扑或辅助能力在部署前阻断。不得把产品 SlowDown 或低 quorum 下的预期错误泛化成 infra_error。必选能力缺失、未知选项、缺失参数实例不能用 unsupported/skipped 充 PASS；不适用项只能在运行前的 applicability manifest 中说明，环境尚未准备好不是“不适用”。 + +## 84 个基础用例 + +编号用于设计评审；`heal.*` 为稳定 catalog ID。topology、fixture、fault 中逗号表示必须展开记录的适用实例/后端集合，不表示同时向全部对象注入所有故障。对应的组合、先后和最小 quorum 必须在编译后的 variant plan 中明确，未证明的一项不得计为通过。 + +| 分类 | 设计条目数 | +| --- | ---: | +| A 控制面、任务与权限契约 | 10 | +| B 磁盘故障、替换与 quorum 边界 | 10 | +| C 分片损坏、元数据与权威选择 | 11 | +| D 对象类型与 S3 语义 | 11 | +| E 触发路径与修复责任 | 6 | +| F 中断、持久化、恢复与升级 | 9 | +| G 与前台和后台操作并发 | 8 | +| H 多 Pool、移动与多 set 隔离 | 7 | +| I 资源、依赖故障与调度 | 6 | +| J 可观测性、幂等、长稳与性能 | 6 | +| 合计 | 84 | + + + +### A. 控制面、任务与权限契约 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| A01 · `heal.admin-healthy-noop` | P0 | H1,H2 / DB / F0 | 对健康对象分别执行 normal/deep，再重复查询 | 不修改用户数据或版本；Normal 只接受 `metadata_healthy`，Deep 才要求 `verified_healthy`，healthy 与 repaired 分开计数，不能把检查盘数当修复盘数。 | +| A02 · `heal.admin-scope-isolation` | P0 | H1,H2 / DB,DS / F1 | 分别以 root、bucket、prefix、object、pool/set 选择器启动 | 目标范围实际遍历；相邻前缀、其他桶及未选 set 的故障保持原状；按对象清单而非目录数量验收。 | +| A03 · `heal.admin-token-cross-node-status` | P0 | H2 / DB / F1 | 从节点甲启动，经乙及稳定入口按原 token 查询 | 任务身份、scope、终态及核心计数有界收敛；状态缺失/未知不能解释为完成。 | +| A04 · `heal.admin-dry-run` | P0 | H1,H2 / DB,DV / F1,F2 | 对缺片与损坏对象执行 dryRun，参数化 remove/recreate | 有明确 dry_run_observed 诊断；用户对象分片、xl.meta、版本与删除标记不变；后台修复干扰须排除。 | +| A05 · `heal.admin-overlap-and-force-start` | P1 | H1,H2 / DB,DS / F1 | 活动中重发同范围、父子前缀和不相交请求，再 forceStart | 按已固定 overlap policy 验证合并返回原 token 或明确冲突；forceStart 不能泄漏旧任务责任或并发破坏对象。 | +| A06 · `heal.admin-stop-and-stale-token` | P0 | H1,H2 / DB / F1 | 非零修复进度时 stop，稳定后新建任务，再发旧 token 的查询/停止 | 停止需等在途写结束并稳定；新任务身份独立，旧请求不得终止新任务；重复 stop 按公开契约处理。 | +| A07 · `heal.admin-invalid-request` | P0 | H1,H2 / DB / F0 | 发送非法 selector、重复 query、非法布尔/scanMode、冲突 flags、超长 body | 服务端返回已核实的结构化错误；无误启动、范围扩大或持久化副作用；CLI 参数错误不能替代服务端负例。 | +| A08 · `heal.admin-authorization` | P0 | H1,H2 / DB / F0 | 匿名、坏签名、过期签名、无 HealAdminAction 用户与最小授权用户分别请求 | 未授权 start/status/stop 均不能越权或泄漏对象信息；授权用户可完成规定操作，凭据不得写入证据。 | +| A09 · `heal.admin-nolock-safety` | P0 | H1,H2 / DL,DV / F1,F9 | 通过签名 API 传 nolock=true，并与同对象提交制造确定性争用 | 兼容字段不得跳过存储层对象锁；最后成功写入和已确认删除保持正确，原生屏障与真机竞争证据分开。 | +| A10 · `heal.admin-option-contract` | P1 | H1,H2 / DB,DV / F1,F2 | 参数化 recursive、recreate、remove、updateParity、readRepair 与合法/非法 pool/set 组合 | 仅测试已核实的候选语义；normal/deep 与危险 remove 的副作用明确；API 独有选项不得伪称 rc 已覆盖。 | + + + +### B. 磁盘故障、替换与 quorum 边界 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| B01 · `heal.disk-loss-and-repair` | P0 | H1,H2 / DB / F4 | 让一个已确认 set 成员盘对 heal 服务不可用，持续读取，恢复原盘 | 记录真实故障窗口和源/目标盘身份；在可恢复条件下最终补齐目标分片及完整 manifest，不只验证降级 GET。 | +| B02 · `heal.disk-readonly-and-permission` | P1 | H1,H2 / DB / F4,F5 | 仅对独占测试卷制造只读/拒绝访问，恢复后重试 | 不可写目标不得 reported repaired；错误及待修责任可见；恢复后真实写入目标且不落到系统盘或其他目录。 | +| B03 · `heal.disk-replacement-formatted` | P0 | H1,H2 / DB,DV / F8 | 换入空的新逻辑盘，确认新 disk UUID/正确格式后触发重建 | 新盘与旧盘身份可区分且属于正确 pool/set/slot；逐版本重建完成，不能从隔离备份复制回来冒充 heal。 | +| B04 · `heal.disk-replacement-unformatted` | P1 | H1,H2 / DB / F8 | 引入空未格式化测试盘，并与带异集群格式的盘分别比较 | 按候选支持的初始化契约安全接纳或明确拒绝；不得覆盖外来身份或将未就绪盘标为可写健康。 | +| B05 · `heal.multiple-shard-loss` | P0 | H1,H2 / DB,DV / F1 | 按实际同版本 EC 布局依次缺失 1..M 个 shard，覆盖 data/parity 角色组合 | 每个参数实例单独满足读取与写入 quorum；满足修复条件时全部目标重建，不把跨版本或跨 set 的 shard 合并。 | +| B06 · `heal.read-quorum-boundary` | P0 | H1,H2 / DB / F1 | 保留恰好 R 个同版本权威 shard，测试读取及修复提交，再恢复写 quorum | 读取边界和修复写入边界分别判定；不能因还能 GET 就要求在写 quorum 不足时提交成功。 | +| B07 · `heal.below-read-quorum` | P0 | H1,H2 / DB,DV / F1 | 使权威来源小于 R，执行 GET 与 deep heal，再恢复被隔离源 | 故障期间不得返回伪造的成功内容、空对象或误删记录；明确 unresolved/失败；恢复后原对象和版本完整。 | +| B08 · `heal.below-write-quorum` | P0 | H1,H2 / DB,DL / F4 | 构造读 quorum 尚存而写/锁 quorum 不足，执行 heal 与新 PUT | 不能虚报持久化修复或成功写入；分别记录对象、元数据、分布式锁 quorum；恢复后核对所有已确认写入。 | +| B09 · `heal.node-outage-rejoin` | P0 | H2 / DB,DV,DL / F6 | 停止同一跨节点 set 的一个节点，执行允许的降级 I/O，再重入 | 按实际丢失盘数判定可用性；缺失期间成功提交的对象补齐；节点重新 ready 不等于所有对象已重建。 | +| B10 · `heal.disk-and-node-flapping` | P1 | H2 / DB,DL / F4,F6 | 重复短暂离线/在线并在重建途中再次失联 | 重试与队列有界、身份稳定，无重复任务风暴；最终故障解除后收敛，所有已确认对象保持正确。 | + + + +### C. 分片损坏、元数据与权威选择 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| C01 · `heal.missing-part-file` | P0 | H1,H2 / DB,DM / F1 | 仅隔离指定对象某个 part/shard，保留 format 与其他元数据，执行 deep heal | 对应数据或校验分片重新生成，part/版本身份准确；不得只看到 xl.meta 重现就通过。 | +| C02 · `heal.data-shard-bitrot` | P0 | H1,H2 / DB,DM / F2 | 在指定 data shard 的已知偏移翻转字节，包含首/中/尾位置 | 故障前后摘要证明注入生效；deep 或 read-repair 发现损坏并修复；独立 bitrot/EC 及 S3 全内容验证通过。 | +| C03 · `heal.parity-shard-bitrot` | P0 | H1,H2 / DB,DM / F2 | 损坏 parity shard，使普通 GET 仍可能成功，再执行 deep heal | 必须识别并重建损坏 parity；只有 GET 正确、无目标分片修复不能通过。 | +| C04 · `heal.truncated-and-zeroed-shard` | P0 | H1,H2 / DB,DM / F2 | 分别截断 shard、置零和破坏校验头，保留正确元数据仲裁来源 | 短读、全零和校验损坏被准确识别；修复不能接受截断内容或错误长度。 | +| C05 · `heal.missing-or-corrupt-xlmeta` | P0 | H1,H2 / DB,DV / F3 | 在少数成员上移走或损坏 xl.meta/inline 内容，保留足够权威来源 | 恢复同一对象的全部应存版本、数据引用与删除标记；不以新建空元数据替代真正修复。 | +| C06 · `heal.stale-version-rejoin` | P0 | H1,H2 / DV,DL / F4 | 盘离线后执行覆盖/版本删除，再让携带旧元数据的盘重入 | 以已确认的新代次/版本为准；旧当前值与已删版本不复活，旧盘被纠正而非成为新的错误权威。 | +| C07 · `heal.conflicting-subquorum-authority` | P0 | H1,H2 / DV / F3,F9 | 用受控原生夹具构造互不构成 quorum 的版本/代次集合，再做对应真机反例 | 禁止把不同版本的 shard 数量并集当成权威；无法证明时安全拒绝且保留可恢复证据，不猜测较新时间戳。 | +| C08 · `heal.metadata-payload-mismatch` | P1 | H1,H2 / DB,DV / F2,F3 | 构造元数据声称某长度/摘要但实际 part 不匹配的少数副本 | 元数据与内容联合判断；不把可解析的 xl.meta 当健康；权威不足时拒绝覆盖正确副本。 | +| C09 · `heal.foreign-format-and-unknown-schema` | P0 | H1,H2 / DB / F3,F8 | 提供外来 disk/set 身份、未来 schema 或不支持的记录格式 | 明确隔离/拒绝且不破坏原记录，不把未知当 missing 后直接重建；解除故障后原数据可用。 | +| C10 · `heal.dangling-and-orphan-safety` | P1 | H1,H2 / DV,DM / F1,F3 | 构造 dangling part、旧 data_dir 与共享历史引用，分别 remove=false/true、宽限期前后检查 | 默认不得破坏仍可恢复数据；显式 remove 只清除具有充分删除证明且过期的本次孤儿，不删共享版本或 MPU。 | +| C11 · `heal.bucket-incarnation-recreation` | P0 | H1,H2 / DV / F4,F9 | 盘离线时完成旧桶合法清空/删除，再同名建桶写入新对象，恢复旧盘与旧任务 | 旧 bucket incarnation 的 checkpoint/MRF/元数据不得污染新桶；旧任务不得删除或恢复新 incarnation 的对象。 | + + + +### D. 对象类型与 S3 语义 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| D01 · `heal.object-size-boundaries` | P0 | H1,H2 / DB / F1,F2 | 覆盖 0/1 字节、页/inline/EC block 边界±1以及大对象 | 每个实际存储形态均确认后注入；长度、SHA-256、ETag、HEAD/LIST 与边界 Range 正确，无假设阈值。 | +| D02 · `heal.completed-multipart-object` | P0 | H1,H2 / DM / F1,F2 | 修复已完成的不等长多 part 对象，分别损坏首/中/末 part | 完整内容、part 顺序、原版本/ETag/元数据和跨 part Range 不变；multipart ETag 不能当整对象 MD5。 | +| D03 · `heal.historical-versions` | P0 | H1,H2 / DV / F1,F3 | 不同版本分布不同损坏位置，按对象和递归范围修复 | 所有原 versionId 可寻址且内容正确，最新版本身份不漂移，无多余版本或遗漏历史版本。 | +| D04 · `heal.delete-marker-semantics` | P0 | H1,H2 / DV / F3,F4 | 缺失/陈旧 marker 副本后 heal，并按当前值与精确 marker versionId 查询 | 当前 GET/HEAD 与指定 marker 的状态码及身份头符合契约；marker 不复活数据，LIST versions 无重复。 | +| D05 · `heal.versioning-suspended-null` | P1 | H1,H2 / DV / F1,F3 | 在版本开启、暂停、null 版本覆盖之间建夹具再修复 | null 和非 null 版本不混淆，不新增假版本；暂停前历史对象仍准确可读。 | +| D06 · `heal.object-metadata-and-tags` | P1 | H1,H2 / DS / F1,F3 | 包含用户 metadata、Content-Type/Encoding/Disposition、tags 的对象修复 | 对外 metadata、tags 和相关响应头逐项相同；不要求无关内部元数据字节完全一致。 | +| D07 · `heal.range-head-and-conditions` | P1 | H1,H2 / DB,DM / F1,F2 | 修复前后验证 HEAD、合法/越界 Range、If-Match/If-None-Match 及时间条件 | 200/206/304/412/416 等按逐请求契约判断，绝不返回旧内容、错误范围或错误 ETag。 | +| D08 · `heal.encrypted-objects` | P0 | H1,H2 / DE / F1,F2 | 按候选支持的 SSE-S3/SSE-KMS/SSE-C 建加密对象并修复 | 授权解密后原内容及版本一致，物理验证使用正确编码/密文层；密钥不可用时不误删、不降级明文，证据脱敏。 | +| D09 · `heal.object-lock-retention` | P0 | H1,H2 / DO,DV / F1,F3 | 对开启 retention/legal hold 的受保护版本修复，期间测试非法删除 | heal 不绕过保留/hold，也不丢失其元数据；拒绝删除与合法修复均有明确证据。 | +| D10 · `heal.special-keys-and-pagination` | P1 | H1,H2 / DS / F1,F3 | 覆盖中文、空格、百分号、编码斜线、多层前缀和多页对象列表 | 逻辑 key 与物理路径映射准确，不跨桶/前缀；分页无遗漏重复，不能把用户 key 当可执行 shell 或裸文件路径。 | +| D11 · `heal.bucket-metadata-integrity` | P0 | H1,H2 / DS,DV / F3 | 在隔离桶白名单内测试可支持的桶元数据恢复，并核对外部 bucket 配置 | 版本设置、策略/标签/保留配置按契约保持；不得遍历凭据配置或把未知内部记录当可重建数据。 | + + + +### E. 触发路径与修复责任 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| E01 · `heal.manual-normal-attribution` | P1 | H1,H2 / DB / F1 | 禁止预读目标，保留 format，隔离背景干扰后显式 normal heal | 证明本次 Admin token 实际覆盖目标并修复可由 normal 检出的缺失；不要求 normal 发现所有静默 bitrot。 | +| E02 · `heal.manual-deep-attribution` | P0 | H1,H2 / DB / F2 | 在普通读取仍可成功的 parity 损坏上显式 deep heal | 按 token/对象/版本及物理变更证明 deep 检查和修复，不把后台提前修复算给手工任务。 | +| E03 · `heal.read-repair-attribution` | P0 | H1,H2 / DB,DV / F1,F2 | 不发 Admin heal，对已确认的特定损坏对象发一次可追踪 GET/Range | 读取正确且后续真实补片；若走异步队列需证实 source=read-repair，不能只有 GET 成功。 | +| E04 · `heal.scanner-cold-object-repair` | P0 | H1,H2 / DB / F1,F2 | 不 GET/HEAD 损坏目标、不发手工 heal，仅等待已启用的 scanner 路径 | 冷对象被发现并修复，来源和遍历窗口明确；scanner 周期结束但目标没检查不能通过。 | +| E05 · `heal.mrf-offline-write-replay` | P0 | H2 / DL,DV / F4 | 成员离线时完成有 quorum 的 PUT/DELETE，确认待修责任后重入 | durable MRF 责任持久且身份精确；只在 repaired/verified healthy/authoritatively absent 等已证明结果后释放，成功提交不丢失。 | +| E06 · `heal.replacement-auto-rebuild` | P0 | H1,H2 / DB,DV / F8 | 替换盘接纳后不发 Admin/目标 GET，观察自动 replacement heal | 替换代次、ready/completion 与盘上全部目标版本一致；不能用服务 ready、队列空或计数重置代替完成。 | + + + +### F. 中断、持久化、恢复与升级 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| F01 · `heal.root-graceful-coordinator-restart` | P0 | H1,H2 / DX / F6 | root heal 非零进度时优雅重启原协调者，保留原拓扑与记录盘 | 在明确支持的边界内恢复原 task ID 和剩余责任/预算，完成后无遗留活动记录；不扩张为协调盘丢失后的故障转移。 | +| F02 · `heal.target-graceful-restart` | P0 | H2 / DB,DV,DX / F6,F8 | 替换目标仅完成部分重建时优雅重启目标节点 | 故障确实发生于部分完成窗口；重启后续建同一目标盘/代次，最终 manifest 与全部目标分片正确。 | +| F03 · `heal.target-process-crash` | P0 | H2 / DB,DV,DX / F6,F8 | 部分重建时 SIGKILL 测试进程，再正常启动 | unclean shutdown 与修复进度有真实证据；已提交对象不丢失，未完成目标不会被当完整；明确不是断电测试。 | +| F04 · `heal.coordinator-process-crash` | P0 | H2 / DB,DX / F6 | 协调者非优雅退出并恢复，分别观察 root、MRF、replacement | 按各机制已核实的持久化边界判断；不得把丢失的易失 token 当成功，需显式记录剩余责任并可安全重扫恢复。 | +| F05 · `heal.full-cluster-restart` | P0 | H2 / DB,DV,DM / F6 | 有未完成修复责任时重启全部测试节点，固定原拓扑/磁盘身份 | 有持久化保证的责任不遗失；全部端点恢复一致，旧版本/marker/MPU 与目标分片完整，重复重启不扩散错误。 | +| F06 · `heal.cancelled-task-restart` | P0 | H1,H2 / DB,DX / F6 | 完成取消及在途退出后重启，随后 forceStart/新任务正常修复 | 已取消的任务不自动复活；旧 checkpoint/token 不能取消或覆盖新任务，旧责任不会误标为修复完成。 | +| F07 · `heal.checkpoint-and-mrf-corruption` | P0 | H1,H2 / DB,DV / F9 | 受控构造 checkpoint/MRF 截断、坏校验、未来 schema、陈旧代次及写入失败 | 已确认持久化责任不得静默删除或伪 ACK；未知记录保留且可诊断；精确崩溃窗口由原生屏障证明，不伪称真机命中。 | +| F08 · `heal.mixed-version-upgrade-rollback` | P1 | H2 / DB,DV / F6,F9 | 固定前后两个明确兼容候选，覆盖滚动升级、混合 peer、回滚与重启 | 能力不足必须显示 partial/unsupported 而非完整成功；按格式兼容契约保留责任，旧版本不能误退休新版本记录。 | +| F09 · `heal.dedicated-host-forced-restart` | P1 | H6 / DB,DV,DX / F10 | 经专项授权在部分修复/持续写入时强制重启独占测试 VM/主机 | 保留平台事件与 ACK ledger；验证落盘和恢复，不把 SIGKILL 当断电；VM 强制停止也不等同于物理控制器断电。 | + + + +### G. 与前台和后台操作并发 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| G01 · `heal.race-unversioned-overwrite` | P0 | H1,H2 / DL / F1,F2,F9 | 同 key 连续覆盖与 heal 同时推进，包含提交尾部清理窗口 | 按可证明的提交顺序验证最终最后一次成功写入；不能回滚到旧 data_dir，未确认超时写入单独解析。 | +| G02 · `heal.race-versioned-put` | P0 | H1,H2 / DV,DL / F1,F9 | 修复历史版本时创建新版本并重复读取各 versionId | 新旧版本不混片、不串内容；已返回成功的每个版本都保留，最新值按确定的先后关系验收。 | +| G03 · `heal.race-version-delete` | P0 | H1,H2 / DV / F1,F2,F9 | 对损坏的精确历史 versionId 发 DELETE，与 deep heal 竞争 | 已确认删除的版本不能被复活，其他历史版本不受影响；原生临界点测试与真机竞争分别留证。 | +| G04 · `heal.race-markers-and-batch-delete` | P0 | H1,H2 / DV / F3,F9 | 与创建/删除 marker、批量版本删除和无版本删除分别交错 | 按逐项 DELETE 回执核对版本/marker 集合，无复活、误删或遗漏批量错误。 | +| G05 · `heal.race-multipart-complete-abort` | P0 | H1,H2 / DM,DL / F1,F9 | heal 期间对不同已知 UploadId 分别续传、Complete、Abort | 成功 Complete 内容只出现一次；Abort 不破坏已提交对象或别的 upload；未完成 part 不被伪造为完整对象。 | +| G06 · `heal.race-copy-and-metadata-update` | P1 | H1,H2 / DS,DL / F1,F9 | COPY 源/目标和标签/元数据更新与修复交错 | 最终对象内容与对应提交的 metadata 一致，读修复发现新代次应重验/放弃陈旧提交而非覆盖。 | +| G07 · `heal.race-ilm-expiry-and-transition` | P0 | H2 / DV,DE / F1,F9,F11 | 分别与当前/历史版本过期、远端 transition/restore 同窗口执行 | 已授权过期不被恢复；迁移状态与远端版本引用正确；只有实际观察到交叠才计并发覆盖，tier 未配置需明确阻断对应实例。 | +| G08 · `heal.race-replication-resync` | P1 | H2 / DV,DL / F1,F9,F11 | 在隔离复制两端与 heal 同时运行新增/删除/重同步 | 对端版本、删除语义及源数据最终一致，无无限回环或重复传播；不得依赖仅源端 GET 宣称复制正确。 | + + + +### H. 多 Pool、移动与多 set 隔离 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| H01 · `heal.with-expansion-rebalance` | P0 | H5 / DB,DV / F1,F2 | 先证明源 pool 持有夹具，再扩容并与修复同时 rebalance | 修复目标不得指向错误 pool/set；新旧数据完整、移动终态正确，保留 scanner backlog 提交权威检查。 | +| H02 · `heal.with-rebalance-stop-resume` | P1 | H5 / DB,DL / F1,F6 | heal 活动期间 stop/restart/resume rebalance | 两类任务状态和身份独立；停止/恢复移动不误终止修复或丢失对象，恢复后两者均满足各自契约。 | +| H03 · `heal.with-decommission` | P0 | H5 / DB,DV,DM / F1,F2 | 分别在待退役源和存活目标制造可恢复缺片，再执行退役 | 不得提前删源或以临时副本冒充迁移成功；按 Pool 既定写入限制验收；只剩 survivor 后仍完整可读且有物理证明。 | +| H04 · `heal.with-decommission-cancel-clear` | P0 | H5 / DB,DL / F1,F9 | heal/移动争用中取消退役、Clear、恢复，并覆盖容量锁超时恢复 | 保持持久化后通知及在途退出顺序；不误取消新代次；对 #2423 的重试窗口单独注明真机是否实际触发，不只看最终绿色。 | +| H05 · `heal.after-retired-pool-removal` | P0 | H5 / DB,DV / F1,F3 | 源 pool 完全退役移除并停机后，只在 survivor 上触发修复 | 不再向已移除源写入或依赖其兜底；来源权威和版本正确；#2422 路径不能出现无权威覆盖或错误回退。 | +| H06 · `heal.multi-pool-target-isolation` | P0 | H5 / DB,DV / F1,F4 | 仅破坏选定 pool/set，并在另一 pool 保留不同对象与控制故障 | 选择器、任务、容量/配额和目标 drive 一致；未选 pool 不受错误修复，跨 pool 存在不等于该目标已修复。 | +| H07 · `heal.ec84-multi-set-restart` | P0 | H3,H4 / DB,DV,DX / F6,F8 | 固定 EC8+4 单 set 与至少两 set，部分重建时重启/崩溃目标 | 每个 set 的故障覆盖和重建分别证明，不跨 set 拼 shard；必须满足真实盘数及映射，不能用 4 盘用例替代 EC8+4。 | + + + +### I. 资源、依赖故障与调度 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| I01 · `heal.slow-io-and-eio` | P1 | H1,H2 / DB,DX / F5 | 在独占故障层注入有界延迟、短读、EIO，解除后观察恢复 | 暂态与永久错误分开，重试有界且无忙循环；解除故障后实际修复，期间不得提前标健康。 | +| I02 · `heal.enospc-inodes-and-quota` | P0 | H1,H2 / DB,DM / F5,F11 | 限制专用测试卷空间/inode，并参数化本次桶配额 | 仅填专用受限卷、不耗尽共享真实盘；失败不破坏好副本或释放尚需容量，释放空间后正确续修。 | +| I03 · `heal.network-partition-and-lock-quorum` | P0 | H2 / DB,DV,DL / F7 | 仅隔离测试服务节点间流量，覆盖少数派、读/写/锁 quorum 边界及恢复 | 少数派不得承诺无安全性的修复或写入；cluster coverage 不完整不能显示全局健康；恢复无分叉、跨代覆盖。 | +| I04 · `heal.queue-pressure-and-admission` | P1 | H1,H2 / DX / F11 | 多来源请求冲击有限队列，混合重复任务、不同优先级和多个桶 | Full/Dropped/Merged 语义与实际任务一致；丢弃不能丢 durable 责任；在预先声明负载窗口内无无限堆积或健康误报。 | +| I05 · `heal.lock-timeout-and-caller-abort` | P0 | H1,H2 / DL,DV / F5,F9 | 真实对象锁争用超过预算及调用方断开，再释放锁恢复 | 取消/超时不得提前释放在途所有权或重复提交；陈旧 read-repair 必须重验代次；恢复后无泄漏和错误覆盖。 | +| I06 · `heal.kms-and-tier-outage` | P1 | H2 / DE,DV / F11 | 仅让已分配测试 KMS/远端 tier 暂不可用，持续查询与修复 | 依赖故障不得解释为对象不存在/可删/已修复，不暴露密钥；恢复后原版本可解密或恢复，责任未丢失。 | + + + +### J. 可观测性、幂等、长稳与性能 + +| 编号 / 稳定 ID | P | 拓扑 / 夹具 / 故障 | 操作与触发 | 附加通过判据 | +| --- | --- | --- | --- | --- | +| J01 · `heal.status-partial-disabled-unknown` | P0 | H1,H2 / DB / F0,F6 | 分别检查 enabled/disabled/uninitialized、全体可达及缺失 peer 的状态 | idle 只在覆盖完整且真实无活动时成立；partial/unknown/degraded/disabled 不被客户端转换为全部完成。 | +| J02 · `heal.outcome-counters-and-error-truth` | P0 | H1,H2 / DB,DV / F1,F2 | 同批次混合健康、可修、不可修、deferred、dry-run 对象 | execution/coverage/disposition/计数和逐对象物理结果相符；unknown、skipped 或 completed_with_errors 不能算修复成功。 | +| J03 · `heal.incremental-cursor-and-truncation` | P0 | H1,H2 / DS,DX / F1 | 生成超过结果保留上限的任务，分页轮询 sinceSeq，并跨节点/终态读取 | nextSeq/minSeq 单调且 gap/truncation 明示；有界 items 不是完整责任清单，最终须用独立 manifest/计数补证，不静默丢错误。 | +| J04 · `heal.repeat-idempotency` | P1 | H1,H2 / DB,DV / F1 | 第一次修复后重复 normal/deep、重启再检查 | versioned 首次允许所有 identity repaired 或至少一个 repaired 加其余正健康态，marker 仅 repaired/metadata_healthy，计数必须与 receipt 集合一致；normal repeat 接受 `metadata_healthy`，deep repeat 要求 `verified_healthy`，repeat marker 为 `metadata_healthy` 且 healed=0；unknown/skipped/failed 不得成为 PASS。 | +| J05 · `heal.long-soak` | P2 | H2,H5 / DX,DL,DV / F4,F6,F11 | 独占环境运行预注册 24h/72h 窗口，混合修复、掉线、写入和版本删除 | 每次 ACK 可追踪；最终全量 manifest 正确，队列/内存/FD/临时空间无无界增长，无后台 panic 或未解释持久责任。 | +| J06 · `heal.performance-abba` | P2 | H2 / DX,DL / F1,F11 | 同硬件同候选按 ABBA 运行无 heal 基线和有 heal 负载，重复并固定限速 | 分别报告修复吞吐、GET/PUT P95/P99、错误率和 CPU/IO；阈值事先冻结，不能事后挑轮次或把功能 PASS 充性能验收。 | + +## 与主仓库既有验证的复用关系 + +以相同源 SHA 的实现为准，先读脚本的 scope、环境要求和故障 oracle，再适配到固定 fleet；**不直接在共享机器运行原 E2E 的清盘/建进程脚本**。已有覆盖只是复用线索，不能直接标记本目录 planned 用例为 implemented。 + +| 已有资产 | 可复用内容 / 本矩阵对应 | 不能混淆的边界 | +| --- | --- | --- | +| `crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs` | runtime wiped disk、Admin deep、remote replacement，B/C/E/F | 进程 E2E 与固定真机、同节点目录盘与独立 XFS 物理盘需分别标记；检查其 skip 分支，必选故障能力缺失不能通过。 | +| registry 的 background-target-restart / crash | F02/F03、目标部分进度与 replacement shard 证据 | SIGKILL 不是断电；原夹具无版本覆盖不能自动扩张为 DV 已覆盖。 | +| registry 的 EC8+4 single-set / multi-set / multi-pool 项 | H07/H06 与相应 F02/F03 参数 | 精确盘数、set/pool 分布与同一故障窗口要匹配，4 盘 PASS 不能覆盖 G14。 | +| W13 durable MRF 与 W16 recovery-intent/quota 证据 | E05、F07、I02、J02 | 责任 ACK、恢复意图、配额与终态内容都需要原始实测输入，不只运行 JSON assembler。 | +| G05/G06 status/outcome、G02 checkpoint、G09 upgrade 证据生产脚本 | A03、J01-J03、F07/F08 | schema 自检、descriptor 组装和真实故障执行是三种不同证据，不可互相代替。 | +| ECStore/heal 原生 runtime-fence、版本删除/覆盖竞争回归 | A09、C07、G01-G06、I05 | 原生可控屏障覆盖精确窗口；没有在 fleet 触发该窗口时只记录功能覆盖，不把两类日志混为真机因果证明。 | + +来源：[必选 E2E registry](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/.config/scanner-heal-required-tests.json)、[磁盘重建 E2E](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs)、[Linux evidence plan](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/scripts/run_scanner_heal_linux_evidence_plan.py)、[脚本分类](https://github.com/rustfs/rustfs/blob/940e221988694da33fe005054bb86086bae88cd6/scripts/README.md)。其中 plan/preflight 不产生 measured evidence，assembler 不替代采集。 + +## 当前框架的执行方式 + +Functional Chain 在共享节点并发锁内调用 auto-testing 的 canonical dispatcher。普通 live 执行消费候选 deb 和工具产物，无需 RustFS checkout；只读 metadata inspector 和原生屏障测试须在构建阶段绑定精确候选源码，单独留证。 + +尚未自动化的分支由 `matrix.py plan` 固定全部适用拓扑、源断言、操作顺序、参数绑定与恢复步骤。`matrix.py audit` 校验九类证据的摘要和分支完整性，结果仍须逐项审阅，绝不自行生成 canonical PASS。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md new file mode 100644 index 00000000..c66522fc --- /dev/null +++ b/docs/validation-heal-operations.md @@ -0,0 +1,630 @@ +# Heal 真机执行手册：84 项操作卡的公共命令 + +此手册和 [84 项逐例操作卡](validation-heal-cases.md) 一起构成可执行的**人工操作方案 + 脚本辅助**。不是 84 个无人值守自动测试，也不是通过报告。操作卡列出 301 个分支模板；真实盘角色、阈值、损坏数量和拓扑仍需在每批 `parameterBindings` 展开，不能把模板数量当实测次数。 + +公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh heal.admin-healthy-noop CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 + +## 1. 使用入口与交付物 + +```bash +python scripts/validation_heal/matrix.py list +python scripts/validation_heal/matrix.py show C02 +python scripts/validation_heal/matrix.py show F02,F03,H07 +python scripts/validation_heal/matrix.py plan --selection all \ + --run-id manual-heal-20260910-a --out reports/validation-heal-84-plan.json +python scripts/validation_heal/matrix.py plan --selection A01,A04,C01,C02,C04,J04 \ + --topologies H1 --run-id manual-heal-20260910-b --out reports/heal-h1-plan.json +``` + +`plan` 只冻结选择/分支和对应操作，不部署、不评分。`show` 同时给出步骤、命令入口、验收和恢复要求。不同拓扑是不同实例；未选拓扑会保留 `scopeExclusions`，不能据局部结果声称全矩阵通过。 + +| 工具 | 已实现的职责 | 不代表什么 | +| --- | --- | --- | +| `matrix.py` | 84 项/301 分支索引、逐例操作卡、冻结计划、证据完整性审计 | 不自动分配拓扑，不根据文件存在就判 PASS | +| `heal_probe.py` | 带签名/匿名/坏签名的原始 Admin；受限 S3 操作；全版本/标签/内容摘要清单及比较 | HTTP 检查通过不代表 heal 成功；inventory 的 GET 会触发读修复 | +| `heal_workload.py` | DB/DV/DM/DS/DX 真实夹具；有界并发、同key前台修改串行、请求前及ACK后持久落盘 | 不替用户推导竞态后的最终版本；超时写必须单独解析 | +| `heal_fault.py` | Linux 描述符相对寻址、mount ID 保护、单文件 missing/bitflip/truncate/zero/header、快照和恢复 | 文件故障不是物理掉盘；不修改系统 metadata；不盲目恢复新代次 | +| `prepare_heal_meta_inspector.py` | 为精确候选源码准备只读 Rust metadata 解析器工程 | 需在批准的构建机编译；解析 metadata 不是验证 bitrot/EC | + +## 2. 先绑定环境，禁止直接猜节点 + +live 命令使用当前 auto-testing 的 `smoke-testing` runner、`RUSTFS_NODES` 和 `RUSTFS_SSH_USER`，复用 `rustfs_heal_test.sh` 的 Heal 节点。运行前取得与 Functional Chain 一致的共享节点并发锁。H1–H6 是测试拓扑要求，不是另一组 SSH 主机；不得按源仓库历史主机名或 IP 选择机器。 + +用 `matrix.py plan` 冻结当前 SSH 节点与所有适用操作分支，再根据本次只读 preflight 填写私有 `lab.json`。计划中的 `verified:false` 不可直接改成资源验证通过。H4 需要至少 24 个已分配独立盘端点；H6 需要独占主机及平台事件/恢复绑定。缺少资源的分支保留为 BLOCKED。 + +填写内容须包含： + +- 本批 run ID、截止时间、独占服务/端口、jump route、实际主机名、每个磁盘的 mount UUID/major:minor、服务 unit/原 PID/数据根； +- 与当前 validation lane 相同的不可变 candidate manifest、源码 commit、包与已安装二进制 SHA-256；当前 rc 路径与二进制 SHA-256；auto-testing commit；保存真实构建清单，不只填摘要字符串； +- H1–H6 实際形状、每个对象 N/data/parity/R/W/lock quorum、pool/set/slot/盘 UUID；H4 至少 24 个实际盘端点； +- 最长修复/重启/故障时限、对象/字节/请求/并发预算、所有危险动作的精确目标及恢复人； +- 包含平台操作、独占故障设备、旧版本、KMS/tier 或原生屏障的实例，先提供能力及授权证据。 + +`binding-check` 只是语法/租约检查，不是资源或包身份的远程证明： + +```bash +python scripts/validation_heal/heal_probe.py --binding lab.json --out reports/binding-check.json binding-check +``` + +先在跳板只读核验 `hostname`、`findmnt -J`、`lsblk -J`、`ss -lnt`、二进制 `sha256sum`/`--version`、实际运行配置与 PID。保存输出为 preflight。租约没有发放、拓扑不符、端口占用、磁盘 UUID 漂移时在这里 BLOCKED，不先注入后跳过。 + +### 独立实验服务的部署 + +H1 使用当前 Heal 节点经核验的四个独立 mount，**另建** `/data/rustfsN/heal-validation//data` 和同级 `quarantine`，root-owned 0700；每个 `` 根写入包含 `runId` 的 root-owned `owner.json`。先检查每个根不存在；已存在的根不能覆盖。复用 `rustfs_heal_test.sh` 的 candidate deb 下载、SHA 校验和安装流程，把已安装且摘要已核验的 `/usr/bin/rustfs` 放在本批 state 下；执行用例不需要 RustFS 源码 checkout。凭据文件为 0600。以下专用 unit 仍运行在当前 Heal 节点，使用已分配的 9000 测试端口： + +```bash +# 在目标节点；所有变量来自已审核的 lab binding，不接受任意在线输入。 +sudo systemd-run --unit="$HEAL_UNIT" --property=Type=exec \ + --property=PrivateMounts=yes --property=KillMode=control-group \ + --property=TimeoutStopSec=30 --property=RuntimeMaxSec=3600 \ + --property=MemoryMax=2G --property=CPUQuota=200% \ + --property=WorkingDirectory="$HEAL_STATE" \ + --property=EnvironmentFile="$HEAL_STATE/service.env" \ + "$HEAL_STATE/rustfs" "${HEAL_DATA_ROOTS[@]}" +``` + +需重启的场景保留 unit 定义、env、二进制及原始 endpoint 参数，不使用 `--collect`；RuntimeMaxSec 按该批预算冻结，J05 必须单独分配长时服务。`HEAL_DATA_ROOTS` 是已审核的 Bash 数组，不从未审核字符串 eval。H2/H3/H4/H5 用各节点一致的 endpoint 序列/显式 pool 分组配置；启动后从每盘 `format.json` 读回，不能从启动参数猜拓扑成功。不得直接运行会清理旧 Pool 根的部署/reset 命令。 + +### 命令约定 + +下列函数在**跳板机**执行。`HEAL_BINDING`、`HEAL_REPORT` 是该实例私有绝对路径；`B` 是绑定 prefix 下的一个桶，`K` 为逻辑 key，`V` 为先前 ACK 的精确版本。每条命令的 label 唯一，已有证据拒绝覆盖。凭据只通过受控环境注入 `HEAL_ACCESS_KEY`/`HEAL_SECRET_KEY`/可选 session token，禁用 `set -x`。 + +```bash +set -euo pipefail +umask 077 +hp() { + local label=$1 + shift + python scripts/validation_heal/heal_probe.py --binding "$HEAL_BINDING" \ + --out "$HEAL_REPORT/$label.json" "$@" +} +hw() { + local label=$1 + shift + python scripts/validation_heal/heal_workload.py --binding "$HEAL_BINDING" \ + --out "$HEAL_REPORT/$label" "$@" +} +B=$(jq -er .bucketPrefix "$HEAL_BINDING") +RUN=$(jq -er .runId "$HEAL_BINDING") +``` + +特权故障命令在**经跳板连接的目标节点**执行。`ROOT` 是本例目标盘 `` 根；`NODE` 是已绑定 SSH alias；把该 SHA 的 `heal_fault.py` 放在 root-owned 的本批 state 下并核对摘要。不得从逻辑 key 拼 SSH shell。以下 `hf` 仅在该目标节点的交互 shell 定义： + +```bash +hf() { sudo python3 "$HEAL_STATE/heal_fault.py" --root "$ROOT" --run-id "$RUN" "$@"; } +hf init --bucket-prefix "$B" --lease-expires-at "$LEASE_EXPIRES_AT" +``` + +初始化只增加故障所有权标记，不创建/格式化数据盘。每盘只 init 一次。系统重启导致 mount ID 变化时暂停注入；先重新核对 UUID/major:minor/所有权并人工重新绑定，不绕过检查。 + +## 3. 数据与客户端命令 + + + + + + + +### 确定性夹具 + +每个命令使用新桶，禁止在现有夹具桶上重复 seed 后称同一轮。 + +```bash +hw seed-db seed --fixture DB --bucket "$B-db" +hw seed-dv seed --fixture DV --bucket "$B-dv" +hw seed-dm seed --fixture DM --bucket "$B-dm" +hw seed-ds seed --fixture DS --bucket "$B-ds" +hw seed-dx seed --fixture DX --bucket "$B-dx" +# 替换为候选确认的真实 inline/block 边界；可另外生成一桶： +hw seed-boundary seed --fixture DB --bucket "$B-boundary" \ + --count 128 --sizes "$BOUNDARY_SIZES" +``` + +DB 默认 128 项，包含 0/1/4095/4096/4097、1MiB±1、多MiB，非重复 SHAKE 数据；1MiB 不是预设产品阈值。DV 默认 4 key，每 key 三个版本，交替创建当前 marker。DM 有已完成三 part（前两 part 分别 5MiB+17、5MiB+93，末 part 1MiB+47）及两个独立 pending upload。DS 默认 257 特殊/多前缀 key；DX 默认 257 外部对象。自动 seed 不包括 suspended/null、特殊压缩、SSE/retention 等额外分支，按下列 S3 命令补充并保存 ACK。 + +从 ACK 文件取得真正的 UploadId/VersionId/part ETag；不要在多个上传之间复用猜测的 ID。总字节预算包含所有版本/part，不足时先扩展经批准的预算，不自动跳过对象。 + +后续命令的 `B` 必须改为该实例实际桶，例如 `B="$(jq -er .bucketPrefix "$HEAL_BINDING")-db"`，不要对没有创建的裸 prefix 做 inventory。根目录 owner 仍绑定原始 run prefix，以允许本run的各后缀桶。 + +seed 完成后自动以完整 GET/版本清单对账确定性原文与创建 ACK，保存 `expected-fixture.json`、`baseline.json`；没有通过基线校验就不准进入注入。这个 GET 发生在故障前，不可拿 seed 命令监测 scanner/read-repair 故障窗口。 + + + + +### 单个 S3 操作与精确版本 + +```bash +hp put s3 --operation put_object --size 5243003 --seed "$RUN/new/1" \ + --params "$(jq -nc --arg b "$B" --arg k "$K" '{Bucket:$b,Key:$k,ContentType:"application/octet-stream",Metadata:{nonce:"1"}}')" +hp get s3 --operation get_object \ + --params "$(jq -nc --arg b "$B" --arg k "$K" --arg v "$V" '{Bucket:$b,Key:$k,VersionId:$v}')" +hp suspend s3 --operation put_bucket_versioning \ + --params "$(jq -nc --arg b "$B" '{Bucket:$b,VersioningConfiguration:{Status:"Suspended"}}')" +hp delete-version s3 --operation delete_object --expect-http 204 \ + --params "$(jq -nc --arg b "$B" --arg k "$K" --arg v "$V" '{Bucket:$b,Key:$k,VersionId:$v}')" +``` + +请求复杂时 `--params-file` 使用 JSON。接口参数为 boto3 snake_case 操作名、原生 CamelCase 参数；只允许本 run 桶、受限操作，copy 源也必须在本 run。GET 只保存完整内容的 hash/长度，不把对象内容写入日志。PUT/Complete 超时记 resultUnknown，下一步用相同唯一 key/version/UploadId 查询解析，不静默重试。 + +`--expect-http` 必须使用该请求预注册的精确值。例如 create/put/list 常见 200，delete/abort 常见 204，Range 206；负例按候选契约分别 304/403/404/405/412/416。CLI 返回 0 只说明该次预期 HTTP 匹配，永不宣称整个 case PASS。SSE-C 参数值可使用字面量 `$HEAL_SSE_C_KEY`，脚本只在内存替换，输出脱敏。 + + + + +### 全版本基线和最终核对 + +```bash +hp baseline inventory --bucket "$B" +# 先完成触发归属和 PHYSICAL，再调用下面 GET 型检查： +hp post-heal inventory --bucket "$B" +hp s3-comparison compare --before "$HEAL_REPORT/baseline.json" \ + --after "$HEAL_REPORT/post-heal.json" +``` + +inventory 分页遍历全部版本/marker，按每个 versionId GET 全内容、tags 和公开 metadata，再用分页 current LIST 交叉核对；对象上限 10,000、单对象 64MiB，批次字节预算来自绑定文件。更大对象场景需单独预注册流式工具和预算，不能截断后称全量。 + +并发、生命周期、合法删除场景的预期清单会变化：人工先按逐请求 ACK 因果关系生成 `expected-post.json`，记录每项增删对应的 ACK 文件；再比较实际清单，不能直接拿静态基线判失败或拿实际输出复制成期望。未知写入未解析则不可通过。加密/归档对象先用授权 GET/restore 参数逐版本取证；默认 inventory 不会自动解锁、传 SSE-C 或 restore。恢复型 GET 必须保存密文/明文两层对应关系。 + + + +### Range、HEAD、条件请求 + +```bash +hp head s3 --operation head_object --params-file request-head.json +hp range s3 --operation get_object --expect-http 206 --params-file request-range.json +hp if-none-match s3 --operation get_object --expect-http 304 --params-file request-if-none-match.json +hp if-match-reject s3 --operation get_object --expect-http 412 --params-file request-if-match-bad.json +hp range-outside s3 --operation get_object --expect-http 416 --params-file request-range-outside.json +``` + +JSON 都包含本例 Bucket/Key/可选 VersionId；另外分别添加 `Range:"bytes=5242879-5242882"`、`IfNoneMatch:<原ETag>`、`IfMatch:<错误但语法合法ETag>`、超出对象长度的 Range。时间条件用 `IfModifiedSince`/`IfUnmodifiedSince` 的 ISO 日期；从 HEAD 的真实 LastModified 推导早/晚两个控制值。验证 ContentRange、响应长度和从确定性原文计算的片段 hash，不能只验状态码。 + + + + + +### SSE、Object Lock 和外部依赖 + +```bash +hp sse-s3-put s3 --operation put_object --size 5243003 --seed "$RUN/sse" --params-file sse-s3-put.json +hp sse-kms-put s3 --operation put_object --size 5243003 --seed "$RUN/kms" --params-file sse-kms-put.json +hp sse-c-put s3 --operation put_object --size 5243003 --seed "$RUN/ssec" --params-file sse-c-put.json +hp locked-bucket s3 --operation create_bucket --params-file locked-bucket.json +hp retained-put s3 --operation put_object --size 5243003 --seed "$RUN/lock" --params-file retained-put.json +hp retention s3 --operation get_object_retention --params-file protected-version.json +hp hold s3 --operation get_object_legal_hold --params-file protected-version.json +hp reject-delete s3 --operation delete_object --expect-http 403 --params-file protected-version.json +``` + +SSE JSON 分别加 `ServerSideEncryption:"AES256"`、`ServerSideEncryption:"aws:kms",SSEKMSKeyId:<本例专用key>`、`SSECustomerAlgorithm:"AES256",SSECustomerKey:"$HEAL_SSE_C_KEY"`。SSE-C GET 也提供同一密钥，先用原请求证明可解密。所有外部 key/endpoint 是经过审批的测试租户，不复用生产依赖。 + +锁桶 JSON 为 `{Bucket:...,ObjectLockEnabledForBucket:true}`。受保护 PUT 使用 `ObjectLockMode:"GOVERNANCE"` 或 `"COMPLIANCE"`、实际未来 UTC `ObjectLockRetainUntilDate`，或 `ObjectLockLegalHoldStatus:"ON"`。必须验证无 bypass 角色在保留期间真的不能删除；脚本拒绝 BypassGovernanceRetention=true。Legal hold 解除使用 `put_object_legal_hold` 的 `LegalHold:{Status:"OFF"}`，仅作用于本例版本。Compliance 等真实到期，不改主机时间、不内部删文件。 + +KMS/tier outage 使用 NETWORK 在本例专用 net namespace 内仅阻断对应依赖的 IP/port，或仅撤销本例租户凭据；恢复原规则/权限后重试原 version。tier 缺失一个远端版本可通过远端本例桶 API 删除并保留ACK/原备份，但这是“远端对象丢失”负例，不等同网络中断。 + +## 4. Admin 触发、状态与窗口 + + + +实际服务配置要取证，但**不要转储包含凭据的全环境**。manual 模式记录以下非敏感字段： + +```text +RUSTFS_HEAL_ENABLED=true +RUSTFS_SCANNER_ENABLED=false +RUSTFS_HEAL_AUTO_HEAL_ENABLE=false +RUSTFS_HEAL_MRF_ENABLE=false +RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE=false +``` + +scanner/自动/MRF 场景按操作卡逐项开启相应字段，并冻结 scanner cycle/start delay/bitrot cycle。配置变更只改专用 service.env，重启本例 unit 后证明实际生效。read-repair 所需队列依赖按候选源码核实，不能为了“排除后台”关掉被测路径。 + + + + + + + +### 原始 Admin 命令 + +`SCOPE_PATH` 为编码后的 `/rustfs/admin/v3/heal//`；key 的每个字节按 URL 编码，root 为 `/rustfs/admin/v3/heal/`。JSON 中 pool/set 是实际零基索引，不能在 H5 固定填 0。例： + +```bash +hp start admin --path "$SCOPE_PATH" \ + --body '{"recursive":false,"dryRun":false,"remove":false,"recreate":false,"scanMode":2,"updateParity":false,"nolock":false,"readRepair":false,"pool":0,"set":0}' +TOKEN=$(jq -er '.observation.response.clientToken | select(type=="string" and length>0)' "$HEAL_REPORT/start.json") +hp status-001 admin --path "$SCOPE_PATH" --query "clientToken=$TOKEN" +hp stop admin --path "$SCOPE_PATH" --query "clientToken=$TOKEN" --query forceStop=true +hp force-start admin --path "$SCOPE_PATH" --query forceStart=true --body-file heal-options.json +# 保留重复 query，真正送到服务端验证： +hp invalid-duplicate admin --path "$SCOPE_PATH" --query clientToken=a \ + --query clientToken=b --expect-http 400 +``` + +WAIT 用有界循环保存每一响应，不仅保存最后绿色状态；普通 repair 900 秒、重启后 1800 秒是起始测试预算，须预注册，不是产品 SLO： + +```bash +for i in $(seq 1 900); do + label=$(printf 'wait-%04d' "$i") + hp "$label" admin --path "$SCOPE_PATH" --query "clientToken=$TOKEN" + summary=$(jq -er '.observation.response.summary' "$HEAL_REPORT/$label.json") + if [ "$summary" != running ]; then break; fi + sleep 1 +done +``` + +循环退出后人工/候选 adapter 检查：execution completed、coverage complete；failed/unknown/attemptFailures=0；不溢出；逐对象 disposition 与设计匹配；非 dry-run 的 incarnation/版本/pool/set 正确。dry-run 只允许 observed，healed=0，随后无物理变更。负例以预注册错误/未完成契约判，不套正向 completed。 + +`summary=finished` 或 HTTP200 单独不够；`truncated=false` 可以省略，null/true 不算 false；status.settings 是查询默认值，不代表原任务实际 scanMode。outcome 的 128 项窗口不是完整对象清单。当前 rc 会简化部分字段，以 raw Admin + 独立 manifest 为准。 + + + + + +```bash +hp background admin --path /rustfs/admin/v3/background-heal/status +hp replacement admin --method GET --path /rustfs/admin/v4/heal/replacement-recovery +"$RC_BIN" --json --no-color admin heal status "$RC_ALIAS" +``` + +rc 必须是本批冻结二进制，私有 RC_CONFIG_DIR/alias 只指向本例服务，stdout/stderr/exitCode 都留存。查询其他节点：直接运行 heal_probe 的全局 `--endpoint n1`/`--endpoint stable`；保存实际连接 peer 身份。健康状态需要 expected/responded/unknown 与所有 peer 覆盖，不允许缺 peer 时 idle 假绿。 + + + + +### 部分进度与增量游标 + +```bash +hp cursor-001 admin --path "$SCOPE_PATH" --query "clientToken=$TOKEN" --query sinceSeq=0 +NEXT=$(jq -er '.observation.response.nextSeq' "$HEAL_REPORT/cursor-001.json") +hp cursor-002 admin --path "$SCOPE_PATH" --query "clientToken=$TOKEN" --query "sinceSeq=$NEXT" +``` + +WINDOW 的必要证据是 running 且 `0 + +### 权限和过期签名 + +在专用服务先保存当前 rc 的 `admin user --help`、`admin policy --help`、`admin heal --help`。使用候选 IAM 支持的 Admin Heal action 创建最小策略（从候选 `HealAdminAction` 的实际字符串导出，不猜拼写）；另建明确不含该 action 的角色。示例用户/策略动作按本批 rc help 的参数顺序执行并保存原始回执，凭据用私有输入通道，不开 shell trace。 + +已核对参考候选 action 为 `admin:Heal`，最小策略主体为 `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:Heal"],"Resource":["arn:aws:s3:::*"]}]}`；RustFS policy parser 要求带类型的 ARN，裸 `"*"` 会被拒绝。将这个权限仅赋予本例临时身份，不改共享管理员策略。新候选仍需核对 action 和 rc attach 语法。 + +```bash +hp anonymous-start admin --path "$SCOPE_PATH" --auth anonymous --expect-http 403 --body-file heal-options.json +hp bad-signature admin --path "$SCOPE_PATH" --auth bad-signature --expect-http 403 --body-file heal-options.json +# 无权限/最小权限实例：更换本次调用环境中的 HEAL_ACCESS_KEY/HEAL_SECRET_KEY，执行相同请求。 +``` + +过期签名：保存一次已签请求，在签名有效期外重放到**相同本例地址/路径/请求体**，仅等待客户端签名超期，不改变服务器时钟。先用新签名同请求做控制。时限按候选签名校验窗口预注册；签名本身含敏感身份，原请求存私有0600文件，只归档脱敏结果和请求摘要。对 start/status/stop 都测试；token 不能替代权限。结束时通过正常 IAM API 删除本例用户/策略并验证旧凭据失效。 + +## 5. 物理证据与故障后端 + + + + +### 真实布局、角色和物理快照 + +在目标节点逐盘读取本例 `data/.rustfs.sys/format.json`（只读）及 mount UUID/major:minor，保存 pool/set/this 映射。不能把同一设备的多个目录作为独立盘。逐对象 xl.meta 使用与候选相同源码的只读解析器： + +```bash +python scripts/validation_heal/prepare_heal_meta_inspector.py --source "$CANDIDATE_SOURCE" \ + --commit "$CANDIDATE_COMMIT" --out "$INSPECTOR_BUILD" +# 仅批准的构建机；生成独立 helper 的锁文件后冻结，和服务构建清单分开。 +cargo build --release --manifest-path "$INSPECTOR_BUILD/Cargo.toml" +sha256sum "$INSPECTOR_BUILD/Cargo.lock" "$INSPECTOR_BUILD/target/release/heal-meta-inspect" +"$INSPECTOR_BUILD/target/release/heal-meta-inspect" "$XL_META_EVIDENCE_COPY" "$B" "$K" +``` + +解析器不输出用户metadata或密钥，只输出版本、data_dir、parts、EC参数/分片角色。首先对同批健康样本和损坏样本分别证明成功/拒绝；不支持的版本不得换别的版本解析器后宣称同候选验证。源 checkout 必须为固定 SHA 且无已跟踪修改。 + +```bash +# 目标节点。所有输出重定向到本例证据文件，不能覆盖已有阶段。 +hf inspect --path "$TARGET_RELATIVE_PATH" +hf snapshot --bucket "$B" +``` + +快照对整桶每个文件保存长度/hash，目录描述符逐层拒绝 symlink/hardlink/不同mount ID（含同设备bind mount）；最多10,000文件/8GiB，每文件64MiB。四阶段 before/fault/after/restart 每盘各一份。外部part与**同一个角色、同版本的健康原片**字节完全相同是独立重建对照；数据应不重复且四片摘要不同。不得读取 quarantine 来辅助服务器重建。 + +xl.meta 可合法重写；比较解析后的版本/引用，而非要求所有内部字节不变。允许的 xl.meta.bkp 必须属于同目标data_dir且摘要等于原xl.meta，不能泛化豁免任意额外文件。metadata恢复/换代后涉及bitrot重新编码时，要用匹配候选验证器/固定合法解码集证明；此解析器不验证HighwayHash或重新编码，报告必须如实写验证深度，不能只因解析成功记深度物理PASS。 + + + +### 单文件故障（已有脚本） + +先停止本例写入并让测试进程静止；格式映射已确认后选择精确物理路径，不能只找第一个part。注入/恢复时 `--service-quiescent` 是操作者确认，工具不会替你停止共享服务。 + +```bash +hf apply --path "$TARGET_RELATIVE_PATH" --fault-id C02-data-middle \ + --kind bitflip --offset "$BYTE_OFFSET" --expected-sha256 "$BASELINE_SHA256" --service-quiescent +# 其他实例各用新故障ID：--kind missing / truncate / zero / header +hf snapshot --bucket "$B" +``` + +`truncate` 保留一半；`header` 翻转前32字节；bitflip 可指定任意有效offset；zero 若不会改变字节则拒绝。先持久化intent/原件，再变更，再见证；重复apply拒绝覆盖。备份在同run隔离目录，绝不在RustFS数据根中。 + +文件故障为 F1/F2/对象级F3，不是整盘故障。helper 明确禁止 `.rustfs.sys`、format、IAM、任意shell或设备操作。用例需要系统记录/特殊格式时走下面人工白名单后端。 + + + + + +### 格式/系统记录故障（人工白名单后端） + +先 `PROCESS stop`，从候选源码/真实task定位**本例**精确记录，生成 `record-plan.json`：绝对路径、run/task/bucket/incarnation、盘UUID、原sha/size、变更offset或替换源sha、允许新增/删除字段、备份和undo。第二人/批准的操作人逐项审核；禁止递归 `.rustfs.sys` 修改。 + +可操作的配方： + +1. **near-tail/长度不符**：复制目标part到本例quarantine并校验sha；在该目标副本上 `truncate -s "$NEW_SIZE" "$TARGET"`，其中 NEW_SIZE=原size−1或指定长度。核对新size/hash和原metadata未变。停止服务期间执行，目标路径来自已审核清单。 +2. **外来format/set**：同run的第二个隔离小集群正常启动产生format，停止两服务。仅把其format安装到本例空替换根，不改健康盘；记录原/外来id、this、sets。启动后期待安全拒绝，原外来format不能被改写。 +3. **未来format版本**：在本例替换根的format JSON副本，用 `jq` 只更改已核实的版本字段为不支持值；原件与变更diff归档。未来对象/checkpoint schema 按候选编解码结构更改对应版本头，不能随机翻位后称“未来schema”。如果存在校验，必须重算后证明结构可解析且schema未知；否则只计坏校验实例。 +4. **冲突代次/quorum**：串行合法写V1/V2或两个incarnation，分别保存全套本例副本；停服务后按预注册盘分配表安装各自不足R的集合。重启前用FORMAT逐盘核对，证明没有任一同版本quorum。 +5. **checkpoint/MRF坏记录**：只读保存 `.rustfs.sys/root-heal-.json` 与候选实际journal路径；确认属于本run。截断、替换为从本run早期阶段保留的陈旧代次、或单独更改校验字段。不得凭目录名删除整个queue。写失败通过IO_FAULT构造，不改权限后未经见证就称失败。 +6. **bucket metadata**：只列入对应本例桶的记录；建立同run健康备份，分别缺失/截断后重启/heal。原策略、tags、版本/lock配置通过S3核对，其他系统文件摘要不变。 +7. **inline偏移**：用匹配候选的codec在证据副本中定位目标version的inline map/value；保存解析offset/长度及反向映射，之后只修改该value。不会解析时不能任意翻xl.meta头冒充inline损坏；改用已明确定义的“坏metadata”实例，原inline实例仍未覆盖。 +8. **orphan/dangling**：通过正常完成/覆盖/删除形成历史目录，FORMAT证明所有活version/MPU引用后再测试remove。宽限期按真实时间/本例服务配置等待，不改主机时间。共享引用控制必须保留。 + +上述人工操作每一步均需精确原件备份、变更前后hash、操作返回值与复查；不能直接 `rm` 记录。用 `mv -n` 同盘移到隔离路径可保留原件，但必须核实源消失/目标摘要相同，因为 `mv -n` 跳过未必报错。恢复时先保留失败后新记录，若有新ACK/代次，禁止把旧metadata拷回去。 + + + +### 逻辑盘不可达、只读、权限 + +仅在已设置 PrivateMounts 的**本例服务 mount namespace**使用；先证明目标PID仍属于本例unit、`/proc/$PID/ns/mnt` 与host不同，且目标挂载非shared。不能进入host namespace卸载 `/data/rustfsN`。原理参考 [Linux nsenter](https://www.man7.org/linux/man-pages/man1/nsenter.1.html)。 + +只读配方（在目标节点，DATA为本例完整数据根）： + +```bash +sudo nsenter -t "$HEAL_PID" -m -- mount --bind "$DATA" "$DATA" +sudo nsenter -t "$HEAL_PID" -m -- mount -o remount,bind,ro "$DATA" +sudo nsenter -t "$HEAL_PID" -m -- findmnt -T "$DATA" -o TARGET,SOURCE,OPTIONS +# 以服务实际UID在该namespace做本例临时文件写探针，必须观察EROFS。 +# 解除仅此额外的bind层： +sudo nsenter -t "$HEAL_PID" -m -- umount "$DATA" +``` + +不可达用经批准的该服务 mount namespace 故障层覆盖此数据根（返回EIO的专用device-mapper设备，见IO_FAULT）；不能把空目录覆盖当作掉盘，因为可能触发新盘初始化。权限实例可使用本例非root UID +精确目录ACL拒绝，在相同UID实测EACCES；root服务单纯chmod不算有效注入。恢复后重新验证mount/磁盘身份和读写可达；在 namespace 内 helper 的跨mount拒绝是正确保护，不应关闭它。 + + + +### 新逻辑盘替换 + +停止本例unit及写入，校验 ROOT/data 与owner/run/mount；将**精确 data 根**同盘移至本例 quarantine 的唯一 old-disk 目录，新建同位置空data根。其他盘不动。重启后读取format，确认新disk UUID与slot；格式初始化失败要如实记录，不手动补出一个UUID。 + +```bash +# 仅已审核的本例 ROOT；先验证备份目标不存在，不递归删除任何目录。 +sudo mv -n -- "$ROOT/data" "$ROOT/quarantine/$FAULT_ID-old-disk" +sudo install -d -m 0700 -- "$ROOT/data" +``` + +检查旧根确实移走、摘要清单完整、服务无法访问quarantine；再正常启动。rebuild期间不得从old-disk复制part。失败救援时先停止unit并把新失败data根移至另一个保留目录，再恢复旧根；该恢复明确为rescue，不是heal结果。真实设备detach/attach仅属于另行批准的H6，不由此命令模拟。 + + + + +### 进程停止、崩溃、重启、升级 + +```bash +sudo systemctl show "$HEAL_UNIT" -p MainPID -p ExecMainStartTimestampMonotonic -p FragmentPath -p ActiveState +sudo systemctl stop "$HEAL_UNIT" +sudo systemctl start "$HEAL_UNIT" +# 独立 crash 实例，必须先 WINDOW、记录被杀的实际MainPID： +sudo systemctl kill --kill-whom=main --signal=SIGKILL "$HEAL_UNIT" +sudo systemctl show "$HEAL_UNIT" -p ExecMainCode -p ExecMainStatus -p Result +sudo systemctl reset-failed "$HEAL_UNIT" +sudo systemctl start "$HEAL_UNIT" +``` + +每次操作前核对 MainPID/exe 摘要/starttime，拒绝0或已替换PID，不 `pkill rustfs`。保存 `/proc/sys/kernel/random/boot_id`，SIGKILL前后boot_id相同，不能说断电。 + +UPGRADE 对本例节点逐一stop，用已经验证的第二个optimized release二进制替换该unit的exec路径（保留旧二进制），原env/endpoint/data不变，start后核对每个peer实际版本。旧新两份构建清单、包sha、库格式兼容表都须预注册；未知格式禁止直接回滚，改测安全拒绝。精确升级期间责任按CHECKPOINT验证，不依赖版本字符串判断兼容。 + + + +### 测试端口分区 / 依赖故障 + +优先预分配专用net namespace，证明它与host不同；需要业务节点连接的路由已配置，SSH仍走host。只对本例API/RPC端口或本例KMS/tier endpoint加规则。先保存规则/路由，设置独立超时恢复任务。例： + +```bash +sudo nsenter -t "$HEAL_PID" -n -- iptables -w -I OUTPUT 1 \ + -d "$TEST_PEER_IP" -p tcp --dport "$TEST_SERVICE_PORT" \ + -m comment --comment "$FAULT_ID" -j DROP +sudo nsenter -t "$HEAL_PID" -n -- iptables -w -nvL OUTPUT --line-numbers +# 相同namespace、完整相同规则删除，不flush任何链： +sudo nsenter -t "$HEAL_PID" -n -- iptables -w -D OUTPUT \ + -d "$TEST_PEER_IP" -p tcp --dport "$TEST_SERVICE_PORT" \ + -m comment --comment "$FAULT_ID" -j DROP +``` + +双向分区分别在绑定双方namespace应用，不能扩大到22、管理平面或host OUTPUT。记录丢包计数与双端探针。无独立net namespace或专项host规则批准时该实例BLOCKED，不能全局断网凑结果。 + + + + + +### 慢IO/EIO/容量与锁窗口 + +此组**只在预分配独占故障盘/有硬上限的专用loop设备**操作，绝不对 `/dev/nvme*` 共享生产设备运行。操作前记录backing file、loop/device-mapper名字、major:minor、mount UUID、上限和唯一FAULT_ID。设备创建/format需要本批单独的资源批准，不是case默认授权。 + +已分配专用 `HEAL_DM_NAME` / `HEAL_BACKING_DEV` 的device-mapper配方： + +```bash +sudo dmsetup table "$HEAL_DM_NAME" +sudo dmsetup suspend "$HEAL_DM_NAME" +sudo dmsetup reload "$HEAL_DM_NAME" --table "0 $HEAL_SECTORS delay $HEAL_BACKING_DEV 0 200" +sudo dmsetup resume "$HEAL_DM_NAME" +# EIO实例用完全返回error的表；只用于专用故障层： +sudo dmsetup suspend "$HEAL_DM_NAME" +sudo dmsetup reload "$HEAL_DM_NAME" --table "0 $HEAL_SECTORS error" +sudo dmsetup resume "$HEAL_DM_NAME" +``` + +恢复reload预先保存的精确原table，不猜linear偏移；在同服务namespace做定长读写见证（记录耗时/errno），并验证源设备可用。块设备EIO不等同应用层短读，短读实例使用FAULT truncate和真实读错误证据；若需“无内容变化的短读”必须有已分配的faultfs后端，另记能力条件。 + +ENOSPC仅写本例**硬上限卷**中的占位文件，先 `findmnt`/`df -B1` 和 inode计数确认不会耗尽外层XFS；按上限分块填充至返回ENOSPC并记录实际服务写失败。inode实例用创建空文件耗尽该卷的inode预算，不能在共享根创建无限文件。释放只删除已列举的本例占位文件或回收专用卷，不使用宽目录递归删除。 + +桶配额使用冻结rc的 `quota --help`，对本例桶 `quota set`/`quota info`；前后查询证明限额真实生效，结束恢复原配置。禁止把客户端本地限速当服务器quota。 + +```bash +"$RC_BIN" --json --no-color quota info "$RC_ALIAS/$B" +"$RC_BIN" --json --no-color quota set "$RC_ALIAS/$B" 64M +# 只在原配置没有quota且本例结束时清除；否则恢复原限额： +"$RC_BIN" --json --no-color quota clear "$RC_ALIAS/$B" +``` + +LOCK_WINDOW：慢IO层作用于一个实际持有对象写锁的请求，另发同key heal/PUT。用日志/原生runtime fence证明锁获取、超时、调用方退出、在途提交退出顺序；没有可观察锁窗口则黑盒实例为NOT_OBSERVED。客户端断开可由有界 `timeout` 结束**本次请求进程**，不杀服务；之后继续采集服务端在途状态。精确所有权证明依赖NATIVE，不能从一次超时推导锁已释放。 + + + +### 强制停机 + +只对H6已批准的精确Azure资源ID执行，先核验该VM独占、没有其他服务、磁盘持久化策略及带外恢复；记录当前boot_id和平台状态： + +```bash +az vm get-instance-view --ids "$APPROVED_EXCLUSIVE_VM_ID" +az vm stop --ids "$APPROVED_EXCLUSIVE_VM_ID" --skip-shutdown +az vm start --ids "$APPROVED_EXCLUSIVE_VM_ID" +az monitor activity-log list --resource-id "$APPROVED_EXCLUSIVE_VM_ID" \ + --start-time "$RUN_STARTED_AT" --end-time "$RUN_FINISHED_AT" +``` + +这是真实VM强制停止，不证明物理控制器掉电。物理主机power-cycle必须使用该独占机器已批准的带外控制台，记录BMC事件和电源周期；不能给未知机型编造通用命令。无授权不执行，也不把SIGKILL替代成通过。 + +## 6. 并发、Pool、ILM、复制及原生证据 + + + + + + +### 有界负载与观测 + +`trace.json` 是按序的S3操作数组，例如： + +```json +[ + {"operation":"put_object","params":{"Bucket":"rv-heal-20260910-a01","Key":"race-${worker}"},"size":5243003,"expectHttp":200}, + {"operation":"head_object","params":{"Bucket":"rv-heal-20260910-a01","Key":"race-${worker}"},"expectHttp":200} +] +``` + +```bash +hw writer run --trace trace.json --rounds 100 --workers 2 --interval 0.1 --duration 300 +``` + +同worker串行；不同worker用不同key可明确先后，测试真正同key竞争时人工按ACK/不确定结果建立期望。`${round}`、`${worker}` 只做JSON字符串替换，不eval。上传内容由run/worker/round/step确定，大小/摘要写入ACK。每次ACK立即独占写文件；意外错误停止后续入场并保留在途结果。不因服务返回错误而无限重试。 + +长稳分支增加已批准 `--duration` 至86400/259200、足够rounds及限速，并核对实际elapsed，提前耗尽rounds/字节预算不算跑满窗口；每批最多1,000,000请求/16worker/64GiB写入预算，可按预注册多个分段运行并保留连续时序。不可通过事后拼接零散短跑冒充连续24h。 + +Admin队列压力使用同样的限量并发方式调用hp（例如shell后台最多4个固定任务），先冻结请求集合，独立label保存每个返回；不能把S3负载当Admin admission覆盖。 + +```bash +# 在目标节点按固定频率采样，时间窗口与本例一致；不要采集全环境凭据。 +sudo journalctl -u "$HEAL_UNIT" --since "$RUN_STARTED_AT" --until "$RUN_FINISHED_AT" --no-pager +sudo systemctl show "$HEAL_UNIT" -p MemoryCurrent -p TasksCurrent -p CPUUsageNSec +pidstat -dru -p "$HEAL_PID" 1 60 +iostat -dx 1 60 +``` + +同时记录FD数量、目标IO、队列/持久记录及临时空间；工具未安装先预检阻断所需性能证据，不跑完才省略。性能ABBA每组四轮、至少三组：warmup、并发、内容、缓存/初始队列、负载窗口和阈值固定。分别统计GET/PUT P95/P99、错误率（所有失败/超时都在总分母）、修复吞吐、CPU/IO。没有预注册门限只能出测量报告，不能记性能通过。 + + + +### Pool交互 + +使用**本例H5服务**的rc alias，已有13项Pool回归结果仅为前置控制，不直接调用其会reset旧fleet的部署脚本。扩容/移除拓扑复用已审阅的endpoint分组方案，但根目录替换为本例heal命名空间。所有poolId从实际status和format映射获得。 + +```bash +"$RC_BIN" --json --no-color admin rebalance start "$RC_ALIAS" +"$RC_BIN" --json --no-color admin rebalance status "$RC_ALIAS" +"$RC_BIN" --json --no-color admin rebalance stop "$RC_ALIAS" +"$RC_BIN" --json --no-color admin decommission start "$RC_ALIAS" "$POOL_ID" --by-id +"$RC_BIN" --json --no-color admin decommission status "$RC_ALIAS" +"$RC_BIN" --json --no-color admin decommission cancel "$RC_ALIAS" "$POOL_ID" --by-id +"$RC_BIN" --json --no-color admin decommission clear "$RC_ALIAS" "$POOL_ID" --by-id +``` + +不是把以上所有操作依次跑完：按H01–H06操作卡在相应窗口执行。每次保存rc stdout/stderr/exitCode、相关Admin状态及前后manifest。退役写入限制沿用既定产品契约；仅survivor验证时源必须真实停机且配置移除。#2422/#2423的临界点是否实际触发单独登记。 + + + + +### ILM、远端tier、复制 + +使用本仓库已实现lane作为**控制实验**核对候选功能，并在本例独立服务重新建立专用桶/远端租户。配置JSON只含本例资源，API操作如下： + +```bash +hp ilm-put s3 --operation put_bucket_lifecycle_configuration --params-file lifecycle.json +hp ilm-get s3 --operation get_bucket_lifecycle_configuration --params-file bucket.json +hp restore s3 --operation restore_object --params-file restore-version.json --expect-http 202 +hp replication-put s3 --operation put_bucket_replication --params-file replication.json +hp replication-get s3 --operation get_bucket_replication --params-file bucket.json +``` + +`lifecycle.json` 使用已验证候选的真实期限；规则ID含run，Filter仅本例prefix，分别配置Expiration/NoncurrentVersionExpiration/Transition。不要用未合并的分钟级测试hook冒充release。远端tier的初始化依照当前rc `ilm tier add --help` 和已有 `scripts/validation_ilm/ilm_behavior.py` 的实际调用，填写本例endpoint/bucket/region与受控凭据；原始凭据不归档。 + +复制配置含本例role/destination ARN、Filter和DeleteMarkerReplication；先证明对端版本映射/删除语义。resync使用本批rc `replicate resync --help` 所支持的实际命令，保存启动ID和两端状态。观察真实重叠窗口；没有tier/第二端，相关分支BLOCKED而非删除该项。结束通过API删除本例lifecycle/replication配置，远端本例数据按版本清单清理，不改共享服务规则。 + +已核对参考rc的resync操作（仍须新批help核验）： + +```bash +"$RC_BIN" --json --no-color replicate resync start "$RC_ALIAS/$B" \ + --target-arn "$TEST_TARGET_ARN" --reset-id "$RUN" --yes +"$RC_BIN" --json --no-color replicate resync status "$RC_ALIAS/$B" --target-arn "$TEST_TARGET_ARN" +``` + + + +### 精确屏障与原生回归补证 + +F9类窗口不能靠普通release黑盒保证命中。使用同源码SHA的**独立native回归二进制**，绝不换掉被测optimized release服务。先列出真实测试并固定完整测试名、crate、源码位置和预期屏障事件： + +```bash +cargo test -p rustfs-heal -- --list +cargo test -p rustfs-ecstore -- --list +cargo test -p e2e_test -- --list +# NATIVE_TEST必须逐字来自上面列表，日志必须显示恰好1项且屏障命中： +cargo test -p "$NATIVE_CRATE" "$NATIVE_TEST" -- --exact --nocapture +``` + +已有replacement回归函数包含 `test_cluster_root_heal_recovers_remote_shards_after_background_target_restart`、`..._crash`、EC8+4和multi-set版本，完整限定名从该commit --list取得。它们默认创建本地E2E临时目录，**不是远端真实独立盘部署器**；不通过环境变量把这些会清理临时目录的测试指到现有fleet。原生锁/代次/checkpoint测试按所选操作卡在同commit源码中找到精确回归和屏障标记后运行，不存在对应屏障的分支要保留“需源侧增加确定性测试”的能力缺口，不能给计划生成器喂手写JSON声称measured。 + +归档 candidate SHA、native测试源码SHA、完整命令、测试数量、exitCode、运行时间及屏障事件。native与fleet分别计分；缺失精确native证据不影响已经完成的黑盒分支真实性，但不能宣布该case所有分支齐全。 + +## 7. 恢复、报告和安全交还 + + + +先保存出错时的状态、日志和PHYSICAL，停止本例负载/任务/进程；再按fault receipt撤销规则/只读层/替换根。对已失效lease允许受限恢复，不允许继续注入。单文件恢复： + +```bash +hf recover --fault-id C02-data-middle --service-quiescent +``` + +只恢复同run的备份，原件摘要不符则拒绝；目标如果已经是原始正确字节不重写；如果变成既不是原始也不是注入后字节（可能有新ACK或合法metadata重写），拒绝盲目覆盖，交人工保留现场。恢复工具输出 `physicalRepairPassAllowed=false`；任何 `restoredFromBackup=true` 都不能作为产品补片证据。 + +强制故障设置第二条恢复通道/超时撤销；本地 finally 不是网络或进程被杀时可靠的唯一恢复办法。退出前枚举本例网络规则、bind层、dm表、测试unit、临时IAM和未完成MPU，逐条撤销并复核。保留失败数据/backup，retention未到期则保留专用资源；记录剩余路径/到期安排。未恢复为 DIRTY/QUARANTINED，不继续下一用例。 + +每个**实际参数实例**都填写人工结论和证据索引，不只填一个基础case总绿： + +```text +preflight/ 基础设施/包/rc/harness/lease/拓扑/预注册参数 +baseline/ S3版本清单、物理基线、ACK、桶配置 +fault/ 精确plan、intent、原件摘要、故障有效性见证 +trigger/ 原始Admin/GET/后台来源及活动窗口 +control/ token/epoch/coverage/counter/负向错误/游标原始响应 +physical/ after/restart分片、metadata映射、独立重建对照 +s3/ 所有应存/应删版本与ACK解析、完整内容和配置 +recovery/ 故障解除、停止服务、retained路径与安全交还 +logs/ 全部相关节点的有界日志/资源/平台或native事件 +``` + +人工结论为 MANUAL_PASS/FAIL/INFRA_ERROR/BLOCKED/NOT_OBSERVED，列评审人、每条断言及引用。`evidence.json`包含runId、instanceId、九类evidence的数组，每个引用为相对于该实例目录的`path`和`sha256`。全部分支完成才能评审基础case；缺资源/窗口未命中/证据不全都不是PASS。 + +```bash +python scripts/validation_heal/matrix.py audit --plan reports/validation-heal-84-plan.json \ + --evidence-dir reports/heal-84 --out reports/heal-84-evidence-audit.json +``` + +audit检查完整性/路径/摘要，输出`EVIDENCE_COMPLETE_REVIEW_REQUIRED`仍不是产品通过报告。原始失败批次永久保留，修复后新批次另记。本文提供全84项的可操作方案，但实际机器、专用依赖和native屏障的可用性必须单独核验，不能把“方案就绪”改写成“环境就绪”或“已实测”。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index a19ad4cd..34a4dce4 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -50,14 +50,18 @@ live PASS evidence in RustFS Actions runs `38044956285`, `38045047518`, `ilm.delete-all-versions`, `ilm.delete-marker-expiration`, `ilm.multipart-expiration`, and `ilm.newer-noncurrent-retention` also passed live in runs `38049740183`, `38049817172`, `38049890130`, and `38049965929`. -The remaining four ILM executors still require live evidence. +`ilm.noncurrent-expiration`, `ilm.restart-persistence`, +`ilm.transition-and-restore`, and `ilm.versioned-current-expiration` passed in +runs `38050040606`, `38050434607`, `38050720293`, and `38051148291`. +All 16 ILM/scanner cases now have live PASS evidence. All 13 Pool lifecycle IDs now dispatch to the source black-box pool checks. The driver reuses the existing auto-testing Pool suite's two-node seed and expansion operations, supplies marker-guarded reset/topology/restart/physical census hooks, and requires source case evidence plus physical cleanup before canonical PASS. `pool.status` passed live in RustFS Actions run `38047001285`; -the other twelve Pool cases still require live evidence. +`pool.decommission` also passed in run `38051227313`; the other eleven Pool +cases still require live evidence. The 18 bucket-replication and 20 site-replication IDs now dispatch to the source black-box checks. The driver uses the existing auto-testing four-node @@ -84,13 +88,11 @@ Actions run `38042759074`. on the Tier suite's `single-single` topology. Event delivery is counted for a unique object after a quiet window; audit checks matching PUT/GET records, distinct request IDs, successful status, and absence of known secrets. -`events.delivery` passed live in RustFS Actions run `38046470999`. The audit -attempt in run `38046574073` failed its secret check because the environment -used the same value for the public access key and secret key. The audit case -now generates a distinct secret key before deployment. Run `38046860333` -exposed an executor parsing mismatch: the candidate emits `s3:PutObject` and -`s3:GetObject`. The parser now recognizes those exact names and waits for both -operations on the selected object; the case still requires a live rerun. +`events.delivery` passed live in RustFS Actions run `38046470999`. +`audit.request-correlation` passed in run `38050138583` with the candidate's +`Records` envelope and exact S3 operation names. The audit fixture uses a +distinct random secret and retains the secret-leak, request-ID, +object-identity and successful-status assertions. `oidc.login-and-session` uses the Security suite's current single-node deployment and a run-owned Keycloak container with a pinned image. It performs @@ -99,8 +101,10 @@ for STS credentials, and checks the readonly identity policy plus a narrower session policy on existing objects. Missing/invalid session tokens, tampered identity tokens, and wrong audiences must fail. Fixtures, containers, and the RustFS deployment must all be cleaned before PASS. Runtime credentials are -excluded from evidence. This executor does not require a RustFS checkout and -still requires live validation. +excluded from evidence. This executor does not require a RustFS checkout. +Its complete live candidate run passed in RustFS Actions run `38051425083`. +Policy denials require `AccessDenied`; absent or mismatched temporary session +tokens require the candidate authentication contract's `InvalidRequest`. `iam.policy-enforcement` has a dedicated executor on the Security suite's `single-single` deployment. It uses one principal with a policy that allows @@ -128,10 +132,24 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 74 currently dispatchable IDs are the S3, two scanner, -KMS, tiering, events, audit, IAM, 14 additional ILM, 13 Pool, and 38 -replication cases and OIDC above. Nineteen have live PASS evidence; the other 139 IDs -remain unverified. +case evidence. The 75 currently dispatchable IDs include all 74 non-Heal +cases and `heal.admin-healthy-noop`. Twenty-six have live PASS evidence; +the other 132 remain unverified. + +All 84 Heal recipes, 301 operator branches and 532 applicable topology branch +templates are preserved under `scripts/validation_heal`, with the public +contract, operation book, signed probe, ACK-logged workload, guarded file-fault +tool and source-bound metadata inspector source. `matrix.py plan` binds current +auto-testing SSH nodes and required resources; its audit checks every branch +and nine evidence stages without manufacturing PASS. +`heal.admin-healthy-noop` now has an automatic H1/H2 executor using the existing +Heal suite's immutable deb download, installed binary digest, independent disk +mounts, run-owned services/data roots, exact normal/deep responsibility +receipts, repeated status, physical hashes and complete S3 inventories. It +still requires live validation. The other 83 Heal cases remain +operator-assisted and are not claimed as migrated automatic executors. +H4 needs at least 24 independent disk endpoints; H6 and native-fence branches +retain their platform and source-bound evidence requirements. The eight planned cases have these concrete reuse candidates: @@ -141,9 +159,9 @@ The eight planned cases have these concrete reuse candidates: | `kms.sse-kms-round-trip` | KMS-104, KMS-301, KMS-214 | Dedicated executor and live PASS evidence. | | `tiering.transition-and-restore` | TIER-105, TIER-107 | Dedicated executor and live PASS evidence. | | `events.delivery` | EVT-103, EVT-104 | Dedicated executor and live PASS evidence. | -| `audit.request-correlation` | AUD-102, AUD-104, AUD-106 | Dedicated executor; live evidence pending. | +| `audit.request-correlation` | AUD-102, AUD-104, AUD-106 | Dedicated executor and live PASS evidence. | | `iam.policy-enforcement` | IAM-105, IAM-109 | Dedicated executor and live PASS evidence. | -| `oidc.login-and-session` | OIDC-103 | Dedicated executor using real Keycloak login and STS session boundaries; live evidence pending. | +| `oidc.login-and-session` | OIDC-103 | Dedicated executor using real Keycloak login and STS session boundaries; live PASS evidence. | | `s3.basic-object-round-trip` | S3C-101, 201, 203, 205, 206, 207 | Dedicated executor and live PASS evidence. | Migration work on an ID is complete only when its result contains the exact diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 9d58634b..864f0e47 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,6 +29,8 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; + heal.admin-healthy-noop) + SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 exit 2 ;; diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py new file mode 100644 index 00000000..e96b85a6 --- /dev/null +++ b/scripts/validation_heal/fleet_node.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Run-owned Heal deployment and physical evidence on current auto-testing nodes. + +The runner sends this module and a structured request over SSH stdin. Paths, +units and mounts are derived here; no shell program or fault path is accepted. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import socket +import stat +import subprocess + + +MOUNTS = tuple(Path(f"/data/rustfs{index}") for index in range(1, 5)) +STATE_ROOT = Path("/var/lib/rustfs-auto-testing/heal") +RUN_PATTERN = re.compile(r"gh-[1-9][0-9]*-[1-9][0-9]*-(?:h1|h2)") +BACKGROUND_CONFIG = {"RUSTFS_SCANNER_ENABLED": "false", "RUSTFS_HEAL_ENABLED": "true", + "RUSTFS_HEAL_AUTO_HEAL_ENABLE": "false", "RUSTFS_HEAL_MRF_ENABLE": "false"} + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def command(*args, check=True): + result = subprocess.run(args, capture_output=True, text=True, timeout=90) + require(not check or result.returncode == 0, f"{args[0]} failed: {result.stderr[-1000:]}") + return result + + +def checked(path, boundary): + require(path.is_absolute() and path.is_relative_to(boundary) and ".." not in path.parts, + "path is outside the run boundary") + for entry in (path, *path.parents): + require(not entry.is_symlink(), "symlink in run path") + require(path.resolve() == path, "noncanonical run path") + if boundary.exists(): + for entry in (path, *path.parents): + if entry == boundary: + break + require(not entry.exists() or entry.stat().st_dev == boundary.stat().st_dev, + "nested mount in run path") + return path + + +def digest(path): + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size <= 2 * 1024**3, + "expected bounded regular file") + result = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024**2): + result.update(chunk) + return result.hexdigest() + + +def paths(run_id): + require(bool(RUN_PATTERN.fullmatch(run_id)), "invalid auto-testing Heal run ID") + state = checked(STATE_ROOT / run_id, STATE_ROOT) + roots = tuple(checked(mount / "heal-validation" / run_id, mount) for mount in MOUNTS) + return state, roots + + +def unit(run_id): + paths(run_id) + return "rustfs-validation-heal-" + run_id + ".service" + + +def mounts(): + observed = [] + for mount in MOUNTS: + checked(mount, mount) + entries = json.loads(command("findmnt", "--json", "--mountpoint", str(mount), "--output", + "TARGET,SOURCE,FSTYPE,UUID,MAJ:MIN,OPTIONS").stdout).get("filesystems", []) + require(len(entries) == 1, "four explicit disk mountpoints are required") + value = entries[0] + require(value["target"] == str(mount) and value.get("uuid") and value.get("maj:min") + and "rw" in value["options"].split(","), "disk mount identity or write capability is missing") + observed.append(value) + require(len({value["uuid"] for value in observed}) == 4 + and len({value["maj:min"] for value in observed}) == 4, + "four independent drives are required; directories on one drive do not qualify") + return observed + + +def preflight(request): + state, roots = paths(request["runId"]) + require(not state.exists() and all(not root.exists() for root in roots), "run namespace already exists") + observed = mounts() + require(all(shutil.disk_usage(mount).free >= 1024**3 for mount in MOUNTS), "1 GiB free per drive required") + return {"hostname": socket.gethostname(), "mounts": observed, "statePath": str(state), + "dataRoots": [str(root / "data") for root in roots], "mutated": False} + + +def reserve(request): + run_id = request["runId"] + observed = preflight(request) + require(observed["hostname"] == request["hostname"] and observed["mounts"] == request["mounts"], + "preflight host or disk identities changed") + require(command("systemctl", "is-active", "--quiet", "rustfs", check=False).returncode != 0, + "shared RustFS service must be stopped before allocating port 9000") + listeners = command("ss", "-H", "-lnt").stdout + require(not any(len(parts := line.split()) >= 4 and parts[3].endswith(":9000") + for line in listeners.splitlines()), "port 9000 remains occupied") + binary_sha = digest(Path("/usr/bin/rustfs")) + require(binary_sha == request["binarySha256"], "installed candidate binary differs") + for name in ("access", "secret"): + require(bool(re.fullmatch(r"[A-Za-z0-9]{16,80}", request[name])), "invalid runtime credential") + volumes = request["volumes"] + require(isinstance(volumes, list) and len(volumes) in {4, 16}, "unsupported automatic Heal topology") + local_paths = [str(root / "data") for root in paths(run_id)[1]] + h1 = run_id.endswith("-h1") + expected = local_paths if h1 else [f"http://rustfs-node{node}:9000{path}" + for node in range(1, 5) for path in local_paths] + require(volumes == expected, "endpoint sequence differs from H1/H2 deployment contract") + state, roots = paths(run_id) + owner = {"runId": run_id, "unit": unit(run_id), "hostname": observed["hostname"], + "mounts": observed["mounts"], "binarySha256": binary_sha, + "bucket": request["bucket"], "volumes": volumes} + require(bool(re.fullmatch(r"rv-heal-[a-f0-9]{16}", owner["bucket"])), "invalid run bucket") + state.mkdir(parents=True, mode=0o700) + (state / "owner.json").write_text(json.dumps(owner)) + for root in roots: + root.mkdir(parents=True, mode=0o700) + (root / "owner.json").write_text(json.dumps(owner)) + (root / "data").mkdir(mode=0o700) + (root / "quarantine").mkdir(mode=0o700) + shutil.copyfile("/usr/bin/rustfs", state / "rustfs") + (state / "rustfs").chmod(0o500) + require(digest(state / "rustfs") == binary_sha, "copied candidate binary differs") + config = {**BACKGROUND_CONFIG, "RUSTFS_ACCESS_KEY": request["access"], "RUSTFS_SECRET_KEY": request["secret"], + "RUSTFS_ADDRESS": ":9000", "RUSTFS_CONSOLE_ENABLE": "false", + "RUSTFS_HEAL_TASK_TIMEOUT_SECS": "300", "RUSTFS_OBS_LOGGER_LEVEL": "info"} + env_file = state / "service.env" + descriptor = os.open(env_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w") as stream: + stream.write("\n".join(f"{key}={value}" for key, value in config.items()) + "\n") + return {"owner": owner, "reserved": True} + + +def owned(run_id): + state, roots = paths(run_id) + marker = checked(state / "owner.json", state) + require(marker.stat().st_uid == 0 and marker.stat().st_nlink == 1, "root-owned marker required") + owner = json.loads(marker.read_text()) + require(owner["runId"] == run_id and owner["unit"] == unit(run_id) + and owner["hostname"] == socket.gethostname(), "run owner or hostname differs") + current = mounts() + for actual, previous in zip(current, owner["mounts"], strict=True): + require(all(actual[key] == previous[key] for key in ("target", "uuid", "maj:min", "source", "fstype")), + "disk identity changed") + for root in roots: + marker = checked(root / "owner.json", root) + require(marker.stat().st_uid == 0 and json.loads(marker.read_text()) == owner, "drive owner differs") + return state, roots, owner + + +def start(request): + state, _, owner = owned(request["runId"]) + require(digest(state / "rustfs") == owner["binarySha256"], "candidate changed before start") + command("systemd-run", "--unit=" + owner["unit"], "--property=Type=exec", + "--property=KillMode=control-group", "--property=TimeoutStopSec=30", + "--property=RuntimeMaxSec=1800", "--property=MemoryMax=4G", + "--property=WorkingDirectory=" + str(state), + "--property=EnvironmentFile=" + str(state / "service.env"), str(state / "rustfs"), *owner["volumes"]) + return {"unit": owner["unit"], "started": True} + + +def snapshot(request): + state, roots, owner = owned(request["runId"]) + pid = int(command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"]).stdout.strip()) + require(pid > 0, "run-owned service has no live PID") + executable = Path(f"/proc/{pid}/exe").resolve(strict=True) + require(executable == state / "rustfs" and digest(executable) == owner["binarySha256"], + "running binary differs from candidate") + environment = dict(value.split("=", 1) for value in Path(f"/proc/{pid}/environ").read_bytes().decode().split("\0") + if "=" in value) + require(all(environment.get(key) == value for key, value in BACKGROUND_CONFIG.items()), + "background repair isolation configuration differs") + files, formats = [], [] + for index, root in enumerate(roots): + data = root / "data" + folder = checked(data / owner["bucket"], data) + if folder.exists(): + for directory, children, names in os.walk(folder, followlinks=False): + for name in (*children, *names): + checked(Path(directory) / name, data) + for name in sorted(names): + path = Path(directory) / name + require(len(files) < 4096 and path.stat().st_size <= 64 * 1024**2, "fixture exceeds physical evidence budget") + files.append({"disk": index, "path": str(path.relative_to(data)), + "size": path.stat().st_size, "sha256": digest(path)}) + path = checked(data / ".rustfs.sys/format.json", data) + require(path.stat().st_size < 65536, "disk format exceeds budget") + formats.append({"disk": index, "sha256": digest(path), "value": json.loads(path.read_text())}) + return {"hostname": owner["hostname"], "unit": owner["unit"], "pid": pid, + "binarySha256": owner["binarySha256"], "backgroundConfig": BACKGROUND_CONFIG, + "files": sorted(files, key=lambda value: (value["disk"], value["path"])), "formats": formats} + + +def logs(request): + _, _, owner = owned(request["runId"]) + return {"unit": owner["unit"], "journal": command("journalctl", "-u", owner["unit"], "--no-pager", "-n", "2000").stdout} + + +def cleanup(request): + state, roots, owner = owned(request["runId"]) + command("systemctl", "stop", owner["unit"], check=False) + pid = command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"], check=False).stdout.strip() + require(pid in {"", "0"}, "run-owned service remains active") + command("systemctl", "reset-failed", owner["unit"], check=False) + for root in (*roots, state): + for directory, children, names in os.walk(root, followlinks=False): + for name in (*children, *names): + checked(Path(directory) / name, root) + shutil.rmtree(root) + require(not state.exists() and all(not root.exists() for root in roots), "run namespace remains") + return {"cleanup": "PASS", "unitStopped": True, "rootsRemoved": True, "hostname": owner["hostname"]} + + +def main(request): + require(os.geteuid() == 0, "node adapter requires passwordless sudo") + actions = {"preflight": preflight, "reserve": reserve, "start": start, "snapshot": snapshot, + "logs": logs, "cleanup": cleanup} + require(request.get("action") in actions, "unsupported node operation") + print(json.dumps(actions[request["action"]](request))) diff --git a/scripts/validation_heal/heal_fault.py b/scripts/validation_heal/heal_fault.py new file mode 100644 index 00000000..5e3ade73 --- /dev/null +++ b/scripts/validation_heal/heal_fault.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Linux-only, descriptor-relative file faults in a dedicated Heal run root. + +No arbitrary shell, disk formatting, shared mount changes, or internal system +metadata. Keep the service quiescent while applying/undoing a file fault. +""" + +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from datetime import UTC, datetime +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import sys + + +LIMIT = 64 * 1024**2 +ROOT_PATTERN = re.compile(r"/data/rustfs[1-9][0-9]*/heal-validation/((?:gh|manual)-[a-z0-9-]{5,60})") +FAULTS = ("missing", "bitflip", "truncate", "zero", "header") + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def digest(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def mount_id(fd: int) -> int: + text = Path(f"/proc/self/fdinfo/{fd}").read_text() + values = re.findall(r"^mnt_id:\s*([0-9]+)$", text, re.MULTILINE) + require(len(values) == 1, "Linux mount identity unavailable") + return int(values[0]) + + +def target_parts(relative: str, prefix: str) -> list[str]: + parts = relative.split("/") + require(3 <= len(parts) <= 48 and all(part not in {"", ".", ".."} for part in parts), + "invalid relative object path") + require(parts[0] == prefix or parts[0].startswith(prefix + "-"), "target bucket not owned by run") + require(not any(part.startswith(".rustfs") for part in parts), "system metadata faults are prohibited") + require(parts[-1] == "xl.meta" or bool(re.fullmatch(r"part\.[1-9][0-9]*", parts[-1])), + "only explicit object xl.meta/part files are supported") + return parts + + +@contextmanager +def directory(path: Path): + require(path.is_absolute() and ".." not in path.parts, "noncanonical root") + fd = os.open("/", os.O_RDONLY | os.O_DIRECTORY) + try: + for part in path.parts[1:]: + child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + os.close(fd) + fd = child + yield fd + finally: + os.close(fd) + + +@contextmanager +def parent(root_fd: int, parts: list[str]): + fd = os.dup(root_fd) + expected_mount = mount_id(root_fd) + try: + for part in parts: + child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + os.close(fd) + fd = child + require(mount_id(fd) == expected_mount, "nested mount or bind-mount escape") + yield fd + finally: + os.close(fd) + + +def read_file(fd: int, name: str, limit: int = LIMIT) -> bytes: + opened = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd) + try: + info = os.fstat(opened) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size <= limit, + "expected bounded regular non-hardlinked file") + require(mount_id(opened) == mount_id(fd), "file bind-mount escape") + with os.fdopen(opened, "rb", closefd=False) as stream: + value = stream.read(limit + 1) + after = os.fstat(opened) + require(len(value) <= limit and info.st_size == len(value) + and (info.st_size, info.st_mtime_ns, info.st_ctime_ns) + == (after.st_size, after.st_mtime_ns, after.st_ctime_ns), "file changed while reading") + return value + finally: + os.close(opened) + + +def new_file(fd: int, name: str, value: bytes) -> None: + opened = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=fd) + with os.fdopen(opened, "wb") as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + os.fsync(fd) + + +def encode(value: dict) -> bytes: + return (json.dumps(value, ensure_ascii=False, indent=2) + "\n").encode() + + +def mutation(value: bytes, kind: str, offset: int | None = None) -> bytes | None: + require(kind in FAULTS, "unknown fault") + if kind == "missing": + return None + require(bool(value), "empty file cannot witness this fault") + if kind == "truncate": + return value[:len(value) // 2] + if kind == "zero": + changed = bytes(len(value)) + elif kind == "header": + changed = bytes(byte ^ 0xFF for byte in value[:32]) + value[32:] + else: + position = len(value) // 2 if offset is None else offset + require(type(position) is int and 0 <= position < len(value), "bitflip offset outside file") + changed = value[:position] + bytes([value[position] ^ 1]) + value[position + 1:] + require(changed != value, "fault would not change the target") + return changed + + +def checked_owner(root_fd: int, run_id: str, recovery: bool = False) -> dict: + owner = json.loads(read_file(root_fd, "fault-owner.json", 16384)) + require(owner.get("runId") == run_id and owner.get("schemaVersion") == 1, "owner mismatch") + require(owner.get("mountId") == mount_id(root_fd) and owner.get("device") == os.fstat(root_fd).st_dev, + "mount identity changed; rebind only after an operator audit") + expiry = datetime.fromisoformat(owner["leaseExpiresAt"].replace("Z", "+00:00")) + require(expiry.tzinfo is not None, "lease must have timezone") + require(recovery or expiry > datetime.now(UTC), "fault lease expired") + return owner + + +def apply_fault(root_fd: int, owner: dict, relative: str, fault_id: str, kind: str, + expected_sha: str, offset: int | None = None) -> dict: + parts = target_parts(relative, owner["bucketPrefix"]) + require(bool(re.fullmatch(r"[A-J][0-9]{2}-[a-z0-9-]{1,48}", fault_id)), "invalid fault ID") + require(bool(re.fullmatch(r"[0-9a-f]{64}", expected_sha)), "baseline SHA-256 required") + with parent(root_fd, ["data", *parts[:-1]]) as target_fd, parent(root_fd, ["quarantine"]) as backup_fd: + before = read_file(target_fd, parts[-1]) + require(digest(before) == expected_sha, "target differs from frozen baseline") + changed = mutation(before, kind, offset) + receipt = {"schemaVersion": 1, "runId": owner["runId"], "faultId": fault_id, + "relativePath": relative, "kind": kind, "offset": offset, + "before": {"sha256": digest(before), "size": len(before)}, + "fault": None if changed is None else {"sha256": digest(changed), "size": len(changed)}, + "createdAt": datetime.now(UTC).isoformat(), "backup": fault_id + ".original", + "backend": "logical-file-fault", "requiresQuiescentService": True} + # Intent and recoverable bytes precede the mutation. An interrupted + # apply remains recoverable even when no completion receipt was emitted. + new_file(backup_fd, fault_id + ".intent.json", encode(receipt)) + new_file(backup_fd, receipt["backup"], before) + require(read_file(target_fd, parts[-1]) == before, "target changed before injection") + if changed is None: + os.unlink(parts[-1], dir_fd=target_fd) + else: + temporary = ".heal-fault-" + fault_id + new_file(target_fd, temporary, changed) + os.rename(temporary, parts[-1], src_dir_fd=target_fd, dst_dir_fd=target_fd) + os.fsync(target_fd) + if changed is None: + require(not os.path.lexists(f"/proc/self/fd/{target_fd}/{parts[-1]}"), "missing fault not observed") + else: + require(read_file(target_fd, parts[-1]) == changed, "changed bytes not observed") + receipt["witnessedAt"] = datetime.now(UTC).isoformat() + new_file(backup_fd, fault_id + ".applied.json", encode(receipt)) + return receipt + + +def recover_fault(root_fd: int, owner: dict, fault_id: str) -> dict: + require(bool(re.fullmatch(r"[A-J][0-9]{2}-[a-z0-9-]{1,48}", fault_id)), "invalid fault ID") + with parent(root_fd, ["quarantine"]) as backup_fd: + receipt = json.loads(read_file(backup_fd, fault_id + ".intent.json", 16384)) + require(receipt["runId"] == owner["runId"] and receipt["faultId"] == fault_id + and receipt["backup"] == fault_id + ".original", "recovery receipt mismatch") + parts = target_parts(receipt["relativePath"], owner["bucketPrefix"]) + before = read_file(backup_fd, receipt["backup"]) + require(digest(before) == receipt["before"]["sha256"] and len(before) == receipt["before"]["size"], + "backup corrupted; preserve scene") + with parent(root_fd, ["data", *parts[:-1]]) as target_fd: + try: + current = read_file(target_fd, parts[-1]) + except FileNotFoundError: + current = None + restored = current != before + if restored: + # A new generation or a legitimate metadata rewrite is not + # ours to overwrite. Preserve it for an operator-led rescue. + expected_fault = receipt["fault"] + require(current is None or expected_fault is not None + and len(current) == expected_fault["size"] + and digest(current) == expected_fault["sha256"], + "target changed after fault; refusing to overwrite possible acknowledged data") + # Never erase post-failure bytes. Exact recovery is a rescue, + # not proof that the product healed the target. + if current is not None: + preserved = fault_id + ".failed-" + digest(current) + try: + new_file(backup_fd, preserved, current) + except FileExistsError: + require(read_file(backup_fd, preserved) == current, "preserved bytes differ") + temporary = ".heal-recovery-" + fault_id + try: + new_file(target_fd, temporary, before) + except FileExistsError: + require(read_file(target_fd, temporary) == before, "incomplete recovery temporary") + os.rename(temporary, parts[-1], src_dir_fd=target_fd, dst_dir_fd=target_fd) + os.fsync(target_fd) + require(read_file(target_fd, parts[-1]) == before, "recovery verification failed") + return {"runId": owner["runId"], "faultId": fault_id, "restoredFromBackup": restored, + "physicalRepairPassAllowed": False, "retainedBackup": receipt["backup"], + "recoveredAt": datetime.now(UTC).isoformat()} + + +def snapshot(root_fd: int, owner: dict, bucket: str) -> dict: + require(bucket == owner["bucketPrefix"] or bucket.startswith(owner["bucketPrefix"] + "-"), + "snapshot bucket outside run") + require(bool(re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket)), "invalid bucket") + files, total = [], 0 + + def walk(fd: int, relative: list[str], depth: int = 0) -> None: + nonlocal total + require(depth <= 48, "snapshot depth exceeds budget") + for name in sorted(os.listdir(fd)): + info = os.stat(name, dir_fd=fd, follow_symlinks=False) + if stat.S_ISDIR(info.st_mode): + with parent(fd, [name]) as child: + walk(child, [*relative, name], depth + 1) + else: + value = read_file(fd, name) + total += len(value) + require(total <= 8 * 1024**3 and len(files) < 10000, "snapshot budget exceeded") + files.append({"path": "/".join([*relative, name]), "size": len(value), "sha256": digest(value)}) + + with parent(root_fd, ["data", bucket]) as bucket_fd: + walk(bucket_fd, [bucket]) + return {"runId": owner["runId"], "bucket": bucket, "files": files, "bytes": total, + "capturedAt": datetime.now(UTC).isoformat(), "decodedMetadata": False, + "physicalRepairPassAllowed": False} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--run-id", required=True) + sub = parser.add_subparsers(dest="action", required=True) + init = sub.add_parser("init") + init.add_argument("--bucket-prefix", required=True) + init.add_argument("--lease-expires-at", required=True) + inspect = sub.add_parser("inspect") + inspect.add_argument("--path", required=True) + capture = sub.add_parser("snapshot") + capture.add_argument("--bucket", required=True) + apply = sub.add_parser("apply") + apply.add_argument("--path", required=True) + apply.add_argument("--fault-id", required=True) + apply.add_argument("--kind", choices=FAULTS, required=True) + apply.add_argument("--expected-sha256", required=True) + apply.add_argument("--offset", type=int) + apply.add_argument("--service-quiescent", action="store_true", required=True) + recover = sub.add_parser("recover") + recover.add_argument("--fault-id", required=True) + recover.add_argument("--service-quiescent", action="store_true", required=True) + args = parser.parse_args() + require(sys.platform == "linux" and os.geteuid() == 0, "run only with sudo on the Linux jump-host target") + match = ROOT_PATTERN.fullmatch(str(args.root)) + require(match is not None and match.group(1) == args.run_id, "root is not a dedicated Heal run root") + with directory(args.root) as root_fd: + require(os.fstat(root_fd).st_uid == 0 and os.fstat(root_fd).st_mode & 0o022 == 0, + "run root must be root-owned and not group/world writable") + if args.action == "init": + require(bool(re.fullmatch(r"rv-heal-[a-z0-9-]{6,35}", args.bucket_prefix)), "invalid bucket prefix") + expiry = datetime.fromisoformat(args.lease_expires_at.replace("Z", "+00:00")) + require(expiry.tzinfo is not None and expiry > datetime.now(UTC), "invalid lease") + # Existing run ownership must already be provisioned by the lab + # operator (or pilot setup). This tool never provisions a data disk. + existing = json.loads(read_file(root_fd, "owner.json", 16384)) + require(existing.get("runId") == args.run_id, "missing lab ownership marker") + with parent(root_fd, ["data"]), parent(root_fd, ["quarantine"]): + pass + result = {"schemaVersion": 1, "runId": args.run_id, "bucketPrefix": args.bucket_prefix, + "leaseExpiresAt": args.lease_expires_at, "mountId": mount_id(root_fd), + "device": os.fstat(root_fd).st_dev} + new_file(root_fd, "fault-owner.json", encode(result)) + else: + owner = checked_owner(root_fd, args.run_id, recovery=args.action == "recover") + if args.action == "snapshot": + result = snapshot(root_fd, owner, args.bucket) + elif args.action == "inspect": + parts = target_parts(args.path, owner["bucketPrefix"]) + with parent(root_fd, ["data", *parts[:-1]]) as target_fd: + value = read_file(target_fd, parts[-1]) + result = {"runId": args.run_id, "path": args.path, "size": len(value), "sha256": digest(value)} + elif args.action == "apply": + result = apply_fault(root_fd, owner, args.path, args.fault_id, args.kind, + args.expected_sha256, args.offset) + else: + result = recover_fault(root_fd, owner, args.fault_id) + print(json.dumps(result, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (ValueError, OSError) as error: + print(json.dumps({"state": "INFRA_ERROR", "error": str(error)}), file=sys.stderr) + raise SystemExit(1) from error diff --git a/scripts/validation_heal/heal_probe.py b/scripts/validation_heal/heal_probe.py new file mode 100644 index 00000000..4c0af320 --- /dev/null +++ b/scripts/validation_heal/heal_probe.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python3 +"""Bounded S3/Admin evidence collector for operator-assisted Heal fleet runs. + +This is an operation tool, not an 84-case PASS generator. Run on the jump-host +runner, using a reviewed, expiring lab binding and credentials from environment. +""" + +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from datetime import UTC, datetime +import hashlib +import json +import os +from pathlib import Path +import re +import time +import uuid +from urllib import request +from urllib.error import HTTPError +from urllib.parse import unquote, urlencode, urlsplit + + +MAX_BODY = 64 * 1024**2 +# Only compare monotonic observations from this collector process. Other +# processes/hosts have a different domain even if wall clocks look aligned. +CLOCK_DOMAIN = str(uuid.uuid4()) + + +@contextmanager +def observation_phase(observation, name): + """Measure an optional collector phase without adding evidence writes.""" + if observation is None: + yield + return + phase = {'startedMonotonicNs': time.monotonic_ns(), 'completed': False} + observation['phases'][name] = phase + try: + yield + phase['completed'] = True + finally: + phase['finishedMonotonicNs'] = time.monotonic_ns() + + +S3_OPERATIONS = frozenset(""" +create_bucket head_bucket delete_bucket put_object get_object head_object delete_object +delete_objects list_objects_v2 list_object_versions copy_object +put_bucket_versioning get_bucket_versioning put_bucket_tagging get_bucket_tagging +put_bucket_policy get_bucket_policy delete_bucket_policy +put_object_tagging get_object_tagging delete_object_tagging +create_multipart_upload upload_part list_parts complete_multipart_upload +abort_multipart_upload list_multipart_uploads +put_bucket_encryption get_bucket_encryption delete_bucket_encryption +put_object_retention get_object_retention put_object_legal_hold get_object_legal_hold +put_object_lock_configuration get_object_lock_configuration +put_bucket_lifecycle_configuration get_bucket_lifecycle_configuration +delete_bucket_lifecycle put_bucket_replication get_bucket_replication +delete_bucket_replication restore_object get_bucket_location +""".split()) +SECRET_FIELDS = {"authorization", "ssecustomerkey", "ssecustomerkeymd5", + "copysourcessecustomerkey", "copysourcessecustomerkeymd5", + "aws_access_key_id", "aws_secret_access_key", "aws_session_token"} + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def canonical(value: object) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + + +def redact(value): + if isinstance(value, dict): + return {key: "[REDACTED]" if key.lower() in SECRET_FIELDS else redact(item) + for key, item in value.items()} + if isinstance(value, list): + return [redact(item) for item in value] + if isinstance(value, str): + for name in ("HEAL_ACCESS_KEY", "HEAL_SECRET_KEY", "HEAL_SESSION_TOKEN", "HEAL_SSE_C_KEY"): + secret = os.environ.get(name) + if secret: + value = value.replace(secret, "[REDACTED]") + return value + + +def save(path: Path, value: dict) -> None: + # Never overwrite earlier observations, including failed operations. + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x", encoding="utf-8") as output: + output.write(json.dumps(redact(value), ensure_ascii=False, indent=2, default=str) + "\n") + output.flush() + os.fsync(output.fileno()) + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def validate_binding(binding: dict, now: datetime | None = None) -> None: + require(binding.get("schemaVersion") == 1, "unsupported lab binding") + require(bool(re.fullmatch(r"(?:gh|manual)-[a-z0-9-]{5,60}", binding.get("runId", ""))), + "invalid run ID") + require(binding.get("jumpHostOnly") is True and binding.get("exclusiveHealService") is True, + "an exclusive Heal service and jump-host route must be assigned") + require(binding.get("assignmentRequired", False) is False, "example/unassigned lab cannot run") + expiry = datetime.fromisoformat(binding["leaseExpiresAt"].replace("Z", "+00:00")) + require(expiry.tzinfo is not None and expiry > (now or datetime.now(UTC)), "fleet lease expired") + require(bool(re.fullmatch(r"rv-heal-[a-z0-9-]{6,35}", binding.get("bucketPrefix", ""))), + "dedicated bucket prefix required") + endpoints = binding.get("endpoints", {}) + require(bool(endpoints) and binding.get("defaultEndpoint") in endpoints, "missing endpoint binding") + for name, endpoint in endpoints.items(): + require(bool(re.fullmatch(r"[a-zA-Z0-9_-]+", name)), "invalid endpoint name") + parsed = urlsplit(endpoint) + require(parsed.scheme in {"http", "https"} and parsed.hostname is not None + and parsed.username is None and parsed.password is None and not parsed.query + and not parsed.fragment and parsed.path in {"", "/"}, "invalid endpoint URL") + require(parsed.port is not None and 1024 <= parsed.port <= 65535, + "an explicitly allocated test port is required") + for name in ("candidateCommit", "harnessCommit"): + require(bool(re.fullmatch(r"[0-9a-f]{40}", binding.get(name, ""))), f"missing pinned {name}") + for name in ("candidatePackageSha256", "candidateBinarySha256", "rcBinarySha256"): + require(bool(re.fullmatch(r"[0-9a-f]{64}", binding.get(name, ""))), f"missing pinned {name}") + require(type(binding.get("maxInventoryObjects")) is int + and binding["maxInventoryObjects"] in range(1, 10001), "invalid inventory object budget") + require(type(binding.get("maxInventoryBytes")) is int + and 1 <= binding["maxInventoryBytes"] <= 64 * 1024**3, "invalid inventory byte budget") + + +def bucket_allowed(binding: dict, bucket: str) -> None: + prefix = binding["bucketPrefix"] + require(isinstance(bucket, str) and len(bucket) <= 63 + and (bucket == prefix or bucket.startswith(prefix + "-")) + and bool(re.fullmatch(r"[a-z0-9][a-z0-9.-]+[a-z0-9]", bucket)), "bucket is outside this run") + + +def validate_s3_request(binding: dict, operation: str, params: dict) -> None: + require(operation in S3_OPERATIONS, "S3 operation is not allowlisted") + bucket_allowed(binding, params.get("Bucket", "")) + source = params.get("CopySource") + if source is not None: + require(isinstance(source, dict), "CopySource must use the structured SDK form") + bucket_allowed(binding, source.get("Bucket", "")) + if operation == "delete_objects": + items = params.get("Delete", {}).get("Objects", []) + require(0 < len(items) <= 1000 and all(isinstance(item.get("Key"), str) for item in items), + "invalid bounded batch deletion") + require(not params.get("BypassGovernanceRetention"), "retention bypass is prohibited") + require("Body" not in params, "use --size/--seed or --body-file; do not put payloads in JSON") + + +def payload(seed: str, size: int) -> bytes: + require(type(size) is int and 0 <= size <= MAX_BODY, "payload exceeds 64 MiB budget") + return hashlib.shake_256(seed.encode()).digest(size) + + +class NoRedirect(request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise ValueError("Admin redirects are prohibited") + + +class Probe: + def __init__(self, binding: dict, endpoint: str | None = None): + validate_binding(binding) + self.binding = binding + self.endpoint_name = endpoint or binding["defaultEndpoint"] + require(self.endpoint_name in binding["endpoints"], "endpoint is not bound") + self.endpoint = binding["endpoints"][self.endpoint_name].rstrip("/") + + def s3_client(self): + import boto3 + from botocore.config import Config + return boto3.client("s3", endpoint_url=self.endpoint, + aws_access_key_id=os.environ["HEAL_ACCESS_KEY"], + aws_secret_access_key=os.environ["HEAL_SECRET_KEY"], + aws_session_token=os.environ.get("HEAL_SESSION_TOKEN"), + region_name="us-east-1", + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, + retries={"max_attempts": 0}, connect_timeout=5, + read_timeout=30, proxies={})) + + def s3(self, operation: str, params: dict, body: bytes | None = None) -> dict: + validate_binding(self.binding) # Recheck expiry before every mutation. + validate_s3_request(self.binding, operation, params) + outgoing = dict(params) + for key in ("SSECustomerKey", "CopySourceSSECustomerKey"): + if outgoing.get(key) == "$HEAL_SSE_C_KEY": + outgoing[key] = os.environ["HEAL_SSE_C_KEY"] + if body is not None: + require(operation in {"put_object", "upload_part"}, "body not allowed for this operation") + outgoing["Body"] = body + started = time.monotonic() + evidence = {"operation": operation, "params": redact(params), "endpoint": self.endpoint_name, + "startedAt": datetime.now(UTC).isoformat(), "clockDomain": CLOCK_DOMAIN, + "callStartedMonotonicNs": time.monotonic_ns(), + "clientCallInvoked": False, + "timingScope": "client-call; not a server commit/lock witness"} + if body is not None: + evidence["payload"] = {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()} + client = None + try: + client = self.s3_client() + evidence['callStartedMonotonicNs'] = time.monotonic_ns() + evidence['clientCallInvoked'] = True + response = getattr(client, operation)(**outgoing) + stream = response.pop("Body", None) + if stream is not None: + digest, count = hashlib.sha256(), 0 + try: + while chunk := stream.read(1024**2): + count += len(chunk) + require(count <= MAX_BODY, "GET exceeds per-object evidence budget") + digest.update(chunk) + finally: + stream.close() + evidence["payload"] = {"size": count, "sha256": digest.hexdigest()} + require(response.get("ContentLength") == count, "GET length differs from response ContentLength") + evidence.update(response=response, acknowledged=True) + except Exception as exc: + response = getattr(exc, "response", None) + evidence.update(response=response, acknowledged=False, + resultUnknown=response is None, + error={"type": type(exc).__name__, "message": str(exc)}) + finally: + evidence["callFinishedMonotonicNs"] = time.monotonic_ns() + if client is not None: + try:client.close() + except Exception as exc: + evidence['clientCleanupError']={'type':type(exc).__name__,'message':str(exc)} + evidence["elapsedSeconds"] = time.monotonic() - started + evidence["finishedAt"] = datetime.now(UTC).isoformat() + return evidence + + def admin(self, path: str, query: list[tuple[str, str]], body: bytes, + method: str = "POST", auth: str = "signed", *, observation=None) -> dict: + from botocore.awsrequest import AWSRequest + try: + from .heal_signing import sign_admin_request + except ImportError: + from heal_signing import sign_admin_request + with observation_phase(observation, 'requestPreparation'): + validate_binding(self.binding) + require(path.startswith("/rustfs/admin/v3/heal/") or path in { + "/rustfs/admin/v3/background-heal/status", "/rustfs/admin/v4/heal/replacement-recovery"}, + "only Heal Admin routes are allowed") + require("?" not in path and "#" not in path and len(body) <= 2 * 1024**2, + "invalid Admin path or body budget") + require(not any(part in {".", ".."} for part in unquote(path).split("/")), + "Admin route traversal is prohibited") + # Root traversal is safe only because the binding attests a dedicated + # service. Bucket paths are still independently bounded to this run. + suffix = path.removeprefix("/rustfs/admin/v3/heal/") + if path.startswith("/rustfs/admin/v3/heal/") and suffix: + bucket_allowed(self.binding, unquote(suffix.split("/", 1)[0])) + url = self.endpoint + path + ("?" + urlencode(query) if query else "") + headers = {"Content-Type": "application/json", "x-amz-content-sha256": hashlib.sha256(body).hexdigest()} + with observation_phase(observation, 'signing'): + if auth != "anonymous": + secret = os.environ["HEAL_SECRET_KEY"] if auth == "signed" else "deliberately-invalid-heal-signature" + outgoing = AWSRequest(method=method, url=url, data=body, headers=headers) + sign_admin_request(outgoing, os.environ["HEAL_ACCESS_KEY"], secret, os.environ.get("HEAL_SESSION_TOKEN")) + headers = dict(outgoing.headers.items()) + started = time.monotonic() + evidence = {"path": path, "query": query, "requestBody": body.decode(errors="replace"), + "method": method, "auth": auth, "endpoint": self.endpoint_name, + "startedAt": datetime.now(UTC).isoformat(), "clockDomain": CLOCK_DOMAIN, + "callStartedMonotonicNs": time.monotonic_ns(), + "timingScope": "client-call; not a server commit/lock witness"} + if observation is not None: + evidence['statusObservation'] = dict(sessionId=observation['sessionId'], sequence=observation['sequence']) + observation['callStartedMonotonicNs'] = evidence['callStartedMonotonicNs'] + try: + response = self._open_admin(request.Request(url, data=body if method != "GET" else None, + method=method, headers=headers)) + except HTTPError as exc: + response = exc + with observation_phase(observation, 'responseBody'): + with response: + raw = response.read(2 * 1024**2 + 1) + require(len(raw) <= 2 * 1024**2, "Admin response exceeds evidence budget") + length = response.headers.get('Content-Length') + require(length is None or (isinstance(length, str) and length.isdigit() and int(length) == len(raw)), + 'Admin response Content-Length differs from complete body') + evidence["httpStatus"] = response.code + evidence["responseHeaders"] = dict(response.headers.items()) + with observation_phase(observation, 'responseParsing'): + try: + evidence["response"] = json.loads(raw) + except (ValueError, UnicodeDecodeError): + evidence["response"] = raw.decode(errors="replace") + evidence["elapsedSeconds"] = time.monotonic() - started + evidence["callFinishedMonotonicNs"] = time.monotonic_ns() + evidence["finishedAt"] = datetime.now(UTC).isoformat() + if observation is not None: + observation.update(callFinishedMonotonicNs=evidence['callFinishedMonotonicNs'], responseObserved=True) + return evidence + + def _open_admin(self, outgoing): + opener = request.build_opener(request.ProxyHandler({}), NoRedirect()) + return opener.open(outgoing, timeout=30) + + def inventory(self, bucket: str, *, unversioned_current_tags: bool = False) -> dict: + """Enumerate every version/marker, then hash complete version-addressed GETs. + + Only call before injection or after attribution/physical proof: a GET can + itself trigger read repair. Never use this to watch scanner progress. + """ + bucket_allowed(self.binding, bucket) + client, entries, count, byte_count = self.s3_client(), [], 0, 0 + if unversioned_current_tags: + mode = self.s3("get_bucket_versioning", {"Bucket": bucket}) + require(mode["acknowledged"] and mode["response"].get("Status") is None, + "current-tag addressing requires a never-versioned fixture bucket") + for page in client.get_paginator("list_object_versions").paginate(Bucket=bucket): + require(not unversioned_current_tags or not page.get("DeleteMarkers"), + "unversioned fixture unexpectedly has delete markers") + for marker in page.get("DeleteMarkers", []): + entries.append({"key": marker["Key"], "versionId": marker["VersionId"], + "isLatest": marker["IsLatest"], "deleteMarker": True}) + count += 1 + for version in page.get("Versions", []): + count += 1 + require(count <= self.binding["maxInventoryObjects"], "inventory object budget exceeded") + params = {"Bucket": bucket, "Key": version["Key"], "VersionId": version["VersionId"]} + result = self.s3("get_object", params) + require(result["acknowledged"], "version GET failed: " + canonical(result)) + tag_params = dict(params) + if unversioned_current_tags: + require(version["VersionId"] == "null" and version["IsLatest"] is True, + "current-tag addressing cannot read historical versions") + tag_params.pop("VersionId") + tags = self.s3("get_object_tagging", tag_params) + require(tags["acknowledged"], "version tag GET failed: " + canonical(tags)) + response = result["response"] + byte_count += result["payload"]["size"] + require(byte_count <= self.binding["maxInventoryBytes"], "inventory byte budget exceeded") + entries.append({"key": version["Key"], "versionId": version["VersionId"], + "isLatest": version["IsLatest"], "deleteMarker": False, + "payload": result["payload"], + "properties": {key: response[key] for key in ( + "ETag", "ContentLength", "ContentType", "ContentEncoding", + "ContentDisposition", "CacheControl", "Metadata", "StorageClass", + "ServerSideEncryption", "ObjectLockMode", "ObjectLockRetainUntilDate", + "ObjectLockLegalHoldStatus") if key in response}, + "tags": sorted(tags["response"].get("TagSet", []), key=lambda item: item["Key"])}) + require(count <= self.binding["maxInventoryObjects"], "inventory object budget exceeded") + identities = [(entry["key"], entry["versionId"]) for entry in entries] + require(len(set(identities)) == len(identities), "duplicate version/marker in pagination") + # Cross-check current LIST against the separately paginated version view. + current = [] + for page in client.get_paginator("list_objects_v2").paginate(Bucket=bucket): + current.extend((item["Key"], item["Size"], item["ETag"]) for item in page.get("Contents", [])) + require(len(current) <= self.binding["maxInventoryObjects"], "current LIST exceeds budget") + expected = [(item["key"], item["payload"]["size"], item["properties"]["ETag"]) + for item in entries if item["isLatest"] and not item["deleteMarker"]] + require(sorted(current) == sorted(expected), "current LIST differs from version manifest") + if unversioned_current_tags: + mode = self.s3("get_bucket_versioning", {"Bucket": bucket}) + require(mode["acknowledged"] and mode["response"].get("Status") is None, + "fixture versioning changed during inventory") + return {"bucket": bucket, "entries": sorted(entries, key=lambda item: (item["key"], item["versionId"])), + "objectsAndMarkers": count, "bytesRead": byte_count, + "tagAddressing": "current-never-versioned-fixture" if unversioned_current_tags else "explicit-version"} + + +def compare_inventories(before: dict, after: dict) -> dict: + require(before["bucket"] == after["bucket"] and bool(before["entries"]), "wrong or empty baseline") + require(canonical(before["entries"]) == canonical(after["entries"]), "S3 versions/content/metadata/tags changed") + return {"equal": True, "objectsAndMarkers": len(before["entries"]), + "proof": "full-version-content-and-public-metadata", "physicalRepairProved": False} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binding", type=Path, required=True) + parser.add_argument("--endpoint") + parser.add_argument("--out", type=Path, required=True) + subs = parser.add_subparsers(dest="command", required=True) + admin = subs.add_parser("admin") + admin.add_argument("--path", required=True) + admin.add_argument("--method", choices=("POST", "GET"), default="POST") + admin.add_argument("--query", action="append", default=[]) + admin.add_argument("--body", default="") + admin.add_argument("--body-file", type=Path) + admin.add_argument("--auth", choices=("signed", "anonymous", "bad-signature"), default="signed") + admin.add_argument("--expect-http", type=int, default=200) + s3 = subs.add_parser("s3") + s3.add_argument("--operation", choices=sorted(S3_OPERATIONS), required=True) + s3.add_argument("--params", default="{}") + s3.add_argument("--params-file", type=Path) + s3.add_argument("--size", type=int) + s3.add_argument("--seed", default="heal") + s3.add_argument("--body-file", type=Path) + s3.add_argument("--expect-http", type=int, default=200) + inventory = subs.add_parser("inventory") + inventory.add_argument("--bucket", required=True) + compare = subs.add_parser("compare") + compare.add_argument("--before", type=Path, required=True) + compare.add_argument("--after", type=Path, required=True) + subs.add_parser("binding-check") + args = parser.parse_args() + require(not args.out.exists(), "output already exists; preserve earlier evidence") + binding = json.loads(args.binding.read_text()) + evidence = {"schemaVersion": 1, "runId": binding.get("runId"), + "bindingSha256": hashlib.sha256(args.binding.read_bytes()).hexdigest(), + "capturedAt": datetime.now(UTC).isoformat(), "command": args.command, + "casePassClaimed": False} + exit_code = 0 + try: + probe = Probe(binding, args.endpoint) + if args.command == "admin": + query = [] + for item in args.query: + require("=" in item, "query must be name=value; repeats are preserved") + query.append(tuple(item.split("=", 1))) + body = args.body_file.read_bytes() if args.body_file else args.body.encode() + evidence["observation"] = probe.admin(args.path, query, body, args.method, args.auth) + require(evidence["observation"]["httpStatus"] == args.expect_http, "unexpected Admin HTTP status") + elif args.command == "s3": + params = json.loads(args.params_file.read_text() if args.params_file else args.params) + require(not (args.body_file and args.size is not None), "choose file or generated body") + if args.body_file: + require(args.body_file.stat().st_size <= MAX_BODY, "body file exceeds budget") + body = args.body_file.read_bytes() + else: + body = payload(args.seed, args.size) if args.size is not None else None + result = probe.s3(args.operation, params, body) + evidence["observation"] = result + actual = (result.get("response") or {}).get("ResponseMetadata", {}).get("HTTPStatusCode") + require(actual == args.expect_http, "unexpected S3 HTTP status (unknown writes require ACK reconciliation)") + elif args.command == "inventory": + evidence["inventory"] = probe.inventory(args.bucket) + elif args.command == "compare": + before, after = (json.loads(path.read_text())["inventory"] for path in (args.before, args.after)) + evidence["comparison"] = compare_inventories(before, after) + else: + evidence["bindingCheck"] = {"valid": True, "liveEnvironmentVerified": False} + evidence["operationCheck"] = "PASS" + except Exception as exc: + evidence.update(operationCheck="FAIL", error={"type": type(exc).__name__, "message": str(exc)}) + exit_code = 1 + save(args.out, evidence) + print(json.dumps({"evidence": str(args.out), "operationCheck": evidence["operationCheck"], + "casePassClaimed": False})) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation_heal/heal_signing.py b/scripts/validation_heal/heal_signing.py new file mode 100644 index 00000000..085b8eef --- /dev/null +++ b/scripts/validation_heal/heal_signing.py @@ -0,0 +1,19 @@ +"""MinIO-compatible S3 signing for raw Heal Admin requests.""" + +from urllib.parse import quote_from_bytes, unquote_to_bytes + + +def sign_admin_request(request, access_key: str, secret_key: str, session_token: str | None = None) -> None: + # Keep SDK imports lazy so offline plan/binding tools need only stdlib. + from botocore.auth import S3SigV4Auth + from botocore.credentials import Credentials + + class AdminS3Auth(S3SigV4Auth): + def _normalize_url_path(self, path): + # MinIO signs the decoded URL.Path after S3 URI encoding. Do not + # normalize dot segments or decode twice: %252F is a literal %2F + # key, while %2F and / identify the same separator. The wire URL + # stays untouched so the server's routing/decoding is still tested. + return quote_from_bytes(unquote_to_bytes(path), safe="/~") + + AdminS3Auth(Credentials(access_key, secret_key, session_token), "s3", "us-east-1").add_auth(request) diff --git a/scripts/validation_heal/heal_workload.py b/scripts/validation_heal/heal_workload.py new file mode 100644 index 00000000..821aeeb0 --- /dev/null +++ b/scripts/validation_heal/heal_workload.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +"""Create deterministic Heal fixtures or run a bounded, ACK-logged S3 workload.""" + +from __future__ import annotations + +import argparse +import copy +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime +import json +import hashlib +from pathlib import Path +import threading +import time + +try: + from .heal_probe import CLOCK_DOMAIN, MAX_BODY, Probe, bucket_allowed, payload, require, save, validate_s3_request + from .heal_workload_journal import AdmissionExpired, MutationOrder, batch_receipts, validate_batch_request +except ImportError: + from heal_probe import CLOCK_DOMAIN, MAX_BODY, Probe, bucket_allowed, payload, require, save, validate_s3_request + from heal_workload_journal import AdmissionExpired, MutationOrder, batch_receipts, validate_batch_request + + +class FixtureStateMismatch(ValueError): + """Observed S3 state disagrees with valid, successful fixture ACKs.""" + + +class RecordedCallError(ValueError): + """Failed check with an already durable, exactly bound observation. + + Callers can retain known per-item batch effects without treating the whole + call as success or guessing its result from a later LIST. Never a retry hint. + """ + def __init__(self,message,operation,params,observation,request_sha256): + super().__init__(message) + self.operation=operation;self.params=copy.deepcopy(params) + self.observation=copy.deepcopy(observation);self.request_sha256=request_sha256 + + +class Recorder: + def __init__(self, probe: Probe, output: Path, *, writer=None): + self.probe, self.output = probe, output + self.writer = writer + output.mkdir(parents=True, exist_ok=False) + self.sequence, self.failures, self.bytes_written = 0, 0, 0 + self.lock = threading.Lock() + self.mutations = MutationOrder() + + def call(self, operation: str, params: dict, body: bytes | None = None, + identity: dict | None = None, expect_http: int | None = None, *, admission_deadline=None) -> dict: + params=copy.deepcopy(params);identity=copy.deepcopy(identity) + validate_s3_request(self.probe.binding, operation, params) + if operation == 'delete_objects':validate_batch_request(params) + require(body is None or isinstance(body,bytes) and len(body)<=MAX_BODY and operation in {'put_object','upload_part'}, + 'invalid foreground payload/type/budget') + with self.mutations.hold(operation, params,admission_deadline): + try: + return self._call(operation, params, body, identity, expect_http) + except BaseException: + # Already in-flight calls drain and retain their own evidence; + # no later foreground mutation starts after an uncertain write. + self.mutations.poison('request/receipt failed') + raise + + def _call(self, operation, params, body, identity, expect_http): + with self.lock: + self.sequence += 1 + sequence = self.sequence + self.bytes_written += len(body) if body is not None else 0 + require(self.bytes_written <= self.probe.binding["maxInventoryBytes"], "batch payload byte budget exceeded") + request = {"schemaVersion": 2, "runId": self.probe.binding["runId"], "sequence": sequence, + "operation": operation, "params": params, "identity": identity, "expectHttp": expect_http, + "clockDomain": CLOCK_DOMAIN, "preparedMonotonicNs": time.monotonic_ns(), + "preparedAt": datetime.now(UTC).isoformat(), "casePassClaimed": False} + if body is not None:request['payload']={'size':len(body),'sha256':hashlib.sha256(body).hexdigest()} + request_path = self.output / f"{sequence:06d}-request.json" + # fsync file+directory before allowing any network invocation. A + # request without a durable result remains UNKNOWN, never disappears. + (self.writer or save)(request_path, request) + request_sha = hashlib.sha256(request_path.read_bytes()).hexdigest() + invoked = time.monotonic_ns() + try: + result = self.probe.s3(operation, params, body) + except Exception as exc: + result = {"acknowledged": False, "resultUnknown": True, "response": getattr(exc,'response',None), + "error": {"type": type(exc).__name__, "message": str(exc)}} + finished = time.monotonic_ns() + status = (result.get("response") or {}).get("ResponseMetadata", {}).get("HTTPStatusCode") + checked = status == expect_http if expect_http is not None else ( + result["acknowledged"] and type(status) is int and 200<=status<300) + checked = checked and not result.get('clientCleanupError') + per_item = None + if operation == 'delete_objects' and status == 200: + per_item = batch_receipts(params, result['response']) + checked = checked and not per_item['errors'] + (self.writer or save)(self.output / f"{sequence:06d}-ack.json", {"schemaVersion": 2, + "runId": self.probe.binding["runId"], "sequence": sequence, "identity": identity, + "requestFile":request_path.name,"requestSha256":request_sha,"batchDelete":per_item, + "clockDomain":CLOCK_DOMAIN,"recordedMonotonicNs":time.monotonic_ns(), + "collectorCallStartedMonotonicNs":invoked,"collectorCallFinishedMonotonicNs":finished, + "operationCheck": "PASS" if checked else "FAIL", "observation": result, + "casePassClaimed": False}) + with self.lock: + self.failures += not checked + if not checked: + raise RecordedCallError(f"unexpected {operation} result; inspect ACK {sequence:06d}, do not silently retry", + operation,params,result,request_sha) + return result.get("response") or {} + + +def seed(recorder: Recorder, bucket: str, fixture: str, count: int, sizes: list[int], + *, key_prefix: str = "objects/") -> list[dict]: + bucket_allowed(recorder.probe.binding, bucket) + require(1 <= count <= 2048 and bool(sizes) and all(0 <= size <= 64 * 1024**2 for size in sizes), + "invalid bounded fixture dimensions") + require(key_prefix in {"objects/", "objects-"} and (key_prefix == "objects/" or fixture == "DB"), + "flat keys are only supported for the bounded DB fixture") + recorder.call("create_bucket", {"Bucket": bucket}) + expected = [] + if fixture == "DV": + recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) + if fixture == "DM": + for index in range(count): + key = f"multipart/{index:04d}.bin" + upload = recorder.call("create_multipart_upload", {"Bucket": bucket, "Key": key, + "Metadata": {"heal-fixture": "DM"}, "ContentType": "application/octet-stream"}) + parts = [] + digest, full_size = hashlib.sha256(), 0 + for number, size in enumerate((5 * 1024**2 + 17, 5 * 1024**2 + 93, 1024**2 + 47), 1): + body = payload(f"{bucket}/{key}/{number}", size) + digest.update(body) + full_size += len(body) + result = recorder.call("upload_part", {"Bucket": bucket, "Key": key, + "UploadId": upload["UploadId"], "PartNumber": number}, + body) + parts.append({"PartNumber": number, "ETag": result["ETag"]}) + completed = recorder.call("complete_multipart_upload", {"Bucket": bucket, "Key": key, + "UploadId": upload["UploadId"], "MultipartUpload": {"Parts": parts}}) + expected.append({"key": key, "versionId": completed.get("VersionId", "null"), + "deleteMarker": False, "isLatest": True, + "payload": {"sha256": digest.hexdigest(), "size": full_size}}) + # Separate uploads for Complete/Abort races. Their actual IDs live in + # the ACK ledger, never inferred from a later upload's ID. + for name in ("pending-a", "pending-b"): + response = recorder.call("create_multipart_upload", {"Bucket": bucket, "Key": name}) + recorder.call("upload_part", {"Bucket": bucket, "Key": name, "UploadId": response["UploadId"], + "PartNumber": 1}, payload(f"{bucket}/{name}", 5 * 1024**2 + 3)) + return expected + for index in range(count): + key = f"{key_prefix}{index:04d}.bin" + if fixture == "DS": + prefixes = ("中文/文件", "space key", "literal%2Fslash", "actual/slash", "a/child", "ab/control", "b/other") + key = f"{prefixes[index % len(prefixes)]}/{index:04d}.bin" + versions = 3 if fixture == "DV" else 1 + for version in range(versions): + size = sizes[index % len(sizes)] + body = payload(f"{bucket}/{key}/{version}", size) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "ContentType": "application/octet-stream", + "Metadata": {"heal-fixture": fixture, "nonce": f"{index}-{version}"}, + "Tagging": "suite=heal&fixture=" + fixture}, + body) + require(fixture != "DV" or response.get("VersionId") not in {None, "null"}, + "enabled versioning PUT omitted a real version ID") + expected.append({"key": key, "versionId": response.get("VersionId", "null"), + "isLatest": version == versions - 1, "deleteMarker": False, + "payload": {"sha256": hashlib.sha256(body).hexdigest(), "size": len(body)}}) + if fixture == "DV" and index % 2 == 0: + response = recorder.call("delete_object", {"Bucket": bucket, "Key": key}) + require(response.get("DeleteMarker") is True and response.get("VersionId"), "marker ACK is incomplete") + expected[-1]["isLatest"] = False + expected.append({"key": key, "versionId": response["VersionId"], "isLatest": True, "deleteMarker": True}) + return expected + + +def verify_seed(expected: list[dict], inventory: dict) -> dict: + actual = {(entry["key"], entry["versionId"]): entry for entry in inventory["entries"]} + wanted = {(entry["key"], entry["versionId"]): entry for entry in expected} + require(bool(wanted) and len(wanted) == len(expected), "fixture ACK identities are empty or duplicate") + if len(actual) != len(inventory["entries"]) or set(wanted) != set(actual): + raise FixtureStateMismatch("fixture version/marker set differs from ACKs") + for identity, entry in wanted.items(): + if not all(actual[identity].get(key) == value for key, value in entry.items()): + raise FixtureStateMismatch("baseline content/version/marker differs from deterministic fixture ACK") + return {"expectedEntries": len(expected), "seedAckAndFullGetAgree": True, "casePassClaimed": False} + + +def substitute(value, round_number: int, worker: int): + if isinstance(value, dict): + return {key: substitute(item, round_number, worker) for key, item in value.items()} + if isinstance(value, list): + return [substitute(item, round_number, worker) for item in value] + if isinstance(value, str): + return value.replace("${round}", str(round_number)).replace("${worker}", str(worker)) + return value + + +def run_trace(recorder: Recorder, trace: list[dict], rounds: int, workers: int, + interval: float, duration: int) -> dict: + require(1 <= rounds <= 100000 and 1 <= workers <= 16 and 0 <= interval <= 60 + and 1 <= duration <= 72 * 3600 and 1 <= len(trace) <= 100, + "invalid bounded workload dimensions") + require(rounds * workers * len(trace) <= 1_000_000, "request count exceeds batch cap") + # Reject the whole trace before its first mutation. A later cross-run + # bucket or invalid payload must not leave an earlier partial workload. + for step in trace: + require(isinstance(step, dict) and set(step) <= {"operation", "params", "size", "expectHttp"}, + "invalid trace fields") + validate_s3_request(recorder.probe.binding, step["operation"], substitute(step["params"], 0, 0)) + if step['operation']=='delete_objects':validate_batch_request(step['params']) + if "size" in step: + require(type(step["size"]) is int and 0 <= step["size"] <= 64 * 1024**2 + and step["operation"] in {"put_object", "upload_part"}, "invalid trace payload") + if "expectHttp" in step: + require(type(step["expectHttp"]) is int and 100 <= step["expectHttp"] <= 599, "invalid expected HTTP code") + deadline, abort = time.monotonic() + duration, threading.Event() + + def worker_run(worker: int) -> dict: + completed = 0 + for round_number in range(rounds): + for index, step in enumerate(trace): + if abort.is_set() or time.monotonic() >= deadline: + return {'worker':worker,'completedOperations':completed,'stopReason':'abort' if abort.is_set() else 'deadline'} + identity = {"worker": worker, "round": round_number, "step": index} + params = substitute(step["params"], round_number, worker) + body = payload(f"{recorder.probe.binding['runId']}/{worker}/{round_number}/{index}", step["size"]) if "size" in step else None + try: + recorder.call(step["operation"], params, body, identity, step.get("expectHttp"),admission_deadline=deadline) + except AdmissionExpired: + return {'worker':worker,'completedOperations':completed,'stopReason':'admission-deadline'} + except Exception: + abort.set() + raise + completed += 1 + if interval: + abort.wait(min(interval, max(0, deadline - time.monotonic()))) + return {'worker':worker,'completedOperations':completed,'stopReason':'round-limit'} + + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = [pool.submit(worker_run, worker) for worker in range(workers)] + outcomes = [future.result() for future in futures] + return {'workers':outcomes,'plannedOperationUpperBound':rounds*workers*len(trace), + 'completedOperations':sum(v['completedOperations'] for v in outcomes), + 'allRoundsCompleted':all(v['stopReason']=='round-limit' for v in outcomes),'casePassClaimed':False} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binding", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--endpoint") + sub = parser.add_subparsers(dest="command", required=True) + fixture = sub.add_parser("seed") + fixture.add_argument("--fixture", choices=("DB", "DM", "DV", "DS", "DX"), required=True) + fixture.add_argument("--bucket", required=True) + fixture.add_argument("--count", type=int) + fixture.add_argument("--sizes", help="comma-separated actual byte sizes, including candidate-specific boundaries") + workload = sub.add_parser("run") + workload.add_argument("--trace", type=Path, required=True) + workload.add_argument("--rounds", type=int, required=True) + workload.add_argument("--workers", type=int, default=1) + workload.add_argument("--interval", type=float, default=0.1, help="minimum seconds between each worker's requests") + workload.add_argument("--duration", type=int, required=True, help="hard admission deadline in seconds") + args = parser.parse_args() + binding = json.loads(args.binding.read_text()) + recorder = Recorder(Probe(binding, args.endpoint), args.out) + summary = {"runId": binding["runId"], "startedAt": datetime.now(UTC).isoformat(), + "command": args.command, "casePassClaimed": False, "state": "RUNNING"} + start = time.monotonic() + try: + if args.command == "seed": + defaults = {"DB": 128, "DM": 1, "DV": 4, "DS": 257, "DX": 257} + sizes = [int(value) for value in args.sizes.split(",")] if args.sizes else ( + [0, 1, 4095, 4096, 4097, 1024**2 - 1, 1024**2, 1024**2 + 1, 5 * 1024**2 + 123] + if args.fixture == "DB" else [5 * 1024**2 + 123]) + expected = seed(recorder, args.bucket, args.fixture, args.count if args.count is not None else defaults[args.fixture], sizes) + save(args.out / "expected-fixture.json", {"runId": binding["runId"], "bucket": args.bucket, "entries": expected}) + baseline = recorder.probe.inventory(args.bucket) + save(args.out / "baseline.json", {"runId": binding["runId"], "inventory": baseline}) + summary["baselineVerification"] = verify_seed(expected, baseline) + else: + summary['workload'] = run_trace(recorder, json.loads(args.trace.read_text()), args.rounds, args.workers, args.interval, args.duration) + summary["state"] = "OPERATIONS_COMPLETE" + except Exception as exc: + summary.update(state="FAIL", error={"type": type(exc).__name__, "message": str(exc)}) + summary.update(requests=recorder.sequence, failedOperations=recorder.failures, + elapsedSeconds=time.monotonic() - start, finishedAt=datetime.now(UTC).isoformat()) + save(args.out / "summary.json", summary) + print(json.dumps(summary)) + return 0 if summary["state"] == "OPERATIONS_COMPLETE" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation_heal/heal_workload_journal.py b/scripts/validation_heal/heal_workload_journal.py new file mode 100644 index 00000000..2877ff9e --- /dev/null +++ b/scripts/validation_heal/heal_workload_journal.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Foreground mutation ordering and per-item DELETE ACKs; no repair verdict.""" +from contextlib import contextmanager +import threading +import time + +try: + from .heal_probe import require +except ImportError: + from heal_probe import require + + +class AdmissionExpired(TimeoutError): + """No call was sent: the admission deadline expired while waiting.""" + + +def mutation_resources(operation,params): + if operation.startswith(('get_','head_','list_')):return () + bucket=params['Bucket'] + if operation=='delete_objects': + keys=[v['Key'] for v in params['Delete']['Objects']] + return tuple(sorted({(bucket,k) for k in keys})) + if 'Key' in params: + require(isinstance(params['Key'],str) and params['Key'],'empty/invalid mutation key') + return ((bucket,params['Key']),) + # Bucket configuration changes exclude all foreground mutations in that + # bucket, while different keys and buckets can otherwise run concurrently. + return ((bucket,None),) + + +def conflicts(left,right): + return any(a==b and (x is None or y is None or x==y) for a,x in left for b,y in right) + + +class MutationOrder: + def __init__(self): + self.condition=threading.Condition();self.active={};self.poisoned=None;self.next_ticket=0 + + @contextmanager + def hold(self,operation,params,deadline=None): + resources=mutation_resources(operation,params);ticket=None + if resources: + with self.condition: + while any(conflicts(resources,r) for r in self.active.values()) and self.poisoned is None: + remaining=None if deadline is None else deadline-time.monotonic() + if remaining is not None and remaining<=0:raise AdmissionExpired('foreground admission deadline expired') + self.condition.wait(timeout=remaining) + require(self.poisoned is None,'foreground mutations halted after prior failure/unknown result') + if deadline is not None and time.monotonic()>=deadline:raise AdmissionExpired('foreground admission deadline expired') + ticket=self.next_ticket;self.next_ticket+=1;self.active[ticket]=resources + elif deadline is not None and time.monotonic()>=deadline:raise AdmissionExpired('foreground admission deadline expired') + try:yield resources + finally: + if ticket is not None: + with self.condition: + del self.active[ticket];self.condition.notify_all() + + def poison(self,reason): + with self.condition:self.poisoned=self.poisoned or reason;self.condition.notify_all() + + +def validate_batch_request(params): + delete=params['Delete'];items=delete['Objects'] + require(delete.get('Quiet',False) is False,'DELETE batch requires explicit per-item receipts; Quiet is unsupported') + identities=[(v['Key'],v.get('VersionId')) for v in items] + require(all(isinstance(k,str) and k and (v is None or isinstance(v,str) and v) for k,v in identities),'invalid DELETE batch identity') + require(len(identities)==len(set(identities)),'duplicate DELETE batch identity') + + +def batch_receipts(params,response): + """HTTP 200 is not an acknowledgement of every member of a DELETE batch.""" + validate_batch_request(params);items=params['Delete']['Objects'];out=[];seen=set();errors=[] + wanted={(v['Key'],v.get('VersionId')) for v in items};by_identity={} + for name,state in (('Deleted','ACK'),('Errors','REJECTED')): + for row in response.get(name,[]): + key=(row.get('Key'),row.get('VersionId')) + if key not in wanted or key in seen:errors.append('unbound/duplicate DELETE result: '+repr(key)) + seen.add(key);by_identity.setdefault(key,[]).append({'state':state,'receipt':row}) + for item in items: + matches=by_identity.get((item['Key'],item.get('VersionId')),[]) + value=matches[0] if len(matches)==1 else {'state':'UNKNOWN','receipts':matches} + out.append({'request':item,**value}) + if value['state']!='ACK':errors.append('DELETE member not acknowledged: '+repr(item)) + return {'items':out,'errors':errors} diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py new file mode 100644 index 00000000..3b5fe4f7 --- /dev/null +++ b/scripts/validation_heal/healthy.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""Execute every A01 normal/deep branch on the current H1 and H2 Heal fleet.""" + +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import subprocess +import sys +import time +from urllib import request +from urllib.parse import quote + +try: + from .heal_probe import Probe, canonical, compare_inventories, require, save + from .heal_workload import Recorder, seed, verify_seed +except ImportError: + from heal_probe import Probe, canonical, compare_inventories, require, save + from heal_workload import Recorder, seed, verify_seed + + +NODE_SOURCE = Path(__file__).with_name("fleet_node.py") +ROOT = Path(__file__).resolve().parents[2] + + +def physical_content(snapshot): + return {node: {key: value[key] for key in ("files", "formats")} for node, value in snapshot.items()} + + +def validate_topology(snapshot, topology): + require(topology in {"H1", "H2"} and len(snapshot) == (1 if topology == "H1" else 4), + "wrong node count for Heal topology") + records = [(node, value) for node, entry in snapshot.items() for value in entry["formats"]] + require(len(records) == (4 if topology == "H1" else 16), "wrong physical disk count") + values = [item["value"] for _, item in records] + sets = values[0]["xl"]["sets"] + require(len(sets) == 1 and len(sets[0]) == len(records) and len(set(sets[0])) == len(records), + "A01 requires one complete independently formatted erasure set") + require(all(value["format"] == "xl" and value["xl"]["sets"] == sets + and value["id"] == values[0]["id"] for value in values) + and {value["xl"]["this"] for value in values} == set(sets[0]), + "physical disk UUID, pool and set mapping differs") + if topology == "H2": + require(all(len({item["value"]["xl"]["this"] for owner, item in records if owner == node}) == 4 + for node in snapshot), "H2 set does not span every node's four drives") + return {"topology": topology, "nodes": len(snapshot), "independentDrives": len(records), + "poolIndex": 0, "setIndex": 0, "diskUUIDs": sets[0]} + + +def validate_healthy(status, bucket, key, disposition): + outcome = status.get("outcome") or {} + require(status.get("summary") == "finished" and status.get("truncated", False) is False + and outcome.get("execution", {}).get("state") == "completed" + and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False, + "Heal did not prove complete terminal coverage") + counters = outcome.get("counters", {}) + expected = {"processed": 1, "healed": 0, "unchanged": 1, "skipped": 0, + "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False} + require(all(type(counters.get(name)) is type(value) and counters[name] == value + for name, value in expected.items()), "healthy object counters do not prove exactly one unchanged object") + objects = outcome.get("objects", []) + require(len(objects) == 1, "healthy task returned incomplete or additional object receipts") + identity = objects[0].get("identity", {}) + require(identity.get("bucket") == bucket and identity.get("object") == key + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0 + and bool(identity.get("bucketIncarnationId")), "healthy receipt lacks the exact object responsibility identity") + require(objects[0].get("disposition", {}).get("state") == disposition, + f"expected {disposition} receipt; observed {objects[0].get('disposition')}") + + +class Fleet: + def __init__(self, nodes, evidence, run_id): + self.nodes, self.evidence, self.run_id = nodes, evidence, run_id + self.sequence = 0 + self.reserved = [] + + def record(self, label, value): + self.sequence += 1 + save(self.evidence / f"{self.sequence:05d}-{label}.json", value) + return value + + def remote(self, node, action, **params): + value = {"runId": self.run_id, "action": action, **params} + source = NODE_SOURCE.read_text() + "\nmain(" + repr(value) + ")\n" + result = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", + "-o", "StrictHostKeyChecking=accept-new", "-p", os.environ.get("RUSTFS_SSH_PORT", "22"), + os.environ.get("RUSTFS_SSH_USER", "azureuser") + "@" + node, + "sudo -n python3 -"], input=source, capture_output=True, text=True, timeout=180) + require(result.returncode == 0, f"Heal node {node} {action} failed: {result.stderr[-1200:]}") + return json.loads(result.stdout) + + def snapshot(self, label): + return self.record(label, {node: self.remote(node, "snapshot") for node in self.nodes}) + + def deploy(self, binary_sha, bucket, access, secret): + checked = {node: self.remote(node, "preflight") for node in self.nodes} + self.record("preflight", checked) + paths = checked[self.nodes[0]]["dataRoots"] + volumes = paths if len(self.nodes) == 1 else [f"http://rustfs-node{index}:9000{path}" + for index in range(1, 5) for path in paths] + for node in self.nodes: + # Reserve may partially mutate before an SSH failure. Preserve its + # namespace unless the root-owned marker proves safe finalization. + self.reserved.append(node) + value = self.remote(node, "reserve", hostname=checked[node]["hostname"], mounts=checked[node]["mounts"], + binarySha256=binary_sha, bucket=bucket, volumes=volumes, access=access, secret=secret) + self.record("reserved", {"node": node, **value}) + with ThreadPoolExecutor(max_workers=len(self.nodes)) as executor: + results = list(executor.map(lambda node: self.remote(node, "start"), self.nodes)) + self.record("started", {node: value for node, value in zip(self.nodes, results, strict=True)}) + opener = request.build_opener(request.ProxyHandler({})) + deadline = time.monotonic() + 180 + pending = {f"http://rustfs-node{index}:9000" for index in range(1, len(self.nodes) + 1)} + while pending and time.monotonic() < deadline: + for endpoint in tuple(pending): + try: + with opener.open(endpoint + "/health/ready", timeout=5) as response: + if response.status == 200: + pending.remove(endpoint) + except OSError: + pass + if pending: + time.sleep(2) + require(not pending, "Heal deployment did not make every node ready") + + def cleanup(self): + failures = [] + for node in self.reserved: + try: + self.record("logs", {"node": node, **self.remote(node, "logs")}) + except Exception as exc: + self.record("logs-error", {"node": node, "error": str(exc)}) + try: + self.record("cleanup", {"node": node, **self.remote(node, "cleanup")}) + except Exception as exc: + failures.append({"node": node, "error": str(exc)}) + self.record("recovery", {"cleanup": "FAIL" if failures else "PASS", "errors": failures}) + return not failures + + +def run_topology(args, topology, nodes): + directory = args.evidence / topology + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + candidate = json.loads(args.candidate.read_text()) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), + "bucketPrefix": bucket, "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" + for index in range(1, len(nodes) + 1)}, "defaultEndpoint": "node1", + "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 128, "maxInventoryBytes": 512 * 1024**2} + fleet.record("binding", binding) + result = {"topology": topology, "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} + active = [] + probe = Probe(binding) + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + initial = fleet.snapshot("deployed-topology") + fleet.record("topology-proof", validate_topology(initial, topology)) + recorder = Recorder(probe, directory / "fixture") + expected = seed(recorder, bucket, "DB", 128, + [0, 1, 4095, 4096, 4097, 1024**2 - 1, 1024**2, 1024**2 + 1, 5 * 1024**2 + 123]) + baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + before = fleet.snapshot("physical-baseline") + fleet.record("fault", {"fault": "F0", "injected": False, "backgroundRepairDisabled": True}) + for branch, mode, disposition in (("normal", 1, "metadata_healthy"), ("deep", 2, "verified_healthy")): + receipts = [] + for entry in expected: + key = entry["key"] + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + quote(key, safe="/") + options = {"recursive": False, "remove": False, "recreate": False, "scanMode": mode, + "dryRun": False, "poolIndex": 0, "setIndex": 0} + started = fleet.record("trigger-" + branch, probe.admin(path, [], json.dumps(options).encode())) + token = (started.get("response") or {}).get("clientToken") + require(started.get("httpStatus") == 200 and isinstance(token, str) and bool(token), + "healthy Heal task admission failed") + active.append((path, token)) + deadline = time.monotonic() + 120 + while True: + observed = fleet.record("status-" + branch, probe.admin(path, [("clientToken", token)], b"")) + require(observed.get("httpStatus") == 200, "Heal status request failed") + terminal = observed.get("response") or {} + if terminal.get("summary") != "running": + break + require(time.monotonic() < deadline, "healthy Heal task exceeded deadline") + time.sleep(0.2) + validate_healthy(terminal, bucket, key, disposition) + active.remove((path, token)) + for _ in range(2): + repeated = fleet.record("repeat-" + branch, probe.admin(path, [("clientToken", token)], b"")) + require(repeated.get("httpStatus") == 200, "repeated Heal status request failed") + validate_healthy(repeated.get("response") or {}, bucket, key, disposition) + require(canonical(repeated["response"]["outcome"]) == canonical(terminal["outcome"]), + "terminal responsibility changed on repeated status") + receipts.append({"key": key, "token": token, "disposition": disposition}) + after = fleet.snapshot("physical-after-" + branch) + require(physical_content(before) == physical_content(after), "healthy Heal changed physical fixture or format") + inventory = fleet.record("s3-after-" + branch, probe.inventory(bucket, unversioned_current_tags=True)) + comparison = compare_inventories(baseline, inventory) + result["branches"][branch] = {"verdict": "PASS", "receipts": receipts, + "physicalUnchanged": True, "s3": comparison} + result["verdict"] = "PASS" + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + if fleet.reserved: + try: + fleet.snapshot("physical-failure-scene") + except Exception as capture: + fleet.record("failure-scene-error", {"error": str(capture)}) + finally: + for path, token in active: + try: + fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: + fleet.record("stop-error", {"error": str(exc)}) + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("topology-result", result) + return result + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f"Heal workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(set(nodes)) == len(nodes) + and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), + "A01 H1/H2 requires the four current auto-testing Heal nodes") + require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid installed binary digest") + require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") + results = [] + try: + for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): + result = run_topology(args, topology, selected) + results.append(result) + if result["verdict"] != "PASS" or result["cleanup"] != "PASS": + break + finally: + save(args.evidence / "healthy-summary.json", {"caseId": "heal.admin-healthy-noop", "results": results, + "cleanup": "PASS" if results and all(value["cleanup"] == "PASS" for value in results) else "FAIL", + "complete": len(results) == 2 and all(value["verdict"] == "PASS" and value["cleanup"] == "PASS" for value in results)}) + require(len(results) == 2 and all(value["verdict"] == "PASS" and value["cleanup"] == "PASS" for value in results), + "A01 requires both H1/H2 and normal/deep; inspect topology-result evidence") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"Heal validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py new file mode 100644 index 00000000..9ac97d30 --- /dev/null +++ b/scripts/validation_heal/matrix.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""Preserve all Heal procedures and audit branch evidence on the auto-testing fleet.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import socket +import sys + + +ROOT = Path(__file__).resolve().parents[2] +RECIPES = Path(__file__).with_name("recipes.json") +CATALOG = ROOT / "validation-cases.json" +OPERATIONS = ROOT / "docs/validation-heal-operations.md" +CONTRACT = ROOT / "docs/validation-heal-contract.md" +TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} +STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} +AUTOMATIC_CASES = {"heal.admin-healthy-noop"} + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def save(path: Path, value: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x", encoding="utf-8") as stream: + stream.write(json.dumps(value, ensure_ascii=False, indent=2) + "\n") + + +def variant_topologies(item: dict) -> dict[str, set[str]]: + allowed = set(item["contract"]["probes"]["topology"].split(",")) + require(bool(allowed) and allowed <= TOPOLOGIES, "invalid Heal topology") + if "variantTopologies" not in item: + return {variant: allowed for variant in item["variants"]} + mapping = item["variantTopologies"] + require(set(mapping) == set(item["variants"]), "variant topology mapping omits a branch") + require(bool(item.get("topologyApplicabilityReason")), "missing topology applicability reason") + for values in mapping.values(): + require(isinstance(values, list) and bool(values) and len(values) == len(set(values)) + and set(values) <= allowed, "invalid variant topology mapping") + require(set().union(*(set(values) for values in mapping.values())) == allowed, + "variant topology mapping omits a required topology") + return {variant: set(values) for variant, values in mapping.items()} + + +def cases() -> list[dict]: + catalog = json.loads(CATALOG.read_text()) + designed = {case["id"]: case for module in catalog["modules"] for case in module["cases"] + if case["id"].startswith("heal.")} + document = json.loads(RECIPES.read_text()) + require(document.get("schemaVersion") == 1 and document.get("executionMode") == "operator-assisted", + "invalid Heal procedure contract") + observed = document["cases"] + require(len(observed) == 84 and {item["id"] for item in observed} == set(designed), + "Heal procedures must cover exactly the 84 canonical source cases") + require(len({item["code"] for item in observed}) == 84, "duplicate Heal procedure code") + anchors = set(re.findall(r'', OPERATIONS.read_text())) + for item in observed: + contract = designed[item["id"]] + require(item["code"] == contract["probes"]["designCode"], "Heal procedure identity mismatch") + require(bool(item["variants"]) and len(item["variants"]) == len(set(item["variants"])) + and all(re.fullmatch(r"[a-z0-9][a-z0-9-]*", variant) for variant in item["variants"]), + "invalid Heal procedure variants") + require(len(item["steps"]) >= 3 and set(item["requiredEvidence"]) == STAGES, + "incomplete Heal procedure or evidence stages") + require(bool(item["commands"]) and set(item["commands"]) <= anchors, + "Heal procedure refers to an undefined operation") + require(bool(item["recovery"]) and bool(item["verdict"]), "missing Heal recovery/verdict contract") + item["contract"] = contract + variant_topologies(item) + return sorted(observed, key=lambda item: item["code"]) + + +def select(selection: str) -> list[dict]: + inventory = cases() + if selection == "all": + return inventory + requested = selection.split(",") + require(len(requested) == len(set(requested)), "duplicate Heal case selection") + lookup = {key: item for item in inventory for key in (item["id"], item["code"])} + require(all(key in lookup for key in requested), "unknown Heal case") + selected = [lookup[key] for key in requested] + require(len({item["id"] for item in selected}) == len(selected), "same Heal case selected by code and ID") + return selected + + +def fleet_binding() -> dict: + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(nodes) == len(set(nodes)) + and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), + "Heal requires the four distinct current auto-testing SSH nodes") + user = os.environ.get("RUSTFS_SSH_USER", "azureuser") + require(bool(re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", user)), "invalid auto-testing SSH user") + return {"provider": "auto-testing", "jumpHost": socket.gethostname(), "sshUser": user, + "sshNodes": nodes, "servicePort": 9000, "verified": False, + "topologies": { + "H1": {"nodes": nodes[:1], "drivesPerNode": 4, "requiredSets": 1}, + "H2": {"nodes": nodes[:4], "drivesPerNode": 4, "crossNodeSetRequired": True}, + "H3": {"nodes": nodes[:3], "drivesPerNode": 4, "dataShards": 8, "parityShards": 4, "requiredSets": 1}, + "H4": {"nodes": nodes[:3], "drivesPerNode": 8, "dataShards": 8, "parityShards": 4, + "minimumSets": 2, "requiresExistingAdditionalDrives": True}, + "H5": {"nodes": nodes[:4], "minimumPools": 2, "poolAndSetMappingRequired": True, + "nodeRedundancyMustBeMeasured": True}, + "H6": {"nodes": [], "requiresDedicatedHostBinding": True, + "requiresPlatformEventAndRecoveryEvidence": True}}, + "beforeMutation": ["workflow fleet lock and unexpired environment lease", "actual hostname and unit/PID", + "independent mount identities and disk UUIDs", "actual pool/set/slot mapping and EC quorums", + "run-owned roots, fixture ACKs, fault witnesses and undo plan"]} + + +def compile_plan(selected: list[dict], topologies: set[str], run_id: str) -> dict: + require(bool(re.fullmatch(r"(?:gh|manual)-[a-z0-9-]{5,60}", run_id)), "invalid Heal run ID") + require(bool(topologies) and topologies <= TOPOLOGIES, "unknown Heal topology") + instances, excluded, non_applicable = [], [], [] + for item in selected: + allowed = set(item["contract"]["probes"]["topology"].split(",")) + mapping = variant_topologies(item) + for topology in sorted(allowed): + if topology not in topologies: + excluded.append({"caseId": item["id"], "topology": topology, "state": "OUT_OF_SELECTED_SCOPE"}) + for variant in item["variants"]: + if topology not in mapping[variant]: + non_applicable.append({"caseId": item["id"], "topology": topology, "branch": variant, + "state": "NOT_APPLICABLE_TO_TOPOLOGY", + "reason": item["topologyApplicabilityReason"]}) + elif topology in topologies: + instances.append({"instanceId": f"{item['code']}.{topology}.{variant}", "caseId": item["id"], + "code": item["code"], "topology": topology, "branch": variant, + "state": "NOT_RUN", "parameterBindings": {}, "sourceAssertions": item["contract"]["assertions"], + "operatorSteps": item["steps"], "operations": item["commands"], + "requiredEvidence": item["requiredEvidence"], "caseVerdict": None}) + require(bool(instances), "selected Heal cases have no requested topology") + return {"schemaVersion": 1, "runId": run_id, "createdAt": datetime.now(timezone.utc).isoformat(), + "executionMode": "operator-assisted", "state": "PLAN_ONLY", "automaticExecution": False, + "livePassClaimed": False, "recipeSha256": digest(RECIPES), "catalogSha256": digest(CATALOG), + "operationBookSha256": digest(OPERATIONS), "publicContractSha256": digest(CONTRACT), + "selectedBaseCaseCount": len(selected), + "selectedCaseIds": [item["id"] for item in selected], "selectedTopologies": sorted(topologies), + "branchTemplateCount": len(instances), "scopeExclusions": excluded, + "nonApplicableCombinations": non_applicable, "environmentBinding": fleet_binding(), "instances": instances} + + +def audit_evidence(plan: dict, directory: Path) -> dict: + require(plan.get("schemaVersion") == 1 and plan.get("state") == "PLAN_ONLY" + and plan.get("automaticExecution") is False, "invalid Heal operator plan") + require(plan.get("recipeSha256") == digest(RECIPES) and plan.get("catalogSha256") == digest(CATALOG) + and plan.get("operationBookSha256") == digest(OPERATIONS) + and plan.get("publicContractSha256") == digest(CONTRACT), "Heal plan contract changed") + expected = compile_plan(select(",".join(plan["selectedCaseIds"])), set(plan["selectedTopologies"]), plan["runId"]) + require(plan["instances"] == expected["instances"] and plan["scopeExclusions"] == expected["scopeExclusions"], + "Heal plan omitted or changed required branches") + root = directory.resolve(strict=True) + problems, recorded = [], [] + for instance in plan["instances"]: + identity = instance["instanceId"] + index = root / identity / "evidence.json" + if not index.is_file() or index.is_symlink(): + problems.append({"instanceId": identity, "reason": "missing evidence index"}) + continue + require(index.resolve().is_relative_to(root), "Heal evidence index escaped archive") + value = json.loads(index.read_text()) + require(value.get("runId") == plan["runId"] and value.get("instanceId") == identity, + "Heal evidence identity mismatch") + entries = value.get("evidence", {}) + missing = STAGES - set(entries) + if missing: + problems.append({"instanceId": identity, "reason": "missing evidence stages", "stages": sorted(missing)}) + for stage, references in entries.items(): + require(stage in STAGES and isinstance(references, list) and bool(references), "invalid Heal evidence stage") + for reference in references: + relative = Path(reference["path"]) + require(not relative.is_absolute() and ".." not in relative.parts, "unsafe Heal evidence path") + file = index.parent / relative + require(file.resolve().is_relative_to(index.parent.resolve()) and not file.is_symlink() and file.is_file(), + "missing or escaped Heal evidence file") + require(digest(file) == reference["sha256"], "Heal evidence digest mismatch") + recorded.append(identity) + return {"state": "EVIDENCE_INCOMPLETE" if problems else "EVIDENCE_COMPLETE_REVIEW_REQUIRED", + "expectedBranches": len(plan["instances"]), "recordedBranches": len(recorded), "problems": problems, + "livePassClaimed": False, "canonicalVerdict": None, + "note": "Evidence completeness does not prove physical repair or satisfy the source assertions."} + + +def render(selected: list[dict]) -> str: + lines = ["# Heal validation procedures", "", "These source procedures require live execution and branch evidence on the current auto-testing Heal fleet.", + "Use [the operations reference](validation-heal-operations.md) before injecting a fault. A procedure card or complete archive is not a PASS.", ""] + for item in selected: + lines += [f'', "", f'## {item["code"]}: {item["id"]}', "", + f'Topologies: {item["contract"]["probes"]["topology"]}. Branches: {", ".join(item["variants"])}.', ""] + lines += [f"{index}. {step}" for index, step in enumerate(item["steps"], 1)] + lines += ["", "Operations: " + " → ".join(f"[{command}](validation-heal-operations.md#{command})" for command in item["commands"]), + "", "Source assertions:", ""] + lines += [f"- {assertion}" for assertion in item["contract"]["assertions"]] + lines += ["", item["recovery"], "", item["verdict"], ""] + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("check") + sub.add_parser("list") + show = sub.add_parser("show") + show.add_argument("selection") + rendering = sub.add_parser("render") + rendering.add_argument("--selection", default="all") + rendering.add_argument("--out", type=Path) + plan = sub.add_parser("plan") + plan.add_argument("--selection", default="all") + plan.add_argument("--topologies", default="H1,H2,H3,H4,H5,H6") + plan.add_argument("--run-id", required=True) + plan.add_argument("--out", type=Path, required=True) + audit = sub.add_parser("audit") + audit.add_argument("--plan", type=Path, required=True) + audit.add_argument("--evidence-dir", type=Path, required=True) + audit.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + if args.command in {"check", "list"}: + inventory = cases() + full = compile_plan(inventory, TOPOLOGIES, "manual-heal-contract-check") + result = {"baseCases": len(inventory), "operatorBranches": sum(len(item["variants"]) for item in inventory), + "topologyBranchTemplates": len(full["instances"]), "automaticCaseCount": len(AUTOMATIC_CASES), "livePassClaimed": False} + if args.command == "list": + result["cases"] = [{"code": item["code"], "id": item["id"], "variants": item["variants"]} for item in inventory] + print(json.dumps(result, ensure_ascii=False)) + elif args.command == "show": + print(json.dumps(select(args.selection), ensure_ascii=False, indent=2)) + elif args.command == "render": + text = render(select(args.selection)) + if args.out: + args.out.write_text(text + "\n") + else: + print(text) + elif args.command == "plan": + save(args.out, compile_plan(select(args.selection), set(args.topologies.split(",")), args.run_id)) + else: + result = audit_evidence(json.loads(args.plan.read_text()), args.evidence_dir) + save(args.out, result) + return 0 if not result["problems"] else 1 + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, KeyError, ValueError) as exc: + print(f"Heal validation contract error: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/prepare_heal_meta_inspector.py b/scripts/validation_heal/prepare_heal_meta_inspector.py new file mode 100644 index 00000000..08026115 --- /dev/null +++ b/scripts/validation_heal/prepare_heal_meta_inspector.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Prepare (do not silently build) a read-only metadata inspector for one SHA.""" + +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + + +def prepare(source: Path, commit: str, output: Path) -> None: + if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise ValueError("full candidate commit required") + source = source.resolve(strict=True) + head = subprocess.run(["git", "-C", str(source), "rev-parse", "HEAD"], check=True, + text=True, capture_output=True).stdout.strip() + dirty = subprocess.run(["git", "-C", str(source), "status", "--porcelain", "--untracked-files=no"], + check=True, text=True, capture_output=True).stdout + if head != commit or dirty.strip(): + raise ValueError("inspector requires an unmodified checkout of the exact candidate SHA") + crate = source / "crates/filemeta" + if not (crate / "Cargo.toml").is_file(): + raise ValueError("candidate has no rustfs-filemeta crate") + output.mkdir(parents=True, exist_ok=False) + (output / "src").mkdir() + helper = Path(__file__).resolve().parent / "tools/heal_meta_inspect.rs" + text = helper.read_text() + (output / "src/main.rs").write_text(text) + manifest = ('[package]\nname = "heal-meta-inspect"\nversion = "0.1.0"\nedition = "2024"\n' + '[workspace]\n[dependencies]\nserde_json = "1"\n' + 'rustfs-filemeta = { path = ' + json.dumps(str(crate)) + ' }\n') + (output / "Cargo.toml").write_text(manifest) + (output / "source.json").write_text(json.dumps({"candidateCommit": head, + "candidateLockSha256": hashlib.sha256((source / "Cargo.lock").read_bytes()).hexdigest(), + "helperSourceSha256": hashlib.sha256(text.encode()).hexdigest(), + "role": "read-only decoder; not the RustFS service under test", + "built": False}, indent=2) + "\n") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--commit", required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + prepare(args.source, args.commit, args.out) + print(f"Prepared {args.out}; build on an approved host, then record its Cargo.lock and binary SHA-256.") + + +if __name__ == "__main__": + main() diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json new file mode 100644 index 00000000..e9483d49 --- /dev/null +++ b/scripts/validation_heal/recipes.json @@ -0,0 +1,3303 @@ +{ + "schemaVersion": 1, + "executionMode": "operator-assisted", + "scoring": "manual verdicts; not canonical automated dispatch", + "cases": [ + { + "code": "A01", + "id": "heal.admin-healthy-noop", + "variants": [ + "normal", + "deep" + ], + "steps": [ + "用 DB 夹具生成完整 manifest；先对每个对象保存 S3 基线和四盘分片摘要，关闭 scanner/MRF/auto-heal。", + "依次执行 START object（scanMode=1、2），WAIT 原 token 直至终态；再 STATUS 同 token 两次。", + "对每个对象的 outcome 区分 Normal 的 metadata_healthy 与 Deep 的 verified_healthy；两者均要求 healed=0、unchanged>0，Normal 不得冒充 payload 完整性证明。比较目标/控制对象物理摘要和 S3 清单，重复请求不得新建版本。" + ], + "commands": [ + "SEED_DB", + "BASELINE", + "START", + "WAIT", + "STATUS", + "VERIFY", + "PHYSICAL" + ], + "requiredEvidence": [ + "preflight", + "baseline", + "fault", + "trigger", + "control", + "physical", + "s3", + "recovery", + "logs" + ], + "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + }, + { + "code": "A02", + "id": "heal.admin-scope-isolation", + "variants": [ + "root", + "bucket", + "prefix", + "object", + "pool-set" + ], + "steps": [ + "用 DS 在两个桶的 a/、ab/、b/ 写入不同内容；另在未选 set 放控制对象；逐对象预先记录实际 pool/set。", + "分别隔离选中对象、相邻前缀及另一桶/未选 set 的一个 part。每个 selector 使用新夹具和故障 ID。", + "对象请求路径到完整 key；prefix 路径到 a/ 且 recursive=true；bucket 路径只含桶；root 为 /；pool/set 写入 body。START 后 WAIT。", + "先 PHYSICAL：选择范围内全部重建、范围外控制故障仍缺失；然后 GET 验证，避免验证请求提前修好控制故障。root 必须包含整个专用服务清单。" + ], + "commands": [ + "SEED_DS", + "BASELINE", + "FAULT", + "START", + "WAIT", + "PHYSICAL", + "VERIFY" + ], + "requiredEvidence": [ + "preflight", + "baseline", + "fault", + "trigger", + "control", + "physical", + "s3", + "recovery", + "logs" + ], + "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + }, + { + "code": "A03", + "id": "heal.admin-token-cross-node-status", + "variants": [ + "coordinator-to-peer", + "coordinator-to-stable-endpoint" + ], + "steps": [ + "在 H2 同一跨节点 set 写入 DB；保存协调者、目标、稳定入口分别对应的端点，不把同一节点的两个别名当跨节点。", + "隔离目标节点 part；从 n0 START，记录原始 clientToken；通过 n1 和 stable 对同一路径、同 token 执行 STATUS。", + "按同一任务身份对齐三端状态，最多等候冻结的状态收敛预算；终态及核心计数必须一致，404/unknown 不能视为结束。", + "WAIT 后先物理证明目标重建，再从所有 S3 入口 VERIFY；保存每个请求实际连接的节点日志。" + ], + "commands": [ + "SEED_DB", + "FAULT", + "START", + "STATUS", + "WAIT", + "PHYSICAL", + "VERIFY" + ], + "requiredEvidence": [ + "preflight", + "baseline", + "fault", + "trigger", + "control", + "physical", + "s3", + "recovery", + "logs" + ], + "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + }, + { + "code": "A04", + "id": "heal.admin-dry-run", + "variants": [ + "missing-remove0-recreate0", + "missing-remove1-recreate0", + "bitflip-remove0-recreate1", + "bitflip-remove1-recreate1", + "versioned-missing", + "versioned-bitflip" + ], + "steps": [ + "分别用 DB 与 DV 建夹具，冻结全部 versionId/marker；关闭后台来源，不在注入后 GET。", + "在原始 part 上做 missing/bitflip，每组独立新目录；保存故障后的 PHYSICAL 快照作为 dry-run 前基线。", + "START body 同时传 dryRun=true、scanMode=2 和该实例的 remove/recreate；WAIT 仅接受 dry_run_observed，不能要求 dry-run 持有 incarnation。", + "前后所有分片/元数据/marker 字节与版本清单相同；没有 repaired ACK。完成非变更证明后恢复故障，再 VERIFY。" + ], + "commands": [ + "SEED_DB", + "SEED_DV", + "BASELINE", + "FAULT", + "PHYSICAL", + "START", + "WAIT", + "RECOVER", + "VERIFY" + ], + "requiredEvidence": [ + "preflight", + "baseline", + "fault", + "trigger", + "control", + "physical", + "s3", + "recovery", + "logs" + ], + "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + }, + { + "code": "A05", + "id": "heal.admin-overlap-and-force-start", + "variants": [ + "same-scope", + "parent-child", + "disjoint", + "force-start" + ], + "steps": [ + "用 DX 超过 128 项的外部分片夹具注入多对象缺片；预先从固定候选记录 overlap policy（复用 token 或冲突错误），禁止接受任意结果。", + "START 递归范围，WINDOW 证明 0> "${evidence}/cleanup.log" 2>&1 || true + fi + exit "${exit_code}" +} +trap on_exit EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +(bootstrap) > "${evidence}/setup.log" 2>&1 || bootstrap_rc=$? +cat "${evidence}/setup.log" +if [ "${bootstrap_rc}" -eq 0 ]; then + for node in "${NODES[@]}"; do + node_sha="$(ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" sha256sum /usr/bin/rustfs | awk '{print $1}')" + [[ "${node_sha}" =~ ^[0-9a-f]{64}$ ]] || die "installed candidate digest unavailable on ${node}" + if [ -n "${binary_sha}" ] && [ "${binary_sha}" != "${node_sha}" ]; then die "node candidate digests differ"; fi + binary_sha="${node_sha}" + done + python3 -B "${ROOT}/scripts/validation_heal/healthy.py" --candidate "${candidate}" \ + --testing-commit "${testing_commit}" --binary-sha "${binary_sha}" --rc-bin "$(command -v rc)" \ + --evidence "${evidence}" > "${evidence}/test.log" 2>&1 || case_rc=$? + cat "${evidence}/test.log" + jq -e '.cleanup == "PASS"' "${evidence}/healthy-summary.json" >/dev/null 2>&1 || cleanup_status=FAIL +else + case_rc=1 +fi +(finalize_package) > "${evidence}/cleanup.log" 2>&1 || cleanup_status=FAIL +package_active=0 +cat "${evidence}/cleanup.log" +[ -n "${binary_sha}" ] || die "candidate was not installed; retain RUNNING result and setup evidence" +verdict=FAIL reason="Heal source assertions did not pass on every required H1/H2 branch" +if [ "${case_rc}" -eq 0 ] && [ "${cleanup_status}" = PASS ]; then + verdict=PASS; reason="" +elif [ "${case_rc}" -eq 0 ]; then + verdict=INFRA_ERROR; reason="Heal environment finalization failed" +fi +args=("${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool + --result-file "${artifacts}/cases/${case_id}.json" --verdict "${verdict}" --cleanup-status "${cleanup_status}" + --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --binary-sha256 "${binary_sha}" + --rc-bin "$(command -v rc)" --evidence "evidence/${case_id}/setup.log" + --evidence "evidence/${case_id}/test.log" --evidence "evidence/${case_id}/cleanup.log" + --evidence "evidence/${case_id}/healthy-summary.json") +if [ -n "${reason}" ]; then args+=(--reason "${reason}"); fi +python3 -B "${args[@]}" +[ "${verdict}" = PASS ] diff --git a/scripts/validation_heal/source.json b/scripts/validation_heal/source.json new file mode 100644 index 00000000..cf75fd8a --- /dev/null +++ b/scripts/validation_heal/source.json @@ -0,0 +1,55 @@ +{ + "source_commit": "524eb5b32c3236497fdd8a9271df0494fa3d9974", + "files": [ + { + "source_path": "scenarios/heal-recipes.json", + "target_path": "scripts/validation_heal/recipes.json", + "source_sha256": "4d81c83fa63c5b0720ff5f031ee595ea3c1f9e39f712e6d131bc50b9b12b164f" + }, + { + "source_path": "scripts/heal_probe.py", + "target_path": "scripts/validation_heal/heal_probe.py", + "source_sha256": "3262c6744c6ae387c3f56095817740b3a34de7a1b9d34cb463b9167df68cef43" + }, + { + "source_path": "scripts/heal_fault.py", + "target_path": "scripts/validation_heal/heal_fault.py", + "source_sha256": "c38ba8f9d129b67eacb8dd7920e534f6bbd40ec261870ebdde6823f067b98fd7" + }, + { + "source_path": "scripts/heal_workload.py", + "target_path": "scripts/validation_heal/heal_workload.py", + "source_sha256": "216e9ca52545f1d52dd6f8fd63a7d634209e0780198355597d2489df61ece383" + }, + { + "source_path": "scripts/heal_workload_journal.py", + "target_path": "scripts/validation_heal/heal_workload_journal.py", + "source_sha256": "a07fe6c130c60b523cca71d8f9d2c2fb4bd270eb052619a5a58d15373370041c" + }, + { + "source_path": "docs/heal-operations.md", + "target_path": "docs/validation-heal-operations.md", + "source_sha256": "23f561ba81ff06c6450533ce1f8f9a3182a576504f5e6d6aca5494127d324bad" + }, + { + "source_path": "scripts/prepare_heal_meta_inspector.py", + "target_path": "scripts/validation_heal/prepare_heal_meta_inspector.py", + "source_sha256": "e940b807f2ce37b8a04270913577221038808ff9e19454048251361fd9d16211" + }, + { + "source_path": "tools/heal_meta_inspect.rs", + "target_path": "scripts/validation_heal/tools/heal_meta_inspect.rs", + "source_sha256": "f102a00ce5690fb6a286315b76f925cd27e0d2143c250a949e43da10d03faaab" + }, + { + "source_path": "scripts/heal_signing.py", + "target_path": "scripts/validation_heal/heal_signing.py", + "source_sha256": "179394da660abad9c9a2f03b3eed7d4d46f5114940244f0ab8bbdb5e48f15537" + }, + { + "source_path": "docs/heal-validation.md", + "target_path": "docs/validation-heal-contract.md", + "source_sha256": "c82cf8142e6400431eeeb58c936494ba2614a194d5bf92cde806cccf4aed3a85" + } + ] +} diff --git a/scripts/validation_heal/tools/heal_meta_inspect.rs b/scripts/validation_heal/tools/heal_meta_inspect.rs new file mode 100644 index 00000000..e173c5cf --- /dev/null +++ b/scripts/validation_heal/tools/heal_meta_inspect.rs @@ -0,0 +1,82 @@ +// Read-only inspector linked to the exact candidate's rustfs-filemeta crate. +// Do not print inline bytes, user metadata, encryption envelopes, or IAM data. +use rustfs_filemeta::FileMeta; +use serde_json::json; + +fn main() -> Result<(), Box> { + let args: Vec = std::env::args().collect(); + if args.len() != 4 { + return Err("usage: heal-meta-inspect XL_META_FILE BUCKET KEY".into()); + } + let metadata = std::fs::symlink_metadata(&args[1])?; + if !metadata.is_file() || metadata.len() > 64 * 1024 * 1024 { + return Err("expected bounded, non-symlink xl.meta evidence copy".into()); + } + let bytes = std::fs::read(&args[1])?; + let meta = FileMeta::load(&bytes)?; + if meta.versions.is_empty() { + return Err("empty metadata is not an authoritative version manifest".into()); + } + let decoded = meta.into_file_info_versions(&args[2], &args[3], true)?; + meta.data.validate()?; + let versions: Result, Box> = decoded + .versions + .iter() + .map(|info| -> Result<_, Box> { + let role = if info.erasure.index == 0 { + "not-an-erasure-shard" + } else if info.erasure.index <= info.erasure.data_blocks { + "data" + } else { + "parity" + }; + // Locate only the exact decoded inline value. Exposing the bytes + // would leak user content; offset/length plus an occurrence count + // lets a bounded injector target the value instead of guessing at + // an xl.meta position. + let data_key = info + .version_id + .filter(|version_id| !version_id.is_nil()) + .map(|version_id| version_id.to_string()) + .unwrap_or_else(|| "null".to_string()); + let decoded_inline = if info.inline_data() { + meta.data.find(&data_key)? + } else { + None + }; + let inline = decoded_inline.as_ref().filter(|data| !data.is_empty()).map(|data| { + let offsets: Vec<_> = bytes + .windows(data.len()) + .enumerate() + .filter_map(|(offset, candidate)| (candidate == data.as_slice()).then_some(offset)) + .collect(); + json!({ + "length": data.len(), + "rawOccurrenceCount": offsets.len(), + "rawOffset": (offsets.len() == 1).then_some(offsets[0]) + }) + }); + Ok(json!({ + "versionId": info.version_id.map(|id| id.to_string()), + "isLatest": info.is_latest, "deleted": info.deleted, + "dataDir": info.data_dir.map(|id| id.to_string()), "size": info.size, + "inline": inline, + "erasure": info.erasure, "role": role, + "parts": info.parts.iter().map(|part| json!({ + "number": part.number, "size": part.size, "actualSize": part.actual_size, + "etag": part.etag + })).collect::>() + })) + }) + .collect(); + let versions = versions?; + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "schemaVersion": 1, "bucket": args[2], "key": args[3], + "metadataVersion": meta.meta_ver, "versions": versions, + "physicalBitrotVerified": false, "authorityQuorumProved": false + }))? + ); + Ok(()) +} diff --git a/test_validation_heal.py b/test_validation_heal.py new file mode 100644 index 00000000..48200b95 --- /dev/null +++ b/test_validation_heal.py @@ -0,0 +1,137 @@ +"""Check preservation of Heal branches and prevent evidence from becoming PASS.""" + +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +import copy +from unittest.mock import patch + +from scripts.validation_heal.matrix import STAGES, TOPOLOGIES, audit_evidence, cases, compile_plan, select +from scripts.validation_heal.healthy import validate_healthy, validate_topology +from scripts.validation_heal.fleet_node import paths + + +class HealContractTest(unittest.TestCase): + def test_preserves_all_source_cases_and_applicable_branches(self): + inventory = cases() + plan = compile_plan(inventory, TOPOLOGIES, "manual-heal-unit-test") + self.assertEqual(len(inventory), 84) + self.assertEqual(sum(len(item["variants"]) for item in inventory), 301) + self.assertEqual(plan["branchTemplateCount"], 532) + self.assertEqual(len(plan["nonApplicableCombinations"]), 4) + self.assertFalse(plan["automaticExecution"]) + self.assertFalse(plan["livePassClaimed"]) + self.assertFalse(plan["environmentBinding"]["verified"]) + self.assertTrue(plan["environmentBinding"]["topologies"]["H4"]["requiresExistingAdditionalDrives"]) + + def test_narrowed_topology_keeps_missing_scope_explicit(self): + plan = compile_plan(select("A01,A03"), {"H1"}, "manual-heal-unit-test") + self.assertEqual(len(plan["instances"]), 2) + self.assertEqual({item["caseId"] for item in plan["scopeExclusions"]}, + {"heal.admin-healthy-noop", "heal.admin-token-cross-node-status"}) + self.assertFalse(plan["livePassClaimed"]) + with self.assertRaisesRegex(ValueError, "same Heal case"): + select("A01,heal.admin-healthy-noop") + + def fixture_archive(self, root, plan): + for instance in plan["instances"]: + directory = root / instance["instanceId"] + directory.mkdir() + proof = {"runId": plan["runId"], "instanceId": instance["instanceId"], "evidence": {}} + for stage in STAGES: + value = (stage + " live observation").encode() + (directory / (stage + ".txt")).write_bytes(value) + proof["evidence"][stage] = [{"path": stage + ".txt", "sha256": hashlib.sha256(value).hexdigest()}] + (directory / "evidence.json").write_text(json.dumps(proof)) + + def test_complete_archive_requires_review_and_never_manufactures_pass(self): + plan = compile_plan(select("A01"), {"H1"}, "manual-heal-unit-test") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.fixture_archive(root, plan) + result = audit_evidence(plan, root) + self.assertEqual(result["state"], "EVIDENCE_COMPLETE_REVIEW_REQUIRED") + self.assertFalse(result["livePassClaimed"]) + self.assertIsNone(result["canonicalVerdict"]) + first = root / plan["instances"][0]["instanceId"] + (first / "physical.txt").write_text("different evidence") + with self.assertRaisesRegex(ValueError, "digest mismatch"): + audit_evidence(plan, root) + + + def test_rejects_dropped_branch_and_escaped_evidence(self): + plan = compile_plan(select("A01"), {"H1"}, "manual-heal-unit-test") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.fixture_archive(root, plan) + dropped = {**plan, "instances": plan["instances"][:1]} + with self.assertRaisesRegex(ValueError, "omitted or changed"): + audit_evidence(dropped, root) + first = root / plan["instances"][0]["instanceId"] + index = first / "evidence.json" + proof = json.loads(index.read_text()) + proof["evidence"]["physical"][0]["path"] = "../outside.txt" + index.write_text(json.dumps(proof)) + with self.assertRaisesRegex(ValueError, "unsafe Heal evidence"): + audit_evidence(plan, root) + + +class HealHealthyOracleTest(unittest.TestCase): + def receipt(self, disposition="metadata_healthy"): + return {"summary": "finished", "outcome": {"execution": {"state": "completed"}, + "coverage": "complete", "objectsTruncated": False, + "counters": {"processed": 1, "healed": 0, "unchanged": 1, "skipped": 0, + "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False}, + "objects": [{"identity": {"bucket": "fixture", "object": "object.bin", "poolIndex": 0, + "setIndex": 0, "bucketIncarnationId": "incarnation"}, + "disposition": {"state": disposition}}]}} + + def test_metadata_is_not_deep_integrity_and_legacy_finish_is_not_success(self): + healthy = self.receipt() + validate_healthy(healthy, "fixture", "object.bin", "metadata_healthy") + with self.assertRaisesRegex(ValueError, "expected verified_healthy"): + validate_healthy(healthy, "fixture", "object.bin", "verified_healthy") + validate_healthy(self.receipt("verified_healthy"), "fixture", "object.bin", "verified_healthy") + for changed in ({"summary": "finished"}, {**healthy, "truncated": True}): + with self.assertRaises(ValueError): + validate_healthy(changed, "fixture", "object.bin", "metadata_healthy") + + def test_rejects_wrong_identity_unknown_counters_and_duplicate_receipts(self): + for kind in ("wrong-object", "unknown", "missing-incarnation", "duplicate", "boolean-counter"): + value = copy.deepcopy(self.receipt()) + outcome = value["outcome"] + if kind == "wrong-object": outcome["objects"][0]["identity"]["object"] = "other.bin" + elif kind == "unknown": outcome["counters"]["unknown"] = 1 + elif kind == "missing-incarnation": outcome["objects"][0]["identity"].pop("bucketIncarnationId") + elif kind == "duplicate": outcome["objects"].append(outcome["objects"][0]) + else: outcome["counters"]["processed"] = True + with self.subTest(kind=kind), self.assertRaises(ValueError): + validate_healthy(value, "fixture", "object.bin", "metadata_healthy") + + def test_h2_requires_one_set_spanning_all_sixteen_disks(self): + disks = [str(index) for index in range(16)] + snapshot = {f"node{node}": {"formats": [{"value": {"format": "xl", "id": "pool", + "xl": {"sets": [disks], "this": disks[(node - 1) * 4 + disk]}}} for disk in range(4)]} + for node in range(1, 5)} + self.assertEqual(validate_topology(snapshot, "H2")["independentDrives"], 16) + bad = copy.deepcopy(snapshot) + bad["node4"]["formats"][0]["value"]["xl"]["this"] = "0" + with self.assertRaisesRegex(ValueError, "mapping differs"): + validate_topology(bad, "H2") + with self.assertRaisesRegex(ValueError, "node count"): + validate_topology({"node1": snapshot["node1"]}, "H2") + + def test_node_paths_cannot_select_shared_roots_or_arbitrary_run(self): + with tempfile.TemporaryDirectory() as directory, patch("scripts.validation_heal.fleet_node.STATE_ROOT", Path(directory).resolve()): + state, roots = paths("gh-12345-1-h1") + self.assertTrue(all(root.parts[-2:] == ("heal-validation", "gh-12345-1-h1") for root in roots)) + for value in ("../mnmd", "manual-arbitrary", "gh-12345-1-h4"): + with self.assertRaisesRegex(ValueError, "run ID"): + paths(value) + + + +if __name__ == "__main__": + unittest.main() From 054949940da5e504e272e730935f4b8021c6c04d Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 20:30:41 +0800 Subject: [PATCH 021/343] fix(validation): handle a clean fleet before Heal deployment --- scripts/validation_heal/run_case.sh | 28 +++++++++++++++++++++++++--- scripts/validation_result.py | 5 +++-- test_validation_migration.py | 9 +++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 9e816663..11e6a704 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -40,7 +40,9 @@ python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profi bootstrap() { local node - service_action_all stop + for node in "${NODES[@]}"; do + stop_existing "${node}" + done step1_download for node in "${NODES[@]}"; do { @@ -52,11 +54,24 @@ bootstrap() { done } +stop_existing() { + local node="$1" + run_remote "${node}" <<'REMOTE' +set -euo pipefail +load_state="$(systemctl show rustfs -p LoadState --value)" +if [ "${load_state}" != not-found ]; then + sudo -n systemctl stop rustfs +fi +! systemctl is-active --quiet rustfs +REMOTE +} + finalize_package() { local node failed=0 for node in "${NODES[@]}"; do + (stop_existing "${node}") || failed=1 ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ - 'sudo -n systemctl stop rustfs && sudo -n dpkg -P rustfs' || failed=1 + 'if dpkg-query -W -f="${Status}" rustfs 2>/dev/null | grep -q "install ok installed"; then sudo -n dpkg -P rustfs; fi' || failed=1 ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ '! systemctl is-active --quiet rustfs' || failed=1 done @@ -96,7 +111,14 @@ fi (finalize_package) > "${evidence}/cleanup.log" 2>&1 || cleanup_status=FAIL package_active=0 cat "${evidence}/cleanup.log" -[ -n "${binary_sha}" ] || die "candidate was not installed; retain RUNNING result and setup evidence" +[ -n "${binary_sha}" ] || { + python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ + --result-file "${artifacts}/cases/${case_id}.json" --verdict INFRA_ERROR --cleanup-status "${cleanup_status}" \ + --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --rc-bin "$(command -v rc)" \ + --evidence "evidence/${case_id}/setup.log" --evidence "evidence/${case_id}/cleanup.log" \ + --reason "Heal candidate bootstrap failed before a verified binary was installed" + exit 1 +} verdict=FAIL reason="Heal source assertions did not pass on every required H1/H2 branch" if [ "${case_rc}" -eq 0 ] && [ "${cleanup_status}" = PASS ]; then verdict=PASS; reason="" diff --git a/scripts/validation_result.py b/scripts/validation_result.py index 2fb24f12..4d074bcf 100644 --- a/scripts/validation_result.py +++ b/scripts/validation_result.py @@ -63,9 +63,10 @@ def main() -> int: parser.error("terminal results require candidate manifest, rc binary, and testing commit") candidate = json.loads(args.candidate_manifest.read_text(encoding="utf-8")) binary_sha256 = args.binary_sha256 or candidate.get("binary_sha256") - if not isinstance(binary_sha256, str) or len(binary_sha256) != 64 or any( + bootstrap_error = args.verdict == "INFRA_ERROR" and binary_sha256 is None + if not bootstrap_error and (not isinstance(binary_sha256, str) or len(binary_sha256) != 64 or any( character not in "0123456789abcdef" for character in binary_sha256 - ): + )): parser.error("terminal results require a valid installed binary SHA-256") if candidate.get("binary_sha256") and candidate["binary_sha256"] != binary_sha256: parser.error("installed binary SHA-256 differs from candidate manifest") diff --git a/test_validation_migration.py b/test_validation_migration.py index 6a999998..e62a7262 100644 --- a/test_validation_migration.py +++ b/test_validation_migration.py @@ -181,6 +181,15 @@ def test_writer_and_report_contract(self) -> None: ] subprocess.run(command + ["--verdict", "RUNNING"], check=True, capture_output=True) self.assertEqual("RUNNING", json.loads((results / "s3.basic-object-round-trip.json").read_text())["verdict"]) + identity = ["--candidate-manifest", str(candidate), "--rc-bin", str(rc), "--testing-commit", "d" * 40] + subprocess.run(command + identity + ["--verdict", "INFRA_ERROR", "--cleanup-status", "PASS", + "--reason", "candidate install failed"], check=True, capture_output=True) + bootstrap = json.loads((results / "s3.basic-object-round-trip.json").read_text()) + self.assertEqual(bootstrap["verdict"], "INFRA_ERROR") + self.assertIsNone(bootstrap["provenance"]["binary_sha256"]) + invalid_pass = subprocess.run(command + identity + ["--verdict", "PASS", "--cleanup-status", "PASS", + "--evidence", "evidence.json"], capture_output=True) + self.assertNotEqual(invalid_pass.returncode, 0) subprocess.run(command + [ "--verdict", "PASS", "--cleanup-status", "PASS", "--candidate-manifest", str(candidate), From 45bd5cd9d6149f150951b9f96d2e135d2d20a8c5 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 20:35:16 +0800 Subject: [PATCH 022/343] fix(validation): send the complete candidate Heal options --- scripts/validation_heal/healthy.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 3b5fe4f7..aff48755 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -185,16 +185,19 @@ def run_topology(args, topology, nodes): key = entry["key"] path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + quote(key, safe="/") options = {"recursive": False, "remove": False, "recreate": False, "scanMode": mode, - "dryRun": False, "poolIndex": 0, "setIndex": 0} + "dryRun": False, "updateParity": False, "nolock": False, + "readRepair": False, "pool": 0, "set": 0} started = fleet.record("trigger-" + branch, probe.admin(path, [], json.dumps(options).encode())) - token = (started.get("response") or {}).get("clientToken") - require(started.get("httpStatus") == 200 and isinstance(token, str) and bool(token), - "healthy Heal task admission failed") + require(started.get("httpStatus") == 200 and isinstance(started.get("response"), dict), + f"healthy Heal task admission failed (HTTP {started.get('httpStatus')})") + token = started["response"].get("clientToken") + require(isinstance(token, str) and bool(token), "healthy Heal task admission omitted its token") active.append((path, token)) deadline = time.monotonic() + 120 while True: observed = fleet.record("status-" + branch, probe.admin(path, [("clientToken", token)], b"")) - require(observed.get("httpStatus") == 200, "Heal status request failed") + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), + "Heal status request failed") terminal = observed.get("response") or {} if terminal.get("summary") != "running": break From 8da6bbc0dfc3cfef8681f3b770bc71a3950fd06e Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 20:37:59 +0800 Subject: [PATCH 023/343] feat(validation): migrate all Heal invalid-request branches --- .../workflows/validation-catalog-tests.yml | 23 +++++++ scripts/run_validation_case.sh | 2 +- scripts/validation_heal/healthy.py | 66 +++++++++++++++++-- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 6 +- test_validation_heal.py | 10 ++- 6 files changed, 99 insertions(+), 10 deletions(-) diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 3c28b7d5..5b544b62 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -15,6 +15,16 @@ on: - rustfs_pool_expand.sh - test_validation_replication.py - scripts/validation_replication/** + - test_validation_heal.py + - scripts/validation_heal/** + - docs/validation-heal-*.md + - rustfs_heal_test.sh + - test_validation_oidc.py + - scripts/validation_oidc.py + - scripts/validation_oidc/** + - test_validation_webhooks.py + - scripts/validation_webhooks.py + - scripts/run_validation_case.sh - rustfs-replication-test.sh - rustfs-storage-test.sh - test_validation_scanner_usage.sh @@ -34,6 +44,16 @@ on: - rustfs_pool_expand.sh - test_validation_replication.py - scripts/validation_replication/** + - test_validation_heal.py + - scripts/validation_heal/** + - docs/validation-heal-*.md + - rustfs_heal_test.sh + - test_validation_oidc.py + - scripts/validation_oidc.py + - scripts/validation_oidc/** + - test_validation_webhooks.py + - scripts/validation_webhooks.py + - scripts/run_validation_case.sh - rustfs-replication-test.sh - rustfs-storage-test.sh - test_validation_scanner_usage.sh @@ -55,5 +75,8 @@ jobs: - run: python3 -B -m unittest -v test_validation_ilm.py - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B scripts/validation_heal/matrix.py check + - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh - run: bash test_validation_scanner_usage.sh diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 864f0e47..4534715d 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop) + heal.admin-healthy-noop|heal.admin-invalid-request) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index aff48755..2baa7b11 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -18,6 +18,7 @@ import time from urllib import request from urllib.parse import quote +import xml.etree.ElementTree as ET try: from .heal_probe import Probe, canonical, compare_inventories, require, save @@ -29,6 +30,55 @@ NODE_SOURCE = Path(__file__).with_name("fleet_node.py") ROOT = Path(__file__).resolve().parents[2] +BASE_OPTIONS = {"recursive": False, "remove": False, "recreate": False, "scanMode": 2, + "dryRun": False, "updateParity": False, "nolock": False, "readRepair": False, "pool": 0, "set": 0} + + +def validate_rejection(observed, expected_code): + require(observed.get("httpStatus") == 400, "invalid Heal request was not rejected with HTTP 400") + body = observed.get("response") + if isinstance(body, str): + code = next((element.text for element in ET.fromstring(body).iter() + if element.tag.rsplit("}", 1)[-1] == "Code"), None) + elif isinstance(body, dict): + code = body.get("Code") or body.get("code") + require(not body.get("clientToken"), "invalid request admitted a Heal task") + else: + code = None + require(code == expected_code, f"invalid Heal request expected {expected_code}; observed {code}") + + +def invalid_requests(probe, fleet, bucket, before, baseline): + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/objects/0000.bin" + variants = [ + ("duplicate-query", BASE_OPTIONS, [("clientToken", "a"), ("clientToken", "b")], "InvalidArgument", None), + ("invalid-bool", BASE_OPTIONS, [("forceStop", "not-a-bool")], "InvalidArgument", None), + ("invalid-scan", {**BASE_OPTIONS, "scanMode": 99}, [], "InvalidRequest", None), + ("conflicting-flags", BASE_OPTIONS, [("forceStart", "true"), ("forceStop", "true")], "InvalidRequest", None), + ("invalid-selector", {**BASE_OPTIONS, "pool": -1}, [], "InvalidRequest", None), + ("oversize-body", BASE_OPTIONS, [], "EntityTooLarge", b" " * (1024**2 + 1)), + ] + branches = {} + for name, options, query, code, raw in variants: + observed = fleet.record("trigger-" + name, probe.admin(path, query, raw if raw is not None else json.dumps(options).encode())) + validate_rejection(observed, code) + for endpoint in probe.binding["endpoints"]: + state = fleet.record("control-" + name, Probe(probe.binding, endpoint).admin( + "/rustfs/admin/v3/background-heal/status", [], b"")) + require(state.get("httpStatus") == 200 and isinstance(state.get("response"), dict), + "invalid-request task isolation status is unavailable") + value = state["response"] + coverage = value.get("coverage") or {} + require(value.get("clusterStatusComplete") is True + and coverage.get("expected") == len(fleet.nodes) and coverage.get("responded") == len(fleet.nodes) + and coverage.get("unknown") == 0 and value.get("healActiveTasks") == 0 and value.get("healQueueLength") == 0, + "invalid request left active responsibility or incomplete peer coverage") + branches[name] = {"verdict": "PASS", "httpStatus": 400, "code": code, "noActiveResponsibility": True} + after = fleet.snapshot("physical-after-negatives") + require(physical_content(before) == physical_content(after), "invalid requests changed the physical fixture") + inventory = fleet.record("s3-after-negatives", probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("negative-isolation-proof", {"physicalUnchanged": True, "s3": compare_inventories(baseline, inventory)}) + return branches def physical_content(snapshot): @@ -179,14 +229,16 @@ def run_topology(args, topology, nodes): fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) before = fleet.snapshot("physical-baseline") fleet.record("fault", {"fault": "F0", "injected": False, "backgroundRepairDisabled": True}) - for branch, mode, disposition in (("normal", 1, "metadata_healthy"), ("deep", 2, "verified_healthy")): + if args.case_id == "heal.admin-invalid-request": + result["branches"] = invalid_requests(probe, fleet, bucket, before, baseline) + healthy_branches = (("normal", 1, "metadata_healthy"), ("deep", 2, "verified_healthy")) \ + if args.case_id == "heal.admin-healthy-noop" else () + for branch, mode, disposition in healthy_branches: receipts = [] for entry in expected: key = entry["key"] path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + quote(key, safe="/") - options = {"recursive": False, "remove": False, "recreate": False, "scanMode": mode, - "dryRun": False, "updateParity": False, "nolock": False, - "readRepair": False, "pool": 0, "set": 0} + options = {**BASE_OPTIONS, "scanMode": mode} started = fleet.record("trigger-" + branch, probe.admin(path, [], json.dumps(options).encode())) require(started.get("httpStatus") == 200 and isinstance(started.get("response"), dict), f"healthy Heal task admission failed (HTTP {started.get('httpStatus')})") @@ -243,6 +295,8 @@ def interrupted(signum, frame): signal.signal(signal.SIGTERM, interrupted) parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=("heal.admin-healthy-noop", "heal.admin-invalid-request"), + default="heal.admin-healthy-noop") parser.add_argument("--testing-commit", required=True) parser.add_argument("--binary-sha", required=True) parser.add_argument("--rc-bin", type=Path, required=True) @@ -262,11 +316,11 @@ def interrupted(signum, frame): if result["verdict"] != "PASS" or result["cleanup"] != "PASS": break finally: - save(args.evidence / "healthy-summary.json", {"caseId": "heal.admin-healthy-noop", "results": results, + save(args.evidence / "healthy-summary.json", {"caseId": args.case_id, "results": results, "cleanup": "PASS" if results and all(value["cleanup"] == "PASS" for value in results) else "FAIL", "complete": len(results) == 2 and all(value["verdict"] == "PASS" and value["cleanup"] == "PASS" for value in results)}) require(len(results) == 2 and all(value["verdict"] == "PASS" and value["cleanup"] == "PASS" for value in results), - "A01 requires both H1/H2 and normal/deep; inspect topology-result evidence") + "Heal case requires every H1/H2 branch; inspect topology-result evidence") if __name__ == "__main__": diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 9ac97d30..929f546d 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.admin-healthy-noop"} +AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 11e6a704..fbb912fd 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -16,7 +16,10 @@ while [ "$#" -gt 0 ]; do *) die "unknown Heal validation option: $1" ;; esac done -[ "${case_id}" = heal.admin-healthy-noop ] || die "automatic Heal executor is not migrated: ${case_id}" +case "${case_id}" in + heal.admin-healthy-noop|heal.admin-invalid-request) ;; + *) die "automatic Heal executor is not migrated: ${case_id}" ;; +esac [ "${#NODES[@]}" -ge 4 ] || die "A01 H1/H2 requires four current auto-testing Heal nodes" NODES=("${NODES[0]}" "${NODES[1]}" "${NODES[2]}" "${NODES[3]}") if [ "${DRY_RUN}" -eq 1 ]; then @@ -101,6 +104,7 @@ if [ "${bootstrap_rc}" -eq 0 ]; then binary_sha="${node_sha}" done python3 -B "${ROOT}/scripts/validation_heal/healthy.py" --candidate "${candidate}" \ + --case-id "${case_id}" \ --testing-commit "${testing_commit}" --binary-sha "${binary_sha}" --rc-bin "$(command -v rc)" \ --evidence "${evidence}" > "${evidence}/test.log" 2>&1 || case_rc=$? cat "${evidence}/test.log" diff --git a/test_validation_heal.py b/test_validation_heal.py index 48200b95..fa2b139e 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -9,7 +9,7 @@ from unittest.mock import patch from scripts.validation_heal.matrix import STAGES, TOPOLOGIES, audit_evidence, cases, compile_plan, select -from scripts.validation_heal.healthy import validate_healthy, validate_topology +from scripts.validation_heal.healthy import validate_healthy, validate_topology, validate_rejection from scripts.validation_heal.fleet_node import paths @@ -79,6 +79,14 @@ def test_rejects_dropped_branch_and_escaped_evidence(self): class HealHealthyOracleTest(unittest.TestCase): + def test_request_rejection_needs_the_exact_server_code(self): + good = {"httpStatus": 400, "response": "InvalidArgument"} + validate_rejection(good, "InvalidArgument") + for bad in ({**good, "httpStatus": 200}, {"httpStatus": 400, "response": "AccessDenied"}, + {"httpStatus": 400, "response": {"Code": "InvalidArgument", "clientToken": "admitted"}}): + with self.assertRaises(ValueError): + validate_rejection(bad, "InvalidArgument") + def receipt(self, disposition="metadata_healthy"): return {"summary": "finished", "outcome": {"execution": {"state": "completed"}, "coverage": "complete", "objectsTruncated": False, From 656ae2b09a051edbae3655fb76470ced0271d513 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 20:46:38 +0800 Subject: [PATCH 024/343] feat(validation): consume candidate-bound Heal metadata tools --- docs/validation-heal-contract.md | 2 +- docs/validation-heal-operations.md | 2 ++ docs/validation-migration-map.md | 13 ++++---- scripts/validation_heal/fleet_node.py | 28 ++++++++++++++-- scripts/validation_heal/healthy.py | 36 +++++++++++++++----- scripts/validation_heal/inspector.py | 48 +++++++++++++++++++++++++++ scripts/validation_heal/run_case.sh | 5 +++ test_validation_heal.py | 27 +++++++++++++++ 8 files changed, 143 insertions(+), 18 deletions(-) create mode 100644 scripts/validation_heal/inspector.py diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index df96c542..de2df2b8 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -`heal.admin-healthy-noop` 已接入自动分发，覆盖 H1/H2 的 normal/deep；其余操作卡保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 +`heal.admin-healthy-noop` 和 `heal.admin-invalid-request` 已接入自动分发，分别覆盖 H1/H2 的 normal/deep 和六项非法请求分支；其余操作卡保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index c66522fc..4bd5c017 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -18,6 +18,8 @@ python scripts/validation_heal/matrix.py plan --selection A01,A04,C01,C02,C04,J0 `plan` 只冻结选择/分支和对应操作，不部署、不评分。`show` 同时给出步骤、命令入口、验收和恢复要求。不同拓扑是不同实例；未选拓扑会保留 `scopeExclusions`，不能据局部结果声称全矩阵通过。 +自动用例通过 RustFS Functional Chain 的 `validation_case` 和精确 `auto_testing_sha` 分发。先用同一不可变候选的 `build_run_id`/`build_run_attempt`、`validation_tools_only=true` 构建只读 metadata inspector；live 分发提供该成功构建的 `validation_tools_run_id` 与 `validation_tools_run_attempt`。工具的 candidate/helper SHA、Linux 二进制 SHA 和独立 Cargo.lock 摘要均须通过检查，再复制到本例节点。普通 live 执行消费构建产物，不需要 RustFS checkout。 + | 工具 | 已实现的职责 | 不代表什么 | | --- | --- | --- | | `matrix.py` | 84 项/301 分支索引、逐例操作卡、冻结计划、证据完整性审计 | 不自动分配拓扑，不根据文件存在就判 PASS | diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 34a4dce4..39b0ca50 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -60,8 +60,7 @@ The driver reuses the existing auto-testing Pool suite's two-node seed and expansion operations, supplies marker-guarded reset/topology/restart/physical census hooks, and requires source case evidence plus physical cleanup before canonical PASS. `pool.status` passed live in RustFS Actions run `38047001285`; -`pool.decommission` also passed in run `38051227313`; the other eleven Pool -cases still require live evidence. +`pool.decommission` and `pool.decommission-cancel-clear-resume` also passed in runs `38051227313` and `38052304396`; the other ten Pool cases still require live evidence. The 18 bucket-replication and 20 site-replication IDs now dispatch to the source black-box checks. The driver uses the existing auto-testing four-node @@ -132,9 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 75 currently dispatchable IDs include all 74 non-Heal -cases and `heal.admin-healthy-noop`. Twenty-six have live PASS evidence; -the other 132 remain unverified. +case evidence. The 76 currently dispatchable IDs include all 74 non-Heal cases, `heal.admin-healthy-noop`, and `heal.admin-invalid-request`. Twenty-eight have live PASS evidence; the other 130 remain unverified. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -146,11 +143,15 @@ and nine evidence stages without manufacturing PASS. Heal suite's immutable deb download, installed binary digest, independent disk mounts, run-owned services/data roots, exact normal/deep responsibility receipts, repeated status, physical hashes and complete S3 inventories. It -still requires live validation. The other 83 Heal cases remain +failed live in run `38052673980`: the candidate returned an `unknown` receipt for a fresh zero-byte object with no bucket-incarnation identity, violating the source healthy-object contract. Cleanup passed; the oracle remains strict. +`heal.admin-invalid-request` passed all six branches on both H1 and H2 in run `38052868969`, with complete peer coverage and unchanged physical/S3 fixtures. +The other 82 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. +The RustFS tool-build lane produced the candidate-bound metadata inspector in run `38052536366`. Set `validation_tools_only=true` with the pinned build and auto-testing SHA to build it. Heal live runs consume `validation_tools_run_id`/`validation_tools_run_attempt`; the workflow verifies the successful build attempt, candidate/helper identities and binary/lock digests. The node adapter installs the verified tool under its run-owned root and uses it only to decode object metadata. The decoder explicitly does not claim physical bitrot verification. Ordinary live execution needs no RustFS source checkout. + The eight planned cases have these concrete reuse candidates: | Validation ID | Candidate IDs | Specific item to verify | diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index e96b85a6..17adb982 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -125,6 +125,9 @@ def reserve(request): owner = {"runId": run_id, "unit": unit(run_id), "hostname": observed["hostname"], "mounts": observed["mounts"], "binarySha256": binary_sha, "bucket": request["bucket"], "volumes": volumes} + if request.get("inspectorSha256"): + require(bool(re.fullmatch(r"[0-9a-f]{64}", request["inspectorSha256"])), "invalid inspector digest") + owner["inspectorSha256"] = request["inspectorSha256"] require(bool(re.fullmatch(r"rv-heal-[a-f0-9]{16}", owner["bucket"])), "invalid run bucket") state.mkdir(parents=True, mode=0o700) (state / "owner.json").write_text(json.dumps(owner)) @@ -174,6 +177,14 @@ def start(request): return {"unit": owner["unit"], "started": True} +def install_inspector(request): + state, _, owner = owned(request["runId"]) + tool = checked(state / "heal-meta-inspect", state) + require(digest(tool) == owner.get("inspectorSha256"), "uploaded inspector digest differs") + tool.chmod(0o500) + return {"inspectorSha256": owner["inspectorSha256"], "installed": True} + + def snapshot(request): state, roots, owner = owned(request["runId"]) pid = int(command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"]).stdout.strip()) @@ -186,6 +197,9 @@ def snapshot(request): require(all(environment.get(key) == value for key, value in BACKGROUND_CONFIG.items()), "background repair isolation configuration differs") files, formats = [], [] + tool = state / "heal-meta-inspect" + if owner.get("inspectorSha256"): + require(digest(checked(tool, state)) == owner["inspectorSha256"], "metadata inspector changed") for index, root in enumerate(roots): data = root / "data" folder = checked(data / owner["bucket"], data) @@ -196,8 +210,16 @@ def snapshot(request): for name in sorted(names): path = Path(directory) / name require(len(files) < 4096 and path.stat().st_size <= 64 * 1024**2, "fixture exceeds physical evidence budget") - files.append({"disk": index, "path": str(path.relative_to(data)), - "size": path.stat().st_size, "sha256": digest(path)}) + entry = {"disk": index, "path": str(path.relative_to(data)), + "size": path.stat().st_size, "sha256": digest(path)} + if path.name == "xl.meta" and owner.get("inspectorSha256"): + key = path.parent.relative_to(folder).as_posix() + decoded = json.loads(command(str(tool), str(path), owner["bucket"], key).stdout) + require(decoded.get("bucket") == owner["bucket"] and decoded.get("key") == key + and decoded.get("physicalBitrotVerified") is False, + "metadata decoder identity or verification role differs") + entry["metadata"] = decoded + files.append(entry) path = checked(data / ".rustfs.sys/format.json", data) require(path.stat().st_size < 65536, "disk format exceeds budget") formats.append({"disk": index, "sha256": digest(path), "value": json.loads(path.read_text())}) @@ -229,6 +251,6 @@ def cleanup(request): def main(request): require(os.geteuid() == 0, "node adapter requires passwordless sudo") actions = {"preflight": preflight, "reserve": reserve, "start": start, "snapshot": snapshot, - "logs": logs, "cleanup": cleanup} + "logs": logs, "cleanup": cleanup, "install-inspector": install_inspector} require(request.get("action") in actions, "unsupported node operation") print(json.dumps(actions[request["action"]](request))) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 2baa7b11..e447e500 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Execute every A01 normal/deep branch on the current H1 and H2 Heal fleet.""" +"""Execute A01 healthy and A07 invalid-request branches on the current H1/H2 fleet.""" from __future__ import annotations @@ -23,9 +23,11 @@ try: from .heal_probe import Probe, canonical, compare_inventories, require, save from .heal_workload import Recorder, seed, verify_seed + from .inspector import verify as verify_inspector except ImportError: from heal_probe import Probe, canonical, compare_inventories, require, save from heal_workload import Recorder, seed, verify_seed + from inspector import verify as verify_inspector NODE_SOURCE = Path(__file__).with_name("fleet_node.py") @@ -128,8 +130,9 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: - def __init__(self, nodes, evidence, run_id): + def __init__(self, nodes, evidence, run_id, tool_directory=None): self.nodes, self.evidence, self.run_id = nodes, evidence, run_id + self.tool_directory = tool_directory self.sequence = 0 self.reserved = [] @@ -138,13 +141,16 @@ def record(self, label, value): save(self.evidence / f"{self.sequence:05d}-{label}.json", value) return value + def ssh(self, node): + return ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=accept-new", + "-p", os.environ.get("RUSTFS_SSH_PORT", "22"), + os.environ.get("RUSTFS_SSH_USER", "azureuser") + "@" + node] + def remote(self, node, action, **params): value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() + "\nmain(" + repr(value) + ")\n" - result = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", - "-o", "StrictHostKeyChecking=accept-new", "-p", os.environ.get("RUSTFS_SSH_PORT", "22"), - os.environ.get("RUSTFS_SSH_USER", "azureuser") + "@" + node, - "sudo -n python3 -"], input=source, capture_output=True, text=True, timeout=180) + result = subprocess.run([*self.ssh(node), "sudo -n python3 -"], + input=source, capture_output=True, text=True, timeout=180) require(result.returncode == 0, f"Heal node {node} {action} failed: {result.stderr[-1200:]}") return json.loads(result.stdout) @@ -161,9 +167,20 @@ def deploy(self, binary_sha, bucket, access, secret): # Reserve may partially mutate before an SSH failure. Preserve its # namespace unless the root-owned marker proves safe finalization. self.reserved.append(node) + inspector_sha = hashlib.sha256((self.tool_directory / "heal-meta-inspect").read_bytes()).hexdigest() \ + if self.tool_directory else None value = self.remote(node, "reserve", hostname=checked[node]["hostname"], mounts=checked[node]["mounts"], - binarySha256=binary_sha, bucket=bucket, volumes=volumes, access=access, secret=secret) + binarySha256=binary_sha, bucket=bucket, volumes=volumes, access=access, secret=secret, + inspectorSha256=inspector_sha) self.record("reserved", {"node": node, **value}) + if self.tool_directory: + tool = self.tool_directory / "heal-meta-inspect" + target = "/var/lib/rustfs-auto-testing/heal/" + self.run_id + "/heal-meta-inspect" + transfer = subprocess.run([*self.ssh(node), "sudo -n dd of=" + target + + " conv=excl,fsync oflag=nofollow bs=1M status=none"], + input=tool.read_bytes(), capture_output=True, timeout=120) + require(transfer.returncode == 0, "exclusive inspector upload failed: " + transfer.stderr.decode()[-800:]) + self.record("inspector-upload", {"node": node, **self.remote(node, "install-inspector")}) with ThreadPoolExecutor(max_workers=len(self.nodes)) as executor: results = list(executor.map(lambda node: self.remote(node, "start"), self.nodes)) self.record("started", {node: value for node, value in zip(self.nodes, results, strict=True)}) @@ -201,11 +218,13 @@ def run_topology(args, topology, nodes): directory = args.evidence / topology directory.mkdir() run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" - fleet = Fleet(nodes, directory, run_id) + fleet = Fleet(nodes, directory, run_id, args.tool_directory) bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) candidate = json.loads(args.candidate.read_text()) + if args.tool_directory: + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" @@ -301,6 +320,7 @@ def interrupted(signum, frame): parser.add_argument("--binary-sha", required=True) parser.add_argument("--rc-bin", type=Path, required=True) parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path) args = parser.parse_args() nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() require(len(nodes) >= 4 and len(set(nodes)) == len(nodes) diff --git a/scripts/validation_heal/inspector.py b/scripts/validation_heal/inspector.py new file mode 100644 index 00000000..a9c20e13 --- /dev/null +++ b/scripts/validation_heal/inspector.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Verify a source-bound Linux Heal metadata decoder before distributing it.""" + +import argparse +import hashlib +import json +from pathlib import Path +import re +import stat + + +def verify(directory: Path, candidate: dict) -> dict: + root = directory.resolve(strict=True) + paths = {name: root / name for name in ("heal-meta-inspect", "heal-meta-inspect.json", "inspector-Cargo.lock")} + for name, path in paths.items(): + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_size > 256 * 1024**2: + raise ValueError(f"invalid inspector artifact file: {name}") + manifest = json.loads(paths["heal-meta-inspect.json"].read_text()) + if (manifest.get("built") is not True or manifest.get("candidateCommit") != candidate["source_sha"] + or manifest.get("target") != "x86_64-unknown-linux-gnu" + or manifest.get("role") != "read-only decoder; not the RustFS service under test"): + raise ValueError("inspector candidate or build role differs") + helper = Path(__file__).with_name("tools") / "heal_meta_inspect.rs" + digests = {"helperSourceSha256": hashlib.sha256(helper.read_bytes()).hexdigest(), + "binarySha256": hashlib.sha256(paths["heal-meta-inspect"].read_bytes()).hexdigest(), + "inspectorLockSha256": hashlib.sha256(paths["inspector-Cargo.lock"].read_bytes()).hexdigest()} + if any(manifest.get(name) != value for name, value in digests.items()): + raise ValueError("inspector source, binary or lock digest differs") + if not re.fullmatch(r"[0-9a-f]{64}", manifest.get("candidateLockSha256", "")): + raise ValueError("inspector omitted the candidate lock identity") + with paths["heal-meta-inspect"].open("rb") as stream: + header = stream.read(20) + if header[:6] != b"\x7fELF\x02\x01" or header[18:20] != b"\x3e\x00": + raise ValueError("inspector is not Linux x86_64 ELF") + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--candidate", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(verify(args.directory, json.loads(args.candidate.read_text())), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index fbb912fd..f0a653e0 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -36,6 +36,10 @@ PACKAGE_URL="$(jq -er '.package_url' "${candidate}")" PACKAGE_SHA256="$(jq -er '.package_sha256' "${candidate}")" [[ "${PACKAGE_SHA256}" =~ ^[0-9a-f]{64}$ ]] || die "invalid candidate package SHA-256" export RUSTFS_NODES="${NODES[*]}" RUSTFS_SSH_USER="${SSH_USER}" RUSTFS_SSH_PORT="${SSH_PORT}" +tool_args=() +if [ -n "${VALIDATION_HEAL_TOOL_DIRECTORY:-}" ]; then + tool_args=(--tool-directory "${VALIDATION_HEAL_TOOL_DIRECTORY}") +fi evidence="${artifacts}/evidence/${case_id}" mkdir -p "${artifacts}/cases" "${evidence}" python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ @@ -105,6 +109,7 @@ if [ "${bootstrap_rc}" -eq 0 ]; then done python3 -B "${ROOT}/scripts/validation_heal/healthy.py" --candidate "${candidate}" \ --case-id "${case_id}" \ + "${tool_args[@]}" \ --testing-commit "${testing_commit}" --binary-sha "${binary_sha}" --rc-bin "$(command -v rc)" \ --evidence "${evidence}" > "${evidence}/test.log" 2>&1 || case_rc=$? cat "${evidence}/test.log" diff --git a/test_validation_heal.py b/test_validation_heal.py index fa2b139e..513c045f 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -11,6 +11,7 @@ from scripts.validation_heal.matrix import STAGES, TOPOLOGIES, audit_evidence, cases, compile_plan, select from scripts.validation_heal.healthy import validate_healthy, validate_topology, validate_rejection from scripts.validation_heal.fleet_node import paths +from scripts.validation_heal.inspector import verify as verify_inspector class HealContractTest(unittest.TestCase): @@ -79,6 +80,32 @@ def test_rejects_dropped_branch_and_escaped_evidence(self): class HealHealthyOracleTest(unittest.TestCase): + def test_inspector_rejects_other_candidate_tampered_binary_and_symlinks(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + header = bytearray(20) + header[:6] = b"\x7fELF\x02\x01" + header[18:20] = b"\x3e\x00" + (root / "heal-meta-inspect").write_bytes(header) + (root / "inspector-Cargo.lock").write_text("unit fixture lock") + helper = Path(__file__).parent / "scripts/validation_heal/tools/heal_meta_inspect.rs" + manifest = {"built": True, "candidateCommit": "a" * 40, "candidateLockSha256": "b" * 64, + "role": "read-only decoder; not the RustFS service under test", "target": "x86_64-unknown-linux-gnu", + "binarySha256": hashlib.sha256(header).hexdigest(), + "helperSourceSha256": hashlib.sha256(helper.read_bytes()).hexdigest(), + "inspectorLockSha256": hashlib.sha256((root / "inspector-Cargo.lock").read_bytes()).hexdigest()} + (root / "heal-meta-inspect.json").write_text(json.dumps(manifest)) + self.assertEqual(verify_inspector(root, {"source_sha": "a" * 40}), manifest) + with self.assertRaisesRegex(ValueError, "candidate"): + verify_inspector(root, {"source_sha": "c" * 40}) + (root / "heal-meta-inspect").write_bytes(header + b"different bytes") + with self.assertRaisesRegex(ValueError, "digest"): + verify_inspector(root, {"source_sha": "a" * 40}) + (root / "heal-meta-inspect").unlink() + (root / "heal-meta-inspect").symlink_to(root / "inspector-Cargo.lock") + with self.assertRaisesRegex(ValueError, "artifact file"): + verify_inspector(root, {"source_sha": "a" * 40}) + def test_request_rejection_needs_the_exact_server_code(self): good = {"httpStatus": 400, "response": "InvalidArgument"} validate_rejection(good, "InvalidArgument") From fb132bb3f286cdfdebf4ba48b992c9ca690d9f69 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 21:11:32 +0800 Subject: [PATCH 025/343] feat(validation): automate external Heal shard fault cases --- docs/validation-heal-contract.md | 2 +- docs/validation-heal-operations.md | 6 +- docs/validation-migration-map.md | 9 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/check_bitrot_tool.py | 60 +++ scripts/validation_heal/fleet_node.py | 117 ++++- scripts/validation_heal/heal_fault.py | 4 +- scripts/validation_heal/heal_workload.py | 9 +- scripts/validation_heal/healthy.py | 38 +- scripts/validation_heal/matrix.py | 4 +- .../prepare_heal_meta_inspector.py | 4 +- scripts/validation_heal/run_case.sh | 12 +- scripts/validation_heal/shards.py | 424 ++++++++++++++++++ .../tools/heal_meta_inspect.rs | 140 +++++- test_validation_heal.py | 105 ++++- 15 files changed, 887 insertions(+), 49 deletions(-) create mode 100644 scripts/validation_heal/check_bitrot_tool.py create mode 100644 scripts/validation_heal/shards.py diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index de2df2b8..3341da97 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -`heal.admin-healthy-noop` 和 `heal.admin-invalid-request` 已接入自动分发，分别覆盖 H1/H2 的 normal/deep 和六项非法请求分支；其余操作卡保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 +9 个用例已接入自动分发：A01、A07、C01–C04、D02、E01、E02。分片执行器保留新增 7 项的全部 24 个源分支和 H1/H2 拓扑，使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。其余 75 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 4bd5c017..c813765b 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -2,7 +2,7 @@ 此手册和 [84 项逐例操作卡](validation-heal-cases.md) 一起构成可执行的**人工操作方案 + 脚本辅助**。不是 84 个无人值守自动测试，也不是通过报告。操作卡列出 301 个分支模板；真实盘角色、阈值、损坏数量和拓扑仍需在每批 `parameterBindings` 展开，不能把模板数量当实测次数。 -公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh heal.admin-healthy-noop CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 +公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 支持 A01、A07、C01–C04、D02、E01、E02，使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 ## 1. 使用入口与交付物 @@ -27,6 +27,8 @@ python scripts/validation_heal/matrix.py plan --selection A01,A04,C01,C02,C04,J0 | `heal_workload.py` | DB/DV/DM/DS/DX 真实夹具；有界并发、同key前台修改串行、请求前及ACK后持久落盘 | 不替用户推导竞态后的最终版本；超时写必须单独解析 | | `heal_fault.py` | Linux 描述符相对寻址、mount ID 保护、单文件 missing/bitflip/truncate/zero/header、快照和恢复 | 文件故障不是物理掉盘；不修改系统 metadata；不盲目恢复新代次 | | `prepare_heal_meta_inspector.py` | 为精确候选源码准备只读 Rust metadata 解析器工程 | 需在批准的构建机编译；解析 metadata 不是验证 bitrot/EC | +| `shards.py` | C01–C04、D02、E01、E02 全 24 分支；静默故障、真实角色、同角色原片与 HighwayHash、先物理后 GET/Range、MPU control | 有自动入口不等于候选通过；任务 unknown、目标只恢复 metadata 或救援备份均不能记 PASS | +| `check_bitrot_tool.py` | 在固定 MinIO/历史 RustFS 产物上检查 24 个健康片和 12 个坏片负例 | 工具验证不等于 live 用例通过；不调用被测 RustFS 服务 | ## 2. 先绑定环境，禁止直接猜节点 @@ -329,7 +331,7 @@ hf snapshot --bucket "$B" 快照对整桶每个文件保存长度/hash，目录描述符逐层拒绝 symlink/hardlink/不同mount ID（含同设备bind mount）；最多10,000文件/8GiB，每文件64MiB。四阶段 before/fault/after/restart 每盘各一份。外部part与**同一个角色、同版本的健康原片**字节完全相同是独立重建对照；数据应不重复且四片摘要不同。不得读取 quarantine 来辅助服务器重建。 -xl.meta 可合法重写；比较解析后的版本/引用，而非要求所有内部字节不变。允许的 xl.meta.bkp 必须属于同目标data_dir且摘要等于原xl.meta，不能泛化豁免任意额外文件。metadata恢复/换代后涉及bitrot重新编码时，要用匹配候选验证器/固定合法解码集证明；此解析器不验证HighwayHash或重新编码，报告必须如实写验证深度，不能只因解析成功记深度物理PASS。 +xl.meta 可合法重写；比较解析后的版本/引用，而非要求所有内部字节不变。允许的 xl.meta.bkp 必须属于同目标data_dir且摘要等于原xl.meta，不能泛化豁免任意额外文件。普通解析模式不验证 payload；`--verify-part XL_META BUCKET KEY PART_FILE VERSION_ID PART_NUMBER` 按候选 metadata 的实际版本、part 和 legacy/current framing 逐块验证 HighwayHash，同时拒绝短读、尾随字节和不支持的格式。分片执行器还逐项要求所有同版本片与故障前原始同角色长度/hash 完全一致，并确认基线全文、真实 set slot 及不重复分片摘要。它不支持加密/压缩、inline 或改变编码形态的物理验收；这些用例仍保留各自工具要求。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 39b0ca50..7fadb319 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -60,7 +60,7 @@ The driver reuses the existing auto-testing Pool suite's two-node seed and expansion operations, supplies marker-guarded reset/topology/restart/physical census hooks, and requires source case evidence plus physical cleanup before canonical PASS. `pool.status` passed live in RustFS Actions run `38047001285`; -`pool.decommission` and `pool.decommission-cancel-clear-resume` also passed in runs `38051227313` and `38052304396`; the other ten Pool cases still require live evidence. +`pool.decommission` and `pool.decommission-cancel-clear-resume` also passed in runs `38051227313` and `38052304396`. `pool.decommission-live-io`, `pool.decommission-restart-recovery`, `pool.decommission-versioned-integrity`, `pool.expand-and-rebalance`, and `pool.expansion-restart-persistence` passed in runs `38053444515`, `38053814812`, `38054128863`, `38054208114`, and `38054287896`. `pool.decommission-multipart-integrity` failed in run `38053715867`: no fresh owned-bucket MPU drain failure evidence was found; cleanup passed. The remaining four Pool cases still require live evidence. The 18 bucket-replication and 20 site-replication IDs now dispatch to the source black-box checks. The driver uses the existing auto-testing four-node @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 76 currently dispatchable IDs include all 74 non-Heal cases, `heal.admin-healthy-noop`, and `heal.admin-invalid-request`. Twenty-eight have live PASS evidence; the other 130 remain unverified. +case evidence. The 83 currently dispatchable IDs include all 74 non-Heal cases and nine Heal cases. Thirty-three have live PASS evidence; the other 125 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -145,12 +145,13 @@ mounts, run-owned services/data roots, exact normal/deep responsibility receipts, repeated status, physical hashes and complete S3 inventories. It failed live in run `38052673980`: the candidate returned an `unknown` receipt for a fresh zero-byte object with no bucket-incarnation identity, violating the source healthy-object contract. Cleanup passed; the oracle remains strict. `heal.admin-invalid-request` passed all six branches on both H1 and H2 in run `38052868969`, with complete peer coverage and unchanged physical/S3 fixtures. -The other 82 Heal cases remain +The external-shard executor also dispatches C01–C04, D02, E01 and E02 with all 24 source branches on H1/H2. It freezes exact version/dataDir/part, actual erasure role/set slot/disk UUID and full original shard hashes, stops all run-owned services before fault injection, witnesses the same fault immediately before the manual token, and checks complete physical repair plus streaming HighwayHash before any target GET. Full S3 inventories, cross-part Range, HEAD and pending MPU controls retain source semantics. Backups are used only for failed-case rescue or finalization; an unexpected new generation keeps the namespace DIRTY. These seven executors still require live evidence. +The other 75 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. -The RustFS tool-build lane produced the candidate-bound metadata inspector in run `38052536366`. Set `validation_tools_only=true` with the pinned build and auto-testing SHA to build it. Heal live runs consume `validation_tools_run_id`/`validation_tools_run_attempt`; the workflow verifies the successful build attempt, candidate/helper identities and binary/lock digests. The node adapter installs the verified tool under its run-owned root and uses it only to decode object metadata. The decoder explicitly does not claim physical bitrot verification. Ordinary live execution needs no RustFS source checkout. +The RustFS tool-build lane produced the initial candidate-bound metadata inspector in run `38052536366`; A07 also passed with that tool in run `38053214984`. Set `validation_tools_only=true` with the pinned build and auto-testing SHA to build the current helper. Heal live runs consume `validation_tools_run_id`/`validation_tools_run_attempt`; the workflow verifies the successful build attempt, candidate/helper identities and binary/lock digests. The node adapter installs the verified tool under its run-owned root. Plain metadata decoding makes no payload-integrity claim; the new explicit part-verification mode validates current and legacy streaming HighwayHash. The build checks 24 healthy shards captured from external released processes and rejects 12 controlled corruptions. Ordinary live execution needs no RustFS source checkout. The eight planned cases have these concrete reuse candidates: diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 4534715d..eaa6b2e5 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.manual-normal-attribution|heal.manual-deep-attribution) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/check_bitrot_tool.py b/scripts/validation_heal/check_bitrot_tool.py new file mode 100644 index 00000000..bbb6ca97 --- /dev/null +++ b/scripts/validation_heal/check_bitrot_tool.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Check the read-only bitrot tool against pinned external release captures.""" + +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import tempfile + + +def check(binary: Path, source: Path): + verified, rejected = 0, 0 + for kind in ("minio", "legacy"): + root = source / "crates/ecstore/tests/fixtures/erasure-shards" / kind + manifest = json.loads((root / "manifest.json").read_text()) + for relative, digest in manifest["files"].items(): + if hashlib.sha256((root / relative).read_bytes()).hexdigest() != digest: + raise ValueError("external corpus differs from its pinned file manifest") + for relative in sorted(manifest["files"]): + part = root / relative + if part.name != "part.1": + continue + metadata = part.parent.parent / "xl.meta" + decoded = json.loads(subprocess.check_output([str(binary), str(metadata), manifest["bucket"], manifest["object"]], text=True)) + version = decoded["versions"][0].get("versionId") or "null" + args = [str(binary), "--verify-part", str(metadata), manifest["bucket"], manifest["object"]] + result = json.loads(subprocess.check_output([*args, str(part), version, "1"], text=True)) + if result.get("physicalBitrotVerified") is not True or result["physicalSize"] != part.stat().st_size: + raise ValueError("external healthy shard did not pass the bitrot oracle") + verified += 1 + if verified not in {1, 13}: + continue + original = part.read_bytes() + changes = {"first": bytes([original[0] ^ 1]) + original[1:], + "middle": original[:len(original) // 2] + bytes([original[len(original) // 2] ^ 1]) + original[len(original) // 2 + 1:], + "last": original[:-1] + bytes([original[-1] ^ 1]), + "truncate": original[:-1], "zero": bytes(len(original)), "trailing": original + b"x"} + with tempfile.TemporaryDirectory(prefix="heal-bitrot-oracle-") as directory: + copied = Path(directory) / part.parent.name / part.name + copied.parent.mkdir() + for name, value in changes.items(): + copied.write_bytes(value) + bad = subprocess.run([*args, str(copied), version, "1"], capture_output=True, text=True) + if bad.returncode == 0 or not any(message in bad.stderr for message in + ("hash mismatch", "short shard read", "trailing shard bytes")): + raise ValueError(f"{kind} {name} corruption was not rejected by the integrity oracle") + rejected += 1 + if verified != 24 or rejected != 12: + raise ValueError("external bitrot corpus coverage is incomplete") + return {"externalHealthyShardsVerified": verified, "corruptedShardsRejected": rejected, + "corpora": ["MinIO released GF8", "RustFS released legacy GF16"], "liveCasePassClaimed": False} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", type=Path, required=True) + parser.add_argument("--source", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(check(args.binary.resolve(strict=True), args.source.resolve(strict=True)))) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 17adb982..7d0429df 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -10,6 +10,7 @@ import hashlib import json import os +from datetime import datetime, timedelta, timezone from pathlib import Path import re import shutil @@ -168,6 +169,8 @@ def owned(run_id): def start(request): state, _, owner = owned(request["runId"]) + stopped(owner) + command("systemctl", "reset-failed", owner["unit"], check=False) require(digest(state / "rustfs") == owner["binarySha256"], "candidate changed before start") command("systemd-run", "--unit=" + owner["unit"], "--property=Type=exec", "--property=KillMode=control-group", "--property=TimeoutStopSec=30", @@ -177,6 +180,90 @@ def start(request): return {"unit": owner["unit"], "started": True} +def stopped(owner): + values = command("systemctl", "show", owner["unit"], "-p", "MainPID", "-p", "ActiveState", + "-p", "ControlGroup", check=False).stdout + state = dict(line.split("=", 1) for line in values.splitlines() if "=" in line) + require(state.get("MainPID") == "0" and state.get("ActiveState") in {"inactive", "failed"}, + "file operations require the run-owned service to be stopped") + group = state.get("ControlGroup", "") + if group: + require(group.startswith("/") and ".." not in Path(group).parts, "invalid systemd control group") + root = Path("/sys/fs/cgroup") / group.lstrip("/") + if root.exists(): + require(all(not entry.read_text().strip() for entry in root.rglob("cgroup.procs")), + "run-owned control group still has live processes") + return state + + +def stop(request): + _, _, owner = owned(request["runId"]) + command("systemctl", "stop", owner["unit"], check=False) + return {"unit": owner["unit"], "quiescent": True, "service": stopped(owner)} + + +def fault_module(): + # SSH requests carry this exact repository module in memory; local imports + # support contract tests without creating an importable file on a node. + if "fault" in globals(): + return globals()["fault"] + from . import heal_fault + return heal_fault + + +def fault_root(request): + _, roots, owner = owned(request["runId"]) + stopped(owner) + index = request.get("disk") + require(type(index) is int and 0 <= index < len(roots), "invalid physical disk index") + return roots[index], owner + + +def apply_fault(request): + root, owner = fault_root(request) + module = fault_module() + with module.directory(root) as fd: + marker = root / "fault-owner.json" + if not marker.exists(): + module.new_file(fd, marker.name, module.encode({"schemaVersion": 1, "runId": owner["runId"], + "bucketPrefix": owner["bucket"], "mountId": module.mount_id(fd), "device": os.fstat(fd).st_dev, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat()})) + binding = module.checked_owner(fd, owner["runId"]) + return module.apply_fault(fd, binding, request["relativePath"], request["faultId"], request["kind"], + request["expectedSha256"], request.get("offset")) + + +def recover_fault(request): + root, owner = fault_root(request) + module = fault_module() + with module.directory(root) as fd: + return module.recover_fault(fd, module.checked_owner(fd, owner["runId"], recovery=True), request["faultId"]) + + +def verify_part(request): + state, roots, owner = owned(request["runId"]) + stopped(owner) + require(owner.get("inspectorSha256"), "candidate-bound bitrot verifier is required") + module = fault_module() + relative = request["relativePath"] + parts = module.target_parts(relative, owner["bucket"]) + require(parts[-1] != "xl.meta" and type(request.get("partNumber")) is int + and parts[-1] == f"part.{request['partNumber']}", "part identity differs from request") + root, _ = fault_root(request) + data = root / "data" + part = checked(data / relative, data) + metadata = checked(part.parent.parent / "xl.meta", data) + key = metadata.parent.relative_to(data / owner["bucket"]).as_posix() + tool = checked(state / "heal-meta-inspect", state) + require(digest(tool) == owner["inspectorSha256"], "bitrot verifier changed") + result = json.loads(command(str(tool), "--verify-part", str(metadata), owner["bucket"], key, + str(part), request["versionId"], str(request["partNumber"])).stdout) + require(result.get("physicalBitrotVerified") is True and result.get("key") == key + and result.get("versionId") == request["versionId"] + and result.get("partNumber") == request["partNumber"], "bitrot proof identity differs") + return {**result, "sha256": digest(part), "relativePath": relative, "disk": request["disk"]} + + def install_inspector(request): state, _, owner = owned(request["runId"]) tool = checked(state / "heal-meta-inspect", state) @@ -187,15 +274,19 @@ def install_inspector(request): def snapshot(request): state, roots, owner = owned(request["runId"]) - pid = int(command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"]).stdout.strip()) - require(pid > 0, "run-owned service has no live PID") - executable = Path(f"/proc/{pid}/exe").resolve(strict=True) - require(executable == state / "rustfs" and digest(executable) == owner["binarySha256"], - "running binary differs from candidate") - environment = dict(value.split("=", 1) for value in Path(f"/proc/{pid}/environ").read_bytes().decode().split("\0") - if "=" in value) - require(all(environment.get(key) == value for key, value in BACKGROUND_CONFIG.items()), - "background repair isolation configuration differs") + pid = int(command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"], check=False).stdout.strip() or "0") + if request.get("quiescent") is True: + stopped(owner) + require(digest(state / "rustfs") == owner["binarySha256"], "stopped candidate changed") + else: + require(pid > 0, "run-owned service has no live PID") + executable = Path(f"/proc/{pid}/exe").resolve(strict=True) + require(executable == state / "rustfs" and digest(executable) == owner["binarySha256"], + "running binary differs from candidate") + environment = dict(value.split("=", 1) for value in Path(f"/proc/{pid}/environ").read_bytes().decode().split("\0") + if "=" in value) + require(all(environment.get(key) == value for key, value in BACKGROUND_CONFIG.items()), + "background repair isolation configuration differs") files, formats = [], [] tool = state / "heal-meta-inspect" if owner.get("inspectorSha256"): @@ -224,7 +315,8 @@ def snapshot(request): require(path.stat().st_size < 65536, "disk format exceeds budget") formats.append({"disk": index, "sha256": digest(path), "value": json.loads(path.read_text())}) return {"hostname": owner["hostname"], "unit": owner["unit"], "pid": pid, - "binarySha256": owner["binarySha256"], "backgroundConfig": BACKGROUND_CONFIG, + "binarySha256": owner["binarySha256"], "quiescent": request.get("quiescent") is True, + "backgroundConfig": None if request.get("quiescent") is True else BACKGROUND_CONFIG, "files": sorted(files, key=lambda value: (value["disk"], value["path"])), "formats": formats} @@ -250,7 +342,8 @@ def cleanup(request): def main(request): require(os.geteuid() == 0, "node adapter requires passwordless sudo") - actions = {"preflight": preflight, "reserve": reserve, "start": start, "snapshot": snapshot, - "logs": logs, "cleanup": cleanup, "install-inspector": install_inspector} + actions = {"preflight": preflight, "reserve": reserve, "start": start, "stop": stop, "snapshot": snapshot, + "logs": logs, "cleanup": cleanup, "install-inspector": install_inspector, + "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part} require(request.get("action") in actions, "unsupported node operation") print(json.dumps(actions[request["action"]](request))) diff --git a/scripts/validation_heal/heal_fault.py b/scripts/validation_heal/heal_fault.py index 5e3ade73..358a95d2 100644 --- a/scripts/validation_heal/heal_fault.py +++ b/scripts/validation_heal/heal_fault.py @@ -117,7 +117,9 @@ def mutation(value: bytes, kind: str, offset: int | None = None) -> bytes | None return None require(bool(value), "empty file cannot witness this fault") if kind == "truncate": - return value[:len(value) // 2] + length = len(value) // 2 if offset is None else offset + require(type(length) is int and 0 <= length < len(value), "truncate length outside file") + return value[:length] if kind == "zero": changed = bytes(len(value)) elif kind == "header": diff --git a/scripts/validation_heal/heal_workload.py b/scripts/validation_heal/heal_workload.py index 821aeeb0..f24a28bf 100644 --- a/scripts/validation_heal/heal_workload.py +++ b/scripts/validation_heal/heal_workload.py @@ -109,13 +109,18 @@ def _call(self, operation, params, body, identity, expect_http): def seed(recorder: Recorder, bucket: str, fixture: str, count: int, sizes: list[int], - *, key_prefix: str = "objects/") -> list[dict]: + *, key_prefix: str = "objects/", existing_bucket: bool = False) -> list[dict]: bucket_allowed(recorder.probe.binding, bucket) require(1 <= count <= 2048 and bool(sizes) and all(0 <= size <= 64 * 1024**2 for size in sizes), "invalid bounded fixture dimensions") require(key_prefix in {"objects/", "objects-"} and (key_prefix == "objects/" or fixture == "DB"), "flat keys are only supported for the bounded DB fixture") - recorder.call("create_bucket", {"Bucket": bucket}) + if not existing_bucket: + recorder.call("create_bucket", {"Bucket": bucket}) + else: + mode = recorder.call("get_bucket_versioning", {"Bucket": bucket}) + require(fixture == "DM" and mode.get("Status") is None, + "appended multipart fixtures require the run's never-versioned bucket") expected = [] if fixture == "DV": recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index e447e500..46da22ef 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -135,6 +135,7 @@ def __init__(self, nodes, evidence, run_id, tool_directory=None): self.tool_directory = tool_directory self.sequence = 0 self.reserved = [] + self.faults = [] def record(self, label, value): self.sequence += 1 @@ -149,13 +150,27 @@ def ssh(self, node): def remote(self, node, action, **params): value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() + "\nmain(" + repr(value) + ")\n" + if action in {"apply-fault", "recover-fault", "verify-part"}: + source = "import types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( + NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" + source result = subprocess.run([*self.ssh(node), "sudo -n python3 -"], input=source, capture_output=True, text=True, timeout=180) require(result.returncode == 0, f"Heal node {node} {action} failed: {result.stderr[-1200:]}") return json.loads(result.stdout) - def snapshot(self, label): - return self.record(label, {node: self.remote(node, "snapshot") for node in self.nodes}) + def snapshot(self, label, *, quiescent=False): + return self.record(label, {node: self.remote(node, "snapshot", quiescent=quiescent) for node in self.nodes}) + + def stop(self): + with ThreadPoolExecutor(max_workers=len(self.nodes)) as executor: + results = list(executor.map(lambda node: self.remote(node, "stop"), self.nodes)) + return self.record("quiescent", dict(zip(self.nodes, results, strict=True))) + + def start(self): + with ThreadPoolExecutor(max_workers=len(self.nodes)) as executor: + results = list(executor.map(lambda node: self.remote(node, "start"), self.nodes)) + self.record("started", dict(zip(self.nodes, results, strict=True))) + self.wait_ready() def deploy(self, binary_sha, bucket, access, secret): checked = {node: self.remote(node, "preflight") for node in self.nodes} @@ -181,9 +196,9 @@ def deploy(self, binary_sha, bucket, access, secret): input=tool.read_bytes(), capture_output=True, timeout=120) require(transfer.returncode == 0, "exclusive inspector upload failed: " + transfer.stderr.decode()[-800:]) self.record("inspector-upload", {"node": node, **self.remote(node, "install-inspector")}) - with ThreadPoolExecutor(max_workers=len(self.nodes)) as executor: - results = list(executor.map(lambda node: self.remote(node, "start"), self.nodes)) - self.record("started", {node: value for node, value in zip(self.nodes, results, strict=True)}) + self.start() + + def wait_ready(self): opener = request.build_opener(request.ProxyHandler({})) deadline = time.monotonic() + 180 pending = {f"http://rustfs-node{index}:9000" for index in range(1, len(self.nodes) + 1)} @@ -206,10 +221,19 @@ def cleanup(self): self.record("logs", {"node": node, **self.remote(node, "logs")}) except Exception as exc: self.record("logs-error", {"node": node, "error": str(exc)}) + if self.faults: try: - self.record("cleanup", {"node": node, **self.remote(node, "cleanup")}) + self.stop() + for node, params in self.faults: + self.record("fault-recovery", {"node": node, **self.remote(node, "recover-fault", **params)}) except Exception as exc: - failures.append({"node": node, "error": str(exc)}) + failures.append({"error": str(exc), "state": "DIRTY", "namespacesRetained": True}) + if not failures: + for node in self.reserved: + try: + self.record("cleanup", {"node": node, **self.remote(node, "cleanup")}) + except Exception as exc: + failures.append({"node": node, "error": str(exc)}) self.record("recovery", {"cleanup": "FAIL" if failures else "PASS", "errors": failures}) return not failures diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 929f546d..8795e284 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,9 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request"} +AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.missing-part-file", + "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", + "heal.completed-multipart-object", "heal.manual-normal-attribution", "heal.manual-deep-attribution"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/prepare_heal_meta_inspector.py b/scripts/validation_heal/prepare_heal_meta_inspector.py index 08026115..4f5069f1 100644 --- a/scripts/validation_heal/prepare_heal_meta_inspector.py +++ b/scripts/validation_heal/prepare_heal_meta_inspector.py @@ -29,7 +29,9 @@ def prepare(source: Path, commit: str, output: Path) -> None: (output / "src/main.rs").write_text(text) manifest = ('[package]\nname = "heal-meta-inspect"\nversion = "0.1.0"\nedition = "2024"\n' '[workspace]\n[dependencies]\nserde_json = "1"\n' - 'rustfs-filemeta = { path = ' + json.dumps(str(crate)) + ' }\n') + 'rustfs-filemeta = { path = ' + json.dumps(str(crate)) + ' }\n' + 'rustfs-utils = { path = ' + json.dumps(str(source / "crates/utils")) + + ', default-features = false, features = ["hash"] }\n') (output / "Cargo.toml").write_text(manifest) (output / "source.json").write_text(json.dumps({"candidateCommit": head, "candidateLockSha256": hashlib.sha256((source / "Cargo.lock").read_bytes()).hexdigest(), diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index f0a653e0..67f499d1 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -5,6 +5,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" RUSTFS_HEAL_SCRIPT_SOURCE_ONLY=1 source "${ROOT}/rustfs_heal_test.sh" ASSUME_YES=1 candidate="" testing_commit="" artifacts="" case_id="" +driver=healthy.py while [ "$#" -gt 0 ]; do case "$1" in --validation-case) case_id="$2"; shift 2 ;; @@ -18,13 +19,15 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request) ;; + heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.manual-normal-attribution|heal.manual-deep-attribution) + driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; esac -[ "${#NODES[@]}" -ge 4 ] || die "A01 H1/H2 requires four current auto-testing Heal nodes" +[ "${#NODES[@]}" -ge 4 ] || die "H1/H2 requires four current auto-testing Heal nodes" NODES=("${NODES[0]}" "${NODES[1]}" "${NODES[2]}" "${NODES[3]}") if [ "${DRY_RUN}" -eq 1 ]; then - log "A01: current Heal nodes ${NODES[*]}, immutable deb install, run-owned H1/H2 service and four roots per node" - log "DB ACK ledger, exact normal metadata_healthy/deep verified_healthy receipts, repeated status, unchanged physical/S3 inventories" + log "${case_id}: current Heal nodes ${NODES[*]}, immutable deb install, run-owned H1/H2 service and four roots per node" + log "Source branches, ACK ledger, candidate-bound physical evidence, exact task outcomes, full S3 verification and cleanup" exit 0 fi for program in python3 ssh curl jq rc; do need_cmd "${program}" "Heal validation"; done @@ -40,6 +43,7 @@ tool_args=() if [ -n "${VALIDATION_HEAL_TOOL_DIRECTORY:-}" ]; then tool_args=(--tool-directory "${VALIDATION_HEAL_TOOL_DIRECTORY}") fi +[ "${driver}" != shards.py ] || [ "${#tool_args[@]}" -gt 0 ] || die "external-shard cases require candidate-bound bitrot tools" evidence="${artifacts}/evidence/${case_id}" mkdir -p "${artifacts}/cases" "${evidence}" python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ @@ -107,7 +111,7 @@ if [ "${bootstrap_rc}" -eq 0 ]; then if [ -n "${binary_sha}" ] && [ "${binary_sha}" != "${node_sha}" ]; then die "node candidate digests differ"; fi binary_sha="${node_sha}" done - python3 -B "${ROOT}/scripts/validation_heal/healthy.py" --candidate "${candidate}" \ + python3 -B "${ROOT}/scripts/validation_heal/${driver}" --candidate "${candidate}" \ --case-id "${case_id}" \ "${tool_args[@]}" \ --testing-commit "${testing_commit}" --binary-sha "${binary_sha}" --rc-bin "$(command -v rc)" \ diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py new file mode 100644 index 00000000..b9fab8ea --- /dev/null +++ b/scripts/validation_heal/shards.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +"""Run external-shard faults with frozen identities and physical repair oracles.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +from urllib.parse import quote +import uuid + +try: + from .heal_probe import Probe, canonical, compare_inventories, payload, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select +except ImportError: + from heal_probe import Probe, canonical, compare_inventories, payload, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + + +# Every branch uses a distinct object. Positions refer to physical bytes, +# including the checksum header; multipart numbers come from decoded metadata. +BRANCHES = { + "heal.missing-part-file": [ + ("data-single-part", "objects/0008.bin", "data", 1, "missing", None, 2), + ("parity-single-part", "objects/0017.bin", "parity", 1, "missing", None, 2), + ("multipart-first", "multipart/0000.bin", "data", 1, "missing", None, 2), + ("multipart-middle", "multipart/0001.bin", "data", 2, "missing", None, 2), + ("multipart-last", "multipart/0002.bin", "data", 3, "missing", None, 2)], + "heal.data-shard-bitrot": [ + ("data-first-byte", "objects/0008.bin", "data", 1, "bitflip", "first", 2), + ("data-middle-byte", "objects/0017.bin", "data", 1, "bitflip", "middle", 2), + ("data-last-byte", "objects/0026.bin", "data", 1, "bitflip", "last", 2), + ("multipart-data", "multipart/0000.bin", "data", 2, "bitflip", "middle", 2)], + "heal.parity-shard-bitrot": [ + ("parity-first-byte", "objects/0008.bin", "parity", 1, "bitflip", "first", 2), + ("parity-middle-byte", "objects/0017.bin", "parity", 1, "bitflip", "middle", 2), + ("parity-last-byte", "objects/0026.bin", "parity", 1, "bitflip", "last", 2)], + "heal.truncated-and-zeroed-shard": [ + ("truncate-half", "objects/0008.bin", "data", 1, "truncate", None, 2), + ("truncate-near-tail", "objects/0017.bin", "data", 1, "truncate", "last", 2), + ("zero-all", "objects/0026.bin", "data", 1, "zero", None, 2), + ("checksum-header", "objects/0035.bin", "data", 1, "header", None, 2)], + "heal.completed-multipart-object": [ + ("first-part", "multipart/0000.bin", "data", 1, "missing", None, 2), + ("middle-part", "multipart/0001.bin", "data", 2, "missing", None, 2), + ("last-part", "multipart/0002.bin", "data", 3, "missing", None, 2), + ("cross-part-range", "multipart/0003.bin", "data", 2, "bitflip", "middle", 2)], + "heal.manual-normal-attribution": [ + ("normal-missing-data", "objects/0008.bin", "data", 1, "missing", None, 1), + ("normal-missing-parity", "objects/0017.bin", "parity", 1, "missing", None, 1)], + "heal.manual-deep-attribution": [ + ("deep-parity-bitrot", "objects/0008.bin", "parity", 1, "bitflip", "middle", 2), + ("deep-data-bitrot", "objects/0017.bin", "data", 1, "bitflip", "middle", 2)], +} +SIZES = [0, 1, 4095, 4096, 4097, 1024**2 - 1, 1024**2, 1024**2 + 1, 5 * 1024**2 + 123] +MULTIPART_SIZES = (5 * 1024**2 + 17, 5 * 1024**2 + 93, 1024**2 + 47) + + +def files(snapshot): + entries = {(node, entry["disk"], entry["path"]): entry + for node, observation in snapshot.items() for entry in observation["files"]} + require(len(entries) == sum(len(value["files"]) for value in snapshot.values()), + "duplicate physical file identity") + return entries + + +def map_target(snapshot, bucket, key, role, part_number, version_id="null"): + validate_topology(snapshot, "H1" if len(snapshot) == 1 else "H2") + indexed = files(snapshot) + cohort, logical = [], None + for node, observation in snapshot.items(): + for format_entry in observation["formats"]: + disk = format_entry["disk"] + identity = (node, disk, f"{bucket}/{key}/xl.meta") + metadata = indexed.get(identity, {}).get("metadata", {}) + require(metadata.get("bucket") == bucket and metadata.get("key") == key, + "missing authoritative metadata identity on a baseline drive") + versions = metadata.get("versions", []) + require(len(versions) == 1 and (versions[0].get("versionId") or "null") == version_id, + "baseline metadata version differs from acknowledged fixture") + version = versions[0] + require(version.get("isLatest") is True and version.get("deleted") is False + and version.get("inline") is None and version.get("dataDir"), "expected live external object") + data_dir = version["dataDir"] + require(str(uuid.UUID(data_dir)) == data_dir, "noncanonical data directory") + erasure = version["erasure"] + data, parity = erasure["data_blocks"], erasure["parity_blocks"] + distribution = erasure["distribution"] + require(type(data) is int and type(parity) is int and data > 0 and parity > 0 + and data + parity == len(distribution) + and sorted(distribution) == list(range(1, data + parity + 1)), "invalid erasure geometry") + layout = format_entry["value"]["xl"] + slot = layout["sets"][0].index(layout["this"]) + require(erasure["index"] == distribution[slot], "shard role does not match the disk's actual set slot") + actual_role = "data" if erasure["index"] <= data else "parity" + require(version.get("role") == actual_role, "decoded shard role differs from erasure index") + logical_entry = {name: version.get(name) for name in + ("versionId", "isLatest", "deleted", "dataDir", "size", "parts", "usesLegacyChecksum")} + logical_entry["erasure"] = {name: erasure[name] for name in + ("data_blocks", "parity_blocks", "block_size", "distribution", "algorithm")} + require(logical is None or logical == logical_entry, "baseline versions or part geometry lack one full-set authority") + logical = logical_entry + parts = [part for part in version["parts"] if part["number"] == part_number] + require(len(parts) == 1 and parts[0]["size"] == parts[0]["actualSize"] > 0, + "unsupported compressed/encrypted or ambiguous fixture part") + relative = f"{bucket}/{key}/{data_dir}/part.{part_number}" + entry = indexed.get((node, disk, relative)) + require(entry and entry["size"] > 32, "external part is missing from the healthy baseline") + cohort.append({"node": node, "disk": disk, "relativePath": relative, "role": actual_role, + "index": erasure["index"], "versionId": version_id, "partNumber": part_number, + "dataDir": data_dir, "diskUUID": layout["this"], "setSlot": slot, + "sha256": entry["sha256"], "size": entry["size"], "metadata": metadata, + "baselineMetadataSha256": indexed[identity]["sha256"]}) + require(len(cohort) == logical["erasure"]["data_blocks"] + logical["erasure"]["parity_blocks"] + and len({entry["index"] for entry in cohort}) == len(cohort) + and len({entry["sha256"] for entry in cohort}) == len(cohort), + "independent full-set shard identity or nonrepeating content is missing") + target = next((entry for entry in cohort if entry["role"] == role), None) + require(target is not None, "requested shard role is not present") + return {**target, "bucket": bucket, "key": key, "poolIndex": 0, "setIndex": 0, + "cohort": cohort, "logical": logical} + + +def verify_cohort(fleet, target, label): + proofs = [] + for entry in target["cohort"]: + proof = fleet.record(label, {"node": entry["node"], **fleet.remote(entry["node"], "verify-part", + disk=entry["disk"], relativePath=entry["relativePath"], + versionId=entry["versionId"], partNumber=entry["partNumber"])}) + require(proof["sha256"] == entry["sha256"] and proof["physicalSize"] == entry["size"], + "physical part differs from the frozen same-role reconstruction oracle") + proofs.append(proof) + return {"allShardBitrotVerified": True, "sameRoleOriginalShardBytesRestored": True, + "proof": "complete pre-fault shard oracle, distinct shards, verified streaming checksums", "shards": proofs} + + +def validate_fault(before, after, target, receipt): + left, right = files(before), files(after) + identity = (target["node"], target["disk"], target["relativePath"]) + require(receipt.get("relativePath") == target["relativePath"] + and receipt.get("before") == {"sha256": target["sha256"], "size": target["size"]}, + "fault receipt differs from the frozen target") + for node in before: + require(before[node]["formats"] == after[node]["formats"], "file fault changed disk format") + expected = receipt.get("fault") + if expected is None: + require(identity not in right, "missing target was not physically witnessed") + else: + require(identity in right and {name: right[identity][name] for name in ("sha256", "size")} == expected + and expected != receipt["before"], "changed shard was not physically witnessed") + require({key: value for key, value in left.items() if key != identity} + == {key: value for key, value in right.items() if key != identity}, + "file fault changed metadata, controls or another shard") + + +def validate_repaired_physical(before, after, target): + left, right = files(before), files(after) + meta_path = f"{target['bucket']}/{target['key']}/xl.meta" + for node in before: + require(before[node]["formats"] == after[node]["formats"], "repair changed disk format") + for identity, entry in left.items(): + actual = right.get(identity) + require(actual is not None, "repair lost a baseline file or control") + if identity[2] != meta_path: + require(actual == entry, "repair did not restore exact part bytes or changed a control") + else: + def semantics(metadata): + return {"bucket": metadata["bucket"], "key": metadata["key"], "versions": [ + {**version, "erasure": {name: value for name, value in version["erasure"].items() if name != "checksums"}} + for version in metadata["versions"]]} + require(actual.get("metadata") and semantics(actual["metadata"]) == semantics(entry["metadata"]), + "repair changed decoded version/part identity") + for identity in set(right) - set(left): + original = left.get((identity[0], identity[1], meta_path)) + allowed = f"{target['bucket']}/{target['key']}/{target['dataDir']}/xl.meta.bkp" + require(identity[2] == allowed and original + and right[identity]["sha256"] == original["sha256"] and right[identity]["size"] == original["size"], + "repair created an unexpected physical file") + return {"targetLengthAndFullSha256Restored": True, "controlFilesUnchanged": True, + "decodedVersionsUnchanged": True, "formatUnchanged": True, "readBeforeRepair": False} + + +def validate_repaired_status(status, target): + outcome = status.get("outcome") or {} + require(status.get("summary") == "finished" and status.get("truncated", False) is False + and outcome.get("execution", {}).get("state") == "completed" + and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False, + "repair did not prove complete terminal coverage") + expected = {"processed": 1, "healed": 1, "unchanged": 0, "skipped": 0, + "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False} + counters = outcome.get("counters", {}) + require(all(type(counters.get(name)) is type(value) and counters[name] == value + for name, value in expected.items()), "repair counters do not prove exactly one repaired object") + objects = outcome.get("objects", []) + require(len(objects) == 1, "repair lacks one complete object receipt") + identity = objects[0].get("identity", {}) + require(identity.get("bucket") == target["bucket"] and identity.get("object") == target["key"] + and (identity.get("versionId") or "null") == target["versionId"] + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == target["poolIndex"] + and type(identity.get("setIndex")) is int and identity["setIndex"] == target["setIndex"] + and bool(identity.get("bucketIncarnationId")), "repair responsibility identity is incomplete or different") + require(objects[0].get("disposition", {}).get("state") == "repaired", + "task finish or unknown receipt does not prove repair") + + +def run_task(probe, fleet, target, mode, active): + path = "/rustfs/admin/v3/heal/" + quote(target["bucket"], safe="") + "/" + quote(target["key"], safe="/") + started = fleet.record("trigger", probe.admin(path, [], json.dumps({**BASE_OPTIONS, "scanMode": mode}).encode())) + require(started.get("httpStatus") == 200 and isinstance(started.get("response"), dict), "repair admission failed") + token = started["response"].get("clientToken") + require(isinstance(token, str) and token, "repair admission omitted token") + active.append((path, token)) + deadline = time.monotonic() + 180 + while True: + observation = fleet.record("control", probe.admin(path, [("clientToken", token)], b"")) + require(observation.get("httpStatus") == 200 and isinstance(observation.get("response"), dict), "repair status failed") + status = observation["response"] + if status.get("summary") != "running": + break + require(time.monotonic() < deadline, "repair task deadline exceeded") + time.sleep(0.2) + active.remove((path, token)) + for _ in range(2): + repeat = fleet.record("control-repeat", probe.admin(path, [("clientToken", token)], b"")) + require(repeat.get("httpStatus") == 200 and repeat.get("response", {}).get("summary") == status.get("summary") + and canonical(repeat["response"].get("outcome")) == canonical(status.get("outcome")), + "terminal repair responsibility changed") + return status + + +def pending_inventory(probe, bucket): + response = probe.s3("list_multipart_uploads", {"Bucket": bucket}) + require(response["acknowledged"] and not response["response"].get("IsTruncated"), "MPU control listing failed") + entries = [] + for upload in response["response"].get("Uploads", []): + require(upload["Key"] in {"pending-a", "pending-b"}, "unexpected pending MPU") + parts = probe.s3("list_parts", {"Bucket": bucket, "Key": upload["Key"], "UploadId": upload["UploadId"]}) + require(parts["acknowledged"] and not parts["response"].get("IsTruncated"), "pending MPU part listing failed") + entries.append({"key": upload["Key"], "uploadId": upload["UploadId"], "parts": [ + {name: part[name] for name in ("PartNumber", "ETag", "Size")} for part in parts["response"].get("Parts", [])]}) + require(len(entries) == 2 and {entry["key"] for entry in entries} == {"pending-a", "pending-b"} + and all(len(entry["parts"]) == 1 for entry in entries), "acknowledged pending MPU controls are missing") + return sorted(entries, key=lambda entry: entry["key"]) + + +def verify_ranges(probe, fleet, bucket, key): + if key.startswith("multipart/"): + body = b"".join(payload(f"{bucket}/{key}/{number}", size) for number, size in enumerate(MULTIPART_SIZES, 1)) + boundaries = (MULTIPART_SIZES[0], sum(MULTIPART_SIZES[:2])) + else: + body = payload(f"{bucket}/{key}/0", SIZES[int(key.rsplit("/", 1)[1].split(".")[0]) % len(SIZES)]) + boundaries = (1024**2, len(body) // 2) + proofs = [] + for start, end in [(point - 17, point + 29) for point in boundaries] + [(len(body) - 47, len(body) - 1)]: + require(0 <= start <= end < len(body), "invalid fixture range") + observed = fleet.record("s3-range", probe.s3("get_object", {"Bucket": bucket, "Key": key, + "VersionId": "null", "Range": f"bytes={start}-{end}"})) + wanted = body[start:end + 1] + require(observed["acknowledged"] and observed["response"]["ResponseMetadata"]["HTTPStatusCode"] == 206 + and observed["response"].get("ContentRange") == f"bytes {start}-{end}/{len(body)}" + and observed["payload"] == {"size": len(wanted), "sha256": hashlib.sha256(wanted).hexdigest()}, + "range content or part ordering differs from independent fixture") + proofs.append({"start": start, "end": end, "sha256": hashlib.sha256(wanted).hexdigest()}) + return proofs + + +def run_topology(args, topology, nodes): + directory = args.evidence / topology + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory) + candidate = json.loads(args.candidate.read_text()) + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 256, "maxInventoryBytes": 512 * 1024**2} + fleet.record("binding", binding) + recipe = select(args.case_id)[0] + require([branch[0] for branch in BRANCHES[args.case_id]] == recipe["variants"], "executor omitted a source branch") + result = {"topology": topology, "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} + active = [] + probe = Probe(binding) + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) + recorder = Recorder(probe, directory / "fixture") + expected = seed(recorder, bucket, "DB", 128, SIZES) + needs_multipart = "DM" in recipe["contract"]["probes"]["fixture"].split(",") + if needs_multipart: + expected += seed(recorder, bucket, "DM", 4, SIZES, existing_bucket=True) + baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + pending = fleet.record("mpu-baseline", pending_inventory(probe, bucket)) if needs_multipart else None + for entry in baseline["entries"]: + head = recorder.call("head_object", {"Bucket": bucket, "Key": entry["key"], "VersionId": entry["versionId"]}) + require(all(head.get(name) == value for name, value in entry["properties"].items()), "HEAD differs from GET baseline") + fleet.stop() + original = fleet.snapshot("physical-baseline", quiescent=True) + targets = [(branch, map_target(original, bucket, branch[1], branch[2], branch[3])) + for branch in BRANCHES[args.case_id]] + fleet.record("fault-plan", {"caseId": args.case_id, "topology": topology, + "branches": [{"variant": branch[0], "faultBackend": "logical-file-fault", "target": target, + "kind": branch[4], "position": branch[5], "scanMode": branch[6]} for branch, target in targets]}) + for branch, target in targets: + name, key, role, number, kind, position, mode = branch + fault_id = recipe["code"] + "-" + name + recovery = {"disk": target["disk"], "faultId": fault_id} + registered = False + try: + fleet.stop() + before = fleet.snapshot("physical-before-" + name, quiescent=True) + verify_cohort(fleet, target, "baseline-bitrot-" + name) + offset = {"first": 0, "middle": target["size"] // 2, "last": target["size"] - 1}.get(position) + fleet.faults.append((target["node"], recovery)) + registered = True + receipt = fleet.record("fault-" + name, fleet.remote(target["node"], "apply-fault", + **recovery, relativePath=target["relativePath"], expectedSha256=target["sha256"], kind=kind, offset=offset)) + after_fault = fleet.snapshot("physical-fault-" + name, quiescent=True) + validate_fault(before, after_fault, target, receipt) + fleet.start() + witness = fleet.snapshot("physical-before-trigger-" + name) + require(physical_content(after_fault) == physical_content(witness), + "ATTRIBUTION_NOT_OBSERVED: fault changed before the manual task") + status = run_task(probe, fleet, target, mode, active) + fleet.stop() + repaired = fleet.snapshot("physical-after-" + name, quiescent=True) + physical = validate_repaired_physical(before, repaired, target) + bitrot = verify_cohort(fleet, target, "repaired-bitrot-" + name) + validate_repaired_status(status, target) + fleet.start() + current = fleet.record("s3-after-" + name, probe.inventory(bucket, unversioned_current_tags=True)) + comparison = compare_inventories(baseline, current) + ranges = verify_ranges(probe, fleet, bucket, key) + if pending is not None: + require(fleet.record("mpu-after-" + name, pending_inventory(probe, bucket)) == pending, + "repair changed pending MPU identities or parts") + result["branches"][name] = {"verdict": "PASS", "physical": physical, "bitrot": bitrot, + "s3": comparison, "ranges": ranges, "trigger": "manual-normal" if mode == 1 else "manual-deep"} + except (Exception, KeyboardInterrupt) as exc: + result["branches"][name] = {"verdict": "FAIL", "failure": {"type": type(exc).__name__, "message": str(exc)}} + try: + fleet.stop() + fleet.snapshot("physical-failure-" + name, quiescent=True) + if registered: + rescue = fleet.record("rescue-" + name, fleet.remote(target["node"], "recover-fault", **recovery)) + fleet.faults.remove((target["node"], recovery)) + result["branches"][name]["rescue"] = rescue + except Exception as recovery_error: + result["branches"][name]["recoveryError"] = str(recovery_error) + raise + if isinstance(exc, KeyboardInterrupt) or active: + raise + result["verdict"] = "PASS" if all(value["verdict"] == "PASS" for value in result["branches"].values()) else "FAIL" + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + finally: + for path, token in active: + try: + fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: + fleet.record("stop-error", {"error": str(exc)}) + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("topology-result", result) + return result + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f"Heal workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=tuple(BRANCHES), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path, required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(set(nodes)) == len(nodes) + and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") + require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") + require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") + results = [] + try: + for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): + results.append(run_topology(args, topology, selected)) + if results[-1]["cleanup"] != "PASS": + break + finally: + complete = len(results) == 2 and all(value["verdict"] == "PASS" and value["cleanup"] == "PASS" for value in results) + save(args.evidence / "healthy-summary.json", {"caseId": args.case_id, "results": results, + "cleanup": "PASS" if results and all(value["cleanup"] == "PASS" for value in results) else "FAIL", "complete": complete}) + require(complete, "Heal case requires every source H1/H2 branch; inspect topology-result evidence") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"Heal shard validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/tools/heal_meta_inspect.rs b/scripts/validation_heal/tools/heal_meta_inspect.rs index e173c5cf..7151eb67 100644 --- a/scripts/validation_heal/tools/heal_meta_inspect.rs +++ b/scripts/validation_heal/tools/heal_meta_inspect.rs @@ -1,10 +1,120 @@ // Read-only inspector linked to the exact candidate's rustfs-filemeta crate. // Do not print inline bytes, user metadata, encryption envelopes, or IAM data. use rustfs_filemeta::FileMeta; +use rustfs_utils::HashAlgorithm; use serde_json::json; +fn bounded_file(path: &str) -> Result, Box> { + let metadata = std::fs::symlink_metadata(path)?; + if !metadata.is_file() || metadata.len() > 64 * 1024 * 1024 { + return Err("expected bounded, non-symlink evidence file".into()); + } + Ok(std::fs::read(path)?) +} + +// This path checks stored framing without invoking the service or repairing +// anything. EC reconstruction is checked separately against the frozen shard. +fn verify_part(args: &[String]) -> Result<(), Box> { + if args.len() != 8 { + return Err( + "usage: --verify-part XL_META BUCKET KEY PART_FILE VERSION_ID PART_NUMBER".into(), + ); + } + let meta = FileMeta::load(&bounded_file(&args[2])?)?; + let versions = meta.into_file_info_versions(&args[3], &args[4], true)?; + let matches: Vec<_> = versions + .versions + .iter() + .filter(|info| { + info.version_id + .filter(|id| !id.is_nil()) + .map(|id| id.to_string()) + .unwrap_or_else(|| "null".to_owned()) + == args[6] + }) + .collect(); + if matches.len() != 1 { + return Err("part verification requires one exact metadata version".into()); + } + let info = matches[0]; + let number: usize = args[7].parse()?; + let part = info + .parts + .iter() + .find(|part| part.number == number) + .ok_or("part is absent from metadata")?; + let path = std::path::Path::new(&args[5]); + let data_dir = info + .data_dir + .ok_or("external part has no data directory")? + .to_string(); + if info.deleted + || info.inline_data() + || info.erasure.data_blocks == 0 + || info.erasure.block_size == 0 + || part.size <= 0 + || path.file_name().and_then(|name| name.to_str()) + != Some(format!("part.{number}").as_str()) + || path + .parent() + .and_then(|parent| parent.file_name()) + .and_then(|name| name.to_str()) + != Some(data_dir.as_str()) + { + return Err("unsupported external part identity or geometry".into()); + } + let algorithm = if info.uses_legacy_checksum { + HashAlgorithm::HighwayHash256SLegacy + } else { + info.erasure.get_checksum_info(number).algorithm + }; + if !matches!( + algorithm, + HashAlgorithm::HighwayHash256S | HashAlgorithm::HighwayHash256SLegacy + ) { + return Err("unsupported bitrot framing; no integrity claim".into()); + } + let bytes = bounded_file(&args[5])?; + let mut remaining = usize::try_from(part.size)?; + let mut offset: usize = 0; + let mut blocks: usize = 0; + while remaining > 0 { + let logical = remaining.min(info.erasure.block_size); + let length = logical.div_ceil(info.erasure.data_blocks); + let length = if info.uses_legacy_checksum { + (length + 1) & !1 + } else { + length + }; + let end = offset + .checked_add(32 + length) + .ok_or("part size overflow")?; + let block = bytes.get(offset..end).ok_or("short shard read")?; + if algorithm.hash_encode(&block[32..]).as_ref() != &block[..32] { + return Err(format!("bitrot hash mismatch in block {blocks}").into()); + } + offset = end; + remaining -= logical; + blocks += 1; + } + if offset != bytes.len() { + return Err("unexpected trailing shard bytes".into()); + } + println!( + "{}", + json!({"schemaVersion": 1, "bucket": args[3], "key": args[4], + "versionId": args[6], "partNumber": number, "dataDir": data_dir, + "algorithm": algorithm, "blocksVerified": blocks, "physicalSize": bytes.len(), + "physicalBitrotVerified": true, "ecReconstructionVerified": false}) + ); + Ok(()) +} + fn main() -> Result<(), Box> { let args: Vec = std::env::args().collect(); + if args.get(1).is_some_and(|value| value == "--verify-part") { + return verify_part(&args); + } if args.len() != 4 { return Err("usage: heal-meta-inspect XL_META_FILE BUCKET KEY".into()); } @@ -44,22 +154,28 @@ fn main() -> Result<(), Box> { } else { None }; - let inline = decoded_inline.as_ref().filter(|data| !data.is_empty()).map(|data| { - let offsets: Vec<_> = bytes - .windows(data.len()) - .enumerate() - .filter_map(|(offset, candidate)| (candidate == data.as_slice()).then_some(offset)) - .collect(); - json!({ - "length": data.len(), - "rawOccurrenceCount": offsets.len(), - "rawOffset": (offsets.len() == 1).then_some(offsets[0]) - }) - }); + let inline = decoded_inline + .as_ref() + .filter(|data| !data.is_empty()) + .map(|data| { + let offsets: Vec<_> = bytes + .windows(data.len()) + .enumerate() + .filter_map(|(offset, candidate)| { + (candidate == data.as_slice()).then_some(offset) + }) + .collect(); + json!({ + "length": data.len(), + "rawOccurrenceCount": offsets.len(), + "rawOffset": (offsets.len() == 1).then_some(offsets[0]) + }) + }); Ok(json!({ "versionId": info.version_id.map(|id| id.to_string()), "isLatest": info.is_latest, "deleted": info.deleted, "dataDir": info.data_dir.map(|id| id.to_string()), "size": info.size, + "usesLegacyChecksum": info.uses_legacy_checksum, "inline": inline, "erasure": info.erasure, "role": role, "parts": info.parts.iter().map(|part| json!({ diff --git a/test_validation_heal.py b/test_validation_heal.py index 513c045f..339186cb 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -6,12 +6,15 @@ import tempfile import unittest import copy +import types from unittest.mock import patch from scripts.validation_heal.matrix import STAGES, TOPOLOGIES, audit_evidence, cases, compile_plan, select from scripts.validation_heal.healthy import validate_healthy, validate_topology, validate_rejection -from scripts.validation_heal.fleet_node import paths +from scripts.validation_heal.fleet_node import paths, stopped from scripts.validation_heal.inspector import verify as verify_inspector +from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status +from scripts.validation_heal.heal_fault import mutation class HealContractTest(unittest.TestCase): @@ -167,6 +170,106 @@ def test_node_paths_cannot_select_shared_roots_or_arbitrary_run(self): paths(value) +class HealShardOracleTest(unittest.TestCase): + def snapshot(self): + bucket, key, data_dir = "rv-heal-0123456789abcdef", "objects/0008.bin", "12345678-1234-1234-1234-123456789abc" + disks, distribution = [str(index) for index in range(4)], [4, 1, 2, 3] + observation = {"formats": [], "files": []} + for disk, index in enumerate(distribution): + observation["formats"].append({"disk": disk, "value": {"format": "xl", "id": "pool", + "xl": {"sets": [disks], "this": disks[disk]}}}) + metadata = {"bucket": bucket, "key": key, "versions": [{"versionId": None, "isLatest": True, + "deleted": False, "dataDir": data_dir, "size": 100, "inline": None, "usesLegacyChecksum": False, + "role": "data" if index <= 2 else "parity", "parts": [{"number": 1, "size": 100, "actualSize": 100}], + "erasure": {"index": index, "data_blocks": 2, "parity_blocks": 2, "block_size": 1048576, + "distribution": distribution, "algorithm": "rs-vandermonde"}}]} + observation["files"].extend([ + {"disk": disk, "path": f"{bucket}/{key}/xl.meta", "size": 600, "sha256": "m" + str(disk), "metadata": metadata}, + {"disk": disk, "path": f"{bucket}/{key}/{data_dir}/part.1", "size": 82, "sha256": "s" + str(disk)}, + {"disk": disk, "path": f"{bucket}/control/xl.meta", "size": 600, "sha256": "c" + str(disk)}]) + return {"node1": observation} + + def target(self, before, role="data"): + return map_target(before, "rv-heal-0123456789abcdef", "objects/0008.bin", role, 1) + + def test_executor_preserves_each_source_branch_and_uses_distinct_objects(self): + inventory = {item["id"]: item for item in cases()} + for case_id, branches in BRANCHES.items(): + self.assertEqual([branch[0] for branch in branches], inventory[case_id]["variants"]) + self.assertEqual(len({branch[1] for branch in branches}), len(branches)) + self.assertEqual(inventory[case_id]["contract"]["probes"]["topology"], "H1,H2") + + def test_target_role_comes_from_metadata_and_actual_format_slot(self): + before = self.snapshot() + self.assertEqual(self.target(before)["disk"], 1) + self.assertEqual(self.target(before, "parity")["disk"], 0) + for invalid in ("index", "role", "dataDir", "missing-part", "same-shards"): + changed = copy.deepcopy(before) + metadata = changed["node1"]["files"][0]["metadata"]["versions"][0] + if invalid == "index": metadata["erasure"]["index"] = 1 + elif invalid == "role": metadata["role"] = "data" + elif invalid == "dataDir": metadata["dataDir"] = "../control" + elif invalid == "missing-part": changed["node1"]["files"].pop(1) + else: + for entry in changed["node1"]["files"]: + if entry["path"].endswith("part.1"): entry["sha256"] = "same" + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + self.target(changed) + + def test_fault_requires_exact_target_and_unchanged_controls_and_metadata(self): + before = self.snapshot() + target = self.target(before) + after = copy.deepcopy(before) + after["node1"]["files"] = [entry for entry in after["node1"]["files"] + if not (entry["disk"] == target["disk"] and entry["path"] == target["relativePath"])] + receipt = {"relativePath": target["relativePath"], "before": {"sha256": target["sha256"], "size": 82}, "fault": None} + validate_fault(before, after, target, receipt) + with self.assertRaisesRegex(ValueError, "baseline file"): + validate_repaired_physical(before, after, target) + changed = copy.deepcopy(after) + changed["node1"]["files"][0]["sha256"] = "metadata changed" + with self.assertRaisesRegex(ValueError, "metadata, controls"): + validate_fault(before, changed, target, receipt) + validate_repaired_physical(before, before, target) + rewritten = copy.deepcopy(before) + rewritten["node1"]["files"][0]["sha256"] = "legitimate rewrite" + rewritten["node1"]["files"][0]["metadata"]["versions"][0]["erasure"]["checksums"] = ["new checksum record"] + validate_repaired_physical(before, rewritten, target) + rewritten["node1"]["files"][0]["metadata"]["versions"][0]["dataDir"] = "other generation" + with self.assertRaisesRegex(ValueError, "version/part identity"): + validate_repaired_physical(before, rewritten, target) + changed = copy.deepcopy(before) + changed["node1"]["files"][-1]["sha256"] = "control changed" + with self.assertRaisesRegex(ValueError, "changed a control"): + validate_repaired_physical(before, changed, target) + + def test_unknown_or_wrong_version_receipt_cannot_certify_physical_repair(self): + target = self.target(self.snapshot()) + receipt = HealHealthyOracleTest().receipt("repaired") + receipt["outcome"]["counters"].update(healed=1, unchanged=0) + receipt["outcome"]["objects"][0]["identity"].update(bucket=target["bucket"], object=target["key"], versionId=None) + validate_repaired_status(receipt, target) + for invalid in ("unknown", "wrong-version", "missing-incarnation"): + changed = copy.deepcopy(receipt) + if invalid == "unknown": changed["outcome"]["counters"]["unknown"] = 1 + elif invalid == "wrong-version": changed["outcome"]["objects"][0]["identity"]["versionId"] = "another" + else: changed["outcome"]["objects"][0]["identity"].pop("bucketIncarnationId") + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + validate_repaired_status(changed, target) + + def test_fault_requires_actual_service_quiescence_and_exact_truncate_length(self): + for value in ("MainPID=99\nActiveState=inactive\nControlGroup=\n", "MainPID=0\nActiveState=active\n", ""): + with patch("scripts.validation_heal.fleet_node.command", return_value=types.SimpleNamespace(stdout=value)): + with self.assertRaisesRegex(ValueError, "service to be stopped"): + stopped({"unit": "owned.service"}) + with patch("scripts.validation_heal.fleet_node.command", return_value=types.SimpleNamespace( + stdout="MainPID=0\nActiveState=inactive\nControlGroup=\n")): + stopped({"unit": "owned.service"}) + self.assertEqual(mutation(b"abcdef", "truncate", 5), b"abcde") + for length in (-1, 6, True): + with self.assertRaisesRegex(ValueError, "truncate length"): + mutation(b"abcdef", "truncate", length) + if __name__ == "__main__": unittest.main() From 9a01686ee0dfe15772a7dcffa1c2122e5bc4f59b Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 21:17:05 +0800 Subject: [PATCH 026/343] feat(validation): preserve Heal cross-node token status branches --- docs/validation-heal-contract.md | 2 +- docs/validation-heal-operations.md | 4 +-- docs/validation-migration-map.md | 10 +++--- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/healthy.py | 2 ++ scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/shards.py | 55 +++++++++++++++++++++++++---- test_validation_heal.py | 17 +++++++-- 9 files changed, 77 insertions(+), 19 deletions(-) diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index 3341da97..1863b41e 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -9 个用例已接入自动分发：A01、A07、C01–C04、D02、E01、E02。分片执行器保留新增 7 项的全部 24 个源分支和 H1/H2 拓扑，使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。其余 75 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 +10 个用例已接入自动分发：A01、A03、A07、C01–C04、D02、E01、E02。分片执行器保留新增 7 项的全部 24 个源分支和 H1/H2 拓扑，使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。其余 74 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index c813765b..32869c99 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -2,7 +2,7 @@ 此手册和 [84 项逐例操作卡](validation-heal-cases.md) 一起构成可执行的**人工操作方案 + 脚本辅助**。不是 84 个无人值守自动测试，也不是通过报告。操作卡列出 301 个分支模板；真实盘角色、阈值、损坏数量和拓扑仍需在每批 `parameterBindings` 展开，不能把模板数量当实测次数。 -公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 支持 A01、A07、C01–C04、D02、E01、E02，使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 +公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 支持 A01、A03、A07、C01–C04、D02、E01、E02，使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 ## 1. 使用入口与交付物 @@ -27,7 +27,7 @@ python scripts/validation_heal/matrix.py plan --selection A01,A04,C01,C02,C04,J0 | `heal_workload.py` | DB/DV/DM/DS/DX 真实夹具；有界并发、同key前台修改串行、请求前及ACK后持久落盘 | 不替用户推导竞态后的最终版本；超时写必须单独解析 | | `heal_fault.py` | Linux 描述符相对寻址、mount ID 保护、单文件 missing/bitflip/truncate/zero/header、快照和恢复 | 文件故障不是物理掉盘；不修改系统 metadata；不盲目恢复新代次 | | `prepare_heal_meta_inspector.py` | 为精确候选源码准备只读 Rust metadata 解析器工程 | 需在批准的构建机编译；解析 metadata 不是验证 bitrot/EC | -| `shards.py` | C01–C04、D02、E01、E02 全 24 分支；静默故障、真实角色、同角色原片与 HighwayHash、先物理后 GET/Range、MPU control | 有自动入口不等于候选通过；任务 unknown、目标只恢复 metadata 或救援备份均不能记 PASS | +| `shards.py` | C01–C04、D02、E01、E02 全 24 分支以及 A03 的两个 H2 跨节点状态分支；静默故障、真实角色、同角色原片与 HighwayHash、先物理后 GET/Range、MPU control | 有自动入口不等于候选通过；任务 unknown、目标只恢复 metadata 或救援备份均不能记 PASS | | `check_bitrot_tool.py` | 在固定 MinIO/历史 RustFS 产物上检查 24 个健康片和 12 个坏片负例 | 工具验证不等于 live 用例通过；不调用被测 RustFS 服务 | ## 2. 先绑定环境，禁止直接猜节点 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 7fadb319..79e22b6d 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -60,7 +60,7 @@ The driver reuses the existing auto-testing Pool suite's two-node seed and expansion operations, supplies marker-guarded reset/topology/restart/physical census hooks, and requires source case evidence plus physical cleanup before canonical PASS. `pool.status` passed live in RustFS Actions run `38047001285`; -`pool.decommission` and `pool.decommission-cancel-clear-resume` also passed in runs `38051227313` and `38052304396`. `pool.decommission-live-io`, `pool.decommission-restart-recovery`, `pool.decommission-versioned-integrity`, `pool.expand-and-rebalance`, and `pool.expansion-restart-persistence` passed in runs `38053444515`, `38053814812`, `38054128863`, `38054208114`, and `38054287896`. `pool.decommission-multipart-integrity` failed in run `38053715867`: no fresh owned-bucket MPU drain failure evidence was found; cleanup passed. The remaining four Pool cases still require live evidence. +`pool.decommission` and `pool.decommission-cancel-clear-resume` also passed in runs `38051227313` and `38052304396`. `pool.decommission-live-io`, `pool.decommission-restart-recovery`, `pool.decommission-versioned-integrity`, `pool.expand-and-rebalance`, and `pool.expansion-restart-persistence` passed in runs `38053444515`, `38053814812`, `38054128863`, `38054208114`, and `38054287896`. `pool.decommission-multipart-integrity` failed in run `38053715867`: no fresh owned-bucket MPU drain failure evidence was found; cleanup passed. `pool.operation-guards` and `pool.rebalance-live-io` passed in runs `38054540360` and `38054696080`. Two Pool cases still await live evidence; the multipart failure remains visible. The 18 bucket-replication and 20 site-replication IDs now dispatch to the source black-box checks. The driver uses the existing auto-testing four-node @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 83 currently dispatchable IDs include all 74 non-Heal cases and nine Heal cases. Thirty-three have live PASS evidence; the other 125 have no live PASS claim. +case evidence. The 84 currently dispatchable IDs include all 74 non-Heal cases and ten Heal cases. Thirty-five have live PASS evidence; the other 123 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -145,13 +145,13 @@ mounts, run-owned services/data roots, exact normal/deep responsibility receipts, repeated status, physical hashes and complete S3 inventories. It failed live in run `38052673980`: the candidate returned an `unknown` receipt for a fresh zero-byte object with no bucket-incarnation identity, violating the source healthy-object contract. Cleanup passed; the oracle remains strict. `heal.admin-invalid-request` passed all six branches on both H1 and H2 in run `38052868969`, with complete peer coverage and unchanged physical/S3 fixtures. -The external-shard executor also dispatches C01–C04, D02, E01 and E02 with all 24 source branches on H1/H2. It freezes exact version/dataDir/part, actual erasure role/set slot/disk UUID and full original shard hashes, stops all run-owned services before fault injection, witnesses the same fault immediately before the manual token, and checks complete physical repair plus streaming HighwayHash before any target GET. Full S3 inventories, cross-part Range, HEAD and pending MPU controls retain source semantics. Backups are used only for failed-case rescue or finalization; an unexpected new generation keeps the namespace DIRTY. These seven executors still require live evidence. -The other 75 Heal cases remain +The external-shard executor also dispatches C01–C04, D02, E01 and E02 with all 24 source branches on H1/H2. It freezes exact version/dataDir/part, actual erasure role/set slot/disk UUID and full original shard hashes, stops all run-owned services before fault injection, witnesses the same fault immediately before the manual token, and checks complete physical repair plus streaming HighwayHash before any target GET. Full S3 inventories, cross-part Range, HEAD and pending MPU controls retain source semantics. Backups are used only for failed-case rescue or finalization; an unexpected new generation keeps the namespace DIRTY. These seven executors still require live evidence. A03 also has both H2 branches: a non-coordinator shard fault, exact token queries through coordinator/peer/fixed current-fleet entry, a 30-second convergence budget for task start and full terminal outcome, physical proof and S3 inventories through all four entries. +The other 74 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. -The RustFS tool-build lane produced the initial candidate-bound metadata inspector in run `38052536366`; A07 also passed with that tool in run `38053214984`. Set `validation_tools_only=true` with the pinned build and auto-testing SHA to build the current helper. Heal live runs consume `validation_tools_run_id`/`validation_tools_run_attempt`; the workflow verifies the successful build attempt, candidate/helper identities and binary/lock digests. The node adapter installs the verified tool under its run-owned root. Plain metadata decoding makes no payload-integrity claim; the new explicit part-verification mode validates current and legacy streaming HighwayHash. The build checks 24 healthy shards captured from external released processes and rejects 12 controlled corruptions. Ordinary live execution needs no RustFS source checkout. +The RustFS tool-build lane produced the initial candidate-bound metadata inspector in run `38052536366`; A07 also passed with that tool in run `38053214984`. The extended bitrot helper was successfully built against the same candidate in run `38054812014`, including the released-shard oracle checks. Set `validation_tools_only=true` with the pinned build and auto-testing SHA to build the current helper. Heal live runs consume `validation_tools_run_id`/`validation_tools_run_attempt`; the workflow verifies the successful build attempt, candidate/helper identities and binary/lock digests. The node adapter installs the verified tool under its run-owned root. Plain metadata decoding makes no payload-integrity claim; the new explicit part-verification mode validates current and legacy streaming HighwayHash. The build checks 24 healthy shards captured from external released processes and rejects 12 controlled corruptions. Ordinary live execution needs no RustFS source checkout. The eight planned cases have these concrete reuse candidates: diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index eaa6b2e5..530c4a8b 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.manual-normal-attribution|heal.manual-deep-attribution) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.manual-normal-attribution|heal.manual-deep-attribution) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 46da22ef..7e3f6410 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -175,6 +175,8 @@ def start(self): def deploy(self, binary_sha, bucket, access, secret): checked = {node: self.remote(node, "preflight") for node in self.nodes} self.record("preflight", checked) + require(len({value["hostname"] for value in checked.values()}) == len(self.nodes), + "Heal node aliases must resolve to distinct actual hosts") paths = checked[self.nodes[0]]["dataRoots"] volumes = paths if len(self.nodes) == 1 else [f"http://rustfs-node{index}:9000{path}" for index in range(1, 5) for path in paths] diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 8795e284..4603272d 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.manual-normal-attribution", "heal.manual-deep-attribution"} diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 67f499d1..d6f450c3 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -19,7 +19,7 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request) ;; - heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.manual-normal-attribution|heal.manual-deep-attribution) + heal.admin-token-cross-node-status|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; esac diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index b9fab8ea..f8839d26 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -34,6 +34,9 @@ # Every branch uses a distinct object. Positions refer to physical bytes, # including the checksum header; multipart numbers come from decoded metadata. BRANCHES = { + "heal.admin-token-cross-node-status": [ + ("coordinator-to-peer", "objects/0008.bin", "data", 1, "missing", None, 2), + ("coordinator-to-stable-endpoint", "objects/0017.bin", "data", 1, "missing", None, 2)], "heal.missing-part-file": [ ("data-single-part", "objects/0008.bin", "data", 1, "missing", None, 2), ("parity-single-part", "objects/0017.bin", "parity", 1, "missing", None, 2), @@ -78,7 +81,7 @@ def files(snapshot): return entries -def map_target(snapshot, bucket, key, role, part_number, version_id="null"): +def map_target(snapshot, bucket, key, role, part_number, version_id="null", *, preferred_nodes=None): validate_topology(snapshot, "H1" if len(snapshot) == 1 else "H2") indexed = files(snapshot) cohort, logical = [], None @@ -129,7 +132,8 @@ def map_target(snapshot, bucket, key, role, part_number, version_id="null"): and len({entry["index"] for entry in cohort}) == len(cohort) and len({entry["sha256"] for entry in cohort}) == len(cohort), "independent full-set shard identity or nonrepeating content is missing") - target = next((entry for entry in cohort if entry["role"] == role), None) + target = next((entry for entry in cohort if entry["role"] == role + and (preferred_nodes is None or entry["node"] in preferred_nodes)), None) require(target is not None, "requested shard role is not present") return {**target, "bucket": bucket, "key": key, "poolIndex": 0, "setIndex": 0, "cohort": cohort, "logical": logical} @@ -217,7 +221,15 @@ def validate_repaired_status(status, target): "task finish or unknown receipt does not prove repair") -def run_task(probe, fleet, target, mode, active): +def peer_status_equal(coordinator, peer): + return (coordinator.get("summary") == "finished" and peer.get("summary") == "finished" + and coordinator.get("startTime") is not None and peer.get("startTime") == coordinator["startTime"] + and canonical(coordinator.get("outcome")) == canonical(peer.get("outcome")) + and coordinator.get("outcome", {}).get("coverage") == "complete" + and peer.get("truncated", False) is False) + + +def run_task(probe, fleet, target, mode, active, *, status_endpoints=()): path = "/rustfs/admin/v3/heal/" + quote(target["bucket"], safe="") + "/" + quote(target["key"], safe="/") started = fleet.record("trigger", probe.admin(path, [], json.dumps({**BASE_OPTIONS, "scanMode": mode}).encode())) require(started.get("httpStatus") == 200 and isinstance(started.get("response"), dict), "repair admission failed") @@ -239,6 +251,22 @@ def run_task(probe, fleet, target, mode, active): require(repeat.get("httpStatus") == 200 and repeat.get("response", {}).get("summary") == status.get("summary") and canonical(repeat["response"].get("outcome")) == canonical(status.get("outcome")), "terminal repair responsibility changed") + convergence_deadline = time.monotonic() + 30 + for endpoint in status_endpoints: + remote_probe = Probe(probe.binding, endpoint) + require(remote_probe.endpoint != probe.endpoint, "peer status must query another actual endpoint") + while True: + observed = fleet.record("cross-node-control", remote_probe.admin(path, [("clientToken", token)], b"")) + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), + "cross-node status is missing; 404/unknown cannot certify task completion") + if peer_status_equal(status, observed["response"]): + break + require(time.monotonic() < convergence_deadline, "peer responsibility did not converge within 30 seconds") + time.sleep(0.2) + if status_endpoints: + fleet.record("cross-node-convergence", {"path": path, "clientToken": token, + "coordinator": probe.endpoint_name, "queriedEndpoints": list(status_endpoints), "budgetSeconds": 30, + "taskStartTimeAndCompleteOutcomeEqual": True}) return status @@ -317,7 +345,13 @@ def run_topology(args, topology, nodes): require(all(head.get(name) == value for name, value in entry["properties"].items()), "HEAD differs from GET baseline") fleet.stop() original = fleet.snapshot("physical-baseline", quiescent=True) - targets = [(branch, map_target(original, bucket, branch[1], branch[2], branch[3])) + cross_node = args.case_id == "heal.admin-token-cross-node-status" + if cross_node: + fleet.record("cross-node-binding", {"coordinator": "node1", "peer": "node2", "stableEndpoint": "node3", + "endpoints": binding["endpoints"], "stableEndpointKind": "fixed current auto-testing node address", + "faultTargetExcludesCoordinator": True}) + targets = [(branch, map_target(original, bucket, branch[1], branch[2], branch[3], + preferred_nodes=nodes[1:] if cross_node else None)) for branch in BRANCHES[args.case_id]] fleet.record("fault-plan", {"caseId": args.case_id, "topology": topology, "branches": [{"variant": branch[0], "faultBackend": "logical-file-fault", "target": target, @@ -342,7 +376,8 @@ def run_topology(args, topology, nodes): witness = fleet.snapshot("physical-before-trigger-" + name) require(physical_content(after_fault) == physical_content(witness), "ATTRIBUTION_NOT_OBSERVED: fault changed before the manual task") - status = run_task(probe, fleet, target, mode, active) + status = run_task(probe, fleet, target, mode, active, + status_endpoints=("node2", "node3") if cross_node else ()) fleet.stop() repaired = fleet.snapshot("physical-after-" + name, quiescent=True) physical = validate_repaired_physical(before, repaired, target) @@ -351,6 +386,11 @@ def run_topology(args, topology, nodes): fleet.start() current = fleet.record("s3-after-" + name, probe.inventory(bucket, unversioned_current_tags=True)) comparison = compare_inventories(baseline, current) + if cross_node: + for endpoint in binding["endpoints"]: + peer_s3 = fleet.record("s3-entry-" + endpoint, Probe(binding, endpoint).inventory( + bucket, unversioned_current_tags=True)) + compare_inventories(baseline, peer_s3) ranges = verify_ranges(probe, fleet, bucket, key) if pending is not None: require(fleet.record("mpu-after-" + name, pending_inventory(probe, bucket)) == pending, @@ -404,13 +444,16 @@ def interrupted(signum, frame): require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") results = [] + topologies = set(select(args.case_id)[0]["contract"]["probes"]["topology"].split(",")) try: for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): + if topology not in topologies: + continue results.append(run_topology(args, topology, selected)) if results[-1]["cleanup"] != "PASS": break finally: - complete = len(results) == 2 and all(value["verdict"] == "PASS" and value["cleanup"] == "PASS" for value in results) + complete = len(results) == len(topologies) and all(value["verdict"] == "PASS" and value["cleanup"] == "PASS" for value in results) save(args.evidence / "healthy-summary.json", {"caseId": args.case_id, "results": results, "cleanup": "PASS" if results and all(value["cleanup"] == "PASS" for value in results) else "FAIL", "complete": complete}) require(complete, "Heal case requires every source H1/H2 branch; inspect topology-result evidence") diff --git a/test_validation_heal.py b/test_validation_heal.py index 339186cb..5b164dbd 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -13,7 +13,7 @@ from scripts.validation_heal.healthy import validate_healthy, validate_topology, validate_rejection from scripts.validation_heal.fleet_node import paths, stopped from scripts.validation_heal.inspector import verify as verify_inspector -from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status +from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal from scripts.validation_heal.heal_fault import mutation @@ -197,7 +197,20 @@ def test_executor_preserves_each_source_branch_and_uses_distinct_objects(self): for case_id, branches in BRANCHES.items(): self.assertEqual([branch[0] for branch in branches], inventory[case_id]["variants"]) self.assertEqual(len({branch[1] for branch in branches}), len(branches)) - self.assertEqual(inventory[case_id]["contract"]["probes"]["topology"], "H1,H2") + self.assertEqual(inventory[case_id]["contract"]["probes"]["topology"], + "H2" if case_id == "heal.admin-token-cross-node-status" else "H1,H2") + + def test_peer_status_cannot_converge_with_wrong_task_unknown_or_incomplete_receipts(self): + coordinator = {**HealHealthyOracleTest().receipt("repaired"), "startTime": "exact-task-start"} + self.assertTrue(peer_status_equal(coordinator, copy.deepcopy(coordinator))) + for invalid in ("task-start", "summary", "coverage", "outcome", "truncated"): + peer = copy.deepcopy(coordinator) + if invalid == "task-start": peer["startTime"] = "different task" + elif invalid == "summary": peer["summary"] = "unknown" + elif invalid == "coverage": peer["outcome"]["coverage"] = "partial" + elif invalid == "outcome": peer["outcome"]["counters"]["unknown"] = 1 + else: peer["truncated"] = True + with self.subTest(invalid=invalid): self.assertFalse(peer_status_equal(coordinator, peer)) def test_target_role_comes_from_metadata_and_actual_format_slot(self): before = self.snapshot() From c7b7e7cc5b654c7892abc950d6d2e2f49ca2fe36 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 21:20:51 +0800 Subject: [PATCH 027/343] fix(validation): compile remote Heal fault driver safely --- scripts/validation_heal/healthy.py | 7 ++++--- test_validation_heal.py | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 7e3f6410..f70ccf05 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -149,10 +149,11 @@ def ssh(self, node): def remote(self, node, action, **params): value = {"runId": self.run_id, "action": action, **params} - source = NODE_SOURCE.read_text() + "\nmain(" + repr(value) + ")\n" + source = NODE_SOURCE.read_text() if action in {"apply-fault", "recover-fault", "verify-part"}: - source = "import types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( - NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" + source + source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( + NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" + source += "\nmain(" + repr(value) + ")\n" result = subprocess.run([*self.ssh(node), "sudo -n python3 -"], input=source, capture_output=True, text=True, timeout=180) require(result.returncode == 0, f"Heal node {node} {action} failed: {result.stderr[-1200:]}") diff --git a/test_validation_heal.py b/test_validation_heal.py index 5b164dbd..ce92b038 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -10,7 +10,7 @@ from unittest.mock import patch from scripts.validation_heal.matrix import STAGES, TOPOLOGIES, audit_evidence, cases, compile_plan, select -from scripts.validation_heal.healthy import validate_healthy, validate_topology, validate_rejection +from scripts.validation_heal.healthy import validate_healthy, validate_topology, validate_rejection, Fleet from scripts.validation_heal.fleet_node import paths, stopped from scripts.validation_heal.inspector import verify as verify_inspector from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal @@ -171,6 +171,19 @@ def test_node_paths_cannot_select_shared_roots_or_arbitrary_run(self): class HealShardOracleTest(unittest.TestCase): + def test_ssh_script_compiles_with_fault_module_and_keeps_structured_request(self): + observed = [] + def invoke(args, **params): + source = params["input"] + compile(source, "ssh-node-script", "exec") + observed.append(source) + return types.SimpleNamespace(returncode=0, stdout='{"checked":true}', stderr="") + with tempfile.TemporaryDirectory() as directory, patch("scripts.validation_heal.healthy.subprocess.run", side_effect=invoke): + fleet = Fleet(["node1"], Path(directory), "gh-12345-1-h1") + for action in ("preflight", "verify-part", "apply-fault", "recover-fault"): + self.assertTrue(fleet.remote("node1", action, disk=1, relativePath="literal$()'part.1")["checked"]) + self.assertEqual(len(observed), 4) + def snapshot(self): bucket, key, data_dir = "rv-heal-0123456789abcdef", "objects/0008.bin", "12345678-1234-1234-1234-123456789abc" disks, distribution = [str(index) for index in range(4)], [4, 1, 2, 3] From 45954a4a5a614ec700008b683cc56d5d45cdb18e Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 21:29:43 +0800 Subject: [PATCH 028/343] feat(validation): automate all Heal dry-run variants --- docs/validation-heal-contract.md | 2 +- docs/validation-heal-operations.md | 4 +- docs/validation-migration-map.md | 5 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/shards.py | 120 ++++++++++++++++++++++------ test_validation_heal.py | 20 ++++- 8 files changed, 124 insertions(+), 33 deletions(-) diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index 1863b41e..6531a466 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -10 个用例已接入自动分发：A01、A03、A07、C01–C04、D02、E01、E02。分片执行器保留新增 7 项的全部 24 个源分支和 H1/H2 拓扑，使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。其余 74 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 +11 个用例已接入自动分发：A01、A03、A04、A07、C01–C04、D02、E01、E02。分片执行器保留新增 7 项的全部 24 个源分支和 H1/H2 拓扑，使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。其余 73 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 32869c99..10f4bc08 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -2,7 +2,7 @@ 此手册和 [84 项逐例操作卡](validation-heal-cases.md) 一起构成可执行的**人工操作方案 + 脚本辅助**。不是 84 个无人值守自动测试，也不是通过报告。操作卡列出 301 个分支模板；真实盘角色、阈值、损坏数量和拓扑仍需在每批 `parameterBindings` 展开，不能把模板数量当实测次数。 -公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 支持 A01、A03、A07、C01–C04、D02、E01、E02，使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 +公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 支持 A01、A03、A04、A07、C01–C04、D02、E01、E02，使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 ## 1. 使用入口与交付物 @@ -27,7 +27,7 @@ python scripts/validation_heal/matrix.py plan --selection A01,A04,C01,C02,C04,J0 | `heal_workload.py` | DB/DV/DM/DS/DX 真实夹具；有界并发、同key前台修改串行、请求前及ACK后持久落盘 | 不替用户推导竞态后的最终版本；超时写必须单独解析 | | `heal_fault.py` | Linux 描述符相对寻址、mount ID 保护、单文件 missing/bitflip/truncate/zero/header、快照和恢复 | 文件故障不是物理掉盘；不修改系统 metadata；不盲目恢复新代次 | | `prepare_heal_meta_inspector.py` | 为精确候选源码准备只读 Rust metadata 解析器工程 | 需在批准的构建机编译；解析 metadata 不是验证 bitrot/EC | -| `shards.py` | C01–C04、D02、E01、E02 全 24 分支以及 A03 的两个 H2 跨节点状态分支；静默故障、真实角色、同角色原片与 HighwayHash、先物理后 GET/Range、MPU control | 有自动入口不等于候选通过；任务 unknown、目标只恢复 metadata 或救援备份均不能记 PASS | +| `shards.py` | C01–C04、D02、E01、E02 全 24 分支以及 A03 的两个 H2 跨节点状态分支、A04 的六个 DB/DV dry-run 分支；静默故障、真实角色、同角色原片与 HighwayHash、先物理后 GET/Range、MPU control | 有自动入口不等于候选通过；任务 unknown、目标只恢复 metadata 或救援备份均不能记 PASS | | `check_bitrot_tool.py` | 在固定 MinIO/历史 RustFS 产物上检查 24 个健康片和 12 个坏片负例 | 工具验证不等于 live 用例通过；不调用被测 RustFS 服务 | ## 2. 先绑定环境，禁止直接猜节点 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 79e22b6d..d157c258 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 84 currently dispatchable IDs include all 74 non-Heal cases and ten Heal cases. Thirty-five have live PASS evidence; the other 123 have no live PASS claim. +case evidence. The 85 currently dispatchable IDs include all 74 non-Heal cases and eleven Heal cases. Thirty-five have live PASS evidence; the other 123 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -146,7 +146,8 @@ receipts, repeated status, physical hashes and complete S3 inventories. It failed live in run `38052673980`: the candidate returned an `unknown` receipt for a fresh zero-byte object with no bucket-incarnation identity, violating the source healthy-object contract. Cleanup passed; the oracle remains strict. `heal.admin-invalid-request` passed all six branches on both H1 and H2 in run `38052868969`, with complete peer coverage and unchanged physical/S3 fixtures. The external-shard executor also dispatches C01–C04, D02, E01 and E02 with all 24 source branches on H1/H2. It freezes exact version/dataDir/part, actual erasure role/set slot/disk UUID and full original shard hashes, stops all run-owned services before fault injection, witnesses the same fault immediately before the manual token, and checks complete physical repair plus streaming HighwayHash before any target GET. Full S3 inventories, cross-part Range, HEAD and pending MPU controls retain source semantics. Backups are used only for failed-case rescue or finalization; an unexpected new generation keeps the namespace DIRTY. These seven executors still require live evidence. A03 also has both H2 branches: a non-coordinator shard fault, exact token queries through coordinator/peer/fixed current-fleet entry, a 30-second convergence budget for task start and full terminal outcome, physical proof and S3 inventories through all four entries. -The other 74 Heal cases remain +A04 dispatches all six dry-run branches with separate DB/DV fixtures on H1/H2. It freezes version and marker identities, requires complete dry_run_observed receipts and zero repaired counters, proves byte-for-byte no mutation, then rescues the injected fault before full S3 verification. It does not require a dry-run incarnation or claim backup restoration as product repair. The first C01/C02 runs (`38055041726`, `38055254447`) stopped before injection because the remote script had an invalid future-import placement; their FAIL records and clean finalization are retained. The script compiler regression is fixed, and fresh runs use the corrected executor. +The other 73 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 530c4a8b..33dfb87d 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.manual-normal-attribution|heal.manual-deep-attribution) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.manual-normal-attribution|heal.manual-deep-attribution) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 4603272d..77bce140 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.manual-normal-attribution", "heal.manual-deep-attribution"} diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index d6f450c3..e5cbe961 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -19,7 +19,7 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request) ;; - heal.admin-token-cross-node-status|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.manual-normal-attribution|heal.manual-deep-attribution) + heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; esac diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index f8839d26..d80306b0 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -34,6 +34,13 @@ # Every branch uses a distinct object. Positions refer to physical bytes, # including the checksum header; multipart numbers come from decoded metadata. BRANCHES = { + "heal.admin-dry-run": [ + ("missing-remove0-recreate0", "objects/0008.bin", "data", 1, "missing", None, 2), + ("missing-remove1-recreate0", "objects/0017.bin", "data", 1, "missing", None, 2), + ("bitflip-remove0-recreate1", "objects/0026.bin", "data", 1, "bitflip", "middle", 2), + ("bitflip-remove1-recreate1", "objects/0035.bin", "data", 1, "bitflip", "middle", 2), + ("versioned-missing", "objects/0000.bin", "data", 1, "missing", None, 2), + ("versioned-bitflip", "objects/0001.bin", "data", 1, "bitflip", "middle", 2)], "heal.admin-token-cross-node-status": [ ("coordinator-to-peer", "objects/0008.bin", "data", 1, "missing", None, 2), ("coordinator-to-stable-endpoint", "objects/0017.bin", "data", 1, "missing", None, 2)], @@ -81,7 +88,7 @@ def files(snapshot): return entries -def map_target(snapshot, bucket, key, role, part_number, version_id="null", *, preferred_nodes=None): +def map_target(snapshot, bucket, key, role, part_number, version_id="null", *, preferred_nodes=None, latest_required=True): validate_topology(snapshot, "H1" if len(snapshot) == 1 else "H2") indexed = files(snapshot) cohort, logical = [], None @@ -92,11 +99,11 @@ def map_target(snapshot, bucket, key, role, part_number, version_id="null", *, p metadata = indexed.get(identity, {}).get("metadata", {}) require(metadata.get("bucket") == bucket and metadata.get("key") == key, "missing authoritative metadata identity on a baseline drive") - versions = metadata.get("versions", []) - require(len(versions) == 1 and (versions[0].get("versionId") or "null") == version_id, + versions = [version for version in metadata.get("versions", []) if (version.get("versionId") or "null") == version_id] + require(len(versions) == 1, "baseline metadata version differs from acknowledged fixture") version = versions[0] - require(version.get("isLatest") is True and version.get("deleted") is False + require((not latest_required or version.get("isLatest") is True) and version.get("deleted") is False and version.get("inline") is None and version.get("dataDir"), "expected live external object") data_dir = version["dataDir"] require(str(uuid.UUID(data_dir)) == data_dir, "noncanonical data directory") @@ -221,6 +228,31 @@ def validate_repaired_status(status, target): "task finish or unknown receipt does not prove repair") +def validate_dry_run_status(status, target, expected): + outcome = status.get("outcome") or {} + identities = {(entry["key"], entry["versionId"]) for entry in expected if entry["key"] == target["key"]} + require(status.get("summary") == "finished" and status.get("truncated", False) is False + and outcome.get("execution", {}).get("state") == "completed" and outcome.get("coverage") == "complete" + and outcome.get("objectsTruncated") is False and identities, "dry-run did not prove complete terminal coverage") + objects = outcome.get("objects", []) + require(len(objects) == len(identities), "dry-run omitted versions or markers") + actual = set() + for entry in objects: + identity = entry.get("identity", {}) + actual.add((identity.get("object"), identity.get("versionId") or "null")) + require(identity.get("bucket") == target["bucket"] + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0 + and entry.get("disposition", {}).get("state") == "dry_run_observed", + "dry-run admitted a different responsibility or claimed repair") + require(actual == identities, "dry-run responsibility versions differ from acknowledged fixture") + wanted = {"processed": len(identities), "healed": 0, "unchanged": 0, "skipped": len(identities), + "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False} + counters = outcome.get("counters", {}) + require(all(type(counters.get(name)) is type(value) and counters[name] == value for name, value in wanted.items()), + "dry-run counters do not match the complete dry_run_observed responsibility set") + + def peer_status_equal(coordinator, peer): return (coordinator.get("summary") == "finished" and peer.get("summary") == "finished" and coordinator.get("startTime") is not None and peer.get("startTime") == coordinator["startTime"] @@ -229,9 +261,10 @@ def peer_status_equal(coordinator, peer): and peer.get("truncated", False) is False) -def run_task(probe, fleet, target, mode, active, *, status_endpoints=()): +def run_task(probe, fleet, target, mode, active, *, status_endpoints=(), option_overrides=None): path = "/rustfs/admin/v3/heal/" + quote(target["bucket"], safe="") + "/" + quote(target["key"], safe="/") - started = fleet.record("trigger", probe.admin(path, [], json.dumps({**BASE_OPTIONS, "scanMode": mode}).encode())) + started = fleet.record("trigger", probe.admin(path, [], json.dumps( + {**BASE_OPTIONS, "scanMode": mode, **(option_overrides or {})}).encode())) require(started.get("httpStatus") == 200 and isinstance(started.get("response"), dict), "repair admission failed") token = started["response"].get("clientToken") require(isinstance(token, str) and token, "repair admission omitted token") @@ -285,18 +318,19 @@ def pending_inventory(probe, bucket): return sorted(entries, key=lambda entry: entry["key"]) -def verify_ranges(probe, fleet, bucket, key): +def verify_ranges(probe, fleet, bucket, key, *, version_id="null", expected_body=None): if key.startswith("multipart/"): body = b"".join(payload(f"{bucket}/{key}/{number}", size) for number, size in enumerate(MULTIPART_SIZES, 1)) boundaries = (MULTIPART_SIZES[0], sum(MULTIPART_SIZES[:2])) else: - body = payload(f"{bucket}/{key}/0", SIZES[int(key.rsplit("/", 1)[1].split(".")[0]) % len(SIZES)]) + body = expected_body if expected_body is not None else payload( + f"{bucket}/{key}/0", SIZES[int(key.rsplit("/", 1)[1].split(".")[0]) % len(SIZES)]) boundaries = (1024**2, len(body) // 2) proofs = [] for start, end in [(point - 17, point + 29) for point in boundaries] + [(len(body) - 47, len(body) - 1)]: require(0 <= start <= end < len(body), "invalid fixture range") observed = fleet.record("s3-range", probe.s3("get_object", {"Bucket": bucket, "Key": key, - "VersionId": "null", "Range": f"bytes={start}-{end}"})) + "VersionId": version_id, "Range": f"bytes={start}-{end}"})) wanted = body[start:end + 1] require(observed["acknowledged"] and observed["response"]["ResponseMetadata"]["HTTPStatusCode"] == 206 and observed["response"].get("ContentRange") == f"bytes {start}-{end}/{len(body)}" @@ -306,7 +340,7 @@ def verify_ranges(probe, fleet, bucket, key): return proofs -def run_topology(args, topology, nodes): +def run_topology(args, topology, nodes, *, fixture="DB"): directory = args.evidence / topology directory.mkdir() run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" @@ -326,21 +360,24 @@ def run_topology(args, topology, nodes): fleet.record("binding", binding) recipe = select(args.case_id)[0] require([branch[0] for branch in BRANCHES[args.case_id]] == recipe["variants"], "executor omitted a source branch") - result = {"topology": topology, "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} + result = {"topology": topology, "fixture": fixture, "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} active = [] probe = Probe(binding) try: fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) recorder = Recorder(probe, directory / "fixture") - expected = seed(recorder, bucket, "DB", 128, SIZES) + expected = seed(recorder, bucket, fixture, 128 if fixture == "DB" else 4, + SIZES if fixture == "DB" else [5 * 1024**2 + 123]) needs_multipart = "DM" in recipe["contract"]["probes"]["fixture"].split(",") if needs_multipart: expected += seed(recorder, bucket, "DM", 4, SIZES, existing_bucket=True) - baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=True)) + baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) pending = fleet.record("mpu-baseline", pending_inventory(probe, bucket)) if needs_multipart else None for entry in baseline["entries"]: + if entry["deleteMarker"]: + continue head = recorder.call("head_object", {"Bucket": bucket, "Key": entry["key"], "VersionId": entry["versionId"]}) require(all(head.get(name) == value for name, value in entry["properties"].items()), "HEAD differs from GET baseline") fleet.stop() @@ -350,9 +387,14 @@ def run_topology(args, topology, nodes): fleet.record("cross-node-binding", {"coordinator": "node1", "peer": "node2", "stableEndpoint": "node3", "endpoints": binding["endpoints"], "stableEndpointKind": "fixed current auto-testing node address", "faultTargetExcludesCoordinator": True}) - targets = [(branch, map_target(original, bucket, branch[1], branch[2], branch[3], - preferred_nodes=nodes[1:] if cross_node else None)) - for branch in BRANCHES[args.case_id]] + dry_run = args.case_id == "heal.admin-dry-run" + selected_branches = [branch for branch in BRANCHES[args.case_id] + if not dry_run or branch[0].startswith("versioned-") == (fixture == "DV")] + targets = [] + for branch in selected_branches: + version = next(entry for entry in expected if entry["key"] == branch[1] and not entry["deleteMarker"]) + targets.append((branch, map_target(original, bucket, branch[1], branch[2], branch[3], version["versionId"], + preferred_nodes=nodes[1:] if cross_node else None, latest_required=fixture != "DV"))) fleet.record("fault-plan", {"caseId": args.case_id, "topology": topology, "branches": [{"variant": branch[0], "faultBackend": "logical-file-fault", "target": target, "kind": branch[4], "position": branch[5], "scanMode": branch[6]} for branch, target in targets]}) @@ -377,21 +419,43 @@ def run_topology(args, topology, nodes): require(physical_content(after_fault) == physical_content(witness), "ATTRIBUTION_NOT_OBSERVED: fault changed before the manual task") status = run_task(probe, fleet, target, mode, active, - status_endpoints=("node2", "node3") if cross_node else ()) + status_endpoints=("node2", "node3") if cross_node else (), + option_overrides={"dryRun": True, "remove": "remove1" in name, "recreate": "recreate1" in name} + if dry_run else None) + listing = fleet.record("dry-run-version-list", probe.s3("list_object_versions", {"Bucket": bucket})) if dry_run else None fleet.stop() repaired = fleet.snapshot("physical-after-" + name, quiescent=True) - physical = validate_repaired_physical(before, repaired, target) - bitrot = verify_cohort(fleet, target, "repaired-bitrot-" + name) - validate_repaired_status(status, target) + if dry_run: + require(physical_content(after_fault) == physical_content(repaired), + "dry-run changed object shards, metadata or delete markers") + validate_dry_run_status(status, target, expected) + require(listing["acknowledged"] and not listing["response"].get("IsTruncated"), "dry-run version listing incomplete") + observed_versions = {(entry["Key"], entry["VersionId"], entry["IsLatest"], deleted) + for field, deleted in (("Versions", False), ("DeleteMarkers", True)) + for entry in listing["response"].get(field, [])} + require(observed_versions == {(entry["key"], entry["versionId"], entry["isLatest"], entry["deleteMarker"]) + for entry in expected}, "dry-run changed version or delete-marker manifest") + fleet.record("dry-run-rescue", fleet.remote(target["node"], "recover-fault", **recovery)) + fleet.faults.remove((target["node"], recovery)) + registered = False + recovered = fleet.snapshot("physical-recovered-" + name, quiescent=True) + require(physical_content(before) == physical_content(recovered), "dry-run fault recovery differs from baseline") + physical = {"dryRunPhysicalUnchanged": True, "repairedAck": False, "backupUsedOnlyAfterNoMutationProof": True} + else: + physical = validate_repaired_physical(before, repaired, target) + bitrot = verify_cohort(fleet, target, ("recovered-bitrot-" if dry_run else "repaired-bitrot-") + name) + if not dry_run: + validate_repaired_status(status, target) fleet.start() - current = fleet.record("s3-after-" + name, probe.inventory(bucket, unversioned_current_tags=True)) + current = fleet.record("s3-after-" + name, probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) comparison = compare_inventories(baseline, current) if cross_node: for endpoint in binding["endpoints"]: peer_s3 = fleet.record("s3-entry-" + endpoint, Probe(binding, endpoint).inventory( bucket, unversioned_current_tags=True)) compare_inventories(baseline, peer_s3) - ranges = verify_ranges(probe, fleet, bucket, key) + ranges = verify_ranges(probe, fleet, bucket, key, version_id=target["versionId"], + expected_body=payload(f"{bucket}/{key}/0", 5 * 1024**2 + 123) if fixture == "DV" else None) if pending is not None: require(fleet.record("mpu-after-" + name, pending_inventory(probe, bucket)) == pending, "repair changed pending MPU identities or parts") @@ -445,15 +509,23 @@ def interrupted(signum, frame): require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") results = [] topologies = set(select(args.case_id)[0]["contract"]["probes"]["topology"].split(",")) + fixtures = ("DB", "DV") if args.case_id == "heal.admin-dry-run" else ("DB",) try: for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): if topology not in topologies: continue - results.append(run_topology(args, topology, selected)) + for fixture in fixtures: + scoped_args = argparse.Namespace(**vars(args)) + if len(fixtures) > 1: + scoped_args.evidence = args.evidence / fixture + scoped_args.evidence.mkdir(exist_ok=True) + results.append(run_topology(scoped_args, topology, selected, fixture=fixture)) + if results[-1]["cleanup"] != "PASS": + break if results[-1]["cleanup"] != "PASS": break finally: - complete = len(results) == len(topologies) and all(value["verdict"] == "PASS" and value["cleanup"] == "PASS" for value in results) + complete = len(results) == len(topologies) * len(fixtures) and all(value["verdict"] == "PASS" and value["cleanup"] == "PASS" for value in results) save(args.evidence / "healthy-summary.json", {"caseId": args.case_id, "results": results, "cleanup": "PASS" if results and all(value["cleanup"] == "PASS" for value in results) else "FAIL", "complete": complete}) require(complete, "Heal case requires every source H1/H2 branch; inspect topology-result evidence") diff --git a/test_validation_heal.py b/test_validation_heal.py index ce92b038..40da86d2 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -13,7 +13,7 @@ from scripts.validation_heal.healthy import validate_healthy, validate_topology, validate_rejection, Fleet from scripts.validation_heal.fleet_node import paths, stopped from scripts.validation_heal.inspector import verify as verify_inspector -from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal +from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal, validate_dry_run_status from scripts.validation_heal.heal_fault import mutation @@ -283,6 +283,24 @@ def test_unknown_or_wrong_version_receipt_cannot_certify_physical_repair(self): with self.subTest(invalid=invalid), self.assertRaises(ValueError): validate_repaired_status(changed, target) + def test_dry_run_requires_all_version_marker_receipts_and_never_claims_repair(self): + target = self.target(self.snapshot()) + expected = [{"key": target["key"], "versionId": version} for version in ("old", "new", "marker")] + receipt = HealHealthyOracleTest().receipt("dry_run_observed") + receipt["outcome"]["counters"].update(processed=3, unchanged=0, skipped=3) + original = receipt["outcome"]["objects"][0] + original["identity"].update(bucket=target["bucket"], object=target["key"], bucketIncarnationId=None) + receipt["outcome"]["objects"] = [copy.deepcopy(original) for _ in expected] + for entry, version in zip(receipt["outcome"]["objects"], expected): entry["identity"]["versionId"] = version["versionId"] + validate_dry_run_status(receipt, target, expected) + for invalid in ("missing-marker", "repaired", "unknown", "healed-counter"): + value = copy.deepcopy(receipt) + if invalid == "missing-marker": value["outcome"]["objects"].pop() + elif invalid == "healed-counter": value["outcome"]["counters"]["healed"] = 1 + else: value["outcome"]["objects"][0]["disposition"]["state"] = invalid + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + validate_dry_run_status(value, target, expected) + def test_fault_requires_actual_service_quiescence_and_exact_truncate_length(self): for value in ("MainPID=99\nActiveState=inactive\nControlGroup=\n", "MainPID=0\nActiveState=active\n", ""): with patch("scripts.validation_heal.fleet_node.command", return_value=types.SimpleNamespace(stdout=value)): From 52ee289b60ff6873a6f9dec5ab41d44651741b3b Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 21:39:51 +0800 Subject: [PATCH 029/343] feat(validation): verify Heal conditions on protected fixtures --- docs/validation-heal-contract.md | 5 +- docs/validation-heal-operations.md | 7 ++- docs/validation-migration-map.md | 8 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 3 +- scripts/validation_heal/healthy.py | 6 +++ scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/shards.py | 77 ++++++++++++++++++++++++++- test_validation_heal.py | 25 +++++++++ 10 files changed, 126 insertions(+), 11 deletions(-) diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index 6531a466..138ad438 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -11 个用例已接入自动分发：A01、A03、A04、A07、C01–C04、D02、E01、E02。分片执行器保留新增 7 项的全部 24 个源分支和 H1/H2 拓扑，使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。其余 73 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 +12 个用例已接入自动分发：A01、A03、A04、A07、C01–C04、D02、D07、E01、E02。分片执行器保留新增 7 项的全部 24 个源分支和 H1/H2 拓扑，使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。D07 保留七个 HEAD/Range/ETag/时间条件分支，先冻结健康请求的具体 200/206/304/412/416 预期，再在物理修复后重复真实请求。其余 72 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 @@ -302,3 +302,6 @@ fault-plan 至少包含 run/case/variant ID、拓扑指纹、对象/版本/incar Functional Chain 在共享节点并发锁内调用 auto-testing 的 canonical dispatcher。普通 live 执行消费候选 deb 和工具产物，无需 RustFS checkout；只读 metadata inspector 和原生屏障测试须在构建阶段绑定精确候选源码，单独留证。 尚未自动化的分支由 `matrix.py plan` 固定全部适用拓扑、源断言、操作顺序、参数绑定与恢复步骤。`matrix.py audit` 校验九类证据的摘要和分支完整性，结果仍须逐项审阅，绝不自行生成 canonical PASS。 + + +正向 H1/H2 夹具显式启用 `RUSTFS_SHARD_INTEGRITY_WRITE=true` 和 `RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED=true`。该确认只针对本例私有 endpoint 集合：所有参与节点先安装并校验同一候选二进制，再启动专用服务；配置从真实进程环境读回。候选默认值为 false，默认 legacy 对象的 unknown 回执已单独保留为失败证据，不能改写为 repaired，也不从保护模式推断默认部署已满足这些正向证明契约。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 10f4bc08..8c9f558f 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -2,7 +2,7 @@ 此手册和 [84 项逐例操作卡](validation-heal-cases.md) 一起构成可执行的**人工操作方案 + 脚本辅助**。不是 84 个无人值守自动测试，也不是通过报告。操作卡列出 301 个分支模板；真实盘角色、阈值、损坏数量和拓扑仍需在每批 `parameterBindings` 展开，不能把模板数量当实测次数。 -公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 支持 A01、A03、A04、A07、C01–C04、D02、E01、E02，使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 +公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 支持 A01、A03、A04、A07、C01–C04、D02、D07、E01、E02，使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 ## 1. 使用入口与交付物 @@ -27,7 +27,7 @@ python scripts/validation_heal/matrix.py plan --selection A01,A04,C01,C02,C04,J0 | `heal_workload.py` | DB/DV/DM/DS/DX 真实夹具；有界并发、同key前台修改串行、请求前及ACK后持久落盘 | 不替用户推导竞态后的最终版本；超时写必须单独解析 | | `heal_fault.py` | Linux 描述符相对寻址、mount ID 保护、单文件 missing/bitflip/truncate/zero/header、快照和恢复 | 文件故障不是物理掉盘；不修改系统 metadata；不盲目恢复新代次 | | `prepare_heal_meta_inspector.py` | 为精确候选源码准备只读 Rust metadata 解析器工程 | 需在批准的构建机编译；解析 metadata 不是验证 bitrot/EC | -| `shards.py` | C01–C04、D02、E01、E02 全 24 分支以及 A03 的两个 H2 跨节点状态分支、A04 的六个 DB/DV dry-run 分支；静默故障、真实角色、同角色原片与 HighwayHash、先物理后 GET/Range、MPU control | 有自动入口不等于候选通过；任务 unknown、目标只恢复 metadata 或救援备份均不能记 PASS | +| `shards.py` | C01–C04、D02、D07、E01、E02 全 24 分支以及 A03 的两个 H2 跨节点状态分支、A04 的六个 DB/DV dry-run 分支；静默故障、真实角色、同角色原片与 HighwayHash、先物理后 GET/Range、MPU control | 有自动入口不等于候选通过；任务 unknown、目标只恢复 metadata 或救援备份均不能记 PASS | | `check_bitrot_tool.py` | 在固定 MinIO/历史 RustFS 产物上检查 24 个健康片和 12 个坏片负例 | 工具验证不等于 live 用例通过；不调用被测 RustFS 服务 | ## 2. 先绑定环境，禁止直接猜节点 @@ -632,3 +632,6 @@ python scripts/validation_heal/matrix.py audit --plan reports/validation-heal-84 ``` audit检查完整性/路径/摘要，输出`EVIDENCE_COMPLETE_REVIEW_REQUIRED`仍不是产品通过报告。原始失败批次永久保留，修复后新批次另记。本文提供全84项的可操作方案，但实际机器、专用依赖和native屏障的可用性必须单独核验，不能把“方案就绪”改写成“环境就绪”或“已实测”。 + + +自动正向夹具固定使用独立分片承诺配置；`integrity-fleet-attestation` 记录本例 endpoint/节点/候选摘要，启动前逐节点 reserve 校验候选，启动后实际进程环境读回两项开关。此配置只作用于 run-owned 服务。默认 legacy 配置的失败批次独立保留；保护模式不代表旧版本混部或默认环境能力。D07 的条件请求以服务器 HEAD 的 LastModified 和基线 ETag 冻结预期，不改主机时间；304/412/416 不能包含成功 payload 或未知传输结果。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index d157c258..4d41467b 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 85 currently dispatchable IDs include all 74 non-Heal cases and eleven Heal cases. Thirty-five have live PASS evidence; the other 123 have no live PASS claim. +case evidence. The 86 currently dispatchable IDs include all 74 non-Heal cases and twelve Heal cases. Thirty-five have live PASS evidence; the other 123 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -147,7 +147,8 @@ failed live in run `38052673980`: the candidate returned an `unknown` receipt fo `heal.admin-invalid-request` passed all six branches on both H1 and H2 in run `38052868969`, with complete peer coverage and unchanged physical/S3 fixtures. The external-shard executor also dispatches C01–C04, D02, E01 and E02 with all 24 source branches on H1/H2. It freezes exact version/dataDir/part, actual erasure role/set slot/disk UUID and full original shard hashes, stops all run-owned services before fault injection, witnesses the same fault immediately before the manual token, and checks complete physical repair plus streaming HighwayHash before any target GET. Full S3 inventories, cross-part Range, HEAD and pending MPU controls retain source semantics. Backups are used only for failed-case rescue or finalization; an unexpected new generation keeps the namespace DIRTY. These seven executors still require live evidence. A03 also has both H2 branches: a non-coordinator shard fault, exact token queries through coordinator/peer/fixed current-fleet entry, a 30-second convergence budget for task start and full terminal outcome, physical proof and S3 inventories through all four entries. A04 dispatches all six dry-run branches with separate DB/DV fixtures on H1/H2. It freezes version and marker identities, requires complete dry_run_observed receipts and zero repaired counters, proves byte-for-byte no mutation, then rescues the injected fault before full S3 verification. It does not require a dry-run incarnation or claim backup restoration as product repair. The first C01/C02 runs (`38055041726`, `38055254447`) stopped before injection because the remote script had an invalid future-import placement; their FAIL records and clean finalization are retained. The script compiler regression is fixed, and fresh runs use the corrected executor. -The other 73 Heal cases remain +`heal.range-head-and-conditions` preserves all seven D07 branches, with real baseline/post-repair HEAD, cross-part Range, exact ETag conditions, and server-time-derived conditions including 304/412/416 negatives. Completed MPU target-side stale `part.N.meta` cleanup is allowed only on the repaired drive/dataDir and for parts still represented by the unchanged authoritative xl.meta; controls and payload shards remain exact. +The other 72 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -180,3 +181,6 @@ shared fleet need the RustFS workflow concurrency group and an environment lease. Special Heal topologies and source-bound tools are provided by separate RustFS build/environment jobs; ordinary case execution consumes their immutable artifacts. + + +The corrected default-mode C01 run `38055597354` executed every H1/H2 fault branch with clean finalization. Single-part bytes and HighwayHash were restored, but the candidate returned unknown responsibility with no incarnation. The candidate explicitly defaults independent-shard writes off; current positive fixtures now enable both shard-integrity write and fleet-confirmation knobs only after every private peer is bound to the same candidate. Their values are read back from the actual processes and archived. The default-mode failure remains separate and never becomes a positive proof. New protected-mode runs are required before any new PASS claim. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 33dfb87d..33433512 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.manual-normal-attribution|heal.manual-deep-attribution) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 7d0429df..b238ff81 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -23,7 +23,8 @@ STATE_ROOT = Path("/var/lib/rustfs-auto-testing/heal") RUN_PATTERN = re.compile(r"gh-[1-9][0-9]*-[1-9][0-9]*-(?:h1|h2)") BACKGROUND_CONFIG = {"RUSTFS_SCANNER_ENABLED": "false", "RUSTFS_HEAL_ENABLED": "true", - "RUSTFS_HEAL_AUTO_HEAL_ENABLE": "false", "RUSTFS_HEAL_MRF_ENABLE": "false"} + "RUSTFS_HEAL_AUTO_HEAL_ENABLE": "false", "RUSTFS_HEAL_MRF_ENABLE": "false", + "RUSTFS_SHARD_INTEGRITY_WRITE": "true", "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED": "true"} def require(condition, message): diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index f70ccf05..a63a9e2b 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -181,6 +181,12 @@ def deploy(self, binary_sha, bucket, access, secret): paths = checked[self.nodes[0]]["dataRoots"] volumes = paths if len(self.nodes) == 1 else [f"http://rustfs-node{index}:9000{path}" for index in range(1, 5) for path in paths] + self.record("integrity-fleet-attestation", {"scope": "this run's private H1/H2 endpoint set", + "allParticipatingNodesUseCandidate": True, "binarySha256": binary_sha, + "nodes": self.nodes, "volumes": volumes, "mixedVersionsAllowed": False, + "precondition": "reserve verifies the candidate copy on every peer before any service starts", + "newFixtureWrites": "protected independent shard commitments", + "legacyDefaultResult": "retained separately; unknown is never a successful positive receipt"}) for node in self.nodes: # Reserve may partially mutate before an SSH failure. Preserve its # namespace unless the root-owned marker proves safe finalization. diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 77bce140..c82ef2c3 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -23,7 +23,7 @@ STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", - "heal.completed-multipart-object", "heal.manual-normal-attribution", "heal.manual-deep-attribution"} + "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index e5cbe961..3c631ae8 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -19,7 +19,7 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request) ;; - heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.manual-normal-attribution|heal.manual-deep-attribution) + heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; esac diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index d80306b0..6c208dde 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -34,6 +34,14 @@ # Every branch uses a distinct object. Positions refer to physical bytes, # including the checksum header; multipart numbers come from decoded metadata. BRANCHES = { + "heal.range-head-and-conditions": [ + ("head", "objects/0008.bin", "data", 1, "missing", None, 2), + ("range-valid", "multipart/0000.bin", "data", 2, "bitflip", "middle", 2), + ("range-outside", "objects/0017.bin", "data", 1, "missing", None, 2), + ("if-match", "objects/0026.bin", "data", 1, "bitflip", "middle", 2), + ("if-none-match", "objects/0035.bin", "data", 1, "missing", None, 2), + ("if-modified-since", "objects/0044.bin", "data", 1, "bitflip", "middle", 2), + ("if-unmodified-since", "objects/0053.bin", "data", 1, "missing", None, 2)], "heal.admin-dry-run": [ ("missing-remove0-recreate0", "objects/0008.bin", "data", 1, "missing", None, 2), ("missing-remove1-recreate0", "objects/0017.bin", "data", 1, "missing", None, 2), @@ -185,6 +193,13 @@ def validate_repaired_physical(before, after, target): require(before[node]["formats"] == after[node]["formats"], "repair changed disk format") for identity, entry in left.items(): actual = right.get(identity) + # Complete MPU commits the authoritative part manifest into xl.meta. + # A healed target directory can discard only its stale per-part sidecars; + # other versions, drives, pending uploads and payload files stay exact. + completed_sidecars = {f"{target['bucket']}/{target['key']}/{target['dataDir']}/part.{part['number']}.meta" + for part in target["logical"]["parts"]} + if actual is None and identity[:2] == (target["node"], target["disk"]) and identity[2] in completed_sidecars: + continue require(actual is not None, "repair lost a baseline file or control") if identity[2] != meta_path: require(actual == entry, "repair did not restore exact part bytes or changed a control") @@ -340,6 +355,53 @@ def verify_ranges(probe, fleet, bucket, key, *, version_id="null", expected_body return proofs +def validate_conditional_response(observed, status, baseline, *, code=None, full_body=False): + response = observed.get("response") or {} + require(response.get("ResponseMetadata", {}).get("HTTPStatusCode") == status, + f"conditional request expected HTTP {status}") + if status >= 300: + require(observed.get("acknowledged") is False and observed.get("resultUnknown") is False + and "payload" not in observed, "conditional denial returned success data or unknown transport outcome") + if code is not None: + require(response.get("Error", {}).get("Code") == code, "conditional response has the wrong structured error") + else: + require(observed.get("acknowledged") is True and response.get("ETag") == baseline["properties"]["ETag"], + "conditional request returned the wrong object ETag") + if full_body: + require(observed.get("payload") == baseline["payload"], "conditional GET returned wrong full content") + + +def verify_conditions(probe, fleet, branch, baseline, head, label): + name, key = branch[:2] + entry = next(entry for entry in baseline["entries"] if entry["key"] == key) + params = {"Bucket": baseline["bucket"], "Key": key, "VersionId": entry["versionId"]} + etag, modified = head["ETag"], head["LastModified"] + require(isinstance(modified, datetime) and modified.tzinfo is not None, "baseline LastModified must be server time") + requests = { + "head": [("head_object", {}, 200, None)], + "range-outside": [("get_object", {"Range": f"bytes={entry['payload']['size']}-"}, 416, "InvalidRange")], + "if-match": [("get_object", {"IfMatch": etag}, 200, None), + ("get_object", {"IfMatch": '"00000000000000000000000000000000"'}, 412, "PreconditionFailed")], + "if-none-match": [("get_object", {"IfNoneMatch": etag}, 304, None), + ("get_object", {"IfNoneMatch": '"00000000000000000000000000000000"'}, 200, None)], + "if-modified-since": [("get_object", {"IfModifiedSince": modified + timedelta(seconds=2)}, 304, None), + ("get_object", {"IfModifiedSince": modified - timedelta(seconds=2)}, 200, None)], + "if-unmodified-since": [("get_object", {"IfUnmodifiedSince": modified - timedelta(seconds=2)}, 412, "PreconditionFailed"), + ("get_object", {"IfUnmodifiedSince": modified + timedelta(seconds=2)}, 200, None)]} + if name == "range-valid": + return verify_ranges(probe, fleet, baseline["bucket"], key) + results = [] + for operation, extra, status, code in requests[name]: + observed = fleet.record(label + "-" + name, probe.s3(operation, {**params, **extra})) + validate_conditional_response(observed, status, entry, code=code, full_body=operation == "get_object" and status == 200) + if operation == "head_object": + require(observed["response"].get("LastModified") == modified + and all(observed["response"].get(field) == value for field, value in entry["properties"].items()), + "HEAD content properties or server LastModified changed") + results.append({"operation": operation, "status": status, "condition": list(extra)}) + return results + + def run_topology(args, topology, nodes, *, fixture="DB"): directory = args.evidence / topology directory.mkdir() @@ -375,10 +437,12 @@ def run_topology(args, topology, nodes, *, fixture="DB"): baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) pending = fleet.record("mpu-baseline", pending_inventory(probe, bucket)) if needs_multipart else None + heads = {} for entry in baseline["entries"]: if entry["deleteMarker"]: continue head = recorder.call("head_object", {"Bucket": bucket, "Key": entry["key"], "VersionId": entry["versionId"]}) + heads[entry["key"]] = head require(all(head.get(name) == value for name, value in entry["properties"].items()), "HEAD differs from GET baseline") fleet.stop() original = fleet.snapshot("physical-baseline", quiescent=True) @@ -388,8 +452,14 @@ def run_topology(args, topology, nodes, *, fixture="DB"): "endpoints": binding["endpoints"], "stableEndpointKind": "fixed current auto-testing node address", "faultTargetExcludesCoordinator": True}) dry_run = args.case_id == "heal.admin-dry-run" + conditions = args.case_id == "heal.range-head-and-conditions" selected_branches = [branch for branch in BRANCHES[args.case_id] if not dry_run or branch[0].startswith("versioned-") == (fixture == "DV")] + if conditions: + fleet.start() + for branch in selected_branches: + verify_conditions(probe, fleet, branch, baseline, heads[branch[1]], "baseline-condition") + fleet.stop() targets = [] for branch in selected_branches: version = next(entry for entry in expected if entry["key"] == branch[1] and not entry["deleteMarker"]) @@ -444,8 +514,6 @@ def run_topology(args, topology, nodes, *, fixture="DB"): else: physical = validate_repaired_physical(before, repaired, target) bitrot = verify_cohort(fleet, target, ("recovered-bitrot-" if dry_run else "repaired-bitrot-") + name) - if not dry_run: - validate_repaired_status(status, target) fleet.start() current = fleet.record("s3-after-" + name, probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) comparison = compare_inventories(baseline, current) @@ -456,11 +524,16 @@ def run_topology(args, topology, nodes, *, fixture="DB"): compare_inventories(baseline, peer_s3) ranges = verify_ranges(probe, fleet, bucket, key, version_id=target["versionId"], expected_body=payload(f"{bucket}/{key}/0", 5 * 1024**2 + 123) if fixture == "DV" else None) + condition_proof = verify_conditions(probe, fleet, branch, baseline, heads[key], "repaired-condition") if conditions else None if pending is not None: require(fleet.record("mpu-after-" + name, pending_inventory(probe, bucket)) == pending, "repair changed pending MPU identities or parts") + if not dry_run: + validate_repaired_status(status, target) result["branches"][name] = {"verdict": "PASS", "physical": physical, "bitrot": bitrot, "s3": comparison, "ranges": ranges, "trigger": "manual-normal" if mode == 1 else "manual-deep"} + if condition_proof is not None: + result["branches"][name]["conditions"] = condition_proof except (Exception, KeyboardInterrupt) as exc: result["branches"][name] = {"verdict": "FAIL", "failure": {"type": type(exc).__name__, "message": str(exc)}} try: diff --git a/test_validation_heal.py b/test_validation_heal.py index 40da86d2..9434f97d 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -15,6 +15,7 @@ from scripts.validation_heal.inspector import verify as verify_inspector from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal, validate_dry_run_status from scripts.validation_heal.heal_fault import mutation +from scripts.validation_heal.shards import validate_conditional_response class HealContractTest(unittest.TestCase): @@ -171,6 +172,18 @@ def test_node_paths_cannot_select_shared_roots_or_arbitrary_run(self): class HealShardOracleTest(unittest.TestCase): + def test_condition_denial_is_not_a_transport_failure_or_success_payload(self): + baseline = {"properties": {"ETag": '"etag"'}, "payload": {"size": 3, "sha256": "correct"}} + good = {"response": {"ResponseMetadata": {"HTTPStatusCode": 412}, "Error": {"Code": "PreconditionFailed"}}, + "acknowledged": False, "resultUnknown": False} + validate_conditional_response(good, 412, baseline, code="PreconditionFailed") + for changed in ({**good, "payload": {"size": 3}}, {**good, "resultUnknown": True}, {**good, "acknowledged": True}): + with self.assertRaises(ValueError): validate_conditional_response(changed, 412, baseline, code="PreconditionFailed") + success = {"response": {"ResponseMetadata": {"HTTPStatusCode": 200}, "ETag": '"etag"'}, + "acknowledged": True, "payload": baseline["payload"]} + validate_conditional_response(success, 200, baseline, full_body=True) + with self.assertRaisesRegex(ValueError, "full content"): + validate_conditional_response({**success, "payload": {"size": 3, "sha256": "wrong"}}, 200, baseline, full_body=True) def test_ssh_script_compiles_with_fault_module_and_keeps_structured_request(self): observed = [] def invoke(args, **params): @@ -269,6 +282,18 @@ def test_fault_requires_exact_target_and_unchanged_controls_and_metadata(self): with self.assertRaisesRegex(ValueError, "changed a control"): validate_repaired_physical(before, changed, target) + def test_completed_mpu_sidecar_cleanup_is_bound_to_target_drive_and_manifest(self): + before = self.snapshot() + target = self.target(before) + path = target["relativePath"] + ".meta" + before["node1"]["files"].append({"disk": target["disk"], "path": path, "size": 100, "sha256": "old staging"}) + after = copy.deepcopy(before) + after["node1"]["files"].pop() + validate_repaired_physical(before, after, target) + before["node1"]["files"][-1]["disk"] = 0 + with self.assertRaisesRegex(ValueError, "lost a baseline file"): + validate_repaired_physical(before, after, target) + def test_unknown_or_wrong_version_receipt_cannot_certify_physical_repair(self): target = self.target(self.snapshot()) receipt = HealHealthyOracleTest().receipt("repaired") From b070bf22d8a91fdb7d091433d2e0ab83c4e214cb Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 22:00:09 +0800 Subject: [PATCH 030/343] test(validation): migrate version repair and repeat Heal cases --- docs/validation-migration-map.md | 27 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 3 +- scripts/validation_heal/run_case.sh | 3 +- scripts/validation_heal/shards.py | 28 ++- scripts/validation_heal/versions.py | 358 ++++++++++++++++++++++++++++ test_validation_heal.py | 102 ++++++++ 7 files changed, 506 insertions(+), 17 deletions(-) create mode 100644 scripts/validation_heal/versions.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 4d41467b..e31e922a 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 86 currently dispatchable IDs include all 74 non-Heal cases and twelve Heal cases. Thirty-five have live PASS evidence; the other 123 have no live PASS claim. +case evidence. The 88 currently dispatchable IDs include all 74 non-Heal cases and fourteen Heal cases. Thirty-seven have live PASS evidence; the other 121 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -145,10 +145,22 @@ mounts, run-owned services/data roots, exact normal/deep responsibility receipts, repeated status, physical hashes and complete S3 inventories. It failed live in run `38052673980`: the candidate returned an `unknown` receipt for a fresh zero-byte object with no bucket-incarnation identity, violating the source healthy-object contract. Cleanup passed; the oracle remains strict. `heal.admin-invalid-request` passed all six branches on both H1 and H2 in run `38052868969`, with complete peer coverage and unchanged physical/S3 fixtures. -The external-shard executor also dispatches C01–C04, D02, E01 and E02 with all 24 source branches on H1/H2. It freezes exact version/dataDir/part, actual erasure role/set slot/disk UUID and full original shard hashes, stops all run-owned services before fault injection, witnesses the same fault immediately before the manual token, and checks complete physical repair plus streaming HighwayHash before any target GET. Full S3 inventories, cross-part Range, HEAD and pending MPU controls retain source semantics. Backups are used only for failed-case rescue or finalization; an unexpected new generation keeps the namespace DIRTY. These seven executors still require live evidence. A03 also has both H2 branches: a non-coordinator shard fault, exact token queries through coordinator/peer/fixed current-fleet entry, a 30-second convergence budget for task start and full terminal outcome, physical proof and S3 inventories through all four entries. +The external-shard executor also dispatches C01–C04, D02, E01 and E02 with all 24 source branches on H1/H2. It freezes exact version/dataDir/part, actual erasure role/set slot/disk UUID and full original shard hashes, stops all run-owned services before fault injection, witnesses the same fault immediately before the manual token, and checks complete physical repair plus streaming HighwayHash before any target GET. Full S3 inventories, cross-part Range, HEAD and pending MPU controls retain source semantics. Backups are used only for failed-case rescue or finalization; an unexpected new generation keeps the namespace DIRTY. Protected-mode C01 passed all ten H1/H2 branches in run `38057373420`; the other six executors still require live evidence. A03 also has both H2 branches: a non-coordinator shard fault, exact token queries through coordinator/peer/fixed current-fleet entry, a 30-second convergence budget for task start and full terminal outcome, physical proof and S3 inventories through all four entries. A04 dispatches all six dry-run branches with separate DB/DV fixtures on H1/H2. It freezes version and marker identities, requires complete dry_run_observed receipts and zero repaired counters, proves byte-for-byte no mutation, then rescues the injected fault before full S3 verification. It does not require a dry-run incarnation or claim backup restoration as product repair. The first C01/C02 runs (`38055041726`, `38055254447`) stopped before injection because the remote script had an invalid future-import placement; their FAIL records and clean finalization are retained. The script compiler regression is fixed, and fresh runs use the corrected executor. `heal.range-head-and-conditions` preserves all seven D07 branches, with real baseline/post-repair HEAD, cross-part Range, exact ETag conditions, and server-time-derived conditions including 304/412/416 negatives. Completed MPU target-side stale `part.N.meta` cleanup is allowed only on the repaired drive/dataDir and for parts still represented by the unchanged authoritative xl.meta; controls and payload shards remain exact. -The other 72 Heal cases remain +`heal.historical-versions` now preserves all four D03 branches: historical and +current part faults on distinct physical roles/drives, a recursive bucket fault +with missing historical part and another drive's complete xl.meta, and multiple +versions faulted together. All acknowledged versions and markers must have +exact, complete responsibility receipts and unchanged S3/physical references. +`heal.repeat-idempotency` preserves all four J04 branches. It freezes the first +repair's physical/S3 result, repeats normal/deep with independent tokens, and +restarts the same run-owned units for the process branch. The versioned branch +removes a whole xl.meta with three data versions and a marker; first repair may +attribute all identities or a mixture of repaired and positive health, while +repeat permits zero repairs and mode-specific health only. Counters must agree +with the complete receipt set. These two executors still require live evidence. +The other 70 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -184,3 +196,12 @@ their immutable artifacts. The corrected default-mode C01 run `38055597354` executed every H1/H2 fault branch with clean finalization. Single-part bytes and HighwayHash were restored, but the candidate returned unknown responsibility with no incarnation. The candidate explicitly defaults independent-shard writes off; current positive fixtures now enable both shard-integrity write and fleet-confirmation knobs only after every private peer is bound to the same candidate. Their values are read back from the actual processes and archived. The default-mode failure remains separate and never becomes a positive proof. New protected-mode runs are required before any new PASS claim. + +Protected-mode A01 run `38056636793` passed all 512 exact H1/H2 normal/deep +object responsibility receipts and clean finalization. This proves only the +explicit, uniformly enabled shard-integrity fixture configuration; the default +deployment failure remains retained. A04 run `38056884022` passed DB branches +but failed DV receipt completeness with cleanup PASS. The executor had used a +nonrecursive object request, which gives latest-object responsibility rather +than all historical versions. DV requests now explicitly set recursive=true +for the exact object prefix; the original FAIL remains and a new run is required. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 33433512..319207d1 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index c82ef2c3..6b6a7359 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -23,7 +23,8 @@ STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", - "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution"} + "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", + "heal.historical-versions", "heal.repeat-idempotency"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 3c631ae8..72e61f34 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -19,6 +19,7 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request) ;; + heal.historical-versions|heal.repeat-idempotency) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; @@ -43,7 +44,7 @@ tool_args=() if [ -n "${VALIDATION_HEAL_TOOL_DIRECTORY:-}" ]; then tool_args=(--tool-directory "${VALIDATION_HEAL_TOOL_DIRECTORY}") fi -[ "${driver}" != shards.py ] || [ "${#tool_args[@]}" -gt 0 ] || die "external-shard cases require candidate-bound bitrot tools" +[ "${driver}" = healthy.py ] || [ "${#tool_args[@]}" -gt 0 ] || die "physical Heal cases require candidate-bound bitrot tools" evidence="${artifacts}/evidence/${case_id}" mkdir -p "${artifacts}/cases" "${evidence}" python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index 6c208dde..f489cc84 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -186,9 +186,10 @@ def validate_fault(before, after, target, receipt): "file fault changed metadata, controls or another shard") -def validate_repaired_physical(before, after, target): +def validate_repaired_physical(before, after, target, *, repair_targets=None): left, right = files(before), files(after) - meta_path = f"{target['bucket']}/{target['key']}/xl.meta" + repair_targets = repair_targets or [target] + meta_paths = {f"{item['bucket']}/{item['key']}/xl.meta" for item in repair_targets} for node in before: require(before[node]["formats"] == after[node]["formats"], "repair changed disk format") for identity, entry in left.items(): @@ -196,12 +197,13 @@ def validate_repaired_physical(before, after, target): # Complete MPU commits the authoritative part manifest into xl.meta. # A healed target directory can discard only its stale per-part sidecars; # other versions, drives, pending uploads and payload files stay exact. - completed_sidecars = {f"{target['bucket']}/{target['key']}/{target['dataDir']}/part.{part['number']}.meta" - for part in target["logical"]["parts"]} - if actual is None and identity[:2] == (target["node"], target["disk"]) and identity[2] in completed_sidecars: + completed_sidecars = {(item["node"], item["disk"], + f"{item['bucket']}/{item['key']}/{item['dataDir']}/part.{part['number']}.meta") + for item in repair_targets for part in item["logical"]["parts"]} + if actual is None and identity in completed_sidecars: continue require(actual is not None, "repair lost a baseline file or control") - if identity[2] != meta_path: + if identity[2] not in meta_paths: require(actual == entry, "repair did not restore exact part bytes or changed a control") else: def semantics(metadata): @@ -211,9 +213,10 @@ def semantics(metadata): require(actual.get("metadata") and semantics(actual["metadata"]) == semantics(entry["metadata"]), "repair changed decoded version/part identity") for identity in set(right) - set(left): - original = left.get((identity[0], identity[1], meta_path)) - allowed = f"{target['bucket']}/{target['key']}/{target['dataDir']}/xl.meta.bkp" - require(identity[2] == allowed and original + allowed = {(item["node"], item["disk"], f"{item['bucket']}/{item['key']}/{item['dataDir']}/xl.meta.bkp") + for item in repair_targets} + original = left.get((identity[0], identity[1], identity[2].rsplit("/", 2)[0] + "/xl.meta")) + require(identity in allowed and original and right[identity]["sha256"] == original["sha256"] and right[identity]["size"] == original["size"], "repair created an unexpected physical file") return {"targetLengthAndFullSha256Restored": True, "controlFilesUnchanged": True, @@ -276,13 +279,15 @@ def peer_status_equal(coordinator, peer): and peer.get("truncated", False) is False) -def run_task(probe, fleet, target, mode, active, *, status_endpoints=(), option_overrides=None): +def run_task(probe, fleet, target, mode, active, *, status_endpoints=(), option_overrides=None, admissions=None): path = "/rustfs/admin/v3/heal/" + quote(target["bucket"], safe="") + "/" + quote(target["key"], safe="/") started = fleet.record("trigger", probe.admin(path, [], json.dumps( {**BASE_OPTIONS, "scanMode": mode, **(option_overrides or {})}).encode())) require(started.get("httpStatus") == 200 and isinstance(started.get("response"), dict), "repair admission failed") token = started["response"].get("clientToken") require(isinstance(token, str) and token, "repair admission omitted token") + if admissions is not None: + admissions.append({"path": path, "clientToken": token}) active.append((path, token)) deadline = time.monotonic() + 180 while True: @@ -490,7 +495,8 @@ def run_topology(args, topology, nodes, *, fixture="DB"): "ATTRIBUTION_NOT_OBSERVED: fault changed before the manual task") status = run_task(probe, fleet, target, mode, active, status_endpoints=("node2", "node3") if cross_node else (), - option_overrides={"dryRun": True, "remove": "remove1" in name, "recreate": "recreate1" in name} + option_overrides={"dryRun": True, "recursive": fixture == "DV", + "remove": "remove1" in name, "recreate": "recreate1" in name} if dry_run else None) listing = fleet.record("dry-run-version-list", probe.s3("list_object_versions", {"Bucket": bucket})) if dry_run else None fleet.stop() diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py new file mode 100644 index 00000000..ce238360 --- /dev/null +++ b/scripts/validation_heal/versions.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +"""D03/J04: frozen version cohorts, recursive repair and repeat responsibility.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys + +try: + from .heal_probe import Probe, compare_inventories, payload, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .shards import (SIZES, files, map_target, run_task, validate_fault, + validate_repaired_physical, verify_cohort, verify_ranges) +except ImportError: + from heal_probe import Probe, compare_inventories, payload, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from shards import (SIZES, files, map_target, run_task, validate_fault, + validate_repaired_physical, verify_cohort, verify_ranges) + + +BRANCHES = { + "heal.historical-versions": ("old-version-part", "current-version-part", + "recursive-all-versions", "multiple-version-targets"), + "heal.repeat-idempotency": ("normal-repeat", "deep-repeat", "repeat-after-restart", "versioned-repeat"), +} +VERSION_SIZE = 5 * 1024**2 + 123 + + +def version_identity(entry): + return entry["key"], entry.get("versionId") or "null" + + +def validate_version_receipts(status, bucket, expected, mode, *, required_repairs=(), metadata_repairs=()): + """Prove the complete responsibility set; counters alone never certify repair.""" + outcome = status.get("outcome") or {} + require(status.get("summary") == "finished" and status.get("truncated", False) is False + and outcome.get("execution", {}).get("state") == "completed" + and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False, + "version task lacks complete terminal coverage") + wanted = {version_identity(entry): entry for entry in expected} + require(wanted and len(wanted) == len(expected), "fixture has duplicate version responsibility") + objects = outcome.get("objects", []) + require(len(objects) == len(wanted), "version task omitted or duplicated a version/marker receipt") + actual, incarnation = {}, None + required_repairs, metadata_repairs = set(required_repairs), set(metadata_repairs) + require(required_repairs <= set(wanted) and metadata_repairs <= {key for key, _ in wanted}, + "repair plan escapes the selected responsibility set") + for receipt in objects: + identity = receipt.get("identity", {}) + pair = identity.get("object"), identity.get("versionId") or "null" + require(pair in wanted and pair not in actual and identity.get("bucket") == bucket + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0 + and isinstance(identity.get("bucketIncarnationId"), str) and identity["bucketIncarnationId"], + "version responsibility has a wrong, duplicate or incomplete identity") + incarnation = incarnation or identity["bucketIncarnationId"] + require(identity["bucketIncarnationId"] == incarnation, "version receipts mix bucket incarnations") + state = receipt.get("disposition", {}).get("state") + healthy = "metadata_healthy" if wanted[pair]["deleteMarker"] or mode == 1 else "verified_healthy" + if pair in required_repairs: + require(state == "repaired", "faulted version lacks a repaired responsibility") + elif pair[0] in metadata_repairs: + require(state in {"repaired", healthy}, "metadata recovery returned unknown or a false healthy disposition") + else: + require(state == healthy, "repeat or control version claimed repair, unknown, or insufficient integrity") + actual[pair] = state + require(set(actual) == set(wanted), "version responsibility set differs from acknowledged fixture") + for key in metadata_repairs: + require(any(pair[0] == key and state == "repaired" for pair, state in actual.items()), + "missing metadata was never attributed as repaired") + healed = sum(state == "repaired" for state in actual.values()) + counters = outcome.get("counters", {}) + wanted_counters = {"processed": len(wanted), "healed": healed, "unchanged": len(wanted) - healed, + "skipped": 0, "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False} + require(all(type(counters.get(name)) is type(value) and counters[name] == value + for name, value in wanted_counters.items()), "version counters differ from complete receipt dispositions") + return {"completeVersionAndMarkerResponsibility": True, "receipts": len(actual), "repaired": healed, + "mode": mode, "bucketIncarnationId": incarnation} + + +def choose_target(snapshot, bucket, entry, role, *, excluded_drives=()): + target = map_target(snapshot, bucket, entry["key"], role, 1, entry["versionId"], latest_required=False) + selected = next((shard for shard in target["cohort"] if shard["role"] == role + and (shard["node"], shard["disk"]) not in excluded_drives), None) + require(selected is not None, "distinct physical version fault position is unavailable") + return {**target, **selected} + + +def metadata_target(snapshot, part_target): + relative = f"{part_target['bucket']}/{part_target['key']}/xl.meta" + entry = files(snapshot)[part_target["node"], part_target["disk"], relative] + return {**part_target, "relativePath": relative, "sha256": entry["sha256"], "size": entry["size"]} + + +def branch_plan(case_id, branch, snapshot, bucket, expected, fixture): + """Plans are derived from ACK order and decoded physical roles, never disk numbers.""" + branch_index = BRANCHES[case_id].index(branch) + key = f"objects/{branch_index:04d}.bin" if fixture == "DV" else f"objects/{8 + 9 * branch_index:04d}.bin" + if branch == "versioned-repeat": + key = "objects/0000.bin" + selected = [entry for entry in expected if entry["key"] == key] + data = [entry for entry in selected if not entry["deleteMarker"]] + require(len(data) == (3 if fixture == "DV" else 1), "branch is missing its distinct acknowledged payload versions") + old = choose_target(snapshot, bucket, data[0], "data") + if case_id == "heal.repeat-idempotency": + if fixture == "DV": + proofs = [choose_target(snapshot, bucket, entry, "data") for entry in data] + # A whole xl.meta contains all three versions and the marker. Keep + # every version's target-drive maintenance references in the oracle. + proofs = [{**item, **next(shard for shard in item["cohort"] + if (shard["node"], shard["disk"]) == (old["node"], old["disk"]))} for item in proofs] + faults = [metadata_target(snapshot, old)] + required, metadata_keys = set(), {key} + else: + proofs, faults, required, metadata_keys = [old], [old], {version_identity(data[0])}, set() + return {"scope": {"bucket": bucket, "key": key}, "recursive": fixture == "DV", + "expected": selected, "faults": faults, "proofs": proofs, + "requiredRepairs": required, "metadataRepairs": metadata_keys} + current = choose_target(snapshot, bucket, data[-1], "parity", excluded_drives={(old["node"], old["disk"])}) + faults, proofs, metadata_keys = [old], [old], set() + if branch == "current-version-part": + require(data[-1]["isLatest"] is True, "current-version branch selected a historical version") + faults, proofs = [current], [current] + elif branch == "recursive-all-versions": + # Missing historical part plus another drive's complete metadata tests + # both F1 and F3 with an intact same-version authority quorum. + faults, proofs = [old, metadata_target(snapshot, current)], [old, current] + metadata_keys = {key} + elif branch == "multiple-version-targets": + faults, proofs = [old, current], [old, current] + recursive_bucket = branch == "recursive-all-versions" + return {"scope": {"bucket": bucket, "key": "" if recursive_bucket else key}, "recursive": True, + "expected": expected if recursive_bucket else selected, "faults": faults, "proofs": proofs, + "requiredRepairs": {version_identity(item) for item in faults if not item["relativePath"].endswith("/xl.meta")}, + "metadataRepairs": metadata_keys} + + +def verify_version_s3(probe, fleet, bucket, expected, baseline, label, *, fixture="DV"): + current = fleet.record(label, probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + comparison = compare_inventories(baseline, current) + ranges = [] + for entry in expected: + if entry["deleteMarker"]: + continue + head = fleet.record(label + "-head", probe.s3("head_object", { + "Bucket": bucket, "Key": entry["key"], "VersionId": entry["versionId"]})) + observed = next(item for item in baseline["entries"] if version_identity(item) == version_identity(entry)) + require(head["acknowledged"] and all(head["response"].get(name) == value + for name, value in observed["properties"].items()), "version HEAD differs from frozen public properties") + if entry["payload"]["size"] < 1024**2: + continue + nonce = next(index for index, item in enumerate( + item for item in expected if item["key"] == entry["key"] and not item["deleteMarker"]) + if item["versionId"] == entry["versionId"]) + ranges.append({"key": entry["key"], "versionId": entry["versionId"], "proof": verify_ranges( + probe, fleet, bucket, entry["key"], version_id=entry["versionId"], + expected_body=payload(f"{bucket}/{entry['key']}/{nonce}", entry["payload"]["size"]))}) + return {"inventory": comparison, "historicalRanges": ranges} + + +def run_topology(args, topology, nodes, fixture): + directory = args.evidence / fixture / topology + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory) + candidate = json.loads(args.candidate.read_text()) + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 256, "maxInventoryBytes": 512 * 1024**2} + fleet.record("binding", binding) + recipe = select(args.case_id)[0] + require(list(BRANCHES[args.case_id]) == recipe["variants"], "executor omitted a source version branch") + branches = [branch for branch in BRANCHES[args.case_id] + if args.case_id != "heal.repeat-idempotency" or (branch == "versioned-repeat") == (fixture == "DV")] + result = {"topology": topology, "fixture": fixture, "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} + active, admissions = [], [] + probe = Probe(binding) + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) + expected = seed(Recorder(probe, directory / "fixture"), bucket, fixture, + 4 if fixture == "DV" else 128, [VERSION_SIZE] if fixture == "DV" else SIZES) + baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + for branch in branches: + registrations = [] + try: + fleet.stop() + before = fleet.snapshot("physical-before-" + branch, quiescent=True) + plan = branch_plan(args.case_id, branch, before, bucket, expected, fixture) + fleet.record("fault-plan-" + branch, {**plan, "requiredRepairs": sorted(plan["requiredRepairs"]), + "metadataRepairs": sorted(plan["metadataRepairs"])}) + for target in plan["proofs"]: + verify_cohort(fleet, target, "baseline-bitrot-" + branch) + prior = before + for index, target in enumerate(plan["faults"]): + params = {"disk": target["disk"], "faultId": recipe["code"] + "-" + branch + "-" + str(index)} + registration = target["node"], params + fleet.faults.append(registration) + registrations.append(registration) + receipt = fleet.record("fault-" + branch, fleet.remote(target["node"], "apply-fault", + **params, relativePath=target["relativePath"], expectedSha256=target["sha256"], kind="missing")) + witnessed = fleet.snapshot("physical-fault-" + branch, quiescent=True) + validate_fault(prior, witnessed, target, receipt) + prior = witnessed + fleet.start() + witness = fleet.snapshot("physical-before-trigger-" + branch) + require(physical_content(prior) == physical_content(witness), + "ATTRIBUTION_NOT_OBSERVED: version fault changed before the manual token") + status = run_task(probe, fleet, plan["scope"], 2, active, + option_overrides={"recursive": plan["recursive"]}, admissions=admissions) + fleet.stop() + repaired = fleet.snapshot("physical-after-" + branch, quiescent=True) + physical = validate_repaired_physical(before, repaired, plan["proofs"][0], repair_targets=plan["proofs"]) + bitrot = [verify_cohort(fleet, target, "repaired-bitrot-" + branch) for target in plan["proofs"]] + fleet.start() + s3 = verify_version_s3(probe, fleet, bucket, expected, baseline, "s3-after-" + branch, fixture=fixture) + # These faults have now been independently repaired and fully + # proved. Finalization discards their backups; it never writes + # old metadata over a legitimate product re-encoding. + for registration in registrations: + fleet.faults.remove(registration) + registrations.clear() + fleet.record("fault-finalization-proof", {"variant": branch, "allOriginalPartBytesVerified": True, + "decodedVersionReferencesAndS3Equal": True, "backupRestorationUsed": False, + "canonicalVerdict": "still requires the exact responsibility oracle"}) + responsibility = validate_version_receipts(status, bucket, plan["expected"], 2, + required_repairs=plan["requiredRepairs"], metadata_repairs=plan["metadataRepairs"]) + repeats = [] + if args.case_id == "heal.repeat-idempotency": + fleet.stop() + post_heal = fleet.snapshot("post-heal-repeat-baseline-" + branch, quiescent=True) + fleet.start() + modes = (1,) if branch == "normal-repeat" else (2,) + if branch in {"repeat-after-restart", "versioned-repeat"}: + modes = (1, 2) + first_token = admissions[-1]["clientToken"] + for mode in modes: + # Restart the same run-owned units, keeping disks and + # candidate identity, before the process repeat branch. + if branch == "repeat-after-restart": + old_processes = fleet.snapshot("before-process-restart") + fleet.stop() + fleet.start() + new_processes = fleet.snapshot("restarted-process") + require(all(old_processes[node]["unit"] == new_processes[node]["unit"] + and old_processes[node]["pid"] != new_processes[node]["pid"] + for node in nodes), "process repeat did not restart the same original units") + fleet.record("process-restart-proof", {"before": old_processes, "after": new_processes}) + repeat = run_task(probe, fleet, plan["scope"], mode, active, + option_overrides={"recursive": plan["recursive"]}, admissions=admissions) + require(admissions[-1]["clientToken"] != first_token + and len({entry["clientToken"] for entry in admissions}) == len(admissions), + "repeat task reused a token instead of admitting independent responsibility") + proof = validate_version_receipts(repeat, bucket, plan["expected"], mode) + fleet.stop() + after_repeat = fleet.snapshot("physical-repeat-" + branch, quiescent=True) + validate_repaired_physical(post_heal, after_repeat, plan["proofs"][0], repair_targets=plan["proofs"]) + for target in plan["proofs"]: + verify_cohort(fleet, target, "repeat-bitrot-" + branch) + fleet.start() + proof["s3"] = verify_version_s3(probe, fleet, bucket, expected, baseline, "s3-repeat-" + branch, fixture=fixture) + repeats.append(proof) + result["branches"][branch] = {"verdict": "PASS", "physical": physical, "bitrot": bitrot, + "s3": s3, "responsibility": responsibility, "repeats": repeats} + except (Exception, KeyboardInterrupt) as exc: + result["branches"][branch] = {"verdict": "FAIL", "failure": {"type": type(exc).__name__, "message": str(exc)}} + try: + fleet.stop() + fleet.snapshot("physical-failure-" + branch, quiescent=True) + for registration in reversed(registrations): + node, params = registration + fleet.record("rescue-" + branch, fleet.remote(node, "recover-fault", **params)) + fleet.faults.remove(registration) + except Exception as recovery_error: + result["branches"][branch]["recoveryError"] = str(recovery_error) + raise + if isinstance(exc, KeyboardInterrupt) or active: + raise + result["verdict"] = "PASS" if len(result["branches"]) == len(branches) and all( + item["verdict"] == "PASS" for item in result["branches"].values()) else "FAIL" + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + finally: + for path, token in active: + try: + fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: + fleet.record("stop-error", {"error": str(exc)}) + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("topology-result", result) + return result + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f"Heal workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=tuple(BRANCHES), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path, required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(set(nodes)) == len(nodes) + and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") + require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") + require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") + fixtures = ("DV",) if args.case_id == "heal.historical-versions" else ("DB", "DV") + results = [] + try: + for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): + for fixture in fixtures: + results.append(run_topology(args, topology, selected, fixture)) + if results[-1]["cleanup"] != "PASS": + break + if results[-1]["cleanup"] != "PASS": + break + finally: + complete = len(results) == 2 * len(fixtures) and all( + item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) + save(args.evidence / "healthy-summary.json", {"caseId": args.case_id, "results": results, + "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", "complete": complete}) + require(complete, "version case requires every H1/H2 source branch; inspect topology-result evidence") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"Heal version validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/test_validation_heal.py b/test_validation_heal.py index 9434f97d..ea25ffdc 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -16,6 +16,7 @@ from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal, validate_dry_run_status from scripts.validation_heal.heal_fault import mutation from scripts.validation_heal.shards import validate_conditional_response +from scripts.validation_heal.versions import BRANCHES as VERSION_BRANCHES, validate_version_receipts, branch_plan class HealContractTest(unittest.TestCase): @@ -225,6 +226,8 @@ def test_executor_preserves_each_source_branch_and_uses_distinct_objects(self): self.assertEqual(len({branch[1] for branch in branches}), len(branches)) self.assertEqual(inventory[case_id]["contract"]["probes"]["topology"], "H2" if case_id == "heal.admin-token-cross-node-status" else "H1,H2") + for case_id, branches in VERSION_BRANCHES.items(): + self.assertEqual(list(branches), inventory[case_id]["variants"]) def test_peer_status_cannot_converge_with_wrong_task_unknown_or_incomplete_receipts(self): coordinator = {**HealHealthyOracleTest().receipt("repaired"), "startTime": "exact-task-start"} @@ -340,5 +343,104 @@ def test_fault_requires_actual_service_quiescence_and_exact_truncate_length(self mutation(b"abcdef", "truncate", length) +class HealVersionOracleTest(unittest.TestCase): + def fixture(self): + expected = [{"key": "object", "versionId": version, "deleteMarker": version == "marker"} + for version in ("old", "middle", "current", "marker")] + status = HealHealthyOracleTest().receipt() + original = status["outcome"]["objects"][0] + original["identity"].update(bucket="fixture", object="object") + status["outcome"]["objects"] = [copy.deepcopy(original) for _ in expected] + for receipt, entry in zip(status["outcome"]["objects"], expected): + receipt["identity"]["versionId"] = entry["versionId"] + receipt["disposition"]["state"] = "metadata_healthy" if entry["deleteMarker"] else "verified_healthy" + status["outcome"]["counters"].update(processed=4, unchanged=4) + return status, expected + + def test_all_version_receipts_and_counters_are_required(self): + status, expected = self.fixture() + status["outcome"]["objects"][0]["disposition"]["state"] = "repaired" + status["outcome"]["counters"].update(healed=1, unchanged=3) + validate_version_receipts(status, "fixture", expected, 2, required_repairs={("object", "old")}) + for kind in ("missing-marker", "duplicate", "wrong-version", "unknown", "incarnation", "boolean-counter", "healthy-fault"): + bad = copy.deepcopy(status) + objects = bad["outcome"]["objects"] + if kind == "missing-marker": objects.pop() + elif kind == "duplicate": objects[-1] = copy.deepcopy(objects[0]) + elif kind == "wrong-version": objects[0]["identity"]["versionId"] = "another" + elif kind == "unknown": objects[0]["disposition"]["state"] = "unknown" + elif kind == "incarnation": objects[0]["identity"]["bucketIncarnationId"] = "other" + elif kind == "boolean-counter": bad["outcome"]["counters"]["healed"] = True + else: objects[0]["disposition"]["state"] = "verified_healthy" + with self.subTest(kind=kind), self.assertRaises(ValueError): + validate_version_receipts(bad, "fixture", expected, 2, required_repairs={("object", "old")}) + + def test_repeat_requires_mode_specific_health_and_zero_repairs(self): + status, expected = self.fixture() + validate_version_receipts(status, "fixture", expected, 2) + with self.assertRaises(ValueError): validate_version_receipts(status, "fixture", expected, 1) + for entry in status["outcome"]["objects"]: entry["disposition"]["state"] = "metadata_healthy" + validate_version_receipts(status, "fixture", expected, 1) + with self.assertRaises(ValueError): validate_version_receipts(status, "fixture", expected, 2) + status["outcome"]["objects"][0]["disposition"]["state"] = "repaired" + status["outcome"]["counters"].update(healed=1, unchanged=3) + with self.assertRaises(ValueError): validate_version_receipts(status, "fixture", expected, 1) + + def test_metadata_repair_may_repair_all_receipts_but_marker_cannot_claim_deep_payload(self): + status, expected = self.fixture() + with self.assertRaisesRegex(ValueError, "never attributed"): + validate_version_receipts(status, "fixture", expected, 2, metadata_repairs={"object"}) + for entry in status["outcome"]["objects"]: entry["disposition"]["state"] = "repaired" + status["outcome"]["counters"].update(healed=4, unchanged=0) + validate_version_receipts(status, "fixture", expected, 2, metadata_repairs={"object"}) + status["outcome"]["objects"][-1]["disposition"]["state"] = "verified_healthy" + status["outcome"]["counters"].update(healed=3, unchanged=1) + with self.assertRaises(ValueError): + validate_version_receipts(status, "fixture", expected, 2, metadata_repairs={"object"}) + + def test_version_fault_plan_uses_ack_versions_distinct_disks_and_exact_recursive_scope(self): + original = HealShardOracleTest().snapshot() + observation = original["node1"] + templates = {entry["disk"]: entry for entry in observation["files"] if "metadata" in entry} + shards = {entry["disk"]: entry for entry in observation["files"] if entry["path"].endswith("part.1")} + snapshot = {"node1": {"formats": observation["formats"], "files": []}} + bucket, expected = "rv-heal-0123456789abcdef", [] + for key_index in range(4): + key = f"objects/{key_index:04d}.bin" + version_ids = [f"00000000-0000-0000-0000-{key_index * 3 + index + 1:012d}" for index in range(3)] + for index, version in enumerate(version_ids): + expected.append({"key": key, "versionId": version, "deleteMarker": False, + "isLatest": index == 2 and key_index % 2 == 1}) + if key_index % 2 == 0: + expected.append({"key": key, "versionId": "marker-" + str(key_index), "deleteMarker": True, "isLatest": True}) + for disk in range(4): + meta = copy.deepcopy(templates[disk]) + meta["path"] = f"{bucket}/{key}/xl.meta" + meta["metadata"]["key"] = key + version_template = meta["metadata"]["versions"][0] + meta["metadata"]["versions"] = [] + for index, version in enumerate(version_ids): + value = copy.deepcopy(version_template) + value.update(versionId=version, dataDir=version, isLatest=index == 2 and key_index % 2 == 1) + meta["metadata"]["versions"].append(value) + shard = copy.deepcopy(shards[disk]) + shard.update(path=f"{bucket}/{key}/{version}/part.1", sha256=f"{key_index}-{index}-{disk}") + snapshot["node1"]["files"].append(shard) + snapshot["node1"]["files"].append(meta) + for branch in VERSION_BRANCHES["heal.historical-versions"]: + plan = branch_plan("heal.historical-versions", branch, snapshot, bucket, expected, "DV") + self.assertTrue(plan["recursive"]) + if branch == "recursive-all-versions": + self.assertEqual(plan["scope"]["key"], "") + self.assertEqual(plan["expected"], expected) + self.assertTrue(any(item["relativePath"].endswith("xl.meta") for item in plan["faults"])) + if len(plan["faults"]) == 2: + self.assertEqual(len({(item["node"], item["disk"]) for item in plan["faults"]}), 2) + plan = branch_plan("heal.repeat-idempotency", "versioned-repeat", snapshot, bucket, expected, "DV") + self.assertEqual(len(plan["expected"]), 4) + self.assertEqual(len(plan["proofs"]), 3) + self.assertTrue(any(entry["deleteMarker"] for entry in plan["expected"])) + + if __name__ == "__main__": unittest.main() From 2a9c936e5e7f502a3e5227e10fdb2e34332a7237 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 22:06:33 +0800 Subject: [PATCH 031/343] test(validation): migrate suspended null-version Heal branches --- docs/validation-migration-map.md | 12 +++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 88 ++++++++++++++++++++++++++++- test_validation_heal.py | 33 ++++++++++- 6 files changed, 130 insertions(+), 9 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index e31e922a..81c25482 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 88 currently dispatchable IDs include all 74 non-Heal cases and fourteen Heal cases. Thirty-seven have live PASS evidence; the other 121 have no live PASS claim. +case evidence. The 89 currently dispatchable IDs include all 74 non-Heal cases and fifteen Heal cases. Thirty-seven have live PASS evidence; the other 121 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -159,8 +159,14 @@ restarts the same run-owned units for the process branch. The versioned branch removes a whole xl.meta with three data versions and a marker; first repair may attribute all identities or a mixture of repaired and positive health, while repeat permits zero repairs and mode-specific health only. Counters must agree -with the complete receipt set. These two executors still require live evidence. -The other 70 Heal cases remain +with the complete receipt set. D05 `heal.versioning-suspended-null` now uses real +S3 versioning suspension, consecutive null overwrites and acknowledged null +marker creation/deletion. It faults a live null data/parity part or an entire +xl.meta containing the current null marker and historical versions, verifies +every exact identity, then re-enables versioning and checks a new non-null +version plus the unchanged original version set. These three executors still +require live evidence. +The other 69 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 319207d1..9b45bd90 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 6b6a7359..bd6d37ee 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -24,7 +24,7 @@ AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", - "heal.historical-versions", "heal.repeat-idempotency"} + "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 72e61f34..9594b9b6 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -19,7 +19,7 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request) ;; - heal.historical-versions|heal.repeat-idempotency) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index ce238360..a398ee83 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import copy from datetime import datetime, timedelta, timezone import hashlib import json @@ -36,6 +37,7 @@ "heal.historical-versions": ("old-version-part", "current-version-part", "recursive-all-versions", "multiple-version-targets"), "heal.repeat-idempotency": ("normal-repeat", "deep-repeat", "repeat-after-restart", "versioned-repeat"), + "heal.versioning-suspended-null": ("enabled-to-suspended", "null-overwrite", "null-delete-marker"), } VERSION_SIZE = 5 * 1024**2 + 123 @@ -106,6 +108,65 @@ def metadata_target(snapshot, part_target): return {**part_target, "relativePath": relative, "sha256": entry["sha256"], "size": entry["size"]} +def suspend_fixture(recorder, bucket, historical): + expected = copy.deepcopy(historical) + recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Suspended"}}) + require(recorder.call("get_bucket_versioning", {"Bucket": bucket}).get("Status") == "Suspended", + "versioning suspension was not persisted") + for key in sorted({entry["key"] for entry in historical}): + body = payload(f"{bucket}/{key}/3", VERSION_SIZE) + params = {"Bucket": bucket, "Key": key, "ContentType": "application/octet-stream", + "Metadata": {"heal-fixture": "suspended-null"}, "Tagging": "suite=heal&fixture=DV"} + for content in (payload(f"{bucket}/{key}/superseded-null", VERSION_SIZE), body): + response = recorder.call("put_object", params, content) + require(response.get("VersionId", "null") == "null", "suspended PUT created a non-null version") + # Create/delete one null marker on every key, then rebuild the live null + # value. This witnesses actual suspended overwrite and deletion ACKs; + # it does not invent history by editing xl.meta. + marker = recorder.call("delete_object", {"Bucket": bucket, "Key": key}) + require(marker.get("DeleteMarker") is True and marker.get("VersionId", "null") == "null", + "suspended delete did not acknowledge a null marker") + deleted = recorder.call("delete_object", {"Bucket": bucket, "Key": key, "VersionId": "null"}) + require(deleted.get("DeleteMarker") is True and deleted.get("VersionId", "null") == "null", + "precise null-marker removal was not acknowledged") + response = recorder.call("put_object", params, body) + require(response.get("VersionId", "null") == "null", "suspended overwrite changed null identity") + for entry in expected: + if entry["key"] == key: + entry["isLatest"] = False + null_entry = {"key": key, "versionId": "null", "deleteMarker": False, "isLatest": True, + "payload": {"sha256": hashlib.sha256(body).hexdigest(), "size": len(body)}} + if key == "objects/0002.bin": + marker = recorder.call("delete_object", {"Bucket": bucket, "Key": key}) + require(marker.get("DeleteMarker") is True and marker.get("VersionId", "null") == "null", + "final suspended delete lacks the exact null marker") + null_entry = {"key": key, "versionId": "null", "deleteMarker": True, "isLatest": True} + expected.append(null_entry) + return expected + + +def reenable_and_verify(recorder, fleet, bucket, baseline): + recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) + require(recorder.call("get_bucket_versioning", {"Bucket": bucket}).get("Status") == "Enabled", + "versioning was not re-enabled after null repair") + key = "objects/re-enabled.bin" + body = payload(f"{bucket}/{key}/0", VERSION_SIZE) + put = recorder.call("put_object", {"Bucket": bucket, "Key": key}, body) + require(put.get("VersionId") not in {None, "null"}, "re-enabled versioning did not create a real new version") + current = fleet.record("re-enabled-s3", recorder.probe.inventory(bucket)) + new_entries = [entry for entry in current["entries"] if entry["key"] == key] + require(len(new_entries) == 1 and new_entries[0]["versionId"] == put["VersionId"] + and new_entries[0]["payload"] == {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()} + and new_entries[0]["isLatest"] is True and not new_entries[0]["deleteMarker"], + "re-enabled version identity or independent payload differs") + # The inventory's current listing also contains the new key. Compare + # original entries directly, and require one exact additional entry. + require(len(current["entries"]) == len(baseline["entries"]) + 1 + and [entry for entry in current["entries"] if entry["key"] != key] == baseline["entries"], + "re-enabling versioning changed prior null, historical or marker identities/content") + return {"reenabled": True, "newVersionId": put["VersionId"], "allOriginalVersionsStillAddressable": True} + + def branch_plan(case_id, branch, snapshot, bucket, expected, fixture): """Plans are derived from ACK order and decoded physical roles, never disk numbers.""" branch_index = BRANCHES[case_id].index(branch) @@ -114,6 +175,24 @@ def branch_plan(case_id, branch, snapshot, bucket, expected, fixture): key = "objects/0000.bin" selected = [entry for entry in expected if entry["key"] == key] data = [entry for entry in selected if not entry["deleteMarker"]] + if case_id == "heal.versioning-suspended-null": + historical = [entry for entry in data if entry["versionId"] != "null"] + require(len(historical) == 3, "suspended fixture lost enabled-period historical versions") + if branch == "null-delete-marker": + old = choose_target(snapshot, bucket, historical[0], "data") + proofs = [choose_target(snapshot, bucket, entry, "data") for entry in historical] + proofs = [{**item, **next(shard for shard in item["cohort"] + if (shard["node"], shard["disk"]) == (old["node"], old["disk"]))} for item in proofs] + require(any(entry["versionId"] == "null" and entry["deleteMarker"] and entry["isLatest"] + for entry in selected), "null-marker branch lacks its current marker") + faults, required, metadata_keys = [metadata_target(snapshot, old)], set(), {key} + else: + null = next((entry for entry in data if entry["versionId"] == "null" and entry["isLatest"]), None) + require(null is not None, "suspended fixture lost its live null identity") + target = choose_target(snapshot, bucket, null, "data" if branch_index == 0 else "parity") + faults, proofs, required, metadata_keys = [target], [target], {version_identity(null)}, set() + return {"scope": {"bucket": bucket, "key": key}, "recursive": True, "expected": selected, + "faults": faults, "proofs": proofs, "requiredRepairs": required, "metadataRepairs": metadata_keys} require(len(data) == (3 if fixture == "DV" else 1), "branch is missing its distinct acknowledged payload versions") old = choose_target(snapshot, bucket, data[0], "data") if case_id == "heal.repeat-idempotency": @@ -200,8 +279,11 @@ def run_topology(args, topology, nodes, fixture): try: fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) - expected = seed(Recorder(probe, directory / "fixture"), bucket, fixture, + recorder = Recorder(probe, directory / "fixture") + expected = seed(recorder, bucket, fixture, 4 if fixture == "DV" else 128, [VERSION_SIZE] if fixture == "DV" else SIZES) + if args.case_id == "heal.versioning-suspended-null": + expected = suspend_fixture(recorder, bucket, expected) baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) for branch in branches: @@ -301,6 +383,8 @@ def run_topology(args, topology, nodes, fixture): raise result["verdict"] = "PASS" if len(result["branches"]) == len(branches) and all( item["verdict"] == "PASS" for item in result["branches"].values()) else "FAIL" + if args.case_id == "heal.versioning-suspended-null" and result["verdict"] == "PASS": + result["reenable"] = reenable_and_verify(recorder, fleet, bucket, baseline) except (Exception, KeyboardInterrupt) as exc: result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) finally: @@ -332,7 +416,7 @@ def interrupted(signum, frame): and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") - fixtures = ("DV",) if args.case_id == "heal.historical-versions" else ("DB", "DV") + fixtures = ("DB", "DV") if args.case_id == "heal.repeat-idempotency" else ("DV",) results = [] try: for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): diff --git a/test_validation_heal.py b/test_validation_heal.py index ea25ffdc..8d5fca75 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -16,7 +16,7 @@ from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal, validate_dry_run_status from scripts.validation_heal.heal_fault import mutation from scripts.validation_heal.shards import validate_conditional_response -from scripts.validation_heal.versions import BRANCHES as VERSION_BRANCHES, validate_version_receipts, branch_plan +from scripts.validation_heal.versions import BRANCHES as VERSION_BRANCHES, validate_version_receipts, branch_plan, suspend_fixture class HealContractTest(unittest.TestCase): @@ -344,6 +344,37 @@ def test_fault_requires_actual_service_quiescence_and_exact_truncate_length(self class HealVersionOracleTest(unittest.TestCase): + def test_suspended_null_fixture_retains_historical_identities_and_requires_real_marker_acks(self): + historical = [{"key": f"objects/{index:04d}.bin", "versionId": f"history-{index}-{version}", + "deleteMarker": False, "isLatest": version == 2} + for index in range(3) for version in range(3)] + calls = [] + def invoke(operation, params, body=None): + calls.append((operation, copy.deepcopy(params))) + if operation == "get_bucket_versioning": return {"Status": "Suspended"} + if operation == "put_object": return {"VersionId": "null"} + if operation == "delete_object": return {"VersionId": "null", "DeleteMarker": True} + return {} + recorder = types.SimpleNamespace(call=invoke) + expected = suspend_fixture(recorder, "fixture", historical) + self.assertEqual(len(expected), 12) + self.assertEqual({item["versionId"] for item in expected if item["versionId"] != "null"}, + {item["versionId"] for item in historical}) + self.assertTrue(all(not item["isLatest"] for item in expected if item["versionId"] != "null")) + markers = [item for item in expected if item["deleteMarker"]] + self.assertEqual(markers, [{"key": "objects/0002.bin", "versionId": "null", "isLatest": True, "deleteMarker": True}]) + self.assertEqual(sum(operation == "delete_object" and params.get("VersionId") == "null" + for operation, params in calls), 3) + for kind in ("fake-version", "missing-marker"): + def invalid(operation, params, body=None): + result = invoke(operation, params, body) + if kind == "fake-version" and operation == "put_object": result["VersionId"] = "unexpected-new-version" + elif kind == "missing-marker" and operation == "delete_object": result["DeleteMarker"] = False + return result + with self.subTest(kind=kind), self.assertRaises(ValueError): + suspend_fixture(types.SimpleNamespace(call=invalid), "fixture", historical) + self.assertTrue(all(item["isLatest"] == (index % 3 == 2) for index, item in enumerate(historical))) + def fixture(self): expected = [{"key": "object", "versionId": version, "deleteMarker": version == "marker"} for version in ("old", "middle", "current", "marker")] From dfa84c6b2b4f1b2f0cd9ed31901d5f55e4183eb0 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 22:13:46 +0800 Subject: [PATCH 032/343] test(validation): prove below-quorum failures against candidate contract --- docs/validation-migration-map.md | 18 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- .../prepare_heal_meta_inspector.py | 35 ++++++ scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 112 +++++++++++++++++- test_validation_heal.py | 59 ++++++++- 7 files changed, 221 insertions(+), 9 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 81c25482..af2ec6fa 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 89 currently dispatchable IDs include all 74 non-Heal cases and fifteen Heal cases. Thirty-seven have live PASS evidence; the other 121 have no live PASS claim. +case evidence. The 90 currently dispatchable IDs include all 74 non-Heal cases and sixteen Heal cases. Thirty-eight have live PASS evidence; the other 120 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -166,7 +166,18 @@ xl.meta containing the current null marker and historical versions, verifies every exact identity, then re-enables versioning and checks a new non-null version plus the unchanged original version set. These three executors still require live evidence. -The other 69 Heal cases remain +B07 `heal.below-read-quorum` now retains exactly R-1 same-version authorities +and isolates all other part files, separately for never-versioned and historical +fixtures. The exact candidate tool build freezes the recognized metadata read +quorum rule, S3 HTTP/error mapping and source digests. Unrecognized or stale +contracts cannot dispatch the negative case. GET must return the frozen refusal +without payload or an unknown transport outcome; manual deep Heal must retain +the exact failed version and complete truthful counters. All good parts and +metadata must remain unchanged. Only after that failure proof may backups be +restored and a new token verify recovered integrity; recovery never proves +product repair during the negative window. This executor still requires live +evidence and a fresh tool build containing the quorum contract. +The other 68 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -211,3 +222,6 @@ but failed DV receipt completeness with cleanup PASS. The executor had used a nonrecursive object request, which gives latest-object responsibility rather than all historical versions. DV requests now explicitly set recursive=true for the exact object prefix; the original FAIL remains and a new run is required. +The corrected recursive DV request passed all DB/DV H1/H2 branches in run +`38057908339` with clean finalization, including complete dry_run_observed +version/marker receipts and byte-for-byte no-mutation proof. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 9b45bd90..d5367de6 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index bd6d37ee..e8a80186 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -24,7 +24,7 @@ AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", - "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null"} + "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/prepare_heal_meta_inspector.py b/scripts/validation_heal/prepare_heal_meta_inspector.py index 4f5069f1..af2e5216 100644 --- a/scripts/validation_heal/prepare_heal_meta_inspector.py +++ b/scripts/validation_heal/prepare_heal_meta_inspector.py @@ -9,6 +9,40 @@ import subprocess +def quorum_contract(source: Path, commit: str) -> dict: + """Freeze only recognized candidate rules; unfamiliar source stays unsupported.""" + paths = ("rustfs/src/error.rs", "crates/ecstore/src/set_disk/metadata.rs") + contents = {path: (source / path).read_text() for path in paths} + code_only = {path: re.sub(r'"(?:\\.|[^"\\])*"', '""', + re.sub(r"/\*.*?\*/|//[^\n]*", "", value, flags=re.S)) for path, value in contents.items()} + compact = {path: re.sub(r"\s+", "", value) for path, value in code_only.items()} + metadata = compact[paths[1]] + signature = "pub(super)fnobject_quorum_from_meta(" + start = metadata.find(signature) + function = "" + if start >= 0: + opening = metadata.find("{", start) + if opening >= 0: + depth = 0 + for index in range(opening, len(metadata)): + depth += (metadata[index] == "{") - (metadata[index] == "}") + if depth == 0: + function = metadata[opening:index + 1] + break + errors = compact[paths[0]] + known_read = ("letdata_blocks=parts_metadata.len()asi32-parity_blocks;" in function + and function.endswith("Ok((data_blocks,write_quorum))}")) + known_error = ('StorageError::ErasureReadQuorum|StorageError::InsufficientReadQuorum(_,_)=>{' + 'S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into())}' in errors + and 'S3ErrorCode::Custom(custom)if&**custom==SLOW_DOWN_READ_CODE=>Some(StatusCode::SERVICE_UNAVAILABLE)' in errors) + code = re.search(r'const\s+SLOW_DOWN_READ_CODE\s*:\s*&str\s*=\s*"([A-Za-z0-9]+)"\s*;', contents[paths[0]]) + return {"candidateCommit": commit, "recognized": bool(known_read and known_error and code), + "readQuorumRule": "decoded-data-blocks" if known_read else None, + "unrecoverableGet": {"httpStatus": 503, "code": code[1]} if known_error and code else None, + "sourceFiles": {path: hashlib.sha256(value.encode()).hexdigest() for path, value in contents.items()}, + "extractorSourceSha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest()} + + def prepare(source: Path, commit: str, output: Path) -> None: if not re.fullmatch(r"[0-9a-f]{40}", commit): raise ValueError("full candidate commit required") @@ -36,6 +70,7 @@ def prepare(source: Path, commit: str, output: Path) -> None: (output / "source.json").write_text(json.dumps({"candidateCommit": head, "candidateLockSha256": hashlib.sha256((source / "Cargo.lock").read_bytes()).hexdigest(), "helperSourceSha256": hashlib.sha256(text.encode()).hexdigest(), + "quorumContract": quorum_contract(source, head), "role": "read-only decoder; not the RustFS service under test", "built": False}, indent=2) + "\n") diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 9594b9b6..09c90e90 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -19,7 +19,7 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request) ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index a398ee83..1172d499 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -38,6 +38,7 @@ "recursive-all-versions", "multiple-version-targets"), "heal.repeat-idempotency": ("normal-repeat", "deep-repeat", "repeat-after-restart", "versioned-repeat"), "heal.versioning-suspended-null": ("enabled-to-suspended", "null-overwrite", "null-delete-marker"), + "heal.below-read-quorum": ("r-minus-1-unversioned", "r-minus-1-historical"), } VERSION_SIZE = 5 * 1024**2 + 123 @@ -94,6 +95,68 @@ def validate_version_receipts(status, bucket, expected, mode, *, required_repair "mode": mode, "bucketIncarnationId": incarnation} +def verify_quorum_contract(manifest, candidate): + contract = manifest.get("quorumContract") or {} + extractor = Path(__file__).with_name("prepare_heal_meta_inspector.py") + require(contract.get("recognized") is True and contract.get("candidateCommit") == candidate["source_sha"] + and contract.get("readQuorumRule") == "decoded-data-blocks" + and contract.get("extractorSourceSha256") == hashlib.sha256(extractor.read_bytes()).hexdigest() + and set(contract.get("sourceFiles", {})) == {"rustfs/src/error.rs", "crates/ecstore/src/set_disk/metadata.rs"} + and all(re.fullmatch(r"[0-9a-f]{64}", value) for value in contract["sourceFiles"].values()) + and contract.get("unrecoverableGet", {}).get("httpStatus") == 503 + and isinstance(contract.get("unrecoverableGet", {}).get("code"), str), + "negative quorum case requires a fresh recognized exact-candidate contract artifact") + return contract + + +def validate_unrecoverable_get(observation, contract): + expected = contract["unrecoverableGet"] + response = observation.get("response") or {} + require(observation.get("acknowledged") is False and observation.get("resultUnknown") is False + and "payload" not in observation + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == expected["httpStatus"] + and response.get("Error", {}).get("Code") == expected["code"], + "below-quorum GET returned data, an unknown transport outcome or a different frozen S3 error") + + +def validate_failed_version(status, bucket, expected, target): + outcome = status.get("outcome") or {} + require(status.get("summary") == "finished" and status.get("truncated", False) is False + and outcome.get("execution", {}).get("state") == "completed_with_errors" + and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False, + "negative Heal task hid failed execution or incomplete responsibility") + objects = outcome.get("objects", []) + wanted = {version_identity(entry): entry for entry in expected} + require(version_identity(target) in wanted and len(objects) == len(wanted), "negative Heal omitted a version or marker") + actual = set() + for receipt in objects: + identity = receipt.get("identity", {}) + pair = identity.get("object"), identity.get("versionId") or "null" + require(pair in wanted and pair not in actual and identity.get("bucket") == bucket + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0, + "negative Heal attributed the failure to another version or set") + disposition = receipt.get("disposition", {}) + if pair == version_identity(target): + require(disposition.get("state") == "failed" + and disposition.get("details") in {"recoverable", "retry_exhausted", "permanent"}, + "unrecoverable version was reported repaired, healthy, absent, skipped or unknown") + else: + require(disposition.get("state") == ("metadata_healthy" if wanted[pair]["deleteMarker"] else "verified_healthy"), + "negative Heal lost another historical version's positive integrity") + actual.add(pair) + require(actual == set(wanted), "negative Heal changed the responsibility set") + wanted_counters = {"processed": len(wanted), "healed": 0, "unchanged": len(wanted) - 1, + "failed": 1, "skipped": 0, "unknown": 0, "overflowed": False} + counters = outcome.get("counters", {}) + require(all(type(counters.get(name)) is type(value) and counters[name] == value + for name, value in wanted_counters.items()) + and type(counters.get("attemptFailures")) is int and counters["attemptFailures"] >= 1, + "negative Heal failure counters hide attempts or claim repair") + return {"failedVersion": version_identity(target), "completeResponsibility": True, + "attemptFailures": counters["attemptFailures"], "repairPassClaimed": False} + + def choose_target(snapshot, bucket, entry, role, *, excluded_drives=()): target = map_target(snapshot, bucket, entry["key"], role, 1, entry["versionId"], latest_required=False) selected = next((shard for shard in target["cohort"] if shard["role"] == role @@ -175,6 +238,16 @@ def branch_plan(case_id, branch, snapshot, bucket, expected, fixture): key = "objects/0000.bin" selected = [entry for entry in expected if entry["key"] == key] data = [entry for entry in selected if not entry["deleteMarker"]] + if case_id == "heal.below-read-quorum": + target = choose_target(snapshot, bucket, data[0], "data") + read_quorum = target["logical"]["erasure"]["data_blocks"] + ordered = sorted(target["cohort"], key=lambda entry: entry["index"]) + retained, missing = ordered[:read_quorum - 1], ordered[read_quorum - 1:] + require(len(retained) == read_quorum - 1 and missing, "below-quorum fault did not retain exactly R-1 authorities") + return {"scope": {"bucket": bucket, "key": key}, "recursive": True, "expected": selected, + "faults": [{**target, **shard} for shard in missing], "proofs": [target], + "requiredRepairs": set(), "metadataRepairs": set(), "negative": True, + "readQuorum": read_quorum, "retainedShards": retained} if case_id == "heal.versioning-suspended-null": historical = [entry for entry in data if entry["versionId"] != "null"] require(len(historical) == 3, "suspended fixture lost enabled-period historical versions") @@ -257,7 +330,8 @@ def run_topology(args, topology, nodes, fixture): run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" fleet = Fleet(nodes, directory, run_id, args.tool_directory) candidate = json.loads(args.candidate.read_text()) - fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + tool_manifest = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + contract = verify_quorum_contract(tool_manifest, candidate) if args.case_id == "heal.below-read-quorum" else None bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) @@ -272,7 +346,8 @@ def run_topology(args, topology, nodes, fixture): recipe = select(args.case_id)[0] require(list(BRANCHES[args.case_id]) == recipe["variants"], "executor omitted a source version branch") branches = [branch for branch in BRANCHES[args.case_id] - if args.case_id != "heal.repeat-idempotency" or (branch == "versioned-repeat") == (fixture == "DV")] + if (args.case_id != "heal.repeat-idempotency" or (branch == "versioned-repeat") == (fixture == "DV")) + and (args.case_id != "heal.below-read-quorum" or (branch == "r-minus-1-historical") == (fixture == "DV"))] result = {"topology": topology, "fixture": fixture, "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} active, admissions = [], [] probe = Probe(binding) @@ -311,10 +386,37 @@ def run_topology(args, topology, nodes, fixture): witness = fleet.snapshot("physical-before-trigger-" + branch) require(physical_content(prior) == physical_content(witness), "ATTRIBUTION_NOT_OBSERVED: version fault changed before the manual token") + if plan.get("negative"): + target = plan["proofs"][0] + fleet.record("quorum-contract", {"contract": contract, "readQuorum": plan["readQuorum"], + "availableSameVersionShards": len(plan["retainedShards"]), "poolIndex": 0, "setIndex": 0}) + denied = fleet.record("negative-get", probe.s3("get_object", { + "Bucket": bucket, "Key": target["key"], "VersionId": target["versionId"]})) + validate_unrecoverable_get(denied, contract) + require(physical_content(prior) == physical_content(fleet.snapshot("physical-after-negative-get")), + "negative read widened damage or silently healed below-quorum fragments") status = run_task(probe, fleet, plan["scope"], 2, active, option_overrides={"recursive": plan["recursive"]}, admissions=admissions) fleet.stop() repaired = fleet.snapshot("physical-after-" + branch, quiescent=True) + negative_proof = None + if plan.get("negative"): + require(physical_content(prior) == physical_content(repaired), + "below-quorum Heal deleted metadata, changed good shards or falsely persisted repair") + negative_proof = validate_failed_version(status, bucket, plan["expected"], plan["proofs"][0]) + fleet.record("negative-before-rescue-proof", negative_proof) + for registration in reversed(registrations): + node, params = registration + fleet.record("negative-rescue", fleet.remote(node, "recover-fault", **params)) + fleet.faults.remove(registration) + registrations.clear() + recovered = fleet.snapshot("physical-backup-recovered-" + branch, quiescent=True) + require(physical_content(before) == physical_content(recovered), "negative fixture rescue differs from original") + fleet.start() + status = run_task(probe, fleet, plan["scope"], 2, active, + option_overrides={"recursive": True}, admissions=admissions) + fleet.stop() + repaired = fleet.snapshot("physical-after-negative-recovery-task", quiescent=True) physical = validate_repaired_physical(before, repaired, plan["proofs"][0], repair_targets=plan["proofs"]) bitrot = [verify_cohort(fleet, target, "repaired-bitrot-" + branch) for target in plan["proofs"]] fleet.start() @@ -367,6 +469,10 @@ def run_topology(args, topology, nodes, fixture): repeats.append(proof) result["branches"][branch] = {"verdict": "PASS", "physical": physical, "bitrot": bitrot, "s3": s3, "responsibility": responsibility, "repeats": repeats} + if negative_proof is not None: + result["branches"][branch]["negative"] = negative_proof + result["branches"][branch]["physical"]["backupUsedAfterFailureProof"] = True + result["branches"][branch]["physical"]["productRepairClaimed"] = False except (Exception, KeyboardInterrupt) as exc: result["branches"][branch] = {"verdict": "FAIL", "failure": {"type": type(exc).__name__, "message": str(exc)}} try: @@ -416,7 +522,7 @@ def interrupted(signum, frame): and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") - fixtures = ("DB", "DV") if args.case_id == "heal.repeat-idempotency" else ("DV",) + fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum"} else ("DV",) results = [] try: for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): diff --git a/test_validation_heal.py b/test_validation_heal.py index 8d5fca75..1da740fe 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -16,7 +16,9 @@ from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal, validate_dry_run_status from scripts.validation_heal.heal_fault import mutation from scripts.validation_heal.shards import validate_conditional_response -from scripts.validation_heal.versions import BRANCHES as VERSION_BRANCHES, validate_version_receipts, branch_plan, suspend_fixture +from scripts.validation_heal.versions import (BRANCHES as VERSION_BRANCHES, validate_version_receipts, + branch_plan, suspend_fixture, validate_failed_version, validate_unrecoverable_get, verify_quorum_contract) +from scripts.validation_heal.prepare_heal_meta_inspector import quorum_contract class HealContractTest(unittest.TestCase): @@ -344,6 +346,55 @@ def test_fault_requires_actual_service_quiescence_and_exact_truncate_length(self class HealVersionOracleTest(unittest.TestCase): + def test_below_quorum_needs_exact_failure_and_never_accepts_transport_or_false_repair(self): + status, expected = self.fixture() + outcome = status["outcome"] + outcome["execution"]["state"] = "completed_with_errors" + outcome["objects"][0]["disposition"] = {"state": "failed", "details": "retry_exhausted"} + outcome["counters"].update(unchanged=3, failed=1, attemptFailures=2) + validate_failed_version(status, "fixture", expected, expected[0]) + for kind in ("completed", "unknown", "repaired", "wrong-version", "hidden-attempts", "wrong-counter"): + value = copy.deepcopy(status) + if kind == "completed": value["outcome"]["execution"]["state"] = "completed" + elif kind in {"unknown", "repaired"}: value["outcome"]["objects"][0]["disposition"]["state"] = kind + elif kind == "wrong-version": value["outcome"]["objects"][0]["identity"]["versionId"] = "different" + elif kind == "hidden-attempts": value["outcome"]["counters"]["attemptFailures"] = 0 + else: value["outcome"]["counters"]["failed"] = True + with self.subTest(kind=kind), self.assertRaises(ValueError): + validate_failed_version(value, "fixture", expected, expected[0]) + contract = {"unrecoverableGet": {"httpStatus": 503, "code": "SlowDownRead"}} + observation = {"acknowledged": False, "resultUnknown": False, + "response": {"ResponseMetadata": {"HTTPStatusCode": 503}, "Error": {"Code": "SlowDownRead"}}} + validate_unrecoverable_get(observation, contract) + for changed in ({**observation, "resultUnknown": True}, {**observation, "payload": {"size": 0}}, + {**observation, "acknowledged": True}): + with self.assertRaises(ValueError): validate_unrecoverable_get(changed, contract) + + def test_tool_freezes_recognized_candidate_quorum_and_rejects_stale_or_changed_contract(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + error = root / "rustfs/src/error.rs" + metadata = root / "crates/ecstore/src/set_disk/metadata.rs" + error.parent.mkdir(parents=True) + metadata.parent.mkdir(parents=True) + error.write_text('const SLOW_DOWN_READ_CODE: &str = "SlowDownRead";\n' + 'StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _) => ' + '{ S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into()) }\n' + 'S3ErrorCode::Custom(custom) if &**custom == SLOW_DOWN_READ_CODE => Some(StatusCode::SERVICE_UNAVAILABLE)') + metadata.write_text('pub(super) fn object_quorum_from_meta() {\n' + 'let data_blocks = parts_metadata.len() as i32 - parity_blocks;\nOk((data_blocks, write_quorum))\n}') + contract = quorum_contract(root, "a" * 40) + self.assertTrue(contract["recognized"]) + verify_quorum_contract({"quorumContract": contract}, {"source_sha": "a" * 40}) + for changed in ({**contract, "candidateCommit": "b" * 40}, + {**contract, "extractorSourceSha256": "0" * 64}, {**contract, "recognized": False}): + with self.assertRaises(ValueError): + verify_quorum_contract({"quorumContract": changed}, {"source_sha": "a" * 40}) + metadata.write_text('pub(super) fn object_quorum_from_meta() {\n' + 'let data_blocks = parts_metadata.len() as i32 - parity_blocks;\n' + '// Ok((data_blocks, write_quorum))\nOk((data_blocks + 1, write_quorum))\n}') + self.assertFalse(quorum_contract(root, "a" * 40)["recognized"]) + def test_suspended_null_fixture_retains_historical_identities_and_requires_real_marker_acks(self): historical = [{"key": f"objects/{index:04d}.bin", "versionId": f"history-{index}-{version}", "deleteMarker": False, "isLatest": version == 2} @@ -471,6 +522,12 @@ def test_version_fault_plan_uses_ack_versions_distinct_disks_and_exact_recursive self.assertEqual(len(plan["expected"]), 4) self.assertEqual(len(plan["proofs"]), 3) self.assertTrue(any(entry["deleteMarker"] for entry in plan["expected"])) + plan = branch_plan("heal.below-read-quorum", "r-minus-1-historical", snapshot, bucket, expected, "DV") + self.assertEqual(plan["readQuorum"], 2) + self.assertEqual(len(plan["retainedShards"]), 1) + self.assertEqual(len(plan["faults"]), 3) + self.assertEqual(len({entry["versionId"] for entry in plan["faults"]}), 1) + self.assertTrue(all(entry["versionId"] == "00000000-0000-0000-0000-000000000004" for entry in plan["faults"])) if __name__ == "__main__": From 6dae2ca6c3d3f589e6edf62740932cd287b129c8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 22:15:35 +0800 Subject: [PATCH 033/343] fix(validation): use registered bucket Heal route --- docs/validation-migration-map.md | 6 ++++++ scripts/validation_heal/shards.py | 4 +++- test_validation_heal.py | 21 +++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index af2ec6fa..d02084a0 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -225,3 +225,9 @@ for the exact object prefix; the original FAIL remains and a new run is required The corrected recursive DV request passed all DB/DV H1/H2 branches in run `38057908339` with clean finalization, including complete dry_run_observed version/marker receipts and byte-for-byte no-mutation proof. + +D03 run `38058420193` passed historical/current/multiple-version branches on +both H1/H2, but its bucket-recursive branch used an extra trailing slash and +received HTTP 501 before admission. Cleanup passed. The task URL now uses the +registered exact bucket route; this harness failure remains retained and D03 +needs a fresh full run before a canonical PASS claim. diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index f489cc84..b0c9164d 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -280,7 +280,9 @@ def peer_status_equal(coordinator, peer): def run_task(probe, fleet, target, mode, active, *, status_endpoints=(), option_overrides=None, admissions=None): - path = "/rustfs/admin/v3/heal/" + quote(target["bucket"], safe="") + "/" + quote(target["key"], safe="/") + path = "/rustfs/admin/v3/heal/" + quote(target["bucket"], safe="") + if target["key"]: + path += "/" + quote(target["key"], safe="/") started = fleet.record("trigger", probe.admin(path, [], json.dumps( {**BASE_OPTIONS, "scanMode": mode, **(option_overrides or {})}).encode())) require(started.get("httpStatus") == 200 and isinstance(started.get("response"), dict), "repair admission failed") diff --git a/test_validation_heal.py b/test_validation_heal.py index 1da740fe..c8046149 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -16,6 +16,7 @@ from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal, validate_dry_run_status from scripts.validation_heal.heal_fault import mutation from scripts.validation_heal.shards import validate_conditional_response +from scripts.validation_heal.shards import run_task from scripts.validation_heal.versions import (BRANCHES as VERSION_BRANCHES, validate_version_receipts, branch_plan, suspend_fixture, validate_failed_version, validate_unrecoverable_get, verify_quorum_contract) from scripts.validation_heal.prepare_heal_meta_inspector import quorum_contract @@ -175,6 +176,26 @@ def test_node_paths_cannot_select_shared_roots_or_arbitrary_run(self): class HealShardOracleTest(unittest.TestCase): + def test_bucket_and_literal_object_routes_keep_the_original_token_for_status(self): + for key, expected_path in (("", "/rustfs/admin/v3/heal/fixture"), + ("literal%2F/key", "/rustfs/admin/v3/heal/fixture/literal%252F/key")): + calls = [] + def admin(path, query, body): + calls.append((path, copy.deepcopy(query), body)) + if not query: + return {"httpStatus": 200, "response": {"clientToken": "original-token"}} + return {"httpStatus": 200, "response": HealHealthyOracleTest().receipt()} + active, admissions = [], [] + run_task(types.SimpleNamespace(admin=admin), types.SimpleNamespace(record=lambda label, value: value), + {"bucket": "fixture", "key": key}, 2, active, + option_overrides={"recursive": True}, admissions=admissions) + self.assertEqual(len(calls), 4) + self.assertTrue(all(path == expected_path for path, _, _ in calls)) + self.assertTrue(all(query == [("clientToken", "original-token")] for _, query, _ in calls[1:])) + self.assertTrue(json.loads(calls[0][2])["recursive"]) + self.assertEqual(active, []) + self.assertEqual(admissions, [{"path": expected_path, "clientToken": "original-token"}]) + def test_condition_denial_is_not_a_transport_failure_or_success_payload(self): baseline = {"properties": {"ETag": '"etag"'}, "payload": {"size": 3, "sha256": "correct"}} good = {"response": {"ResponseMetadata": {"HTTPStatusCode": 412}, "Error": {"Code": "PreconditionFailed"}}, From 015421e0956b18bbf8e94863bfb36c9a20a31eb5 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 22:20:48 +0800 Subject: [PATCH 034/343] fix(validation): bound ranges to actual fixture lengths --- docs/validation-migration-map.md | 10 ++++++++++ scripts/validation_heal/shards.py | 6 +++++- test_validation_heal.py | 20 +++++++++++++++++++- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index d02084a0..a093c792 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -231,3 +231,13 @@ both H1/H2, but its bucket-recursive branch used an extra trailing slash and received HTTP 501 before admission. Cleanup passed. The task URL now uses the registered exact bucket route; this harness failure remains retained and D03 needs a fresh full run before a canonical PASS claim. + +J04 run `38058699913` retained FAIL with clean finalization. Its DB checks hit +an executor Range bug on exact/near-1MiB controls; positive ranges now use +actual bounded intervals and have boundary regression coverage. The DV H1 +branch passed first repair and both normal/deep repeats. DV H2 physically +restored all original bytes, decoded version references and S3 manifests, but +the candidate attributed the current delete marker as unknown while three data +versions were repaired. That genuine receipt failure remains strict and visible. +Tool build `38058949922` successfully produced the current decoder plus the +recognized exact-candidate quorum/error contract for B07. diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index b0c9164d..9f923ad7 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -349,7 +349,11 @@ def verify_ranges(probe, fleet, bucket, key, *, version_id="null", expected_body f"{bucket}/{key}/0", SIZES[int(key.rsplit("/", 1)[1].split(".")[0]) % len(SIZES)]) boundaries = (1024**2, len(body) // 2) proofs = [] - for start, end in [(point - 17, point + 29) for point in boundaries] + [(len(body) - 47, len(body) - 1)]: + require(len(body) > 0, "positive ranges require a nonempty fixture") + intervals = [(max(0, point - 17), min(len(body) - 1, point + 29)) + for point in boundaries if 0 < point < len(body)] + intervals.append((max(0, len(body) - 47), len(body) - 1)) + for start, end in intervals: require(0 <= start <= end < len(body), "invalid fixture range") observed = fleet.record("s3-range", probe.s3("get_object", {"Bucket": bucket, "Key": key, "VersionId": version_id, "Range": f"bytes={start}-{end}"})) diff --git a/test_validation_heal.py b/test_validation_heal.py index c8046149..e0c6a2d1 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -16,7 +16,7 @@ from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal, validate_dry_run_status from scripts.validation_heal.heal_fault import mutation from scripts.validation_heal.shards import validate_conditional_response -from scripts.validation_heal.shards import run_task +from scripts.validation_heal.shards import run_task, verify_ranges from scripts.validation_heal.versions import (BRANCHES as VERSION_BRANCHES, validate_version_receipts, branch_plan, suspend_fixture, validate_failed_version, validate_unrecoverable_get, verify_quorum_contract) from scripts.validation_heal.prepare_heal_meta_inspector import quorum_contract @@ -176,6 +176,24 @@ def test_node_paths_cannot_select_shared_roots_or_arbitrary_run(self): class HealShardOracleTest(unittest.TestCase): + def test_ranges_at_one_megabyte_boundary_stay_inside_the_real_payload(self): + for size in (1024**2 - 1, 1024**2, 1024**2 + 1): + body = b"x" * size + requested = [] + def s3(operation, params): + start, end = map(int, params["Range"].removeprefix("bytes=").split("-")) + self.assertTrue(0 <= start <= end < size) + requested.append((start, end)) + value = body[start:end + 1] + return {"acknowledged": True, "response": {"ResponseMetadata": {"HTTPStatusCode": 206}, + "ContentRange": f"bytes {start}-{end}/{size}"}, + "payload": {"size": len(value), "sha256": hashlib.sha256(value).hexdigest()}} + verify_ranges(types.SimpleNamespace(s3=s3), types.SimpleNamespace(record=lambda label, value: value), + "fixture", "objects/0000.bin", expected_body=body) + self.assertEqual(requested[-1][1], size - 1) + if size > 1024**2: + self.assertTrue(any(start < 1024**2 <= end for start, end in requested)) + def test_bucket_and_literal_object_routes_keep_the_original_token_for_status(self): for key, expected_path in (("", "/rustfs/admin/v3/heal/fixture"), ("literal%2F/key", "/rustfs/admin/v3/heal/fixture/literal%252F/key")): From 2022619c36bfdf92f7cafc7b07c29fa7bb4df0d7 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 22:33:45 +0800 Subject: [PATCH 035/343] test(validation): migrate metadata and inline Heal faults --- docs/validation-migration-map.md | 22 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/check_bitrot_tool.py | 28 ++++ scripts/validation_heal/fleet_node.py | 40 ++++- scripts/validation_heal/healthy.py | 7 +- scripts/validation_heal/matrix.py | 3 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/shards.py | 4 +- .../tools/heal_meta_inspect.rs | 150 ++++++++++++++---- scripts/validation_heal/versions.py | 96 +++++++++-- test_validation_heal.py | 51 +++++- 11 files changed, 345 insertions(+), 60 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index a093c792..91b25830 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 90 currently dispatchable IDs include all 74 non-Heal cases and sixteen Heal cases. Thirty-eight have live PASS evidence; the other 120 have no live PASS claim. +case evidence. The 91 currently dispatchable IDs include all 74 non-Heal cases and seventeen Heal cases. Thirty-eight have live PASS evidence; the other 120 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -177,7 +177,19 @@ metadata must remain unchanged. Only after that failure proof may backups be restored and a new token verify recovered integrity; recovery never proves product repair during the negative window. This executor still requires live evidence and a fresh tool build containing the quorum contract. -The other 68 Heal cases remain +C05 `heal.missing-or-corrupt-xlmeta` also has all five automatic branches on +H1/H2: missing/truncated/header-corrupt xl.meta, a flip inside the unique decoded +inline value, and whole metadata loss with three historical data versions and a +marker. The tool now freezes each inline value's physical offset/length/SHA-256 +and verifies its streaming HighwayHash framing. Complete original inline shard +digests, version/dataDir/part/role references and all S3 versions must be restored. +A decoder error can be captured only when its file length/hash exactly matches +an injected fault receipt; it never becomes valid metadata or a repair proof. +Legal metadata re-encoding may move an inline value while its exact length/hash +stay fixed. The extended helper passed the 24/12 external shard oracle and one +frozen released-process inline fixture with three corruption negatives locally; +this executor still needs a new candidate-bound tool build and live evidence. +The other 67 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -241,3 +253,9 @@ the candidate attributed the current delete marker as unknown while three data versions were repaired. That genuine receipt failure remains strict and visible. Tool build `38058949922` successfully produced the current decoder plus the recognized exact-candidate quorum/error contract for B07. + +The corrected D03 run `38059278988` admitted every source branch, passed H1 +completely and passed H2 historical/current/recursive branches. H2's simultaneous +multiple-version fault left one original current-version parity part missing. +The exact physical oracle retained FAIL and clean backup-assisted finalization; +that restoration does not count as product repair. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index d5367de6..49197d58 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/check_bitrot_tool.py b/scripts/validation_heal/check_bitrot_tool.py index bbb6ca97..b68ab41a 100644 --- a/scripts/validation_heal/check_bitrot_tool.py +++ b/scripts/validation_heal/check_bitrot_tool.py @@ -48,7 +48,35 @@ def check(binary: Path, source: Path): rejected += 1 if verified != 24 or rejected != 12: raise ValueError("external bitrot corpus coverage is incomplete") + captured = bytes.fromhex((source / "crates/filemeta/tests/fixtures/issue_2288_legacy_xlmeta.hex").read_text()) + if hashlib.sha256(captured).hexdigest() != "5997675004972399e2d279a83b725c360a12f117edbf9dffc8c02d76c8753452": + raise ValueError("captured inline fixture differs from its frozen released-process bytes") + with tempfile.TemporaryDirectory(prefix="heal-inline-oracle-") as directory: + metadata = Path(directory) / "xl.meta" + metadata.write_bytes(captured) + decoded = json.loads(subprocess.check_output([str(binary), str(metadata), "fixture", "object"], text=True)) + inline_versions = [version for version in decoded["versions"] if version.get("inline")] + if len(inline_versions) != 1 or inline_versions[0]["inline"]["rawOccurrenceCount"] != 1: + raise ValueError("captured inline identity is ambiguous") + version = inline_versions[0] + inline = version["inline"] + value = captured[inline["rawOffset"]:inline["rawOffset"] + inline["length"]] + if hashlib.sha256(value).hexdigest() != inline["sha256"]: + raise ValueError("decoded inline value is not its exact physical slice") + args = [str(binary), "--verify-inline", str(metadata), "fixture", "object", version["versionId"] or "null"] + good = json.loads(subprocess.check_output(args, text=True)) + if good.get("physicalBitrotVerified") is not True or good["inlineSha256"] != inline["sha256"]: + raise ValueError("captured inline framing did not verify") + inline_rejected = 0 + for position in (0, inline["length"] // 2, inline["length"] - 1): + offset = inline["rawOffset"] + position + metadata.write_bytes(captured[:offset] + bytes([captured[offset] ^ 1]) + captured[offset + 1:]) + bad = subprocess.run(args, capture_output=True, text=True) + if bad.returncode == 0 or "hash mismatch" not in bad.stderr: + raise ValueError("inline corruption was not rejected by the independent framing checker") + inline_rejected += 1 return {"externalHealthyShardsVerified": verified, "corruptedShardsRejected": rejected, + "releasedInlineVerified": 1, "corruptedInlineValuesRejected": inline_rejected, "corpora": ["MinIO released GF8", "RustFS released legacy GF16"], "liveCasePassClaimed": False} diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index b238ff81..f52008a4 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -265,6 +265,22 @@ def verify_part(request): return {**result, "sha256": digest(part), "relativePath": relative, "disk": request["disk"]} +def verify_inline(request): + state, _, owner = owned(request["runId"]) + root, _ = fault_root(request) + module = fault_module() + parts = module.target_parts(request["relativePath"], owner["bucket"]) + require(parts[-1] == "xl.meta", "inline verification requires the exact object metadata") + path = checked(root / "data" / request["relativePath"], root / "data") + key = "/".join(parts[1:-1]) + tool = checked(state / "heal-meta-inspect", state) + require(digest(tool) == owner.get("inspectorSha256"), "inline verifier changed") + result = json.loads(command(str(tool), "--verify-inline", str(path), owner["bucket"], key, request["versionId"]).stdout) + require(result.get("physicalBitrotVerified") is True and result.get("key") == key + and result.get("versionId") == request["versionId"], "inline proof identity differs") + return {**result, "relativePath": request["relativePath"], "disk": request["disk"]} + + def install_inspector(request): state, _, owner = owned(request["runId"]) tool = checked(state / "heal-meta-inspect", state) @@ -273,6 +289,21 @@ def install_inspector(request): return {"inspectorSha256": owner["inspectorSha256"], "installed": True} +def inspect_metadata(tool, path, bucket, key, entry, metadata_faults): + observed = command(str(tool), str(path), bucket, key, check=False) + if observed.returncode: + expected_fault = metadata_faults.get(entry["path"]) + require(expected_fault == {name: entry[name] for name in ("sha256", "size")}, + "metadata decoder failed outside an exact witnessed file fault") + return {"metadataError": {"code": observed.returncode, "message": observed.stderr[-500:], + "exactWitnessedFault": True, "validMetadataClaimed": False}} + decoded = json.loads(observed.stdout) + require(decoded.get("bucket") == bucket and decoded.get("key") == key + and decoded.get("physicalBitrotVerified") is False, + "metadata decoder identity or verification role differs") + return {"metadata": decoded} + + def snapshot(request): state, roots, owner = owned(request["runId"]) pid = int(command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"], check=False).stdout.strip() or "0") @@ -306,11 +337,7 @@ def snapshot(request): "size": path.stat().st_size, "sha256": digest(path)} if path.name == "xl.meta" and owner.get("inspectorSha256"): key = path.parent.relative_to(folder).as_posix() - decoded = json.loads(command(str(tool), str(path), owner["bucket"], key).stdout) - require(decoded.get("bucket") == owner["bucket"] and decoded.get("key") == key - and decoded.get("physicalBitrotVerified") is False, - "metadata decoder identity or verification role differs") - entry["metadata"] = decoded + entry.update(inspect_metadata(tool, path, owner["bucket"], key, entry, request.get("metadataFaults", {}))) files.append(entry) path = checked(data / ".rustfs.sys/format.json", data) require(path.stat().st_size < 65536, "disk format exceeds budget") @@ -345,6 +372,7 @@ def main(request): require(os.geteuid() == 0, "node adapter requires passwordless sudo") actions = {"preflight": preflight, "reserve": reserve, "start": start, "stop": stop, "snapshot": snapshot, "logs": logs, "cleanup": cleanup, "install-inspector": install_inspector, - "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part} + "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part, + "verify-inline": verify_inline} require(request.get("action") in actions, "unsupported node operation") print(json.dumps(actions[request["action"]](request))) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index a63a9e2b..ab801a23 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -150,7 +150,7 @@ def ssh(self, node): def remote(self, node, action, **params): value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() - if action in {"apply-fault", "recover-fault", "verify-part"}: + if action in {"apply-fault", "recover-fault", "verify-part", "verify-inline"}: source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" source += "\nmain(" + repr(value) + ")\n" @@ -159,8 +159,9 @@ def remote(self, node, action, **params): require(result.returncode == 0, f"Heal node {node} {action} failed: {result.stderr[-1200:]}") return json.loads(result.stdout) - def snapshot(self, label, *, quiescent=False): - return self.record(label, {node: self.remote(node, "snapshot", quiescent=quiescent) for node in self.nodes}) + def snapshot(self, label, *, quiescent=False, metadata_faults=None): + return self.record(label, {node: self.remote(node, "snapshot", quiescent=quiescent, + metadataFaults=(metadata_faults or {}).get(node, {})) for node in self.nodes}) def stop(self): with ThreadPoolExecutor(max_workers=len(self.nodes)) as executor: diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index e8a80186..99ca5eea 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -24,7 +24,8 @@ AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", - "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum"} + "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", + "heal.missing-or-corrupt-xlmeta"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 09c90e90..e2391977 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -19,7 +19,7 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request) ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index 9f923ad7..dbbb02e8 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -208,7 +208,9 @@ def validate_repaired_physical(before, after, target, *, repair_targets=None): else: def semantics(metadata): return {"bucket": metadata["bucket"], "key": metadata["key"], "versions": [ - {**version, "erasure": {name: value for name, value in version["erasure"].items() if name != "checksums"}} + {**version, "inline": {name: version["inline"].get(name) for name in ("length", "sha256")} + if version.get("inline") else None, + "erasure": {name: value for name, value in version["erasure"].items() if name != "checksums"}} for version in metadata["versions"]]} require(actual.get("metadata") and semantics(actual["metadata"]) == semantics(entry["metadata"]), "repair changed decoded version/part identity") diff --git a/scripts/validation_heal/tools/heal_meta_inspect.rs b/scripts/validation_heal/tools/heal_meta_inspect.rs index 7151eb67..5fada4c4 100644 --- a/scripts/validation_heal/tools/heal_meta_inspect.rs +++ b/scripts/validation_heal/tools/heal_meta_inspect.rs @@ -12,6 +12,113 @@ fn bounded_file(path: &str) -> Result, Box> { Ok(std::fs::read(path)?) } +fn sha256(bytes: &[u8]) -> String { + HashAlgorithm::SHA256 + .hash_encode(bytes) + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn verify_framing( + bytes: &[u8], + logical_size: usize, + block_size: usize, + data_blocks: usize, + legacy: bool, + algorithm: &HashAlgorithm, +) -> Result> { + if logical_size == 0 || block_size == 0 || data_blocks == 0 { + return Err("positive bitrot verification requires supported nonempty geometry".into()); + } + if !matches!( + algorithm, + HashAlgorithm::HighwayHash256S | HashAlgorithm::HighwayHash256SLegacy + ) { + return Err("unsupported bitrot framing; no integrity claim".into()); + } + let mut remaining = logical_size; + let mut offset: usize = 0; + let mut blocks: usize = 0; + while remaining > 0 { + let logical = remaining.min(block_size); + let length = logical.div_ceil(data_blocks); + let length = if legacy { (length + 1) & !1 } else { length }; + let end = offset + .checked_add(32 + length) + .ok_or("part size overflow")?; + let block = bytes.get(offset..end).ok_or("short shard read")?; + if algorithm.hash_encode(&block[32..]).as_ref() != &block[..32] { + return Err(format!("bitrot hash mismatch in block {blocks}").into()); + } + offset = end; + remaining -= logical; + blocks += 1; + } + if offset != bytes.len() { + return Err("unexpected trailing shard bytes".into()); + } + Ok(blocks) +} + +fn verify_inline(args: &[String]) -> Result<(), Box> { + if args.len() != 6 { + return Err("usage: --verify-inline XL_META BUCKET KEY VERSION_ID".into()); + } + let bytes = bounded_file(&args[2])?; + let meta = FileMeta::load(&bytes)?; + meta.data.validate()?; + let decoded = meta.into_file_info_versions(&args[3], &args[4], true)?; + let matches: Vec<_> = decoded + .versions + .iter() + .filter(|info| { + info.version_id + .filter(|id| !id.is_nil()) + .map(|id| id.to_string()) + .unwrap_or_else(|| "null".to_owned()) + == args[5] + }) + .collect(); + if matches.len() != 1 { + return Err("inline verification requires one exact metadata version".into()); + } + let info = matches[0]; + if info.deleted + || !info.inline_data() + || info.parts.len() != 1 + || info.parts[0].size == 0 + || usize::try_from(info.parts[0].actual_size).ok() != Some(info.parts[0].size) + || usize::try_from(info.size).ok() != Some(info.parts[0].size) + { + return Err("unsupported inline version or part identity".into()); + } + let data = meta.data.find(&args[5])?.ok_or("inline value is missing")?; + let algorithm = if info.uses_legacy_checksum { + HashAlgorithm::HighwayHash256SLegacy + } else { + info.erasure + .get_checksum_info(info.parts[0].number) + .algorithm + }; + let blocks = verify_framing( + &data, + usize::try_from(info.parts[0].size)?, + info.erasure.block_size, + info.erasure.data_blocks, + info.uses_legacy_checksum, + &algorithm, + )?; + println!( + "{}", + json!({"schemaVersion": 1, "bucket": args[3], "key": args[4], + "versionId": args[5], "physicalBitrotVerified": true, "ecReconstructionVerified": false, + "algorithm": algorithm, "blocksVerified": blocks, "inlineLength": data.len(), "inlineSha256": sha256(&data)}) + ); + Ok(()) +} + // This path checks stored framing without invoking the service or repairing // anything. EC reconstruction is checked separately against the frozen shard. fn verify_part(args: &[String]) -> Result<(), Box> { @@ -68,38 +175,15 @@ fn verify_part(args: &[String]) -> Result<(), Box> { } else { info.erasure.get_checksum_info(number).algorithm }; - if !matches!( - algorithm, - HashAlgorithm::HighwayHash256S | HashAlgorithm::HighwayHash256SLegacy - ) { - return Err("unsupported bitrot framing; no integrity claim".into()); - } let bytes = bounded_file(&args[5])?; - let mut remaining = usize::try_from(part.size)?; - let mut offset: usize = 0; - let mut blocks: usize = 0; - while remaining > 0 { - let logical = remaining.min(info.erasure.block_size); - let length = logical.div_ceil(info.erasure.data_blocks); - let length = if info.uses_legacy_checksum { - (length + 1) & !1 - } else { - length - }; - let end = offset - .checked_add(32 + length) - .ok_or("part size overflow")?; - let block = bytes.get(offset..end).ok_or("short shard read")?; - if algorithm.hash_encode(&block[32..]).as_ref() != &block[..32] { - return Err(format!("bitrot hash mismatch in block {blocks}").into()); - } - offset = end; - remaining -= logical; - blocks += 1; - } - if offset != bytes.len() { - return Err("unexpected trailing shard bytes".into()); - } + let blocks = verify_framing( + &bytes, + usize::try_from(part.size)?, + info.erasure.block_size, + info.erasure.data_blocks, + info.uses_legacy_checksum, + &algorithm, + )?; println!( "{}", json!({"schemaVersion": 1, "bucket": args[3], "key": args[4], @@ -115,6 +199,9 @@ fn main() -> Result<(), Box> { if args.get(1).is_some_and(|value| value == "--verify-part") { return verify_part(&args); } + if args.get(1).is_some_and(|value| value == "--verify-inline") { + return verify_inline(&args); + } if args.len() != 4 { return Err("usage: heal-meta-inspect XL_META_FILE BUCKET KEY".into()); } @@ -167,6 +254,7 @@ fn main() -> Result<(), Box> { .collect(); json!({ "length": data.len(), + "sha256": sha256(data), "rawOccurrenceCount": offsets.len(), "rawOffset": (offsets.len() == 1).then_some(offsets[0]) }) diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 1172d499..bdbbbd0c 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -39,6 +39,7 @@ "heal.repeat-idempotency": ("normal-repeat", "deep-repeat", "repeat-after-restart", "versioned-repeat"), "heal.versioning-suspended-null": ("enabled-to-suspended", "null-overwrite", "null-delete-marker"), "heal.below-read-quorum": ("r-minus-1-unversioned", "r-minus-1-historical"), + "heal.missing-or-corrupt-xlmeta": ("missing-xlmeta", "truncated-xlmeta", "corrupt-xlmeta", "inline-payload", "versioned-xlmeta"), } VERSION_SIZE = 5 * 1024**2 + 123 @@ -171,6 +172,59 @@ def metadata_target(snapshot, part_target): return {**part_target, "relativePath": relative, "sha256": entry["sha256"], "size": entry["size"]} +def map_inline_target(snapshot, bucket, entry): + validate_topology(snapshot, "H1" if len(snapshot) == 1 else "H2") + indexed, cohort, authority = files(snapshot), [], None + path = f"{bucket}/{entry['key']}/xl.meta" + for node, observation in snapshot.items(): + for physical in observation["formats"]: + file = indexed.get((node, physical["disk"], path), {}) + metadata = file.get("metadata", {}) + versions = [version for version in metadata.get("versions", []) + if (version.get("versionId") or "null") == entry["versionId"]] + require(metadata.get("bucket") == bucket and metadata.get("key") == entry["key"] and len(versions) == 1, + "inline baseline lacks an exact full-set metadata authority") + version = versions[0] + inline, erasure = version.get("inline") or {}, version["erasure"] + layout = physical["value"]["xl"] + slot = layout["sets"][0].index(layout["this"]) + require(version.get("deleted") is False and version.get("size") == entry["payload"]["size"] > 0 + and inline.get("rawOccurrenceCount") == 1 and type(inline.get("rawOffset")) is int + and type(inline.get("length")) is int and inline["length"] > 32 + and 0 <= inline["rawOffset"] < inline["rawOffset"] + inline["length"] <= file["size"] + and re.fullmatch(r"[0-9a-f]{64}", inline.get("sha256", "")) + and erasure["index"] == erasure["distribution"][slot], "inline location, digest or physical role is unsupported") + role = "data" if erasure["index"] <= erasure["data_blocks"] else "parity" + require(version.get("role") == role, "inline erasure index and role differ") + logical = {name: version.get(name) for name in ("versionId", "isLatest", "deleted", "dataDir", "size", "parts", "usesLegacyChecksum")} + logical["erasure"] = {name: erasure[name] for name in ("data_blocks", "parity_blocks", "block_size", "distribution", "algorithm")} + require(authority is None or logical == authority, "inline version or part geometry lacks full-set authority") + authority = logical + cohort.append({"node": node, "disk": physical["disk"], "relativePath": path, + "versionId": entry["versionId"], "partNumber": 1, "dataDir": version.get("dataDir"), + "diskUUID": layout["this"], "setSlot": slot, "role": role, "index": erasure["index"], + "sha256": file["sha256"], "size": file["size"], "inlineValue": inline}) + require(len(cohort) == authority["erasure"]["data_blocks"] + authority["erasure"]["parity_blocks"] + and len({item["index"] for item in cohort}) == len(cohort), "inline set is incomplete or duplicated") + target = next(item for item in cohort if item["role"] == "data") + return {**target, "bucket": bucket, "key": entry["key"], "poolIndex": 0, "setIndex": 0, + "inlineProof": True, "cohort": cohort, "logical": authority} + + +def verify_proof(fleet, target, label): + if not target.get("inlineProof"): + return verify_cohort(fleet, target, label) + proofs = [] + for entry in target["cohort"]: + proof = fleet.record(label, {"node": entry["node"], **fleet.remote(entry["node"], "verify-inline", + disk=entry["disk"], relativePath=entry["relativePath"], versionId=entry["versionId"])}) + require(proof.get("physicalBitrotVerified") is True and proof.get("inlineLength") == entry["inlineValue"]["length"] + and proof.get("inlineSha256") == entry["inlineValue"]["sha256"], + "recovered inline shard differs from frozen same-role bytes") + proofs.append(proof) + return {"allInlineShardsBitrotVerified": True, "originalInlineSha256Restored": True, "shards": proofs} + + def suspend_fixture(recorder, bucket, historical): expected = copy.deepcopy(historical) recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Suspended"}}) @@ -236,8 +290,25 @@ def branch_plan(case_id, branch, snapshot, bucket, expected, fixture): key = f"objects/{branch_index:04d}.bin" if fixture == "DV" else f"objects/{8 + 9 * branch_index:04d}.bin" if branch == "versioned-repeat": key = "objects/0000.bin" + if case_id == "heal.missing-or-corrupt-xlmeta": + key = "objects/0000.bin" if fixture == "DV" else f"objects/{branch_index + 1:04d}.bin" selected = [entry for entry in expected if entry["key"] == key] data = [entry for entry in selected if not entry["deleteMarker"]] + if case_id == "heal.missing-or-corrupt-xlmeta": + if fixture == "DB": + target = map_inline_target(snapshot, bucket, data[0]) + faults, proofs = [target], [target] + else: + target = choose_target(snapshot, bucket, data[0], "data") + faults = [metadata_target(snapshot, target)] + proofs = [choose_target(snapshot, bucket, entry, "data") for entry in data] + proofs = [{**item, **next(shard for shard in item["cohort"] + if (shard["node"], shard["disk"]) == (target["node"], target["disk"]))} for item in proofs] + kind = {"truncated-xlmeta": "truncate", "corrupt-xlmeta": "header", "inline-payload": "bitflip"}.get(branch, "missing") + offset = target["inlineValue"]["rawOffset"] + 32 if branch == "inline-payload" else None + return {"scope": {"bucket": bucket, "key": key}, "recursive": fixture == "DV", "expected": selected, + "faults": faults, "proofs": proofs, "requiredRepairs": {version_identity(data[0])} if branch == "inline-payload" else set(), + "metadataRepairs": {key}, "kind": kind, "offset": offset} if case_id == "heal.below-read-quorum": target = choose_target(snapshot, bucket, data[0], "data") read_quorum = target["logical"]["erasure"]["data_blocks"] @@ -347,7 +418,8 @@ def run_topology(args, topology, nodes, fixture): require(list(BRANCHES[args.case_id]) == recipe["variants"], "executor omitted a source version branch") branches = [branch for branch in BRANCHES[args.case_id] if (args.case_id != "heal.repeat-idempotency" or (branch == "versioned-repeat") == (fixture == "DV")) - and (args.case_id != "heal.below-read-quorum" or (branch == "r-minus-1-historical") == (fixture == "DV"))] + and (args.case_id != "heal.below-read-quorum" or (branch == "r-minus-1-historical") == (fixture == "DV")) + and (args.case_id != "heal.missing-or-corrupt-xlmeta" or (branch == "versioned-xlmeta") == (fixture == "DV"))] result = {"topology": topology, "fixture": fixture, "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} active, admissions = [], [] probe = Probe(binding) @@ -363,6 +435,7 @@ def run_topology(args, topology, nodes, fixture): fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) for branch in branches: registrations = [] + metadata_faults = {} try: fleet.stop() before = fleet.snapshot("physical-before-" + branch, quiescent=True) @@ -370,7 +443,7 @@ def run_topology(args, topology, nodes, fixture): fleet.record("fault-plan-" + branch, {**plan, "requiredRepairs": sorted(plan["requiredRepairs"]), "metadataRepairs": sorted(plan["metadataRepairs"])}) for target in plan["proofs"]: - verify_cohort(fleet, target, "baseline-bitrot-" + branch) + verify_proof(fleet, target, "baseline-bitrot-" + branch) prior = before for index, target in enumerate(plan["faults"]): params = {"disk": target["disk"], "faultId": recipe["code"] + "-" + branch + "-" + str(index)} @@ -378,12 +451,15 @@ def run_topology(args, topology, nodes, fixture): fleet.faults.append(registration) registrations.append(registration) receipt = fleet.record("fault-" + branch, fleet.remote(target["node"], "apply-fault", - **params, relativePath=target["relativePath"], expectedSha256=target["sha256"], kind="missing")) - witnessed = fleet.snapshot("physical-fault-" + branch, quiescent=True) + **params, relativePath=target["relativePath"], expectedSha256=target["sha256"], + kind=plan.get("kind", "missing"), offset=plan.get("offset"))) + if target["relativePath"].endswith("/xl.meta") and receipt.get("fault"): + metadata_faults.setdefault(target["node"], {})[target["relativePath"]] = receipt["fault"] + witnessed = fleet.snapshot("physical-fault-" + branch, quiescent=True, metadata_faults=metadata_faults) validate_fault(prior, witnessed, target, receipt) prior = witnessed fleet.start() - witness = fleet.snapshot("physical-before-trigger-" + branch) + witness = fleet.snapshot("physical-before-trigger-" + branch, metadata_faults=metadata_faults) require(physical_content(prior) == physical_content(witness), "ATTRIBUTION_NOT_OBSERVED: version fault changed before the manual token") if plan.get("negative"): @@ -398,7 +474,7 @@ def run_topology(args, topology, nodes, fixture): status = run_task(probe, fleet, plan["scope"], 2, active, option_overrides={"recursive": plan["recursive"]}, admissions=admissions) fleet.stop() - repaired = fleet.snapshot("physical-after-" + branch, quiescent=True) + repaired = fleet.snapshot("physical-after-" + branch, quiescent=True, metadata_faults=metadata_faults) negative_proof = None if plan.get("negative"): require(physical_content(prior) == physical_content(repaired), @@ -418,7 +494,7 @@ def run_topology(args, topology, nodes, fixture): fleet.stop() repaired = fleet.snapshot("physical-after-negative-recovery-task", quiescent=True) physical = validate_repaired_physical(before, repaired, plan["proofs"][0], repair_targets=plan["proofs"]) - bitrot = [verify_cohort(fleet, target, "repaired-bitrot-" + branch) for target in plan["proofs"]] + bitrot = [verify_proof(fleet, target, "repaired-bitrot-" + branch) for target in plan["proofs"]] fleet.start() s3 = verify_version_s3(probe, fleet, bucket, expected, baseline, "s3-after-" + branch, fixture=fixture) # These faults have now been independently repaired and fully @@ -463,7 +539,7 @@ def run_topology(args, topology, nodes, fixture): after_repeat = fleet.snapshot("physical-repeat-" + branch, quiescent=True) validate_repaired_physical(post_heal, after_repeat, plan["proofs"][0], repair_targets=plan["proofs"]) for target in plan["proofs"]: - verify_cohort(fleet, target, "repeat-bitrot-" + branch) + verify_proof(fleet, target, "repeat-bitrot-" + branch) fleet.start() proof["s3"] = verify_version_s3(probe, fleet, bucket, expected, baseline, "s3-repeat-" + branch, fixture=fixture) repeats.append(proof) @@ -477,7 +553,7 @@ def run_topology(args, topology, nodes, fixture): result["branches"][branch] = {"verdict": "FAIL", "failure": {"type": type(exc).__name__, "message": str(exc)}} try: fleet.stop() - fleet.snapshot("physical-failure-" + branch, quiescent=True) + fleet.snapshot("physical-failure-" + branch, quiescent=True, metadata_faults=metadata_faults) for registration in reversed(registrations): node, params = registration fleet.record("rescue-" + branch, fleet.remote(node, "recover-fault", **params)) @@ -522,7 +598,7 @@ def interrupted(signum, frame): and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") - fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum"} else ("DV",) + fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta"} else ("DV",) results = [] try: for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): diff --git a/test_validation_heal.py b/test_validation_heal.py index e0c6a2d1..6783d19d 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -11,14 +11,15 @@ from scripts.validation_heal.matrix import STAGES, TOPOLOGIES, audit_evidence, cases, compile_plan, select from scripts.validation_heal.healthy import validate_healthy, validate_topology, validate_rejection, Fleet -from scripts.validation_heal.fleet_node import paths, stopped +from scripts.validation_heal.fleet_node import paths, stopped, inspect_metadata from scripts.validation_heal.inspector import verify as verify_inspector from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal, validate_dry_run_status from scripts.validation_heal.heal_fault import mutation from scripts.validation_heal.shards import validate_conditional_response from scripts.validation_heal.shards import run_task, verify_ranges from scripts.validation_heal.versions import (BRANCHES as VERSION_BRANCHES, validate_version_receipts, - branch_plan, suspend_fixture, validate_failed_version, validate_unrecoverable_get, verify_quorum_contract) + branch_plan, suspend_fixture, validate_failed_version, validate_unrecoverable_get, verify_quorum_contract, + map_inline_target, verify_proof) from scripts.validation_heal.prepare_heal_meta_inspector import quorum_contract @@ -235,9 +236,9 @@ def invoke(args, **params): return types.SimpleNamespace(returncode=0, stdout='{"checked":true}', stderr="") with tempfile.TemporaryDirectory() as directory, patch("scripts.validation_heal.healthy.subprocess.run", side_effect=invoke): fleet = Fleet(["node1"], Path(directory), "gh-12345-1-h1") - for action in ("preflight", "verify-part", "apply-fault", "recover-fault"): + for action in ("preflight", "verify-part", "verify-inline", "apply-fault", "recover-fault"): self.assertTrue(fleet.remote("node1", action, disk=1, relativePath="literal$()'part.1")["checked"]) - self.assertEqual(len(observed), 4) + self.assertEqual(len(observed), 5) def snapshot(self): bucket, key, data_dir = "rv-heal-0123456789abcdef", "objects/0008.bin", "12345678-1234-1234-1234-123456789abc" @@ -385,6 +386,48 @@ def test_fault_requires_actual_service_quiescence_and_exact_truncate_length(self class HealVersionOracleTest(unittest.TestCase): + def test_decoder_failure_is_evidence_only_for_the_exact_witnessed_corruption(self): + entry = {"path": "fixture/key/xl.meta", "sha256": "changed-sha", "size": 9} + fault = {entry["path"]: {"sha256": "changed-sha", "size": 9}} + rejected = types.SimpleNamespace(returncode=1, stderr="invalid captured metadata", stdout="") + with patch("scripts.validation_heal.fleet_node.command", return_value=rejected): + observed = inspect_metadata("tool", "path", "fixture", "key", entry, fault) + self.assertFalse(observed["metadataError"]["validMetadataClaimed"]) + self.assertNotIn("metadata", observed) + for incorrect in ({}, {entry["path"]: {"sha256": "different", "size": 9}}, + {entry["path"]: {"sha256": "changed-sha", "size": 10}}): + with self.assertRaises(ValueError): inspect_metadata("tool", "path", "fixture", "key", entry, incorrect) + decoded = {"bucket": "fixture", "key": "another", "physicalBitrotVerified": False} + with patch("scripts.validation_heal.fleet_node.command", return_value=types.SimpleNamespace( + returncode=0, stdout=json.dumps(decoded))): + with self.assertRaises(ValueError): inspect_metadata("tool", "path", "fixture", "key", entry, fault) + + def test_inline_fault_uses_unique_candidate_decoded_slice_and_frozen_role_digest(self): + original = HealShardOracleTest().snapshot() + bucket, key = "rv-heal-0123456789abcdef", "objects/0008.bin" + for entry in original["node1"]["files"]: + if "metadata" in entry: + entry["metadata"]["versions"][0]["inline"] = {"length": 82, "rawOffset": 200, + "rawOccurrenceCount": 1, "sha256": str(entry["disk"]) * 64} + expected = {"key": key, "versionId": "null", "payload": {"size": 100}} + target = map_inline_target(original, bucket, expected) + self.assertEqual(target["disk"], 1) + def remote(node, action, **params): + entry = next(entry for entry in target["cohort"] if entry["disk"] == params["disk"]) + return {"physicalBitrotVerified": True, "inlineLength": entry["inlineValue"]["length"], "inlineSha256": entry["inlineValue"]["sha256"]} + fleet = types.SimpleNamespace(record=lambda label, value: value, remote=remote) + self.assertTrue(verify_proof(fleet, target, "inline")['originalInlineSha256Restored']) + with patch.object(fleet, "remote", return_value={"inlineLength": 82, "inlineSha256": "wrong"}): + with self.assertRaises(ValueError): verify_proof(fleet, target, "inline") + for kind in ("ambiguous", "outside", "wrong-role", "wrong-digest"): + bad = copy.deepcopy(original) + value = next(entry["metadata"]["versions"][0] for entry in bad["node1"]["files"] if "metadata" in entry) + if kind == "ambiguous": value["inline"]["rawOccurrenceCount"] = 2 + elif kind == "outside": value["inline"]["rawOffset"] = 590 + elif kind == "wrong-role": value["role"] = "data" + else: value["inline"]["sha256"] = "invalid" + with self.subTest(kind=kind), self.assertRaises(ValueError): map_inline_target(bad, bucket, expected) + def test_below_quorum_needs_exact_failure_and_never_accepts_transport_or_false_repair(self): status, expected = self.fixture() outcome = status["outcome"] From f9c153f7b8104951054827934ea24f3cd9aae5c9 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 22:40:24 +0800 Subject: [PATCH 036/343] test(validation): migrate public metadata and versioned-tag Heal checks --- docs/validation-migration-map.md | 20 +++++++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 72 +++++++++++++++++++++++++++-- test_validation_heal.py | 27 +++++++++++ 6 files changed, 117 insertions(+), 8 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 91b25830..64eb2551 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 91 currently dispatchable IDs include all 74 non-Heal cases and seventeen Heal cases. Thirty-eight have live PASS evidence; the other 120 have no live PASS claim. +case evidence. The 92 currently dispatchable IDs include all 74 non-Heal cases and eighteen Heal cases. Thirty-nine have live PASS evidence; the other 119 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -189,7 +189,15 @@ Legal metadata re-encoding may move an inline value while its exact length/hash stay fixed. The extended helper passed the 24/12 external shard oracle and one frozen released-process inline fixture with three corruption negatives locally; this executor still needs a new candidate-bound tool build and live evidence. -The other 67 Heal cases remain +D06 `heal.object-metadata-and-tags` preserves all three H1/H2 branches with +three historical payloads per Unicode/space/literal-percent key, actual gzip +content, encoded Chinese user metadata and two distinct tag groups. It checks +the initial normalized metadata/Content-Type/Encoding/Disposition/Cache-Control +and precise versioned tags, then faults an external data part, an entire xl.meta +header, or a parity part. Exact physical/version references, complete S3 +inventories, HEAD and ranges must agree after repair. This executor still +requires live evidence. +The other 66 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -259,3 +267,11 @@ completely and passed H2 historical/current/recursive branches. H2's simultaneou multiple-version fault left one original current-version parity part missing. The exact physical oracle retained FAIL and clean backup-assisted finalization; that restoration does not count as product repair. + +Fresh corrected J04 run `38059567206` passed every DB/DV H1/H2 source branch, +normal/deep independent-token repeats and same-unit process restarts, with clean +finalization. The earlier same-candidate H2 unknown-marker failure in run +`38058699913` is still retained; the new PASS does not rewrite that evidence or +claim a product fix. Tool build `38060217084` successfully built the extended +inline verifier and checked the 24 healthy/12 corrupt external shards plus the +one healthy/three corrupt released-process inline samples. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 49197d58..10302801 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 99ca5eea..b401d621 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index e2391977..4c35f66b 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -19,7 +19,7 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request) ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index bdbbbd0c..0f0d7cc5 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -5,6 +5,7 @@ import argparse import copy +import gzip from datetime import datetime, timedelta, timezone import hashlib import json @@ -14,6 +15,7 @@ import secrets import signal import sys +from urllib.parse import quote try: from .heal_probe import Probe, compare_inventories, payload, require, save @@ -40,8 +42,47 @@ "heal.versioning-suspended-null": ("enabled-to-suspended", "null-overwrite", "null-delete-marker"), "heal.below-read-quorum": ("r-minus-1-unversioned", "r-minus-1-historical"), "heal.missing-or-corrupt-xlmeta": ("missing-xlmeta", "truncated-xlmeta", "corrupt-xlmeta", "inline-payload", "versioned-xlmeta"), + "heal.object-metadata-and-tags": ("user-metadata", "content-headers", "versioned-tags"), } VERSION_SIZE = 5 * 1024**2 + 123 +SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") + + +def semantic_body(bucket, key, nonce): + return gzip.compress(payload(f"{bucket}/{key}/{nonce}", VERSION_SIZE), mtime=0) + + +def seed_semantics(recorder, bucket): + recorder.call("create_bucket", {"Bucket": bucket}) + recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) + expected, public = [], [] + for key in SEMANTIC_KEYS: + for nonce in range(3): + body = semantic_body(bucket, key, nonce) + metadata = {"UserKey": quote(f"中文值-{nonce}", safe=""), "generation": str(nonce)} + headers = {"ContentType": "application/octet-stream", "ContentEncoding": "gzip", + "ContentDisposition": "attachment; filename*=UTF-8''" + quote("中文文件.bin", safe=""), + "CacheControl": "no-store"} + color = "blue" if nonce % 2 == 0 else "red" + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, "Metadata": metadata, + **headers, "Tagging": f"color={color}&generation=v{nonce}"}, body) + require(response.get("VersionId") not in {None, "null"}, "semantic fixture omitted its precise historical version") + expected.append({"key": key, "versionId": response["VersionId"], "deleteMarker": False, "isLatest": nonce == 2, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + public.append({"key": key, "versionId": response["VersionId"], + "properties": {**headers, "Metadata": {name.lower(): value for name, value in metadata.items()}}, + "tags": [{"Key": "color", "Value": color}, {"Key": "generation", "Value": f"v{nonce}"}], + "payloadEncoding": "actual deterministic gzip bytes; no server-side compression claim"}) + return expected, public + + +def verify_semantics(public, baseline): + for spec in public: + entry = next((entry for entry in baseline["entries"] if version_identity(entry) == version_identity(spec)), None) + require(entry is not None and entry["tags"] == spec["tags"] + and all(entry["properties"].get(name) == value for name, value in spec["properties"].items()), + "semantic baseline lost normalized user metadata, actual encoding headers or exact versioned tags") + return {"versionedTagSetsAndPublicHeadersVerified": True, "expected": public} def version_identity(entry): @@ -292,8 +333,23 @@ def branch_plan(case_id, branch, snapshot, bucket, expected, fixture): key = "objects/0000.bin" if case_id == "heal.missing-or-corrupt-xlmeta": key = "objects/0000.bin" if fixture == "DV" else f"objects/{branch_index + 1:04d}.bin" + if case_id == "heal.object-metadata-and-tags": + key = SEMANTIC_KEYS[branch_index] selected = [entry for entry in expected if entry["key"] == key] data = [entry for entry in selected if not entry["deleteMarker"]] + if case_id == "heal.object-metadata-and-tags": + require(len(data) == 3, "metadata/tag fixture lacks three independently acknowledged versions") + target = choose_target(snapshot, bucket, data[0] if branch == "user-metadata" else data[-1], + "parity" if branch == "versioned-tags" else "data") + proofs, faults, required, metadata_keys = [target], [target], {version_identity(target)}, set() + if branch == "content-headers": + faults, required, metadata_keys = [metadata_target(snapshot, target)], set(), {key} + proofs = [choose_target(snapshot, bucket, entry, "data") for entry in data] + proofs = [{**item, **next(shard for shard in item["cohort"] + if (shard["node"], shard["disk"]) == (target["node"], target["disk"]))} for item in proofs] + return {"scope": {"bucket": bucket, "key": key}, "recursive": True, "expected": selected, + "faults": faults, "proofs": proofs, "requiredRepairs": required, "metadataRepairs": metadata_keys, + "kind": {"user-metadata": "missing", "content-headers": "header", "versioned-tags": "bitflip"}[branch]} if case_id == "heal.missing-or-corrupt-xlmeta": if fixture == "DB": target = map_inline_target(snapshot, bucket, data[0]) @@ -389,9 +445,11 @@ def verify_version_s3(probe, fleet, bucket, expected, baseline, label, *, fixtur nonce = next(index for index, item in enumerate( item for item in expected if item["key"] == entry["key"] and not item["deleteMarker"]) if item["versionId"] == entry["versionId"]) + body = semantic_body(bucket, entry["key"], nonce) if fixture == "DS" else payload( + f"{bucket}/{entry['key']}/{nonce}", entry["payload"]["size"]) ranges.append({"key": entry["key"], "versionId": entry["versionId"], "proof": verify_ranges( probe, fleet, bucket, entry["key"], version_id=entry["versionId"], - expected_body=payload(f"{bucket}/{entry['key']}/{nonce}", entry["payload"]["size"]))}) + expected_body=body)}) return {"inventory": comparison, "historicalRanges": ranges} @@ -427,12 +485,18 @@ def run_topology(args, topology, nodes, fixture): fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) recorder = Recorder(probe, directory / "fixture") - expected = seed(recorder, bucket, fixture, - 4 if fixture == "DV" else 128, [VERSION_SIZE] if fixture == "DV" else SIZES) + public = None + if fixture == "DS": + expected, public = seed_semantics(recorder, bucket) + else: + expected = seed(recorder, bucket, fixture, + 4 if fixture == "DV" else 128, [VERSION_SIZE] if fixture == "DV" else SIZES) if args.case_id == "heal.versioning-suspended-null": expected = suspend_fixture(recorder, bucket, expected) baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + if public is not None: + fleet.record("semantic-baseline", verify_semantics(public, baseline)) for branch in branches: registrations = [] metadata_faults = {} @@ -599,6 +663,8 @@ def interrupted(signum, frame): require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta"} else ("DV",) + if args.case_id == "heal.object-metadata-and-tags": + fixtures = ("DS",) results = [] try: for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): diff --git a/test_validation_heal.py b/test_validation_heal.py index 6783d19d..8be5a02a 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -6,6 +6,7 @@ import tempfile import unittest import copy +import gzip import types from unittest.mock import patch @@ -20,6 +21,7 @@ from scripts.validation_heal.versions import (BRANCHES as VERSION_BRANCHES, validate_version_receipts, branch_plan, suspend_fixture, validate_failed_version, validate_unrecoverable_get, verify_quorum_contract, map_inline_target, verify_proof) +from scripts.validation_heal.versions import seed_semantics, verify_semantics from scripts.validation_heal.prepare_heal_meta_inspector import quorum_contract @@ -386,6 +388,31 @@ def test_fault_requires_actual_service_quiescence_and_exact_truncate_length(self class HealVersionOracleTest(unittest.TestCase): + def test_metadata_fixture_has_real_encoded_content_and_exact_distinct_version_tags(self): + uploaded = [] + def call(operation, params, body=None): + if operation == "put_object": + self.assertEqual(params["ContentEncoding"], "gzip") + self.assertTrue(gzip.decompress(body)) + self.assertEqual(len(gzip.decompress(body)), 5 * 1024**2 + 123) + self.assertTrue(all(value.isascii() for value in params["Metadata"].values())) + uploaded.append((params["Key"], params["Metadata"]["generation"])) + return {"VersionId": "version-" + str(len(uploaded))} + return {} + expected, public = seed_semantics(types.SimpleNamespace(call=call), "fixture") + self.assertEqual(len(expected), 9) + self.assertEqual(len({entry["versionId"] for entry in expected}), 9) + self.assertEqual(len({key for key, _ in uploaded}), 3) + baseline = {"entries": [{"key": spec["key"], "versionId": spec["versionId"], + "properties": copy.deepcopy(spec["properties"]), "tags": copy.deepcopy(spec["tags"])} for spec in public]} + verify_semantics(public, baseline) + for kind in ("header", "metadata", "versioned-tags"): + changed = copy.deepcopy(baseline) + if kind == "header": changed["entries"][0]["properties"]["ContentEncoding"] = "identity" + elif kind == "metadata": changed["entries"][0]["properties"]["Metadata"]["userkey"] = "another" + else: changed["entries"][0]["tags"] = copy.deepcopy(changed["entries"][1]["tags"]) + with self.subTest(kind=kind), self.assertRaises(ValueError): verify_semantics(public, changed) + def test_decoder_failure_is_evidence_only_for_the_exact_witnessed_corruption(self): entry = {"path": "fixture/key/xl.meta", "sha256": "changed-sha", "size": 9} fault = {entry["path"]: {"sha256": "changed-sha", "size": 9}} From d745f24020ddc5c1fe26eda0f81441f69bfc5b2e Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 22:46:45 +0800 Subject: [PATCH 037/343] fix(validation): normalize typed null version receipts --- docs/validation-migration-map.md | 7 +++++++ scripts/validation_heal/versions.py | 13 ++++++++++--- test_validation_heal.py | 13 +++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 64eb2551..e6759956 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -275,3 +275,10 @@ finalization. The earlier same-candidate H2 unknown-marker failure in run claim a product fix. Tool build `38060217084` successfully built the extended inline verifier and checked the 24 healthy/12 corrupt external shards plus the one healthy/three corrupt released-process inline samples. + +D05 run `38060400818` exercised all H1/H2 suspended/null branches and cleaned +up successfully, but its receipt oracle compared the Admin typed nil UUID with +the S3 `null` spelling literally. The candidate source defines these as the same +null identity. The oracle now normalizes only those representations and still +rejects every other version mismatch. The original FAIL remains; a fresh full +run is required. Auto-testing CI passed all six lanes at commit `f9c153f`. diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 0f0d7cc5..6ea28bfe 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -86,7 +86,14 @@ def verify_semantics(public, baseline): def version_identity(entry): - return entry["key"], entry.get("versionId") or "null" + return entry["key"], public_version_id(entry.get("versionId")) + + +def public_version_id(value): + # The candidate's typed Admin identity serializes Uuid::nil(), whereas + # the S3 identity and ACK ledger expose that same null version as "null". + # Normalize only those representations; every non-null UUID stays exact. + return "null" if value in {None, "", "null", "00000000-0000-0000-0000-000000000000"} else value def validate_version_receipts(status, bucket, expected, mode, *, required_repairs=(), metadata_repairs=()): @@ -106,7 +113,7 @@ def validate_version_receipts(status, bucket, expected, mode, *, required_repair "repair plan escapes the selected responsibility set") for receipt in objects: identity = receipt.get("identity", {}) - pair = identity.get("object"), identity.get("versionId") or "null" + pair = identity.get("object"), public_version_id(identity.get("versionId")) require(pair in wanted and pair not in actual and identity.get("bucket") == bucket and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 and type(identity.get("setIndex")) is int and identity["setIndex"] == 0 @@ -173,7 +180,7 @@ def validate_failed_version(status, bucket, expected, target): actual = set() for receipt in objects: identity = receipt.get("identity", {}) - pair = identity.get("object"), identity.get("versionId") or "null" + pair = identity.get("object"), public_version_id(identity.get("versionId")) require(pair in wanted and pair not in actual and identity.get("bucket") == bucket and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 and type(identity.get("setIndex")) is int and identity["setIndex"] == 0, diff --git a/test_validation_heal.py b/test_validation_heal.py index 8be5a02a..74340d59 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -577,6 +577,19 @@ def test_repeat_requires_mode_specific_health_and_zero_repairs(self): status["outcome"]["counters"].update(healed=1, unchanged=3) with self.assertRaises(ValueError): validate_version_receipts(status, "fixture", expected, 1) + def test_admin_nil_uuid_maps_only_to_the_acknowledged_s3_null_version(self): + status, expected = self.fixture() + expected[0]["versionId"] = "null" + status["outcome"]["objects"][0]["identity"]["versionId"] = "00000000-0000-0000-0000-000000000000" + validate_version_receipts(status, "fixture", expected, 2) + wrong = copy.deepcopy(status) + wrong["outcome"]["objects"][0]["identity"]["versionId"] = "00000000-0000-0000-0000-000000000001" + with self.assertRaises(ValueError): + validate_version_receipts(wrong, "fixture", expected, 2) + expected[0]["versionId"] = "old" + with self.assertRaises(ValueError): + validate_version_receipts(status, "fixture", expected, 2) + def test_metadata_repair_may_repair_all_receipts_but_marker_cannot_claim_deep_payload(self): status, expected = self.fixture() with self.assertRaisesRegex(ValueError, "never attributed"): From c935afa17e1665ef304d97e3c55168f8c1088978 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 22:51:35 +0800 Subject: [PATCH 038/343] feat(validation): execute actual EC multi-shard loss matrix --- docs/validation-migration-map.md | 19 +++++++++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 59 +++++++++++++++++++++++++++-- test_validation_heal.py | 26 +++++++++++++ 6 files changed, 102 insertions(+), 8 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index e6759956..20e3fd28 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 92 currently dispatchable IDs include all 74 non-Heal cases and eighteen Heal cases. Thirty-nine have live PASS evidence; the other 119 have no live PASS claim. +case evidence. The 93 currently dispatchable IDs include all 74 non-Heal cases and nineteen Heal cases. Thirty-nine have live PASS evidence; the other 119 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 66 Heal cases remain +The other 65 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -282,3 +282,18 @@ the S3 `null` spelling literally. The candidate source defines these as the same null identity. The oracle now normalizes only those representations and still rejects every other version mismatch. The original FAIL remains; a fresh full run is required. Auto-testing CI passed all six lanes at commit `f9c153f`. + +B05 `heal.multiple-shard-loss` now expands 1..M losses from the actual decoded +EC geometry, including every possible data/parity role count. Each parameter +instance uses a distinct acknowledged object; DB and DV both run on H1/H2. +All drives remain online for the write/lock condition while selected same-version +parts are isolated. Exact full original shard hashes, streaming checksums, +complete version/marker responsibility and S3 controls must all pass. B05 has +not yet been validated live. + +B07 run `38060686737` retained FAIL with cleanup PASS. At R-1 shards H1 +returned HTTP 500/InternalError instead of the recognized candidate read-quorum +refusal. H2 returned 503/SlowDownRead, but Heal recorded a grace-window deferred +receipt and completed execution rather than the source's required failed +responsibility. Good-shard preservation and the raw task outcome remain +archived; neither branch is promoted to PASS. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 10302801..c4a2e423 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index b401d621..acd2a878 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 4c35f66b..35cdfdff 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -19,7 +19,7 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request) ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 6ea28bfe..f91288f4 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -43,6 +43,7 @@ "heal.below-read-quorum": ("r-minus-1-unversioned", "r-minus-1-historical"), "heal.missing-or-corrupt-xlmeta": ("missing-xlmeta", "truncated-xlmeta", "corrupt-xlmeta", "inline-payload", "versioned-xlmeta"), "heal.object-metadata-and-tags": ("user-metadata", "content-headers", "versioned-tags"), + "heal.multiple-shard-loss": ("loss-1-data", "loss-1-parity", "loss-2-through-m-data-parity"), } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") @@ -220,6 +221,46 @@ def metadata_target(snapshot, part_target): return {**part_target, "relativePath": relative, "sha256": entry["sha256"], "size": entry["size"]} +def multi_loss_instances(snapshot, bucket, expected, fixture): + sample = next(entry for entry in expected if not entry["deleteMarker"]) + target = choose_target(snapshot, bucket, sample, "data") + geometry = target["logical"]["erasure"] + data, parity = geometry["data_blocks"], geometry["parity_blocks"] + require(data > 0 and parity > 0 and data + parity == len(target["cohort"]), + "multi-loss matrix needs a complete same-version EC authority") + instances = [] + for loss in range(1, parity + 1): + for data_loss in range(max(0, loss - parity), min(loss, data) + 1): + source_branch = ("loss-1-data" if data_loss else "loss-1-parity") if loss == 1 else "loss-2-through-m-data-parity" + instances.append({"name": f"{source_branch}-n{loss}-d{data_loss}-p{loss - data_loss}", + "sourceBranch": source_branch, "loss": loss, "dataLoss": data_loss, "parityLoss": loss - data_loss, + "key": f"objects/{len(instances):04d}.bin", "fixture": fixture, + "readQuorum": data, "remainingSameVersionShards": data + parity - loss, + "writeCommitCondition": "all physical members online; only selected part files isolated"}) + require({entry["sourceBranch"] for entry in instances} == set(BRANCHES["heal.multiple-shard-loss"]), + "actual EC geometry cannot cover every required multi-loss branch") + require(len({entry["key"] for entry in instances}) == len(instances) + and all(entry["remainingSameVersionShards"] >= data for entry in instances), + "multi-loss matrix reused a fixture or crossed below its read quorum") + return instances + + +def multi_loss_plan(instance, snapshot, bucket, expected, fixture): + selected = [entry for entry in expected if entry["key"] == instance["key"]] + data = [entry for entry in selected if not entry["deleteMarker"]] + require(len(data) == (3 if fixture == "DV" else 1), "multi-loss instance lacks its fresh ACK cohort") + target = choose_target(snapshot, bucket, data[0], "data") + cohort = sorted(target["cohort"], key=lambda shard: shard["index"]) + missing = [shard for shard in cohort if shard["role"] == "data"][:instance["dataLoss"]] + missing += [shard for shard in cohort if shard["role"] == "parity"][:instance["parityLoss"]] + require(len(missing) == instance["loss"] and len(cohort) - len(missing) >= instance["readQuorum"], + "actual multi-loss target roles or quorum differ from the frozen parameter matrix") + faults = [{**target, **shard} for shard in missing] + return {"scope": {"bucket": bucket, "key": instance["key"]}, "recursive": fixture == "DV", + "expected": selected, "faults": faults, "proofs": faults, + "requiredRepairs": {version_identity(data[0])}, "metadataRepairs": set(), "parameters": instance} + + def map_inline_target(snapshot, bucket, entry): validate_topology(snapshot, "H1" if len(snapshot) == 1 else "H2") indexed, cohort, authority = files(snapshot), [], None @@ -497,20 +538,30 @@ def run_topology(args, topology, nodes, fixture): expected, public = seed_semantics(recorder, bucket) else: expected = seed(recorder, bucket, fixture, - 4 if fixture == "DV" else 128, [VERSION_SIZE] if fixture == "DV" else SIZES) + 16 if args.case_id == "heal.multiple-shard-loss" else 4 if fixture == "DV" else 128, + [VERSION_SIZE] if args.case_id == "heal.multiple-shard-loss" or fixture == "DV" else SIZES) if args.case_id == "heal.versioning-suspended-null": expected = suspend_fixture(recorder, bucket, expected) baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) if public is not None: fleet.record("semantic-baseline", verify_semantics(public, baseline)) + instances = None + if args.case_id == "heal.multiple-shard-loss": + fleet.stop() + instances = multi_loss_instances(fleet.snapshot("multi-loss-geometry", quiescent=True), bucket, expected, fixture) + fleet.record("multi-loss-parameter-matrix", {"instances": instances, "allLossCountsAndRoleCompositions": True}) + branches = [entry["name"] for entry in instances] + fleet.start() for branch in branches: registrations = [] metadata_faults = {} try: fleet.stop() before = fleet.snapshot("physical-before-" + branch, quiescent=True) - plan = branch_plan(args.case_id, branch, before, bucket, expected, fixture) + plan = multi_loss_plan(next(entry for entry in instances if entry["name"] == branch), + before, bucket, expected, fixture) if instances else branch_plan( + args.case_id, branch, before, bucket, expected, fixture) fleet.record("fault-plan-" + branch, {**plan, "requiredRepairs": sorted(plan["requiredRepairs"]), "metadataRepairs": sorted(plan["metadataRepairs"])}) for target in plan["proofs"]: @@ -616,6 +667,8 @@ def run_topology(args, topology, nodes, fixture): repeats.append(proof) result["branches"][branch] = {"verdict": "PASS", "physical": physical, "bitrot": bitrot, "s3": s3, "responsibility": responsibility, "repeats": repeats} + if instances: + result["branches"][branch]["parameters"] = plan["parameters"] if negative_proof is not None: result["branches"][branch]["negative"] = negative_proof result["branches"][branch]["physical"]["backupUsedAfterFailureProof"] = True @@ -669,7 +722,7 @@ def interrupted(signum, frame): and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") - fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta"} else ("DV",) + fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.multiple-shard-loss"} else ("DV",) if args.case_id == "heal.object-metadata-and-tags": fixtures = ("DS",) results = [] diff --git a/test_validation_heal.py b/test_validation_heal.py index 74340d59..3d05d949 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -22,6 +22,7 @@ branch_plan, suspend_fixture, validate_failed_version, validate_unrecoverable_get, verify_quorum_contract, map_inline_target, verify_proof) from scripts.validation_heal.versions import seed_semantics, verify_semantics +from scripts.validation_heal.versions import multi_loss_instances, multi_loss_plan from scripts.validation_heal.prepare_heal_meta_inspector import quorum_contract @@ -388,6 +389,31 @@ def test_fault_requires_actual_service_quiescence_and_exact_truncate_length(self class HealVersionOracleTest(unittest.TestCase): + def test_multi_loss_covers_every_actual_loss_count_and_role_composition(self): + cohort = {"logical": {"erasure": {"data_blocks": 12, "parity_blocks": 4}}, "cohort": list(range(16))} + expected = [{"key": "object", "versionId": "old", "deleteMarker": False}] + with patch("scripts.validation_heal.versions.choose_target", return_value=cohort): + instances = multi_loss_instances({}, "fixture", expected, "DV") + self.assertEqual(len(instances), 14) + self.assertEqual({(item["loss"], item["dataLoss"], item["parityLoss"]) for item in instances}, + {(loss, data, loss - data) for loss in range(1, 5) for data in range(loss + 1)}) + self.assertEqual(len({item["key"] for item in instances}), 14) + self.assertEqual({item["sourceBranch"] for item in instances}, set(VERSION_BRANCHES["heal.multiple-shard-loss"])) + for item in instances: + self.assertGreaterEqual(item["remainingSameVersionShards"], item["readQuorum"]) + before = HealShardOracleTest().snapshot() + entry = {"key": "objects/0008.bin", "versionId": "null", "deleteMarker": False} + instance = {**instances[2], "key": entry["key"], "dataLoss": 1, "parityLoss": 1, + "loss": 2, "readQuorum": 2} + plan = multi_loss_plan(instance, before, "rv-heal-0123456789abcdef", [entry], "DB") + self.assertEqual({item["role"] for item in plan["faults"]}, {"data", "parity"}) + self.assertEqual(len({item["disk"] for item in plan["faults"]}), 2) + self.assertEqual(plan["requiredRepairs"], {(entry["key"], "null")}) + self.assertTrue(all(item["versionId"] == "null" for item in plan["faults"])) + with self.assertRaisesRegex(ValueError, "quorum"): + multi_loss_plan({**instance, "dataLoss": 3, "loss": 4}, before, + "rv-heal-0123456789abcdef", [entry], "DB") + def test_metadata_fixture_has_real_encoded_content_and_exact_distinct_version_tags(self): uploaded = [] def call(operation, params, body=None): From 537e981f63c470ee02bd322ecdcc89138975be57 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 22:54:29 +0800 Subject: [PATCH 039/343] feat(validation): exercise special keys and explicit list pagination --- docs/validation-migration-map.md | 12 +++++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/heal_probe.py | 19 +++++++-- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 61 +++++++++++++++++++++++++-- test_validation_heal.py | 40 +++++++++++++++++- 7 files changed, 124 insertions(+), 14 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 20e3fd28..82f000c1 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 93 currently dispatchable IDs include all 74 non-Heal cases and nineteen Heal cases. Thirty-nine have live PASS evidence; the other 119 have no live PASS claim. +case evidence. The 94 currently dispatchable IDs include all 74 non-Heal cases and twenty Heal cases. Thirty-nine have live PASS evidence; the other 119 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 65 Heal cases remain +The other 64 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -297,3 +297,11 @@ refusal. H2 returned 503/SlowDownRead, but Heal recorded a grace-window deferred receipt and completed execution rather than the source's required failed responsibility. Good-shard preservation and the raw task outcome remain archived; neither branch is promoted to PASS. + +D10 `heal.special-keys-and-pagination` now creates 265 unique objects, including +Unicode, spaces, literal percent/encoded slash, real slash, nested paths and +adjacent `a/`/`ab/` controls. Both version and current LIST explicitly request +MaxKeys=17 and retain observed page sizes; the complete ACK manifest must match +without missing or duplicate keys. Exact logical-key physical mappings precede +part/xl.meta faults and signed object requests. Every control remains exact. +D10 has not yet been validated live. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index c4a2e423..2b9a80fa 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/heal_probe.py b/scripts/validation_heal/heal_probe.py index 4c0af320..d51efe10 100644 --- a/scripts/validation_heal/heal_probe.py +++ b/scripts/validation_heal/heal_probe.py @@ -303,19 +303,26 @@ def _open_admin(self, outgoing): opener = request.build_opener(request.ProxyHandler({}), NoRedirect()) return opener.open(outgoing, timeout=30) - def inventory(self, bucket: str, *, unversioned_current_tags: bool = False) -> dict: + def inventory(self, bucket: str, *, unversioned_current_tags: bool = False, page_size: int | None = None) -> dict: """Enumerate every version/marker, then hash complete version-addressed GETs. Only call before injection or after attribution/physical proof: a GET can itself trigger read repair. Never use this to watch scanner progress. """ bucket_allowed(self.binding, bucket) + require(page_size is None or type(page_size) is int and 1 <= page_size <= 1000, + "invalid explicit inventory page size") client, entries, count, byte_count = self.s3_client(), [], 0, 0 + pagination = {"requestedPageSize": page_size, "versionPageSizes": [], "currentPageSizes": []} + options = {"PaginationConfig": {"PageSize": page_size}} if page_size else {} if unversioned_current_tags: mode = self.s3("get_bucket_versioning", {"Bucket": bucket}) require(mode["acknowledged"] and mode["response"].get("Status") is None, "current-tag addressing requires a never-versioned fixture bucket") - for page in client.get_paginator("list_object_versions").paginate(Bucket=bucket): + for page in client.get_paginator("list_object_versions").paginate(Bucket=bucket, **options): + page_count = len(page.get("Versions", [])) + len(page.get("DeleteMarkers", [])) + require(page_size is None or page_count <= page_size, "version LIST exceeded the requested page size") + pagination["versionPageSizes"].append(page_count) require(not unversioned_current_tags or not page.get("DeleteMarkers"), "unversioned fixture unexpectedly has delete markers") for marker in page.get("DeleteMarkers", []): @@ -352,7 +359,10 @@ def inventory(self, bucket: str, *, unversioned_current_tags: bool = False) -> d require(len(set(identities)) == len(identities), "duplicate version/marker in pagination") # Cross-check current LIST against the separately paginated version view. current = [] - for page in client.get_paginator("list_objects_v2").paginate(Bucket=bucket): + for page in client.get_paginator("list_objects_v2").paginate(Bucket=bucket, **options): + require(page_size is None or len(page.get("Contents", [])) <= page_size, + "current LIST exceeded the requested page size") + pagination["currentPageSizes"].append(len(page.get("Contents", []))) current.extend((item["Key"], item["Size"], item["ETag"]) for item in page.get("Contents", [])) require(len(current) <= self.binding["maxInventoryObjects"], "current LIST exceeds budget") expected = [(item["key"], item["payload"]["size"], item["properties"]["ETag"]) @@ -364,7 +374,8 @@ def inventory(self, bucket: str, *, unversioned_current_tags: bool = False) -> d "fixture versioning changed during inventory") return {"bucket": bucket, "entries": sorted(entries, key=lambda item: (item["key"], item["versionId"])), "objectsAndMarkers": count, "bytesRead": byte_count, - "tagAddressing": "current-never-versioned-fixture" if unversioned_current_tags else "explicit-version"} + "tagAddressing": "current-never-versioned-fixture" if unversioned_current_tags else "explicit-version", + "pagination": pagination} def compare_inventories(before: dict, after: dict) -> dict: diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index acd2a878..0989f807 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 35cdfdff..cf019940 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -19,7 +19,7 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request) ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index f91288f4..0b47d6e2 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -44,9 +44,42 @@ "heal.missing-or-corrupt-xlmeta": ("missing-xlmeta", "truncated-xlmeta", "corrupt-xlmeta", "inline-payload", "versioned-xlmeta"), "heal.object-metadata-and-tags": ("user-metadata", "content-headers", "versioned-tags"), "heal.multiple-shard-loss": ("loss-1-data", "loss-1-parity", "loss-2-through-m-data-parity"), + "heal.special-keys-and-pagination": ("unicode-space-percent", "encoded-slash", "adjacent-prefixes", "multi-page-list"), } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") +SPECIAL_KEYS = ("中文/空 格%25.bin", "space key/object.bin", "literal%2Fslash/object.bin", + "literal/slash/object.bin", "a/object.bin", "ab/object.bin", "b/object.bin", "nested/a/b/object.bin") + + +def seed_special_keys(recorder, bucket): + recorder.call("create_bucket", {"Bucket": bucket}) + expected = [] + keys = [*SPECIAL_KEYS, *(f"pages/{index:04d}.bin" for index in range(257))] + for key in keys: + body = payload(f"{bucket}/{key}/0", VERSION_SIZE if key in SPECIAL_KEYS else 128) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": "DS", + "logical-key-sha256": hashlib.sha256(key.encode()).hexdigest()}}, body) + require(response.get("VersionId") in {None, "null"}, "special-key fixture unexpectedly became versioned") + expected.append({"key": key, "versionId": "null", "deleteMarker": False, "isLatest": True, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + require(len(keys) == len(set(keys)) > 257, "special keys collided with the pagination fixture") + return expected + + +def verify_pagination(inventory): + pagination = inventory.get("pagination") or {} + entries = inventory["entries"] + require(pagination.get("requestedPageSize") == 17 and len(entries) > 257 + and {entry["key"] for entry in entries}.issuperset(SPECIAL_KEYS), + "special-key inventory omitted its explicit pagination or distinct logical keys") + for name in ("versionPageSizes", "currentPageSizes"): + pages = pagination.get(name, []) + require(len(pages) > 1 and all(type(count) is int and 0 <= count <= 17 for count in pages) + and sum(pages) == len(entries), "multi-page LIST omitted or duplicated acknowledged objects") + return {"maxKeys": 17, "versionPages": len(pagination["versionPageSizes"]), + "currentPages": len(pagination["currentPageSizes"]), "acknowledgedObjects": len(entries)} def semantic_body(bucket, key, nonce): @@ -383,8 +416,18 @@ def branch_plan(case_id, branch, snapshot, bucket, expected, fixture): key = "objects/0000.bin" if fixture == "DV" else f"objects/{branch_index + 1:04d}.bin" if case_id == "heal.object-metadata-and-tags": key = SEMANTIC_KEYS[branch_index] + if case_id == "heal.special-keys-and-pagination": + key = SPECIAL_KEYS[(0, 2, 4, 7)[branch_index]] selected = [entry for entry in expected if entry["key"] == key] data = [entry for entry in selected if not entry["deleteMarker"]] + if case_id == "heal.special-keys-and-pagination": + require(len(data) == 1 and data[0]["versionId"] == "null", "special-key fault lost its exact logical identity") + target = choose_target(snapshot, bucket, data[0], "parity" if branch == "adjacent-prefixes" else "data") + metadata = branch in {"encoded-slash", "multi-page-list"} + return {"scope": {"bucket": bucket, "key": key}, "recursive": False, "expected": selected, + "faults": [metadata_target(snapshot, target) if metadata else target], "proofs": [target], + "requiredRepairs": set() if metadata else {version_identity(data[0])}, + "metadataRepairs": {key} if metadata else set()} if case_id == "heal.object-metadata-and-tags": require(len(data) == 3, "metadata/tag fixture lacks three independently acknowledged versions") target = choose_target(snapshot, bucket, data[0] if branch == "user-metadata" else data[-1], @@ -477,8 +520,11 @@ def branch_plan(case_id, branch, snapshot, bucket, expected, fixture): def verify_version_s3(probe, fleet, bucket, expected, baseline, label, *, fixture="DV"): - current = fleet.record(label, probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + current = fleet.record(label, probe.inventory(bucket, unversioned_current_tags=fixture in {"DB", "DK"}, + page_size=17 if fixture == "DK" else None)) comparison = compare_inventories(baseline, current) + if fixture == "DK": + comparison["pagination"] = verify_pagination(current) ranges = [] for entry in expected: if entry["deleteMarker"]: @@ -518,7 +564,7 @@ def run_topology(args, topology, nodes, fixture): "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), - "maxInventoryObjects": 256, "maxInventoryBytes": 512 * 1024**2} + "maxInventoryObjects": 512, "maxInventoryBytes": 512 * 1024**2} fleet.record("binding", binding) recipe = select(args.case_id)[0] require(list(BRANCHES[args.case_id]) == recipe["variants"], "executor omitted a source version branch") @@ -534,7 +580,9 @@ def run_topology(args, topology, nodes, fixture): fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) recorder = Recorder(probe, directory / "fixture") public = None - if fixture == "DS": + if fixture == "DK": + expected = seed_special_keys(recorder, bucket) + elif fixture == "DS": expected, public = seed_semantics(recorder, bucket) else: expected = seed(recorder, bucket, fixture, @@ -542,8 +590,11 @@ def run_topology(args, topology, nodes, fixture): [VERSION_SIZE] if args.case_id == "heal.multiple-shard-loss" or fixture == "DV" else SIZES) if args.case_id == "heal.versioning-suspended-null": expected = suspend_fixture(recorder, bucket, expected) - baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=fixture in {"DB", "DK"}, + page_size=17 if fixture == "DK" else None)) fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + if fixture == "DK": + fleet.record("special-key-pagination-baseline", verify_pagination(baseline)) if public is not None: fleet.record("semantic-baseline", verify_semantics(public, baseline)) instances = None @@ -725,6 +776,8 @@ def interrupted(signum, frame): fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.multiple-shard-loss"} else ("DV",) if args.case_id == "heal.object-metadata-and-tags": fixtures = ("DS",) + if args.case_id == "heal.special-keys-and-pagination": + fixtures = ("DK",) results = [] try: for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): diff --git a/test_validation_heal.py b/test_validation_heal.py index 3d05d949..462cd159 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -22,7 +22,8 @@ branch_plan, suspend_fixture, validate_failed_version, validate_unrecoverable_get, verify_quorum_contract, map_inline_target, verify_proof) from scripts.validation_heal.versions import seed_semantics, verify_semantics -from scripts.validation_heal.versions import multi_loss_instances, multi_loss_plan +from scripts.validation_heal.versions import multi_loss_instances, multi_loss_plan, SPECIAL_KEYS, verify_pagination +from scripts.validation_heal.heal_probe import Probe from scripts.validation_heal.prepare_heal_meta_inspector import quorum_contract @@ -389,6 +390,43 @@ def test_fault_requires_actual_service_quiescence_and_exact_truncate_length(self class HealVersionOracleTest(unittest.TestCase): + def test_explicit_small_list_pages_preserve_distinct_special_keys(self): + bucket = "rv-heal-pages-unit" + keys = [*SPECIAL_KEYS, *(f"pages/{index:04d}.bin" for index in range(257))] + for failure in (None, "duplicate-version", "omitted-current", "oversized-page"): + calls = [] + def paginator(operation): + def paginate(**params): + calls.append((operation, params)) + self.assertEqual(params["PaginationConfig"], {"PageSize": 17}) + listed = list(keys) + if failure == "duplicate-version" and operation == "list_object_versions": listed[-1] = listed[0] + if failure == "omitted-current" and operation == "list_objects_v2": listed.pop() + width = 18 if failure == "oversized-page" else 17 + for start in range(0, len(listed), width): + page = listed[start:start + width] + yield {"Versions": [{"Key": key, "VersionId": "null", "IsLatest": True} for key in page]} \ + if operation == "list_object_versions" else {"Contents": [{"Key": key, "Size": 1, "ETag": key} for key in page]} + return types.SimpleNamespace(paginate=paginate) + def s3(operation, params): + if operation == "get_object": + return {"acknowledged": True, "response": {"ETag": params["Key"]}, "payload": {"size": 1, "sha256": "digest"}} + return {"acknowledged": True, "response": {}} + probe = Probe.__new__(Probe) + probe.binding = {"bucketPrefix": bucket, "maxInventoryObjects": 512, "maxInventoryBytes": 512} + probe.s3_client = lambda: types.SimpleNamespace(get_paginator=paginator) + probe.s3 = s3 + if failure: + with self.subTest(failure=failure), self.assertRaises(ValueError): + probe.inventory(bucket, unversioned_current_tags=True, page_size=17) + else: + observed = probe.inventory(bucket, unversioned_current_tags=True, page_size=17) + proof = verify_pagination(observed) + self.assertEqual(proof["acknowledgedObjects"], 265) + self.assertEqual(proof["currentPages"], 16) + self.assertEqual(proof["versionPages"], 16) + self.assertEqual({entry["key"] for entry in observed["entries"]}, set(keys)) + def test_multi_loss_covers_every_actual_loss_count_and_role_composition(self): cohort = {"logical": {"erasure": {"data_blocks": 12, "parity_blocks": 4}}, "cohort": list(range(16))} expected = [{"key": "object", "versionId": "old", "deleteMarker": False}] From 103982b46fbbdafe00f4816082beed2e0cce5b2a Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 23:03:24 +0800 Subject: [PATCH 040/343] feat(validation): derive and exercise candidate size boundaries --- docs/validation-migration-map.md | 27 ++++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/boundaries.py | 107 ++++++++++++++++++ scripts/validation_heal/fleet_node.py | 2 + scripts/validation_heal/matrix.py | 2 +- .../prepare_heal_meta_inspector.py | 51 +++++++++ scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 95 ++++++++++++---- test_validation_heal.py | 93 ++++++++++++++- 9 files changed, 354 insertions(+), 27 deletions(-) create mode 100644 scripts/validation_heal/boundaries.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 82f000c1..a33a1a53 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 94 currently dispatchable IDs include all 74 non-Heal cases and twenty Heal cases. Thirty-nine have live PASS evidence; the other 119 have no live PASS claim. +case evidence. The 95 currently dispatchable IDs include all 74 non-Heal cases and twenty-one Heal cases. Forty have live PASS evidence; the other 118 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 64 Heal cases remain +The other 63 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -305,3 +305,26 @@ MaxKeys=17 and retain observed page sizes; the complete ACK manifest must match without missing or duplicate keys. Exact logical-key physical mappings precede part/xl.meta faults and signed object requests. Every control remains exact. D10 has not yet been validated live. + +C05 run `38061075879` exercised all ten DB/DV H1/H2 branches and retained FAIL +with cleanup PASS. Nine branches passed. H1's truncated xl.meta remained at the +exact injected 1866-byte length instead of its 3732-byte baseline and could not +decode; the physical oracle rejected it before target GET. + +D06 run `38061589839` passed all H1/H2 semantic branches with cleanup PASS: +three distinct versions for each Unicode/space/percent key retained exact public +metadata, actual gzip bytes/headers and versioned tags through data, metadata +and parity faults. Physical original-shard/bitrot and full S3/HEAD/Range checks +passed before receipt promotion. + +D01 `heal.object-size-boundaries` now has a complete automatic parameter matrix +for zero/one, 4KiB, actual inline and EC stripe boundaries plus a multi-block +external object. A tool-build contract freezes recognized candidate policy and +source/extractor digests; live process environment and actual EC metadata derive +the limits. H1/H2 need different inline limits under the recognized current +policy (the runtime derives these rather than guessing 256KiB). Each size tests +both actual data/parity roles with missing and bitflip faults. Zero objects use +metadata loss/header faults and never invent a payload part. Exact byte hashes, +complete responsibility, HEAD/LIST and actual first/tail/stripe Range checks +are required; zero Range is a strict S3 416/InvalidRange negative. D01 requires a +fresh tool artifact carrying this size contract and has not yet run live. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 2b9a80fa..5d56f373 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/boundaries.py b/scripts/validation_heal/boundaries.py new file mode 100644 index 00000000..3067c2c3 --- /dev/null +++ b/scripts/validation_heal/boundaries.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""D01 size parameters from the fixed candidate contract and observed EC geometry.""" + +import hashlib +from pathlib import Path +import re + +try: + from .heal_probe import payload, require +except ImportError: + from heal_probe import payload, require + + +BRANCHES = ("zero", "one", "4kib-minus-exact-plus", "inline-minus-exact-plus", + "stripe-minus-exact-plus", "large-external") + + +def verify_size_contract(manifest, candidate): + contract = manifest.get("sizeContract") or {} + extractor = Path(__file__).with_name("prepare_heal_meta_inspector.py") + require(contract.get("recognized") is True and contract.get("candidateCommit") == candidate["source_sha"] + and contract.get("extractorSourceSha256") == hashlib.sha256(extractor.read_bytes()).hexdigest() + and contract.get("inlineRule") == "ceil(size/dataBlocks)<=effectiveInlineBlock AND size<=actualECBlock" + and contract.get("effectiveDefaultRule") == "min(ceil(2*defaultInlineBlock/dataBlocks),defaultInlineBlock)" + and type(contract.get("defaultInlineBlock")) is int and contract["defaultInlineBlock"] > 0 + and contract.get("defaultObjectBudgetFactor") == 2 + and set(contract.get("sourceFiles", {})) == { + "crates/ecstore/src/config/storageclass.rs", "crates/ecstore/src/set_disk/mod.rs", + "crates/ecstore/src/set_disk/ops/object.rs", "crates/ecstore/src/erasure/coding/erasure.rs"} + and all(re.fullmatch(r"[0-9a-f]{64}", value) for value in contract["sourceFiles"].values()), + "size boundaries require a recognized fresh exact-candidate source contract") + return contract + + +def boundary_sizes(contract, target, processes): + require(processes and all(entry.get("storageClassEnvironment") == { + "RUSTFS_STORAGE_CLASS_INLINE_BLOCK": None, "RUSTFS_STORAGE_CLASS_STANDARD": None} + for entry in processes.values()), "default size policy requires actual process environment confirmation") + geometry = target["logical"]["erasure"] + data, block = geometry["data_blocks"], geometry["block_size"] + require(type(data) is int and data > 0 and type(block) is int and 4097 < block < 12 * 1024**2 + and target["logical"]["usesLegacyChecksum"] is False, + "observed size calibration needs bounded current-codec EC geometry") + default = contract["defaultInlineBlock"] + effective = min((2 * default + data - 1) // data, default) + inline = min(effective * data, block) + require(4097 < inline < block, "candidate policy lacks independently testable inline and stripe boundaries") + groups = {"zero": [0], "one": [1], "4kib-minus-exact-plus": [4095, 4096, 4097], + "inline-minus-exact-plus": [inline - 1, inline, inline + 1], + "stripe-minus-exact-plus": [block - 1, block, block + 1], "large-external": [5 * block + 123]} + require(set(groups) == set(BRANCHES), "size parameter matrix omitted a source boundary") + return {"actualDataBlocks": data, "actualECBlockSize": block, "defaultInlineBlock": default, + "effectiveInlineShardBudget": effective, "actualLogicalInlineLimit": inline, + "sourceBoundRule": contract["inlineRule"], "groups": groups, + "processConfigurationReadBack": True, "codec": "current"} + + +def seed_boundaries(recorder, bucket, parameters): + expected, instances = [], [] + for branch, sizes in parameters["groups"].items(): + for size in sizes: + key = f"boundaries/{len(expected):04d}-{size}.bin" + body = payload(f"{bucket}/{key}/0", size) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "Metadata": {"heal-fixture": "D01", "size": str(size)}, "ContentType": "application/octet-stream"}, body) + require(response.get("VersionId") in {None, "null"}, "boundary fixture became versioned") + entry = {"key": key, "versionId": "null", "deleteMarker": False, "isLatest": True, + "payload": {"size": size, "sha256": hashlib.sha256(body).hexdigest()}} + expected.append(entry) + shape = "zero-metadata" if size == 0 else "inline" if size <= parameters["actualLogicalInlineLimit"] else "external" + for role in ("data", "parity"): + for kind in ("missing", "bitflip"): + instances.append({"name": f"{branch}-s{size}-{role}-{kind}", "sourceBranch": branch, + "size": size, "key": key, "role": role, "kind": kind, "expectedShape": shape, + "ecBlockSize": parameters["actualECBlockSize"]}) + return expected, instances + + +def verify_boundary_ranges(probe, fleet, bucket, expected, block): + proofs = [] + for entry in expected: + key, size = entry["key"], entry["payload"]["size"] + if size == 0: + denied = fleet.record("zero-range", probe.s3("get_object", { + "Bucket": bucket, "Key": key, "VersionId": "null", "Range": "bytes=0-0"})) + response = denied.get("response") or {} + require(denied.get("acknowledged") is False and denied.get("resultUnknown") is False + and "payload" not in denied and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 416 + and response.get("Error", {}).get("Code") == "InvalidRange", + "empty object Range must satisfy the frozen S3 416/InvalidRange contract") + proofs.append({"key": key, "zeroRangeRejected": True, "status": 416, "code": "InvalidRange"}) + continue + body = payload(f"{bucket}/{key}/0", size) + intervals = {(size - 1, size - 1), (0, min(size - 1, 46))} + if size > block: + intervals.add((max(0, block - 17), min(size - 1, block + 29))) + for start, end in sorted(intervals): + observed = fleet.record("size-range", probe.s3("get_object", { + "Bucket": bucket, "Key": key, "VersionId": "null", "Range": f"bytes={start}-{end}"})) + response = observed.get("response") or {} + require(observed.get("acknowledged") is True + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 206 + and response.get("ContentRange") == f"bytes {start}-{end}/{size}" + and observed.get("payload") == {"size": end - start + 1, + "sha256": hashlib.sha256(body[start:end + 1]).hexdigest()}, "actual size/stripe Range changed bytes or bounds") + proofs.append({"key": key, "start": start, "end": end, "httpStatus": 206}) + return proofs diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index f52008a4..3a445e1b 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -345,6 +345,8 @@ def snapshot(request): return {"hostname": owner["hostname"], "unit": owner["unit"], "pid": pid, "binarySha256": owner["binarySha256"], "quiescent": request.get("quiescent") is True, "backgroundConfig": None if request.get("quiescent") is True else BACKGROUND_CONFIG, + "storageClassEnvironment": None if request.get("quiescent") is True else { + key: environment.get(key) for key in ("RUSTFS_STORAGE_CLASS_INLINE_BLOCK", "RUSTFS_STORAGE_CLASS_STANDARD")}, "files": sorted(files, key=lambda value: (value["disk"], value["path"])), "formats": formats} diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 0989f807..aad932fd 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/prepare_heal_meta_inspector.py b/scripts/validation_heal/prepare_heal_meta_inspector.py index af2e5216..3ce52020 100644 --- a/scripts/validation_heal/prepare_heal_meta_inspector.py +++ b/scripts/validation_heal/prepare_heal_meta_inspector.py @@ -43,6 +43,56 @@ def quorum_contract(source: Path, commit: str) -> dict: "extractorSourceSha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest()} +def compact_rust(value: str) -> str: + value = re.sub(r"/\*.*?\*/|//[^\n]*", "", value, flags=re.S) + return re.sub(r"\s+", "", value) + + +def rust_body(value: str, signature: str) -> str: + start = value.find(signature) + if start < 0: + return "" + opening = value.find("{", start) + if opening < 0: + return "" + depth = 0 + for index in range(opening, len(value)): + depth += (value[index] == "{") - (value[index] == "}") + if depth == 0: + return value[opening:index + 1] + return "" + + +def size_contract(source: Path, commit: str) -> dict: + """Recognize the bounded untransformed PUT policy, preserving source digests.""" + paths = ("crates/ecstore/src/config/storageclass.rs", "crates/ecstore/src/set_disk/mod.rs", + "crates/ecstore/src/set_disk/ops/object.rs", "crates/ecstore/src/erasure/coding/erasure.rs") + contents = {path: (source / path).read_text() for path in paths} + storage, admission, put, erasure = (compact_rust(contents[path]) for path in paths) + default = re.search(r"pubstaticDEFAULT_INLINE_BLOCK:usize=([0-9]+)\*([0-9]+);", storage) + should = rust_body(storage, "pubfnshould_inline(") + budget = rust_body(storage, "pub(crate)fneffective_inline_block(") + shard = rust_body(erasure, "pubfncalc_shard_size(") + recognized = bool(default and + "constDEFAULT_INLINE_OBJECT_BUDGET:usize=2*DEFAULT_INLINE_BLOCK;" in storage and + should == "{ifshard_size<0||data_shards==0{returnfalse;}letshard_size=shard_sizeasusize;" + "letinline_block=self.effective_inline_block(data_shards);ifversioned{shard_size<=inline_block/8}" + "else{shard_size<=inline_block}}" and + budget == "{ifdata_shards==0{return0;}ifself.initialized&&self.inline_block_explicit{self.inline_block}" + "else{DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)}}" and + "ifstored_size>=0{returnerasure.shard_file_size(stored_size);}" in rust_body(admission, "fninline_admission_shard_size(") and + "letis_inline_buffer=storage_class_config.should_inline(inline_admission_shard_size(&erasure,put_object_size,data.actual_size())," + "erasure.data_shards,opts.versioned,)&&usize::try_from(inline_admission_size).is_ok_and(|size|size<=erasure.block_size);" in put and + shard == "{block_size.div_ceil(data_shards)}") + return {"candidateCommit": commit, "recognized": recognized, "scope": "known-size untransformed unversioned PUT; current shard codec", + "defaultInlineBlock": int(default[1]) * int(default[2]) if default else None, + "defaultObjectBudgetFactor": 2, "inlineEnvironment": "RUSTFS_STORAGE_CLASS_INLINE_BLOCK", + "inlineRule": "ceil(size/dataBlocks)<=effectiveInlineBlock AND size<=actualECBlock", + "effectiveDefaultRule": "min(ceil(2*defaultInlineBlock/dataBlocks),defaultInlineBlock)", + "sourceFiles": {path: hashlib.sha256(value.encode()).hexdigest() for path, value in contents.items()}, + "extractorSourceSha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest()} + + def prepare(source: Path, commit: str, output: Path) -> None: if not re.fullmatch(r"[0-9a-f]{40}", commit): raise ValueError("full candidate commit required") @@ -71,6 +121,7 @@ def prepare(source: Path, commit: str, output: Path) -> None: "candidateLockSha256": hashlib.sha256((source / "Cargo.lock").read_bytes()).hexdigest(), "helperSourceSha256": hashlib.sha256(text.encode()).hexdigest(), "quorumContract": quorum_contract(source, head), + "sizeContract": size_contract(source, head), "role": "read-only decoder; not the RustFS service under test", "built": False}, indent=2) + "\n") diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index cf019940..fbb814c0 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -19,7 +19,7 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request) ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 0b47d6e2..2f2a424d 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -23,6 +23,8 @@ from .healthy import Fleet, physical_content, validate_topology from .inspector import verify as verify_inspector from .matrix import select + from .boundaries import (BRANCHES as SIZE_BRANCHES, verify_size_contract, boundary_sizes, + seed_boundaries, verify_boundary_ranges) from .shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) except ImportError: @@ -31,6 +33,8 @@ from healthy import Fleet, physical_content, validate_topology from inspector import verify as verify_inspector from matrix import select + from boundaries import (BRANCHES as SIZE_BRANCHES, verify_size_contract, boundary_sizes, + seed_boundaries, verify_boundary_ranges) from shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) @@ -45,6 +49,7 @@ "heal.object-metadata-and-tags": ("user-metadata", "content-headers", "versioned-tags"), "heal.multiple-shard-loss": ("loss-1-data", "loss-1-parity", "loss-2-through-m-data-parity"), "heal.special-keys-and-pagination": ("unicode-space-percent", "encoded-slash", "adjacent-prefixes", "multi-page-list"), + "heal.object-size-boundaries": SIZE_BRANCHES, } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") @@ -294,7 +299,7 @@ def multi_loss_plan(instance, snapshot, bucket, expected, fixture): "requiredRepairs": {version_identity(data[0])}, "metadataRepairs": set(), "parameters": instance} -def map_inline_target(snapshot, bucket, entry): +def map_inline_target(snapshot, bucket, entry, *, role="data", allow_empty=False): validate_topology(snapshot, "H1" if len(snapshot) == 1 else "H2") indexed, cohort, authority = files(snapshot), [], None path = f"{bucket}/{entry['key']}/xl.meta" @@ -310,30 +315,43 @@ def map_inline_target(snapshot, bucket, entry): inline, erasure = version.get("inline") or {}, version["erasure"] layout = physical["value"]["xl"] slot = layout["sets"][0].index(layout["this"]) - require(version.get("deleted") is False and version.get("size") == entry["payload"]["size"] > 0 - and inline.get("rawOccurrenceCount") == 1 and type(inline.get("rawOffset")) is int - and type(inline.get("length")) is int and inline["length"] > 32 - and 0 <= inline["rawOffset"] < inline["rawOffset"] + inline["length"] <= file["size"] - and re.fullmatch(r"[0-9a-f]{64}", inline.get("sha256", "")) - and erasure["index"] == erasure["distribution"][slot], "inline location, digest or physical role is unsupported") - role = "data" if erasure["index"] <= erasure["data_blocks"] else "parity" - require(version.get("role") == role, "inline erasure index and role differ") + empty = allow_empty and entry["payload"]["size"] == 0 + require(version.get("deleted") is False and version.get("size") == entry["payload"]["size"] + and erasure["index"] == erasure["distribution"][slot], "inline identity or physical role is unsupported") + if empty: + require(not inline and version["parts"] and all(part["size"] == part["actualSize"] == 0 for part in version["parts"]) + and not any(identity[0] == node and identity[1] == physical["disk"] + and identity[2].startswith(f"{bucket}/{entry['key']}/") + and identity[2].rsplit("/", 1)[-1].startswith("part.") for identity in indexed), + "zero object unexpectedly has inline bytes or external payload parts") + else: + require(entry["payload"]["size"] > 0 and inline.get("rawOccurrenceCount") == 1 + and type(inline.get("rawOffset")) is int and type(inline.get("length")) is int and inline["length"] > 32 + and 0 <= inline["rawOffset"] < inline["rawOffset"] + inline["length"] <= file["size"] + and re.fullmatch(r"[0-9a-f]{64}", inline.get("sha256", "")), + "inline location or digest is unsupported") + actual_role = "data" if erasure["index"] <= erasure["data_blocks"] else "parity" + require(version.get("role") == actual_role, "inline erasure index and role differ") logical = {name: version.get(name) for name in ("versionId", "isLatest", "deleted", "dataDir", "size", "parts", "usesLegacyChecksum")} logical["erasure"] = {name: erasure[name] for name in ("data_blocks", "parity_blocks", "block_size", "distribution", "algorithm")} require(authority is None or logical == authority, "inline version or part geometry lacks full-set authority") authority = logical cohort.append({"node": node, "disk": physical["disk"], "relativePath": path, "versionId": entry["versionId"], "partNumber": 1, "dataDir": version.get("dataDir"), - "diskUUID": layout["this"], "setSlot": slot, "role": role, "index": erasure["index"], + "diskUUID": layout["this"], "setSlot": slot, "role": actual_role, "index": erasure["index"], "sha256": file["sha256"], "size": file["size"], "inlineValue": inline}) require(len(cohort) == authority["erasure"]["data_blocks"] + authority["erasure"]["parity_blocks"] and len({item["index"] for item in cohort}) == len(cohort), "inline set is incomplete or duplicated") - target = next(item for item in cohort if item["role"] == "data") + target = next(item for item in cohort if item["role"] == role) return {**target, "bucket": bucket, "key": entry["key"], "poolIndex": 0, "setIndex": 0, - "inlineProof": True, "cohort": cohort, "logical": authority} + "inlineProof": not empty, "zeroProof": empty, "cohort": cohort, "logical": authority} def verify_proof(fleet, target, label): + if target.get("zeroProof"): + return fleet.record(label, {"zeroLengthMetadataAuthorityVerified": True, "payloadBytes": 0, + "bitrotBlocksApplicable": False, "physicalPartInvented": False, + "proof": "decoded complete same-version metadata; physical no-part baseline; independent post-repair oracle"}) if not target.get("inlineProof"): return verify_cohort(fleet, target, label) proofs = [] @@ -347,6 +365,24 @@ def verify_proof(fleet, target, label): return {"allInlineShardsBitrotVerified": True, "originalInlineSha256Restored": True, "shards": proofs} +def boundary_plan(instance, snapshot, bucket, expected): + selected = [entry for entry in expected if entry["key"] == instance["key"]] + require(len(selected) == 1 and selected[0]["payload"]["size"] == instance["size"], "boundary lost its ACK size") + shape, kind = instance["expectedShape"], instance["kind"] + target = choose_target(snapshot, bucket, selected[0], instance["role"]) if shape == "external" else map_inline_target( + snapshot, bucket, selected[0], role=instance["role"], allow_empty=shape == "zero-metadata") + offset = None + if shape == "inline" and kind == "bitflip": + offset = target["inlineValue"]["rawOffset"] + 32 + if shape == "zero-metadata" and kind == "bitflip": + kind = "header" + metadata = shape != "external" and not (shape == "inline" and instance["kind"] == "bitflip") + return {"scope": {"bucket": bucket, "key": instance["key"]}, "recursive": False, + "expected": selected, "faults": [target], "proofs": [target], "kind": kind, "offset": offset, + "requiredRepairs": set() if metadata else {version_identity(selected[0])}, + "metadataRepairs": {instance["key"]} if metadata else set(), "parameters": instance} + + def suspend_fixture(recorder, bucket, historical): expected = copy.deepcopy(historical) recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Suspended"}}) @@ -555,11 +591,12 @@ def run_topology(args, topology, nodes, fixture): candidate = json.loads(args.candidate.read_text()) tool_manifest = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) contract = verify_quorum_contract(tool_manifest, candidate) if args.case_id == "heal.below-read-quorum" else None + size_rule = verify_size_contract(tool_manifest, candidate) if args.case_id == "heal.object-size-boundaries" else None bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, - "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45 if size_rule else 30)).isoformat(), "bucketPrefix": bucket, "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, @@ -577,10 +614,19 @@ def run_topology(args, topology, nodes, fixture): probe = Probe(binding) try: fleet.deploy(args.binary_sha, bucket, access, secret) - fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) + processes = fleet.snapshot("deployed-topology") + fleet.record("topology-proof", validate_topology(processes, topology)) recorder = Recorder(probe, directory / "fixture") - public = None - if fixture == "DK": + public, instances, sizes = None, None, None + if size_rule: + expected = seed(recorder, bucket, "DB", 1, [VERSION_SIZE]) + calibration = fleet.snapshot("size-calibration") + target = choose_target(calibration, bucket, expected[0], "data") + sizes = boundary_sizes(size_rule, target, processes) + fleet.record("candidate-size-contract", {"sourceContract": size_rule, "parameters": sizes}) + extra, instances = seed_boundaries(recorder, bucket, sizes) + expected += extra + elif fixture == "DK": expected = seed_special_keys(recorder, bucket) elif fixture == "DS": expected, public = seed_semantics(recorder, bucket) @@ -597,22 +643,27 @@ def run_topology(args, topology, nodes, fixture): fleet.record("special-key-pagination-baseline", verify_pagination(baseline)) if public is not None: fleet.record("semantic-baseline", verify_semantics(public, baseline)) - instances = None if args.case_id == "heal.multiple-shard-loss": fleet.stop() instances = multi_loss_instances(fleet.snapshot("multi-loss-geometry", quiescent=True), bucket, expected, fixture) fleet.record("multi-loss-parameter-matrix", {"instances": instances, "allLossCountsAndRoleCompositions": True}) branches = [entry["name"] for entry in instances] fleet.start() + if size_rule: + branches = [entry["name"] for entry in instances] + fleet.record("boundary-parameter-matrix", {"instances": instances, "sourceBranches": list(SIZE_BRANCHES)}) for branch in branches: registrations = [] metadata_faults = {} try: fleet.stop() before = fleet.snapshot("physical-before-" + branch, quiescent=True) - plan = multi_loss_plan(next(entry for entry in instances if entry["name"] == branch), - before, bucket, expected, fixture) if instances else branch_plan( - args.case_id, branch, before, bucket, expected, fixture) + if size_rule: + plan = boundary_plan(next(entry for entry in instances if entry["name"] == branch), before, bucket, expected) + elif instances: + plan = multi_loss_plan(next(entry for entry in instances if entry["name"] == branch), before, bucket, expected, fixture) + else: + plan = branch_plan(args.case_id, branch, before, bucket, expected, fixture) fleet.record("fault-plan-" + branch, {**plan, "requiredRepairs": sorted(plan["requiredRepairs"]), "metadataRepairs": sorted(plan["metadataRepairs"])}) for target in plan["proofs"]: @@ -670,6 +721,8 @@ def run_topology(args, topology, nodes, fixture): bitrot = [verify_proof(fleet, target, "repaired-bitrot-" + branch) for target in plan["proofs"]] fleet.start() s3 = verify_version_s3(probe, fleet, bucket, expected, baseline, "s3-after-" + branch, fixture=fixture) + if size_rule: + s3["actualBoundaryRanges"] = verify_boundary_ranges(probe, fleet, bucket, expected, sizes["actualECBlockSize"]) # These faults have now been independently repaired and fully # proved. Finalization discards their backups; it never writes # old metadata over a legitimate product re-encoding. @@ -778,6 +831,8 @@ def interrupted(signum, frame): fixtures = ("DS",) if args.case_id == "heal.special-keys-and-pagination": fixtures = ("DK",) + if args.case_id == "heal.object-size-boundaries": + fixtures = ("DB",) results = [] try: for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): diff --git a/test_validation_heal.py b/test_validation_heal.py index 462cd159..089aca61 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -22,9 +22,10 @@ branch_plan, suspend_fixture, validate_failed_version, validate_unrecoverable_get, verify_quorum_contract, map_inline_target, verify_proof) from scripts.validation_heal.versions import seed_semantics, verify_semantics -from scripts.validation_heal.versions import multi_loss_instances, multi_loss_plan, SPECIAL_KEYS, verify_pagination +from scripts.validation_heal.versions import multi_loss_instances, multi_loss_plan, SPECIAL_KEYS, verify_pagination, boundary_plan from scripts.validation_heal.heal_probe import Probe -from scripts.validation_heal.prepare_heal_meta_inspector import quorum_contract +from scripts.validation_heal.prepare_heal_meta_inspector import quorum_contract, size_contract +from scripts.validation_heal.boundaries import verify_size_contract, boundary_sizes, verify_boundary_ranges class HealContractTest(unittest.TestCase): @@ -390,6 +391,94 @@ def test_fault_requires_actual_service_quiescence_and_exact_truncate_length(self class HealVersionOracleTest(unittest.TestCase): + def test_size_contract_derives_distinct_actual_inline_limits_and_rejects_changed_policy(self): + source = { + "crates/ecstore/src/config/storageclass.rs": '''pub static DEFAULT_INLINE_BLOCK: usize = 128 * 1024; + const DEFAULT_INLINE_OBJECT_BUDGET: usize = 2 * DEFAULT_INLINE_BLOCK; + pub fn should_inline(&self, shard_size: i64, data_shards: usize, versioned: bool) -> bool { + if shard_size < 0 || data_shards == 0 { return false; } + let shard_size = shard_size as usize; let inline_block = self.effective_inline_block(data_shards); + if versioned { shard_size <= inline_block / 8 } else { shard_size <= inline_block } + } + pub(crate) fn effective_inline_block(&self, data_shards: usize) -> usize { + if data_shards == 0 { return 0; } + if self.initialized && self.inline_block_explicit { self.inline_block } + else { DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK) } + }''', + "crates/ecstore/src/set_disk/mod.rs": '''fn inline_admission_shard_size(erasure: &coding::Erasure, stored_size: i64, actual_size: i64) -> i64 { + if stored_size >= 0 { return erasure.shard_file_size(stored_size); } actual_size }''', + "crates/ecstore/src/set_disk/ops/object.rs": '''let is_inline_buffer = storage_class_config.should_inline( + inline_admission_shard_size(&erasure, put_object_size, data.actual_size()), erasure.data_shards, opts.versioned, + ) && usize::try_from(inline_admission_size).is_ok_and(|size| size <= erasure.block_size);''', + "crates/ecstore/src/erasure/coding/erasure.rs": '''pub fn calc_shard_size(block_size: usize, data_shards: usize) -> usize { + block_size.div_ceil(data_shards) }'''} + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for path, text in source.items(): + file = root / path + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(text) + contract = size_contract(root, "a" * 40) + verify_size_contract({"sizeContract": contract}, {"source_sha": "a" * 40}) + processes = {"node": {"storageClassEnvironment": { + "RUSTFS_STORAGE_CLASS_INLINE_BLOCK": None, "RUSTFS_STORAGE_CLASS_STANDARD": None}}} + for data, inline in ((2, 262144), (12, 262152)): + target = {"logical": {"usesLegacyChecksum": False, "erasure": {"data_blocks": data, "block_size": 1048576}}} + parameters = boundary_sizes(contract, target, processes) + self.assertEqual(parameters["actualLogicalInlineLimit"], inline) + self.assertEqual(parameters["groups"]["inline-minus-exact-plus"], [inline - 1, inline, inline + 1]) + self.assertEqual(parameters["groups"]["stripe-minus-exact-plus"], [1048575, 1048576, 1048577]) + with self.assertRaises(ValueError): boundary_sizes(contract, target, {"node": {"storageClassEnvironment": {}}}) + for field, value in (("recognized", False), ("extractorSourceSha256", "0" * 64), ("candidateCommit", "b" * 40)): + with self.assertRaises(ValueError): + verify_size_contract({"sizeContract": {**contract, field: value}}, {"source_sha": "a" * 40}) + path = root / "crates/ecstore/src/erasure/coding/erasure.rs" + path.write_text(source[str(path.relative_to(root))].replace("div_ceil", "div_floor")) + self.assertFalse(size_contract(root, "a" * 40)["recognized"]) + + def test_zero_boundary_faults_metadata_and_never_invents_a_part(self): + snapshot = HealShardOracleTest().snapshot() + observation = snapshot["node1"] + observation["files"] = [entry for entry in observation["files"] if "metadata" in entry] + for entry in observation["files"]: + version = entry["metadata"]["versions"][0] + version["size"] = 0 + version["parts"][0].update(size=0, actualSize=0) + expected = [{"key": "objects/0008.bin", "versionId": "null", "deleteMarker": False, + "payload": {"size": 0, "sha256": hashlib.sha256(b"").hexdigest()}}] + instance = {"key": expected[0]["key"], "size": 0, "role": "parity", "expectedShape": "zero-metadata", "kind": "bitflip"} + plan = boundary_plan(instance, snapshot, "rv-heal-0123456789abcdef", expected) + self.assertEqual(plan["kind"], "header") + self.assertTrue(plan["proofs"][0]["zeroProof"]) + self.assertEqual(plan["faults"][0]["role"], "parity") + self.assertTrue(plan["faults"][0]["relativePath"].endswith("/xl.meta")) + observation["files"].append(next(entry for entry in HealShardOracleTest().snapshot()["node1"]["files"] if "metadata" not in entry)) + with self.assertRaisesRegex(ValueError, "zero object"): + boundary_plan(instance, snapshot, "rv-heal-0123456789abcdef", expected) + + def test_actual_size_ranges_require_206_bytes_and_empty_range_requires_exact_416(self): + from scripts.validation_heal.heal_probe import payload + bucket, block = "fixture", 4096 + expected = [{"key": str(size), "payload": {"size": size}} for size in (0, 1, 4097)] + calls = [] + def s3(operation, params): + size = int(params["Key"]) + calls.append(params) + if not size: + return {"acknowledged": False, "resultUnknown": False, "response": { + "ResponseMetadata": {"HTTPStatusCode": 416}, "Error": {"Code": "InvalidRange"}}} + start, end = map(int, params["Range"].removeprefix("bytes=").split("-")) + value = payload(f"{bucket}/{params['Key']}/0", size)[start:end + 1] + return {"acknowledged": True, "response": {"ResponseMetadata": {"HTTPStatusCode": 206}, + "ContentRange": f"bytes {start}-{end}/{size}"}, + "payload": {"size": len(value), "sha256": hashlib.sha256(value).hexdigest()}} + fleet = types.SimpleNamespace(record=lambda label, value: value) + verify_boundary_ranges(types.SimpleNamespace(s3=s3), fleet, bucket, expected, block) + self.assertTrue(any(params["Range"] == "bytes=4079-4096" for params in calls)) + for wrong in ({"acknowledged": False, "resultUnknown": True}, {"acknowledged": True, "payload": {"size": 0}}): + with self.assertRaises(ValueError): + verify_boundary_ranges(types.SimpleNamespace(s3=lambda *args: wrong), fleet, bucket, expected[:1], block) + def test_explicit_small_list_pages_preserve_distinct_special_keys(self): bucket = "rv-heal-pages-unit" keys = [*SPECIAL_KEYS, *(f"pages/{index:04d}.bin" for index in range(257))] From e8991fb8476a05f076352ef10ee3071ffaf172a7 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 23:05:28 +0800 Subject: [PATCH 041/343] docs(validation): record complete suspended null live proof --- docs/validation-migration-map.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index a33a1a53..d2118f5e 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 95 currently dispatchable IDs include all 74 non-Heal cases and twenty-one Heal cases. Forty have live PASS evidence; the other 118 have no live PASS claim. +case evidence. The 95 currently dispatchable IDs include all 74 non-Heal cases and twenty-one Heal cases. Forty-one have live PASS evidence; the other 117 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -328,3 +328,10 @@ metadata loss/header faults and never invent a payload part. Exact byte hashes, complete responsibility, HEAD/LIST and actual first/tail/stripe Range checks are required; zero Range is a strict S3 416/InvalidRange negative. D01 requires a fresh tool artifact carrying this size contract and has not yet run live. + +Corrected D05 run `38061843290` passed every H1/H2 suspended/null branch and +clean finalization. Real enabled-period history, null overwrites and markers +retain precise S3 identities; the typed Admin nil UUID is mapped only to that +acknowledged null version. The candidate then re-enabled versioning and created +a distinct non-null version without changing old payloads or version/marker +identities. The original literal-identity harness failure is still retained. From f4ddaa967ac6a7bd9b42f1704d37e831576f055c Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 23:28:38 +0800 Subject: [PATCH 042/343] fix(validation): bound fault IDs and audit rejected fault cleanup --- scripts/validation_heal/fleet_node.py | 24 ++++- scripts/validation_heal/heal_fault.py | 6 ++ .../validation_heal/recover_environment.py | 93 +++++++++++++++++++ scripts/validation_heal/versions.py | 4 +- test_validation_heal_recovery.py | 54 +++++++++++ 5 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 scripts/validation_heal/recover_environment.py create mode 100644 test_validation_heal_recovery.py diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 3a445e1b..c003afbe 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -370,10 +370,32 @@ def cleanup(request): return {"cleanup": "PASS", "unitStopped": True, "rootsRemoved": True, "hostname": owner["hostname"]} +def audit_finalize(request): + """Finalize only an invalid-ID rejection proved identical to its archived pre-fault scene.""" + _, _, owner = owned(request["runId"]) + stopped(owner) + fault_id = request.get("rejectedFaultId", "") + require(re.fullmatch(r"[A-J][0-9]{2}-[a-z0-9-]{49,128}", fault_id) is not None, + "audited finalization only supports an overlength ID rejected before injection") + require(request.get("binarySha256") == owner["binarySha256"], "audited candidate identity differs") + root, _ = fault_root(request) + for suffix in (".intent.json", ".original", ".applied.json"): + require(not checked(root / "quarantine" / (fault_id + suffix), root).exists(), + "rejected fault unexpectedly has intent or backup bytes; retain scene") + observed = snapshot({**request, "quiescent": True}) + content = {key: observed[key] for key in ("files", "formats")} + scene_sha = hashlib.sha256(json.dumps(content, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + require(re.fullmatch(r"[0-9a-f]{64}", request.get("expectedSceneSha256", "")) is not None + and scene_sha == request["expectedSceneSha256"], "physical scene drifted from archived pre-fault evidence; retain namespace") + result = cleanup(request) + return {**result, "faultNeverApplied": True, "archivedPreFaultSceneEqual": True, "sceneSha256": scene_sha, + "backupRestorationUsed": False, "productRepairClaimed": False} + + def main(request): require(os.geteuid() == 0, "node adapter requires passwordless sudo") actions = {"preflight": preflight, "reserve": reserve, "start": start, "stop": stop, "snapshot": snapshot, - "logs": logs, "cleanup": cleanup, "install-inspector": install_inspector, + "logs": logs, "cleanup": cleanup, "audit-finalize": audit_finalize, "install-inspector": install_inspector, "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part, "verify-inline": verify_inline} require(request.get("action") in actions, "unsupported node operation") diff --git a/scripts/validation_heal/heal_fault.py b/scripts/validation_heal/heal_fault.py index 358a95d2..6fd00974 100644 --- a/scripts/validation_heal/heal_fault.py +++ b/scripts/validation_heal/heal_fault.py @@ -24,6 +24,12 @@ FAULTS = ("missing", "bitflip", "truncate", "zero", "header") +def bounded_fault_id(code: str, branch: str, index: int) -> str: + require(bool(re.fullmatch(r"[A-J][0-9]{2}", code)) and isinstance(branch, str) and branch + and type(index) is int and index >= 0, "invalid fault identity inputs") + return code + "-" + hashlib.sha256(f"{branch}/{index}".encode()).hexdigest()[:32] + + def require(condition: bool, message: str) -> None: if not condition: raise ValueError(message) diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py new file mode 100644 index 00000000..4e3390d5 --- /dev/null +++ b/scripts/validation_heal/recover_environment.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Audit a failed run's rejected overlength fault against its archived physical scene.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re + +try: + from .healthy import Fleet, physical_content + from .heal_probe import require, save +except ImportError: + from healthy import Fleet, physical_content + from heal_probe import require, save + + +def recovery_plan(artifact, failed_run, candidate): + require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") + results = list(artifact.glob("*/run/heal.object-size-boundaries/cases/heal.object-size-boundaries.json")) + require(len(results) == 1, "failed artifact must have one exact D01 canonical result") + result = json.loads(results[0].read_text()) + require(result["verdict"] == "FAIL" and result["cleanup_status"] == "FAIL" + and result["provenance"]["source_commit"] == candidate["source_sha"] + and result["provenance"]["candidate_sha256"] == candidate["package_sha256"], "failed result candidate or cleanup identity differs") + evidence = results[0].parents[1] / "evidence/heal.object-size-boundaries" + summary = json.loads((evidence / "healthy-summary.json").read_text()) + plans = [] + for topology in summary["results"]: + if topology["cleanup"] == "PASS": + continue + require(topology["topology"] in {"H1", "H2"} and topology["fixture"] == "DB" + and "invalid fault ID" in topology.get("failure", {}).get("message", ""), "cleanup failure needs a different recovery procedure") + failed = [(name, branch) for name, branch in topology["branches"].items() + if "invalid fault ID" in branch.get("failure", {}).get("message", "")] + require(len(failed) == 1, "cannot identify one rejected fault branch") + branch, _ = failed[0] + folder = evidence / "DB" / topology["topology"] + files = list(folder.glob("*-physical-before-" + branch + ".json")) + faults = list(folder.glob("*-fault-plan-" + branch + ".json")) + bindings = list(folder.glob("*-binding.json")) + require(len(files) == len(faults) == len(bindings) == 1, "recovery lacks exact archived pre-fault/binding/plan evidence") + before, fault, binding = (json.loads(path.read_text()) for path in (files[0], faults[0], bindings[0])) + require(re.fullmatch(rf"gh-{failed_run}-[1-9][0-9]*-{topology['topology'].lower()}", binding["runId"]) is not None, + "archived run namespace differs from failed Actions run") + require(len(fault["faults"]) == 1, "rejected-ID recovery cannot omit earlier faults in the same branch") + target = fault["faults"][0] + rejected_id = "D01-" + branch + "-0" + require(re.fullmatch(r"D01-[a-z0-9-]{49,128}", rejected_id) is not None, "failed ID was not rejected for excessive length") + require(target["node"] in before and target["sha256"] and binding["candidateBinarySha256"] == result["provenance"]["binary_sha256"], + "fault/candidate physical identity differs") + nodes = list(before) + require(all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "invalid archived node name") + plans.append({"runId": binding["runId"], "nodes": nodes, "node": target["node"], "disk": target["disk"], + "rejectedFaultId": rejected_id, "binarySha256": binding["candidateBinarySha256"], "before": before, + "expectedSceneSha256": hashlib.sha256(json.dumps(physical_content(before)[target["node"]], + sort_keys=True, separators=(",", ":")).encode()).hexdigest()}) + require(plans, "failed artifact has no supported retained environment") + return plans + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--failed-run-id", type=int, required=True) + parser.add_argument("--artifact", type=Path, required=True) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + args = parser.parse_args() + plans = recovery_plan(args.artifact, args.failed_run_id, json.loads(args.candidate.read_text())) + os.environ["RUSTFS_SSH_USER"] = os.environ.get("RUSTFS_SSH_USER") or "azureuser" + assigned = (os.environ.get("RUSTFS_NODES") or "vm000 vm001 vm002 vm003").split() + require(all(set(plan["nodes"]) <= set(assigned) for plan in plans), "archived nodes do not belong to the current auto-testing fleet") + args.evidence.mkdir(parents=True, exist_ok=False) + complete = [] + for plan in plans: + fleet = Fleet(plan["nodes"], args.evidence, plan["runId"]) + fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) + current = fleet.snapshot("audited-physical-scene", quiescent=True) + require(physical_content(current) == physical_content(plan["before"]), "retained fleet drifted from archived pre-fault scene; no cleanup performed") + fleet.record("retained-service-logs", fleet.remote(plan["node"], "logs")) + proof = fleet.record("audited-finalization", fleet.remote(plan["node"], "audit-finalize", **{ + key: plan[key] for key in ("disk", "rejectedFaultId", "binarySha256", "expectedSceneSha256")})) + for node in plan["nodes"]: + if node != plan["node"]: + fleet.record("unchanged-peer-finalization", fleet.remote(node, "cleanup")) + require(proof.get("cleanup") == "PASS", "audited environment finalization failed") + complete.append({"failedRunId": args.failed_run_id, "runId": plan["runId"], "cleanup": "PASS", "originalVerdictRewritten": False}) + save(args.evidence / "recovery-summary.json", {"cleanup": "PASS", "environments": complete, "canonicalPassClaimed": False}) + + +if __name__ == "__main__": + main() diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 2f2a424d..1cb0227a 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -23,6 +23,7 @@ from .healthy import Fleet, physical_content, validate_topology from .inspector import verify as verify_inspector from .matrix import select + from .heal_fault import bounded_fault_id from .boundaries import (BRANCHES as SIZE_BRANCHES, verify_size_contract, boundary_sizes, seed_boundaries, verify_boundary_ranges) from .shards import (SIZES, files, map_target, run_task, validate_fault, @@ -33,6 +34,7 @@ from healthy import Fleet, physical_content, validate_topology from inspector import verify as verify_inspector from matrix import select + from heal_fault import bounded_fault_id from boundaries import (BRANCHES as SIZE_BRANCHES, verify_size_contract, boundary_sizes, seed_boundaries, verify_boundary_ranges) from shards import (SIZES, files, map_target, run_task, validate_fault, @@ -670,7 +672,7 @@ def run_topology(args, topology, nodes, fixture): verify_proof(fleet, target, "baseline-bitrot-" + branch) prior = before for index, target in enumerate(plan["faults"]): - params = {"disk": target["disk"], "faultId": recipe["code"] + "-" + branch + "-" + str(index)} + params = {"disk": target["disk"], "faultId": bounded_fault_id(recipe["code"], branch, index)} registration = target["node"], params fleet.faults.append(registration) registrations.append(registration) diff --git a/test_validation_heal_recovery.py b/test_validation_heal_recovery.py new file mode 100644 index 00000000..2b84bd95 --- /dev/null +++ b/test_validation_heal_recovery.py @@ -0,0 +1,54 @@ +"""A rejected fault ID cannot bypass ownership, intent or physical-scene checks.""" + +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from scripts.validation_heal.heal_fault import bounded_fault_id +from scripts.validation_heal.fleet_node import audit_finalize + + +class RejectedFaultRecoveryTest(unittest.TestCase): + def test_expanded_branches_have_bounded_distinct_fault_ids(self): + branch = "stripe-minus-exact-plus-s1048575-parity-missing" + original = "D01-" + branch + "-0" + self.assertEqual(len(original.split("-", 1)[1]), 49) + identifier = bounded_fault_id("D01", branch, 0) + self.assertRegex(identifier, r"^D01-[a-z0-9-]{1,48}$") + self.assertNotEqual(identifier, bounded_fault_id("D01", branch, 1)) + self.assertNotEqual(identifier, bounded_fault_id("D01", branch + "-another", 0)) + with self.assertRaises(ValueError): bounded_fault_id("bad", branch, 0) + + def test_finalization_needs_no_intent_and_exact_archived_scene(self): + snapshot = {"files": [{"sha256": "actual", "path": "owned/key/xl.meta"}], "formats": [{"disk": "original"}]} + scene = hashlib.sha256(json.dumps(snapshot, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + request = {"runId": "gh-12345-1-h1", "disk": 0, "binarySha256": "a" * 64, + "rejectedFaultId": "D01-" + "a" * 49, "expectedSceneSha256": scene} + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + (root / "quarantine").mkdir() + with patch("scripts.validation_heal.fleet_node.owned", return_value=(None, None, {"binarySha256": "a" * 64})), \ + patch("scripts.validation_heal.fleet_node.stopped"), \ + patch("scripts.validation_heal.fleet_node.fault_root", return_value=(root, {})), \ + patch("scripts.validation_heal.fleet_node.snapshot", return_value=snapshot), \ + patch("scripts.validation_heal.fleet_node.cleanup", return_value={"cleanup": "PASS"}) as cleanup: + result = audit_finalize(request) + self.assertTrue(result["faultNeverApplied"]) + self.assertFalse(result["productRepairClaimed"]) + cleanup.reset_mock() + for changed in ({**request, "expectedSceneSha256": "b" * 64}, + {**request, "binarySha256": "b" * 64}, + {**request, "rejectedFaultId": "D01-valid-length"}): + with self.assertRaises(ValueError): audit_finalize(changed) + cleanup.assert_not_called() + (root / "quarantine" / (request["rejectedFaultId"] + ".intent.json")).write_text("intent") + with self.assertRaisesRegex(ValueError, "intent"): + audit_finalize(request) + cleanup.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From b2f099d89c7261f07d4aa4e0fbe9aed8e2732327 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 23:40:26 +0800 Subject: [PATCH 043/343] feat(validation): exercise Heal authorization and principal cleanup --- .../workflows/validation-catalog-tests.yml | 7 +- docs/validation-heal-contract.md | 2 +- docs/validation-migration-map.md | 34 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/authorization.py | 204 ++++++++++++++++++ scripts/validation_heal/heal_probe.py | 31 ++- scripts/validation_heal/heal_signing.py | 16 +- scripts/validation_heal/healthy.py | 23 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- test_validation_heal_authorization.py | 194 +++++++++++++++++ 11 files changed, 496 insertions(+), 21 deletions(-) create mode 100644 scripts/validation_heal/authorization.py create mode 100644 test_validation_heal_authorization.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 5b544b62..a478a607 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -16,6 +16,8 @@ on: - test_validation_replication.py - scripts/validation_replication/** - test_validation_heal.py + - test_validation_heal_recovery.py + - test_validation_heal_authorization.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -45,6 +47,8 @@ on: - test_validation_replication.py - scripts/validation_replication/** - test_validation_heal.py + - test_validation_heal_recovery.py + - test_validation_heal_authorization.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -75,7 +79,8 @@ jobs: - run: python3 -B -m unittest -v test_validation_ilm.py - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index 138ad438..0caefabf 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -12 个用例已接入自动分发：A01、A03、A04、A07、C01–C04、D02、D07、E01、E02。分片执行器保留新增 7 项的全部 24 个源分支和 H1/H2 拓扑，使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。D07 保留七个 HEAD/Range/ETag/时间条件分支，先冻结健康请求的具体 200/206/304/412/416 预期，再在物理修复后重复真实请求。其余 72 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 +22 个用例已接入自动分发：A01、A03、A04、A07、A08、B05、B07、C01–C05、D01–D03、D05–D07、D10、E01、E02、J04。分片执行器使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。A08 保留五种主体和各自 start/status/stop 操作，使用真正重新计算的过期签名及仅授权 `admin:Heal` 的临时用户，清理后验证凭据失效。其余 62 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index d2118f5e..dc6fe0e3 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 95 currently dispatchable IDs include all 74 non-Heal cases and twenty-one Heal cases. Forty-one have live PASS evidence; the other 117 have no live PASS claim. +case evidence. The 96 currently dispatchable IDs include all 74 non-Heal cases and twenty-two Heal cases. Forty-one have live PASS evidence; the other 117 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 63 Heal cases remain +The other 62 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -326,8 +326,9 @@ policy (the runtime derives these rather than guessing 256KiB). Each size tests both actual data/parity roles with missing and bitflip faults. Zero objects use metadata loss/header faults and never invent a payload part. Exact byte hashes, complete responsibility, HEAD/LIST and actual first/tail/stripe Range checks -are required; zero Range is a strict S3 416/InvalidRange negative. D01 requires a -fresh tool artifact carrying this size contract and has not yet run live. +are required; zero Range is a strict S3 416/InvalidRange negative. Tool build +`38062137032` successfully produced the recognized size and quorum contracts +for the fixed candidate, with current extractor/source and binary identities. Corrected D05 run `38061843290` passed every H1/H2 suspended/null branch and clean finalization. Real enabled-period history, null overwrites and markers @@ -335,3 +336,28 @@ retain precise S3 identities; the typed Admin nil UUID is mapped only to that acknowledged null version. The candidate then re-enabled versioning and created a distinct non-null version without changing old payloads or version/marker identities. The original literal-identity harness failure is still retained. + +A08 `heal.admin-authorization` now dispatches every H1/H2 source principal +branch. Anonymous, deliberately bad, correctly signed but expired, and denied +IAM requests each test start/status/stop with exact HTTP/error refusals and +recursive JSON/XML task-disclosure rejection. A temporary principal with only +`admin:Heal` admits, queries and stops its independent token while S3 reads stay +denied. Temporary users/policy/tasks are finalized and credential revocation is +verified; original physical/S3 fixtures must remain unchanged. Evidence redacts +root and temporary credentials without altering oracle inputs. Nine focused +tests cover independent SigV4 verification, denial leakage, lost IAM admission, +cleanup failures and revocation. A08 still requires full live evidence. + +D01 run `38062610870` retained FAIL after 35 H1 parameter attempts. Three +physical comparisons rejected changed target-side `xl.meta.bkp` files; archived +per-drive comparison shows each new backup is the exact pre-repair authoritative +`xl.meta`, while payload shards and decoded logical identities agree. The +original verdict remains retained pending a new complete run with a bounded +target-backup oracle. A fourth attempt was rejected before injection because +its expanded fault ID exceeded the tool limit; retrying that invalid ID during +finalization caused cleanup FAIL. Fault IDs now use branch/index digests within +the allowed length. Recovery run `38063798238` independently proved stopped +owned service, no injection intent, exact archived pre-fault physical scene and +removed run roots. It records cleanup PASS, `originalVerdictRewritten=false` +and no canonical/product-repair PASS claim. Serial deployment resumed only +after inspecting that recovery evidence. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 5d56f373..7231e419 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/authorization.py b/scripts/validation_heal/authorization.py new file mode 100644 index 00000000..b5c0ad72 --- /dev/null +++ b/scripts/validation_heal/authorization.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""A08 exercises actual IAM principals and every Heal control operation.""" + +import hashlib +import json +import secrets +import time +from urllib import request +from urllib.error import HTTPError +from urllib.parse import quote, urlencode +import xml.etree.ElementTree as ET + +try: + from .heal_probe import Probe, compare_inventories, require, validate_binding + from .heal_signing import sign_admin_request + from .healthy import physical_content +except ImportError: + from heal_probe import Probe, compare_inventories, require, validate_binding + from heal_signing import sign_admin_request + from healthy import physical_content + + +DENIALS = {"anonymous": (400, "InvalidRequest"), "bad-signature": (403, "SignatureDoesNotMatch"), + "expired-signature": (403, "RequestTimeTooSkewed"), "denied-user": (403, "AccessDenied")} + + +def response_code(observation): + body = observation.get("response") + if isinstance(body, dict): + return body.get("Code") or body.get("code") + if isinstance(body, str): + return next((element.text for element in ET.fromstring(body).iter() + if element.tag.rsplit("}", 1)[-1] == "Code"), None) + return None + + +def validate_denial(observation, mode): + status, code = DENIALS[mode] + require(observation.get("httpStatus") == status and response_code(observation) == code, + f"{mode} did not satisfy its frozen {status}/{code} refusal") + body = observation.get("response") + forbidden = {"outcome", "objects", "progress", "clienttoken"} + def disclosed(value): + if isinstance(value, dict): + return any(str(key).lower() in forbidden or disclosed(item) for key, item in value.items()) + if isinstance(value, list): + return any(disclosed(item) for item in value) + return False + if isinstance(body, str): + leaked = any(element.tag.rsplit("}", 1)[-1].lower() in forbidden for element in ET.fromstring(body).iter()) + else: + leaked = disclosed(body) + require(not leaked, "unauthorized response disclosed task responsibility or admitted a token") + return {"status": status, "code": code, "noTaskDisclosure": True} + + +def iam_exchange(probe, fleet, operation, user, *, secret=None, policy=None, document=None): + from botocore.awsrequest import AWSRequest + validate_binding(probe.binding) + require(user.startswith("rvheal") and user.isalnum(), "IAM principal is outside the run namespace") + routes = { + "add-user": ("PUT", "add-user", {"accessKey": user}, {"secretKey": secret, "status": "enabled"}), + "remove-user": ("DELETE", "remove-user", {"accessKey": user}, None), + "add-policy": ("PUT", "add-canned-policy", {"name": policy}, document), + "remove-policy": ("DELETE", "remove-canned-policy", {"name": policy}, None), + "attach": ("POST", "idp/builtin/policy/attach", {}, {"policies": [policy], "user": user}), + "detach": ("POST", "idp/builtin/policy/detach", {}, {"policies": [policy], "user": user}), + } + require(operation in routes and (policy is None or policy.startswith("rvheal") and policy.isalnum()), "invalid owned IAM operation") + method, route, query, body = routes[operation] + raw = json.dumps(body).encode() if body is not None else b"" + url = probe.endpoint + "/rustfs/admin/v3/" + route + ("?" + urlencode(query) if query else "") + outgoing = AWSRequest(method=method, url=url, data=raw, + headers={"Content-Type": "application/json", "x-amz-content-sha256": "UNSIGNED-PAYLOAD"}) + outgoing.context["payload_signing_enabled"] = False + sign_admin_request(outgoing, *probe.credentials()) + try: + response = probe._open_admin(request.Request(url, data=raw if raw else None, method=method, + headers=dict(outgoing.headers.items()))) + except HTTPError as exc: + response = exc + with response: + returned = response.read(64 * 1024 + 1) + require(len(returned) <= 64 * 1024, "IAM response exceeds evidence budget") + observed = {"operation": operation, "httpStatus": response.code, + "principal": "run-owned", "response": returned.decode(errors="replace"), + "requestBodyArchived": False, "credentialValuesArchived": False} + fleet.record("iam-" + operation, observed) + require(observed["httpStatus"] == 200, "owned IAM operation failed: " + operation) + return observed + + +def admit(probe, fleet, path, options, active): + started = probe.admin(path, [], json.dumps(options).encode()) + token = (started.get("response") or {}).get("clientToken") if isinstance(started.get("response"), dict) else None + if isinstance(token, str) and token: + active.append((path, token)) + fleet.record("authorization-admission", started) + require(started.get("httpStatus") == 200 and token, "authorized principal could not admit its own task") + return token + + +def authorization_requests(probe, fleet, bucket, before, baseline, active, options): + suffix = hashlib.sha256(probe.binding["runId"].encode()).hexdigest()[:16] + denied_user, allowed_user, policy = ("rvheal" + suffix + ending for ending in ("denied", "allowed", "policy")) + denied_secret, allowed_secret = secrets.token_hex(24), secrets.token_hex(24) + fleet.secret_values.update({*probe.credentials(), denied_user, allowed_user, denied_secret, allowed_secret}) + denied = Probe(probe.binding, credential_pair=(denied_user, denied_secret)) + allowed = Probe(probe.binding, credential_pair=(allowed_user, allowed_secret)) + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + task_options = {**options, "recursive": True, "dryRun": True} + branches, cleanup_errors = {}, [] + users, policy_created, attached = [], False, False + try: + for user, secret in ((denied_user, denied_secret), (allowed_user, allowed_secret)): + users.append(user) # Preserve ownership even if admission loses its reply. + iam_exchange(probe, fleet, "add-user", user, secret=secret) + policy_created = True + document = {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Action": ["admin:Heal"], "Resource": ["*"]}]} + iam_exchange(probe, fleet, "add-policy", allowed_user, policy=policy, document=document) + attached = True + iam_exchange(probe, fleet, "attach", allowed_user, policy=policy) + for principal in (denied, allowed): + result = fleet.record("least-privilege-s3-denial", principal.s3("get_object", {"Bucket": bucket, "Key": "objects/0008.bin"})) + response = result.get("response") or {} + require(result.get("acknowledged") is False and result.get("resultUnknown") is False and "payload" not in result + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 403 + and response.get("Error", {}).get("Code") == "AccessDenied", "Heal-only/denied principal unexpectedly has S3 read access") + root_token = admit(probe, fleet, path, task_options, active) + for mode in DENIALS: + proofs, failures = {}, [] + client = denied if mode == "denied-user" else probe + auth = "signed" if mode == "denied-user" else mode + for operation, query, raw in ( + ("start", [], json.dumps(task_options).encode()), + ("status", [("clientToken", root_token)], b""), + ("stop", [("clientToken", root_token), ("forceStop", "true")], b"")): + try: + observed = client.admin(path, query, raw, auth=auth) + token = (observed.get("response") or {}).get("clientToken") if isinstance(observed.get("response"), dict) else None + if token and (path, token) not in active: + active.append((path, token)) + fleet.record("auth-" + mode + "-" + operation, observed) + proofs[operation] = validate_denial(observed, mode) + except Exception as exc: + failures.append({"operation": operation, "type": type(exc).__name__, "message": str(exc)}) + branches[mode] = {"verdict": "FAIL" if failures else "PASS", "operations": proofs, "failures": failures} + fleet.record("authorization-negative-branches", branches) + deadline = time.monotonic() + 30 + while True: + root_status = fleet.record("root-task-status-after-denials", probe.admin(path, [("clientToken", root_token)], b"")) + require(root_status.get("httpStatus") == 200 and isinstance(root_status.get("response"), dict), "administrator lost its original token after denied requests") + if root_status["response"].get("summary") == "finished": + break + require(root_status["response"].get("summary") == "running" and time.monotonic() < deadline, + "denied control requests stopped the original task or it did not finish before the independent authorized task") + time.sleep(0.2) + token = admit(allowed, fleet, path, task_options, active) + queried = fleet.record("authorized-status", allowed.admin(path, [("clientToken", token)], b"")) + require(queried.get("httpStatus") == 200 and isinstance(queried.get("response"), dict) + and queried["response"].get("summary") in {"running", "finished", "stopped"}, "Heal-only principal cannot query its admitted token") + stopped = fleet.record("authorized-stop", allowed.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + require(stopped.get("httpStatus") == 200 and isinstance(stopped.get("response"), dict), "Heal-only principal cannot stop its admitted token") + require(stopped["response"].get("summary") in {"stopped", "finished"}, + "Heal-only stop did not return terminal task control") + branches["least-privilege-user"] = {"verdict": "PASS", "admissionStatusStop": True, + "s3ReadDenied": True, "policyActions": ["admin:Heal"]} + finally: + for task_path, token in list(active): + try: + observed = fleet.record("auth-final-stop", probe.admin(task_path, [("clientToken", token), ("forceStop", "true")], b"")) + require(observed.get("httpStatus") == 200, "authorization task finalization failed") + active.remove((task_path, token)) + except Exception as exc: + cleanup_errors.append(str(exc)) + if attached: + try: + iam_exchange(probe, fleet, "detach", allowed_user, policy=policy) + except Exception as exc: + cleanup_errors.append(str(exc)) + for user in users: + try: + iam_exchange(probe, fleet, "remove-user", user) + except Exception as exc: + cleanup_errors.append(str(exc)) + if policy_created: + try: + iam_exchange(probe, fleet, "remove-policy", allowed_user, policy=policy) + except Exception as exc: + cleanup_errors.append(str(exc)) + fleet.record("iam-finalization", {"cleanup": "FAIL" if cleanup_errors else "PASS", "errors": cleanup_errors}) + require(not cleanup_errors, "temporary principals, policy or task finalization failed") + for principal in (denied, allowed): + observed = fleet.record("revoked-credential-denial", principal.admin(path, [], json.dumps(task_options).encode())) + require(observed.get("httpStatus") == 403 and response_code(observed) == "InvalidAccessKeyId", + "removed temporary credentials still authenticate or return an unexpected outcome") + after = fleet.snapshot("physical-after-authorization") + require(physical_content(before) == physical_content(after), "authorization/dry-run controls mutated the object fixture") + current = fleet.record("s3-after-authorization", probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("authorization-controls", {"physicalUnchanged": True, "s3": compare_inventories(baseline, current)}) + fleet.record("authorization-all-branches", branches) + require(set(branches) == {*DENIALS, "least-privilege-user"} + and all(branch["verdict"] == "PASS" for branch in branches.values()), "authorization requires every start/status/stop principal branch") + return branches diff --git a/scripts/validation_heal/heal_probe.py b/scripts/validation_heal/heal_probe.py index d51efe10..9ab30bb0 100644 --- a/scripts/validation_heal/heal_probe.py +++ b/scripts/validation_heal/heal_probe.py @@ -9,7 +9,7 @@ import argparse from contextlib import contextmanager -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta import hashlib import json import os @@ -164,20 +164,26 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): class Probe: - def __init__(self, binding: dict, endpoint: str | None = None): + def __init__(self, binding: dict, endpoint: str | None = None, *, credential_pair=None): validate_binding(binding) self.binding = binding self.endpoint_name = endpoint or binding["defaultEndpoint"] require(self.endpoint_name in binding["endpoints"], "endpoint is not bound") self.endpoint = binding["endpoints"][self.endpoint_name].rstrip("/") + require(credential_pair is None or isinstance(credential_pair, tuple) and len(credential_pair) == 2 + and all(isinstance(value, str) and value for value in credential_pair), "invalid in-memory principal credentials") + self.credential_pair = credential_pair + + def credentials(self): + return self.credential_pair or (os.environ["HEAL_ACCESS_KEY"], os.environ["HEAL_SECRET_KEY"]) def s3_client(self): import boto3 from botocore.config import Config + access, secret = self.credentials() return boto3.client("s3", endpoint_url=self.endpoint, - aws_access_key_id=os.environ["HEAL_ACCESS_KEY"], - aws_secret_access_key=os.environ["HEAL_SECRET_KEY"], - aws_session_token=os.environ.get("HEAL_SESSION_TOKEN"), + aws_access_key_id=access, aws_secret_access_key=secret, + aws_session_token=None if self.credential_pair else os.environ.get("HEAL_SESSION_TOKEN"), region_name="us-east-1", config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, retries={"max_attempts": 0}, connect_timeout=5, @@ -259,10 +265,17 @@ def admin(self, path: str, query: list[tuple[str, str]], body: bytes, url = self.endpoint + path + ("?" + urlencode(query) if query else "") headers = {"Content-Type": "application/json", "x-amz-content-sha256": hashlib.sha256(body).hexdigest()} with observation_phase(observation, 'signing'): + require(auth in {"signed", "anonymous", "bad-signature", "expired-signature"}, "unsupported Admin auth mode") + signing_time = None if auth != "anonymous": - secret = os.environ["HEAL_SECRET_KEY"] if auth == "signed" else "deliberately-invalid-heal-signature" + access, secret = self.credentials() + if auth == "bad-signature": + secret = "deliberately-invalid-heal-signature" + if auth == "expired-signature": + signing_time = (datetime.now(UTC) - timedelta(days=1)).strftime("%Y%m%dT%H%M%SZ") outgoing = AWSRequest(method=method, url=url, data=body, headers=headers) - sign_admin_request(outgoing, os.environ["HEAL_ACCESS_KEY"], secret, os.environ.get("HEAL_SESSION_TOKEN")) + sign_admin_request(outgoing, access, secret, + None if self.credential_pair else os.environ.get("HEAL_SESSION_TOKEN"), timestamp=signing_time) headers = dict(outgoing.headers.items()) started = time.monotonic() evidence = {"path": path, "query": query, "requestBody": body.decode(errors="replace"), @@ -270,6 +283,8 @@ def admin(self, path: str, query: list[tuple[str, str]], body: bytes, "startedAt": datetime.now(UTC).isoformat(), "clockDomain": CLOCK_DOMAIN, "callStartedMonotonicNs": time.monotonic_ns(), "timingScope": "client-call; not a server commit/lock witness"} + if signing_time: + evidence["isolatedClientSigningTimestamp"] = signing_time if observation is not None: evidence['statusObservation'] = dict(sessionId=observation['sessionId'], sequence=observation['sequence']) observation['callStartedMonotonicNs'] = evidence['callStartedMonotonicNs'] @@ -397,7 +412,7 @@ def main() -> int: admin.add_argument("--query", action="append", default=[]) admin.add_argument("--body", default="") admin.add_argument("--body-file", type=Path) - admin.add_argument("--auth", choices=("signed", "anonymous", "bad-signature"), default="signed") + admin.add_argument("--auth", choices=("signed", "anonymous", "bad-signature", "expired-signature"), default="signed") admin.add_argument("--expect-http", type=int, default=200) s3 = subs.add_parser("s3") s3.add_argument("--operation", choices=sorted(S3_OPERATIONS), required=True) diff --git a/scripts/validation_heal/heal_signing.py b/scripts/validation_heal/heal_signing.py index 085b8eef..92bfee31 100644 --- a/scripts/validation_heal/heal_signing.py +++ b/scripts/validation_heal/heal_signing.py @@ -3,12 +3,26 @@ from urllib.parse import quote_from_bytes, unquote_to_bytes -def sign_admin_request(request, access_key: str, secret_key: str, session_token: str | None = None) -> None: +def sign_admin_request(request, access_key: str, secret_key: str, session_token: str | None = None, + *, timestamp: str | None = None) -> None: # Keep SDK imports lazy so offline plan/binding tools need only stdlib. from botocore.auth import S3SigV4Auth from botocore.credentials import Credentials class AdminS3Auth(S3SigV4Auth): + def add_auth(self, request): + if timestamp is None: + return super().add_auth(request) + # Recompute the complete signature for the isolated client's old + # timestamp. Editing an already signed date would test bad signing. + from datetime import datetime + datetime.strptime(timestamp, "%Y%m%dT%H%M%SZ") + request.context["timestamp"] = timestamp + self._modify_request_before_signing(request) + canonical = self.canonical_request(request) + string_to_sign = self.string_to_sign(request, canonical) + self._inject_signature_to_request(request, self.signature(string_to_sign, request)) + def _normalize_url_path(self, path): # MinIO signs the decoded URL.Path after S3 URI encoding. Do not # normalize dot segments or decode twice: %252F is a literal %2F diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index ab801a23..23033beb 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -136,10 +136,21 @@ def __init__(self, nodes, evidence, run_id, tool_directory=None): self.sequence = 0 self.reserved = [] self.faults = [] + self.secret_values = set() def record(self, label, value): self.sequence += 1 - save(self.evidence / f"{self.sequence:05d}-{label}.json", value) + def sanitized(item): + if isinstance(item, str): + for secret in sorted(self.secret_values, key=len, reverse=True): + item = item.replace(secret, "[REDACTED]") + return item + if isinstance(item, dict): + return {sanitized(key): sanitized(entry) for key, entry in item.items()} + if isinstance(item, list): + return [sanitized(entry) for entry in item] + return item + save(self.evidence / f"{self.sequence:05d}-{label}.json", sanitized(value)) return value def ssh(self, node): @@ -284,6 +295,12 @@ def run_topology(args, topology, nodes): fleet.record("fault", {"fault": "F0", "injected": False, "backgroundRepairDisabled": True}) if args.case_id == "heal.admin-invalid-request": result["branches"] = invalid_requests(probe, fleet, bucket, before, baseline) + if args.case_id == "heal.admin-authorization": + try: + from .authorization import authorization_requests + except ImportError: + from authorization import authorization_requests + result["branches"] = authorization_requests(probe, fleet, bucket, before, baseline, active, BASE_OPTIONS) healthy_branches = (("normal", 1, "metadata_healthy"), ("deep", 2, "verified_healthy")) \ if args.case_id == "heal.admin-healthy-noop" else () for branch, mode, disposition in healthy_branches: @@ -348,7 +365,7 @@ def interrupted(signum, frame): signal.signal(signal.SIGTERM, interrupted) parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--candidate", type=Path, required=True) - parser.add_argument("--case-id", choices=("heal.admin-healthy-noop", "heal.admin-invalid-request"), + parser.add_argument("--case-id", choices=("heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-authorization"), default="heal.admin-healthy-noop") parser.add_argument("--testing-commit", required=True) parser.add_argument("--binary-sha", required=True) @@ -367,7 +384,7 @@ def interrupted(signum, frame): for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): result = run_topology(args, topology, selected) results.append(result) - if result["verdict"] != "PASS" or result["cleanup"] != "PASS": + if result["cleanup"] != "PASS": break finally: save(args.evidence / "healthy-summary.json", {"caseId": args.case_id, "results": results, diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index aad932fd..7ba114f8 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index fbb814c0..0c59b324 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -18,7 +18,7 @@ while [ "$#" -gt 0 ]; do esac done case "${case_id}" in - heal.admin-healthy-noop|heal.admin-invalid-request) ;; + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization) ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; diff --git a/test_validation_heal_authorization.py b/test_validation_heal_authorization.py new file mode 100644 index 00000000..75ad4477 --- /dev/null +++ b/test_validation_heal_authorization.py @@ -0,0 +1,194 @@ +"""Exercise signing, denial isolation and temporary-principal finalization.""" + +from datetime import UTC, datetime, timedelta +import hashlib +import hmac +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from botocore.awsrequest import AWSRequest + +from scripts.validation_heal.authorization import DENIALS, authorization_requests, iam_exchange, validate_denial +from scripts.validation_heal.heal_probe import Probe +from scripts.validation_heal.heal_signing import sign_admin_request +from scripts.validation_heal.healthy import BASE_OPTIONS, Fleet +from scripts.validation_heal.matrix import select + + +def binding(): + return {"schemaVersion": 1, "runId": "manual-auth-test", "jumpHostOnly": True, + "exclusiveHealService": True, "leaseExpiresAt": (datetime.now(UTC) + timedelta(hours=1)).isoformat(), + "bucketPrefix": "rv-heal-auth-test", "endpoints": {"node1": "http://127.0.0.1:19000"}, + "defaultEndpoint": "node1", "candidateCommit": "a" * 40, "harnessCommit": "b" * 40, + "candidatePackageSha256": "c" * 64, "candidateBinarySha256": "d" * 64, + "rcBinarySha256": "e" * 64, "maxInventoryObjects": 128, "maxInventoryBytes": 1024**2} + + +class Response(io.BytesIO): + code = 200 + headers = {} + + +class AuthorizationBoundaryTest(unittest.TestCase): + def test_old_timestamp_recomputes_valid_signature_and_credential_scope(self): + timestamp, access, secret = "20000102T030405Z", "isolated-access", "isolated-secret" + body = b'{"dryRun":true}' + digest = hashlib.sha256(body).hexdigest() + outgoing = AWSRequest(method="POST", url="http://127.0.0.1:19000/rustfs/admin/v3/heal/rv-heal-auth-test?clientToken=token", + data=body, headers={"Content-Type": "application/json", "x-amz-content-sha256": digest}) + sign_admin_request(outgoing, access, secret, timestamp=timestamp) + # Independent SigV4 calculation for this fixed wire request. The old + # date participates in canonical headers, credential scope and HMAC. + canonical = "\n".join(("POST", "/rustfs/admin/v3/heal/rv-heal-auth-test", "clientToken=token", + "content-type:application/json\nhost:127.0.0.1:19000\nx-amz-content-sha256:" + digest + "\nx-amz-date:" + timestamp + "\n", + "content-type;host;x-amz-content-sha256;x-amz-date", digest)) + scope = "20000102/us-east-1/s3/aws4_request" + string_to_sign = "\n".join(("AWS4-HMAC-SHA256", timestamp, scope, hashlib.sha256(canonical.encode()).hexdigest())) + key = ("AWS4" + secret).encode() + for value in ("20000102", "us-east-1", "s3", "aws4_request"): + key = hmac.new(key, value.encode(), hashlib.sha256).digest() + signature = hmac.new(key, string_to_sign.encode(), hashlib.sha256).hexdigest() + self.assertEqual(outgoing.headers["X-Amz-Date"], timestamp) + self.assertIn("Credential=" + access + "/" + scope, outgoing.headers["Authorization"]) + self.assertTrue(outgoing.headers["Authorization"].endswith("Signature=" + signature)) + tampered = canonical.replace(timestamp, "20000103T030405Z") + self.assertNotEqual(hashlib.sha256(canonical.encode()).hexdigest(), hashlib.sha256(tampered.encode()).hexdigest()) + + def test_probe_uses_isolated_principal_and_expired_client_clock(self): + probe = Probe(binding(), credential_pair=("client-access", "client-secret")) + sent = [] + def opened(outgoing): + sent.append(outgoing) + return Response(b'{"Code":"RequestTimeTooSkewed"}') + with patch.dict("os.environ", {"HEAL_ACCESS_KEY": "root-access", "HEAL_SECRET_KEY": "root-secret", "HEAL_SESSION_TOKEN": "root-token"}), \ + patch.object(probe, "_open_admin", side_effect=opened): + observed = probe.admin("/rustfs/admin/v3/heal/rv-heal-auth-test", [], b"{}", auth="expired-signature") + signed_at = datetime.strptime(observed["isolatedClientSigningTimestamp"], "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC) + self.assertAlmostEqual((datetime.now(UTC) - signed_at).total_seconds(), 86400, delta=3) + headers = dict(sent[0].header_items()) + self.assertIn("Credential=client-access/", headers["Authorization"]) + self.assertNotIn("X-amz-security-token", headers) + self.assertNotIn("root-access", json.dumps(headers)) + + def test_denial_requires_exact_code_and_no_nested_json_or_xml_disclosure(self): + for mode, (status, code) in DENIALS.items(): + good = {"httpStatus": status, "response": {"Code": code}} + self.assertTrue(validate_denial(good, mode)["noTaskDisclosure"]) + for poisoned in ({**good, "httpStatus": 200}, {**good, "response": {"Code": "InternalError"}}, + {**good, "response": {"Code": code, "detail": [{"objects": []}]}}, + {**good, "response": {"Code": code, "clientToken": "admitted"}}, + {**good, "response": f'{code}'}, + {"response": {"Code": code}, "transportError": "timeout"}): + with self.subTest(mode=mode, response=poisoned), self.assertRaises(ValueError): + validate_denial(poisoned, mode) + + def test_iam_evidence_redacts_credentials_without_modifying_oracle_input(self): + with tempfile.TemporaryDirectory() as directory: + fleet = Fleet([], Path(directory), "manual-auth-test") + fleet.secret_values.update({"root-access", "root-secret", "user-secret"}) + probe = Probe(binding(), credential_pair=("root-access", "root-secret")) + with patch.object(probe, "_open_admin", return_value=Response(b'{"message":"user-secret root-access root-secret"}')): + result = iam_exchange(probe, fleet, "add-user", "rvhealowneduser", secret="user-secret") + self.assertIn("user-secret", result["response"]) + archive = (Path(directory) / "00001-iam-add-user.json").read_text() + for secret in fleet.secret_values: + self.assertNotIn(secret, archive) + self.assertFalse(json.loads(archive)["requestBodyArchived"]) + with self.assertRaisesRegex(ValueError, "namespace"): + iam_exchange(probe, fleet, "remove-user", "unrelateduser") + original = {"user-secret": {"details": ["user-secret"]}} + self.assertIs(fleet.record("secret-key", original), original) + self.assertNotIn("user-secret", (Path(directory) / "00002-secret-key.json").read_text()) + + +class PrincipalLifecycleTest(unittest.TestCase): + def execute(self, directory, *, failed_operation=None, negative_transport=False, revocation_broken=False): + users, iam_calls, negative_calls = set(), [], [] + active = [] + fleet = Fleet([], Path(directory), "manual-auth-test") + scene = {"node1": {"files": [], "formats": []}} + class Client: + def __init__(self, bound, *, credential_pair=None): + self.binding = bound + self.credential_pair = credential_pair + def credentials(self): + return self.credential_pair or ("root-access", "root-secret") + def s3(self, operation, params): + return {"acknowledged": False, "resultUnknown": False, + "response": {"ResponseMetadata": {"HTTPStatusCode": 403}, "Error": {"Code": "AccessDenied"}}} + def admin(self, path, query, body, auth="signed"): + if self.credential_pair and self.credential_pair[0] not in users: + return {"httpStatus": 403, "response": {"Code": "AccessDenied" if revocation_broken else "InvalidAccessKeyId"}} + denied = self.credential_pair and self.credential_pair[0].endswith("denied") + if auth != "signed" or denied: + mode = "denied-user" if denied else auth + operation = "stop" if dict(query).get("forceStop") else "status" if query else "start" + negative_calls.append((mode, operation)) + if negative_transport and (mode, operation) == ("bad-signature", "status"): + raise OSError("isolated transport failure") + status, code = DENIALS[mode] + return {"httpStatus": status, "response": {"Code": code}} + if not query: + return {"httpStatus": 200, "response": {"clientToken": "allowed-token" if self.credential_pair else "root-token"}} + return {"httpStatus": 200, "response": {"summary": "finished"}} + def inventory(self, *args, **kwargs): + return {} + def exchange(probe, recorded, operation, user, **kwargs): + iam_calls.append((operation, user)) + if operation == "add-user": users.add(user) + if operation == failed_operation: raise ValueError("injected IAM failure: " + operation) + if operation == "remove-user": users.discard(user) + self.addCleanup(lambda: self.assertFalse(active, "tokens must be finalized even on IAM failure")) + with patch("scripts.validation_heal.authorization.Probe", Client), \ + patch("scripts.validation_heal.authorization.iam_exchange", side_effect=exchange), \ + patch.object(fleet, "snapshot", return_value=scene), \ + patch("scripts.validation_heal.authorization.compare_inventories", return_value={"unchanged": True}): + try: + branches = authorization_requests(Client(binding()), fleet, "rv-heal-auth-test", scene, {}, active, BASE_OPTIONS) + return branches, iam_calls, negative_calls + finally: + self.last_calls, self.last_negatives = iam_calls, negative_calls + + def test_every_source_principal_controls_start_status_stop_and_is_removed(self): + with tempfile.TemporaryDirectory() as directory: + branches, iam_calls, negatives = self.execute(directory) + self.assertEqual(set(branches), set(select("A08")[0]["variants"])) + self.assertTrue(all(branch["verdict"] == "PASS" for branch in branches.values())) + self.assertEqual(set(negatives), {(mode, operation) for mode in DENIALS for operation in ("start", "status", "stop")}) + self.assertEqual([operation for operation, _ in iam_calls].count("remove-user"), 2) + self.assertEqual([operation for operation, _ in iam_calls][-1], "remove-policy") + + def test_failed_or_ambiguous_admission_keeps_owned_principal_for_cleanup(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, "injected IAM failure"): + self.execute(directory, failed_operation="add-user") + self.assertEqual([operation for operation, _ in self.last_calls], ["add-user", "remove-user"]) + + def test_cleanup_error_cannot_pass_and_other_cleanup_operations_continue(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, "finalization failed"): + self.execute(directory, failed_operation="detach") + self.assertEqual([operation for operation, _ in self.last_calls][-3:], ["remove-user", "remove-user", "remove-policy"]) + result = next(Path(directory).glob("*-iam-finalization.json")) + self.assertEqual(json.loads(result.read_text())["cleanup"], "FAIL") + + def test_one_denial_transport_failure_preserves_all_branches_and_never_passes(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, "every start/status/stop"): + self.execute(directory, negative_transport=True) + self.assertEqual(len(self.last_negatives), 12) + result = next(Path(directory).glob("*-authorization-negative-branches.json")) + self.assertEqual(json.loads(result.read_text())["bad-signature"]["verdict"], "FAIL") + + def test_removed_credentials_must_return_the_exact_revocation_error(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, "removed temporary credentials"): + self.execute(directory, revocation_broken=True) + + +if __name__ == "__main__": + unittest.main() From 107dec600334b703cc5fecadc6751d43408829bf Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 23:41:59 +0800 Subject: [PATCH 044/343] fix(validation): bind repeated repair backups to current metadata --- scripts/validation_heal/shards.py | 24 +++++++++++++++--------- test_validation_heal.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index dbbb02e8..67db57da 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -190,6 +190,15 @@ def validate_repaired_physical(before, after, target, *, repair_targets=None): left, right = files(before), files(after) repair_targets = repair_targets or [target] meta_paths = {f"{item['bucket']}/{item['key']}/xl.meta" for item in repair_targets} + backups = {(item["node"], item["disk"], f"{item['bucket']}/{item['key']}/{item['dataDir']}/xl.meta.bkp"): + left.get((item["node"], item["disk"], f"{item['bucket']}/{item['key']}/xl.meta")) + for item in repair_targets} + completed_sidecars = {(item["node"], item["disk"], + f"{item['bucket']}/{item['key']}/{item['dataDir']}/part.{part['number']}.meta") + for item in repair_targets for part in item["logical"]["parts"]} + def exact_backup(identity, actual): + original = backups.get(identity) + return bool(original and actual and actual["sha256"] == original["sha256"] and actual["size"] == original["size"]) for node in before: require(before[node]["formats"] == after[node]["formats"], "repair changed disk format") for identity, entry in left.items(): @@ -197,13 +206,14 @@ def validate_repaired_physical(before, after, target, *, repair_targets=None): # Complete MPU commits the authoritative part manifest into xl.meta. # A healed target directory can discard only its stale per-part sidecars; # other versions, drives, pending uploads and payload files stay exact. - completed_sidecars = {(item["node"], item["disk"], - f"{item['bucket']}/{item['key']}/{item['dataDir']}/part.{part['number']}.meta") - for item in repair_targets for part in item["logical"]["parts"]} if actual is None and identity in completed_sidecars: continue require(actual is not None, "repair lost a baseline file or control") - if identity[2] not in meta_paths: + if identity in backups and actual != entry: + # Repeated repair replaces only this target's rollback metadata + # with the exact authoritative bytes observed before this repair. + require(exact_backup(identity, actual), "target rollback backup is not the exact pre-repair metadata") + elif identity[2] not in meta_paths: require(actual == entry, "repair did not restore exact part bytes or changed a control") else: def semantics(metadata): @@ -215,11 +225,7 @@ def semantics(metadata): require(actual.get("metadata") and semantics(actual["metadata"]) == semantics(entry["metadata"]), "repair changed decoded version/part identity") for identity in set(right) - set(left): - allowed = {(item["node"], item["disk"], f"{item['bucket']}/{item['key']}/{item['dataDir']}/xl.meta.bkp") - for item in repair_targets} - original = left.get((identity[0], identity[1], identity[2].rsplit("/", 2)[0] + "/xl.meta")) - require(identity in allowed and original - and right[identity]["sha256"] == original["sha256"] and right[identity]["size"] == original["size"], + require(exact_backup(identity, right[identity]), "repair created an unexpected physical file") return {"targetLengthAndFullSha256Restored": True, "controlFilesUnchanged": True, "decodedVersionsUnchanged": True, "formatUnchanged": True, "readBeforeRepair": False} diff --git a/test_validation_heal.py b/test_validation_heal.py index 089aca61..2152a361 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -344,6 +344,37 @@ def test_completed_mpu_sidecar_cleanup_is_bound_to_target_drive_and_manifest(sel with self.assertRaisesRegex(ValueError, "lost a baseline file"): validate_repaired_physical(before, after, target) + def test_repeated_target_backup_requires_exact_current_metadata_and_target_identity(self): + before = self.snapshot() + target = self.target(before) + meta = next(entry for entry in before["node1"]["files"] + if entry["disk"] == target["disk"] and entry["path"] == f"{target['bucket']}/{target['key']}/xl.meta") + backup = {"disk": target["disk"], "path": f"{target['bucket']}/{target['key']}/{target['dataDir']}/xl.meta.bkp", + "size": 599, "sha256": "previous-repair-metadata"} + before["node1"]["files"].append(backup) + after = copy.deepcopy(before) + after["node1"]["files"][-1].update(size=meta["size"], sha256=meta["sha256"]) + validate_repaired_physical(before, after, target) + for invalid in ("bytes", "length", "other-drive", "other-generation", "payload"): + changed_before, changed_after = copy.deepcopy(before), copy.deepcopy(after) + if invalid == "bytes": changed_after["node1"]["files"][-1]["sha256"] = "not-authoritative" + elif invalid == "length": changed_after["node1"]["files"][-1]["size"] += 1 + elif invalid == "other-drive": + for value in (changed_before, changed_after): value["node1"]["files"][-1]["disk"] = 0 + elif invalid == "other-generation": + for value in (changed_before, changed_after): value["node1"]["files"][-1]["path"] = backup["path"].replace(target["dataDir"], "other-generation") + else: + changed_after["node1"]["files"][1]["sha256"] = "changed-payload" + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + validate_repaired_physical(changed_before, changed_after, target) + added_before = self.snapshot() + added_after = copy.deepcopy(added_before) + added_after["node1"]["files"].append(after["node1"]["files"][-1]) + validate_repaired_physical(added_before, added_after, target) + added_after["node1"]["files"][-1]["sha256"] = "unexpected" + with self.assertRaisesRegex(ValueError, "unexpected physical file"): + validate_repaired_physical(added_before, added_after, target) + def test_unknown_or_wrong_version_receipt_cannot_certify_physical_repair(self): target = self.target(self.snapshot()) receipt = HealHealthyOracleTest().receipt("repaired") From 56ffdb2e877e745b09acfe7c08c7b8a878343b71 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 10 Oct 2026 23:55:36 +0800 Subject: [PATCH 045/343] feat(validation): verify node outage ACKs and physical MRF rejoin --- docs/validation-heal-contract.md | 2 +- docs/validation-migration-map.md | 20 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 18 +- scripts/validation_heal/healthy.py | 6 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/node_outage.py | 316 ++++++++++++++++++ .../prepare_heal_meta_inspector.py | 3 + scripts/validation_heal/run_case.sh | 1 + test_validation_heal.py | 112 +++++++ 10 files changed, 470 insertions(+), 12 deletions(-) create mode 100644 scripts/validation_heal/node_outage.py diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index 0caefabf..2ee4c10e 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -22 个用例已接入自动分发：A01、A03、A04、A07、A08、B05、B07、C01–C05、D01–D03、D05–D07、D10、E01、E02、J04。分片执行器使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。A08 保留五种主体和各自 start/status/stop 操作，使用真正重新计算的过期签名及仅授权 `admin:Heal` 的临时用户，清理后验证凭据失效。其余 62 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 +23 个用例已接入自动分发：A01、A03、A04、A07、A08、B05、B07、B09、C01–C05、D01–D03、D05–D07、D10、E01、E02、J04。分片执行器使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。A08 保留五种主体和各自 start/status/stop 操作，使用真正重新计算的过期签名及仅授权 `admin:Heal` 的临时用户，清理后验证凭据失效。B09 使用明确启用 MRF 的专用服务；停止一个跨节点 set 成员节点，记录降级 ACK，并先验证逐盘补片，再执行 S3/手工验证请求。其余 61 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index dc6fe0e3..07845141 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 96 currently dispatchable IDs include all 74 non-Heal cases and twenty-two Heal cases. Forty-one have live PASS evidence; the other 117 have no live PASS claim. +case evidence. The 97 currently dispatchable IDs include all 74 non-Heal cases and twenty-three Heal cases. Forty-one have live PASS evidence; the other 117 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 62 Heal cases remain +The other 61 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -361,3 +361,19 @@ owned service, no injection intent, exact archived pre-fault physical scene and removed run roots. It records cleanup PASS, `originalVerdictRewritten=false` and no canonical/product-repair PASS claim. Serial deployment resumed only after inspecting that recovery evidence. + +B09 `heal.node-outage-rejoin` now has both source H2 branches with independent +DB and DV deployments on the existing four nodes. The candidate tool freezes +the recognized same-set read/write quorum policy; actual metadata maps the +four lost drives before admitting degraded I/O. Exact unit exit and refused +endpoint connections witness the outage. PUT/overwrite/marker/historical-delete +ACKs define the new manifest, while the stopped node's complete physical scene +stays unchanged. Only the explicit run-owned `offline-replay` profile enables +MRF; scanner and automatic disk-rebuild remain disabled. After the same node +and roots rejoin, physical polling precedes any target S3 reads or manual task. +Every drive must contain the exact acknowledged version/marker set; surviving +shard bytes, committed streaming checksums and all-node S3 manifests must agree. +Deleted identities receive exact S3 absence checks. No copied backup contributes +to repair. Six focused tests cover profile boundaries, source-quorum recognition, +ACK ambiguity, per-drive omissions/resurrection and deleted-read refusals. +B09 requires the fresh write-quorum tool artifact and full live evidence. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 7231e419..cae30abb 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index c003afbe..46e6c7fb 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -27,6 +27,11 @@ "RUSTFS_SHARD_INTEGRITY_WRITE": "true", "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED": "true"} +def runtime_config(profile): + require(profile in {"manual", "offline-replay"}, "unsupported Heal runtime profile") + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true" if profile == "offline-replay" else "false"} + + def require(condition, message): if not condition: raise ValueError(message) @@ -104,6 +109,8 @@ def preflight(request): def reserve(request): run_id = request["runId"] + profile = request.get("runtimeProfile", "manual") + background = runtime_config(profile) observed = preflight(request) require(observed["hostname"] == request["hostname"] and observed["mounts"] == request["mounts"], "preflight host or disk identities changed") @@ -126,7 +133,7 @@ def reserve(request): state, roots = paths(run_id) owner = {"runId": run_id, "unit": unit(run_id), "hostname": observed["hostname"], "mounts": observed["mounts"], "binarySha256": binary_sha, - "bucket": request["bucket"], "volumes": volumes} + "bucket": request["bucket"], "volumes": volumes, "runtimeProfile": profile} if request.get("inspectorSha256"): require(bool(re.fullmatch(r"[0-9a-f]{64}", request["inspectorSha256"])), "invalid inspector digest") owner["inspectorSha256"] = request["inspectorSha256"] @@ -141,7 +148,7 @@ def reserve(request): shutil.copyfile("/usr/bin/rustfs", state / "rustfs") (state / "rustfs").chmod(0o500) require(digest(state / "rustfs") == binary_sha, "copied candidate binary differs") - config = {**BACKGROUND_CONFIG, "RUSTFS_ACCESS_KEY": request["access"], "RUSTFS_SECRET_KEY": request["secret"], + config = {**background, "RUSTFS_ACCESS_KEY": request["access"], "RUSTFS_SECRET_KEY": request["secret"], "RUSTFS_ADDRESS": ":9000", "RUSTFS_CONSOLE_ENABLE": "false", "RUSTFS_HEAL_TASK_TIMEOUT_SECS": "300", "RUSTFS_OBS_LOGGER_LEVEL": "info"} env_file = state / "service.env" @@ -317,8 +324,9 @@ def snapshot(request): "running binary differs from candidate") environment = dict(value.split("=", 1) for value in Path(f"/proc/{pid}/environ").read_bytes().decode().split("\0") if "=" in value) - require(all(environment.get(key) == value for key, value in BACKGROUND_CONFIG.items()), - "background repair isolation configuration differs") + expected_config = runtime_config(owner.get("runtimeProfile", "manual")) + require(all(environment.get(key) == value for key, value in expected_config.items()), + "bound Heal runtime configuration differs") files, formats = [], [] tool = state / "heal-meta-inspect" if owner.get("inspectorSha256"): @@ -344,7 +352,7 @@ def snapshot(request): formats.append({"disk": index, "sha256": digest(path), "value": json.loads(path.read_text())}) return {"hostname": owner["hostname"], "unit": owner["unit"], "pid": pid, "binarySha256": owner["binarySha256"], "quiescent": request.get("quiescent") is True, - "backgroundConfig": None if request.get("quiescent") is True else BACKGROUND_CONFIG, + "backgroundConfig": None if request.get("quiescent") is True else expected_config, "storageClassEnvironment": None if request.get("quiescent") is True else { key: environment.get(key) for key in ("RUSTFS_STORAGE_CLASS_INLINE_BLOCK", "RUSTFS_STORAGE_CLASS_STANDARD")}, "files": sorted(files, key=lambda value: (value["disk"], value["path"])), "formats": formats} diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 23033beb..467efdd3 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -130,9 +130,11 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: - def __init__(self, nodes, evidence, run_id, tool_directory=None): + def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual"): + require(runtime_profile in {"manual", "offline-replay"}, "unsupported Heal runtime profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory + self.runtime_profile = runtime_profile self.sequence = 0 self.reserved = [] self.faults = [] @@ -207,7 +209,7 @@ def deploy(self, binary_sha, bucket, access, secret): if self.tool_directory else None value = self.remote(node, "reserve", hostname=checked[node]["hostname"], mounts=checked[node]["mounts"], binarySha256=binary_sha, bucket=bucket, volumes=volumes, access=access, secret=secret, - inspectorSha256=inspector_sha) + inspectorSha256=inspector_sha, runtimeProfile=self.runtime_profile) self.record("reserved", {"node": node, **value}) if self.tool_directory: tool = self.tool_directory / "heal-meta-inspect" diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 7ba114f8..f0f75c1d 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/node_outage.py b/scripts/validation_heal/node_outage.py new file mode 100644 index 00000000..22ff0148 --- /dev/null +++ b/scripts/validation_heal/node_outage.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +"""B09: acknowledged degraded I/O and physical MRF repair on the current H2 fleet.""" + +import argparse +import copy +from datetime import UTC, datetime, timedelta +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +from urllib import request +from urllib.error import URLError + +try: + from .heal_probe import Probe, compare_inventories, payload, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .shards import SIZES, files, map_target, run_task + from .versions import public_version_id, validate_version_receipts, verify_quorum_contract +except ImportError: + from heal_probe import Probe, compare_inventories, payload, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from shards import SIZES, files, map_target, run_task + from versions import public_version_id, validate_version_receipts, verify_quorum_contract + + +BRANCHES = {"one-node-stop": "DB", "rejoin-with-new-versions": "DV"} +SIZE = 1024**2 + 47 + + +def quorum_plan(target, offline_node, contract): + require(contract.get("recognized") is True and contract.get("writeQuorumRule") == "data-plus-one-if-equal-parity", + "node outage requires recognized candidate read/write quorum policies") + erasure = target["logical"]["erasure"] + data, parity = erasure["data_blocks"], erasure["parity_blocks"] + lost = [shard for shard in target["cohort"] if shard["node"] == offline_node] + remaining = data + parity - len(lost) + write = data + int(data == parity) + require(len(lost) == 4 and len(target["cohort"]) == 16 and data + parity == 16 + and len({shard["node"] for shard in target["cohort"]}) == 4 + and remaining >= max(data, write), "selected node outage cannot admit the degraded read/write workload") + return {"offlineNode": offline_node, "offlineDiskUUIDs": [shard["diskUUID"] for shard in lost], + "lostSameSetDrives": len(lost), "remainingSameSetDrives": remaining, + "readQuorum": data, "writeQuorum": write, "policy": contract["writeQuorumRule"], + "distributedLockAdmission": "each successful write ACK also proves its actual lock admission"} + + +def put_ack(expected, key, body, response, fixture): + version = response.get("VersionId", "null") + require(fixture != "DV" or version not in {None, "null", ""}, "degraded versioned PUT omitted its version identity") + require(fixture != "DB" or version in {None, "null"}, "never-versioned PUT unexpectedly created a version") + version = public_version_id(version) + result = copy.deepcopy(expected) + if fixture == "DB": result = [entry for entry in result if entry["key"] != key] + else: + require(not any(entry["key"] == key and entry["versionId"] == version for entry in result), + "degraded PUT reused an acknowledged version") + for entry in result: + if entry["key"] == key: entry["isLatest"] = False + result.append({"key": key, "versionId": version, "deleteMarker": False, "isLatest": True, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + return result + + +def delete_ack(expected, key, response, fixture, *, version=None): + result = copy.deepcopy(expected) + if version is not None: + require(response.get("VersionId") == version and response.get("DeleteMarker") in {None, False}, + "explicit data-version DELETE did not acknowledge the exact identity") + require(any(entry["key"] == key and entry["versionId"] == version and not entry["isLatest"] for entry in result), + "historical delete would change the expected current identity") + return [entry for entry in result if (entry["key"], entry["versionId"]) != (key, version)] + if fixture == "DB": + require(response.get("DeleteMarker") in {None, False} and response.get("VersionId") in {None, "null"}, + "never-versioned DELETE returned a marker or foreign version") + return [entry for entry in result if entry["key"] != key] + marker = response.get("VersionId") + require(response.get("DeleteMarker") is True and marker not in {None, "", "null"} + and not any(entry["key"] == key and entry["versionId"] == marker for entry in result), + "degraded marker DELETE omitted or reused its acknowledged identity") + for entry in result: + if entry["key"] == key: entry["isLatest"] = False + result.append({"key": key, "versionId": marker, "deleteMarker": True, "isLatest": True}) + return result + + +def workload(recorder, bucket, expected, fixture): + manifest = copy.deepcopy(expected) + for index in range(6): + key = f"offline/new-{index:02d}.bin" + body = payload(f"{bucket}/{key}/offline", SIZE + index) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "Metadata": {"heal-fixture": "offline-ack"}, "Tagging": "suite=heal&fixture=offline"}, body) + manifest = put_ack(manifest, key, body, response, fixture) + for index in (1, 3): + key = f"objects/{index:04d}.bin" + body = payload(f"{bucket}/{key}/offline-overwrite", SIZE + index) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "Metadata": {"heal-fixture": "offline-overwrite"}, "Tagging": "suite=heal&fixture=offline"}, body) + manifest = put_ack(manifest, key, body, response, fixture) + key = "objects/0002.bin" if fixture == "DV" else "objects/0006.bin" + response = recorder.call("delete_object", {"Bucket": bucket, "Key": key}) + manifest = delete_ack(manifest, key, response, fixture) + if fixture == "DV": + old = next(entry for entry in expected if entry["key"] == "objects/0001.bin" and not entry["isLatest"]) + response = recorder.call("delete_object", {"Bucket": bucket, "Key": old["key"], "VersionId": old["versionId"]}) + manifest = delete_ack(manifest, old["key"], response, fixture, version=old["versionId"]) + for entry in [entry for entry in manifest if not entry["deleteMarker"]][:6]: + observed = recorder.probe.s3("get_object", {"Bucket": bucket, "Key": entry["key"], "VersionId": entry["versionId"]}) + # Archive failures as well as complete bodies' hashes before checking. + save(recorder.output / ("offline-get-" + hashlib.sha256((entry["key"] + entry["versionId"]).encode()).hexdigest()[:20] + ".json"), observed) + require(observed.get("acknowledged") and observed.get("payload") == entry["payload"], + "degraded historical GET differs from acknowledged bytes") + recorder.call("head_object", {"Bucket": bucket, "Key": entry["key"], "VersionId": entry["versionId"]}) + return manifest + + +def replica_census(snapshot, bucket, expected, offline_snapshot): + validate_topology(snapshot, "H2") + original = files(offline_snapshot) + wanted = {} + for entry in expected: + wanted.setdefault(entry["key"], set()).add((entry["versionId"], entry["deleteMarker"], entry["isLatest"])) + for node, value in snapshot.items(): + for disk in range(4): + observed = {} + for item in value["files"]: + if item["disk"] != disk: continue + metadata = item.get("metadata") + if metadata: + require(metadata.get("bucket") == bucket, "metadata escaped the run bucket") + versions = {(public_version_id(version.get("versionId")), version["deleted"], version["isLatest"]) + for version in metadata["versions"]} + if versions: observed[metadata["key"]] = versions + require(observed == wanted, "rejoined replica lacks or resurrected an acknowledged version/marker") + targets = [] + for entry in expected: + if entry["deleteMarker"]: continue + target = map_target(snapshot, bucket, entry["key"], "data", 1, entry["versionId"], latest_required=False) + for shard in target["cohort"]: + previous = original.get((shard["node"], shard["disk"], shard["relativePath"])) + require(previous is None or (shard["sha256"] == previous["sha256"] and shard["size"] == previous["size"]), + "rejoin changed an authoritative surviving shard") + targets.append(target) + return targets + + +def validate_deleted_read(observed, version): + response = observed.get("response") or {} + require(observed.get("acknowledged") is False and observed.get("resultUnknown") is False and "payload" not in observed + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404 + and response.get("Error", {}).get("Code") == ("NoSuchKey" if version == "null" else "NoSuchVersion"), + "rejoined endpoint returned a deleted identity or an unrecognized refusal") + + +def run_branch(args, branch, nodes): + fixture = BRANCHES[branch] + directory = args.evidence / branch + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-h2" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="offline-replay") + candidate = json.loads(args.candidate.read_text()) + tool = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + contract = verify_quorum_contract(tool, candidate) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(UTC) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, 5)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 512, "maxInventoryBytes": 512 * 1024**2} + fleet.record("binding", binding) + probe, active = Probe(binding), [] + result = {"topology": "H2", "branch": branch, "fixture": fixture, "verdict": "RUNNING", "cleanup": "PENDING"} + try: + require(contract.get("writeQuorumRule") == "data-plus-one-if-equal-parity", + "node outage needs a fresh recognized candidate write-quorum contract") + fleet.deploy(args.binary_sha, bucket, access, secret) + recorder = Recorder(probe, directory / "fixture") + expected = seed(recorder, bucket, fixture, 128 if fixture == "DB" else 4, [SIZE]) + initial = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, initial)}) + before = fleet.snapshot("physical-baseline") + offline_node = nodes[-1] + target = map_target(before, bucket, "objects/0001.bin", "data", 1, + next(entry["versionId"] for entry in expected if entry["key"] == "objects/0001.bin" and entry["isLatest"])) + fleet.record("quorum-fault-plan", quorum_plan(target, offline_node, contract)) + stop = fleet.record("node-stopped", fleet.remote(offline_node, "stop")) + require(stop.get("quiescent") is True and stop.get("service", {}).get("MainPID") == "0", "node stop lacks process exit") + opener = request.build_opener(request.ProxyHandler({})) + try: + with opener.open(binding["endpoints"]["node4"] + "/health/ready", timeout=3): + raise ValueError("stopped node endpoint still accepts requests") + except URLError as exc: + require(isinstance(exc.reason, ConnectionRefusedError), "node outage was not a witnessed connection refusal") + fleet.record("node-endpoint-refused", {"endpoint": "node4", "connectionRefused": True}) + frozen_offline = fleet.record("offline-node-baseline", fleet.remote(offline_node, "snapshot", quiescent=True)) + expected = workload(recorder, bucket, expected, fixture) + acknowledged = fleet.record("s3-degraded-ack-manifest", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + fleet.record("degraded-full-proof", {"expected": expected, **verify_seed(expected, acknowledged)}) + previous = {(entry["key"], entry["versionId"]): entry for entry in initial["entries"]} + retained = {(entry["key"], entry["versionId"]) for entry in acknowledged["entries"]} + for entry in acknowledged["entries"]: + old = previous.get((entry["key"], entry["versionId"])) + if old and not entry["deleteMarker"] and old.get("payload") == entry.get("payload"): + require(old["properties"] == entry["properties"] and old["tags"] == entry["tags"], + "degraded I/O changed an untouched version's public metadata or tags") + still_offline = fleet.record("offline-node-after-acks", fleet.remote(offline_node, "snapshot", quiescent=True)) + require(physical_content({offline_node: frozen_offline}) == physical_content({offline_node: still_offline}), + "offline node received writes before it rejoined") + authorities = fleet.record("physical-offline-authorities", {node: fleet.remote(node, "snapshot", quiescent=node == offline_node) for node in nodes}) + fleet.record("node-rejoined", fleet.remote(offline_node, "start")) + fleet.wait_ready() + fleet.record("trigger", {"source": "MRF", "manualTasksBeforePhysicalRepair": False, + "targetS3ReadsBeforePhysicalRepair": False, "budgetSeconds": 180}) + deadline, last_error = time.monotonic() + 180, None + while True: + scene = fleet.snapshot("physical-mrf-progress") + try: + targets = replica_census(scene, bucket, expected, authorities) + break + except ValueError as exc: + last_error = str(exc) + require(time.monotonic() < deadline, "MRF physical rejoin deadline exceeded: " + last_error) + time.sleep(2) + fleet.stop() + repaired = fleet.snapshot("physical-repaired-quiescent", quiescent=True) + targets = replica_census(repaired, bucket, expected, authorities) + proofs = [] + for target in targets: + for shard in target["cohort"]: + observed = fleet.record("rejoined-streaming-bitrot", fleet.remote(shard["node"], "verify-part", + disk=shard["disk"], relativePath=shard["relativePath"], versionId=shard["versionId"], partNumber=1)) + require(observed.get("physicalBitrotVerified") is True and observed["sha256"] == shard["sha256"] + and observed["physicalSize"] == shard["size"], "rejoined shard failed its exact committed checksum") + proofs.append({"node": shard["node"], "disk": shard["disk"], "key": target["key"], "versionId": shard["versionId"]}) + fleet.start() + status = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, option_overrides={"recursive": True}) + responsibility = validate_version_receipts(status, bucket, expected, 2) + fleet.stop() + final = fleet.snapshot("physical-after-verifier", quiescent=True) + require(physical_content(repaired) == physical_content(final), "verification task unexpectedly repaired or changed the MRF result") + fleet.start() + for endpoint in binding["endpoints"]: + entry_probe = Probe(binding, endpoint) + current = fleet.record("s3-rejoined-" + endpoint, entry_probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + compare_inventories(acknowledged, current) + for key, version in set(previous) - retained: + params = {"Bucket": bucket, "Key": key} + if version != "null": params["VersionId"] = version + observed = fleet.record("deleted-identity-read-" + endpoint, entry_probe.s3("get_object", params)) + validate_deleted_read(observed, version) + result.update(verdict="PASS", allNodeS3ManifestEqual=True, responsibility=responsibility, + physical={"allExpectedReplicaIdentities": True, "survivingShardsUnchanged": True, + "newTargetShardsVerified": True, "streamingShardProofs": len(proofs), + "backupRestorationUsed": False, "source": "MRF"}) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + if fleet.reserved: + try: + fleet.stop() + fleet.snapshot("physical-failure-scene", quiescent=True) + except Exception as capture: + fleet.record("failure-scene-error", {"message": str(capture)}) + finally: + for path, token in active: + try: fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: fleet.record("stop-error", {"message": str(exc)}) + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("topology-result", result) + return result + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f"node-outage workflow signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ("candidate", "rc-bin", "evidence", "tool-directory"): parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--case-id", choices=("heal.node-outage-rejoin",), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) == 4 and len(set(nodes)) == 4 and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four existing Heal nodes required") + require(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha) and re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")), "fixed workflow/candidate identities required") + require(list(BRANCHES) == select(args.case_id)[0]["variants"], "node-outage executor omitted a source branch") + results = [] + try: + for branch in BRANCHES: + results.append(run_branch(args, branch, nodes)) + if results[-1]["cleanup"] != "PASS": break + finally: + complete = len(results) == len(BRANCHES) and all(item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) + save(args.evidence / "healthy-summary.json", {"caseId": args.case_id, "results": results, + "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", "complete": complete}) + require(complete, "B09 requires both complete H2 branches; inspect topology-result evidence") + + +if __name__ == "__main__": + try: main() + except Exception as exc: + print(f"Heal node outage validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/prepare_heal_meta_inspector.py b/scripts/validation_heal/prepare_heal_meta_inspector.py index 3ce52020..47b8390c 100644 --- a/scripts/validation_heal/prepare_heal_meta_inspector.py +++ b/scripts/validation_heal/prepare_heal_meta_inspector.py @@ -32,12 +32,15 @@ def quorum_contract(source: Path, commit: str) -> dict: errors = compact[paths[0]] known_read = ("letdata_blocks=parts_metadata.len()asi32-parity_blocks;" in function and function.endswith("Ok((data_blocks,write_quorum))}")) + known_write = known_read and function.endswith("letdata_blocks=parts_metadata.len()asi32-parity_blocks;" + "letwrite_quorum=ifdata_blocks==parity_blocks{data_blocks+1}else{data_blocks};Ok((data_blocks,write_quorum))}") known_error = ('StorageError::ErasureReadQuorum|StorageError::InsufficientReadQuorum(_,_)=>{' 'S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into())}' in errors and 'S3ErrorCode::Custom(custom)if&**custom==SLOW_DOWN_READ_CODE=>Some(StatusCode::SERVICE_UNAVAILABLE)' in errors) code = re.search(r'const\s+SLOW_DOWN_READ_CODE\s*:\s*&str\s*=\s*"([A-Za-z0-9]+)"\s*;', contents[paths[0]]) return {"candidateCommit": commit, "recognized": bool(known_read and known_error and code), "readQuorumRule": "decoded-data-blocks" if known_read else None, + "writeQuorumRule": "data-plus-one-if-equal-parity" if known_write else None, "unrecoverableGet": {"httpStatus": 503, "code": code[1]} if known_error and code else None, "sourceFiles": {path: hashlib.sha256(value.encode()).hexdigest() for path, value in contents.items()}, "extractorSourceSha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest()} diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 0c59b324..cf2aac7f 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -19,6 +19,7 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization) ;; + heal.node-outage-rejoin) driver=node_outage.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; diff --git a/test_validation_heal.py b/test_validation_heal.py index 2152a361..81c622d7 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -26,6 +26,8 @@ from scripts.validation_heal.heal_probe import Probe from scripts.validation_heal.prepare_heal_meta_inspector import quorum_contract, size_contract from scripts.validation_heal.boundaries import verify_size_contract, boundary_sizes, verify_boundary_ranges +from scripts.validation_heal.node_outage import quorum_plan, put_ack, delete_ack, replica_census, validate_deleted_read +from scripts.validation_heal.fleet_node import runtime_config, BACKGROUND_CONFIG class HealContractTest(unittest.TestCase): @@ -836,5 +838,115 @@ def test_version_fault_plan_uses_ack_versions_distinct_disks_and_exact_recursive self.assertTrue(all(entry["versionId"] == "00000000-0000-0000-0000-000000000004" for entry in plan["faults"])) +class HealNodeOutageTest(unittest.TestCase): + def snapshot(self): + bucket = "rv-heal-0123456789abcdef" + identities = ["00000000-0000-0000-0000-" + f"{index:012d}" for index in range(1, 17)] + expected = [{"key": "new.bin", "versionId": "new-version", "deleteMarker": False, "isLatest": False}, + {"key": "new.bin", "versionId": "marker-version", "deleteMarker": True, "isLatest": True}] + observation = {} + data_dir = "00000000-0000-0000-0000-000000000100" + for node_index in range(4): + node = "node" + str(node_index + 1) + value = {"formats": [], "files": []} + for disk in range(4): + slot = node_index * 4 + disk + value["formats"].append({"disk": disk, "value": {"format": "xl", "id": "pool", + "xl": {"sets": [identities], "this": identities[slot]}}}) + version = {"versionId": "new-version", "deleted": False, "isLatest": False, "inline": None, + "dataDir": data_dir, "size": 1048623, "usesLegacyChecksum": False, + "role": "data" if slot < 8 else "parity", "parts": [{"number": 1, "size": 1048623, "actualSize": 1048623}], + "erasure": {"data_blocks": 8, "parity_blocks": 8, "index": slot + 1, + "distribution": list(range(1, 17)), "block_size": 1048576, "algorithm": "rs-vandermonde"}} + marker = {"versionId": "marker-version", "deleted": True, "isLatest": True} + value["files"].extend([ + {"disk": disk, "path": f"{bucket}/new.bin/xl.meta", "sha256": "meta" + str(slot), "size": 1000, + "metadata": {"bucket": bucket, "key": "new.bin", "versions": [version, marker]}}, + {"disk": disk, "path": f"{bucket}/new.bin/{data_dir}/part.1", "sha256": "shard" + str(slot), "size": 131200}]) + observation[node] = value + return bucket, expected, observation + + def test_mrf_profile_is_explicit_and_cannot_enable_unrelated_background_sources(self): + self.assertEqual(runtime_config("manual"), BACKGROUND_CONFIG) + replay = runtime_config("offline-replay") + self.assertEqual(replay.pop("RUSTFS_HEAL_MRF_ENABLE"), "true") + self.assertEqual(replay, {key: value for key, value in BACKGROUND_CONFIG.items() if key != "RUSTFS_HEAL_MRF_ENABLE"}) + with self.assertRaises(ValueError): runtime_config("arbitrary") + + def test_outage_calculates_same_set_quorum_and_rejects_unrecognized_write_policy(self): + bucket, expected, snapshot = self.snapshot() + target = map_target(snapshot, bucket, "new.bin", "data", 1, "new-version", latest_required=False) + contract = {"recognized": True, "writeQuorumRule": "data-plus-one-if-equal-parity"} + plan = quorum_plan(target, "node4", contract) + self.assertEqual((plan["lostSameSetDrives"], plan["remainingSameSetDrives"], plan["readQuorum"], plan["writeQuorum"]), (4, 12, 8, 9)) + for changed in ({**contract, "writeQuorumRule": None}, {**contract, "recognized": False}): + with self.assertRaises(ValueError): quorum_plan(target, "node4", changed) + with self.assertRaises(ValueError): quorum_plan(target, "foreign-node", contract) + changed = copy.deepcopy(target) + changed["logical"]["erasure"].update(data_blocks=13, parity_blocks=3) + with self.assertRaises(ValueError): quorum_plan(changed, "node4", contract) + + def test_ack_manifest_preserves_history_and_rejects_ambiguous_put_delete_receipts(self): + old = [{"key": "key", "versionId": "v1", "deleteMarker": False, "isLatest": False}, + {"key": "key", "versionId": "v2", "deleteMarker": False, "isLatest": True}] + updated = put_ack(old, "key", b"new bytes", {"VersionId": "v3"}, "DV") + self.assertEqual([entry["versionId"] for entry in updated if entry["isLatest"]], ["v3"]) + self.assertEqual(old[-1]["isLatest"], True) + marked = delete_ack(updated, "key", {"VersionId": "marker", "DeleteMarker": True}, "DV") + self.assertEqual([entry["versionId"] for entry in marked if entry["isLatest"]], ["marker"]) + deleted = delete_ack(marked, "key", {"VersionId": "v1"}, "DV", version="v1") + self.assertEqual({entry["versionId"] for entry in deleted}, {"v2", "v3", "marker"}) + for response in ({}, {"VersionId": "null"}, {"VersionId": "v2"}): + with self.subTest(response=response), self.assertRaises(ValueError): put_ack(old, "key", b"x", response, "DV") + for response in ({}, {"VersionId": "marker", "DeleteMarker": False}): + with self.subTest(response=response), self.assertRaises(ValueError): delete_ack(updated, "key", response, "DV") + with self.assertRaises(ValueError): delete_ack(old, "key", {"VersionId": "v2"}, "DV", version="v2") + unversioned = [{"key": "key", "versionId": "null", "deleteMarker": False, "isLatest": True}] + self.assertEqual(len(put_ack(unversioned, "key", b"x", {}, "DB")), 1) + self.assertEqual(delete_ack(unversioned, "key", {}, "DB"), []) + + def test_physical_census_requires_each_drive_and_preserves_surviving_authority(self): + bucket, expected, repaired = self.snapshot() + offline = copy.deepcopy(repaired) + offline["node4"]["files"] = [] + self.assertEqual(len(replica_census(repaired, bucket, expected, offline)), 1) + for poison in ("missing-one-replica", "missing-marker", "resurrected-version", "changed-survivor", "role-mismatch"): + changed = copy.deepcopy(repaired) + if poison == "missing-one-replica": changed["node4"]["files"].pop(0) + elif poison == "missing-marker": changed["node4"]["files"][0]["metadata"]["versions"].pop() + elif poison == "resurrected-version": changed["node4"]["files"][0]["metadata"]["versions"].append({"versionId": "deleted-version", "deleted": False, "isLatest": False}) + elif poison == "changed-survivor": changed["node1"]["files"][1]["sha256"] = "different" + else: changed["node4"]["files"][0]["metadata"]["versions"][0]["role"] = "data" + with self.subTest(poison=poison), self.assertRaises(ValueError): replica_census(changed, bucket, expected, offline) + + def test_candidate_write_quorum_recognition_is_independent_of_read_contract(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "rustfs/src").mkdir(parents=True) + (root / "crates/ecstore/src/set_disk").mkdir(parents=True) + (root / "rustfs/src/error.rs").write_text('const SLOW_DOWN_READ_CODE: &str = "SlowDownRead";\n' + 'StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _) => { S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into()) }\n' + 'S3ErrorCode::Custom(custom) if &**custom == SLOW_DOWN_READ_CODE => Some(StatusCode::SERVICE_UNAVAILABLE)') + metadata = root / "crates/ecstore/src/set_disk/metadata.rs" + source = 'pub(super) fn object_quorum_from_meta() { let data_blocks = parts_metadata.len() as i32 - parity_blocks; let write_quorum = if data_blocks == parity_blocks { data_blocks + 1 } else { data_blocks }; Ok((data_blocks, write_quorum)) }' + metadata.write_text(source) + self.assertEqual(quorum_contract(root, "a" * 40)["writeQuorumRule"], "data-plus-one-if-equal-parity") + metadata.write_text(source.replace("data_blocks + 1", "data_blocks + 2")) + changed = quorum_contract(root, "a" * 40) + self.assertTrue(changed["recognized"]) + self.assertIsNone(changed["writeQuorumRule"]) + metadata.write_text(source.replace("Ok((data_blocks, write_quorum))", "let write_quorum = write_quorum + 1; Ok((data_blocks, write_quorum))")) + self.assertIsNone(quorum_contract(root, "a" * 40)["writeQuorumRule"]) + + def test_deleted_identity_requires_exact_s3_absence_without_payload_or_transport_ambiguity(self): + for version, code in (("null", "NoSuchKey"), ("deleted-version", "NoSuchVersion")): + good = {"acknowledged": False, "resultUnknown": False, + "response": {"ResponseMetadata": {"HTTPStatusCode": 404}, "Error": {"Code": code}}} + validate_deleted_read(good, version) + for changed in ({**good, "acknowledged": True}, {**good, "resultUnknown": True}, {**good, "payload": {}}, + {**good, "response": {"ResponseMetadata": {"HTTPStatusCode": 403}, "Error": {"Code": "AccessDenied"}}}): + with self.subTest(version=version), self.assertRaises(ValueError): validate_deleted_read(changed, version) + + if __name__ == "__main__": unittest.main() From f25febf1827fc1ef603b4a72f6337ab181e83d23 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 00:10:13 +0800 Subject: [PATCH 046/343] fix(validation): use candidate-compatible Heal Admin policy --- scripts/validation_heal/authorization.py | 4 +++- test_validation_heal_authorization.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/authorization.py b/scripts/validation_heal/authorization.py index b5c0ad72..4a6d8c26 100644 --- a/scripts/validation_heal/authorization.py +++ b/scripts/validation_heal/authorization.py @@ -116,7 +116,9 @@ def authorization_requests(probe, fleet, bucket, before, baseline, active, optio users.append(user) # Preserve ownership even if admission loses its reply. iam_exchange(probe, fleet, "add-user", user, secret=secret) policy_created = True - document = {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Action": ["admin:Heal"], "Resource": ["*"]}]} + # The candidate admits unscoped Admin actions without an S3 Resource; + # its resource parser correctly rejects a bare '*' as an invalid ARN. + document = {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Action": ["admin:Heal"]}]} iam_exchange(probe, fleet, "add-policy", allowed_user, policy=policy, document=document) attached = True iam_exchange(probe, fleet, "attach", allowed_user, policy=policy) diff --git a/test_validation_heal_authorization.py b/test_validation_heal_authorization.py index 75ad4477..b139a9b1 100644 --- a/test_validation_heal_authorization.py +++ b/test_validation_heal_authorization.py @@ -139,6 +139,8 @@ def inventory(self, *args, **kwargs): return {} def exchange(probe, recorded, operation, user, **kwargs): iam_calls.append((operation, user)) + if operation == "add-policy": + self.assertEqual(kwargs["document"], {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Action": ["admin:Heal"]}]}) if operation == "add-user": users.add(user) if operation == failed_operation: raise ValueError("injected IAM failure: " + operation) if operation == "remove-user": users.discard(user) From ea5e46499f07aa20a7e72d505ecf687abaaabfc0 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 00:24:28 +0800 Subject: [PATCH 047/343] fix(validation): freeze unsigned Heal router refusal --- scripts/validation_heal/authorization.py | 4 +++- test_validation_heal_authorization.py | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/authorization.py b/scripts/validation_heal/authorization.py index 4a6d8c26..b32a93bb 100644 --- a/scripts/validation_heal/authorization.py +++ b/scripts/validation_heal/authorization.py @@ -20,7 +20,9 @@ from healthy import physical_content -DENIALS = {"anonymous": (400, "InvalidRequest"), "bad-signature": (403, "SignatureDoesNotMatch"), +# The candidate's S3Router::check_access rejects unsigned Admin traffic +# before authorize_admin_request (whose missing-credential error is 400). +DENIALS = {"anonymous": (403, "AccessDenied"), "bad-signature": (403, "SignatureDoesNotMatch"), "expired-signature": (403, "RequestTimeTooSkewed"), "denied-user": (403, "AccessDenied")} diff --git a/test_validation_heal_authorization.py b/test_validation_heal_authorization.py index b139a9b1..72674fb7 100644 --- a/test_validation_heal_authorization.py +++ b/test_validation_heal_authorization.py @@ -86,6 +86,12 @@ def test_denial_requires_exact_code_and_no_nested_json_or_xml_disclosure(self): with self.subTest(mode=mode, response=poisoned), self.assertRaises(ValueError): validate_denial(poisoned, mode) + def test_unsigned_route_refusal_is_not_the_unreachable_handler_error(self): + self.assertEqual(DENIALS["anonymous"], (403, "AccessDenied")) + validate_denial({"httpStatus": 403, "response": 'AccessDeniedSignature is required'}, "anonymous") + with self.assertRaises(ValueError): + validate_denial({"httpStatus": 400, "response": {"Code": "InvalidRequest"}}, "anonymous") + def test_iam_evidence_redacts_credentials_without_modifying_oracle_input(self): with tempfile.TemporaryDirectory() as directory: fleet = Fleet([], Path(directory), "manual-auth-test") From 251d996339f3fd0ed922fc2b9a347c02bff33efd Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 00:30:15 +0800 Subject: [PATCH 048/343] feat(validation): automate protected Object Lock Heal cases --- docs/validation-heal-contract.md | 2 +- docs/validation-migration-map.md | 41 +++++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/object_lock.py | 189 +++++++++++++++++++++++++ scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 35 ++++- test_validation_heal.py | 162 +++++++++++++++++++++ 8 files changed, 428 insertions(+), 7 deletions(-) create mode 100644 scripts/validation_heal/object_lock.py diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index 2ee4c10e..a7219962 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -23 个用例已接入自动分发：A01、A03、A04、A07、A08、B05、B07、B09、C01–C05、D01–D03、D05–D07、D10、E01、E02、J04。分片执行器使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。A08 保留五种主体和各自 start/status/stop 操作，使用真正重新计算的过期签名及仅授权 `admin:Heal` 的临时用户，清理后验证凭据失效。B09 使用明确启用 MRF 的专用服务；停止一个跨节点 set 成员节点，记录降级 ACK，并先验证逐盘补片，再执行 S3/手工验证请求。其余 61 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 +24 个用例已接入自动分发：A01、A03、A04、A07、A08、B05、B07、B09、C01–C05、D01–D03、D05–D07、D09、D10、E01、E02、J04。分片执行器使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。A08 保留五种主体和各自 start/status/stop 操作，使用真正重新计算的过期签名及仅授权 `admin:Heal` 的临时用户，清理后验证凭据失效。B09 使用明确启用 MRF 的专用服务；停止一个跨节点 set 成员节点，记录降级 ACK，并先验证逐盘补片，再执行 S3/手工验证请求。其余 60 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 07845141..7367afed 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 97 currently dispatchable IDs include all 74 non-Heal cases and twenty-three Heal cases. Forty-one have live PASS evidence; the other 117 have no live PASS claim. +case evidence. The 98 currently dispatchable IDs include all 74 non-Heal cases and twenty-four Heal cases. Forty-one have live PASS evidence; the other 117 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 61 Heal cases remain +The other 60 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -377,3 +377,40 @@ Deleted identities receive exact S3 absence checks. No copied backup contributes to repair. Six focused tests cover profile boundaries, source-quorum recognition, ACK ambiguity, per-drive omissions/resurrection and deleted-read refusals. B09 requires the fresh write-quorum tool artifact and full live evidence. + +D01 run `38064187494` completed all 96 H1/H2 parameters: 76 passed and 20 +failed the then-strict existing-backup oracle; cleanup passed. Independent +forensic comparison of all 20 scenes found only target `xl.meta.bkp` updates +to the exact pre-repair authoritative metadata. The corrected oracle now +bounds both new and existing backup updates to that drive/key/dataDir and +exact size/hash. The original FAIL is retained; a fresh full run is required. + +A08 runs `38066188001` and `38066437827` failed policy creation because bare +`Resource:["*"]` is not an ARN. The fixed candidate accepts `admin:Heal` +without Resource. Corrected run `38066970195` reached every branch: bad and +expired signatures, denied user and least-privilege user passed on H1/H2. +Anonymous traffic returned 403/AccessDenied from S3Router::check_access before +the handler, exposing an incorrect 400/InvalidRequest harness expectation. +The route-level expectation is now exact; all original FAILs retain cleanup +PASS and none is promoted without a fresh complete run. + +Tool build `38065628505` succeeded against the same fixed candidate and carries +the recognized `data-plus-one-if-equal-parity` write-quorum policy. Its helper, +extractor, candidate and binary digests were verified locally; it is used by +the serial live queue for D01/B07/B09. Ordinary runs still need no RustFS checkout. + +D09 `heal.object-lock-retention` now dispatches all four H1/H2 source branches +on real Object Lock/versioning buckets. Three versions per key retain exact +Governance/Compliance metadata or ON legal hold; the ten-minute UTC window is +anchored to the server HTTP Date. A run-owned principal can read but explicitly +cannot bypass Governance. Baseline and pre/post-fault DELETE must return exact +403/AccessDenied while retention is active. Data/parity faults and whole xl.meta +loss cover current/historical versions; all affected version shards, public +properties, tags and responsibility receipts remain independently checked. +Finalization releases hold and Governance only through normal APIs, waits up to +twelve minutes for real Compliance expiry and deletion ACK, proves the bucket +empty/absent, and removes IAM resources. Only then may physical roots be removed. +Incomplete finalization retains a stopped namespace with cleanup FAIL. Partial +seed ownership, real server time, same-drive multi-version proof selection and +protected cleanup failure boundaries have focused tests. Full live evidence +is still required. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index cae30abb..6ccea865 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index f0f75c1d..134f9134 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/object_lock.py b/scripts/validation_heal/object_lock.py new file mode 100644 index 00000000..c9a73177 --- /dev/null +++ b/scripts/validation_heal/object_lock.py @@ -0,0 +1,189 @@ +"""D09 protection proofs and S3 finalization after real Compliance expiry.""" + +from datetime import UTC, timedelta +from email.utils import parsedate_to_datetime +import hashlib +import secrets +import time + +try: + from .authorization import iam_exchange + from .heal_probe import Probe, payload, require +except ImportError: + from authorization import iam_exchange + from heal_probe import Probe, payload, require + + +BRANCHES = ("governance-retention", "compliance-retention", "legal-hold", "historical-protected-version") +KEYS = ("locked/governance.bin", "locked/compliance.bin", "locked/hold.bin", "locked/historical.bin") +SIZE = 5 * 1024**2 + 123 +RETENTION_MINUTES = 10 +CLEANUP_SECONDS = (RETENTION_MINUTES + 2) * 60 + + +def seed_locked(recorder, fleet, bucket, context): + probe = recorder.probe + suffix = hashlib.sha256(probe.binding["runId"].encode()).hexdigest()[:16] + user, policy, secret = "rvheal" + suffix + "locked", "rvheal" + suffix + "lockpolicy", secrets.token_hex(24) + context.update(user=user, policy=policy, principal=Probe(probe.binding, credential_pair=(user, secret)), + policyCreated=False, attached=False, bucketAttempted=False) + fleet.secret_values.update({user, secret, *probe.credentials()}) + iam_exchange(probe, fleet, "add-user", user, secret=secret) + context["policyCreated"] = True + document = {"Version": "2012-10-17", "Statement": [ + {"Effect": "Allow", "Action": ["s3:*"], "Resource": [f"arn:aws:s3:::{bucket}", f"arn:aws:s3:::{bucket}/*"]}, + {"Effect": "Deny", "Action": ["s3:BypassGovernanceRetention"], "Resource": [f"arn:aws:s3:::{bucket}/*"]}]} + iam_exchange(probe, fleet, "add-policy", user, policy=policy, document=document) + context["attached"] = True + iam_exchange(probe, fleet, "attach", user, policy=policy) + context["bucketAttempted"] = True + recorder.call("create_bucket", {"Bucket": bucket, "ObjectLockEnabledForBucket": True}) + mode = recorder.call("get_bucket_versioning", {"Bucket": bucket}) + require(mode.get("Status") == "Enabled", "Object Lock bucket did not enable real versioning") + lock = recorder.call("get_object_lock_configuration", {"Bucket": bucket}) + require(lock.get("ObjectLockConfiguration", {}).get("ObjectLockEnabled") == "Enabled", "Object Lock is not active") + server_date = lock.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("date") + require(isinstance(server_date, str), "protected fixture needs a server-time retention anchor") + server_time = parsedate_to_datetime(server_date) + require(server_time.tzinfo is not None, "retention server time is not timezone-aware") + until = server_time.astimezone(UTC).replace(microsecond=0) + timedelta(minutes=RETENTION_MINUTES) + context["retainUntil"] = until + expected, protections = [], [] + for index, key in enumerate(KEYS): + protection = {"ObjectLockLegalHoldStatus": "ON"} if index == 2 else { + "ObjectLockMode": "COMPLIANCE" if index == 1 else "GOVERNANCE", "ObjectLockRetainUntilDate": until} + for nonce in range(3): + body = payload(f"{bucket}/{key}/{nonce}", SIZE) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "Metadata": {"heal-fixture": "object-lock", "generation": str(nonce)}, + "Tagging": f"suite=heal&generation=v{nonce}", **protection}, body) + version = response.get("VersionId") + require(version not in {None, "", "null"}, "protected PUT omitted its acknowledged version") + expected.append({"key": key, "versionId": version, "deleteMarker": False, "isLatest": nonce == 2, + "payload": {"size": SIZE, "sha256": hashlib.sha256(body).hexdigest()}}) + protections.append({"key": key, "versionId": version, **protection}) + context["protections"] = protections + fleet.record("lock-fixture", {"protections": protections, "complianceUsesRealExpiry": True, + "principalBypassGovernanceDenied": True}) + return expected + + +def validate_protection(observed, spec): + response = observed.get("response") or {} + require(observed.get("acknowledged") is True, "protected metadata request failed") + if "ObjectLockLegalHoldStatus" in spec: + require(response.get("LegalHold", {}).get("Status") == spec["ObjectLockLegalHoldStatus"], "legal hold changed or disappeared") + else: + retention = response.get("Retention", {}) + require(retention.get("Mode") == spec["ObjectLockMode"] and retention.get("RetainUntilDate") == spec["ObjectLockRetainUntilDate"], + "retention mode or exact UTC expiry changed") + server_date = response.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("date") + require(isinstance(server_date, str) and parsedate_to_datetime(server_date) < spec["ObjectLockRetainUntilDate"], + "retention proof did not occur during the real server-time protection window") + + +def validate_protected_delete(observed): + response = observed.get("response") or {} + require(observed.get("acknowledged") is False and observed.get("resultUnknown") is False and "payload" not in observed + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 403 + and response.get("Error", {}).get("Code") == "AccessDenied", "protected DELETE was acknowledged or did not return exact AccessDenied") + + +def prove_locked(probe, fleet, bucket, context, *, key=None, label): + selected = [spec for spec in context["protections"] if key is None or spec["key"] == key] + require(selected, "protected branch has no acknowledged versions") + for spec in selected: + params = {"Bucket": bucket, "Key": spec["key"], "VersionId": spec["versionId"]} + operation = "get_object_legal_hold" if "ObjectLockLegalHoldStatus" in spec else "get_object_retention" + observed = fleet.record(label + "-protection", probe.s3(operation, params)) + validate_protection(observed, spec) + # Read succeeds under the same non-bypass principal, so an IAM-wide + # access denial cannot masquerade as enforced object protection. + readable = fleet.record(label + "-authorized-head", context["principal"].s3("head_object", params)) + require(readable.get("acknowledged") is True and readable["response"].get("VersionId") == spec["versionId"], + "non-bypass principal cannot read the protected version") + denied = fleet.record(label + "-delete-denied", context["principal"].s3("delete_object", params)) + validate_protected_delete(denied) + if spec.get("ObjectLockMode") == "GOVERNANCE": + denied = fleet.record(label + "-bypass-denied", context["principal"].s3("delete_object", {**params, "BypassGovernanceRetention": True})) + validate_protected_delete(denied) + return {"protectedVersions": len(selected), "exactProtectionMetadata": True, "nonBypassDeleteDenied": True, + "serverTimeBeforeExpiry": True} + + +def finalize_locked(probe, fleet, bucket, context): + """Remove protection through APIs; retain the namespace on incomplete cleanup.""" + errors = [] + try: + if fleet.reserved: fleet.stop() + if fleet.faults: + for registration in list(fleet.faults): + node, params = registration + fleet.record("locked-fault-rescue", fleet.remote(node, "recover-fault", **params)) + fleet.faults.remove(registration) + if fleet.reserved: fleet.start() + if context.get("bucketAttempted"): + listing = fleet.record("locked-final-list", probe.s3("list_object_versions", {"Bucket": bucket})) + response = listing.get("response") or {} + absent = not listing.get("acknowledged") and response.get("Error", {}).get("Code") == "NoSuchBucket" + require(absent or listing.get("acknowledged") and not response.get("IsTruncated"), "protected finalization inventory is incomplete") + versions = [] if absent else [{"Key": entry["Key"], "VersionId": entry["VersionId"], "marker": marker} + for field, marker in (("Versions", False), ("DeleteMarkers", True)) for entry in response.get(field, [])] + require(len(versions) <= 32, "protected cleanup exceeded the owned fixture budget") + deadline = time.monotonic() + CLEANUP_SECONDS + for version in versions: + params = {"Bucket": bucket, "Key": version["Key"], "VersionId": version["VersionId"]} + require(version["Key"] in KEYS, "protected finalization found an unowned key") + if not version["marker"]: + hold = fleet.record("locked-final-hold", probe.s3("get_object_legal_hold", params)) + value = (hold.get("response") or {}).get("LegalHold", {}) + if hold.get("acknowledged") and value.get("Status") == "ON": + changed = fleet.record("locked-final-hold-off", probe.s3("put_object_legal_hold", {**params, "LegalHold": {"Status": "OFF"}})) + require(changed.get("acknowledged") is True, "legal hold could not be released by the owner") + retention = fleet.record("locked-final-retention", probe.s3("get_object_retention", params)) + if (retention.get("response") or {}).get("Retention", {}).get("Mode") == "GOVERNANCE": + params["BypassGovernanceRetention"] = True + while True: + deleted = fleet.record("locked-final-delete", probe.s3("delete_object", params)) + if deleted.get("acknowledged"): + require(deleted["response"].get("ResponseMetadata", {}).get("HTTPStatusCode") in {200, 204}, "protected final DELETE had an unexpected status") + break + # A Compliance version stays on disk until the server + # accepts its deletion after actual expiry. No clock edits, + # shortened retention or internal-file removal are used. + validate_protected_delete(deleted) + require(time.monotonic() < deadline, "protected data is still retained past the cleanup budget") + time.sleep(2) + if not absent: + empty = fleet.record("locked-final-empty", probe.s3("list_object_versions", {"Bucket": bucket})) + require(empty.get("acknowledged") and not any(empty["response"].get(field) for field in ("Versions", "DeleteMarkers", "IsTruncated")), + "protected versions or markers remain after API finalization") + removed = fleet.record("locked-final-bucket-delete", probe.s3("delete_bucket", {"Bucket": bucket})) + require(removed.get("acknowledged") is True, "protected bucket finalization failed") + absent = fleet.record("locked-final-bucket-absence", probe.s3("head_bucket", {"Bucket": bucket})) + require(absent.get("acknowledged") is False and absent.get("resultUnknown") is False + and (absent.get("response") or {}).get("ResponseMetadata", {}).get("HTTPStatusCode") == 404, + "protected bucket absence is unproven") + except Exception as exc: + errors.append(str(exc)) + user, policy = context.get("user"), context.get("policy") + for operation, enabled in (("detach", context.get("attached")), ("remove-user", user is not None), ("remove-policy", context.get("policyCreated"))): + if enabled: + try: iam_exchange(probe, fleet, operation, user, policy=policy if operation != "remove-user" else None) + except Exception as exc: errors.append(str(exc)) + if errors: + try: + fleet.stop() + for node in fleet.reserved: fleet.record("locked-retained-logs", {"node": node, **fleet.remote(node, "logs")}) + fleet.snapshot("locked-retained-scene", quiescent=True) + except Exception as exc: errors.append(str(exc)) + fleet.record("locked-finalization", {"cleanup": "FAIL" if errors else "PASS", "errors": errors, + "state": "RETAINED" if errors else "API_FINALIZED", "namespaceDeletionAllowed": not errors, + "complianceExpiryShortened": False, "internalProtectedFileDeletionUsed": False}) + return not errors + + +def cleanup_locked_fleet(probe, fleet, bucket, context): + # Physical namespace deletion is permitted only after protected versions + # and temporary IAM resources have been finalized through their APIs. + return finalize_locked(probe, fleet, bucket, context) and fleet.cleanup() diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index cf2aac7f..3aa660ee 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -20,7 +20,7 @@ done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization) ;; heal.node-outage-rejoin) driver=node_outage.py ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 1cb0227a..57e45b8c 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -26,6 +26,7 @@ from .heal_fault import bounded_fault_id from .boundaries import (BRANCHES as SIZE_BRANCHES, verify_size_contract, boundary_sizes, seed_boundaries, verify_boundary_ranges) + from .object_lock import BRANCHES as LOCK_BRANCHES, KEYS as LOCK_KEYS, seed_locked, prove_locked, cleanup_locked_fleet from .shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) except ImportError: @@ -37,6 +38,7 @@ from heal_fault import bounded_fault_id from boundaries import (BRANCHES as SIZE_BRANCHES, verify_size_contract, boundary_sizes, seed_boundaries, verify_boundary_ranges) + from object_lock import BRANCHES as LOCK_BRANCHES, KEYS as LOCK_KEYS, seed_locked, prove_locked, cleanup_locked_fleet from shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) @@ -52,6 +54,7 @@ "heal.multiple-shard-loss": ("loss-1-data", "loss-1-parity", "loss-2-through-m-data-parity"), "heal.special-keys-and-pagination": ("unicode-space-percent", "encoded-slash", "adjacent-prefixes", "multi-page-list"), "heal.object-size-boundaries": SIZE_BRANCHES, + "heal.object-lock-retention": LOCK_BRANCHES, } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") @@ -446,6 +449,22 @@ def reenable_and_verify(recorder, fleet, bucket, baseline): def branch_plan(case_id, branch, snapshot, bucket, expected, fixture): """Plans are derived from ACK order and decoded physical roles, never disk numbers.""" + if case_id == "heal.object-lock-retention": + index = LOCK_BRANCHES.index(branch) + key = LOCK_KEYS[index] + selected = [entry for entry in expected if entry["key"] == key] + require(len(selected) == 3 and all(not entry["deleteMarker"] for entry in selected), "protected fixture lost its three distinct versions") + target = choose_target(snapshot, bucket, selected[0 if index == 3 else -1], "parity" if index == 1 else "data") + metadata = index >= 2 + proofs = [target] + if metadata: + proofs = [choose_target(snapshot, bucket, entry, "data") for entry in selected] + proofs = [{**item, **next(shard for shard in item["cohort"] + if (shard["node"], shard["disk"]) == (target["node"], target["disk"]))} for item in proofs] + return {"scope": {"bucket": bucket, "key": key}, "recursive": True, "expected": selected, + "faults": [metadata_target(snapshot, target)] if metadata else [target], "proofs": proofs, + "requiredRepairs": set() if metadata else {version_identity(target)}, + "metadataRepairs": {key} if metadata else set()} branch_index = BRANCHES[case_id].index(branch) key = f"objects/{branch_index:04d}.bin" if fixture == "DV" else f"objects/{8 + 9 * branch_index:04d}.bin" if branch == "versioned-repeat": @@ -614,6 +633,7 @@ def run_topology(args, topology, nodes, fixture): result = {"topology": topology, "fixture": fixture, "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} active, admissions = [], [] probe = Probe(binding) + lock_context = {} if args.case_id == "heal.object-lock-retention" else None try: fleet.deploy(args.binary_sha, bucket, access, secret) processes = fleet.snapshot("deployed-topology") @@ -628,6 +648,8 @@ def run_topology(args, topology, nodes, fixture): fleet.record("candidate-size-contract", {"sourceContract": size_rule, "parameters": sizes}) extra, instances = seed_boundaries(recorder, bucket, sizes) expected += extra + elif lock_context is not None: + expected = seed_locked(recorder, fleet, bucket, lock_context) elif fixture == "DK": expected = seed_special_keys(recorder, bucket) elif fixture == "DS": @@ -641,6 +663,8 @@ def run_topology(args, topology, nodes, fixture): baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=fixture in {"DB", "DK"}, page_size=17 if fixture == "DK" else None)) fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + if lock_context is not None: + fleet.record("lock-baseline-proof", prove_locked(probe, fleet, bucket, lock_context, label="lock-baseline")) if fixture == "DK": fleet.record("special-key-pagination-baseline", verify_pagination(baseline)) if public is not None: @@ -658,6 +682,9 @@ def run_topology(args, topology, nodes, fixture): registrations = [] metadata_faults = {} try: + if lock_context is not None: + fleet.record("lock-before-fault", prove_locked(probe, fleet, bucket, lock_context, + key=LOCK_KEYS[LOCK_BRANCHES.index(branch)], label="lock-before-" + branch)) fleet.stop() before = fleet.snapshot("physical-before-" + branch, quiescent=True) if size_rule: @@ -723,6 +750,9 @@ def run_topology(args, topology, nodes, fixture): bitrot = [verify_proof(fleet, target, "repaired-bitrot-" + branch) for target in plan["proofs"]] fleet.start() s3 = verify_version_s3(probe, fleet, bucket, expected, baseline, "s3-after-" + branch, fixture=fixture) + if lock_context is not None: + s3["protection"] = prove_locked(probe, fleet, bucket, lock_context, + key=LOCK_KEYS[LOCK_BRANCHES.index(branch)], label="lock-after-" + branch) if size_rule: s3["actualBoundaryRanges"] = verify_boundary_ranges(probe, fleet, bucket, expected, sizes["actualECBlockSize"]) # These faults have now been independently repaired and fully @@ -805,7 +835,8 @@ def run_topology(args, topology, nodes, fixture): fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) except Exception as exc: fleet.record("stop-error", {"error": str(exc)}) - result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + cleaned = fleet.cleanup() if lock_context is None else cleanup_locked_fleet(probe, fleet, bucket, lock_context) + result["cleanup"] = "PASS" if cleaned else "FAIL" fleet.record("topology-result", result) return result @@ -835,6 +866,8 @@ def interrupted(signum, frame): fixtures = ("DK",) if args.case_id == "heal.object-size-boundaries": fixtures = ("DB",) + if args.case_id == "heal.object-lock-retention": + fixtures = ("DO",) results = [] try: for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): diff --git a/test_validation_heal.py b/test_validation_heal.py index 81c622d7..6d2ce639 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -7,6 +7,7 @@ import unittest import copy import gzip +from datetime import UTC, datetime, timedelta import types from unittest.mock import patch @@ -28,6 +29,8 @@ from scripts.validation_heal.boundaries import verify_size_contract, boundary_sizes, verify_boundary_ranges from scripts.validation_heal.node_outage import quorum_plan, put_ack, delete_ack, replica_census, validate_deleted_read from scripts.validation_heal.fleet_node import runtime_config, BACKGROUND_CONFIG +from scripts.validation_heal.object_lock import (validate_protection, validate_protected_delete, cleanup_locked_fleet, + seed_locked, BRANCHES as LOCK_BRANCHES, KEYS as LOCK_KEYS, CLEANUP_SECONDS) class HealContractTest(unittest.TestCase): @@ -948,5 +951,164 @@ def test_deleted_identity_requires_exact_s3_absence_without_payload_or_transport with self.subTest(version=version), self.assertRaises(ValueError): validate_deleted_read(changed, version) +class HealObjectLockTest(unittest.TestCase): + def seed(self, *, fail_after=None): + calls, context = [], {} + fleet = types.SimpleNamespace(secret_values=set(), record=lambda label, value: value) + principal = types.SimpleNamespace(binding={"runId": "gh-12345-lock"}, credentials=lambda: ("owner", "secret")) + def invoke(operation, params, body=None): + calls.append((operation, copy.deepcopy(params))) + if fail_after is not None and operation == "put_object" and len([x for x in calls if x[0] == operation]) > fail_after: + raise ValueError("partial protected fixture") + if operation == "get_bucket_versioning": return {"Status": "Enabled"} + if operation == "get_object_lock_configuration": + return {"ObjectLockConfiguration": {"ObjectLockEnabled": "Enabled"}, + "ResponseMetadata": {"HTTPHeaders": {"date": "Sat, 10 Oct 2026 12:00:00 GMT"}}} + if operation == "put_object": return {"VersionId": "version-" + str(len(calls))} + return {} + recorder = types.SimpleNamespace(probe=principal, call=invoke) + with patch("scripts.validation_heal.object_lock.Probe", return_value=principal), \ + patch("scripts.validation_heal.object_lock.payload", return_value=b"protected"), \ + patch("scripts.validation_heal.object_lock.iam_exchange") as iam: + if fail_after is None: + expected = seed_locked(recorder, fleet, "rv-heal-lock", context) + else: + with self.assertRaisesRegex(ValueError, "partial protected fixture"): + seed_locked(recorder, fleet, "rv-heal-lock", context) + expected = None + return expected, context, calls, iam.call_args_list + + def test_protected_seed_uses_real_server_time_and_keeps_partial_ownership(self): + expected, context, calls, iam = self.seed() + self.assertEqual(len(expected), 12) + self.assertEqual(len({(entry["key"], entry["versionId"]) for entry in expected}), 12) + self.assertEqual(context["retainUntil"], datetime(2026, 10, 10, 12, 10, tzinfo=UTC)) + self.assertEqual(sum(entry["isLatest"] for entry in expected), 4) + statements = next(call.kwargs["document"]["Statement"] for call in iam if "document" in call.kwargs) + self.assertEqual(statements[-1]["Action"], ["s3:BypassGovernanceRetention"]) + self.assertEqual(statements[-1]["Effect"], "Deny") + for fail_after in (0, 1, 10): + _, partial, _, _ = self.seed(fail_after=fail_after) + self.assertTrue(partial["bucketAttempted"] and partial["policyCreated"] and partial["attached"]) + self.assertTrue(partial["user"].startswith("rvheal")) + + def test_protected_plan_freezes_all_versions_on_the_lost_metadata_drive(self): + base = HealShardOracleTest().snapshot() + bucket = "rv-heal-0123456789abcdef" + expected = [] + snapshot = {"node1": {"formats": base["node1"]["formats"], "files": []}} + for key in LOCK_KEYS: + for entry in base["node1"]["files"]: + if "/control/" in entry["path"]: continue + cloned = copy.deepcopy(entry) + cloned["path"] = cloned["path"].replace("objects/0008.bin", key) + if "metadata" in cloned: + cloned["metadata"]["key"] = key + versions = [] + for nonce in range(3): + version = copy.deepcopy(cloned["metadata"]["versions"][0]) + version.update(versionId=f"v{nonce}", isLatest=nonce == 2, + dataDir=f"12345678-1234-1234-1234-{nonce:012d}") + versions.append(version) + cloned["metadata"]["versions"] = versions + snapshot["node1"]["files"].append(cloned) + else: + for nonce in range(3): + part = copy.deepcopy(cloned) + part["path"] = part["path"].replace("12345678-1234-1234-1234-123456789abc", f"12345678-1234-1234-1234-{nonce:012d}") + part["sha256"] += f"v{nonce}" + snapshot["node1"]["files"].append(part) + expected.extend({"key": key, "versionId": f"v{nonce}", "deleteMarker": False, "isLatest": nonce == 2, + "payload": {"size": 100}} for nonce in range(3)) + for index, branch in enumerate(LOCK_BRANCHES): + plan = branch_plan("heal.object-lock-retention", branch, snapshot, bucket, expected, "DO") + self.assertEqual({item["key"] for item in plan["expected"]}, {LOCK_KEYS[index]}) + self.assertEqual(plan["faults"][0]["versionId"], "v0" if index == 3 else "v2") + if index < 2: + self.assertEqual(plan["faults"][0]["role"], "parity" if index == 1 else "data") + else: + self.assertEqual(len(plan["proofs"]), 3) + self.assertEqual({item["versionId"] for item in plan["proofs"]}, {"v0", "v1", "v2"}) + self.assertEqual(len({(item["node"], item["disk"]) for item in plan["proofs"]}), 1) + self.assertEqual(len({item["sha256"] for item in plan["proofs"]}), 3) + self.assertTrue(plan["faults"][0]["relativePath"].endswith("/xl.meta")) + + def test_retention_proof_needs_exact_mode_expiry_and_active_server_time(self): + expiry = datetime(2026, 10, 10, 12, 5, tzinfo=UTC) + spec = {"ObjectLockMode": "COMPLIANCE", "ObjectLockRetainUntilDate": expiry} + good = {"acknowledged": True, "response": {"Retention": {"Mode": "COMPLIANCE", "RetainUntilDate": expiry}, + "ResponseMetadata": {"HTTPHeaders": {"date": "Sat, 10 Oct 2026 12:00:00 GMT"}}}} + validate_protection(good, spec) + for invalid in ("mode", "expiry", "expired", "missing-server-time", "failed"): + changed = copy.deepcopy(good) + if invalid == "mode": changed["response"]["Retention"]["Mode"] = "GOVERNANCE" + elif invalid == "expiry": changed["response"]["Retention"]["RetainUntilDate"] -= timedelta(seconds=1) + elif invalid == "expired": changed["response"]["ResponseMetadata"]["HTTPHeaders"]["date"] = "Sat, 10 Oct 2026 12:05:00 GMT" + elif invalid == "missing-server-time": changed["response"].pop("ResponseMetadata") + else: changed["acknowledged"] = False + with self.subTest(invalid=invalid), self.assertRaises(ValueError): validate_protection(changed, spec) + validate_protection({"acknowledged": True, "response": {"LegalHold": {"Status": "ON"}}}, {"ObjectLockLegalHoldStatus": "ON"}) + with self.assertRaises(ValueError): + validate_protection({"acknowledged": True, "response": {"LegalHold": {"Status": "OFF"}}}, {"ObjectLockLegalHoldStatus": "ON"}) + + def test_protected_delete_cannot_pass_as_a_success_or_generic_transport_failure(self): + good = {"acknowledged": False, "resultUnknown": False, + "response": {"ResponseMetadata": {"HTTPStatusCode": 403}, "Error": {"Code": "AccessDenied"}}} + validate_protected_delete(good) + for changed in ({**good, "acknowledged": True}, {**good, "payload": {}}, {**good, "resultUnknown": True}, + {**good, "response": {"ResponseMetadata": {"HTTPStatusCode": 404}, "Error": {"Code": "NoSuchVersion"}}}): + with self.assertRaises(ValueError): validate_protected_delete(changed) + + def cleanup(self, *, permanent_denial=False, transport_unknown=False, iam_failure=False): + clock, calls, evidence, removed = [0], [], {}, [False] + fleet = types.SimpleNamespace(reserved=["node1"], faults=[], stop=lambda: None, start=lambda: None, + remote=lambda *args, **kwargs: {}, snapshot=lambda *args, **kwargs: {}, cleanup=unittest.mock.Mock(return_value=True)) + def record(label, value): evidence.setdefault(label, []).append(value); return value + fleet.record = record + def invoke(operation, params): + calls.append((operation, copy.deepcopy(params))) + if operation == "list_object_versions": + return {"acknowledged": True, "response": {} if removed[0] else {"Versions": [{"Key": LOCK_KEYS[1], "VersionId": "protected-version"}]}} + if operation == "get_object_legal_hold": return {"acknowledged": True, "response": {"LegalHold": {"Status": "OFF"}}} + if operation == "get_object_retention": return {"acknowledged": True, "response": {"Retention": {"Mode": "COMPLIANCE"}}} + if operation == "delete_object": + self.assertNotIn("BypassGovernanceRetention", params) + if permanent_denial or transport_unknown or clock[0] < 4: + return {"acknowledged": False, "resultUnknown": transport_unknown, + "response": {"ResponseMetadata": {"HTTPStatusCode": 403}, "Error": {"Code": "AccessDenied"}}} + removed[0] = True + return {"acknowledged": True, "response": {"ResponseMetadata": {"HTTPStatusCode": 204}}} + if operation == "delete_bucket": return {"acknowledged": True, "response": {}} + if operation == "head_bucket": return {"acknowledged": False, "resultUnknown": False, + "response": {"ResponseMetadata": {"HTTPStatusCode": 404}}} + self.fail("unexpected cleanup API: " + operation) + def sleep(seconds): clock[0] += CLEANUP_SECONDS + 1 if permanent_denial else seconds + context = {"bucketAttempted": True} + if iam_failure: context.update(user="rvhealowned", policy="rvhealpolicy", policyCreated=True, attached=True) + with patch("scripts.validation_heal.object_lock.time.monotonic", side_effect=lambda: clock[0]), \ + patch("scripts.validation_heal.object_lock.time.sleep", side_effect=sleep), \ + patch("scripts.validation_heal.object_lock.iam_exchange", side_effect=ValueError("IAM cleanup failure") if iam_failure else None): + result = cleanup_locked_fleet(types.SimpleNamespace(s3=invoke), fleet, "rv-heal-0123456789abcdef", context) + return result, fleet, calls, evidence, clock + + def test_physical_cleanup_waits_for_server_ack_after_real_compliance_expiry(self): + result, fleet, calls, evidence, clock = self.cleanup() + self.assertTrue(result) + fleet.cleanup.assert_called_once() + self.assertEqual(clock[0], 4) + self.assertEqual(len(evidence["locked-final-delete"]), 3) + self.assertTrue(evidence["locked-finalization"][0]["namespaceDeletionAllowed"]) + self.assertFalse(any(operation == "put_object_retention" for operation, params in calls)) + + def test_retained_or_unknown_protection_forbids_namespace_deletion(self): + for options in ({"permanent_denial": True}, {"transport_unknown": True}, {"iam_failure": True}): + with self.subTest(options=options): + result, fleet, calls, evidence, clock = self.cleanup(**options) + self.assertFalse(result) + fleet.cleanup.assert_not_called() + self.assertEqual(evidence["locked-finalization"][0]["state"], "RETAINED") + self.assertFalse(evidence["locked-finalization"][0]["namespaceDeletionAllowed"]) + + if __name__ == "__main__": unittest.main() From 27cd83f3bf07924b7d6cf4da10e05fee3885f7a4 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 00:36:06 +0800 Subject: [PATCH 049/343] feat(validation): automate metadata and payload mismatch Heal cases --- docs/validation-heal-contract.md | 2 +- docs/validation-migration-map.md | 21 +++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 3 +- scripts/validation_heal/heal_fault.py | 28 ++++- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 68 ++++++++++- test_validation_heal.py | 163 +++++++++++++++++++++----- 9 files changed, 251 insertions(+), 40 deletions(-) diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index a7219962..c5b5d63a 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -24 个用例已接入自动分发：A01、A03、A04、A07、A08、B05、B07、B09、C01–C05、D01–D03、D05–D07、D09、D10、E01、E02、J04。分片执行器使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。A08 保留五种主体和各自 start/status/stop 操作，使用真正重新计算的过期签名及仅授权 `admin:Heal` 的临时用户，清理后验证凭据失效。B09 使用明确启用 MRF 的专用服务；停止一个跨节点 set 成员节点，记录降级 ACK，并先验证逐盘补片，再执行 S3/手工验证请求。其余 60 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 +25 个用例已接入自动分发：A01、A03、A04、A07、A08、B05、B07、B09、C01–C05、C08、D01–D03、D05–D07、D09、D10、E01、E02、J04。分片执行器使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。A08 保留五种主体和各自 start/status/stop 操作，使用真正重新计算的过期签名及仅授权 `admin:Heal` 的临时用户，清理后验证凭据失效。B09 使用明确启用 MRF 的专用服务；停止一个跨节点 set 成员节点，记录降级 ACK，并先验证逐盘补片，再执行 S3/手工验证请求。其余 59 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 7367afed..382c793d 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 98 currently dispatchable IDs include all 74 non-Heal cases and twenty-four Heal cases. Forty-one have live PASS evidence; the other 117 have no live PASS claim. +case evidence. The 99 currently dispatchable IDs include all 74 non-Heal cases and twenty-five Heal cases. Forty-one have live PASS evidence; the other 117 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 60 Heal cases remain +The other 59 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -414,3 +414,20 @@ Incomplete finalization retains a stopped namespace with cleanup FAIL. Partial seed ownership, real server time, same-drive multi-version proof selection and protected cleanup failure boundaries have focused tests. Full live evidence is still required. + +C08 `heal.metadata-payload-mismatch` dispatches every H1/H2 branch with three +real versions per distinct key. The donor is another acknowledged version's +same-role/same-slot part: a different physical length for length mismatch, +the same length with different bytes for payload mismatch. Candidate-bound +streaming verification proves both original and donor cohorts before injection. +The descriptor-relative fault layer admits only another owned same-number part +on the same disk, exact donor/target hashes and quiescent service; it rejects +metadata, traversal, symlinks, hardlinks, unowned buckets and identical bytes. +Intent and original bytes are persisted before replacement. Every physical +fault receipt must equal the frozen donor; metadata and controls stay unchanged. +The negative branch leaves exactly R-1 correct same-version shards and requires +exact read refusal, truthful failed responsibility and no physical mutation +before rescue. Restored backups certify cleanup only; a later-generation target +cannot be overwritten. Positive branches require original shard bytes, full +version metadata/contents/tags, streaming checksums and repaired receipts. +This executor has local fault/recovery/authority tests and still needs live proof. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 6ccea865..b932395b 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 46e6c7fb..e5dd776d 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -238,7 +238,8 @@ def apply_fault(request): "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat()})) binding = module.checked_owner(fd, owner["runId"]) return module.apply_fault(fd, binding, request["relativePath"], request["faultId"], request["kind"], - request["expectedSha256"], request.get("offset")) + request["expectedSha256"], request.get("offset"), + donor_relative=request.get("donorRelativePath"), donor_sha=request.get("donorSha256")) def recover_fault(request): diff --git a/scripts/validation_heal/heal_fault.py b/scripts/validation_heal/heal_fault.py index 6fd00974..35e17aa6 100644 --- a/scripts/validation_heal/heal_fault.py +++ b/scripts/validation_heal/heal_fault.py @@ -21,7 +21,7 @@ LIMIT = 64 * 1024**2 ROOT_PATTERN = re.compile(r"/data/rustfs[1-9][0-9]*/heal-validation/((?:gh|manual)-[a-z0-9-]{5,60})") -FAULTS = ("missing", "bitflip", "truncate", "zero", "header") +FAULTS = ("missing", "bitflip", "truncate", "zero", "header", "replace-part") def bounded_fault_id(code: str, branch: str, index: int) -> str: @@ -119,6 +119,7 @@ def encode(value: dict) -> bytes: def mutation(value: bytes, kind: str, offset: int | None = None) -> bytes | None: require(kind in FAULTS, "unknown fault") + require(kind != "replace-part", "replacement requires a verified owned donor") if kind == "missing": return None require(bool(value), "empty file cannot witness this fault") @@ -150,20 +151,36 @@ def checked_owner(root_fd: int, run_id: str, recovery: bool = False) -> dict: def apply_fault(root_fd: int, owner: dict, relative: str, fault_id: str, kind: str, - expected_sha: str, offset: int | None = None) -> dict: + expected_sha: str, offset: int | None = None, *, donor_relative: str | None = None, + donor_sha: str | None = None) -> dict: parts = target_parts(relative, owner["bucketPrefix"]) require(bool(re.fullmatch(r"[A-J][0-9]{2}-[a-z0-9-]{1,48}", fault_id)), "invalid fault ID") require(bool(re.fullmatch(r"[0-9a-f]{64}", expected_sha)), "baseline SHA-256 required") with parent(root_fd, ["data", *parts[:-1]]) as target_fd, parent(root_fd, ["quarantine"]) as backup_fd: before = read_file(target_fd, parts[-1]) require(digest(before) == expected_sha, "target differs from frozen baseline") - changed = mutation(before, kind, offset) + donor = None + if kind == "replace-part": + require(offset is None and isinstance(donor_relative, str) and isinstance(donor_sha, str) + and re.fullmatch(r"[0-9a-f]{64}", donor_sha), "verified replacement donor required") + source = target_parts(donor_relative, owner["bucketPrefix"]) + require(source[-1] == parts[-1] and source[-1].startswith("part.") and source != parts, + "replacement must use another owned generation's same-number part") + with parent(root_fd, ["data", *source[:-1]]) as source_fd: + changed = read_file(source_fd, source[-1]) + require(digest(changed) == donor_sha and changed != before, "donor changed or is identical to target") + donor = {"relativePath": donor_relative, "sha256": donor_sha, "size": len(changed)} + else: + require(donor_relative is None and donor_sha is None, "donor parameters require replace-part") + changed = mutation(before, kind, offset) receipt = {"schemaVersion": 1, "runId": owner["runId"], "faultId": fault_id, "relativePath": relative, "kind": kind, "offset": offset, "before": {"sha256": digest(before), "size": len(before)}, "fault": None if changed is None else {"sha256": digest(changed), "size": len(changed)}, "createdAt": datetime.now(UTC).isoformat(), "backup": fault_id + ".original", "backend": "logical-file-fault", "requiresQuiescentService": True} + if donor is not None: + receipt["donor"] = donor # Intent and recoverable bytes precede the mutation. An interrupted # apply remains recoverable even when no completion receipt was emitted. new_file(backup_fd, fault_id + ".intent.json", encode(receipt)) @@ -275,6 +292,8 @@ def main() -> int: apply.add_argument("--kind", choices=FAULTS, required=True) apply.add_argument("--expected-sha256", required=True) apply.add_argument("--offset", type=int) + apply.add_argument("--donor-path") + apply.add_argument("--donor-sha256") apply.add_argument("--service-quiescent", action="store_true", required=True) recover = sub.add_parser("recover") recover.add_argument("--fault-id", required=True) @@ -311,7 +330,8 @@ def main() -> int: result = {"runId": args.run_id, "path": args.path, "size": len(value), "sha256": digest(value)} elif args.action == "apply": result = apply_fault(root_fd, owner, args.path, args.fault_id, args.kind, - args.expected_sha256, args.offset) + args.expected_sha256, args.offset, + donor_relative=args.donor_path, donor_sha=args.donor_sha256) else: result = recover_fault(root_fd, owner, args.fault_id) print(json.dumps(result, ensure_ascii=False)) diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 134f9134..df9e750e 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 3aa660ee..d725506a 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -20,7 +20,7 @@ done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization) ;; heal.node-outage-rejoin) driver=node_outage.py ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 57e45b8c..cd1831fd 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -55,6 +55,7 @@ "heal.special-keys-and-pagination": ("unicode-space-percent", "encoded-slash", "adjacent-prefixes", "multi-page-list"), "heal.object-size-boundaries": SIZE_BRANCHES, "heal.object-lock-retention": LOCK_BRANCHES, + "heal.metadata-payload-mismatch": ("length-mismatch", "payload-mismatch", "insufficient-authority"), } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") @@ -62,6 +63,55 @@ "literal/slash/object.bin", "a/object.bin", "ab/object.bin", "b/object.bin", "nested/a/b/object.bin") +def seed_mismatch(recorder, bucket): + recorder.call("create_bucket", {"Bucket": bucket}) + recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) + expected = [] + for branch in BRANCHES["heal.metadata-payload-mismatch"]: + key = "mismatch/" + branch + ".bin" + for nonce, size in enumerate((VERSION_SIZE, VERSION_SIZE, 2 * 1024**2 + 147)): + body = payload(f"{bucket}/{key}/{nonce}", size) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "Metadata": {"heal-fixture": "metadata-payload-mismatch", "generation": str(nonce)}, + "Tagging": f"suite=heal&generation=v{nonce}"}, body) + version = response.get("VersionId") + require(version not in {None, "", "null"}, "mismatch fixture omitted a real version ACK") + expected.append({"key": key, "versionId": version, "isLatest": nonce == 2, "deleteMarker": False, + "payload": {"size": size, "sha256": hashlib.sha256(body).hexdigest()}}) + return expected + + +def mismatch_plan(branch, snapshot, bucket, expected): + key = "mismatch/" + branch + ".bin" + selected = [entry for entry in expected if entry["key"] == key] + require(len(selected) == 3 and len({entry["versionId"] for entry in selected}) == 3 + and all(not entry["deleteMarker"] for entry in selected), "mismatch fixture lacks three distinct data versions") + target = choose_target(snapshot, bucket, selected[0], "data") + donor = choose_target(snapshot, bucket, selected[2 if branch == "length-mismatch" else 1], "data") + wanted_size = selected[0]["payload"]["size"] + require((selected[2]["payload"]["size"] != wanted_size) and selected[1]["payload"]["size"] == wanted_size, + "mismatch donor sizes do not distinguish length and payload faults") + erasure = target["logical"]["erasure"] + require(erasure == donor["logical"]["erasure"], "mismatch donor has a different EC layout") + read = erasure["data_blocks"] + ordered = sorted(target["cohort"], key=lambda shard: shard["index"]) + negative = branch == "insufficient-authority" + replaced = ordered[read - 1:] if negative else [target] + faults = [] + for shard in replaced: + source = next(item for item in donor["cohort"] if (item["node"], item["disk"]) == (shard["node"], shard["disk"])) + require(source["index"] == shard["index"] and source["role"] == shard["role"] + and source["dataDir"] != shard["dataDir"] and source["sha256"] != shard["sha256"] + and (source["size"] != shard["size"] if branch == "length-mismatch" else source["size"] == shard["size"]), + "mismatch donor is not the frozen same-role distinct-generation part") + faults.append({**target, **shard, "donorRelativePath": source["relativePath"], "donorSha256": source["sha256"], + "donorSize": source["size"]}) + return {"scope": {"bucket": bucket, "key": key}, "recursive": True, "expected": selected, + "faults": faults, "proofs": [target], "donorProof": donor, "kind": "replace-part", + "requiredRepairs": set() if negative else {version_identity(target)}, "metadataRepairs": set(), + "negative": negative, "readQuorum": read, "retainedShards": ordered[:read - 1] if negative else []} + + def seed_special_keys(recorder, bucket): recorder.call("create_bucket", {"Bucket": bucket}) expected = [] @@ -449,6 +499,8 @@ def reenable_and_verify(recorder, fleet, bucket, baseline): def branch_plan(case_id, branch, snapshot, bucket, expected, fixture): """Plans are derived from ACK order and decoded physical roles, never disk numbers.""" + if case_id == "heal.metadata-payload-mismatch": + return mismatch_plan(branch, snapshot, bucket, expected) if case_id == "heal.object-lock-retention": index = LOCK_BRANCHES.index(branch) key = LOCK_KEYS[index] @@ -611,7 +663,8 @@ def run_topology(args, topology, nodes, fixture): fleet = Fleet(nodes, directory, run_id, args.tool_directory) candidate = json.loads(args.candidate.read_text()) tool_manifest = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) - contract = verify_quorum_contract(tool_manifest, candidate) if args.case_id == "heal.below-read-quorum" else None + contract = verify_quorum_contract(tool_manifest, candidate) if args.case_id in { + "heal.below-read-quorum", "heal.metadata-payload-mismatch"} else None size_rule = verify_size_contract(tool_manifest, candidate) if args.case_id == "heal.object-size-boundaries" else None bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) @@ -650,6 +703,8 @@ def run_topology(args, topology, nodes, fixture): expected += extra elif lock_context is not None: expected = seed_locked(recorder, fleet, bucket, lock_context) + elif args.case_id == "heal.metadata-payload-mismatch": + expected = seed_mismatch(recorder, bucket) elif fixture == "DK": expected = seed_special_keys(recorder, bucket) elif fixture == "DS": @@ -697,6 +752,8 @@ def run_topology(args, topology, nodes, fixture): "metadataRepairs": sorted(plan["metadataRepairs"])}) for target in plan["proofs"]: verify_proof(fleet, target, "baseline-bitrot-" + branch) + if plan.get("donorProof"): + verify_proof(fleet, plan["donorProof"], "baseline-donor-bitrot-" + branch) prior = before for index, target in enumerate(plan["faults"]): params = {"disk": target["disk"], "faultId": bounded_fault_id(recipe["code"], branch, index)} @@ -705,7 +762,14 @@ def run_topology(args, topology, nodes, fixture): registrations.append(registration) receipt = fleet.record("fault-" + branch, fleet.remote(target["node"], "apply-fault", **params, relativePath=target["relativePath"], expectedSha256=target["sha256"], - kind=plan.get("kind", "missing"), offset=plan.get("offset"))) + kind=plan.get("kind", "missing"), offset=plan.get("offset"), + **({"donorRelativePath": target["donorRelativePath"], "donorSha256": target["donorSha256"]} + if plan.get("donorProof") else {}))) + if plan.get("donorProof"): + require(receipt.get("donor") == {"relativePath": target["donorRelativePath"], + "sha256": target["donorSha256"], "size": target["donorSize"]} + and receipt.get("fault") == {"sha256": target["donorSha256"], "size": target["donorSize"]}, + "replacement receipt did not witness the exact frozen donor bytes") if target["relativePath"].endswith("/xl.meta") and receipt.get("fault"): metadata_faults.setdefault(target["node"], {})[target["relativePath"]] = receipt["fault"] witnessed = fleet.snapshot("physical-fault-" + branch, quiescent=True, metadata_faults=metadata_faults) diff --git a/test_validation_heal.py b/test_validation_heal.py index 6d2ce639..64f348c3 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -31,6 +31,41 @@ from scripts.validation_heal.fleet_node import runtime_config, BACKGROUND_CONFIG from scripts.validation_heal.object_lock import (validate_protection, validate_protected_delete, cleanup_locked_fleet, seed_locked, BRANCHES as LOCK_BRANCHES, KEYS as LOCK_KEYS, CLEANUP_SECONDS) +from scripts.validation_heal.versions import mismatch_plan, seed_mismatch +from scripts.validation_heal import heal_fault + + +def versioned_scene(keys, sizes=(100, 100, 61)): + base = HealShardOracleTest().snapshot() + bucket = "rv-heal-0123456789abcdef" + expected = [] + snapshot = {"node1": {"formats": base["node1"]["formats"], "files": []}} + for key in keys: + for entry in base["node1"]["files"]: + if "/control/" in entry["path"]: continue + cloned = copy.deepcopy(entry) + cloned["path"] = cloned["path"].replace("objects/0008.bin", key) + if "metadata" in cloned: + cloned["metadata"]["key"] = key + versions = [] + for nonce, size in enumerate(sizes): + version = copy.deepcopy(cloned["metadata"]["versions"][0]) + version.update(versionId=f"v{nonce}", isLatest=nonce == 2, size=size, + dataDir=f"12345678-1234-1234-1234-{nonce:012d}") + version["parts"][0].update(size=size, actualSize=size) + versions.append(version) + cloned["metadata"]["versions"] = versions + snapshot["node1"]["files"].append(cloned) + else: + for nonce, size in enumerate(sizes): + part = copy.deepcopy(cloned) + part["path"] = part["path"].replace("12345678-1234-1234-1234-123456789abc", f"12345678-1234-1234-1234-{nonce:012d}") + part["sha256"] += f"v{nonce}" + part["size"] = (size + 1) // 2 + 32 + snapshot["node1"]["files"].append(part) + expected.extend({"key": key, "versionId": f"v{nonce}", "deleteMarker": False, "isLatest": nonce == 2, + "payload": {"size": size}} for nonce, size in enumerate(sizes)) + return snapshot, bucket, expected class HealContractTest(unittest.TestCase): @@ -951,6 +986,106 @@ def test_deleted_identity_requires_exact_s3_absence_without_payload_or_transport with self.subTest(version=version), self.assertRaises(ValueError): validate_deleted_read(changed, version) +class HealPayloadMismatchTest(unittest.TestCase): + def test_length_and_payload_donors_keep_original_metadata_and_exact_same_role(self): + branches = VERSION_BRANCHES["heal.metadata-payload-mismatch"] + scene, bucket, expected = versioned_scene(["mismatch/" + branch + ".bin" for branch in branches]) + original = copy.deepcopy(scene) + for branch in branches: + plan = mismatch_plan(branch, scene, bucket, expected) + self.assertEqual(len(plan["expected"]), 3) + self.assertEqual(plan["proofs"][0]["versionId"], "v0") + self.assertEqual(plan["donorProof"]["versionId"], "v2" if branch == "length-mismatch" else "v1") + self.assertEqual(len(plan["faults"]), 3 if plan["negative"] else 1) + if plan["negative"]: + self.assertEqual(len(plan["retainedShards"]), plan["readQuorum"] - 1) + self.assertTrue({shard["index"] for shard in plan["retainedShards"]}.isdisjoint( + {shard["index"] for shard in plan["faults"]})) + for fault in plan["faults"]: + self.assertNotEqual(fault["donorSha256"], fault["sha256"]) + self.assertNotEqual(fault["donorRelativePath"], fault["relativePath"]) + self.assertEqual(fault["donorSize"] != fault["size"], branch == "length-mismatch") + self.assertEqual(scene, original) + broken = copy.deepcopy(scene) + for item in broken["node1"]["files"]: + if "metadata" in item: + for version in item["metadata"]["versions"]: + if version["versionId"] == "v1": + version["erasure"]["block_size"] //= 2 + with self.assertRaisesRegex(ValueError, "different EC layout"): + mismatch_plan("payload-mismatch", broken, bucket, expected) + + def replacement(self, directory, *, invalid=None): + root = Path(directory).resolve() + prefix = "rv-heal-mismatch-unit" + target = prefix + "/key/old/part.1" + donor = prefix + "/key/new/part.1" + before, replacement = b"authoritative-old-shard", b"other-generation-shard" + (root / "quarantine").mkdir() + for relative, value in ((target, before), (donor, replacement)): + path = root / "data" / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(value) + sha = hashlib.sha256(replacement).hexdigest() + if invalid == "hash": sha = "0" * 64 + elif invalid == "traversal": donor = prefix + "/key/../new/part.1" + elif invalid == "bucket": donor = donor.replace(prefix, "unowned-bucket") + elif invalid == "metadata": donor = donor.removesuffix("part.1") + "xl.meta" + elif invalid == "symlink": + link = root / "data" / donor + link.unlink() + link.symlink_to(root / "data" / target) + elif invalid == "hardlink": + import os + os.link(root / "data" / donor, root / "second-link") + elif invalid == "identical": donor = target + owner = {"runId": "manual-mismatch-unit", "bucketPrefix": prefix} + with patch.object(heal_fault, "mount_id", return_value=1), heal_fault.directory(root) as fd: + if invalid: + with self.assertRaises((ValueError, OSError)): + heal_fault.apply_fault(fd, owner, target, "C08-test", "replace-part", hashlib.sha256(before).hexdigest(), + donor_relative=donor, donor_sha=sha) + self.assertEqual((root / "data" / target).read_bytes(), before) + self.assertFalse(list((root / "quarantine").iterdir())) + return + receipt = heal_fault.apply_fault(fd, owner, target, "C08-test", "replace-part", hashlib.sha256(before).hexdigest(), + donor_relative=donor, donor_sha=sha) + self.assertEqual((root / "data" / target).read_bytes(), replacement) + self.assertEqual((root / "data" / donor).read_bytes(), replacement) + self.assertEqual(receipt["donor"]["sha256"], sha) + self.assertEqual((root / "quarantine" / "C08-test.original").read_bytes(), before) + # A later generation is preserved instead of overwritten by rescue. + (root / "data" / target).write_bytes(b"new-committed-generation") + with self.assertRaisesRegex(ValueError, "changed after fault"): + heal_fault.recover_fault(fd, owner, "C08-test") + (root / "data" / target).write_bytes(replacement) + restored = heal_fault.recover_fault(fd, owner, "C08-test") + self.assertFalse(restored["physicalRepairPassAllowed"]) + self.assertEqual((root / "data" / target).read_bytes(), before) + + def test_replacement_is_recoverable_and_cannot_escape_owned_part_files(self): + for invalid in (None, "hash", "traversal", "bucket", "metadata", "symlink", "hardlink", "identical"): + with self.subTest(invalid=invalid), tempfile.TemporaryDirectory() as directory: + self.replacement(directory, invalid=invalid) + + def test_three_version_fixture_has_real_different_length_and_content_acks(self): + observed = [] + def invoke(operation, params, body=None): + observed.append((operation, params, body)) + return {"VersionId": str(len(observed))} if operation == "put_object" else {} + with patch("scripts.validation_heal.versions.payload", side_effect=lambda identity, size: identity.encode()): + expected = seed_mismatch(types.SimpleNamespace(call=invoke), "rv-heal-mismatch") + self.assertEqual(len(expected), 9) + self.assertEqual(sum(entry["isLatest"] for entry in expected), 3) + for key in {entry["key"] for entry in expected}: + group = [entry for entry in expected if entry["key"] == key] + self.assertEqual(group[0]["payload"]["size"], group[1]["payload"]["size"]) + self.assertNotEqual(group[0]["payload"]["size"], group[2]["payload"]["size"]) + self.assertNotEqual(group[0]["payload"]["sha256"], group[1]["payload"]["sha256"]) + with self.assertRaises(ValueError): + seed_mismatch(types.SimpleNamespace(call=lambda *args: {"VersionId": "null"}), "rv-heal-mismatch") + + class HealObjectLockTest(unittest.TestCase): def seed(self, *, fail_after=None): calls, context = [], {} @@ -993,33 +1128,7 @@ def test_protected_seed_uses_real_server_time_and_keeps_partial_ownership(self): self.assertTrue(partial["user"].startswith("rvheal")) def test_protected_plan_freezes_all_versions_on_the_lost_metadata_drive(self): - base = HealShardOracleTest().snapshot() - bucket = "rv-heal-0123456789abcdef" - expected = [] - snapshot = {"node1": {"formats": base["node1"]["formats"], "files": []}} - for key in LOCK_KEYS: - for entry in base["node1"]["files"]: - if "/control/" in entry["path"]: continue - cloned = copy.deepcopy(entry) - cloned["path"] = cloned["path"].replace("objects/0008.bin", key) - if "metadata" in cloned: - cloned["metadata"]["key"] = key - versions = [] - for nonce in range(3): - version = copy.deepcopy(cloned["metadata"]["versions"][0]) - version.update(versionId=f"v{nonce}", isLatest=nonce == 2, - dataDir=f"12345678-1234-1234-1234-{nonce:012d}") - versions.append(version) - cloned["metadata"]["versions"] = versions - snapshot["node1"]["files"].append(cloned) - else: - for nonce in range(3): - part = copy.deepcopy(cloned) - part["path"] = part["path"].replace("12345678-1234-1234-1234-123456789abc", f"12345678-1234-1234-1234-{nonce:012d}") - part["sha256"] += f"v{nonce}" - snapshot["node1"]["files"].append(part) - expected.extend({"key": key, "versionId": f"v{nonce}", "deleteMarker": False, "isLatest": nonce == 2, - "payload": {"size": 100}} for nonce in range(3)) + snapshot, bucket, expected = versioned_scene(LOCK_KEYS, (100, 100, 100)) for index, branch in enumerate(LOCK_BRANCHES): plan = branch_plan("heal.object-lock-retention", branch, snapshot, bucket, expected, "DO") self.assertEqual({item["key"] for item in plan["expected"]}, {LOCK_KEYS[index]}) From 2b48c8ce9f621d61d8055fa6e3499651c88c4c9a Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 00:37:55 +0800 Subject: [PATCH 050/343] fix(validation): allow explicit owned Governance finalization requests --- scripts/validation_heal/heal_probe.py | 12 ++++++++---- scripts/validation_heal/object_lock.py | 6 ++++-- test_validation_heal.py | 22 +++++++++++++++++++++- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/scripts/validation_heal/heal_probe.py b/scripts/validation_heal/heal_probe.py index 9ab30bb0..3c52550e 100644 --- a/scripts/validation_heal/heal_probe.py +++ b/scripts/validation_heal/heal_probe.py @@ -138,7 +138,7 @@ def bucket_allowed(binding: dict, bucket: str) -> None: and bool(re.fullmatch(r"[a-z0-9][a-z0-9.-]+[a-z0-9]", bucket)), "bucket is outside this run") -def validate_s3_request(binding: dict, operation: str, params: dict) -> None: +def validate_s3_request(binding: dict, operation: str, params: dict, *, allow_governance_bypass: bool = False) -> None: require(operation in S3_OPERATIONS, "S3 operation is not allowlisted") bucket_allowed(binding, params.get("Bucket", "")) source = params.get("CopySource") @@ -149,7 +149,11 @@ def validate_s3_request(binding: dict, operation: str, params: dict) -> None: items = params.get("Delete", {}).get("Objects", []) require(0 < len(items) <= 1000 and all(isinstance(item.get("Key"), str) for item in items), "invalid bounded batch deletion") - require(not params.get("BypassGovernanceRetention"), "retention bypass is prohibited") + if params.get("BypassGovernanceRetention"): + require(allow_governance_bypass is True and params["BypassGovernanceRetention"] is True + and operation == "delete_object" and isinstance(params.get("Key"), str) and params["Key"] + and isinstance(params.get("VersionId"), str) and params["VersionId"] not in {"", "null"}, + "retention bypass requires an explicit owned-version DELETE proof or finalizer") require("Body" not in params, "use --size/--seed or --body-file; do not put payloads in JSON") @@ -189,9 +193,9 @@ def s3_client(self): retries={"max_attempts": 0}, connect_timeout=5, read_timeout=30, proxies={})) - def s3(self, operation: str, params: dict, body: bytes | None = None) -> dict: + def s3(self, operation: str, params: dict, body: bytes | None = None, *, allow_governance_bypass: bool = False) -> dict: validate_binding(self.binding) # Recheck expiry before every mutation. - validate_s3_request(self.binding, operation, params) + validate_s3_request(self.binding, operation, params, allow_governance_bypass=allow_governance_bypass) outgoing = dict(params) for key in ("SSECustomerKey", "CopySourceSSECustomerKey"): if outgoing.get(key) == "$HEAL_SSE_C_KEY": diff --git a/scripts/validation_heal/object_lock.py b/scripts/validation_heal/object_lock.py index c9a73177..0dab7bab 100644 --- a/scripts/validation_heal/object_lock.py +++ b/scripts/validation_heal/object_lock.py @@ -105,7 +105,8 @@ def prove_locked(probe, fleet, bucket, context, *, key=None, label): denied = fleet.record(label + "-delete-denied", context["principal"].s3("delete_object", params)) validate_protected_delete(denied) if spec.get("ObjectLockMode") == "GOVERNANCE": - denied = fleet.record(label + "-bypass-denied", context["principal"].s3("delete_object", {**params, "BypassGovernanceRetention": True})) + denied = fleet.record(label + "-bypass-denied", context["principal"].s3("delete_object", + {**params, "BypassGovernanceRetention": True}, allow_governance_bypass=True)) validate_protected_delete(denied) return {"protectedVersions": len(selected), "exactProtectionMetadata": True, "nonBypassDeleteDenied": True, "serverTimeBeforeExpiry": True} @@ -144,7 +145,8 @@ def finalize_locked(probe, fleet, bucket, context): if (retention.get("response") or {}).get("Retention", {}).get("Mode") == "GOVERNANCE": params["BypassGovernanceRetention"] = True while True: - deleted = fleet.record("locked-final-delete", probe.s3("delete_object", params)) + deleted = fleet.record("locked-final-delete", probe.s3("delete_object", params, + **({"allow_governance_bypass": True} if params.get("BypassGovernanceRetention") else {}))) if deleted.get("acknowledged"): require(deleted["response"].get("ResponseMetadata", {}).get("HTTPStatusCode") in {200, 204}, "protected final DELETE had an unexpected status") break diff --git a/test_validation_heal.py b/test_validation_heal.py index 64f348c3..44a7323a 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -24,7 +24,7 @@ map_inline_target, verify_proof) from scripts.validation_heal.versions import seed_semantics, verify_semantics from scripts.validation_heal.versions import multi_loss_instances, multi_loss_plan, SPECIAL_KEYS, verify_pagination, boundary_plan -from scripts.validation_heal.heal_probe import Probe +from scripts.validation_heal.heal_probe import Probe, validate_s3_request from scripts.validation_heal.prepare_heal_meta_inspector import quorum_contract, size_contract from scripts.validation_heal.boundaries import verify_size_contract, boundary_sizes, verify_boundary_ranges from scripts.validation_heal.node_outage import quorum_plan, put_ack, delete_ack, replica_census, validate_deleted_read @@ -1087,6 +1087,26 @@ def invoke(operation, params, body=None): class HealObjectLockTest(unittest.TestCase): + def test_governance_bypass_requires_explicit_owned_version_and_reaches_the_sdk(self): + from test_validation_heal_authorization import binding + bound = binding() + params = {"Bucket": bound["bucketPrefix"], "Key": "locked/governance.bin", "VersionId": "owned-version", + "BypassGovernanceRetention": True} + with self.assertRaises(ValueError): validate_s3_request(bound, "delete_object", params) + validate_s3_request(bound, "delete_object", params, allow_governance_bypass=True) + for operation, changed in (("put_object", params), ("delete_objects", params), + ("delete_object", {**params, "Bucket": "unowned"}), + ("delete_object", {**params, "VersionId": "null"}), + ("delete_object", {**params, "BypassGovernanceRetention": 1})): + with self.subTest(operation=operation), self.assertRaises(ValueError): + validate_s3_request(bound, operation, changed, allow_governance_bypass=True) + sdk = unittest.mock.Mock() + sdk.delete_object.return_value = {"ResponseMetadata": {"HTTPStatusCode": 204}} + probe = Probe(bound) + with patch.object(probe, "s3_client", return_value=sdk): + self.assertTrue(probe.s3("delete_object", params, allow_governance_bypass=True)["acknowledged"]) + sdk.delete_object.assert_called_once_with(**params) + def seed(self, *, fail_after=None): calls, context = [], {} fleet = types.SimpleNamespace(secret_values=set(), record=lambda label, value: value) From 4e60a0c5ff0822b9e442dedb932a5631dcdcd9a8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 00:47:19 +0800 Subject: [PATCH 051/343] feat(validation): attribute automatic read repair to exact source tasks --- docs/validation-heal-contract.md | 2 +- docs/validation-migration-map.md | 21 ++++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 10 ++- scripts/validation_heal/healthy.py | 3 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/read_repair.py | 102 +++++++++++++++++++++++++ scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 26 ++++++- test_validation_heal.py | 87 +++++++++++++++++++++ 10 files changed, 242 insertions(+), 15 deletions(-) create mode 100644 scripts/validation_heal/read_repair.py diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index c5b5d63a..e395974d 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -25 个用例已接入自动分发：A01、A03、A04、A07、A08、B05、B07、B09、C01–C05、C08、D01–D03、D05–D07、D09、D10、E01、E02、J04。分片执行器使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。A08 保留五种主体和各自 start/status/stop 操作，使用真正重新计算的过期签名及仅授权 `admin:Heal` 的临时用户，清理后验证凭据失效。B09 使用明确启用 MRF 的专用服务；停止一个跨节点 set 成员节点，记录降级 ACK，并先验证逐盘补片，再执行 S3/手工验证请求。其余 59 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 +26 个用例已接入自动分发：A01、A03、A04、A07、A08、B05、B07、B09、C01–C05、C08、D01–D03、D05–D07、D09、D10、E01–E03、J04。分片执行器使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。A08 保留五种主体和各自 start/status/stop 操作，使用真正重新计算的过期签名及仅授权 `admin:Heal` 的临时用户，清理后验证凭据失效。B09 使用明确启用 MRF 的专用服务；停止一个跨节点 set 成员节点，记录降级 ACK，并先验证逐盘补片，再执行 S3/手工验证请求。其余 58 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 382c793d..a34be4e3 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 99 currently dispatchable IDs include all 74 non-Heal cases and twenty-five Heal cases. Forty-one have live PASS evidence; the other 117 have no live PASS claim. +case evidence. The 100 currently dispatchable IDs include all 74 non-Heal cases and twenty-six Heal cases. Forty-one have live PASS evidence; the other 117 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 59 Heal cases remain +The other 58 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -431,3 +431,20 @@ before rescue. Restored backups certify cleanup only; a later-generation target cannot be overwritten. Positive branches require original shard bytes, full version metadata/contents/tags, streaming checksums and repaired receipts. This executor has local fault/recovery/authority tests and still needs live proof. + +E03 `heal.read-repair-attribution` dispatches GET-missing and Range-bitrot DB +branches plus an exact historical-version DV read on H1/H2. Its explicit +profile leaves scanner, automatic rebuild and best-effort MRF ingress off, +while enabling candidate JSON debug events; the candidate's durable consumer +stays initialized independently of that ingress switch. Before each single +GET/Range the exact fault remains physically present. Correct bytes, exact +version/ContentRange and a server request ID are required. A new structured +`Read-repair heal request admitted` event must match bucket/key/part/pool/set +and accepted task ID; stale events, text mentions, merges and ambiguous tasks +cannot prove attribution. Only status queries observe that automatic task; +no Admin start/deep verifier repairs the fixture. The task's exact version +receipt, physical original shard hashes, streaming checksums and full public +S3/version controls must all agree. Runtime credentials are redacted from +archived debug journals. Focused tests cover source identity, stale-task +refusal, single-read/no-mutation dispatch, exact content/range/request identity +and runtime profile isolation. Full live proof remains pending. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index b932395b..68249cc7 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index e5dd776d..ad638b79 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -24,12 +24,14 @@ RUN_PATTERN = re.compile(r"gh-[1-9][0-9]*-[1-9][0-9]*-(?:h1|h2)") BACKGROUND_CONFIG = {"RUSTFS_SCANNER_ENABLED": "false", "RUSTFS_HEAL_ENABLED": "true", "RUSTFS_HEAL_AUTO_HEAL_ENABLE": "false", "RUSTFS_HEAL_MRF_ENABLE": "false", - "RUSTFS_SHARD_INTEGRITY_WRITE": "true", "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED": "true"} + "RUSTFS_SHARD_INTEGRITY_WRITE": "true", "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED": "true", + "RUSTFS_OBS_LOGGER_LEVEL": "info"} def runtime_config(profile): - require(profile in {"manual", "offline-replay"}, "unsupported Heal runtime profile") - return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true" if profile == "offline-replay" else "false"} + require(profile in {"manual", "offline-replay", "read-repair"}, "unsupported Heal runtime profile") + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true" if profile == "offline-replay" else "false", + "RUSTFS_OBS_LOGGER_LEVEL": "debug" if profile == "read-repair" else "info"} def require(condition, message): @@ -150,7 +152,7 @@ def reserve(request): require(digest(state / "rustfs") == binary_sha, "copied candidate binary differs") config = {**background, "RUSTFS_ACCESS_KEY": request["access"], "RUSTFS_SECRET_KEY": request["secret"], "RUSTFS_ADDRESS": ":9000", "RUSTFS_CONSOLE_ENABLE": "false", - "RUSTFS_HEAL_TASK_TIMEOUT_SECS": "300", "RUSTFS_OBS_LOGGER_LEVEL": "info"} + "RUSTFS_HEAL_TASK_TIMEOUT_SECS": "300"} env_file = state / "service.env" descriptor = os.open(env_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) with os.fdopen(descriptor, "w") as stream: diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 467efdd3..dc5c8dda 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -131,7 +131,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual"): - require(runtime_profile in {"manual", "offline-replay"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "offline-replay", "read-repair"}, "unsupported Heal runtime profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory self.runtime_profile = runtime_profile @@ -188,6 +188,7 @@ def start(self): self.wait_ready() def deploy(self, binary_sha, bucket, access, secret): + self.secret_values.update({access, secret}) checked = {node: self.remote(node, "preflight") for node in self.nodes} self.record("preflight", checked) require(len({value["hostname"] for value in checked.values()}) == len(self.nodes), diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index df9e750e..90dd688b 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/read_repair.py b/scripts/validation_heal/read_repair.py new file mode 100644 index 00000000..9b76cb65 --- /dev/null +++ b/scripts/validation_heal/read_repair.py @@ -0,0 +1,102 @@ +"""E03: one tracked read, a fresh ReadRepair admission and its exact task.""" + +import hashlib +import json +import time +from urllib.parse import quote + +try: + from .heal_probe import Probe, payload, require +except ImportError: + from heal_probe import Probe, payload, require + + +BRANCHES = ("get-missing", "range-bitrot", "historical-version-read") +ADMISSION_MESSAGE = "Read-repair heal request admitted" +ADMISSION_TARGET = "rustfs_ecstore::set_disk::core::io_primitives" + + +def source_tasks(logs, bucket, key): + """Consume candidate JSON events; free-text mentions cannot certify a source.""" + found = set() + for node, observed in logs.items(): + for line in observed.get("journal", "").splitlines(): + start = line.find("{") + if start < 0: continue + try: event = json.loads(line[start:]) + except json.JSONDecodeError: continue + if (event.get("target") != ADMISSION_TARGET or event.get("level") != "DEBUG" + or event.get("message") != ADMISSION_MESSAGE + or event.get("bucket") != bucket or event.get("object") != key): + continue + require(event.get("admission") == "accepted" and type(event.get("pool_index")) is int + and event["pool_index"] == 0 and type(event.get("set_index")) is int and event["set_index"] == 0 + and type(event.get("part_number")) is int and event["part_number"] == 1 + and isinstance(event.get("request_id"), str) and event["request_id"], + "read-repair event lacks exact accepted task/set/part identity") + found.add((node, event["request_id"])) + return found + + +def validate_read(observed, entry, *, expected_range=None, body=None): + response = observed.get("response") or {} + require(observed.get("acknowledged") is True and observed.get("resultUnknown", False) is False, + "tracked read failed or has an unknown transport outcome") + headers = response.get("ResponseMetadata", {}).get("HTTPHeaders", {}) + request_id = response.get("ResponseMetadata", {}).get("RequestId") or headers.get("x-request-id") or headers.get("x-amz-request-id") + require(isinstance(request_id, str) and request_id, "tracked read omitted a server request ID") + version = response.get("VersionId", "null") + require(version == entry["versionId"], "tracked read returned another version") + if expected_range is None: + require(response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 200 + and observed.get("payload") == entry["payload"], "tracked GET differs from its acknowledged bytes") + else: + start, end = expected_range + selected = body[start:end + 1] + require(response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 206 + and response.get("ContentRange") == f"bytes {start}-{end}/{entry['payload']['size']}" + and observed.get("payload") == {"size": len(selected), "sha256": hashlib.sha256(selected).hexdigest()}, + "tracked Range differs from its exact acknowledged interval") + return request_id + + +def tracked_read_task(probe, fleet, bucket, branch, entry): + require(fleet.runtime_profile == "read-repair", "read repair requires its explicit isolated runtime profile") + def capture(label): + return fleet.record(label, {node: fleet.remote(node, "logs") for node in fleet.nodes}) + baseline = source_tasks(capture("read-source-before"), bucket, entry["key"]) + params = {"Bucket": bucket, "Key": entry["key"]} + if entry["versionId"] != "null": params["VersionId"] = entry["versionId"] + interval = (0, min(1024**2 + 16, entry["payload"]["size"]) - 1) if branch == "range-bitrot" else None + body = payload(f"{bucket}/{entry['key']}/0", entry["payload"]["size"]) if interval else None + if interval: params["Range"] = f"bytes={interval[0]}-{interval[1]}" + read = fleet.record("single-read-trigger", probe.s3("get_object", params)) + request_id = validate_read(read, entry, expected_range=interval, body=body) + deadline = time.monotonic() + 180 + fresh = set() + while not fresh: + fresh = source_tasks(capture("read-source-progress"), bucket, entry["key"]) - baseline + require(time.monotonic() < deadline, "ATTRIBUTION_NOT_OBSERVED: no fresh accepted ReadRepair task for the read target") + if not fresh: time.sleep(0.2) + require(len(fresh) == 1, "read source has ambiguous task ownership") + node, token = next(iter(fresh)) + # Status queries observe the automatically admitted request. No Admin + # start, stop, retry or deep verifier contributes to this repair proof. + endpoint = "node" + str(fleet.nodes.index(node) + 1) + status_probe = Probe(probe.binding, endpoint) + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + quote(entry["key"], safe="/") + while True: + observed = fleet.record("read-source-task-status", status_probe.admin(path, [("clientToken", token)], b"")) + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), + "ReadRepair task cannot be queried by its exact source ID") + status = observed["response"] + if status.get("summary") != "running": break + require(time.monotonic() < deadline, "ReadRepair task did not finish before the physical repair budget") + time.sleep(0.2) + require(status.get("summary") == "finished", "ReadRepair task did not report finished execution") + fleet.record("read-source-binding", {"source": "read-repair", "s3RequestId": request_id, "taskId": token, + "sourceNode": node, "key": entry["key"], "versionId": entry["versionId"], + "singleTargetRead": True, "adminMutationUsed": False, "scannerAutoHealAndBestEffortMrfDisabled": True, + "durableMrf": "candidate consumer stays initialized; no degraded writes create responsibility in this fixture", + "requirement": "still requires exact version receipt, physical bytes, streaming checksum and full S3 proof"}) + return status diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index d725506a..9d3b10c9 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -20,7 +20,7 @@ done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization) ;; heal.node-outage-rejoin) driver=node_outage.py ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index cd1831fd..78b1f95b 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -27,6 +27,7 @@ from .boundaries import (BRANCHES as SIZE_BRANCHES, verify_size_contract, boundary_sizes, seed_boundaries, verify_boundary_ranges) from .object_lock import BRANCHES as LOCK_BRANCHES, KEYS as LOCK_KEYS, seed_locked, prove_locked, cleanup_locked_fleet + from .read_repair import BRANCHES as READ_BRANCHES, tracked_read_task from .shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) except ImportError: @@ -39,6 +40,7 @@ from boundaries import (BRANCHES as SIZE_BRANCHES, verify_size_contract, boundary_sizes, seed_boundaries, verify_boundary_ranges) from object_lock import BRANCHES as LOCK_BRANCHES, KEYS as LOCK_KEYS, seed_locked, prove_locked, cleanup_locked_fleet + from read_repair import BRANCHES as READ_BRANCHES, tracked_read_task from shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) @@ -56,6 +58,7 @@ "heal.object-size-boundaries": SIZE_BRANCHES, "heal.object-lock-retention": LOCK_BRANCHES, "heal.metadata-payload-mismatch": ("length-mismatch", "payload-mismatch", "insufficient-authority"), + "heal.read-repair-attribution": READ_BRANCHES, } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") @@ -499,6 +502,16 @@ def reenable_and_verify(recorder, fleet, bucket, baseline): def branch_plan(case_id, branch, snapshot, bucket, expected, fixture): """Plans are derived from ACK order and decoded physical roles, never disk numbers.""" + if case_id == "heal.read-repair-attribution": + key = "objects/0001.bin" if fixture == "DV" else f"objects/{8 + 9 * READ_BRANCHES.index(branch):04d}.bin" + data = [entry for entry in expected if entry["key"] == key and not entry["deleteMarker"]] + require(len(data) == (3 if fixture == "DV" else 1), "read target lacks its acknowledged version history") + require(branch != "historical-version-read" or not data[0]["isLatest"], "historical read selected a current version") + target = choose_target(snapshot, bucket, data[0], "data") + return {"scope": {"bucket": bucket, "key": key}, "recursive": False, "expected": [data[0]], + "faults": [target], "proofs": [target], "requiredRepairs": {version_identity(target)}, + "metadataRepairs": set(), "kind": "bitflip" if branch == "range-bitrot" else "missing", + "offset": 32 if branch == "range-bitrot" else None} if case_id == "heal.metadata-payload-mismatch": return mismatch_plan(branch, snapshot, bucket, expected) if case_id == "heal.object-lock-retention": @@ -660,7 +673,8 @@ def run_topology(args, topology, nodes, fixture): directory = args.evidence / fixture / topology directory.mkdir(parents=True) run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" - fleet = Fleet(nodes, directory, run_id, args.tool_directory) + fleet = Fleet(nodes, directory, run_id, args.tool_directory, + runtime_profile="read-repair" if args.case_id == "heal.read-repair-attribution" else "manual") candidate = json.loads(args.candidate.read_text()) tool_manifest = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) contract = verify_quorum_contract(tool_manifest, candidate) if args.case_id in { @@ -682,6 +696,7 @@ def run_topology(args, topology, nodes, fixture): branches = [branch for branch in BRANCHES[args.case_id] if (args.case_id != "heal.repeat-idempotency" or (branch == "versioned-repeat") == (fixture == "DV")) and (args.case_id != "heal.below-read-quorum" or (branch == "r-minus-1-historical") == (fixture == "DV")) + and (args.case_id != "heal.read-repair-attribution" or (branch == "historical-version-read") == (fixture == "DV")) and (args.case_id != "heal.missing-or-corrupt-xlmeta" or (branch == "versioned-xlmeta") == (fixture == "DV"))] result = {"topology": topology, "fixture": fixture, "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} active, admissions = [], [] @@ -788,8 +803,11 @@ def run_topology(args, topology, nodes, fixture): validate_unrecoverable_get(denied, contract) require(physical_content(prior) == physical_content(fleet.snapshot("physical-after-negative-get")), "negative read widened damage or silently healed below-quorum fragments") - status = run_task(probe, fleet, plan["scope"], 2, active, - option_overrides={"recursive": plan["recursive"]}, admissions=admissions) + if args.case_id == "heal.read-repair-attribution": + status = tracked_read_task(probe, fleet, bucket, branch, plan["expected"][0]) + else: + status = run_task(probe, fleet, plan["scope"], 2, active, + option_overrides={"recursive": plan["recursive"]}, admissions=admissions) fleet.stop() repaired = fleet.snapshot("physical-after-" + branch, quiescent=True, metadata_faults=metadata_faults) negative_proof = None @@ -923,7 +941,7 @@ def interrupted(signum, frame): and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") - fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.multiple-shard-loss"} else ("DV",) + fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.multiple-shard-loss", "heal.read-repair-attribution"} else ("DV",) if args.case_id == "heal.object-metadata-and-tags": fixtures = ("DS",) if args.case_id == "heal.special-keys-and-pagination": diff --git a/test_validation_heal.py b/test_validation_heal.py index 44a7323a..7856ad49 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -33,6 +33,7 @@ seed_locked, BRANCHES as LOCK_BRANCHES, KEYS as LOCK_KEYS, CLEANUP_SECONDS) from scripts.validation_heal.versions import mismatch_plan, seed_mismatch from scripts.validation_heal import heal_fault +from scripts.validation_heal.read_repair import source_tasks, validate_read, tracked_read_task, ADMISSION_MESSAGE, ADMISSION_TARGET def versioned_scene(keys, sizes=(100, 100, 61)): @@ -986,6 +987,92 @@ def test_deleted_identity_requires_exact_s3_absence_without_payload_or_transport with self.subTest(version=version), self.assertRaises(ValueError): validate_deleted_read(changed, version) +class HealReadRepairTest(unittest.TestCase): + def event(self, **changes): + return {"level": "DEBUG", "target": ADMISSION_TARGET, "message": ADMISSION_MESSAGE, + "bucket": "rv-heal-read-unit", "object": "target.bin", "part_number": 1, + "pool_index": 0, "set_index": 0, "request_id": "automatic-task", "admission": "accepted", **changes} + + def test_source_requires_structured_exact_scope_and_new_task_identity(self): + event = self.event() + self.assertEqual(source_tasks({"node1": {"journal": "journal-prefix " + json.dumps(event)}}, event["bucket"], event["object"]), + {("node1", "automatic-task")}) + for changed in ({**event, "target": "unrelated"}, {**event, "bucket": "another-bucket"}, + {**event, "object": "other-version-key"}, {**event, "message": "Read-repair heal request not admitted"}): + self.assertFalse(source_tasks({"node1": {"journal": json.dumps(changed)}}, event["bucket"], event["object"])) + self.assertFalse(source_tasks({"node1": {"journal": str(event)}}, event["bucket"], event["object"])) + for name, value in (("admission", "merged"), ("pool_index", 1), ("set_index", True), ("part_number", 2), ("request_id", "")): + with self.subTest(field=name), self.assertRaises(ValueError): + source_tasks({"node1": {"journal": json.dumps({**event, name: value})}}, event["bucket"], event["object"]) + + def test_read_bytes_version_and_server_request_id_are_required(self): + entry = {"versionId": "historical", "payload": {"size": 3, "sha256": hashlib.sha256(b"abc").hexdigest()}} + good = {"acknowledged": True, "payload": entry["payload"], "response": {"VersionId": "historical", + "ResponseMetadata": {"HTTPStatusCode": 200, "RequestId": "server-read"}}} + self.assertEqual(validate_read(good, entry), "server-read") + for invalid in ("version", "bytes", "request-id", "transport", "status"): + changed = copy.deepcopy(good) + if invalid == "version": changed["response"]["VersionId"] = "latest" + elif invalid == "bytes": changed["payload"]["sha256"] = "not-acknowledged" + elif invalid == "request-id": changed["response"]["ResponseMetadata"].pop("RequestId") + elif invalid == "transport": changed["resultUnknown"] = True + else: changed["response"]["ResponseMetadata"]["HTTPStatusCode"] = 206 + with self.subTest(invalid=invalid), self.assertRaises(ValueError): validate_read(changed, entry) + ranged = {**good, "payload": {"size": 2, "sha256": hashlib.sha256(b"bc").hexdigest()}, + "response": {**good["response"], "ContentRange": "bytes 1-2/3", + "ResponseMetadata": {"HTTPStatusCode": 206, "RequestId": "server-range"}}} + validate_read(ranged, entry, expected_range=(1, 2), body=b"abc") + with self.assertRaises(ValueError): validate_read(ranged, entry, expected_range=(0, 1), body=b"abc") + + def execute(self, *, stale_only=False): + bucket, key, calls = "rv-heal-read-unit", "target.bin", [] + entry = {"key": key, "versionId": "historical", "payload": {"size": 3, "sha256": hashlib.sha256(b"abc").hexdigest()}} + old = self.event(request_id="old-task") + fresh = self.event() + logs = [json.dumps(old), json.dumps(old) + ("" if stale_only else "\n" + json.dumps(fresh))] + fleet = types.SimpleNamespace(runtime_profile="read-repair", nodes=["host-one"], + record=lambda label, value: value, remote=lambda *args: {"journal": logs.pop(0) if logs else json.dumps(old)}) + def read(operation, params): + calls.append((operation, params)) + return {"acknowledged": True, "payload": entry["payload"], "response": {"VersionId": "historical", + "ResponseMetadata": {"HTTPStatusCode": 200, "RequestId": "server-read"}}} + def status(path, query, body): + calls.append((path, query, body)) + self.assertEqual(query, [("clientToken", "automatic-task")]) + self.assertEqual(body, b"") + return {"httpStatus": 200, "response": {"summary": "finished", "source-receipt": "validated-by-physical-driver"}} + client = types.SimpleNamespace(binding={}, s3=read) + observer = types.SimpleNamespace(admin=status) + ticks = [0, 1, 181] if stale_only else [0, 1] + with patch("scripts.validation_heal.read_repair.Probe", return_value=observer) as factory, \ + patch("scripts.validation_heal.read_repair.time.monotonic", side_effect=ticks), \ + patch("scripts.validation_heal.read_repair.time.sleep"): + if stale_only: + with self.assertRaisesRegex(ValueError, "ATTRIBUTION_NOT_OBSERVED"): + tracked_read_task(client, fleet, bucket, "historical-version-read", entry) + factory.assert_not_called() + else: + self.assertEqual(tracked_read_task(client, fleet, bucket, "historical-version-read", entry)["summary"], "finished") + factory.assert_called_once_with({}, "node1") + self.assertEqual(sum(call[0] == "get_object" for call in calls), 1) + self.assertEqual(calls[0][1], {"Bucket": bucket, "Key": key, "VersionId": "historical"}) + self.assertTrue(all(len(call) == 2 or call[1] == [("clientToken", "automatic-task")] for call in calls)) + + def test_only_one_target_read_and_observation_of_the_fresh_automatic_task(self): + self.execute() + self.execute(stale_only=True) + + def test_read_profile_keeps_other_repair_sources_disabled(self): + profile = runtime_config("read-repair") + self.assertEqual(profile.pop("RUSTFS_OBS_LOGGER_LEVEL"), "debug") + self.assertEqual(profile, {key: value for key, value in BACKGROUND_CONFIG.items() if key != "RUSTFS_OBS_LOGGER_LEVEL"}) + snapshot, bucket, expected = versioned_scene(["objects/0001.bin"]) + plan = branch_plan("heal.read-repair-attribution", "historical-version-read", snapshot, bucket, expected, "DV") + self.assertEqual(plan["expected"][0]["versionId"], "v0") + self.assertFalse(plan["expected"][0]["isLatest"]) + self.assertEqual(len(plan["expected"]), 1) + + class HealPayloadMismatchTest(unittest.TestCase): def test_length_and_payload_donors_keep_original_metadata_and_exact_same_role(self): branches = VERSION_BRANCHES["heal.metadata-payload-mismatch"] From 0789743d439fd91b967c2d6fba85c06e29f844c5 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 00:55:23 +0800 Subject: [PATCH 052/343] docs(validation): record complete object boundary live pass --- docs/validation-migration-map.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index a34be4e3..8712be35 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 100 currently dispatchable IDs include all 74 non-Heal cases and twenty-six Heal cases. Forty-one have live PASS evidence; the other 117 have no live PASS claim. +case evidence. The 100 currently dispatchable IDs include all 74 non-Heal cases and twenty-six Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -448,3 +448,11 @@ S3/version controls must all agree. Runtime credentials are redacted from archived debug journals. Focused tests cover source identity, stale-task refusal, single-read/no-mutation dispatch, exact content/range/request identity and runtime profile isolation. Full live proof remains pending. + +Corrected D01 run `38067139720` passed all 96 H1/H2 parameters (48 per +topology), with canonical PASS, complete summary and cleanup PASS. Its exact +candidate/package/binary/harness identities and each parameter's evidence were +downloaded and independently checked by the canonical report gate. It covers +zero/one/4KiB, actual inline and EC-block boundaries, multi-block payloads, +both roles, missing/bitflip faults and exact Range outcomes. The three prior +D01 failures and independent recovery record remain unchanged. From 49ef48ec63f90cd7ec7a4f2043b130c6b7fe3733 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 01:08:01 +0800 Subject: [PATCH 053/343] feat(validation): isolate readonly and permission drive failures --- docs/validation-heal-contract.md | 2 +- docs/validation-migration-map.md | 20 +++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/drive_gate.py | 153 ++++++++++++++++++++++++++ scripts/validation_heal/fleet_node.py | 29 ++++- scripts/validation_heal/healthy.py | 14 ++- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 42 +++++++ test_validation_heal.py | 93 +++++++++++++++- 10 files changed, 347 insertions(+), 12 deletions(-) create mode 100644 scripts/validation_heal/drive_gate.py diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index e395974d..8d4f283e 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -26 个用例已接入自动分发：A01、A03、A04、A07、A08、B05、B07、B09、C01–C05、C08、D01–D03、D05–D07、D09、D10、E01–E03、J04。分片执行器使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。A08 保留五种主体和各自 start/status/stop 操作，使用真正重新计算的过期签名及仅授权 `admin:Heal` 的临时用户，清理后验证凭据失效。B09 使用明确启用 MRF 的专用服务；停止一个跨节点 set 成员节点，记录降级 ACK，并先验证逐盘补片，再执行 S3/手工验证请求。其余 58 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 +27 个用例已接入自动分发：A01、A03、A04、A07、A08、B02、B05、B07、B09、C01–C05、C08、D01–D03、D05–D07、D09、D10、E01–E03、J04。分片执行器使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。A08 保留五种主体和各自 start/status/stop 操作，使用真正重新计算的过期签名及仅授权 `admin:Heal` 的临时用户，清理后验证凭据失效。B09 使用明确启用 MRF 的专用服务；停止一个跨节点 set 成员节点，记录降级 ACK，并先验证逐盘补片，再执行 S3/手工验证请求。其余 57 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 8712be35..66a393be 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 100 currently dispatchable IDs include all 74 non-Heal cases and twenty-six Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. +case evidence. The 101 currently dispatchable IDs include all 74 non-Heal cases and twenty-seven Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 58 Heal cases remain +The other 57 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -456,3 +456,19 @@ downloaded and independently checked by the canonical report gate. It covers zero/one/4KiB, actual inline and EC-block boundaries, multi-block payloads, both roles, missing/bitflip faults and exact Range outcomes. The three prior D01 failures and independent recovery record remain unchanged. + +B02 `heal.disk-readonly-and-permission` dispatches both H1/H2 branches. The +read-only gate uses systemd PrivateMounts/ReadOnlyPaths for the selected +run-owned data root; host mounts are never remounted. The permission gate +records/restores that root's original 0700 mode and drops both DAC override +capabilities from the root service, so chmod cannot be bypassed. A write probe +enters the actual service mount namespace with the same effective UID/GID and +DAC boundary and requires exact EROFS/EACCES. Original shard isolation precedes +the gate. Failed/deferred exact-version responsibility must show zero repaired +counters and the complete physical fault scene must remain unchanged. After +service exit and gate release, a distinct task must actually repair the +original target; full physical, streaming and public checks follow. Gate intent +precedes chmod/start and is recovered before any fault rescue or physical +cleanup. Root drift or release failure retains the namespace. Focused tests +exercise permission restoration, write refusal identity/errno, no root bypass, +truthful unresolved receipts and cleanup retention. Live proof is pending. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 68249cc7..4fd76727 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/drive_gate.py b/scripts/validation_heal/drive_gate.py new file mode 100644 index 00000000..7dafc9a6 --- /dev/null +++ b/scripts/validation_heal/drive_gate.py @@ -0,0 +1,153 @@ +"""B02 gates in run-owned units; the host's data mounts are never remounted.""" + +import errno +import json +import os +from pathlib import Path +import stat +import subprocess + + +DENY_DAC_CAPS = (1 << 1) | (1 << 2) # CAP_DAC_OVERRIDE, CAP_DAC_READ_SEARCH +PROBE_SOURCE = '''import errno,json,os +path = __import__('sys').argv[1] +result = {"uid": os.geteuid(), "gid": os.getegid()} +try: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) +except OSError as exc: + result["errno"] = exc.errno +else: + os.close(fd) + os.unlink(path) + result["errno"] = 0 +print(json.dumps(result)) +''' + + +def require(value, message): + if not value: raise ValueError(message) + + +def checked_gate(state, roots): + path = state / "drive-gate.json" + if not path.exists(): return None + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_uid == 0 and info.st_nlink == 1 + and info.st_size < 4096 and info.st_mode & 0o077 == 0, "invalid drive gate ownership") + value = json.loads(path.read_text()) + require(value.get("schemaVersion") == 1 and value.get("mode") in {"read-only", "permission-denied"} + and type(value.get("disk")) is int and 0 <= value["disk"] < len(roots) + and value.get("dataPath") == str(roots[value["disk"]] / "data") + and type(value.get("originalMode")) is int and value["originalMode"] == 0o700, + "drive gate no longer matches the run-owned data root") + return value + + +def apply_gate(state, roots, disk, mode): + require(type(disk) is int and 0 <= disk < len(roots) and mode in {"read-only", "permission-denied"}, "invalid drive gate") + require(checked_gate(state, roots) is None, "a drive gate is already active") + folder = roots[disk] / "data" + info = folder.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and stat.S_IMODE(info.st_mode) == 0o700, + "expected private run-owned root with original mode 0700") + value = {"schemaVersion": 1, "disk": disk, "mode": mode, "dataPath": str(folder), + "originalMode": 0o700, "device": info.st_dev, "inode": info.st_ino} + # Intent precedes chmod/start. Cleanup can restore a partially applied gate. + fd = os.open(state / "drive-gate.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(value, stream) + stream.flush() + os.fsync(stream.fileno()) + if mode == "permission-denied": folder.chmod(0) + return {**value, "state": "ARMED", "hostMountRemounted": False, "sharedPermissionsChanged": False} + + +def release_gate(state, roots): + value = checked_gate(state, roots) + if value is None: return {"state": "CLEAR", "gateWasAbsent": True} + folder = Path(value["dataPath"]) + info = folder.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and info.st_dev == value["device"] and info.st_ino == value["inode"], + "gated data root changed; preserve namespace") + if value["mode"] == "permission-denied": + require(stat.S_IMODE(info.st_mode) in {0, value["originalMode"]}, "gated permissions drifted") + folder.chmod(value["originalMode"]) + require(stat.S_IMODE(folder.stat().st_mode) == value["originalMode"], "data root permissions were not restored") + (state / "drive-gate.json").unlink() + return {"state": "CLEAR", "originalPermissionsRestored": True, "hostMountRemounted": False} + + +def start_properties(value): + if value is None: return [] + properties = ["--property=PrivateMounts=true"] + if value["mode"] == "read-only": properties.append("--property=ReadOnlyPaths=" + value["dataPath"]) + else: properties.append("--property=CapabilityBoundingSet=~CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH") + return properties + + +def process_identity(pid): + text = Path(f"/proc/{pid}/status").read_text() + fields = dict(line.split(":", 1) for line in text.splitlines() if ":" in line) + uid, gid = list(map(int, fields["Uid"].split())), list(map(int, fields["Gid"].split())) + return {"uid": uid[1], "gid": gid[1], "capEff": int(fields["CapEff"].strip(), 16)} + + +def validate_probe(observed, mode, identity): + expected = errno.EROFS if mode == "read-only" else errno.EACCES + require(identity["uid"] == observed.get("uid") == 0 and identity["gid"] == observed.get("gid") == 0, + "write probe does not share the service identity") + require(observed.get("errno") == expected, "drive gate did not cause its precise Linux write refusal") + if mode == "permission-denied": + require(identity["capEff"] & DENY_DAC_CAPS == 0, "root service still bypasses filesystem permissions") + return {"errno": expected, "name": errno.errorcode[expected], "sameServiceIdentity": True, + "dacBypassRemoved": mode == "permission-denied"} + + +def probe_gate(state, roots, pid): + value = checked_gate(state, roots) + require(value is not None and type(pid) is int and pid > 0, "active gated service required") + identity = process_identity(pid) + require(Path(f"/proc/{pid}/ns/mnt").readlink() != Path("/proc/self/ns/mnt").readlink(), + "gate requires an independent service mount namespace") + path = value["dataPath"] + "/.validation-write-probe" + args = ["nsenter", "--mount=/proc/" + str(pid) + "/ns/mnt"] + if value["mode"] == "permission-denied": args += ["setpriv", "--bounding-set=-dac_override,-dac_read_search"] + args += ["python3", "-c", PROBE_SOURCE, path] + result = subprocess.run(args, capture_output=True, text=True, timeout=15) + require(result.returncode == 0, "service-namespace write probe did not complete") + observed = json.loads(result.stdout) + return {**validate_probe(observed, value["mode"], identity), "disk": value["disk"], + "privateMountNamespace": True, "hostMountRemounted": False, "observed": observed} + + +def validate_unwritable(status, bucket, entry): + outcome = status.get("outcome") or {} + objects = outcome.get("objects", []) + require(status.get("summary") == "finished" and status.get("truncated", False) is False + and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False + and len(objects) == 1, "unwritable task did not preserve complete exact responsibility") + receipt = objects[0] + identity = receipt.get("identity", {}) + public_version = identity.get("versionId") or "null" + if public_version == "00000000-0000-0000-0000-000000000000": public_version = "null" + require(identity.get("bucket") == bucket and identity.get("object") == entry["key"] + and public_version == entry["versionId"] and identity.get("bucketIncarnationId") + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0, + "unwritable target receipt has a wrong or incomplete identity") + disposition = receipt.get("disposition", {}) + state, detail = disposition.get("state"), disposition.get("details") + require(state == "failed" and detail in {"recoverable", "retry_exhausted", "permanent"} + or state == "deferred" and isinstance(detail, dict) + and detail.get("reason") in {"deadline", "transient_existence_check"}, + "unwritable target was claimed repaired/healthy/absent or has an unrelated pending reason") + failed = state == "failed" + require(outcome.get("execution", {}).get("state") == ("completed_with_errors" if failed else "completed"), + "unwritable execution hides the unresolved result") + wanted = {"processed": 1, "healed": 0, "unchanged": 0, "failed": int(failed), + "skipped": int(not failed), "unknown": 0, "overflowed": False} + counters = outcome.get("counters", {}) + require(all(type(counters.get(name)) is type(value) and counters[name] == value for name, value in wanted.items()) + and type(counters.get("attemptFailures")) is int and counters["attemptFailures"] >= int(failed), + "unwritable responsibility counters claim success or hide a failure") + return {"disposition": disposition, "repairClaimedWhileUnwritable": False, "completeResponsibility": True} diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index ad638b79..7feed560 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -178,7 +178,7 @@ def owned(run_id): def start(request): - state, _, owner = owned(request["runId"]) + state, roots, owner = owned(request["runId"]) stopped(owner) command("systemctl", "reset-failed", owner["unit"], check=False) require(digest(state / "rustfs") == owner["binarySha256"], "candidate changed before start") @@ -186,7 +186,8 @@ def start(request): "--property=KillMode=control-group", "--property=TimeoutStopSec=30", "--property=RuntimeMaxSec=1800", "--property=MemoryMax=4G", "--property=WorkingDirectory=" + str(state), - "--property=EnvironmentFile=" + str(state / "service.env"), str(state / "rustfs"), *owner["volumes"]) + "--property=EnvironmentFile=" + str(state / "service.env"), + *gate_module().start_properties(gate_module().checked_gate(state, roots)), str(state / "rustfs"), *owner["volumes"]) return {"unit": owner["unit"], "started": True} @@ -221,6 +222,27 @@ def fault_module(): return heal_fault +def gate_module(): + if "gate" in globals(): return globals()["gate"] + from . import drive_gate + return drive_gate + + +def drive_gate(request): + state, roots, owner = owned(request["runId"]) + stopped(owner) + if request.get("mode") == "clear": return gate_module().release_gate(state, roots) + return gate_module().apply_gate(state, roots, request.get("disk"), request.get("mode")) + + +def drive_gate_probe(request): + state, roots, owner = owned(request["runId"]) + pid = int(command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"]).stdout.strip()) + require(Path(f"/proc/{pid}/exe").resolve(strict=True) == state / "rustfs", "gate probe candidate process differs") + require(digest(state / "rustfs") == owner["binarySha256"], "gate probe candidate digest differs") + return gate_module().probe_gate(state, roots, pid) + + def fault_root(request): _, roots, owner = owned(request["runId"]) stopped(owner) @@ -371,6 +393,7 @@ def cleanup(request): command("systemctl", "stop", owner["unit"], check=False) pid = command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"], check=False).stdout.strip() require(pid in {"", "0"}, "run-owned service remains active") + gate_module().release_gate(state, roots) command("systemctl", "reset-failed", owner["unit"], check=False) for root in (*roots, state): for directory, children, names in os.walk(root, followlinks=False): @@ -408,6 +431,6 @@ def main(request): actions = {"preflight": preflight, "reserve": reserve, "start": start, "stop": stop, "snapshot": snapshot, "logs": logs, "cleanup": cleanup, "audit-finalize": audit_finalize, "install-inspector": install_inspector, "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part, - "verify-inline": verify_inline} + "verify-inline": verify_inline, "drive-gate": drive_gate, "drive-gate-probe": drive_gate_probe} require(request.get("action") in actions, "unsupported node operation") print(json.dumps(actions[request["action"]](request))) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index dc5c8dda..bddef5ca 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -138,6 +138,7 @@ def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_prof self.sequence = 0 self.reserved = [] self.faults = [] + self.drive_gates = set() self.secret_values = set() def record(self, label, value): @@ -166,6 +167,9 @@ def remote(self, node, action, **params): if action in {"apply-fault", "recover-fault", "verify-part", "verify-inline"}: source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" + if action in {"start", "cleanup", "audit-finalize", "drive-gate", "drive-gate-probe"}: + source += "\nimport types\ngate = types.ModuleType('drive_gate')\nexec(" + repr( + NODE_SOURCE.with_name("drive_gate.py").read_text()) + ", gate.__dict__)\n" source += "\nmain(" + repr(value) + ")\n" result = subprocess.run([*self.ssh(node), "sudo -n python3 -"], input=source, capture_output=True, text=True, timeout=180) @@ -240,12 +244,20 @@ def wait_ready(self): def cleanup(self): failures = [] + if self.drive_gates: + try: + self.stop() + for node in tuple(self.drive_gates): + self.record("gate-recovery", {"node": node, **self.remote(node, "drive-gate", mode="clear")}) + self.drive_gates.remove(node) + except Exception as exc: + failures.append({"error": str(exc), "state": "DIRTY", "namespacesRetained": True}) for node in self.reserved: try: self.record("logs", {"node": node, **self.remote(node, "logs")}) except Exception as exc: self.record("logs-error", {"node": node, "error": str(exc)}) - if self.faults: + if self.faults and not failures: try: self.stop() for node, params in self.faults: diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 90dd688b..7a06a994 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 9d3b10c9..9c253f68 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -20,7 +20,7 @@ done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization) ;; heal.node-outage-rejoin) driver=node_outage.py ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 78b1f95b..ae0005b5 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -28,6 +28,7 @@ seed_boundaries, verify_boundary_ranges) from .object_lock import BRANCHES as LOCK_BRANCHES, KEYS as LOCK_KEYS, seed_locked, prove_locked, cleanup_locked_fleet from .read_repair import BRANCHES as READ_BRANCHES, tracked_read_task + from .drive_gate import validate_unwritable from .shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) except ImportError: @@ -41,6 +42,7 @@ seed_boundaries, verify_boundary_ranges) from object_lock import BRANCHES as LOCK_BRANCHES, KEYS as LOCK_KEYS, seed_locked, prove_locked, cleanup_locked_fleet from read_repair import BRANCHES as READ_BRANCHES, tracked_read_task + from drive_gate import validate_unwritable from shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) @@ -59,6 +61,7 @@ "heal.object-lock-retention": LOCK_BRANCHES, "heal.metadata-payload-mismatch": ("length-mismatch", "payload-mismatch", "insufficient-authority"), "heal.read-repair-attribution": READ_BRANCHES, + "heal.disk-readonly-and-permission": ("read-only", "permission-denied"), } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") @@ -502,6 +505,14 @@ def reenable_and_verify(recorder, fleet, bucket, baseline): def branch_plan(case_id, branch, snapshot, bucket, expected, fixture): """Plans are derived from ACK order and decoded physical roles, never disk numbers.""" + if case_id == "heal.disk-readonly-and-permission": + key = f"objects/{8 + 9 * BRANCHES[case_id].index(branch):04d}.bin" + selected = [entry for entry in expected if entry["key"] == key] + require(len(selected) == 1 and selected[0]["versionId"] == "null", "unwritable fixture lacks a unique unversioned target") + target = choose_target(snapshot, bucket, selected[0], "data") + return {"scope": {"bucket": bucket, "key": key}, "recursive": False, "expected": selected, + "faults": [target], "proofs": [target], "requiredRepairs": {version_identity(target)}, + "metadataRepairs": set(), "driveGate": branch} if case_id == "heal.read-repair-attribution": key = "objects/0001.bin" if fixture == "DV" else f"objects/{8 + 9 * READ_BRANCHES.index(branch):04d}.bin" data = [entry for entry in expected if entry["key"] == key and not entry["deleteMarker"]] @@ -751,6 +762,7 @@ def run_topology(args, topology, nodes, fixture): for branch in branches: registrations = [] metadata_faults = {} + gate_node, gated_proof = None, None try: if lock_context is not None: fleet.record("lock-before-fault", prove_locked(probe, fleet, bucket, lock_context, @@ -790,7 +802,14 @@ def run_topology(args, topology, nodes, fixture): witnessed = fleet.snapshot("physical-fault-" + branch, quiescent=True, metadata_faults=metadata_faults) validate_fault(prior, witnessed, target, receipt) prior = witnessed + if plan.get("driveGate"): + gate_node = plan["proofs"][0]["node"] + fleet.drive_gates.add(gate_node) + fleet.record("drive-gate-armed", fleet.remote(gate_node, "drive-gate", + disk=plan["proofs"][0]["disk"], mode=plan["driveGate"])) fleet.start() + if gate_node is not None: + fleet.record("service-write-denial", fleet.remote(gate_node, "drive-gate-probe")) witness = fleet.snapshot("physical-before-trigger-" + branch, metadata_faults=metadata_faults) require(physical_content(prior) == physical_content(witness), "ATTRIBUTION_NOT_OBSERVED: version fault changed before the manual token") @@ -810,6 +829,22 @@ def run_topology(args, topology, nodes, fixture): option_overrides={"recursive": plan["recursive"]}, admissions=admissions) fleet.stop() repaired = fleet.snapshot("physical-after-" + branch, quiescent=True, metadata_faults=metadata_faults) + if gate_node is not None: + require(physical_content(prior) == physical_content(repaired), + "unwritable window changed object files or wrote outside the target") + gated_proof = validate_unwritable(status, bucket, plan["expected"][0]) + fleet.record("unwritable-responsibility", gated_proof) + fleet.record("drive-gate-released", fleet.remote(gate_node, "drive-gate", mode="clear")) + fleet.drive_gates.remove(gate_node) + gate_node = None + fleet.start() + restored_status = run_task(probe, fleet, plan["scope"], 2, active, + option_overrides={"recursive": False}, admissions=admissions) + require(len(admissions) >= 2 and admissions[-1]["clientToken"] != admissions[-2]["clientToken"], + "restored writable repair reused the failed task identity") + status = restored_status + fleet.stop() + repaired = fleet.snapshot("physical-writable-repair", quiescent=True) negative_proof = None if plan.get("negative"): require(physical_content(prior) == physical_content(repaired), @@ -891,10 +926,15 @@ def run_topology(args, topology, nodes, fixture): result["branches"][branch]["negative"] = negative_proof result["branches"][branch]["physical"]["backupUsedAfterFailureProof"] = True result["branches"][branch]["physical"]["productRepairClaimed"] = False + if gated_proof is not None: + result["branches"][branch]["unwritableWindow"] = gated_proof except (Exception, KeyboardInterrupt) as exc: result["branches"][branch] = {"verdict": "FAIL", "failure": {"type": type(exc).__name__, "message": str(exc)}} try: fleet.stop() + if gate_node is not None: + fleet.record("gate-error-rescue", fleet.remote(gate_node, "drive-gate", mode="clear")) + fleet.drive_gates.remove(gate_node) fleet.snapshot("physical-failure-" + branch, quiescent=True, metadata_faults=metadata_faults) for registration in reversed(registrations): node, params = registration @@ -950,6 +990,8 @@ def interrupted(signum, frame): fixtures = ("DB",) if args.case_id == "heal.object-lock-retention": fixtures = ("DO",) + if args.case_id == "heal.disk-readonly-and-permission": + fixtures = ("DB",) results = [] try: for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): diff --git a/test_validation_heal.py b/test_validation_heal.py index 7856ad49..2b1e7919 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -34,6 +34,9 @@ from scripts.validation_heal.versions import mismatch_plan, seed_mismatch from scripts.validation_heal import heal_fault from scripts.validation_heal.read_repair import source_tasks, validate_read, tracked_read_task, ADMISSION_MESSAGE, ADMISSION_TARGET +from scripts.validation_heal.drive_gate import apply_gate, release_gate, checked_gate, start_properties, validate_probe, validate_unwritable +import errno +import stat def versioned_scene(keys, sizes=(100, 100, 61)): @@ -282,9 +285,9 @@ def invoke(args, **params): return types.SimpleNamespace(returncode=0, stdout='{"checked":true}', stderr="") with tempfile.TemporaryDirectory() as directory, patch("scripts.validation_heal.healthy.subprocess.run", side_effect=invoke): fleet = Fleet(["node1"], Path(directory), "gh-12345-1-h1") - for action in ("preflight", "verify-part", "verify-inline", "apply-fault", "recover-fault"): + for action in ("preflight", "verify-part", "verify-inline", "apply-fault", "recover-fault", "start", "cleanup", "drive-gate", "drive-gate-probe"): self.assertTrue(fleet.remote("node1", action, disk=1, relativePath="literal$()'part.1")["checked"]) - self.assertEqual(len(observed), 5) + self.assertEqual(len(observed), 9) def snapshot(self): bucket, key, data_dir = "rv-heal-0123456789abcdef", "objects/0008.bin", "12345678-1234-1234-1234-123456789abc" @@ -987,6 +990,92 @@ def test_deleted_identity_requires_exact_s3_absence_without_payload_or_transport with self.subTest(version=version), self.assertRaises(ValueError): validate_deleted_read(changed, version) +class HealDriveGateTest(unittest.TestCase): + def test_gate_only_changes_owned_permissions_and_restores_partial_intent(self): + for mode in ("read-only", "permission-denied"): + with self.subTest(mode=mode), tempfile.TemporaryDirectory() as directory: + state = Path(directory) / "state" + root = Path(directory) / "root" + state.mkdir() + (root / "data").mkdir(parents=True, mode=0o700) + other = root / "quarantine" + other.mkdir(mode=0o700) + (root / "data" / "original").write_bytes(b"owned-data") + original_lstat = Path.lstat + def root_owned(path): + observed = original_lstat(path) + return types.SimpleNamespace(st_mode=observed.st_mode, st_uid=0, st_nlink=observed.st_nlink, + st_size=observed.st_size, st_dev=observed.st_dev, st_ino=observed.st_ino) + with patch.object(Path, "lstat", root_owned): + gate = apply_gate(state, (root,), 0, mode) + self.assertEqual(checked_gate(state, (root,))["mode"], mode) + self.assertFalse(gate["hostMountRemounted"]) + self.assertEqual(stat.S_IMODE((root / "data").stat().st_mode), 0 if mode == "permission-denied" else 0o700) + self.assertEqual(stat.S_IMODE(other.stat().st_mode), 0o700) + with self.assertRaisesRegex(ValueError, "already active"): + apply_gate(state, (root,), 0, mode) + props = start_properties(gate) + self.assertIn("--property=PrivateMounts=true", props) + self.assertIn("--property=ReadOnlyPaths=" + str(root / "data") if mode == "read-only" else + "--property=CapabilityBoundingSet=~CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH", props) + self.assertTrue(release_gate(state, (root,))["originalPermissionsRestored"]) + self.assertEqual(stat.S_IMODE((root / "data").stat().st_mode), 0o700) + self.assertEqual((root / "data" / "original").read_bytes(), b"owned-data") + self.assertFalse((state / "drive-gate.json").exists()) + self.assertTrue(release_gate(state, (root,))["gateWasAbsent"]) + + def test_gate_refusal_proof_requires_same_uid_gid_errno_and_no_root_bypass(self): + identity = {"uid": 0, "gid": 0, "capEff": 0} + for mode, error in (("read-only", errno.EROFS), ("permission-denied", errno.EACCES)): + good = {"uid": 0, "gid": 0, "errno": error} + self.assertTrue(validate_probe(good, mode, identity)["sameServiceIdentity"]) + for changed in ({**good, "errno": 0}, {**good, "errno": errno.ENOSPC}, {**good, "uid": 1000}, {**good, "gid": 1000}): + with self.assertRaises(ValueError): validate_probe(changed, mode, identity) + with self.assertRaisesRegex(ValueError, "bypasses"): + validate_probe({"uid": 0, "gid": 0, "errno": errno.EACCES}, "permission-denied", {**identity, "capEff": 1 << 1}) + + def test_unwritable_receipt_never_accepts_repaired_or_hidden_failures(self): + status = HealHealthyOracleTest().receipt("failed") + bucket, entry = "rv-heal-drive-unit", {"key": "target.bin", "versionId": "null"} + receipt = status["outcome"]["objects"][0] + receipt["identity"].update(bucket=bucket, object=entry["key"], versionId=None) + receipt["disposition"]["details"] = "permanent" + status["outcome"]["execution"]["state"] = "completed_with_errors" + status["outcome"]["counters"].update(healed=0, unchanged=0, failed=1, attemptFailures=1) + self.assertFalse(validate_unwritable(status, bucket, entry)["repairClaimedWhileUnwritable"]) + for invalid in ("repaired", "metadata_healthy", "unknown", "wrong-version", "healed-count", "hidden-failure", "incarnation"): + changed = copy.deepcopy(status) + if invalid in {"repaired", "metadata_healthy", "unknown"}: changed["outcome"]["objects"][0]["disposition"]["state"] = invalid + elif invalid == "wrong-version": changed["outcome"]["objects"][0]["identity"]["versionId"] = "foreign" + elif invalid == "healed-count": changed["outcome"]["counters"]["healed"] = 1 + elif invalid == "hidden-failure": changed["outcome"]["execution"]["state"] = "completed" + else: changed["outcome"]["objects"][0]["identity"].pop("bucketIncarnationId") + with self.subTest(invalid=invalid), self.assertRaises(ValueError): validate_unwritable(changed, bucket, entry) + deferred = copy.deepcopy(status) + deferred["outcome"]["execution"]["state"] = "completed" + deferred["outcome"]["counters"].update(failed=0, skipped=1) + deferred["outcome"]["objects"][0]["disposition"] = {"state": "deferred", "details": {"reason": "deadline"}} + validate_unwritable(deferred, bucket, entry) + deferred["outcome"]["objects"][0]["disposition"]["details"]["reason"] = "dangling_delete_grace" + with self.assertRaises(ValueError): validate_unwritable(deferred, bucket, entry) + + def test_gate_cleanup_precedes_fault_rescue_and_preserves_namespace_on_failure(self): + with tempfile.TemporaryDirectory() as directory: + fleet = Fleet(["node1"], Path(directory), "gh-12345-1-h1") + fleet.reserved.append("node1") + fleet.drive_gates.add("node1") + fleet.faults.append(("node1", {"disk": 1, "faultId": "B02-test"})) + calls = [] + def remote(node, action, **params): + calls.append(action) + if action == "drive-gate": raise ValueError("gate root drifted") + return {} + with patch.object(fleet, "remote", side_effect=remote): self.assertFalse(fleet.cleanup()) + self.assertNotIn("recover-fault", calls) + self.assertNotIn("cleanup", calls) + self.assertTrue(fleet.drive_gates) + + class HealReadRepairTest(unittest.TestCase): def event(self, **changes): return {"level": "DEBUG", "target": ADMISSION_TARGET, "message": ADMISSION_MESSAGE, From d80cdaf77f3487dbf977f81f159daefbdaeca9c8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 01:25:14 +0800 Subject: [PATCH 054/343] feat(validation): migrate scanner cold-object repair attribution --- docs/validation-migration-map.md | 20 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 26 +++- scripts/validation_heal/healthy.py | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/scanner_repair.py | 149 ++++++++++++++++++++++ scripts/validation_heal/versions.py | 39 +++++- test_validation_heal.py | 131 +++++++++++++++++++ 9 files changed, 362 insertions(+), 11 deletions(-) create mode 100644 scripts/validation_heal/scanner_repair.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 66a393be..67f49d90 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 101 currently dispatchable IDs include all 74 non-Heal cases and twenty-seven Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. +case evidence. The 102 currently dispatchable IDs include all 74 non-Heal cases and twenty-eight Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 57 Heal cases remain +The other 56 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -472,3 +472,19 @@ precedes chmod/start and is recovered before any fault rescue or physical cleanup. Root drift or release failure retains the namespace. Focused tests exercise permission restoration, write refusal identity/errno, no root bypass, truthful unresolved receipts and cleanup retention. Live proof is pending. + +E04 `heal.scanner-cold-object-repair` now dispatches all three H1/H2 branches. +It reads back the actual private process settings for five-second scanner +cycles, every-cycle bitrot, zero deep cooldown and full object selection, while +replacement auto-heal and best-effort MRF stay disabled. Before any fault, each +cold target needs an exact healthy scanner task and unchanged physical files. +Faults are injected with all run-owned processes stopped; the stopped scene +proves damage immediately before restart. Cursor-bounded journal windows bind +the exact object discovery to an accepted Scanner source request, task start, +completed cycle and terminal task. The cycle-boundary branch requires two +complete post-fault cycles. Observation issues status queries only; no target +GET/HEAD or Admin mutation contributes to the repair. Exact repaired version +responsibility, original physical shard bytes, streaming checksum and complete +S3 inventories still gate PASS. Truncated logs, another source, a later healthy +receipt or cycle counters alone cannot certify repair. The executor passed its +local source/timing/cursor negatives; full live evidence remains pending. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 4fd76727..b8ad4bb1 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 7feed560..e9bca065 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -29,7 +29,13 @@ def runtime_config(profile): - require(profile in {"manual", "offline-replay", "read-repair"}, "unsupported Heal runtime profile") + require(profile in {"manual", "offline-replay", "read-repair", "scanner"}, "unsupported Heal runtime profile") + if profile == "scanner": + return {**BACKGROUND_CONFIG, "RUSTFS_SCANNER_ENABLED": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug", + "RUSTFS_SCANNER_START_DELAY_SECS": "5", "RUSTFS_SCANNER_CYCLE": "5", + "RUSTFS_SCANNER_SPEED": "fastest", "RUSTFS_SCANNER_DELAY": "0", + "RUSTFS_SCANNER_BITROT_CYCLE_SECS": "0", "RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS": "0", + "RUSTFS_HEAL_OBJECT_SELECT_PROB": "1", "RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES": "1"} return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true" if profile == "offline-replay" else "false", "RUSTFS_OBS_LOGGER_LEVEL": "debug" if profile == "read-repair" else "info"} @@ -385,6 +391,24 @@ def snapshot(request): def logs(request): _, _, owner = owned(request["runId"]) + if request.get("checkpoint") is True or "afterCursor" in request: + cursor = request.get("afterCursor") + require(cursor is None or isinstance(cursor, str) and re.fullmatch(r"[A-Za-z0-9=;_-]{1,512}", cursor), + "invalid journal cursor") + args = ["journalctl", "-u", owner["unit"], "--no-pager", "-o", "cat", "--show-cursor"] + args += ["--after-cursor", cursor, "-n", "10001"] if cursor else ["-n", "1"] + output = command(*args).stdout + lines, observed_cursor = [], None + for line in output.splitlines(): + if line.startswith("-- cursor: "): + observed_cursor = line[len("-- cursor: "):].strip() + else: + lines.append(line) + require(len(lines) < 10000, "journal window exceeds its bound; source attribution is incomplete") + require(observed_cursor is not None or cursor is not None and not lines, + "journal did not expose a complete cursor window") + return {"unit": owner["unit"], "journal": "\n".join(lines), "beforeCursor": cursor, + "afterCursor": observed_cursor or cursor, "complete": True} return {"unit": owner["unit"], "journal": command("journalctl", "-u", owner["unit"], "--no-pager", "-n", "2000").stdout} diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index bddef5ca..1100e7bb 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -131,7 +131,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual"): - require(runtime_profile in {"manual", "offline-replay", "read-repair"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "offline-replay", "read-repair", "scanner"}, "unsupported Heal runtime profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory self.runtime_profile = runtime_profile diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 7a06a994..29979d25 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 9c253f68..05be9a04 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -20,7 +20,7 @@ done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization) ;; heal.node-outage-rejoin) driver=node_outage.py ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/scanner_repair.py b/scripts/validation_heal/scanner_repair.py new file mode 100644 index 00000000..a9e02ff2 --- /dev/null +++ b/scripts/validation_heal/scanner_repair.py @@ -0,0 +1,149 @@ +"""E04: cursor-bounded scanner discovery, source admission and exact task proof.""" + +from datetime import datetime, timezone +import json +import time +from urllib.parse import quote + +try: + from .heal_probe import Probe, require +except ImportError: + from heal_probe import Probe, require + + +BRANCHES = ("cold-missing", "cold-bitrot", "scanner-cycle-boundary") +BUDGET_SECONDS = 240 + + +def event_time(event): + value = event.get("timestamp") + if not isinstance(value, str): return None + try: stamp = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: return None + return stamp.astimezone(timezone.utc) if stamp.tzinfo is not None else None + + +def null_version(value): + return value in {None, "", "null", "None", "00000000-0000-0000-0000-000000000000"} + + +class JournalWindow: + def __init__(self, fleet, label): + self.fleet, self.label = fleet, label + self.events = {node: [] for node in fleet.nodes} + self.cursors = {} + for node in fleet.nodes: + observed = fleet.record(label + "-checkpoint", {"node": node, **fleet.remote(node, "logs", checkpoint=True)}) + require(observed.get("complete") is True and observed.get("afterCursor"), "missing source journal checkpoint") + self.cursors[node] = observed["afterCursor"] + + def capture(self): + for node in self.fleet.nodes: + observed = self.fleet.record(self.label + "-window", {"node": node, + **self.fleet.remote(node, "logs", afterCursor=self.cursors[node])}) + require(observed.get("complete") is True and observed.get("beforeCursor") == self.cursors[node] + and observed.get("afterCursor"), "journal window lost its checkpoint or completeness") + self.cursors[node] = observed["afterCursor"] + for line in observed.get("journal", "").splitlines(): + try: event = json.loads(line) + except json.JSONDecodeError: continue + if isinstance(event, dict): self.events[node].append(event) + require(len(self.events[node]) <= 60000, "scanner evidence exceeded its registered event budget") + return self.events + + +def cycle_pairs(events): + """Pair actual timestamps on one leader; completed cycle may serialize as 0.""" + pairs, pending = [], None + for event in events: + if event.get("target") != "rustfs::scanner" or event.get("event") != "scanner_cycle_state": continue + stamp = event_time(event) + if stamp is None: + pending = None + continue + state, cycle = event.get("state"), event.get("cycle") + if state == "started": + pending = (cycle, stamp) if type(cycle) is int and cycle > 0 and event.get("scan_mode") == "Deep" else None + elif state == "completed": + if pending and type(cycle) is int and cycle in (0, pending[0]) and stamp > pending[1]: + pairs.append({"cycle": pending[0], "started": pending[1], "completed": stamp}) + pending = None + elif state in {"failed", "aborted", "budget_exhausted", "deferred"}: + pending = None + return pairs + + +def source_tasks(events, bucket, entry): + """Bind discovery -> channel -> accepted Scanner source -> exact object task.""" + require(entry["versionId"] == "null" and not entry["deleteMarker"], "scanner cold fixture must be unversioned data") + found = [] + for node, rows in events.items(): + pairs = cycle_pairs(rows) + discoveries = [event for event in rows if event.get("target") == "rustfs::scanner::folder" + and event.get("event") == "scanner_heal_admission" and event.get("state") == "request_started" + and event.get("bucket") == bucket and event.get("object") == entry["key"] + and "version_id" in event and null_version(event["version_id"]) and event_time(event) is not None] + for start in rows: + if not (start.get("target") == "rustfs::heal::task" and start.get("message") == "Heal object started" + and start.get("stage") == "start" and start.get("bucket") == bucket + and start.get("object") == entry["key"] and "version_id" in start + and null_version(start["version_id"])): continue + token, stamp = start.get("task_id"), event_time(start) + if not isinstance(token, str) or not token or stamp is None: continue + received = [event for event in rows if event.get("target") == "rustfs::heal::channel" + and event.get("state") == "start_received" and event.get("request_id") == token + and event.get("bucket") == bucket and event.get("object_prefix") == entry["key"]] + accepted = [event for event in rows if event.get("target") == "rustfs::heal::manager" + and event.get("message") == "Heal queue request accepted" and event.get("request_id") == token + and event.get("source") == "scanner" and event.get("result") == "accepted"] + completed = [event for event in rows if event.get("target") == "rustfs::heal::manager" + and event.get("state") == "task_completed" and event.get("task_id") == token] + for receipt in received: + receipt_time = event_time(receipt) + if receipt_time is None: continue + discovery = next((event for event in reversed(discoveries) if event_time(event) <= receipt_time), None) + if discovery is None: continue + admission = next((event for event in accepted if event_time(event) is not None + and receipt_time <= event_time(event) <= stamp), None) + done = next((event for event in completed if event_time(event) is not None and stamp <= event_time(event)), None) + cycle = next((pair for pair in pairs if pair["started"] <= event_time(discovery) + <= pair["completed"] and event_time(discovery) <= stamp), None) + if admission is not None and done is not None and cycle is not None: + found.append({"node": node, "taskId": token, "discovery": discovery, "received": receipt, + "admission": admission, "started": start, "completed": done, + "cycle": {name: value.isoformat() if isinstance(value, datetime) else value for name, value in cycle.items()}}) + break + return found + + +def observe_task(probe, fleet, window, bucket, entry, branch, *, healthy=False): + require(fleet.runtime_profile == "scanner", "scanner proof requires its isolated runtime profile") + deadline = time.monotonic() + BUDGET_SECONDS + while True: + events = window.capture() + tasks = source_tasks(events, bucket, entry) + if tasks: + if branch != "scanner-cycle-boundary" or healthy or any(len(cycle_pairs(events[task["node"]])) >= 2 for task in tasks): + break + require(time.monotonic() < deadline, "ATTRIBUTION_NOT_OBSERVED: scanner did not discover, admit and complete the exact target in its cycle") + time.sleep(1) + # A later healthy scanner pass cannot certify the injected fault. Inspect + # tasks in order and select the first exact repaired receipt for this run. + observations = [] + for task in tasks: + endpoint = "node" + str(fleet.nodes.index(task["node"]) + 1) + observer = Probe(probe.binding, endpoint) + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + quote(entry["key"], safe="/") + observed = fleet.record("scanner-task-status", observer.admin(path, [("clientToken", task["taskId"])], b"")) + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), + "scanner task cannot be queried by its exact source identity") + status = observed["response"] + observations.append({"source": task, "status": status}) + objects = status.get("outcome", {}).get("objects", []) + if len(objects) == 1 and objects[0].get("disposition", {}).get("state") == ("verified_healthy" if healthy else "repaired"): + fleet.record("scanner-source-binding", {"source": "scanner", "branch": branch, "healthyPreflight": healthy, + "budgetSeconds": BUDGET_SECONDS, "task": task, "targetReadsBeforePhysicalProof": 0, + "adminMutations": 0, "observedTasks": observations, + "postFaultCycles": len(cycle_pairs(events[task["node"]]))}) + return status + raise ValueError("scanner tasks did not provide the required exact healthy/repaired responsibility; inspect source evidence") diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index ae0005b5..e017bf8a 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -29,6 +29,7 @@ from .object_lock import BRANCHES as LOCK_BRANCHES, KEYS as LOCK_KEYS, seed_locked, prove_locked, cleanup_locked_fleet from .read_repair import BRANCHES as READ_BRANCHES, tracked_read_task from .drive_gate import validate_unwritable + from .scanner_repair import BRANCHES as SCANNER_BRANCHES, JournalWindow, observe_task as scanner_task from .shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) except ImportError: @@ -43,6 +44,7 @@ from object_lock import BRANCHES as LOCK_BRANCHES, KEYS as LOCK_KEYS, seed_locked, prove_locked, cleanup_locked_fleet from read_repair import BRANCHES as READ_BRANCHES, tracked_read_task from drive_gate import validate_unwritable + from scanner_repair import BRANCHES as SCANNER_BRANCHES, JournalWindow, observe_task as scanner_task from shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) @@ -62,6 +64,7 @@ "heal.metadata-payload-mismatch": ("length-mismatch", "payload-mismatch", "insufficient-authority"), "heal.read-repair-attribution": READ_BRANCHES, "heal.disk-readonly-and-permission": ("read-only", "permission-denied"), + "heal.scanner-cold-object-repair": SCANNER_BRANCHES, } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") @@ -505,6 +508,15 @@ def reenable_and_verify(recorder, fleet, bucket, baseline): def branch_plan(case_id, branch, snapshot, bucket, expected, fixture): """Plans are derived from ACK order and decoded physical roles, never disk numbers.""" + if case_id == "heal.scanner-cold-object-repair": + key = f"objects/{8 + 9 * SCANNER_BRANCHES.index(branch):04d}.bin" + selected = [entry for entry in expected if entry["key"] == key] + require(len(selected) == 1 and selected[0]["versionId"] == "null", "scanner fixture lacks a unique cold target") + target = choose_target(snapshot, bucket, selected[0], "parity" if branch == "cold-bitrot" else "data") + return {"scope": {"bucket": bucket, "key": key}, "recursive": False, "expected": selected, + "faults": [target], "proofs": [target], "requiredRepairs": {version_identity(target)}, + "metadataRepairs": set(), "kind": "bitflip" if branch == "cold-bitrot" else "missing", + "offset": 32 if branch == "cold-bitrot" else None} if case_id == "heal.disk-readonly-and-permission": key = f"objects/{8 + 9 * BRANCHES[case_id].index(branch):04d}.bin" selected = [entry for entry in expected if entry["key"] == key] @@ -685,7 +697,8 @@ def run_topology(args, topology, nodes, fixture): directory.mkdir(parents=True) run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" fleet = Fleet(nodes, directory, run_id, args.tool_directory, - runtime_profile="read-repair" if args.case_id == "heal.read-repair-attribution" else "manual") + runtime_profile="scanner" if args.case_id == "heal.scanner-cold-object-repair" else + "read-repair" if args.case_id == "heal.read-repair-attribution" else "manual") candidate = json.loads(args.candidate.read_text()) tool_manifest = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) contract = verify_quorum_contract(tool_manifest, candidate) if args.case_id in { @@ -717,6 +730,12 @@ def run_topology(args, topology, nodes, fixture): fleet.deploy(args.binary_sha, bucket, access, secret) processes = fleet.snapshot("deployed-topology") fleet.record("topology-proof", validate_topology(processes, topology)) + scanner = args.case_id == "heal.scanner-cold-object-repair" + healthy_window = JournalWindow(fleet, "scanner-healthy") if scanner else None + if scanner: + fleet.record("scanner-registered-budget", {"perBranchSeconds": 240, "sourceBranches": list(SCANNER_BRANCHES), + "actualProcessConfiguration": {node: value["backgroundConfig"] for node, value in processes.items()}, + "otherAutomaticSourcesDisabled": True, "targetReadsAfterFaultBeforePhysicalProof": 0}) recorder = Recorder(probe, directory / "fixture") public, instances, sizes = None, None, None if size_rule: @@ -750,6 +769,14 @@ def run_topology(args, topology, nodes, fixture): fleet.record("special-key-pagination-baseline", verify_pagination(baseline)) if public is not None: fleet.record("semantic-baseline", verify_semantics(public, baseline)) + if scanner: + healthy_before = fleet.snapshot("scanner-healthy-physical-before") + for branch in branches: + plan = branch_plan(args.case_id, branch, healthy_before, bucket, expected, fixture) + status = scanner_task(probe, fleet, healthy_window, bucket, plan["expected"][0], branch, healthy=True) + fleet.record("scanner-healthy-responsibility", validate_version_receipts(status, bucket, plan["expected"], 2)) + require(physical_content(healthy_before) == physical_content(fleet.snapshot("scanner-healthy-physical-after")), + "healthy scanner preflight changed acknowledged object files") if args.case_id == "heal.multiple-shard-loss": fleet.stop() instances = multi_loss_instances(fleet.snapshot("multi-loss-geometry", quiescent=True), bucket, expected, fixture) @@ -769,6 +796,7 @@ def run_topology(args, topology, nodes, fixture): key=LOCK_KEYS[LOCK_BRANCHES.index(branch)], label="lock-before-" + branch)) fleet.stop() before = fleet.snapshot("physical-before-" + branch, quiescent=True) + scanner_window = JournalWindow(fleet, "scanner-fault-" + branch) if scanner else None if size_rule: plan = boundary_plan(next(entry for entry in instances if entry["name"] == branch), before, bucket, expected) elif instances: @@ -811,8 +839,9 @@ def run_topology(args, topology, nodes, fixture): if gate_node is not None: fleet.record("service-write-denial", fleet.remote(gate_node, "drive-gate-probe")) witness = fleet.snapshot("physical-before-trigger-" + branch, metadata_faults=metadata_faults) - require(physical_content(prior) == physical_content(witness), - "ATTRIBUTION_NOT_OBSERVED: version fault changed before the manual token") + if not scanner: + require(physical_content(prior) == physical_content(witness), + "ATTRIBUTION_NOT_OBSERVED: version fault changed before the manual token") if plan.get("negative"): target = plan["proofs"][0] fleet.record("quorum-contract", {"contract": contract, "readQuorum": plan["readQuorum"], @@ -824,6 +853,8 @@ def run_topology(args, topology, nodes, fixture): "negative read widened damage or silently healed below-quorum fragments") if args.case_id == "heal.read-repair-attribution": status = tracked_read_task(probe, fleet, bucket, branch, plan["expected"][0]) + elif scanner: + status = scanner_task(probe, fleet, scanner_window, bucket, plan["expected"][0], branch) else: status = run_task(probe, fleet, plan["scope"], 2, active, option_overrides={"recursive": plan["recursive"]}, admissions=admissions) @@ -990,7 +1021,7 @@ def interrupted(signum, frame): fixtures = ("DB",) if args.case_id == "heal.object-lock-retention": fixtures = ("DO",) - if args.case_id == "heal.disk-readonly-and-permission": + if args.case_id in {"heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair"}: fixtures = ("DB",) results = [] try: diff --git a/test_validation_heal.py b/test_validation_heal.py index 2b1e7919..fbac0fd4 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -35,6 +35,9 @@ from scripts.validation_heal import heal_fault from scripts.validation_heal.read_repair import source_tasks, validate_read, tracked_read_task, ADMISSION_MESSAGE, ADMISSION_TARGET from scripts.validation_heal.drive_gate import apply_gate, release_gate, checked_gate, start_properties, validate_probe, validate_unwritable +from scripts.validation_heal.scanner_repair import (source_tasks as scanner_sources, cycle_pairs, + observe_task as observe_scanner, JournalWindow) +from scripts.validation_heal.fleet_node import logs as node_logs import errno import stat @@ -1162,6 +1165,134 @@ def test_read_profile_keeps_other_repair_sources_disabled(self): self.assertEqual(len(plan["expected"]), 1) +class HealScannerSourceTest(unittest.TestCase): + bucket, key = "rv-heal-scanner-unit", "objects/0008.bin" + + def events(self): + def row(second, **values): + return {"timestamp": f"2026-10-10T12:00:{second:02d}Z", **values} + return [ + row(1, target="rustfs::scanner", event="scanner_cycle_state", state="started", cycle=7, scan_mode="Deep"), + row(2, target="rustfs::scanner::folder", event="scanner_heal_admission", state="request_started", + bucket=self.bucket, object=self.key, version_id="00000000-0000-0000-0000-000000000000"), + row(3, target="rustfs::heal::channel", state="start_received", request_id="task-one", + bucket=self.bucket, object_prefix=self.key), + row(4, target="rustfs::heal::manager", message="Heal queue request accepted", request_id="task-one", + source="scanner", result="accepted"), + row(5, target="rustfs::heal::task", message="Heal object started", stage="start", task_id="task-one", + bucket=self.bucket, object=self.key, version_id="None"), + row(6, target="rustfs::heal::manager", state="task_completed", task_id="task-one"), + row(7, target="rustfs::scanner", event="scanner_cycle_state", state="completed", cycle=0, scan_mode="Deep"), + ] + + def entry(self): + return {"key": self.key, "versionId": "null", "deleteMarker": False} + + def test_source_chain_requires_exact_scanner_admission_version_cycle_and_task(self): + rows = self.events() + self.assertEqual(scanner_sources({"host": rows}, self.bucket, self.entry())[0]["taskId"], "task-one") + mutations = ((1, "bucket", "other"), (1, "object", "other"), (1, "version_id", "historical"), + (2, "object_prefix", "other"), (3, "source", "read-repair"), (3, "result", "merged_duplicate"), + (4, "task_id", "another"), (4, "version_id", "historical"), (5, "task_id", "another"), + (6, "state", "failed"), (0, "scan_mode", "Normal"), (1, "timestamp", "2026-10-10T12:00:08Z")) + for index, field, value in mutations: + changed = copy.deepcopy(rows) + changed[index][field] = value + with self.subTest(index=index, field=field): + self.assertFalse(scanner_sources({"host": changed}, self.bucket, self.entry())) + for index in (1, 4): + changed = copy.deepcopy(rows) + changed[index].pop("version_id") + self.assertFalse(scanner_sources({"host": changed}, self.bucket, self.entry())) + self.assertFalse(scanner_sources({"host": rows[:1] + rows[-1:]}, self.bucket, self.entry())) + self.assertFalse(scanner_sources({"host": rows[:4], "peer": rows[4:]}, self.bucket, self.entry())) + + def test_task_may_start_after_discovery_cycle_but_cannot_precede_admission(self): + rows = self.events() + rows[4]["timestamp"] = "2026-10-10T12:00:08Z" + rows[5]["timestamp"] = "2026-10-10T12:00:09Z" + self.assertTrue(scanner_sources({"host": rows}, self.bucket, self.entry())) + rows[4]["timestamp"] = "2026-10-10T12:00:03Z" + self.assertFalse(scanner_sources({"host": rows}, self.bucket, self.entry())) + + def test_cycle_pair_requires_ordered_positive_start_and_complete_boundary(self): + rows = self.events() + self.assertEqual(cycle_pairs(rows)[0]["cycle"], 7) + for change in ({"cycle": 9}, {"timestamp": "2026-10-10T11:00:00Z"}, {"timestamp": "invalid"}): + changed = copy.deepcopy(rows) + changed[-1].update(change) + self.assertFalse(cycle_pairs(changed)) + failed = {**rows[-1], "state": "failed", "timestamp": "2026-10-10T12:00:06Z"} + self.assertFalse(cycle_pairs(rows[:-1] + [failed, rows[-1]])) + + def test_observer_uses_status_only_and_rejects_healthy_substitution_for_repair(self): + fleet = types.SimpleNamespace(runtime_profile="scanner", nodes=["host"], record=lambda label, value: value) + window = types.SimpleNamespace(capture=lambda: {"host": self.events()}) + def status(state): + return {"summary": "finished", "outcome": {"objects": [{"disposition": {"state": state}}]}} + client = types.SimpleNamespace(binding={}) + calls = [] + def query(path, args, body): + calls.append((path, args, body)) + return {"httpStatus": 200, "response": status("repaired")} + with patch("scripts.validation_heal.scanner_repair.Probe", return_value=types.SimpleNamespace(admin=query)): + self.assertEqual(observe_scanner(client, fleet, window, self.bucket, self.entry(), "cold-missing"), status("repaired")) + self.assertEqual(calls, [("/rustfs/admin/v3/heal/" + self.bucket + "/" + self.key, + [("clientToken", "task-one")], b"")]) + observer = types.SimpleNamespace(admin=lambda *args: {"httpStatus": 200, "response": status("verified_healthy")}) + with patch("scripts.validation_heal.scanner_repair.Probe", return_value=observer): + with self.assertRaisesRegex(ValueError, "required exact"): + observe_scanner(client, fleet, window, self.bucket, self.entry(), "cold-missing") + observe_scanner(client, fleet, window, self.bucket, self.entry(), "cold-missing", healthy=True) + + def test_boundary_branch_requires_two_complete_post_fault_cycles(self): + fleet = types.SimpleNamespace(runtime_profile="scanner", nodes=["host"], record=lambda label, value: value) + window = types.SimpleNamespace(capture=lambda: {"host": self.events()}) + with patch("scripts.validation_heal.scanner_repair.time.monotonic", side_effect=[0, 241]), \ + patch("scripts.validation_heal.scanner_repair.Probe") as observer: + with self.assertRaisesRegex(ValueError, "ATTRIBUTION_NOT_OBSERVED"): + observe_scanner(types.SimpleNamespace(binding={}), fleet, window, self.bucket, self.entry(), "scanner-cycle-boundary") + observer.assert_not_called() + + def test_cursor_windows_are_complete_and_old_events_are_excluded(self): + calls = [] + observations = [{"complete": True, "afterCursor": "cursor-1", "journal": json.dumps(self.events()[0])}, + {"complete": True, "beforeCursor": "cursor-1", "afterCursor": "cursor-2", "journal": json.dumps(self.events()[1])}] + def remote(node, action, **params): + calls.append(params) + return observations.pop(0) + fleet = types.SimpleNamespace(nodes=["host"], record=lambda label, value: value, remote=remote) + window = JournalWindow(fleet, "unit") + self.assertEqual(window.capture(), {"host": [self.events()[1]]}) + self.assertEqual(calls, [{"checkpoint": True}, {"afterCursor": "cursor-1"}]) + window.fleet.remote = lambda *args, **kwargs: {"complete": False, "beforeCursor": "cursor-2", "afterCursor": "cursor-3"} + with self.assertRaisesRegex(ValueError, "completeness"): window.capture() + + def test_remote_log_window_rejects_truncation_and_invalid_cursor(self): + owner = {"unit": "private.service"} + with patch("scripts.validation_heal.fleet_node.owned", return_value=(None, None, owner)), \ + patch("scripts.validation_heal.fleet_node.command") as command: + command.return_value = types.SimpleNamespace(stdout='{"message":"fresh"}\n-- cursor: s=abc;i=def\n') + value = node_logs({"runId": "unit", "afterCursor": "s=previous;i=1"}) + self.assertTrue(value["complete"]) + self.assertEqual(value["afterCursor"], "s=abc;i=def") + self.assertIn("--after-cursor", command.call_args.args) + with self.assertRaisesRegex(ValueError, "invalid journal cursor"): + node_logs({"runId": "unit", "afterCursor": "invalid\n--help"}) + command.return_value = types.SimpleNamespace(stdout="row\n" * 10000 + "-- cursor: s=abc;i=def\n") + with self.assertRaisesRegex(ValueError, "incomplete"): + node_logs({"runId": "unit", "afterCursor": "s=previous;i=1"}) + + def test_scanner_profile_enables_only_scanner_source_and_fixed_deep_timing(self): + config = runtime_config("scanner") + self.assertEqual(config["RUSTFS_SCANNER_ENABLED"], "true") + for name in ("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "RUSTFS_HEAL_MRF_ENABLE"): + self.assertEqual(config[name], "false") + self.assertEqual(config["RUSTFS_SCANNER_BITROT_CYCLE_SECS"], "0") + self.assertEqual(config["RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS"], "0") + self.assertEqual(config["RUSTFS_HEAL_OBJECT_SELECT_PROB"], "1") + + class HealPayloadMismatchTest(unittest.TestCase): def test_length_and_payload_donors_keep_original_metadata_and_exact_same_role(self): branches = VERSION_BRANCHES["heal.metadata-payload-mismatch"] From a5e370163eb921f7c8e706c1d72903f3114d73f2 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 01:36:09 +0800 Subject: [PATCH 055/343] feat(validation): migrate acknowledged delete-marker repair semantics --- docs/validation-migration-map.md | 22 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/drive_gate.py | 23 ++- scripts/validation_heal/markers.py | 240 ++++++++++++++++++++++++++ scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 27 ++- test_validation_heal.py | 119 ++++++++++++- 8 files changed, 429 insertions(+), 8 deletions(-) create mode 100644 scripts/validation_heal/markers.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 67f49d90..2c2edc53 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 102 currently dispatchable IDs include all 74 non-Heal cases and twenty-eight Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. +case evidence. The 103 currently dispatchable IDs include all 74 non-Heal cases and twenty-nine Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 56 Heal cases remain +The other 55 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -488,3 +488,21 @@ responsibility, original physical shard bytes, streaming checksum and complete S3 inventories still gate PASS. Truncated logs, another source, a later healthy receipt or cycle counters alone cannot certify repair. The executor passed its local source/timing/cursor negatives; full live evidence remains pending. + +D04 `heal.delete-marker-semantics` now dispatches its four H1/H2 source +branches as six real instances. Each key has three separately acknowledged +data versions and, where applicable, a current or historical marker. Missing +replicas cover both physical roles. Stale creation and deletion isolate one +original drive with a service-identity read/write EACCES gate, perform one +exact marker mutation, and prove the gated replica stayed unchanged while all +other members carry the ACK authority. The expected metadata is derived only +for comparison from that authority and the original target-drive data +references; the harness never writes synthesized metadata. Formats, data +shards, controls, exact version receipts and streaming checksums must agree. +Every peer checks current-marker 404 and explicit-marker 405 with exact marker +identity headers. Explicit current-marker deletion must reveal the precise +previous data version, remove the marker identity and preserve the full +remaining version/public-property manifest. A failed or ambiguous stale ACK +stops the topology and retains its result; gate cleanup precedes physical +finalization. Local ACK/order/authority/header/gate negatives passed; full live +evidence remains pending. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index b8ad4bb1..0cabaad8 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/drive_gate.py b/scripts/validation_heal/drive_gate.py index 7dafc9a6..90ed8016 100644 --- a/scripts/validation_heal/drive_gate.py +++ b/scripts/validation_heal/drive_gate.py @@ -22,6 +22,17 @@ result["errno"] = 0 print(json.dumps(result)) ''' +READ_PROBE_SOURCE = '''import json,os +result = {"uid": os.geteuid(), "gid": os.getegid()} +try: + fd = os.open(__import__('sys').argv[1], os.O_RDONLY | os.O_NOFOLLOW) +except OSError as exc: + result["errno"] = exc.errno +else: + os.close(fd) + result["errno"] = 0 +print(json.dumps(result)) +''' def require(value, message): @@ -116,7 +127,17 @@ def probe_gate(state, roots, pid): result = subprocess.run(args, capture_output=True, text=True, timeout=15) require(result.returncode == 0, "service-namespace write probe did not complete") observed = json.loads(result.stdout) - return {**validate_probe(observed, value["mode"], identity), "disk": value["disk"], + read = None + if value["mode"] == "permission-denied": + # A denied write alone cannot prove a stale member cannot read. Use + # the same namespace/identity/capability boundary to open its format. + read_args = [*args[:-4], "python3", "-c", READ_PROBE_SOURCE, + value["dataPath"] + "/.rustfs.sys/format.json"] + result = subprocess.run(read_args, capture_output=True, text=True, timeout=15) + require(result.returncode == 0, "service-namespace read probe did not complete") + read = json.loads(result.stdout) + validate_probe(read, value["mode"], identity) + return {**validate_probe(observed, value["mode"], identity), "disk": value["disk"], "readProbe": read, "privateMountNamespace": True, "hostMountRemounted": False, "observed": observed} diff --git a/scripts/validation_heal/markers.py b/scripts/validation_heal/markers.py new file mode 100644 index 00000000..2eac87b4 --- /dev/null +++ b/scripts/validation_heal/markers.py @@ -0,0 +1,240 @@ +"""D04: genuine marker ACKs, missing metadata and same-drive stale transitions.""" + +import copy +import hashlib + +try: + from .heal_probe import Probe, payload, require + from .heal_workload import verify_seed + from .healthy import physical_content + from .shards import files +except ImportError: + from heal_probe import Probe, payload, require + from heal_workload import verify_seed + from healthy import physical_content + from shards import files + + +BRANCHES = ("current-marker", "historical-marker", "missing-marker-replica", "stale-disk-marker") +INSTANCES = ( + {"name": "current-marker", "sourceBranch": "current-marker", "mode": "current", "role": "data"}, + {"name": "historical-marker", "sourceBranch": "historical-marker", "mode": "historical", "role": "parity"}, + {"name": "missing-current-data", "sourceBranch": "missing-marker-replica", "mode": "current", "role": "data"}, + {"name": "missing-historical-parity", "sourceBranch": "missing-marker-replica", "mode": "historical", "role": "parity"}, + {"name": "stale-create-current-marker", "sourceBranch": "stale-disk-marker", "mode": "create", "role": "data"}, + {"name": "stale-delete-current-marker", "sourceBranch": "stale-disk-marker", "mode": "delete", "role": "parity"}, +) + + +def key_for(instance): + return "markers/" + instance["name"] + ".bin" + + +def add_marker(expected, key, response): + marker = response.get("VersionId") + require(response.get("DeleteMarker") is True and isinstance(marker, str) and marker not in {"", "null"} + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 204 + and not any(entry["versionId"] == marker for entry in expected), "marker creation lacks a distinct exact DELETE ACK") + result = copy.deepcopy(expected) + require(any(entry["key"] == key for entry in result), "marker creation has no acknowledged data key") + for entry in result: + if entry["key"] == key: entry["isLatest"] = False + result.append({"key": key, "versionId": marker, "deleteMarker": True, "isLatest": True}) + return result + + +def remove_marker(expected, key, response): + selected = [entry for entry in expected if entry["key"] == key and entry["deleteMarker"] and entry["isLatest"]] + require(len(selected) == 1 and response.get("VersionId") == selected[0]["versionId"] + and response.get("DeleteMarker") is True and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 204, + "current marker removal lacks its exact DELETE ACK") + result = [copy.deepcopy(entry) for entry in expected if entry != selected[0]] + data = [entry for entry in result if entry["key"] == key and not entry["deleteMarker"]] + require(len(data) == 3, "marker removal lost or invented a retained data version") + for entry in result: + if entry["key"] == key: entry["isLatest"] = entry is data[-1] + return result + + +def seed_markers(recorder, bucket, size): + recorder.call("create_bucket", {"Bucket": bucket}) + recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) + expected = [] + for instance in INSTANCES: + key = key_for(instance) + for nonce in range(3): + body = payload(f"{bucket}/{key}/{nonce}", size) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "Metadata": {"heal-fixture": "marker-semantics", "generation": str(nonce)}, + "Tagging": f"suite=heal&generation=v{nonce}"}, body) + version = response.get("VersionId") + require(isinstance(version, str) and version not in {"", "null"} + and not any(entry["versionId"] == version for entry in expected), "marker fixture lacks distinct versioned PUT ACKs") + for entry in expected: + if entry["key"] == key: entry["isLatest"] = False + expected.append({"key": key, "versionId": version, "deleteMarker": False, "isLatest": True, + "payload": {"size": size, "sha256": hashlib.sha256(body).hexdigest()}}) + if instance["mode"] == "historical" and nonce == 1: + expected = add_marker(expected, key, recorder.call("delete_object", {"Bucket": bucket, "Key": key})) + if instance["mode"] in {"current", "delete"}: + expected = add_marker(expected, key, recorder.call("delete_object", {"Bucket": bucket, "Key": key})) + return expected + + +def marker_plan(instance, snapshot, bucket, expected): + # Imported lazily: the shared version executor also uses this module. + try: + from .versions import choose_target, metadata_target + except ImportError: + from versions import choose_target, metadata_target + key = key_for(instance) + selected = [entry for entry in expected if entry["key"] == key] + data = [entry for entry in selected if not entry["deleteMarker"]] + require(len(data) == 3 and len({entry["versionId"] for entry in selected}) == len(selected), "marker plan lacks the three exact data versions") + if instance["mode"] != "create": + require(len([entry for entry in selected if entry["deleteMarker"]]) == 1, "marker plan lacks its acknowledged current/historical marker") + target = choose_target(snapshot, bucket, data[0], instance["role"]) + proofs = [choose_target(snapshot, bucket, entry, "data") for entry in data] + proofs = [{**item, **next(shard for shard in item["cohort"] if + (shard["node"], shard["disk"]) == (target["node"], target["disk"]))} for item in proofs] + return {"scope": {"bucket": bucket, "key": key}, "recursive": True, "expected": selected, + "faults": [] if instance["mode"] in {"create", "delete"} else [metadata_target(snapshot, target)], + "proofs": proofs, "requiredRepairs": set(), "metadataRepairs": {key}, "parameters": instance, + "staleMarker": instance["mode"] if instance["mode"] in {"create", "delete"} else None} + + +def stale_expected_scene(original, authorities, bucket, key, expected, target): + """Derive an oracle from real quorum metadata. This never writes metadata.""" + left, right = files(original), files(authorities) + selected = (target["node"], target["disk"]) + require({node: value["formats"] for node, value in original.items()} == + {node: value["formats"] for node, value in authorities.items()}, "stale window changed original disk/set identities") + require({identity: entry for identity, entry in left.items() if identity[:2] == selected} == + {identity: entry for identity, entry in right.items() if identity[:2] == selected}, "inaccessible member received a marker write") + wanted = {(entry["versionId"], entry["deleteMarker"], entry["isLatest"]) + for entry in expected if entry["key"] == key} + path = f"{bucket}/{key}/xl.meta" + donors = [entry["metadata"] for identity, entry in right.items() if identity[2] == path and identity[:2] != selected] + require(len(donors) == len(target["cohort"]) - 1 and all( + {(version["versionId"], version["deleted"], version["isLatest"]) for version in donor["versions"]} == wanted + for donor in donors), "online marker replicas lack a full same-version ACK authority quorum") + for identity, entry in right.items(): + if identity[2] != path or identity[:2] == selected: continue + previous = left[identity]["metadata"] + actual = entry["metadata"] + require({name: value for name, value in previous.items() if name != "versions"} == + {name: value for name, value in actual.items() if name != "versions"}, + "marker operation changed the metadata format or object identity") + previous_data = {version["versionId"]: {name: value for name, value in version.items() if name != "isLatest"} + for version in previous["versions"] if not version["deleted"]} + actual_data = {version["versionId"]: {name: value for name, value in version.items() if name != "isLatest"} + for version in actual["versions"] if not version["deleted"]} + require(previous_data == actual_data, "marker operation changed a retained data-version reference") + current_meta = left[(*selected, path)]["metadata"] + require({(version["versionId"], version["deleted"], version["isLatest"]) for version in current_meta["versions"]} != wanted, + "marker window did not create a genuinely stale replica") + template = donors[0] + old = {version["versionId"]: version for version in current_meta["versions"]} + versions = [] + for authority in template["versions"]: + if authority["deleted"]: + require(authority.get("dataDir") is None and not authority.get("parts") and authority.get("inline") is None, + "marker authority unexpectedly has live data references") + versions.append(copy.deepcopy(authority)) + else: + require(authority["versionId"] in old and not old[authority["versionId"]]["deleted"], "stale marker changed a data generation") + version = copy.deepcopy(old[authority["versionId"]]) + version["isLatest"] = authority["isLatest"] + versions.append(version) + oracle = copy.deepcopy(authorities) + for entry in oracle[selected[0]]["files"]: + if entry["disk"] == selected[1] and entry["path"] == path: + entry["metadata"]["versions"] = versions + # Data shards and all controls are unchanged throughout the legitimate + # marker operation; only this key's xl.meta is allowed to change online. + require({identity: entry for identity, entry in left.items() if identity[2] != path} == + {identity: entry for identity, entry in right.items() if identity[2] != path}, + "marker ACK changed a data shard, auxiliary file or control") + return oracle + + +def prepare_stale(recorder, fleet, bucket, plan, before, expected): + target = plan["proofs"][0] + node, disk, key = target["node"], target["disk"], plan["scope"]["key"] + fleet.drive_gates.add(node) + fleet.record("marker-member-isolated", fleet.remote(node, "drive-gate", disk=disk, mode="permission-denied")) + fleet.start() + refusal = fleet.record("marker-service-denial", fleet.remote(node, "drive-gate-probe")) + require(refusal.get("readProbe", {}).get("errno") == 13 and refusal.get("dacBypassRemoved") is True, + "stale marker member lacks actual same-service read/write denial") + params = {"Bucket": bucket, "Key": key} + if plan["staleMarker"] == "delete": + params["VersionId"] = next(entry["versionId"] for entry in plan["expected"] if entry["deleteMarker"] and entry["isLatest"]) + response = recorder.call("delete_object", params) + manifest = (add_marker if plan["staleMarker"] == "create" else remove_marker)(expected, key, response) + baseline = fleet.record("marker-online-ack-inventory", recorder.probe.inventory(bucket)) + fleet.record("marker-online-ack-proof", {"expected": manifest, **verify_seed(manifest, baseline)}) + fleet.stop() + authority = fleet.snapshot("marker-stale-authorities", quiescent=True) + oracle = stale_expected_scene(before, authority, bucket, key, manifest, target) + fleet.record("marker-derived-physical-oracle", {"scene": oracle, "metadataWrittenByHarness": False, + "derivation": "acknowledged marker quorum plus original same-drive data-version references"}) + fleet.record("marker-member-rejoined", fleet.remote(node, "drive-gate", mode="clear")) + fleet.drive_gates.remove(node) + plan["expected"] = [entry for entry in manifest if entry["key"] == key] + return oracle, authority, manifest, baseline + + +def validate_marker_error(observed, status, version): + response = observed.get("response") or {} + metadata = response.get("ResponseMetadata", {}) + headers = {name.lower(): value for name, value in metadata.get("HTTPHeaders", {}).items()} + require(observed.get("acknowledged") is False and observed.get("resultUnknown") is False and "payload" not in observed + and metadata.get("HTTPStatusCode") == status and headers.get("x-amz-delete-marker") == "true" + and headers.get("x-amz-version-id") == version, "marker refusal lost its exact HTTP/version/marker identity") + + +def verify_markers(probe, fleet, bucket, key, expected): + selected = [entry for entry in expected if entry["key"] == key] + probes = [] + for endpoint in probe.binding["endpoints"]: + peer = Probe(probe.binding, endpoint) + for entry in [entry for entry in selected if entry["deleteMarker"]]: + requests = [(entry["versionId"], 405)] + ([(None, 404)] if entry["isLatest"] else []) + for selector, status in requests: + for method in ("get_object", "head_object"): + params = {"Bucket": bucket, "Key": key} + if selector: params["VersionId"] = selector + observed = fleet.record("marker-s3-refusal", peer.s3(method, params)) + validate_marker_error(observed, status, entry["versionId"]) + probes.append({"endpoint": endpoint, "method": method, "selector": selector, + "status": status, "markerVersion": entry["versionId"]}) + return {"allPeerMarkerIdentityHeaders": True, "probes": probes} + + +def reveal_current(recorder, fleet, bucket, key, expected, baseline): + current = next((entry for entry in expected if entry["key"] == key and entry["deleteMarker"] and entry["isLatest"]), None) + if current is None: return expected, baseline, {"currentMarkerPresent": False} + response = recorder.call("delete_object", {"Bucket": bucket, "Key": key, "VersionId": current["versionId"]}) + manifest = remove_marker(expected, key, response) + revealed = next(entry for entry in manifest if entry["key"] == key and entry["isLatest"]) + for endpoint in recorder.probe.binding["endpoints"]: + peer = Probe(recorder.probe.binding, endpoint) + read = fleet.record("marker-revealed-data", peer.s3("get_object", {"Bucket": bucket, "Key": key})) + require(read.get("acknowledged") and read.get("payload") == revealed["payload"] + and read.get("response", {}).get("VersionId") == revealed["versionId"], "marker removal exposed another data generation") + deleted = fleet.record("marker-removed-version", peer.s3("get_object", { + "Bucket": bucket, "Key": key, "VersionId": current["versionId"]})) + require(deleted.get("acknowledged") is False and deleted.get("resultUnknown") is False + and "payload" not in deleted and deleted.get("response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode") == 404, + "explicitly removed marker remained addressable") + inventory = fleet.record("marker-removal-inventory", recorder.probe.inventory(bucket)) + verify_seed(manifest, inventory) + old = {(entry["key"], entry["versionId"]): entry for entry in baseline["entries"]} + require(len(inventory["entries"]) == len(baseline["entries"]) - 1, "marker removal created or lost extra versions") + for entry in inventory["entries"]: + previous = copy.deepcopy(old[(entry["key"], entry["versionId"])]) + if entry["key"] == key: previous["isLatest"] = entry["isLatest"] + require(previous == entry, "marker removal changed a retained version or public property") + return manifest, inventory, {"currentMarkerRemoved": current["versionId"], "revealedVersion": revealed["versionId"], + "noAdditionalDataVersions": True, "allPeerCurrentReadsMatch": True} diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 29979d25..23526975 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 05be9a04..0207eca5 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -20,7 +20,7 @@ done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization) ;; heal.node-outage-rejoin) driver=node_outage.py ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index e017bf8a..89753279 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -30,6 +30,8 @@ from .read_repair import BRANCHES as READ_BRANCHES, tracked_read_task from .drive_gate import validate_unwritable from .scanner_repair import BRANCHES as SCANNER_BRANCHES, JournalWindow, observe_task as scanner_task + from .markers import (BRANCHES as MARKER_BRANCHES, INSTANCES as MARKER_INSTANCES, seed_markers, + marker_plan, prepare_stale, verify_markers, reveal_current) from .shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) except ImportError: @@ -45,6 +47,8 @@ from read_repair import BRANCHES as READ_BRANCHES, tracked_read_task from drive_gate import validate_unwritable from scanner_repair import BRANCHES as SCANNER_BRANCHES, JournalWindow, observe_task as scanner_task + from markers import (BRANCHES as MARKER_BRANCHES, INSTANCES as MARKER_INSTANCES, seed_markers, + marker_plan, prepare_stale, verify_markers, reveal_current) from shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) @@ -65,6 +69,7 @@ "heal.read-repair-attribution": READ_BRANCHES, "heal.disk-readonly-and-permission": ("read-only", "permission-denied"), "heal.scanner-cold-object-repair": SCANNER_BRANCHES, + "heal.delete-marker-semantics": MARKER_BRANCHES, } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") @@ -750,6 +755,9 @@ def run_topology(args, topology, nodes, fixture): expected = seed_locked(recorder, fleet, bucket, lock_context) elif args.case_id == "heal.metadata-payload-mismatch": expected = seed_mismatch(recorder, bucket) + elif args.case_id == "heal.delete-marker-semantics": + expected = seed_markers(recorder, bucket, VERSION_SIZE) + instances = list(MARKER_INSTANCES) elif fixture == "DK": expected = seed_special_keys(recorder, bucket) elif fixture == "DS": @@ -786,10 +794,14 @@ def run_topology(args, topology, nodes, fixture): if size_rule: branches = [entry["name"] for entry in instances] fleet.record("boundary-parameter-matrix", {"instances": instances, "sourceBranches": list(SIZE_BRANCHES)}) + if args.case_id == "heal.delete-marker-semantics": + branches = [entry["name"] for entry in instances] + fleet.record("marker-parameter-matrix", {"instances": instances, "sourceBranches": list(MARKER_BRANCHES)}) for branch in branches: registrations = [] metadata_faults = {} gate_node, gated_proof = None, None + stale_marker_started = False try: if lock_context is not None: fleet.record("lock-before-fault", prove_locked(probe, fleet, bucket, lock_context, @@ -799,6 +811,8 @@ def run_topology(args, topology, nodes, fixture): scanner_window = JournalWindow(fleet, "scanner-fault-" + branch) if scanner else None if size_rule: plan = boundary_plan(next(entry for entry in instances if entry["name"] == branch), before, bucket, expected) + elif args.case_id == "heal.delete-marker-semantics": + plan = marker_plan(next(entry for entry in instances if entry["name"] == branch), before, bucket, expected) elif instances: plan = multi_loss_plan(next(entry for entry in instances if entry["name"] == branch), before, bucket, expected, fixture) else: @@ -810,6 +824,9 @@ def run_topology(args, topology, nodes, fixture): if plan.get("donorProof"): verify_proof(fleet, plan["donorProof"], "baseline-donor-bitrot-" + branch) prior = before + if plan.get("staleMarker"): + stale_marker_started = True + before, prior, expected, baseline = prepare_stale(recorder, fleet, bucket, plan, before, expected) for index, target in enumerate(plan["faults"]): params = {"disk": target["disk"], "faultId": bounded_fault_id(recipe["code"], branch, index)} registration = target["node"], params @@ -901,6 +918,8 @@ def run_topology(args, topology, nodes, fixture): if lock_context is not None: s3["protection"] = prove_locked(probe, fleet, bucket, lock_context, key=LOCK_KEYS[LOCK_BRANCHES.index(branch)], label="lock-after-" + branch) + if args.case_id == "heal.delete-marker-semantics": + s3["markerSemantics"] = verify_markers(probe, fleet, bucket, plan["scope"]["key"], expected) if size_rule: s3["actualBoundaryRanges"] = verify_boundary_ranges(probe, fleet, bucket, expected, sizes["actualECBlockSize"]) # These faults have now been independently repaired and fully @@ -914,6 +933,9 @@ def run_topology(args, topology, nodes, fixture): "canonicalVerdict": "still requires the exact responsibility oracle"}) responsibility = validate_version_receipts(status, bucket, plan["expected"], 2, required_repairs=plan["requiredRepairs"], metadata_repairs=plan["metadataRepairs"]) + if args.case_id == "heal.delete-marker-semantics": + expected, baseline, s3["markerRemoval"] = reveal_current( + recorder, fleet, bucket, plan["scope"]["key"], expected, baseline) repeats = [] if args.case_id == "heal.repeat-idempotency": fleet.stop() @@ -967,6 +989,9 @@ def run_topology(args, topology, nodes, fixture): fleet.record("gate-error-rescue", fleet.remote(gate_node, "drive-gate", mode="clear")) fleet.drive_gates.remove(gate_node) fleet.snapshot("physical-failure-" + branch, quiescent=True, metadata_faults=metadata_faults) + for node in tuple(fleet.drive_gates): + fleet.record("marker-gate-error-rescue", fleet.remote(node, "drive-gate", mode="clear")) + fleet.drive_gates.remove(node) for registration in reversed(registrations): node, params = registration fleet.record("rescue-" + branch, fleet.remote(node, "recover-fault", **params)) @@ -974,7 +999,7 @@ def run_topology(args, topology, nodes, fixture): except Exception as recovery_error: result["branches"][branch]["recoveryError"] = str(recovery_error) raise - if isinstance(exc, KeyboardInterrupt) or active: + if isinstance(exc, KeyboardInterrupt) or active or stale_marker_started: raise result["verdict"] = "PASS" if len(result["branches"]) == len(branches) and all( item["verdict"] == "PASS" for item in result["branches"].values()) else "FAIL" diff --git a/test_validation_heal.py b/test_validation_heal.py index fbac0fd4..34a0f9cb 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -15,7 +15,7 @@ from scripts.validation_heal.healthy import validate_healthy, validate_topology, validate_rejection, Fleet from scripts.validation_heal.fleet_node import paths, stopped, inspect_metadata from scripts.validation_heal.inspector import verify as verify_inspector -from scripts.validation_heal.shards import BRANCHES, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal, validate_dry_run_status +from scripts.validation_heal.shards import BRANCHES, files, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal, validate_dry_run_status from scripts.validation_heal.heal_fault import mutation from scripts.validation_heal.shards import validate_conditional_response from scripts.validation_heal.shards import run_task, verify_ranges @@ -38,6 +38,9 @@ from scripts.validation_heal.scanner_repair import (source_tasks as scanner_sources, cycle_pairs, observe_task as observe_scanner, JournalWindow) from scripts.validation_heal.fleet_node import logs as node_logs +from scripts.validation_heal.markers import (INSTANCES as MARKER_INSTANCES, seed_markers, marker_plan, + stale_expected_scene, add_marker, remove_marker, validate_marker_error) +from scripts.validation_heal.drive_gate import probe_gate import errno import stat @@ -1293,6 +1296,120 @@ def test_scanner_profile_enables_only_scanner_source_and_fixed_deep_timing(self) self.assertEqual(config["RUSTFS_HEAL_OBJECT_SELECT_PROB"], "1") +class HealMarkerSemanticsTest(unittest.TestCase): + def response(self, version, **changes): + return {"VersionId": version, "DeleteMarker": True, "ResponseMetadata": {"HTTPStatusCode": 204}, **changes} + + def test_ack_manifest_retains_three_data_versions_and_exact_marker_order(self): + expected = [{"key": "key", "versionId": f"v{nonce}", "deleteMarker": False, "isLatest": nonce == 2} + for nonce in range(3)] + current = add_marker(expected, "key", self.response("marker")) + self.assertEqual(current[-1], {"key": "key", "versionId": "marker", "deleteMarker": True, "isLatest": True}) + self.assertTrue(all(not entry["isLatest"] for entry in current[:-1])) + self.assertEqual(remove_marker(current, "key", self.response("marker")), expected) + for response in (self.response("marker", DeleteMarker=False), self.response("v1"), self.response("null"), + self.response("marker", ResponseMetadata={"HTTPStatusCode": 200})): + with self.subTest(response=response), self.assertRaises(ValueError): add_marker(expected, "key", response) + with self.assertRaises(ValueError): remove_marker(current, "key", self.response("another")) + + def test_seed_has_real_historical_and_current_marker_ack_positions(self): + sequence = 0 + calls = [] + def call(operation, params, body=None): + nonlocal sequence + sequence += 1 + calls.append((operation, params)) + if operation == "put_object": return {"VersionId": f"data-{sequence}"} + if operation == "delete_object": return self.response(f"marker-{sequence}") + return {} + recorder = types.SimpleNamespace(call=call) + expected = seed_markers(recorder, "rv-heal-markers", 100) + self.assertEqual(len([entry for entry in expected if not entry["deleteMarker"]]), 18) + for instance in MARKER_INSTANCES: + key = "markers/" + instance["name"] + ".bin" + entries = [entry for entry in expected if entry["key"] == key] + markers = [entry for entry in entries if entry["deleteMarker"]] + self.assertEqual(len(markers), int(instance["mode"] != "create")) + if markers: + self.assertEqual(markers[0]["isLatest"], instance["mode"] != "historical") + selected = [operation for operation, params in calls if params.get("Key") == key] + self.assertEqual(selected, ["put_object", "put_object", "delete_object", "put_object"] + if instance["mode"] == "historical" else ["put_object"] * 3 + ["delete_object"]) + + def scene(self): + instance = next(item for item in MARKER_INSTANCES if item["mode"] == "create") + key = "markers/" + instance["name"] + ".bin" + before, bucket, expected = versioned_scene([key]) + plan = marker_plan(instance, before, bucket, expected) + target = plan["proofs"][0] + marker = {"versionId": "new-marker", "deleted": True, "isLatest": True, + "dataDir": None, "inline": None, "parts": [], "erasure": {}, "size": 0} + authorities = copy.deepcopy(before) + for entry in authorities[target["node"]]["files"]: + if "metadata" not in entry or entry["disk"] == target["disk"]: continue + for version in entry["metadata"]["versions"]: version["isLatest"] = False + entry["metadata"]["versions"].insert(0, copy.deepcopy(marker)) + entry["sha256"] += "marker-ack" + manifest = add_marker(expected, key, self.response("new-marker")) + return before, authorities, bucket, key, manifest, target + + def test_stale_oracle_uses_online_ack_authority_and_original_target_data_roles(self): + before, authorities, bucket, key, manifest, target = self.scene() + oracle = stale_expected_scene(before, authorities, bucket, key, manifest, target) + identity = (target["node"], target["disk"], f"{bucket}/{key}/xl.meta") + original = files(before)[identity]["metadata"]["versions"] + wanted = files(oracle)[identity]["metadata"]["versions"] + self.assertEqual(wanted[0]["versionId"], "new-marker") + for old, new in zip(original, wanted[1:], strict=True): + self.assertEqual({**old, "isLatest": False}, new) + self.assertEqual(files(before)[identity]["sha256"], files(oracle)[identity]["sha256"]) + self.assertEqual(files(authorities)[identity]["metadata"]["versions"], original) + + def test_stale_oracle_rejects_target_writes_wrong_quorum_and_control_changes(self): + before, authorities, bucket, key, manifest, target = self.scene() + for mutation in ("target", "authority", "data", "format"): + changed = copy.deepcopy(authorities) + entries = changed[target["node"]]["files"] + if mutation == "target": + next(entry for entry in entries if entry["disk"] == target["disk"])["sha256"] += "unexpected" + elif mutation == "authority": + next(entry for entry in entries if entry.get("metadata") and entry["disk"] != target["disk"])["metadata"]["versions"][0]["versionId"] = "foreign" + elif mutation == "data": next(entry for entry in entries if not entry.get("metadata"))["sha256"] += "changed" + else: changed[target["node"]]["formats"][0]["value"]["id"] += "changed" + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + stale_expected_scene(before, changed, bucket, key, manifest, target) + + def test_marker_negative_requires_http_headers_and_no_payload_or_unknown(self): + good = {"acknowledged": False, "resultUnknown": False, "response": {"ResponseMetadata": { + "HTTPStatusCode": 405, "HTTPHeaders": {"x-amz-version-id": "marker", "x-amz-delete-marker": "true"}}}} + validate_marker_error(good, 405, "marker") + for kind in ("status", "version", "marker", "payload", "transport", "ack"): + changed = copy.deepcopy(good) + if kind == "status": changed["response"]["ResponseMetadata"]["HTTPStatusCode"] = 404 + elif kind in {"version", "marker"}: changed["response"]["ResponseMetadata"]["HTTPHeaders"]["x-amz-" + ("version-id" if kind == "version" else "delete-marker")] = "wrong" + elif kind == "payload": changed["payload"] = {"size": 0} + elif kind == "transport": changed["resultUnknown"] = True + else: changed["acknowledged"] = True + with self.subTest(kind=kind), self.assertRaises(ValueError): validate_marker_error(changed, 405, "marker") + + def test_permission_probe_checks_read_and_write_with_same_namespace_and_dac_limit(self): + value = {"mode": "permission-denied", "dataPath": "/private/data", "disk": 1} + output = types.SimpleNamespace(returncode=0, stdout=json.dumps({"uid": 0, "gid": 0, "errno": errno.EACCES})) + with patch("scripts.validation_heal.drive_gate.checked_gate", return_value=value), \ + patch("scripts.validation_heal.drive_gate.process_identity", return_value={"uid": 0, "gid": 0, "capEff": 0}), \ + patch("pathlib.Path.readlink", side_effect=[Path("mnt:[private]"), Path("mnt:[host]")]), \ + patch("scripts.validation_heal.drive_gate.subprocess.run", return_value=output) as invoke: + result = probe_gate(None, None, 123) + self.assertEqual(result["readProbe"]["errno"], errno.EACCES) + self.assertEqual(invoke.call_count, 2) + for args in invoke.call_args_list: + command = args.args[0] + self.assertEqual(command.count("python3"), 1) + self.assertIn("--mount=/proc/123/ns/mnt", command) + self.assertIn("--bounding-set=-dac_override,-dac_read_search", command) + self.assertEqual(invoke.call_args.args[0][-1], "/private/data/.rustfs.sys/format.json") + + class HealPayloadMismatchTest(unittest.TestCase): def test_length_and_payload_donors_keep_original_metadata_and_exact_same_role(self): branches = VERSION_BRANCHES["heal.metadata-payload-mismatch"] From 41ab09382120abbb45e098bde23dccab11b8f0ef Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 01:46:20 +0800 Subject: [PATCH 056/343] feat(validation): migrate raw and rc Heal runtime status truth --- docs/validation-migration-map.md | 23 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 4 +- scripts/validation_heal/healthy.py | 6 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 1 + scripts/validation_heal/status.py | 338 ++++++++++++++++++++++++++ test_validation_heal.py | 106 ++++++++ 8 files changed, 475 insertions(+), 7 deletions(-) create mode 100644 scripts/validation_heal/status.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 2c2edc53..dccb309b 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 103 currently dispatchable IDs include all 74 non-Heal cases and twenty-nine Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. +case evidence. The 104 currently dispatchable IDs include all 74 non-Heal cases and thirty Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 55 Heal cases remain +The other 54 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -506,3 +506,22 @@ remaining version/public-property manifest. A failed or ambiguous stale ACK stops the topology and retains its result; gate cleanup precedes physical finalization. Local ACK/order/authority/header/gate negatives passed; full live evidence remains pending. + +J01 `heal.status-partial-disabled-unknown` now has a dedicated current-fleet +executor for H1/H2. Fresh manual and disabled deployments retain actual process +configuration, complete physical fixtures and both raw Admin and rc JSON/text +outputs. Peer coverage must balance exactly; absent peers need an explicit +unknown count/reason and cannot report idle. Any raw or CLI sample that erases +uncertainty fails immediately, including during bounded convergence. H2 stops +one original peer, witnesses its process exit and connection refusal, and +queries every survivor before restarting the same member. The source's step 2 +explicitly restricts that branch to H2; H1 records its applicability reason +and instead proves an original-service restart followed by complete idle. +A startup observer begins before any service starts and requires raw +uninitialized status to bracket matching rc/replacement observations. A +transport error or an already initialized service cannot prove that window; +NOT_OBSERVED makes the complete case fail. Original formats/object files and +full S3 manifests must remain unchanged. Temporary rc credentials use a 0600 +configuration outside artifacts and are removed during finalization. Local +coverage/uncertainty/startup/config-cleanup negatives passed; live evidence +remains pending. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 0cabaad8..961c8d6b 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index e9bca065..87269342 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -29,7 +29,9 @@ def runtime_config(profile): - require(profile in {"manual", "offline-replay", "read-repair", "scanner"}, "unsupported Heal runtime profile") + require(profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled"}, "unsupported Heal runtime profile") + if profile == "disabled": + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_ENABLED": "false"} if profile == "scanner": return {**BACKGROUND_CONFIG, "RUSTFS_SCANNER_ENABLED": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug", "RUSTFS_SCANNER_START_DELAY_SECS": "5", "RUSTFS_SCANNER_CYCLE": "5", diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 1100e7bb..852af123 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -131,7 +131,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual"): - require(runtime_profile in {"manual", "offline-replay", "read-repair", "scanner"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled"}, "unsupported Heal runtime profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory self.runtime_profile = runtime_profile @@ -191,7 +191,7 @@ def start(self): self.record("started", dict(zip(self.nodes, results, strict=True))) self.wait_ready() - def deploy(self, binary_sha, bucket, access, secret): + def deploy(self, binary_sha, bucket, access, secret, *, before_start=None): self.secret_values.update({access, secret}) checked = {node: self.remote(node, "preflight") for node in self.nodes} self.record("preflight", checked) @@ -224,6 +224,8 @@ def deploy(self, binary_sha, bucket, access, secret): input=tool.read_bytes(), capture_output=True, timeout=120) require(transfer.returncode == 0, "exclusive inspector upload failed: " + transfer.stderr.decode()[-800:]) self.record("inspector-upload", {"node": node, **self.remote(node, "install-inspector")}) + if before_start is not None: + before_start() self.start() def wait_ready(self): diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 23526975..d4c4a2ab 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 0207eca5..d374714f 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -20,6 +20,7 @@ done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization) ;; heal.node-outage-rejoin) driver=node_outage.py ;; + heal.status-partial-disabled-unknown) driver=status.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; diff --git a/scripts/validation_heal/status.py b/scripts/validation_heal/status.py new file mode 100644 index 00000000..45c7405c --- /dev/null +++ b/scripts/validation_heal/status.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""J01: real startup/disabled/missing-peer truth through raw API and rc.""" + +import argparse +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import subprocess +import sys +import tempfile +import threading +import time +from urllib import request +from urllib.error import URLError + +try: + from .heal_probe import Probe, NoRedirect, require, save, compare_inventories + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .shards import SIZES +except ImportError: + from heal_probe import Probe, NoRedirect, require, save, compare_inventories + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from shards import SIZES + + +CASE = "heal.status-partial-disabled-unknown" +BRANCHES = ("enabled-idle", "disabled", "uninitialized", "missing-peer", "recovered-complete") + + +class TimedProbe(Probe): + def __init__(self, *args, timeout=8, **kwargs): + super().__init__(*args, **kwargs) + self.timeout = timeout + + def _open_admin(self, outgoing): + return request.build_opener(request.ProxyHandler({}), NoRedirect()).open(outgoing, timeout=self.timeout) + + +def validate_background(value, branch, peers): + coverage = value.get("coverage") or {} + require(all(type(coverage.get(name)) is int and 0 <= coverage[name] <= peers + for name in ("expected", "responded", "unknown")) and coverage["expected"] == peers + and coverage["responded"] + coverage["unknown"] == peers, + "background status lacks balanced exact peer coverage") + state = value.get("state") + if branch == "missing-peer": + require(value.get("clusterStatusComplete") is False and coverage["responded"] == peers - 1 + and coverage["unknown"] == 1 and state in {"degraded", "active"} + and isinstance(coverage.get("reasons"), list) and coverage["reasons"], + "missing peer was hidden as complete/idle or lacks its reason") + else: + wanted = "disabled" if branch == "disabled" else "uninitialized" if branch == "uninitialized" else "idle" + require(state == wanted and value.get("clusterStatusComplete") is True and coverage["responded"] == peers + and coverage["unknown"] == 0 and coverage.get("reasons") == [], + "background runtime state or complete coverage differs from the actual fixture") + for name in ("healQueueLength", "healActiveTasks"): + require(type(value.get(name)) is int and value[name] == 0, "status fixture has unresolved Heal work") + operations = value.get("healOperations") or {} + require(all(type(operations.get(name)) is int and operations[name] == 0 + for name in ("queueLength", "activeTasks", "retryingTasks")), "status fixture still has queued, active or retrying responsibility") + + +def validate_replacement(value, branch): + cluster = value.get("cluster") or {} + require(value.get("contractVersion") == 2 and value.get("scope") == "cluster_survivor_disks" + and isinstance(cluster.get("records"), list) and not cluster["records"], + "fresh status fixture lacks replacement contract or has residual records") + uncertain = branch in {"missing-peer", "disabled", "uninitialized"} + require(cluster.get("definitive") is (not uncertain) + and (isinstance(cluster.get("reason"), str) and bool(cluster["reason"]) if uncertain else cluster.get("reason") is None), + "replacement status erased uncertainty or claimed a definitive cluster") + + +def validate_cli(json_result, text_result, background, branch, peers): + require(json_result.get("exitCode") == 0 and text_result.get("exitCode") == 0, + "rc status did not complete successfully") + value = json_result.get("json") + require(isinstance(value, dict), "rc omitted parseable JSON status") + allowed = {"degraded", "active", "unknown"} if branch == "missing-peer" else {background["state"]} + require(value.get("state") in allowed, "rc converted actual runtime state to a different completion claim") + if branch == "missing-peer": + require(value.get("clusterStatusComplete") is not True, "rc falsely completed a partial cluster") + # When rc exposes coverage, all counters must agree with the raw API. + if "coverage" in value: + require(value["coverage"] == background["coverage"], "rc peer coverage differs from raw Admin") + if "clusterStatusComplete" in value: + require(value["clusterStatusComplete"] is background["clusterStatusComplete"], "rc completeness differs from raw Admin") + text = text_result.get("stdout", "") + if branch in {"missing-peer", "disabled", "uninitialized"}: + require("Heal Status: Idle" not in text and not re.search(r"(?im)^\s*(?:status|heal status)\s*:\s*(?:completed|complete|idle)\s*$", text), + "rc text presented partial/disabled/uninitialized status as completed/idle") + require(bool(text.strip()), "rc omitted its human-readable status") + return {"rawAndRcRuntimeAgree": True, "expectedPeers": peers, "rawState": background["state"], "rcState": value["state"]} + + +class Cli: + def __init__(self, binary, binding, access, secret): + self.binary, self.binding = binary, binding + self.folder = tempfile.TemporaryDirectory(prefix="validation-heal-rc-") + root = Path(self.folder.name) + root.chmod(0o700) + text = "schema_version = 1\n" + for alias, endpoint in binding["endpoints"].items(): + text += "[[aliases]]\n" + f"name = {json.dumps(alias)}\nendpoint = {json.dumps(endpoint)}\n" + text += f"access_key = {json.dumps(access)}\nsecret_key = {json.dumps(secret)}\nbucket_lookup = \"path\"\n" + text += "[aliases.retry]\nmax_attempts = 0\ninitial_backoff_ms = 100\nmax_backoff_ms = 1000\n" + descriptor = os.open(root / "config.toml", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w") as stream: stream.write(text) + self.env = {**os.environ, "RC_CONFIG_DIR": str(root), "NO_PROXY": "*", "HTTP_PROXY": "", "HTTPS_PROXY": "", "ALL_PROXY": ""} + + def read(self, endpoint, *, as_json, timeout=10): + args = [str(self.binary), *(["--json"] if as_json else []), "--no-color", "admin", "heal", "status", endpoint] + try: + result = subprocess.run(args, env=self.env, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired as exc: + return {"exitCode": None, "timeout": True, "stdout": (exc.stdout or b"").decode(errors="replace") if isinstance(exc.stdout, bytes) else exc.stdout, + "stderr": "rc status deadline", "json": None} + try: value = json.loads(result.stdout) + except ValueError: value = None + return {"exitCode": result.returncode, "stdout": result.stdout, "stderr": result.stderr, "json": value} + + def close(self): + self.folder.cleanup() + require(not Path(self.folder.name).exists(), "private rc credential configuration remains") + + +def observations(fleet, binding, cli, endpoints, *, timeout=8): + observed = {} + for endpoint in endpoints: + probe = TimedProbe(binding, endpoint, timeout=timeout) + def raw(path, method="POST"): + try: return probe.admin(path, [], b"", method=method) + except Exception as exc: return {"httpStatus": None, "error": str(exc)} + observed[endpoint] = {"background": raw("/rustfs/admin/v3/background-heal/status"), + "replacement": raw("/rustfs/admin/v4/heal/replacement-recovery", "GET"), + "rcJson": cli.read(endpoint, as_json=True, timeout=timeout), + "rcText": cli.read(endpoint, as_json=False, timeout=timeout)} + return fleet.record("status-observations", observed) + + +def validate_observations(observed, branch, peers): + proofs = {} + for endpoint, value in observed.items(): + for name in ("background", "replacement"): + require(value[name].get("httpStatus") == 200 and isinstance(value[name].get("response"), dict), + "status observation lacks a successful raw Admin response") + background = value["background"]["response"] + validate_background(background, branch, peers) + validate_replacement(value["replacement"]["response"], branch) + proofs[endpoint] = validate_cli(value["rcJson"], value["rcText"], background, branch, peers) + require(bool(proofs), "status observation has no actual endpoint") + return proofs + + +def reject_unsafe_completion(observed): + for value in observed.values(): + background = value["background"].get("response") + if not isinstance(background, dict): continue + partial = background.get("clusterStatusComplete") is False + uncertain = partial or background.get("state") in {"disabled", "uninitialized", "degraded"} + require(not partial or background.get("state") != "idle", "partial cluster falsely reported idle during convergence") + if not uncertain: continue + cli = value["rcJson"].get("json") + if isinstance(cli, dict): + require(cli.get("state") not in {"idle", "complete", "completed"}, "rc falsely reported uncertain status as completed/idle") + require(not partial or cli.get("clusterStatusComplete") is not True, "rc falsely completed a partial cluster") + text = value["rcText"].get("stdout") or "" + require("Heal Status: Idle" not in text and not re.search(r"(?im)^\s*(?:status|heal status)\s*:\s*(?:completed|complete|idle)\s*$", text), + "rc text erased status uncertainty during convergence") + + +def stable_status(fleet, binding, cli, branch, endpoints): + deadline = time.monotonic() + 180 + while True: + observed = observations(fleet, binding, cli, endpoints) + reject_unsafe_completion(observed) + try: return validate_observations(observed, branch, len(binding["endpoints"])) + except ValueError: + if time.monotonic() >= deadline: raise + time.sleep(0.5) + + +def watch_startup(fleet, binding, cli, done): + deadline, result = time.monotonic() + 30, None + while not done.is_set() and time.monotonic() < deadline: + # Bracket CLI/replacement reads with the same raw uninitialized state; + # a transport failure or a sample from an already initialized service + # cannot serve as startup-window evidence. + observed = observations(fleet, binding, cli, ["node1"], timeout=0.7) + reject_unsafe_completion(observed) + try: + proof = validate_observations(observed, "uninitialized", len(binding["endpoints"])) + after = fleet.record("startup-after-cli", TimedProbe(binding, "node1", timeout=0.7).admin( + "/rustfs/admin/v3/background-heal/status", [], b"")) + require(after.get("httpStatus") == 200 and isinstance(after.get("response"), dict), "startup after-CLI raw status unavailable") + validate_background(after["response"], "uninitialized", len(binding["endpoints"])) + result = {"verdict": "PASS", "proof": proof, "rawBracketedCliWindow": True} + break + except Exception: + time.sleep(0.1) + return result or {"verdict": "NOT_OBSERVED", "reason": "full raw/rc uninitialized startup window was not witnessed; transport failure is not proof"} + + +def run_topology(args, topology, nodes, profile): + directory = args.evidence / topology / profile + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile=profile) + candidate = json.loads(args.candidate.read_text()) + if args.tool_directory: fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), "maxInventoryObjects": 512, + "maxInventoryBytes": 512 * 1024**2} + fleet.record("binding", binding) + result = {"topology": topology, "runtimeProfile": profile, "branches": {}, "verdict": "RUNNING", "cleanup": "PENDING"} + cli, pool, future, done = None, None, None, threading.Event() + try: + cli = Cli(args.rc_bin, binding, access, secret) + if profile == "manual": + pool = ThreadPoolExecutor(max_workers=1) + def before_start(): + nonlocal future + future = pool.submit(watch_startup, fleet, binding, cli, done) + fleet.deploy(args.binary_sha, bucket, access, secret, before_start=before_start) + done.set() + result["branches"]["uninitialized"] = future.result(timeout=10) + else: + fleet.deploy(args.binary_sha, bucket, access, secret) + physical = fleet.snapshot("deployed-runtime-and-topology") + fleet.record("topology-proof", validate_topology(physical, topology)) + recorder = Recorder(Probe(binding), directory / "fixture") + expected = seed(recorder, bucket, "DB", 128, SIZES) + baseline = fleet.record("s3-baseline", recorder.probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + before = fleet.snapshot("physical-baseline") + branch = "disabled" if profile == "disabled" else "enabled-idle" + result["branches"][branch] = {"verdict": "PASS", "proof": stable_status(fleet, binding, cli, branch, binding["endpoints"])} + if profile == "manual": + if topology == "H2": + node = nodes[-1] + stopped = fleet.record("missing-peer-stopped", fleet.remote(node, "stop")) + require(stopped.get("quiescent") is True and stopped.get("service", {}).get("MainPID") == "0", "peer stop lacks actual process exit") + try: + with request.build_opener(request.ProxyHandler({})).open(binding["endpoints"]["node4"] + "/health/ready", timeout=2): + raise ValueError("stopped peer still accepts requests") + except URLError as exc: + require(isinstance(exc.reason, ConnectionRefusedError), "peer outage is not a witnessed endpoint refusal") + fleet.record("missing-peer-refused", {"connectionRefused": True, "endpoint": "node4"}) + result["branches"]["missing-peer"] = {"verdict": "PASS", "proof": stable_status(fleet, binding, cli, "missing-peer", ["node1", "node2", "node3"])} + fleet.record("same-peer-restarted", fleet.remote(node, "start")) + fleet.wait_ready() + else: + result["branches"]["missing-peer"] = {"verdict": "NOT_APPLICABLE", "reason": "source J01 step 2 explicitly requires H2 for stopping one peer while querying survivors"} + fleet.stop() + fleet.start() + result["branches"]["recovered-complete"] = {"verdict": "PASS", "proof": stable_status(fleet, binding, cli, "recovered-complete", binding["endpoints"])} + fleet.stop() + require(physical_content(before) == physical_content(fleet.snapshot("physical-after-status", quiescent=True)), "status/process observations changed object files or format identities") + fleet.start() + after = fleet.record("s3-after-status", recorder.probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("status-full-fixture-proof", compare_inventories(baseline, after)) + wanted = {"disabled"} if profile == "disabled" else set(BRANCHES) - {"disabled"} + result["verdict"] = "PASS" if set(result["branches"]) == wanted and all( + value["verdict"] == "PASS" or branch == "missing-peer" and topology == "H1" and value["verdict"] == "NOT_APPLICABLE" + for branch, value in result["branches"].items()) else "FAIL" + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + finally: + done.set() + if pool: pool.shutdown(wait=True) + cli_clean = True + if cli: + try: cli.close() + except Exception as exc: + cli_clean = False + fleet.record("rc-config-cleanup-error", {"error": str(exc)}) + clean = fleet.cleanup() + result["cleanup"] = "PASS" if clean and cli_clean else "FAIL" + fleet.record("topology-result", result) + return result + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f"status workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=[CASE], required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path) + args = parser.parse_args() + require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") + require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") + require(select(CASE)[0]["variants"] == list(BRANCHES), "status executor omitted a source branch") + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(set(nodes)) == len(nodes) and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") + results = [] + try: + for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): + for profile in ("manual", "disabled"): + results.append(run_topology(args, topology, selected, profile)) + if results[-1]["cleanup"] != "PASS": break + if results[-1]["cleanup"] != "PASS": break + finally: + complete = len(results) == 4 and all(value["verdict"] == "PASS" and value["cleanup"] == "PASS" for value in results) + save(args.evidence / "healthy-summary.json", {"caseId": CASE, "results": results, "complete": complete, + "cleanup": "PASS" if results and all(value["cleanup"] == "PASS" for value in results) else "FAIL"}) + require(complete, "J01 requires every applicable raw/rc startup, disabled and peer-recovery branch") + + +if __name__ == "__main__": + try: main() + except Exception as exc: + print(f"Heal status validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/test_validation_heal.py b/test_validation_heal.py index 34a0f9cb..8622daf9 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -41,6 +41,8 @@ from scripts.validation_heal.markers import (INSTANCES as MARKER_INSTANCES, seed_markers, marker_plan, stale_expected_scene, add_marker, remove_marker, validate_marker_error) from scripts.validation_heal.drive_gate import probe_gate +from scripts.validation_heal.status import (validate_background, validate_replacement, validate_cli, + reject_unsafe_completion, watch_startup, stable_status, Cli) import errno import stat @@ -1410,6 +1412,110 @@ def test_permission_probe_checks_read_and_write_with_same_namespace_and_dac_limi self.assertEqual(invoke.call_args.args[0][-1], "/private/data/.rustfs.sys/format.json") +class HealRuntimeStatusTest(unittest.TestCase): + def background(self, *, state="idle", missing=False): + return {"state": state, "clusterStatusComplete": not missing, + "coverage": {"expected": 4, "responded": 3 if missing else 4, "unknown": int(missing), + "reasons": ["peer_status_unavailable"] if missing else []}, + "healQueueLength": 0, "healActiveTasks": 0, + "healOperations": {"queueLength": 0, "activeTasks": 0, "retryingTasks": 0}} + + def observation(self, *, state="uninitialized", missing=False): + background = self.background(state=state, missing=missing) + uncertain = missing or state in {"disabled", "uninitialized"} + return {"node1": {"background": {"httpStatus": 200, "response": background}, + "replacement": {"httpStatus": 200, "response": {"contractVersion": 2, "scope": "cluster_survivor_disks", + "cluster": {"records": [], "definitive": not uncertain, "reason": "runtime unavailable" if uncertain else None}}}, + "rcJson": {"exitCode": 0, "json": {"state": state, "clusterStatusComplete": not missing, "coverage": background["coverage"]}}, + "rcText": {"exitCode": 0, "stdout": "Heal Status: " + state.capitalize()}}} + + def test_raw_coverage_cannot_hide_peer_loss_retrying_work_or_disabled_state(self): + validate_background(self.background(), "enabled-idle", 4) + validate_background(self.background(state="degraded", missing=True), "missing-peer", 4) + for branch, state in (("disabled", "disabled"), ("uninitialized", "uninitialized")): + validate_background(self.background(state=state), branch, 4) + for mutation in ("bool", "lost-peer", "false-idle", "retry", "disabled-idle", "reason"): + missing = mutation in {"false-idle", "reason"} + value = self.background(state="degraded" if missing else "idle", missing=missing) + branch = "missing-peer" if missing else "disabled" if mutation == "disabled-idle" else "enabled-idle" + if mutation == "bool": value["coverage"]["responded"] = True + elif mutation == "lost-peer": value["coverage"]["unknown"] = 1 + elif mutation == "false-idle": value["state"] = "idle" + elif mutation == "retry": value["healOperations"]["retryingTasks"] = 1 + elif mutation == "reason": value["coverage"]["reasons"] = [] + with self.subTest(mutation=mutation), self.assertRaises(ValueError): validate_background(value, branch, 4) + + def test_replacement_does_not_convert_disabled_uninitialized_or_partial_to_definitive(self): + for branch, state in (("disabled", "disabled"), ("uninitialized", "uninitialized"), ("missing-peer", "degraded")): + value = self.observation(state=state, missing=branch == "missing-peer")["node1"]["replacement"]["response"] + validate_replacement(value, branch) + value["cluster"]["definitive"] = True + with self.assertRaises(ValueError): validate_replacement(value, branch) + + def test_rc_fields_and_text_must_preserve_raw_runtime_and_peer_truth(self): + for branch, state in (("disabled", "disabled"), ("uninitialized", "uninitialized"), ("missing-peer", "degraded")): + value = self.observation(state=state, missing=branch == "missing-peer")["node1"] + validate_cli(value["rcJson"], value["rcText"], value["background"]["response"], branch, 4) + changed = copy.deepcopy(value) + changed["rcJson"]["json"]["state"] = "idle" + with self.assertRaises(ValueError): validate_cli(changed["rcJson"], changed["rcText"], changed["background"]["response"], branch, 4) + changed = copy.deepcopy(value) + changed["rcJson"]["json"]["coverage"] = copy.deepcopy(changed["rcJson"]["json"]["coverage"]) + changed["rcJson"]["json"]["coverage"]["unknown"] += 1 + with self.assertRaises(ValueError): validate_cli(changed["rcJson"], changed["rcText"], changed["background"]["response"], branch, 4) + with self.assertRaises(ValueError): validate_cli(value["rcJson"], {"exitCode": 0, "stdout": "Heal Status: Idle"}, value["background"]["response"], branch, 4) + + def test_unsafe_convergence_sample_fails_immediately_and_is_not_retried_to_green(self): + observed = self.observation(state="degraded", missing=True) + observed["node1"]["rcText"]["stdout"] = "Heal Status: Idle" + with self.assertRaises(ValueError): reject_unsafe_completion(observed) + with patch("scripts.validation_heal.status.observations", return_value=observed) as reads, \ + patch("scripts.validation_heal.status.time.sleep") as sleep: + with self.assertRaises(ValueError): stable_status(None, {"endpoints": {"node1": "url"}}, None, "missing-peer", ["node1"]) + reads.assert_called_once() + sleep.assert_not_called() + + def test_startup_needs_raw_rc_window_bracket_transport_failure_is_not_observation(self): + good = self.observation() + fleet = types.SimpleNamespace(record=lambda label, value: value) + binding = {"endpoints": {"node1": "one", "node2": "two", "node3": "three", "node4": "four"}} + with patch("scripts.validation_heal.status.observations", return_value=good), \ + patch("scripts.validation_heal.status.TimedProbe", return_value=types.SimpleNamespace(admin=lambda *args: good["node1"]["background"])): + result = watch_startup(fleet, binding, None, types.SimpleNamespace(is_set=lambda: False)) + self.assertEqual(result["verdict"], "PASS") + self.assertTrue(result["rawBracketedCliWindow"]) + bad = copy.deepcopy(good) + bad["node1"]["background"] = {"httpStatus": None, "error": "connection refused"} + with patch("scripts.validation_heal.status.observations", return_value=bad), \ + patch("scripts.validation_heal.status.time.monotonic", side_effect=[0, 1, 31]), \ + patch("scripts.validation_heal.status.time.sleep"): + result = watch_startup(fleet, binding, None, types.SimpleNamespace(is_set=lambda: False)) + self.assertEqual(result["verdict"], "NOT_OBSERVED") + with patch("scripts.validation_heal.status.observations", return_value=good), \ + patch("scripts.validation_heal.status.TimedProbe", return_value=types.SimpleNamespace(admin=lambda *args: {"httpStatus": 200, "response": self.background()})), \ + patch("scripts.validation_heal.status.time.monotonic", side_effect=[0, 1, 31]), \ + patch("scripts.validation_heal.status.time.sleep"): + self.assertEqual(watch_startup(fleet, binding, None, types.SimpleNamespace(is_set=lambda: False))["verdict"], "NOT_OBSERVED") + + def test_cli_private_credentials_are_removed_and_never_enter_process_argv(self): + cli = Cli(Path("/usr/bin/rc"), {"endpoints": {"node1": "http://private:9000"}}, "runtime-access", "runtime-secret") + root = Path(cli.folder.name) + self.assertEqual(stat.S_IMODE((root / "config.toml").stat().st_mode), 0o600) + try: + with patch("scripts.validation_heal.status.subprocess.run", return_value=types.SimpleNamespace( + returncode=0, stdout='{"state":"disabled"}', stderr="")) as invoked: + result = cli.read("node1", as_json=True) + self.assertEqual(result["json"]["state"], "disabled") + self.assertNotIn("runtime-access", repr(invoked.call_args.args)) + self.assertNotIn("runtime-secret", repr(invoked.call_args.args)) + self.assertEqual(invoked.call_args.kwargs["env"]["RC_CONFIG_DIR"], str(root)) + finally: cli.close() + self.assertFalse(root.exists()) + config = runtime_config("disabled") + self.assertEqual(config["RUSTFS_HEAL_ENABLED"], "false") + self.assertEqual({**config, "RUSTFS_HEAL_ENABLED": "true"}, BACKGROUND_CONFIG) + + class HealPayloadMismatchTest(unittest.TestCase): def test_length_and_payload_donors_keep_original_metadata_and_exact_same_role(self): branches = VERSION_BRANCHES["heal.metadata-payload-mismatch"] From c99b8426586860d6f67a6e4f589679be702b4a4a Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 02:10:38 +0800 Subject: [PATCH 057/343] test(validation): migrate retained Heal cursor windows and task identity --- docs/validation-heal-operations.md | 4 +- docs/validation-migration-map.md | 27 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/cursor.py | 395 ++++++++++++++++++++++++++ scripts/validation_heal/fleet_node.py | 52 +++- scripts/validation_heal/healthy.py | 14 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 1 + scripts/validation_heal/shards.py | 11 +- test_validation_heal.py | 170 ++++++++++- 10 files changed, 652 insertions(+), 26 deletions(-) create mode 100644 scripts/validation_heal/cursor.py diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 8c9f558f..66842609 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -279,7 +279,7 @@ rc 必须是本批冻结二进制，私有 RC_CONFIG_DIR/alias 只指向本例 ```bash hp cursor-001 admin --path "$SCOPE_PATH" --query "clientToken=$TOKEN" --query sinceSeq=0 -NEXT=$(jq -er '.observation.response.nextSeq' "$HEAL_REPORT/cursor-001.json") +NEXT=$(jq -er '.observation.response.nextSeq - 1' "$HEAL_REPORT/cursor-001.json") hp cursor-002 admin --path "$SCOPE_PATH" --query "clientToken=$TOKEN" --query "sinceSeq=$NEXT" ``` @@ -287,6 +287,8 @@ WINDOW 的必要证据是 running 且 `0 ### 权限和过期签名 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index dccb309b..152f34a5 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 104 currently dispatchable IDs include all 74 non-Heal cases and thirty Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. +case evidence. The 105 currently dispatchable IDs include all 74 non-Heal cases and thirty-one Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 54 Heal cases remain +The other 53 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -525,3 +525,26 @@ full S3 manifests must remain unchanged. Temporary rc credentials use a 0600 configuration outside artifacts and are removed during finalization. Local coverage/uncertainty/startup/config-cleanup negatives passed; live evidence remains pending. + + +J03 `heal.incremental-cursor-and-truncation` now has a current-fleet DX +executor with 1025 pre-registered external objects of 1MiB+73 bytes. The +reference candidate retains 1024 result items, separately from its 128-object +outcome diagnostics; a 257-object task cannot prove the retention-gap branch. +The actual terminal minSeq must move and an old cursor must carry truncation. +Incremental queries treat sinceSeq as exclusive and advance only over returned +items, including when the response payload itself is cropped. Same-task bounds +cannot move backwards or duplicate object/version identities. At least two +useful running windows are mandatory; NOT_OBSERVED prevents a complete PASS. +H2 queries every actual node with the admitted path/token and uninterrupted +deployment epoch; H1 records why a cross-node query is inapplicable. The +candidate renders startTime anew on each response, so A03 also binds its +original query path/token instead of comparing that timestamp. Canonical +counters must cover all 1025 ACKs with the exact 17 real missing-shard repairs, +no failures/unknowns, and complete traversal. Bounded diagnostic receipts remain +explicitly incomplete. Full physical manifests, every original same-role shard +and streaming checksum, and all-peer S3 manifests independently cover every +object before PASS. The case has a fixed 12288-file snapshot budget, 128-part +verification batches, a 45-minute service/lease budget and 2GiB free per drive. +Local cursor/gap/counter/scope/part-omission negatives passed; live evidence +remains pending. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 961c8d6b..395601ea 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/cursor.py b/scripts/validation_heal/cursor.py new file mode 100644 index 00000000..f89861ad --- /dev/null +++ b/scripts/validation_heal/cursor.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python3 +"""J03: real retained-window gaps with independent, complete DX verification.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +from urllib.parse import quote + +try: + from .heal_probe import Probe, canonical, compare_inventories, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .heal_fault import bounded_fault_id + from .shards import files, map_target, validate_repaired_physical + from .versions import public_version_id +except ImportError: + from heal_probe import Probe, canonical, compare_inventories, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from heal_fault import bounded_fault_id + from shards import files, map_target, validate_repaired_physical + from versions import public_version_id + + +CASE_ID = "heal.incremental-cursor-and-truncation" +BRANCHES = ("within-window-cursor", "past-retention-gap", "cross-node-cursor", "terminal-truncation") +RETAINED_ITEMS = 1024 +OBJECTS = RETAINED_ITEMS + 1 +SIZE = 1024**2 + 73 +TASK_SECONDS = 600 +LIMIT_REFERENCE = {"candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", + "path": "crates/heal/src/heal/task.rs", + "sourceSha256": "81e8e2ec8807d1094095e444d0aaf8913f94114836bc1bc9f7953cbeff98d855", + "retainedItems": RETAINED_ITEMS, "diagnosticObjects": 128, + "contract": "sinceSeq is exclusive; nextSeq is the next sequence to allocate", + "differentCandidates": "the measured terminal window must still actually exceed retention; no inferred gap"} + + +def wanted_pairs(expected): + wanted = {(entry["key"], public_version_id(entry["versionId"])): entry for entry in expected} + require(len(wanted) == len(expected) == OBJECTS and all(not entry["deleteMarker"] + and pair[1] == "null" and entry["payload"]["size"] == SIZE for pair, entry in wanted.items()), + "cursor fixture must contain the pre-registered 1025 distinct external-size ACKs") + return wanted + + +class Window: + """One endpoint's cursors, within one admitted task and deployment epoch.""" + + def __init__(self, bucket, expected, token, endpoint, epoch): + self.bucket, self.wanted = bucket, wanted_pairs(expected) + require(isinstance(token, str) and token and isinstance(epoch, str) and epoch, + "cursor lacks an admitted task/deployment epoch") + self.token, self.endpoint, self.epoch = token, endpoint, epoch + self.cursor, self.next_seq, self.min_seq = 0, 1, 1 + self.seen, self.running_windows, self.gaps = set(), 0, [] + + def check(self, status, cursor): + require(type(cursor) is int and cursor >= 0, "invalid exclusive cursor") + next_seq, min_seq = status.get("nextSeq"), status.get("minSeq") + require(type(next_seq) is int and type(min_seq) is int and 1 <= min_seq <= next_seq <= OBJECTS + 1, + "cursor response omitted or corrupted its actual sequence bounds") + require(status.get("summary") in {"running", "finished"}, "cursor task failed, stopped or disappeared") + truncated = status.get("truncated", False) + require(type(truncated) is bool, "cursor truncation flag is not boolean") + gap = cursor + 1 < min_seq + require(not gap or truncated, "past-retention gap was silently omitted") + items = status.get("items", []) + require(isinstance(items, list) and len(items) <= RETAINED_ITEMS, "invalid retained item window") + first = max(cursor + 1, min_seq) + available = max(0, next_seq - first) + require(len(items) <= available and (len(items) == available or truncated), + "cursor items silently skip sequences or exceed their bounds") + pairs = [] + for item in items: + pair = item.get("object"), public_version_id(item.get("versionId")) + require(item.get("bucket") == self.bucket and pair in self.wanted + and item.get("objectSize") == self.wanted[pair]["payload"]["size"], + "retained item has the wrong bucket/key/version or acknowledged size") + pairs.append(pair) + require(len(pairs) == len(set(pairs)), "retained window duplicated an object/version") + return {"nextSeq": next_seq, "minSeq": min_seq, "retentionGap": gap, + "payloadTruncated": len(items) != available, "firstReturnedSeq": first, + "pairs": pairs, "returnedItems": len(items), "truncated": truncated} + + def consume(self, status, cursor, *, token, endpoint, epoch): + require((token, endpoint, epoch) == (self.token, self.endpoint, self.epoch), + "cursor crossed a task, endpoint or deployment epoch") + require(cursor == self.cursor, "cursor jumped over an unconsumed sequence") + observed = self.check(status, cursor) + require(observed["nextSeq"] >= self.next_seq and observed["minSeq"] >= self.min_seq, + "same-task cursor bounds moved backwards") + require(not self.seen.intersection(observed["pairs"]), "incremental cursor duplicated an earlier item") + self.seen.update(observed["pairs"]) + if observed["retentionGap"]: + self.gaps.append({"sinceSeq": cursor, "minSeq": observed["minSeq"], "truncated": True}) + # The API may trim its serialized payload separately from item retention. + # Advance over received items only, preserving any unreturned suffix. + if observed["returnedItems"]: + self.cursor = observed["firstReturnedSeq"] + observed["returnedItems"] - 1 + elif not observed["payloadTruncated"]: + self.cursor = max(self.cursor, observed["nextSeq"] - 1) + counters = (status.get("outcome") or {}).get("counters", {}) + processed = counters.get("processed") + if (status["summary"] == "running" and type(processed) is int and 0 < processed < OBJECTS + and observed["returnedItems"] and not observed["retentionGap"] and not observed["payloadTruncated"]): + self.running_windows += 1 + self.next_seq, self.min_seq = observed["nextSeq"], observed["minSeq"] + return {name: value for name, value in observed.items() if name != "pairs"} | { + "exclusiveNextCursor": self.cursor, "taskId": token, "endpoint": endpoint, "deploymentEpoch": epoch} + + +def terminal_proof(status, bucket, expected, repaired, window): + wanted = wanted_pairs(expected) + require(repaired and set(repaired) <= set(wanted), "cursor repair plan is empty or escapes ACKs") + outcome = status.get("outcome") or {} + require(status.get("summary") == "finished" and outcome.get("execution", {}).get("state") == "completed" + and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is True, + "terminal cursor task omitted its diagnostic truncation or complete coverage") + counters = outcome.get("counters", {}) + counts = {"processed": OBJECTS, "healed": len(repaired), "unchanged": OBJECTS - len(repaired), + "skipped": 0, "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False} + require(all(type(counters.get(name)) is type(value) and counters[name] == value for name, value in counts.items()), + "terminal canonical counters do not cover every frozen object and real missing shard") + receipts = outcome.get("objects") + require(isinstance(receipts, list) and 0 < len(receipts) <= 128 < OBJECTS, + "diagnostic receipts were incorrectly treated as a complete responsibility manifest") + seen, incarnation = set(), None + for receipt in receipts: + identity = receipt.get("identity", {}) + pair = identity.get("object"), public_version_id(identity.get("versionId")) + require(identity.get("bucket") == bucket and pair in wanted and pair not in seen + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0 + and isinstance(identity.get("bucketIncarnationId"), str) and identity["bucketIncarnationId"], + "bounded diagnostic receipt has a wrong, duplicate or incomplete identity") + incarnation = incarnation or identity["bucketIncarnationId"] + require(incarnation == identity["bucketIncarnationId"], "diagnostics mixed bucket incarnations") + require(receipt.get("disposition", {}).get("state") == ("repaired" if pair in repaired else "verified_healthy"), + "bounded diagnostics claimed false repair, unknown responsibility or incomplete integrity") + seen.add(pair) + retained = window.check(status, 0) + require(retained["retentionGap"] and retained["truncated"] and retained["nextSeq"] == OBJECTS + 1, + "terminal task never actually exceeded its retained item window") + return {"canonicalCounters": counters, "bucketIncarnationId": incarnation, + "diagnosticObjects": len(receipts), "diagnosticObjectsAreCompleteManifest": False, + "retainedWindow": {name: value for name, value in retained.items() if name != "pairs"}} + + +def map_cohorts(snapshot, bucket, expected): + """Index once, then use the existing complete-set oracle on each object.""" + wanted_pairs(expected) + by_key = {entry["key"]: {node: {"formats": value["formats"], "files": []} + for node, value in snapshot.items()} for entry in expected} + for node, observation in snapshot.items(): + for entry in observation["files"]: + require(entry["path"].startswith(bucket + "/"), "physical manifest contains an unowned bucket") + suffix = entry["path"][len(bucket) + 1:] + key = suffix.removesuffix("/xl.meta") if suffix.endswith("/xl.meta") else suffix.rsplit("/", 2)[0] + require(key in by_key, "physical baseline contains an unacknowledged object") + by_key[key][node]["files"].append(entry) + return [map_target(by_key[entry["key"]], bucket, entry["key"], "data" if index % 2 == 0 else "parity", 1) + for index, entry in enumerate(expected)] + + +def verify_all_parts(fleet, cohorts, label): + total = 0 + for node in fleet.nodes: + entries = [shard for target in cohorts for shard in target["cohort"] if shard["node"] == node] + for offset in range(0, len(entries), 128): + batch = entries[offset:offset + 128] + observed = fleet.record(label, {"node": node, **fleet.remote(node, "verify-parts", parts=[{ + name: entry[name] for name in ("disk", "relativePath", "versionId", "partNumber")} for entry in batch])}) + require(observed.get("complete") is True and len(observed.get("parts", [])) == len(batch), + "full external-part verifier omitted a batch item") + for entry, proof in zip(batch, observed["parts"], strict=True): + require(proof.get("physicalBitrotVerified") is True + and all(proof.get(name) == entry[name] for name in ("disk", "relativePath", "versionId", "partNumber", "sha256")) + and proof.get("physicalSize") == entry["size"], "external part differs from its frozen streaming/hash oracle") + total += len(batch) + require(total == OBJECTS * (4 if len(fleet.nodes) == 1 else 16), "full external-part verifier missed a disk/object") + return {"allExternalObjects": OBJECTS, "allPartsStreamingVerified": total, + "sameRoleOriginalShardBytesRestored": True} + + +def validate_missing_batch(before, after, targets, receipts): + left, right = files(before), files(after) + require(len(targets) == len(receipts) > 1, "cursor fault group omitted its exact receipts") + removed = set() + for target, receipt in zip(targets, receipts, strict=True): + identity = target["node"], target["disk"], target["relativePath"] + require(identity in left and identity not in right and identity not in removed + and receipt.get("relativePath") == target["relativePath"] and receipt.get("fault") is None + and receipt.get("before") == {"sha256": target["sha256"], "size": target["size"]}, + "cursor missing shard was not physically witnessed on its frozen drive") + removed.add(identity) + require({key: value for key, value in left.items() if key not in removed} == right + and all(before[node]["formats"] == after[node]["formats"] for node in before), + "cursor fault group changed metadata, a control shard or disk format") + return {"exactMissingParts": len(removed), "otherFilesAndFormatsUnchanged": True} + + +def observe_task(probe, fleet, binding, bucket, expected, targets, active): + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/objects/" + start = fleet.record("trigger", probe.admin(path, [], json.dumps({**BASE_OPTIONS, "recursive": True}).encode())) + require(start.get("httpStatus") == 200 and isinstance(start.get("response"), dict), "cursor task admission failed") + token = start["response"].get("clientToken") + require(isinstance(token, str) and token, "cursor task omitted its real token") + active.append((path, token)) + epoch = fleet.run_id + "/" + token + window = Window(bucket, expected, token, "node1", epoch) + fleet.record("cursor-identity", {"taskId": token, "path": path, "deploymentEpoch": epoch, + "processRestartedDuringTask": False, "responseStartTimeIsEpoch": False, + "reason": "candidate renders startTime on each response; use admission token and uninterrupted deployment"}) + deadline = time.monotonic() + TASK_SECONDS + while True: + cursor = window.cursor + response = fleet.record("cursor-control", probe.admin(path, [("clientToken", token), ("sinceSeq", str(cursor))], b"")) + require(response.get("httpStatus") == 200 and isinstance(response.get("response"), dict), "incremental status failed") + status = response["response"] + fleet.record("cursor-window-proof", window.consume(status, cursor, token=token, endpoint="node1", epoch=epoch)) + if status["summary"] == "finished" and window.cursor == window.next_seq - 1: + break + require(time.monotonic() < deadline, "incremental task/retained suffix exceeded its pre-registered deadline") + time.sleep(0.05) + active.remove((path, token)) + repaired = {(entry["key"], entry["versionId"]) for entry in targets} + proofs, old_status = {}, None + for endpoint in binding["endpoints"]: + client = Probe(binding, endpoint) + peer_window = Window(bucket, expected, token, endpoint, epoch) + observed = fleet.record("terminal-old-cursor", client.admin(path, [("clientToken", token), ("sinceSeq", "0")], b"")) + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), + "terminal old cursor failed on an actual bound endpoint") + current = observed["response"] + proofs[endpoint] = terminal_proof(current, bucket, expected, repaired, peer_window) + require(canonical(current.get("outcome")) == canonical(status.get("outcome")), + "terminal responsibility changed or crossed a task on another endpoint") + if old_status is not None: + require(all(canonical(current.get(name)) == canonical(old_status.get(name)) for name in + ("summary", "nextSeq", "minSeq", "items", "truncated", "outcome")), + "same-token terminal cursor window differs across nodes") + old_status = current + caught_up = fleet.record("terminal-caught-up", client.admin(path, + [("clientToken", token), ("sinceSeq", str(current["nextSeq"] - 1))], b"")) + require(caught_up.get("httpStatus") == 200 and isinstance(caught_up.get("response"), dict), "caught-up cursor failed") + empty = peer_window.check(caught_up["response"], current["nextSeq"] - 1) + require(empty["returnedItems"] == 0 and empty["nextSeq"] == current["nextSeq"] + and empty["minSeq"] == current["minSeq"] + and canonical(caught_up["response"].get("outcome")) == canonical(current.get("outcome")), + "caught-up terminal cursor repeated data or changed responsibility") + return {"within-window-cursor": {"verdict": "PASS" if window.running_windows >= 2 else "NOT_OBSERVED", + "nonemptyRunningIncrementalWindows": window.running_windows, "duplicateItems": 0, + "reason": "" if window.running_windows >= 2 else "two useful running windows were not observed; no timing success claim"}, + "past-retention-gap": {"verdict": "PASS", "oldCursor": 0, "endpointProofs": proofs, "incrementalGaps": window.gaps}, + "cross-node-cursor": {"verdict": "PASS" if len(binding["endpoints"]) > 1 else "NOT_APPLICABLE", + "queriedEndpoints": list(binding["endpoints"]), "sameTaskId": token, "sameDeploymentEpoch": epoch, + "reason": "H1 is one physical node; actual cross-node reads are required and performed on H2"}, + "terminal-truncation": {"verdict": "PASS", "endpointProofs": proofs, + "completeManifestEvidenceRequired": True}} + + +def run_topology(args, topology, nodes): + directory = args.evidence / topology + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, resource_profile="cursor") + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + candidate = json.loads(args.candidate.read_text()) + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat(), + "bucketPrefix": bucket, "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": OBJECTS, "maxInventoryBytes": OBJECTS * SIZE} + fleet.record("binding", binding) + fleet.record("pre-registered-budget", {"fixture": "DX", "objects": OBJECTS, "objectBytes": SIZE, + "logicalBytes": OBJECTS * SIZE, "taskSeconds": TASK_SECONDS, "sourceLimitReference": LIMIT_REFERENCE, + "maxSnapshotFilesPerNode": 12288, "snapshotSecondsPerNode": 600, "verificationBatchItems": 128, + "missingIndices": list(range(0, OBJECTS, 64)), "taskScope": "objects/ prefix; one result per ACK", + "faultLeaseMinutes": 30, "serviceRuntimeSeconds": 2700, "topologyLeaseMinutes": 45}) + require(select(args.case_id)[0]["variants"] == list(BRANCHES), "cursor executor omitted a source branch") + result = {"topology": topology, "fixture": "DX", "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} + active, probe = [], Probe(binding) + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) + expected = seed(Recorder(probe, directory / "fixture"), bucket, "DX", OBJECTS, [SIZE]) + wanted_pairs(expected) + baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=True, page_size=127)) + fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + fleet.stop() + original = fleet.snapshot("physical-baseline", quiescent=True) + cohorts = map_cohorts(original, bucket, expected) + fleet.record("all-external-baseline", verify_all_parts(fleet, cohorts, "baseline-part-batch")) + targets = [cohorts[index] for index in range(0, OBJECTS, 64)] + receipts = [] + for index, target in enumerate(targets): + params = {"disk": target["disk"], "relativePath": target["relativePath"], "kind": "missing", + "faultId": bounded_fault_id("J03", "retained-window", index), "expectedSha256": target["sha256"]} + fleet.faults.append((target["node"], params)) + receipts.append(fleet.record("fault", {"node": target["node"], **fleet.remote(target["node"], "apply-fault", **params)})) + faulted = fleet.snapshot("physical-fault", quiescent=True) + fleet.record("fault-proof", validate_missing_batch(original, faulted, targets, receipts)) + fleet.start() + result["branches"] = observe_task(probe, fleet, binding, bucket, expected, targets, active) + fleet.stop() + repaired = fleet.snapshot("physical-repaired-before-get", quiescent=True) + fleet.record("physical-repair-proof", validate_repaired_physical(original, repaired, targets[0], repair_targets=targets)) + fleet.record("all-external-repair-proof", verify_all_parts(fleet, cohorts, "repaired-part-batch")) + fleet.start() + for endpoint in binding["endpoints"]: + inventory = fleet.record("s3-complete-after", {"endpoint": endpoint, **Probe(binding, endpoint).inventory( + bucket, unversioned_current_tags=True, page_size=127)}) + fleet.record("independent-full-manifest-proof", {"endpoint": endpoint, + **verify_seed(expected, inventory), **compare_inventories(baseline, inventory)}) + fleet.stop() + require(physical_content(repaired) == physical_content(fleet.snapshot("physical-after-get", quiescent=True)), + "final GET changed physical files; prior repair attribution was incomplete") + result["branches"]["terminal-truncation"]["completePhysicalAndS3ManifestsVerified"] = True + result["verdict"] = "PASS" if all(value["verdict"] == "PASS" or name == "cross-node-cursor" + and topology == "H1" and value["verdict"] == "NOT_APPLICABLE" for name, value in result["branches"].items()) else "FAIL" + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + try: + fleet.stop() + fleet.snapshot("physical-failure", quiescent=True) + except Exception as evidence_error: + result["failureEvidenceError"] = str(evidence_error) + finally: + for path, token in active: + try: + fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: + fleet.record("stop-error", {"error": str(exc)}) + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("topology-result", result) + return result + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f"Heal workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path, required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(set(nodes)) == len(nodes) + and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") + require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") + require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") + results = [] + try: + for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): + results.append(run_topology(args, topology, selected)) + if results[-1]["cleanup"] != "PASS": + break + finally: + complete = len(results) == 2 and all(item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) + save(args.evidence / "healthy-summary.json", {"caseId": args.case_id, "results": results, + "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", "complete": complete}) + require(complete, "cursor case requires every applicable H1/H2 branch; an unobserved running window is not PASS") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"Heal cursor validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 87269342..18918b9e 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -108,13 +108,22 @@ def mounts(): return observed +def resource_limits(profile): + require(profile in {"standard", "cursor"}, "unsupported Heal resource profile") + return {"maxSnapshotFiles": 12288 if profile == "cursor" else 4096, + "minimumDriveFreeBytes": (2 if profile == "cursor" else 1) * 1024**3, + "runtimeMaxSeconds": 2700 if profile == "cursor" else 1800} + + def preflight(request): state, roots = paths(request["runId"]) require(not state.exists() and all(not root.exists() for root in roots), "run namespace already exists") observed = mounts() - require(all(shutil.disk_usage(mount).free >= 1024**3 for mount in MOUNTS), "1 GiB free per drive required") + limits = resource_limits(request.get("resourceProfile", "standard")) + require(all(shutil.disk_usage(mount).free >= limits["minimumDriveFreeBytes"] for mount in MOUNTS), + "insufficient free space for the explicit Heal resource profile") return {"hostname": socket.gethostname(), "mounts": observed, "statePath": str(state), - "dataRoots": [str(root / "data") for root in roots], "mutated": False} + "dataRoots": [str(root / "data") for root in roots], "resourceLimits": limits, "mutated": False} def reserve(request): @@ -143,7 +152,8 @@ def reserve(request): state, roots = paths(run_id) owner = {"runId": run_id, "unit": unit(run_id), "hostname": observed["hostname"], "mounts": observed["mounts"], "binarySha256": binary_sha, - "bucket": request["bucket"], "volumes": volumes, "runtimeProfile": profile} + "bucket": request["bucket"], "volumes": volumes, "runtimeProfile": profile, + "resourceProfile": request.get("resourceProfile", "standard")} if request.get("inspectorSha256"): require(bool(re.fullmatch(r"[0-9a-f]{64}", request["inspectorSha256"])), "invalid inspector digest") owner["inspectorSha256"] = request["inspectorSha256"] @@ -192,7 +202,8 @@ def start(request): require(digest(state / "rustfs") == owner["binarySha256"], "candidate changed before start") command("systemd-run", "--unit=" + owner["unit"], "--property=Type=exec", "--property=KillMode=control-group", "--property=TimeoutStopSec=30", - "--property=RuntimeMaxSec=1800", "--property=MemoryMax=4G", + "--property=RuntimeMaxSec=" + str(resource_limits(owner.get("resourceProfile", "standard"))["runtimeMaxSeconds"]), + "--property=MemoryMax=4G", "--property=WorkingDirectory=" + str(state), "--property=EnvironmentFile=" + str(state / "service.env"), *gate_module().start_properties(gate_module().checked_gate(state, roots)), str(state / "rustfs"), *owner["volumes"]) @@ -285,18 +296,23 @@ def verify_part(request): state, roots, owner = owned(request["runId"]) stopped(owner) require(owner.get("inspectorSha256"), "candidate-bound bitrot verifier is required") + tool = checked(state / "heal-meta-inspect", state) + require(digest(tool) == owner["inspectorSha256"], "bitrot verifier changed") + return verify_part_owned(request, roots, owner, tool) + + +def verify_part_owned(request, roots, owner, tool): module = fault_module() relative = request["relativePath"] parts = module.target_parts(relative, owner["bucket"]) require(parts[-1] != "xl.meta" and type(request.get("partNumber")) is int and parts[-1] == f"part.{request['partNumber']}", "part identity differs from request") - root, _ = fault_root(request) + require(type(request.get("disk")) is int and 0 <= request["disk"] < len(roots), "invalid physical disk index") + root = roots[request["disk"]] data = root / "data" part = checked(data / relative, data) metadata = checked(part.parent.parent / "xl.meta", data) key = metadata.parent.relative_to(data / owner["bucket"]).as_posix() - tool = checked(state / "heal-meta-inspect", state) - require(digest(tool) == owner["inspectorSha256"], "bitrot verifier changed") result = json.loads(command(str(tool), "--verify-part", str(metadata), owner["bucket"], key, str(part), request["versionId"], str(request["partNumber"])).stdout) require(result.get("physicalBitrotVerified") is True and result.get("key") == key @@ -305,6 +321,21 @@ def verify_part(request): return {**result, "sha256": digest(part), "relativePath": relative, "disk": request["disk"]} +def verify_parts(request): + state, roots, owner = owned(request["runId"]) + stopped(owner) + entries = request.get("parts") + require(isinstance(entries, list) and 1 <= len(entries) <= 128 + and all(isinstance(entry, dict) and set(entry) == {"disk", "relativePath", "versionId", "partNumber"} + for entry in entries), "invalid bounded part verification batch") + require(len({(entry["disk"], entry["relativePath"], entry["versionId"], entry["partNumber"]) + for entry in entries}) == len(entries), "duplicate part in verification batch") + require(owner.get("inspectorSha256"), "candidate-bound bitrot verifier is required") + tool = checked(state / "heal-meta-inspect", state) + require(digest(tool) == owner["inspectorSha256"], "bitrot verifier changed") + return {"complete": True, "parts": [verify_part_owned(entry, roots, owner, tool) for entry in entries]} + + def verify_inline(request): state, _, owner = owned(request["runId"]) root, _ = fault_root(request) @@ -361,6 +392,7 @@ def snapshot(request): require(all(environment.get(key) == value for key, value in expected_config.items()), "bound Heal runtime configuration differs") files, formats = [], [] + limits = resource_limits(owner.get("resourceProfile", "standard")) tool = state / "heal-meta-inspect" if owner.get("inspectorSha256"): require(digest(checked(tool, state)) == owner["inspectorSha256"], "metadata inspector changed") @@ -373,7 +405,8 @@ def snapshot(request): checked(Path(directory) / name, data) for name in sorted(names): path = Path(directory) / name - require(len(files) < 4096 and path.stat().st_size <= 64 * 1024**2, "fixture exceeds physical evidence budget") + require(len(files) < limits["maxSnapshotFiles"] and path.stat().st_size <= 64 * 1024**2, + "fixture exceeds physical evidence budget") entry = {"disk": index, "path": str(path.relative_to(data)), "size": path.stat().st_size, "sha256": digest(path)} if path.name == "xl.meta" and owner.get("inspectorSha256"): @@ -388,6 +421,7 @@ def snapshot(request): "backgroundConfig": None if request.get("quiescent") is True else expected_config, "storageClassEnvironment": None if request.get("quiescent") is True else { key: environment.get(key) for key in ("RUSTFS_STORAGE_CLASS_INLINE_BLOCK", "RUSTFS_STORAGE_CLASS_STANDARD")}, + "resourceLimits": limits, "files": sorted(files, key=lambda value: (value["disk"], value["path"])), "formats": formats} @@ -456,7 +490,7 @@ def main(request): require(os.geteuid() == 0, "node adapter requires passwordless sudo") actions = {"preflight": preflight, "reserve": reserve, "start": start, "stop": stop, "snapshot": snapshot, "logs": logs, "cleanup": cleanup, "audit-finalize": audit_finalize, "install-inspector": install_inspector, - "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part, + "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part, "verify-parts": verify_parts, "verify-inline": verify_inline, "drive-gate": drive_gate, "drive-gate-probe": drive_gate_probe} require(request.get("action") in actions, "unsupported node operation") print(json.dumps(actions[request["action"]](request))) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 852af123..7c3ff91d 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -130,11 +130,13 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: - def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual"): + def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): require(runtime_profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled"}, "unsupported Heal runtime profile") + require(resource_profile in {"standard", "cursor"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory self.runtime_profile = runtime_profile + self.resource_profile = resource_profile self.sequence = 0 self.reserved = [] self.faults = [] @@ -164,15 +166,16 @@ def ssh(self, node): def remote(self, node, action, **params): value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() - if action in {"apply-fault", "recover-fault", "verify-part", "verify-inline"}: + if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline"}: source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" if action in {"start", "cleanup", "audit-finalize", "drive-gate", "drive-gate-probe"}: source += "\nimport types\ngate = types.ModuleType('drive_gate')\nexec(" + repr( NODE_SOURCE.with_name("drive_gate.py").read_text()) + ", gate.__dict__)\n" source += "\nmain(" + repr(value) + ")\n" + timeout = 600 if self.resource_profile == "cursor" and action == "snapshot" else 180 result = subprocess.run([*self.ssh(node), "sudo -n python3 -"], - input=source, capture_output=True, text=True, timeout=180) + input=source, capture_output=True, text=True, timeout=timeout) require(result.returncode == 0, f"Heal node {node} {action} failed: {result.stderr[-1200:]}") return json.loads(result.stdout) @@ -193,7 +196,7 @@ def start(self): def deploy(self, binary_sha, bucket, access, secret, *, before_start=None): self.secret_values.update({access, secret}) - checked = {node: self.remote(node, "preflight") for node in self.nodes} + checked = {node: self.remote(node, "preflight", resourceProfile=self.resource_profile) for node in self.nodes} self.record("preflight", checked) require(len({value["hostname"] for value in checked.values()}) == len(self.nodes), "Heal node aliases must resolve to distinct actual hosts") @@ -214,7 +217,8 @@ def deploy(self, binary_sha, bucket, access, secret, *, before_start=None): if self.tool_directory else None value = self.remote(node, "reserve", hostname=checked[node]["hostname"], mounts=checked[node]["mounts"], binarySha256=binary_sha, bucket=bucket, volumes=volumes, access=access, secret=secret, - inspectorSha256=inspector_sha, runtimeProfile=self.runtime_profile) + inspectorSha256=inspector_sha, runtimeProfile=self.runtime_profile, + resourceProfile=self.resource_profile) self.record("reserved", {"node": node, **value}) if self.tool_directory: tool = self.tool_directory / "heal-meta-inspect" diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index d4c4a2ab..ea33fa73 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index d374714f..cb98fcc4 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -21,6 +21,7 @@ case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization) ;; heal.node-outage-rejoin) driver=node_outage.py ;; heal.status-partial-disabled-unknown) driver=status.py ;; + heal.incremental-cursor-and-truncation) driver=cursor.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index 67db57da..eee17e2e 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -279,9 +279,11 @@ def validate_dry_run_status(status, target, expected): "dry-run counters do not match the complete dry_run_observed responsibility set") -def peer_status_equal(coordinator, peer): +def peer_status_equal(coordinator, peer, *, task, peer_task): + # The candidate renders startTime at response serialization; it is not a + # persistent task epoch. Bind the actual signed queries to admission instead. return (coordinator.get("summary") == "finished" and peer.get("summary") == "finished" - and coordinator.get("startTime") is not None and peer.get("startTime") == coordinator["startTime"] + and isinstance(task, tuple) and len(task) == 2 and all(task) and task == peer_task and canonical(coordinator.get("outcome")) == canonical(peer.get("outcome")) and coordinator.get("outcome", {}).get("coverage") == "complete" and peer.get("truncated", False) is False) @@ -322,14 +324,15 @@ def run_task(probe, fleet, target, mode, active, *, status_endpoints=(), option_ observed = fleet.record("cross-node-control", remote_probe.admin(path, [("clientToken", token)], b"")) require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), "cross-node status is missing; 404/unknown cannot certify task completion") - if peer_status_equal(status, observed["response"]): + if peer_status_equal(status, observed["response"], task=(path, token), peer_task=(path, token)): break require(time.monotonic() < convergence_deadline, "peer responsibility did not converge within 30 seconds") time.sleep(0.2) if status_endpoints: fleet.record("cross-node-convergence", {"path": path, "clientToken": token, "coordinator": probe.endpoint_name, "queriedEndpoints": list(status_endpoints), "budgetSeconds": 30, - "taskStartTimeAndCompleteOutcomeEqual": True}) + "sameOriginalPathAndToken": True, "completeOutcomeEqual": True, + "responseStartTimeIsTaskIdentity": False}) return status diff --git a/test_validation_heal.py b/test_validation_heal.py index 8622daf9..fe215cfd 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -43,6 +43,9 @@ from scripts.validation_heal.drive_gate import probe_gate from scripts.validation_heal.status import (validate_background, validate_replacement, validate_cli, reject_unsafe_completion, watch_startup, stable_status, Cli) +from scripts.validation_heal.cursor import (Window, OBJECTS as CURSOR_OBJECTS, SIZE as CURSOR_SIZE, + BRANCHES as CURSOR_BRANCHES, terminal_proof as cursor_terminal, validate_missing_batch, verify_all_parts) +from scripts.validation_heal.fleet_node import resource_limits, verify_parts as node_verify_parts import errno import stat @@ -330,15 +333,19 @@ def test_executor_preserves_each_source_branch_and_uses_distinct_objects(self): def test_peer_status_cannot_converge_with_wrong_task_unknown_or_incomplete_receipts(self): coordinator = {**HealHealthyOracleTest().receipt("repaired"), "startTime": "exact-task-start"} - self.assertTrue(peer_status_equal(coordinator, copy.deepcopy(coordinator))) + task = ("/rustfs/admin/v3/heal/fixture/object", "original-token") + self.assertTrue(peer_status_equal(coordinator, copy.deepcopy(coordinator), task=task, peer_task=task)) + rerendered = {**copy.deepcopy(coordinator), "startTime": "fresh response timestamp"} + self.assertTrue(peer_status_equal(coordinator, rerendered, task=task, peer_task=task)) for invalid in ("task-start", "summary", "coverage", "outcome", "truncated"): peer = copy.deepcopy(coordinator) - if invalid == "task-start": peer["startTime"] = "different task" + peer_task = task + if invalid == "task-start": peer_task = (task[0], "different-task") elif invalid == "summary": peer["summary"] = "unknown" elif invalid == "coverage": peer["outcome"]["coverage"] = "partial" elif invalid == "outcome": peer["outcome"]["counters"]["unknown"] = 1 else: peer["truncated"] = True - with self.subTest(invalid=invalid): self.assertFalse(peer_status_equal(coordinator, peer)) + with self.subTest(invalid=invalid): self.assertFalse(peer_status_equal(coordinator, peer, task=task, peer_task=peer_task)) def test_target_role_comes_from_metadata_and_actual_format_slot(self): before = self.snapshot() @@ -1769,5 +1776,162 @@ def test_retained_or_unknown_protection_forbids_namespace_deletion(self): self.assertFalse(evidence["locked-finalization"][0]["namespaceDeletionAllowed"]) +class HealCursorOracleTest(unittest.TestCase): + bucket = "rv-heal-cursor-unit" + + def expected(self): + return [{"key": f"objects/{index:04d}.bin", "versionId": "null", "deleteMarker": False, + "payload": {"size": CURSOR_SIZE}} for index in range(CURSOR_OBJECTS)] + + def window(self, endpoint="node1"): + return Window(self.bucket, self.expected(), "task", endpoint, "deployment/task") + + def status(self, start=0, stop=3, *, minimum=1, truncated=False, finished=False): + items = [{"bucket": self.bucket, "object": f"objects/{index:04d}.bin", "versionId": "", + "objectSize": CURSOR_SIZE} for index in range(start, stop)] + return {"summary": "finished" if finished else "running", "nextSeq": stop + 1, "minSeq": minimum, + "items": items, "truncated": truncated, "outcome": {"counters": {"processed": stop}}} + + def consume(self, window, status, cursor): + return window.consume(status, cursor, token="task", endpoint=window.endpoint, epoch="deployment/task") + + def test_exclusive_next_sequence_preserves_the_first_item_of_each_poll(self): + window = self.window() + self.assertEqual(self.consume(window, self.status(), 0)["exclusiveNextCursor"], 3) + self.assertEqual(self.consume(window, self.status(3, 6), 3)["exclusiveNextCursor"], 6) + self.assertEqual(len(window.seen), 6) + self.assertEqual(window.running_windows, 2) + with self.assertRaisesRegex(ValueError, "jumped"): + self.consume(window, self.status(6, 9), 7) + + def test_truncated_payload_advances_only_over_transmitted_items(self): + window, status = self.window(), self.status(0, 8, truncated=True) + status["items"] = status["items"][:4] + self.assertEqual(self.consume(window, status, 0)["exclusiveNextCursor"], 4) + self.assertEqual(self.consume(window, self.status(4, 8, truncated=True), 4)["exclusiveNextCursor"], 8) + self.assertEqual(len(window.seen), 8) + status["truncated"] = False + with self.assertRaisesRegex(ValueError, "silently skip"): + self.window().check(status, 0) + + def test_retention_gap_requires_a_real_moved_minimum_and_flag(self): + status = self.status(1, CURSOR_OBJECTS, minimum=2, truncated=True, finished=True) + proof = self.window().check(status, 0) + self.assertTrue(proof["retentionGap"]) + self.assertEqual(proof["returnedItems"], 1024) + changed = copy.deepcopy(status) + changed["truncated"] = False + with self.assertRaisesRegex(ValueError, "silently omitted"): + self.window().check(changed, 0) + caught_up = self.status(CURSOR_OBJECTS, CURSOR_OBJECTS, minimum=2, truncated=True, finished=True) + self.assertEqual(self.window().check(caught_up, CURSOR_OBJECTS)["returnedItems"], 0) + self.assertFalse(self.window().check(caught_up, CURSOR_OBJECTS)["retentionGap"]) + + def test_duplicate_backward_wrong_scope_and_epoch_are_not_convergence(self): + for kind in ("duplicate", "backward", "bucket", "version", "size", "bounds", "flag", "terminal"): + with self.subTest(kind=kind): + window = self.window() + self.consume(window, self.status(), 0) + status = self.status(3, 6) + if kind == "duplicate": status["items"][0]["object"] = "objects/0000.bin" + if kind == "backward": status.update(nextSeq=3, items=[]) + if kind == "bucket": status["items"][0]["bucket"] = "unowned" + if kind == "version": status["items"][0]["versionId"] = "different-generation" + if kind == "size": status["items"][0]["objectSize"] += 1 + if kind == "bounds": status["nextSeq"] = True + if kind == "flag": status["truncated"] = "false" + if kind == "terminal": status["summary"] = "stopped" + with self.assertRaises(ValueError): self.consume(window, status, 3) + for name, value in (("token", "other-task"), ("endpoint", "node2"), ("epoch", "restarted/task")): + values = {"token": "task", "endpoint": "node1", "epoch": "deployment/task", name: value} + with self.assertRaisesRegex(ValueError, "crossed"): + self.window().consume(self.status(), 0, **values) + + def terminal(self): + status = self.status(1, CURSOR_OBJECTS, minimum=2, truncated=True, finished=True) + receipts = [{"identity": {"bucket": self.bucket, "object": f"objects/{index:04d}.bin", "versionId": None, + "poolIndex": 0, "setIndex": 0, "bucketIncarnationId": "incarnation"}, + "disposition": {"state": "repaired" if index == 1024 else "verified_healthy"}} for index in range(897, 1025)] + status["outcome"].update(execution={"state": "completed"}, coverage="complete", objectsTruncated=True, + objects=receipts, counters={"processed": CURSOR_OBJECTS, "healed": 1, "unchanged": CURSOR_OBJECTS - 1, + "skipped": 0, "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False}) + return status + + def test_terminal_truncation_cannot_hide_error_or_claim_complete_diagnostics(self): + repaired = {("objects/1024.bin", "null")} + proof = cursor_terminal(self.terminal(), self.bucket, self.expected(), repaired, self.window()) + self.assertFalse(proof["diagnosticObjectsAreCompleteManifest"]) + for kind in ("unknown", "failure", "attempt", "counter", "coverage", "truncation", "identity", "repair", "gap"): + status = self.terminal() + if kind == "unknown": status["outcome"]["counters"]["unknown"] = 1 + if kind == "failure": status["outcome"]["counters"]["failed"] = 1 + if kind == "attempt": status["outcome"]["counters"]["attemptFailures"] = 1 + if kind == "counter": status["outcome"]["counters"]["processed"] -= 1 + if kind == "coverage": status["outcome"]["coverage"] = "partial" + if kind == "truncation": status["outcome"]["objectsTruncated"] = False + if kind == "identity": status["outcome"]["objects"][0]["identity"]["bucketIncarnationId"] = "other" + if kind == "repair": status["outcome"]["objects"][-1]["disposition"]["state"] = "verified_healthy" + if kind == "gap": status["minSeq"] = 1 + with self.subTest(kind=kind), self.assertRaises(ValueError): + cursor_terminal(status, self.bucket, self.expected(), repaired, self.window()) + + def test_fault_group_checks_all_controls_and_all_exact_missing_parts(self): + before = HealShardOracleTest().snapshot() + target = map_target(before, "rv-heal-0123456789abcdef", "objects/0008.bin", "data", 1) + second = {**target, **next(shard for shard in target["cohort"] if shard["disk"] != target["disk"])} + targets = [target, second] + receipts = [{"relativePath": item["relativePath"], "fault": None, + "before": {"size": item["size"], "sha256": item["sha256"]}} for item in targets] + after = copy.deepcopy(before) + removed = {(item["disk"], item["relativePath"]) for item in targets} + after["node1"]["files"] = [entry for entry in after["node1"]["files"] if (entry["disk"], entry["path"]) not in removed] + self.assertEqual(validate_missing_batch(before, after, targets, receipts)["exactMissingParts"], 2) + after["node1"]["files"][0]["sha256"] = "changed-control" + with self.assertRaisesRegex(ValueError, "changed metadata"): + validate_missing_batch(before, after, targets, receipts) + + def test_batch_bitrot_verifier_requires_every_exact_part_and_no_duplicates(self): + with patch("scripts.validation_heal.fleet_node.owned", return_value=(Path("/state"), [], {"inspectorSha256": "sha"})), \ + patch("scripts.validation_heal.fleet_node.stopped"), patch("scripts.validation_heal.fleet_node.checked", side_effect=lambda path, boundary: path), \ + patch("scripts.validation_heal.fleet_node.digest", return_value="sha"), \ + patch("scripts.validation_heal.fleet_node.verify_part_owned", side_effect=lambda entry, *args: entry) as reads: + entry = {"disk": 0, "relativePath": "bucket/key/data/part.1", "versionId": "null", "partNumber": 1} + self.assertEqual(node_verify_parts({"runId": "run", "parts": [entry]})["parts"], [entry]) + self.assertEqual(reads.call_count, 1) + for batch in ([], [entry, entry], [entry] * 129, [{**entry, "extra": "unbound"}]): + with self.assertRaises(ValueError): node_verify_parts({"runId": "run", "parts": batch}) + + def test_cursor_resources_are_explicit_bounded_and_source_branches_preserved(self): + self.assertEqual(resource_limits("standard")["maxSnapshotFiles"], 4096) + self.assertEqual(resource_limits("cursor")["maxSnapshotFiles"], 12288) + self.assertEqual(resource_limits("cursor")["minimumDriveFreeBytes"], 2 * 1024**3) + self.assertEqual(list(CURSOR_BRANCHES), select("J03")[0]["variants"]) + with self.assertRaises(ValueError): resource_limits("unbounded") + + def test_complete_part_manifest_rejects_omitted_or_wrong_original_bytes(self): + cohorts = [{"cohort": [{"node": "node1", "disk": disk, "relativePath": f"bucket/{index}/part.1", + "versionId": "null", "partNumber": 1, "sha256": f"sha-{index}-{disk}", "size": 100} + for disk in range(4)]} for index in range(CURSOR_OBJECTS)] + authority = {(item["disk"], item["relativePath"]): item for cohort in cohorts for item in cohort["cohort"]} + def remote(node, action, *, parts): + self.assertEqual(action, "verify-parts") + self.assertLessEqual(len(parts), 128) + return {"complete": True, "parts": [{**part, "sha256": authority[part["disk"], part["relativePath"]]["sha256"], + "physicalBitrotVerified": True, "physicalSize": 100} for part in parts]} + fleet = types.SimpleNamespace(nodes=["node1"], remote=remote, record=lambda label, value: value) + self.assertEqual(verify_all_parts(fleet, cohorts, "proof")["allPartsStreamingVerified"], CURSOR_OBJECTS * 4) + good = fleet.remote + for kind in ("omitted", "wrong-sha", "checksum", "size"): + def invalid(node, action, *, parts): + result = good(node, action, parts=parts) + if kind == "omitted": result["parts"].pop() + if kind == "wrong-sha": result["parts"][0]["sha256"] = "donor" + if kind == "checksum": result["parts"][0]["physicalBitrotVerified"] = False + if kind == "size": result["parts"][0]["physicalSize"] += 1 + return result + fleet.remote = invalid + with self.subTest(kind=kind), self.assertRaises(ValueError): verify_all_parts(fleet, cohorts, "proof") + + if __name__ == "__main__": unittest.main() From 480f2281f98c90a9028e91d2e2d27ff80e495894 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 02:19:48 +0800 Subject: [PATCH 058/343] test(validation): migrate Heal cancellation and stale token isolation --- docs/validation-migration-map.md | 22 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/cursor.py | 24 +-- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 1 + scripts/validation_heal/stop.py | 262 ++++++++++++++++++++++++++++ test_validation_heal.py | 108 ++++++++++++ 7 files changed, 407 insertions(+), 14 deletions(-) create mode 100644 scripts/validation_heal/stop.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 152f34a5..ec9511c8 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 105 currently dispatchable IDs include all 74 non-Heal cases and thirty-one Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. +case evidence. The 106 currently dispatchable IDs include all 74 non-Heal cases and thirty-two Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 53 Heal cases remain +The other 52 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -548,3 +548,21 @@ object before PASS. The case has a fixed 12288-file snapshot budget, 128-part verification batches, a 45-minute service/lease budget and 2GiB free per drive. Local cursor/gap/counter/scope/part-omission negatives passed; live evidence remains pending. + + +A06 `heal.admin-stop-and-stale-token` now uses the same pre-registered 1025 +external-object DX fixture on H1/H2. Cancellation requires actual running, +nonzero partial canonical progress and both repaired and still-missing targets. +The frozen candidate policy requires active stop acknowledgement, retained +aborted/cancelled partial responsibility, and exact 500/InternalError with the +retired token for repeat/stale stops. Three complete physical snapshots with +five-second gaps keep the service running after cancellation; stopping the +process cannot substitute for task write quiescence. Actual restored shard +counts must equal the retained cancellation counters, and remaining shards +must still be missing. A distinct new token must be running before and after +old-token STATUS/STOP; unobserved brackets prevent PASS. The new task repairs +only the remaining defects, and its completion must preserve the original +cancelled outcome. Full streaming/physical and all-peer S3 verification happens +before backup rescue. All four source branches remain explicit. Local +cancellation/counter/quiet-physical/token/error negatives passed; full live +proof remains pending. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 395601ea..e1ac4474 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/cursor.py b/scripts/validation_heal/cursor.py index f89861ad..07839747 100644 --- a/scripts/validation_heal/cursor.py +++ b/scripts/validation_heal/cursor.py @@ -214,7 +214,7 @@ def validate_missing_batch(before, after, targets, receipts): return {"exactMissingParts": len(removed), "otherFilesAndFormatsUnchanged": True} -def observe_task(probe, fleet, binding, bucket, expected, targets, active): +def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, original=None): path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/objects/" start = fleet.record("trigger", probe.admin(path, [], json.dumps({**BASE_OPTIONS, "recursive": True}).encode())) require(start.get("httpStatus") == 200 and isinstance(start.get("response"), dict), "cursor task admission failed") @@ -274,7 +274,7 @@ def observe_task(probe, fleet, binding, bucket, expected, targets, active): "completeManifestEvidenceRequired": True}} -def run_topology(args, topology, nodes): +def run_topology(args, topology, nodes, *, observer=observe_task, branches=BRANCHES): directory = args.evidence / topology directory.mkdir() run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" @@ -297,7 +297,8 @@ def run_topology(args, topology, nodes): "maxSnapshotFilesPerNode": 12288, "snapshotSecondsPerNode": 600, "verificationBatchItems": 128, "missingIndices": list(range(0, OBJECTS, 64)), "taskScope": "objects/ prefix; one result per ACK", "faultLeaseMinutes": 30, "serviceRuntimeSeconds": 2700, "topologyLeaseMinutes": 45}) - require(select(args.case_id)[0]["variants"] == list(BRANCHES), "cursor executor omitted a source branch") + recipe = select(args.case_id)[0] + require(recipe["variants"] == list(branches), "DX executor omitted a source branch") result = {"topology": topology, "fixture": "DX", "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} active, probe = [], Probe(binding) try: @@ -315,13 +316,14 @@ def run_topology(args, topology, nodes): receipts = [] for index, target in enumerate(targets): params = {"disk": target["disk"], "relativePath": target["relativePath"], "kind": "missing", - "faultId": bounded_fault_id("J03", "retained-window", index), "expectedSha256": target["sha256"]} + "faultId": bounded_fault_id(recipe["code"], "dx-window", index), "expectedSha256": target["sha256"]} fleet.faults.append((target["node"], params)) receipts.append(fleet.record("fault", {"node": target["node"], **fleet.remote(target["node"], "apply-fault", **params)})) faulted = fleet.snapshot("physical-fault", quiescent=True) fleet.record("fault-proof", validate_missing_batch(original, faulted, targets, receipts)) fleet.start() - result["branches"] = observe_task(probe, fleet, binding, bucket, expected, targets, active) + result["branches"] = observer(probe, fleet, binding, bucket, expected, targets, active, original=original) + require(set(result["branches"]) == set(branches), "DX observer omitted or invented a source branch") fleet.stop() repaired = fleet.snapshot("physical-repaired-before-get", quiescent=True) fleet.record("physical-repair-proof", validate_repaired_physical(original, repaired, targets[0], repair_targets=targets)) @@ -335,7 +337,9 @@ def run_topology(args, topology, nodes): fleet.stop() require(physical_content(repaired) == physical_content(fleet.snapshot("physical-after-get", quiescent=True)), "final GET changed physical files; prior repair attribution was incomplete") - result["branches"]["terminal-truncation"]["completePhysicalAndS3ManifestsVerified"] = True + result["completePhysicalAndS3ManifestsVerified"] = True + if "terminal-truncation" in result["branches"]: + result["branches"]["terminal-truncation"]["completePhysicalAndS3ManifestsVerified"] = True result["verdict"] = "PASS" if all(value["verdict"] == "PASS" or name == "cross-node-cursor" and topology == "H1" and value["verdict"] == "NOT_APPLICABLE" for name, value in result["branches"].items()) else "FAIL" except (Exception, KeyboardInterrupt) as exc: @@ -356,13 +360,13 @@ def run_topology(args, topology, nodes): return result -def main(): +def main(*, case_id=CASE_ID, observer=observe_task, branches=BRANCHES): def interrupted(signum, frame): raise KeyboardInterrupt(f"Heal workflow interrupted by signal {signum}") signal.signal(signal.SIGTERM, interrupted) parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--candidate", type=Path, required=True) - parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--case-id", choices=(case_id,), required=True) parser.add_argument("--testing-commit", required=True) parser.add_argument("--binary-sha", required=True) parser.add_argument("--rc-bin", type=Path, required=True) @@ -377,14 +381,14 @@ def interrupted(signum, frame): results = [] try: for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): - results.append(run_topology(args, topology, selected)) + results.append(run_topology(args, topology, selected, observer=observer, branches=branches)) if results[-1]["cleanup"] != "PASS": break finally: complete = len(results) == 2 and all(item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) save(args.evidence / "healthy-summary.json", {"caseId": args.case_id, "results": results, "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", "complete": complete}) - require(complete, "cursor case requires every applicable H1/H2 branch; an unobserved running window is not PASS") + require(complete, f"{args.case_id} requires every applicable H1/H2 branch; an unobserved running window is not PASS") if __name__ == "__main__": diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index ea33fa73..73af9642 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index cb98fcc4..1a35ad78 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -22,6 +22,7 @@ case "${case_id}" in heal.node-outage-rejoin) driver=node_outage.py ;; heal.status-partial-disabled-unknown) driver=status.py ;; heal.incremental-cursor-and-truncation) driver=cursor.py ;; + heal.admin-stop-and-stale-token) driver=stop.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; diff --git a/scripts/validation_heal/stop.py b/scripts/validation_heal/stop.py new file mode 100644 index 00000000..4560a16c --- /dev/null +++ b/scripts/validation_heal/stop.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""A06: cancellation, actual quiet writes and stale-token isolation on DX.""" + +from __future__ import annotations + +import copy +import json +import sys +import time +from urllib.parse import quote +import xml.etree.ElementTree as ET + +try: + from .heal_probe import Probe, canonical, require + from .healthy import BASE_OPTIONS, physical_content + from .cursor import OBJECTS, TASK_SECONDS, Window, wanted_pairs, terminal_proof, main as dx_main + from .shards import files, validate_repaired_physical + from .versions import public_version_id + from .status import validate_background +except ImportError: + from heal_probe import Probe, canonical, require + from healthy import BASE_OPTIONS, physical_content + from cursor import OBJECTS, TASK_SECONDS, Window, wanted_pairs, terminal_proof, main as dx_main + from shards import files, validate_repaired_physical + from versions import public_version_id + from status import validate_background + + +CASE_ID = "heal.admin-stop-and-stale-token" +BRANCHES = ("stop-active", "repeat-stop", "old-status", "old-stop-new-task") +QUIET_SECONDS = 5 +POLICY = {"referenceCandidate": "072b57b37559fbe626aeda8abf61617fdcbdd926", + "cancelActive": {"httpStatus": 200, "summary": "stopped"}, + "repeatOrRetiredTokenStop": {"httpStatus": 500, "code": "InternalError", "messagePrefix": "Heal task not found: "}, + "oldStatus": "retained aborted/cancelled outcome, partial coverage; same path/token", + "sources": ["crates/heal/src/heal/manager.rs:cancel_task", "crates/heal/src/heal/channel.rs:process_cancel_request", + "rustfs/src/admin/handlers/heal.rs:force_stop handler"], + "unsupportedCandidates": "fail the frozen HTTP/body contract; do not accept arbitrary alternatives"} + + +def status(probe, fleet, path, token, label): + observed = fleet.record(label, probe.admin(path, [("clientToken", token)], b"")) + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), "token status failed") + return observed["response"] + + +def start(probe, fleet, path, active): + observed = fleet.record("trigger", probe.admin(path, [], json.dumps({**BASE_OPTIONS, "recursive": True}).encode())) + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), "DX task admission failed") + token = observed["response"].get("clientToken") + require(isinstance(token, str) and token, "DX task omitted its admitted token") + active.append((path, token)) + return token + + +def partial_window(status, targets, *, repair_required): + counters = (status.get("outcome") or {}).get("counters", {}) + processed, healed = counters.get("processed"), counters.get("healed") + for name in ("processed", "healed", "unchanged", "skipped", "failed", "unknown", "attemptFailures"): + require(type(counters.get(name)) is int and counters[name] >= 0, "active window omitted typed canonical counters") + require(all(counters[name] == 0 for name in ("failed", "unknown", "attemptFailures")) + and counters.get("overflowed") is False, "active window contains a real failure or unknown responsibility") + return (status.get("summary") == "running" and status.get("outcome", {}).get("execution", {}).get("state") == "running" + and 0 < processed < OBJECTS and (not repair_required or 0 < healed < len(targets))) + + +def wait_partial(probe, fleet, path, token, targets, *, repair_required): + deadline = time.monotonic() + TASK_SECONDS + while True: + current = status(probe, fleet, path, token, "running-window") + if partial_window(current, targets, repair_required=repair_required): + return current + require(current.get("summary") == "running", "NOT_OBSERVED: task finished before the required active window") + require(time.monotonic() < deadline, "NOT_OBSERVED: active window exceeded the frozen budget") + time.sleep(0.05) + + +def validate_retired_stop(observed, token): + require(observed.get("httpStatus") == 500 and not observed.get("transportError"), + "retired token stop did not follow the frozen HTTP contract") + body = observed.get("response") + if isinstance(body, str): + fields = {element.tag.rsplit("}", 1)[-1]: element.text for element in ET.fromstring(body).iter()} + else: + fields = body if isinstance(body, dict) else {} + require((fields.get("Code") or fields.get("code")) == "InternalError" + and (fields.get("Message") or fields.get("message")) == "Heal task not found: " + token, + "retired-token response has the wrong exact structured error/task identity") + return {"httpStatus": 500, "code": "InternalError", "exactRetiredTaskId": token} + + +def validate_cancelled(current, bucket, expected, targets): + wanted = wanted_pairs(expected) + outcome = current.get("outcome") or {} + require(current.get("summary") == "stopped" and outcome.get("execution") == {"state": "aborted", "reason": "cancelled"} + and outcome.get("coverage") == "partial", "cancelled responsibility was incorrectly reported as complete/repaired") + counters = outcome.get("counters", {}) + for name in ("processed", "healed", "unchanged", "skipped", "failed", "unknown", "attemptFailures"): + require(type(counters.get(name)) is int and counters[name] >= 0, "cancelled task omitted typed counters") + require(0 < counters["processed"] < OBJECTS and 0 < counters["healed"] < len(targets) + and counters["processed"] == sum(counters[name] for name in ("healed", "unchanged", "skipped", "failed")) + and all(counters[name] == 0 for name in ("failed", "unknown", "attemptFailures")) + and counters.get("overflowed") is False, "cancelled counters hide partial coverage, errors or unresolved responsibility") + objects, seen, incarnation, dispositions = outcome.get("objects"), set(), None, [] + require(isinstance(objects, list) and 0 < len(objects) <= 128, "cancelled task omitted bounded diagnostic receipts") + repaired = {(entry["key"], entry["versionId"]) for entry in targets} + for receipt in objects: + identity = receipt.get("identity", {}) + pair = identity.get("object"), public_version_id(identity.get("versionId")) + require(pair in wanted and pair not in seen and identity.get("bucket") == bucket + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0 + and isinstance(identity.get("bucketIncarnationId"), str) and identity["bucketIncarnationId"], + "cancelled diagnostic receipt lost its exact responsibility identity") + incarnation = incarnation or identity["bucketIncarnationId"] + require(incarnation == identity["bucketIncarnationId"], "cancelled diagnostics mixed bucket incarnations") + state = receipt.get("disposition", {}).get("state") + require(state in {"repaired", "verified_healthy", "cancelled"} + and (state != "repaired" or pair in repaired) + and (state != "verified_healthy" or pair not in repaired), + "cancelled/unknown/unrepaired object falsely counted as repaired or healthy") + seen.add(pair) + dispositions.append({"key": pair[0], "versionId": pair[1], "state": state}) + return {"canonicalCounters": counters, "coverage": "partial", "bucketIncarnationId": incarnation, + "unstartedObjectsAreNotRepaired": True, "diagnosticObjectsAreCompleteManifest": False, + "diagnosticDispositions": dispositions} + + +def cancelled_physical(original, quiet, targets, cancelled): + actual = files(quiet) + restored, remaining = [], [] + for target in targets: + entry = actual.get((target["node"], target["disk"], target["relativePath"])) + if entry is None: + remaining.append(target) + else: + require(entry["sha256"] == target["sha256"] and entry["size"] == target["size"], + "cancelled task committed incorrect shard bytes") + restored.append(target) + require(restored and remaining and len(restored) == cancelled["canonicalCounters"]["healed"], + "cancelled physical repair count disagrees with its retained outcome") + restored_pairs = {(entry["key"], entry["versionId"]) for entry in restored} + require(isinstance(cancelled.get("diagnosticDispositions"), list) and cancelled["diagnosticDispositions"], + "cancelled physical proof omitted its witnessed diagnostic identities") + for receipt in cancelled["diagnosticDispositions"]: + pair = receipt["key"], receipt["versionId"] + require((receipt["state"] == "repaired") == (pair in restored_pairs), + "cancelled diagnostic disposition disagrees with the actual restored object") + expected = copy.deepcopy(original) + missing = {(entry["node"], entry["disk"], entry["relativePath"]) for entry in remaining} + for node, observation in expected.items(): + observation["files"] = [entry for entry in observation["files"] if (node, entry["disk"], entry["path"]) not in missing] + proof = validate_repaired_physical(expected, quiet, restored[0], repair_targets=restored) + return remaining, {**proof, "actuallyRestoredParts": len(restored), "stillMissingParts": len(remaining), + "backupRescueUsed": False, "cancelledCountersMatchPhysical": True} + + +def idle_fleet(fleet, binding): + deadline = time.monotonic() + 30 + while True: + proofs = {} + for endpoint in binding["endpoints"]: + observed = fleet.record("task-exit-background", {"endpoint": endpoint, **Probe(binding, endpoint).admin( + "/rustfs/admin/v3/background-heal/status", [], b"")}) + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), + "task exit has no complete background status") + value = observed["response"] + require(value.get("state") != "idle" or value.get("clusterStatusComplete") is True + and value.get("coverage", {}).get("unknown") == 0 + and value.get("coverage", {}).get("responded") == len(binding["endpoints"]), + "task exit falsely reports a partial fleet as idle") + try: + validate_background(value, "idle", len(binding["endpoints"])) + proofs[endpoint] = {"state": "idle", "clusterStatusComplete": True, + "coverage": value["coverage"], "healOperations": value["healOperations"]} + except ValueError: + pass + if len(proofs) == len(binding["endpoints"]): + return proofs + require(time.monotonic() < deadline, "task did not drain queued/active/retrying responsibility on every peer") + time.sleep(0.1) + + +def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, original): + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/objects/" + fleet.record("frozen-cancel-policy", POLICY) + old = start(probe, fleet, path, active) + first = wait_partial(probe, fleet, path, old, targets, repair_required=True) + stop_ack = fleet.record("stop-active", probe.admin(path, [("clientToken", old), ("forceStop", "true")], b"")) + require(stop_ack.get("httpStatus") == 200 and stop_ack.get("response", {}).get("summary") == "stopped", + "active cancellation did not follow the frozen acknowledgement contract") + cancelled = status(probe, fleet, path, old, "cancelled-outcome") + cancellation = validate_cancelled(cancelled, bucket, expected, targets) + # Keep the actual service alive. An ACK or stopping its process would mask + # writes that escaped task cancellation. + quiet = [] + exit_proofs = [] + for index in range(3): + exit_proofs.append(idle_fleet(fleet, binding)) + started = time.monotonic_ns() + scene = fleet.snapshot("cancelled-live-quiet") + quiet.append(scene) + fleet.record("quiet-sample-time", {"sample": index, "startedMonotonicNs": started, + "finishedMonotonicNs": time.monotonic_ns(), "serviceKeptRunning": True, "minimumBetweenSamplesSeconds": QUIET_SECONDS}) + unchanged = status(probe, fleet, path, old, "quiet-old-status") + validate_cancelled(unchanged, bucket, expected, targets) + require(canonical(unchanged["outcome"]) == canonical(cancelled["outcome"]), "retired responsibility changed during write quiet window") + if index < 2: + time.sleep(QUIET_SECONDS) + require(all(physical_content(scene) == physical_content(quiet[0]) for scene in quiet[1:]), + "cancelled task continued physical writes after its terminal acknowledgement") + require(all(all((scene[node]["pid"], scene[node]["unit"]) == (quiet[0][node]["pid"], quiet[0][node]["unit"]) + for node in fleet.nodes) for scene in quiet[1:]), + "quiet window replaced the original live service process") + remaining, physical = cancelled_physical(original, quiet[-1], targets, cancellation) + active.remove((path, old)) + repeated = fleet.record("repeat-stop", probe.admin(path, [("clientToken", old), ("forceStop", "true")], b"")) + repeated_proof = validate_retired_stop(repeated, old) + new = start(probe, fleet, path, active) + require(new != old, "new task reused a cancelled token") + before = wait_partial(probe, fleet, path, new, remaining, repair_required=False) + old_status = status(probe, fleet, path, old, "old-status-during-new") + validate_cancelled(old_status, bucket, expected, targets) + require(canonical(old_status["outcome"]) == canonical(cancelled["outcome"]), "new task overwrote old partial responsibility") + stale = fleet.record("old-stop-during-new", probe.admin(path, [("clientToken", old), ("forceStop", "true")], b"")) + stale_proof = validate_retired_stop(stale, old) + after = status(probe, fleet, path, new, "new-after-stale-stop") + require(after.get("summary") != "stopped", "old token actually stopped the new task") + require(partial_window(after, remaining, repair_required=False), + "NOT_OBSERVED: stale-stop request was not bracketed by the same active new task") + after_stale_counters = dict(after["outcome"]["counters"]) + deadline = time.monotonic() + TASK_SECONDS + while after.get("summary") == "running": + require(time.monotonic() < deadline, "new task exceeded the frozen completion budget") + time.sleep(0.05) + after = status(probe, fleet, path, new, "new-task-control") + active.remove((path, new)) + response = fleet.record("new-terminal-old-cursor", probe.admin(path, [("clientToken", new), ("sinceSeq", "0")], b"")) + require(response.get("httpStatus") == 200 and isinstance(response.get("response"), dict), "new task terminal cursor failed") + terminal = terminal_proof(response["response"], bucket, expected, {(entry["key"], entry["versionId"]) for entry in remaining}, + Window(bucket, expected, new, "node1", fleet.run_id + "/" + new)) + require(canonical(response["response"].get("outcome")) == canonical(after.get("outcome")), + "new task terminal responsibility changed") + old_final = status(probe, fleet, path, old, "old-final-status") + require(canonical(old_final.get("outcome")) == canonical(cancelled["outcome"]), "new completion rewrote the original cancellation") + final_exit = idle_fleet(fleet, binding) + return {"stop-active": {"verdict": "PASS", "activeWindow": first["outcome"]["counters"], + "cancelled": cancellation, "physicalQuietProof": physical, "liveQuietSamples": 3, "betweenSamplesSeconds": QUIET_SECONDS, + "sameLiveProcesses": True, "allPeerIdleProofs": exit_proofs}, + "repeat-stop": {"verdict": "PASS", **repeated_proof}, + "old-status": {"verdict": "PASS", "oldTaskId": old, "originalPartialOutcomePreserved": True}, + "old-stop-new-task": {"verdict": "PASS", "oldTaskId": old, "newTaskId": new, + "newRunningBefore": before["outcome"]["counters"], "newRunningAfter": after_stale_counters, + "staleStop": stale_proof, "newCompleteResponsibility": terminal, "allPeerIdleProof": final_exit}} + + +if __name__ == "__main__": + try: + dx_main(case_id=CASE_ID, observer=observe_task, branches=BRANCHES) + except Exception as exc: + print(f"Heal stop validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/test_validation_heal.py b/test_validation_heal.py index fe215cfd..5e0eb920 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -46,6 +46,8 @@ from scripts.validation_heal.cursor import (Window, OBJECTS as CURSOR_OBJECTS, SIZE as CURSOR_SIZE, BRANCHES as CURSOR_BRANCHES, terminal_proof as cursor_terminal, validate_missing_batch, verify_all_parts) from scripts.validation_heal.fleet_node import resource_limits, verify_parts as node_verify_parts +from scripts.validation_heal.stop import (partial_window, validate_retired_stop, validate_cancelled, + cancelled_physical, idle_fleet as stop_idle_fleet, BRANCHES as STOP_BRANCHES) import errno import stat @@ -1933,5 +1935,111 @@ def invalid(node, action, *, parts): with self.subTest(kind=kind), self.assertRaises(ValueError): verify_all_parts(fleet, cohorts, "proof") +class HealStopOracleTest(unittest.TestCase): + def expected(self): + return HealCursorOracleTest().expected() + + def cancelled(self): + status = HealCursorOracleTest().terminal() + objects = status["outcome"]["objects"][-2:] + objects[0]["disposition"]["state"] = "cancelled" + status.update(summary="stopped") + status["outcome"].update(execution={"state": "aborted", "reason": "cancelled"}, coverage="partial", objects=objects) + status["outcome"]["counters"].update(processed=4, healed=1, unchanged=2, skipped=1) + targets = [{"key": "objects/1023.bin", "versionId": "null"}, {"key": "objects/1024.bin", "versionId": "null"}] + return status, targets + + def test_cancelled_receipts_cannot_claim_complete_or_repaired_unresolved_objects(self): + status, targets = self.cancelled() + proof = validate_cancelled(status, HealCursorOracleTest.bucket, self.expected(), targets) + self.assertEqual(proof["coverage"], "partial") + for kind in ("completed", "coverage", "unknown", "counter", "full-heal", "cancelled-as-repaired", "identity"): + changed = copy.deepcopy(status) + if kind == "completed": changed["outcome"]["execution"] = {"state": "completed"} + if kind == "coverage": changed["outcome"]["coverage"] = "complete" + if kind == "unknown": changed["outcome"]["counters"]["unknown"] = 1 + if kind == "counter": changed["outcome"]["counters"]["processed"] += 1 + if kind == "full-heal": changed["outcome"]["counters"]["healed"] = 2 + if kind == "cancelled-as-repaired": changed["outcome"]["objects"][0]["disposition"]["state"] = "unknown" + if kind == "identity": changed["outcome"]["objects"][0]["identity"]["bucket"] = "other" + with self.subTest(kind=kind), self.assertRaises(ValueError): + validate_cancelled(changed, HealCursorOracleTest.bucket, self.expected(), targets) + + def test_active_window_requires_actual_partial_canonical_progress_and_no_errors(self): + status, targets = self.cancelled() + status.update(summary="running") + status["outcome"]["execution"] = {"state": "running"} + self.assertTrue(partial_window(status, targets, repair_required=True)) + for name, value in (("processed", 0), ("processed", CURSOR_OBJECTS), ("healed", 0), ("healed", 2)): + changed = copy.deepcopy(status) + changed["outcome"]["counters"][name] = value + self.assertFalse(partial_window(changed, targets, repair_required=True)) + changed = copy.deepcopy(status) + changed["outcome"]["counters"]["unknown"] = 1 + with self.assertRaises(ValueError): partial_window(changed, targets, repair_required=True) + + def test_repeat_stop_freezes_http_structured_code_and_the_retired_token(self): + response = {"httpStatus": 500, "response": "InternalErrorHeal task not found: old-token"} + self.assertEqual(validate_retired_stop(response, "old-token")["exactRetiredTaskId"], "old-token") + for changed in ({**response, "httpStatus": 200}, {**response, "transportError": "timeout"}, + {**response, "response": response["response"].replace("old-token", "new-token")}, + {**response, "response": response["response"].replace("InternalError", "AccessDenied")}): + with self.assertRaises(ValueError): validate_retired_stop(changed, "old-token") + + def test_cancelled_physical_count_must_match_actual_original_shards(self): + before = HealShardOracleTest().snapshot() + for entry in list(before["node1"]["files"]): + if "objects/0008.bin" not in entry["path"]: continue + cloned = copy.deepcopy(entry) + cloned["path"] = cloned["path"].replace("objects/0008.bin", "objects/0017.bin") + if "metadata" in cloned: cloned["metadata"]["key"] = "objects/0017.bin" + before["node1"]["files"].append(cloned) + target = map_target(before, "rv-heal-0123456789abcdef", "objects/0008.bin", "data", 1) + second = map_target(before, "rv-heal-0123456789abcdef", "objects/0017.bin", "data", 1) + quiet = copy.deepcopy(before) + quiet["node1"]["files"] = [entry for entry in quiet["node1"]["files"] if + (entry["disk"], entry["path"]) != (second["disk"], second["relativePath"])] + responsibility = {"canonicalCounters": {"healed": 1}, "diagnosticDispositions": [ + {"key": target["key"], "versionId": "null", "state": "repaired"}, + {"key": second["key"], "versionId": "null", "state": "cancelled"}]} + remaining, proof = cancelled_physical(before, quiet, [target, second], responsibility) + self.assertEqual(remaining, [second]) + self.assertFalse(proof["backupRescueUsed"]) + with self.assertRaisesRegex(ValueError, "disagrees"): + cancelled_physical(before, quiet, [target, second], {"canonicalCounters": {"healed": 0}}) + wrong_disposition = copy.deepcopy(responsibility) + wrong_disposition["diagnosticDispositions"][0]["state"] = "cancelled" + with self.assertRaisesRegex(ValueError, "diagnostic disposition"): + cancelled_physical(before, quiet, [target, second], wrong_disposition) + next(entry for entry in quiet["node1"]["files"] if "/control/" in entry["path"])["sha256"] = "changed-control" + with self.assertRaises(ValueError): cancelled_physical(before, quiet, [target, second], responsibility) + + def test_every_stop_source_branch_is_retained(self): + self.assertEqual(list(STOP_BRANCHES), select("A06")[0]["variants"]) + + def test_task_exit_waits_for_actual_drain_and_never_hides_partial_idle(self): + good = {"httpStatus": 200, "response": {"state": "idle", "clusterStatusComplete": True, + "coverage": {"expected": 1, "responded": 1, "unknown": 0, "reasons": []}, + "healQueueLength": 0, "healActiveTasks": 0, + "healOperations": {"queueLength": 0, "activeTasks": 0, "retryingTasks": 0}}} + active = copy.deepcopy(good) + active["response"].update(state="active", healActiveTasks=1) + active["response"]["healOperations"]["activeTasks"] = 1 + fleet = types.SimpleNamespace(record=lambda label, value: value) + client = types.SimpleNamespace(admin=types.SimpleNamespace()) + from unittest.mock import Mock + client.admin = Mock(side_effect=[active, good]) + binding = {"endpoints": {"node1": "url"}} + with patch("scripts.validation_heal.stop.Probe", return_value=client), patch("scripts.validation_heal.stop.time.sleep"): + self.assertEqual(stop_idle_fleet(fleet, binding)["node1"]["state"], "idle") + unsafe = copy.deepcopy(good) + unsafe["response"]["clusterStatusComplete"] = False + client.admin = Mock(side_effect=[unsafe, good]) + with patch("scripts.validation_heal.stop.Probe", return_value=client): + with self.assertRaisesRegex(ValueError, "partial fleet"): + stop_idle_fleet(fleet, binding) + self.assertEqual(client.admin.call_count, 1) + + if __name__ == "__main__": unittest.main() From 8253bacd86467b5e6c33463b0fcaf053a0526e39 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 02:30:56 +0800 Subject: [PATCH 059/343] test(validation): migrate cancelled Heal restart and durable journal proof --- docs/validation-migration-map.md | 25 ++++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/cancel_restart.py | 111 ++++++++++++++++++++++ scripts/validation_heal/fleet_node.py | 42 ++++++++ scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 1 + scripts/validation_heal/stop.py | 21 +++- test_validation_heal.py | 68 +++++++++++++ 8 files changed, 263 insertions(+), 9 deletions(-) create mode 100644 scripts/validation_heal/cancel_restart.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index ec9511c8..67e4d45d 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 106 currently dispatchable IDs include all 74 non-Heal cases and thirty-two Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. +case evidence. The 107 currently dispatchable IDs include all 74 non-Heal cases and thirty-three Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 52 Heal cases remain +The other 51 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -566,3 +566,24 @@ cancelled outcome. Full streaming/physical and all-peer S3 verification happens before backup rescue. All four source branches remain explicit. Local cancellation/counter/quiet-physical/token/error negatives passed; full live proof remains pending. + + +F06 `heal.cancelled-task-restart` extends the same real partial cancellation +and write-quiescence proof with a graceful restart of every original H1/H2 +unit. A read-only node action accepts only a canonical admitted task UUID and +three exact candidate journal names under the private run roots; it records +all four drives on every node, bounds terminal/intent to 64KiB and report to +8MiB, and rejects links, hardlinks, oversized or changing files. The actual +cancelled terminal and report need one matching physical owner, exact task and +prefix scope, no runnable intent anywhere, and the settled partial outcome and +valid sequence window. It never synthesizes a checkpoint or edits system +records. After same-root restart, every actual process must be new while its +unit/candidate/formats/objects remain original, all peers must be idle, old +status must still be cancelled, and the exact original journal bytes must be +preserved. A fresh forceStart task remains independent under old STATUS/STOP +and repairs only the remaining witnessed defects. Full physical/streaming/S3 +proof remains mandatory. A06 also now preserves both its initial cancellation +and allowed monotonic in-flight report refinement before freezing the report +after actual drain; later quiet/restart/new-task checks cannot rewrite it. +Local journal-owner/schema/scope/cursor/read-boundary negatives passed; full +live evidence remains pending. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index e1ac4474..397e7f11 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/cancel_restart.py b/scripts/validation_heal/cancel_restart.py new file mode 100644 index 00000000..ca046b8c --- /dev/null +++ b/scripts/validation_heal/cancel_restart.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""F06: cancelled Admin journal survives the same-unit full fleet restart.""" + +from __future__ import annotations + +import sys + +try: + from .heal_probe import canonical, require + from .healthy import physical_content + from .cursor import main as dx_main + from .stop import observe_task as stop_task, status, idle_fleet +except ImportError: + from heal_probe import canonical, require + from healthy import physical_content + from cursor import main as dx_main + from stop import observe_task as stop_task, status, idle_fleet + + +CASE_ID = "heal.cancelled-task-restart" +BRANCHES = ("cancel-restart", "old-token-after-restart", "force-new-after-cancel") +REFERENCE = {"candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", + "terminalSchema": 1, "reportSchema": 1, "intentPrefix": "root-heal-", + "terminalPrefix": "terminal-root-heal-", "reportPrefix": "heal-terminal-report-", + "sources": {"crates/heal/src/heal/manager/root_recovery.rs": "ba6beba9d9d14985e4c5448d85727036388d3d72f28879ac5c67fc5f760048ba", + "crates/heal/src/heal/manager/root_recovery/report.rs": "9beca8d450e426d66315e10ea2cebf4b375719c06d9019a7263e20f559070009"}, + "role": "exact run-owned read-only journal witness; no synthesized checkpoint or record replay"} + + +def validate_journal(observed, token, bucket, cancelled): + present, all_records = {}, [] + for node, value in observed.items(): + require(value.get("readOnly") is True and value.get("taskId") == token + and isinstance(value.get("records"), list) and len(value["records"]) == 12, + "Admin journal did not cover each actual node's four drives and three exact names") + seen = set() + for entry in value["records"]: + identity = (node, entry.get("disk"), entry.get("kind")) + require(type(identity[1]) is int and 0 <= identity[1] < 4 and identity[2] in {"intent", "terminal", "report"} + and identity not in seen and type(entry.get("present")) is bool, "invalid or duplicate journal witness") + seen.add(identity) + if entry["present"]: + require(entry["kind"] not in present, "Admin task has duplicate physical journal owners") + present[entry["kind"]] = (node, entry) + all_records.append({"node": node, **entry}) + require(set(present) == {"terminal", "report"}, "cancelled task retained runnable intent or lost its durable terminal/report") + terminal_node, terminal_entry = present["terminal"] + report_node, report_entry = present["report"] + require((terminal_node, terminal_entry["disk"]) == (report_node, report_entry["disk"]), "journal terminal/report owners disagree") + terminal, report = terminal_entry["value"], report_entry["value"] + require(type(terminal.get("schema")) is int and terminal["schema"] == 1 + and terminal.get("task_id") == token and terminal.get("status") == "Cancelled" + and terminal.get("heal_type") == {"type": "prefix", "bucket": bucket, "prefix": "objects/"}, + "durable terminal has the wrong task, scope, schema or cancelled state") + require(type(report.get("schema")) is int and report["schema"] == 1 and report.get("terminal") == terminal + and canonical(report.get("outcome")) == canonical(cancelled.get("outcome")), + "durable report differs from the settled cancelled responsibility") + require(type(report.get("next_seq")) is int and type(report.get("min_seq")) is int + and 1 <= report["min_seq"] <= report["next_seq"] and isinstance(report.get("seqed_items"), list) + and len(report["seqed_items"]) <= 1024, "durable cancelled report has invalid sequence bounds") + previous = 0 + for item in report["seqed_items"]: + require(isinstance(item, list) and len(item) == 2 and type(item[0]) is int + and max(previous + 1, report["min_seq"]) <= item[0] < report["next_seq"] + and item[1].get("bucket") == bucket and item[1].get("object", "").startswith("objects/"), + "durable report cursor/item crossed the old task responsibility") + previous = item[0] + return {"ownerNode": terminal_node, "ownerDisk": terminal_entry["disk"], "cancelledTaskId": token, + "runnableIntentAbsentOnEveryDrive": True, "terminalAndReportExactlyMatch": True, + "allRecords": all_records} + + +def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, original): + fleet.record("journal-source-contract", REFERENCE) + def restart(path, old, cancelled, quiet): + fleet.stop() + before = fleet.record("cancelled-journal-before-restart", {node: fleet.remote(node, "task-records", taskId=old, + quiescent=True) for node in fleet.nodes}) + journal = validate_journal(before, old, bucket, cancelled) + stopped = fleet.snapshot("cancelled-before-restart", quiescent=True) + require(physical_content(stopped) == physical_content(quiet), "graceful stop changed cancelled object files/formats") + fleet.start() + idle = idle_fleet(fleet, binding) + old_after = status(probe, fleet, path, old, "cancelled-status-after-restart") + require(old_after.get("summary") == "stopped" and canonical(old_after.get("outcome")) == canonical(cancelled["outcome"]), + "cancelled task revived, lost its responsibility or falsely completed after restart") + restarted = fleet.snapshot("cancelled-live-after-restart") + require(physical_content(restarted) == physical_content(quiet) and all( + restarted[node]["pid"] > 0 and restarted[node]["pid"] != quiet[node]["pid"] + and restarted[node]["unit"] == quiet[node]["unit"] and restarted[node]["binarySha256"] == quiet[node]["binarySha256"] + for node in fleet.nodes), "restart replaced topology/candidate/roots or failed to start new processes") + after = fleet.record("cancelled-journal-after-restart", {node: fleet.remote(node, "task-records", taskId=old) + for node in fleet.nodes}) + validate_journal(after, old, bucket, old_after) + require(canonical(before) == canonical(after), "restart rewrote the original cancelled task's journal") + return {"journal": journal, "sameOriginalUnitsRootsAndCandidate": True, "newProcessOnEveryNode": True, + "oldCancelledOutcomePreserved": True, "oldJournalBytesPreserved": True, "allPeerIdle": idle} + proof = stop_task(probe, fleet, binding, bucket, expected, targets, active, original=original, + after_cancelled=restart, force_new=True) + return {"cancel-restart": {"verdict": "PASS", "cancellation": proof["stop-active"]}, + "old-token-after-restart": {"verdict": "PASS", "oldStatus": proof["old-status"], + "retiredStop": proof["old-stop-new-task"]["staleStop"]}, + "force-new-after-cancel": {"verdict": "PASS", "forceStart": True, **proof["old-stop-new-task"]}} + + +if __name__ == "__main__": + try: + dx_main(case_id=CASE_ID, observer=observe_task, branches=BRANCHES) + except Exception as exc: + print(f"Heal cancelled-restart validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 18918b9e..dfee2f7a 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -17,6 +17,7 @@ import socket import stat import subprocess +import uuid MOUNTS = tuple(Path(f"/data/rustfs{index}") for index in range(1, 5)) @@ -486,11 +487,52 @@ def audit_finalize(request): "backupRestorationUsed": False, "productRepairClaimed": False} +def task_records(request): + """Read three exact Admin journal names; never mutate system metadata.""" + state, roots, owner = owned(request["runId"]) + token = request.get("taskId") + require(isinstance(token, str) and str(uuid.UUID(token)) == token, "noncanonical Admin journal task ID") + require(digest(state / "rustfs") == owner["binarySha256"], "journal candidate changed") + if request.get("quiescent") is True: + stopped(owner) + else: + pid = int(command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"]).stdout.strip() or "0") + require(pid > 0 and Path(f"/proc/{pid}/exe").resolve(strict=True) == state / "rustfs", "journal service is not run-owned") + records = [] + for disk, root in enumerate(roots): + data = root / "data" + for kind, prefix, limit in (("intent", "root-heal-", 64 * 1024), + ("terminal", "terminal-root-heal-", 64 * 1024), + ("report", "heal-terminal-report-", 8 * 1024**2)): + relative = ".rustfs.sys/" + prefix + token + ".json" + path = checked(data / relative, data) + entry = {"disk": disk, "kind": kind, "relativePath": relative, "present": False} + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + except FileNotFoundError: + records.append(entry) + continue + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size <= limit, + "Admin journal exceeds its exact regular-file budget") + raw = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + require(len(raw) == info.st_size and len(raw) <= limit + and all(getattr(after, name) == getattr(info, name) for name in + ("st_ino", "st_dev", "st_size", "st_mtime_ns", "st_ctime_ns", "st_nlink")), + "Admin journal changed during its bounded read") + entry.update(present=True, size=len(raw), sha256=hashlib.sha256(raw).hexdigest(), value=json.loads(raw)) + records.append(entry) + return {"taskId": token, "binarySha256": owner["binarySha256"], "readOnly": True, "records": records} + + def main(request): require(os.geteuid() == 0, "node adapter requires passwordless sudo") actions = {"preflight": preflight, "reserve": reserve, "start": start, "stop": stop, "snapshot": snapshot, "logs": logs, "cleanup": cleanup, "audit-finalize": audit_finalize, "install-inspector": install_inspector, "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part, "verify-parts": verify_parts, "verify-inline": verify_inline, "drive-gate": drive_gate, "drive-gate-probe": drive_gate_probe} + actions["task-records"] = task_records require(request.get("action") in actions, "unsupported node operation") print(json.dumps(actions[request["action"]](request))) diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 73af9642..323c2d18 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 1a35ad78..a7e7a798 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -23,6 +23,7 @@ case "${case_id}" in heal.status-partial-disabled-unknown) driver=status.py ;; heal.incremental-cursor-and-truncation) driver=cursor.py ;; heal.admin-stop-and-stale-token) driver=stop.py ;; + heal.cancelled-task-restart) driver=cancel_restart.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; diff --git a/scripts/validation_heal/stop.py b/scripts/validation_heal/stop.py index 4560a16c..cb2d8b61 100644 --- a/scripts/validation_heal/stop.py +++ b/scripts/validation_heal/stop.py @@ -44,8 +44,9 @@ def status(probe, fleet, path, token, label): return observed["response"] -def start(probe, fleet, path, active): - observed = fleet.record("trigger", probe.admin(path, [], json.dumps({**BASE_OPTIONS, "recursive": True}).encode())) +def start(probe, fleet, path, active, *, force=False): + observed = fleet.record("trigger", probe.admin(path, [("forceStart", "true")] if force else [], + json.dumps({**BASE_OPTIONS, "recursive": True}).encode())) require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), "DX task admission failed") token = observed["response"].get("clientToken") require(isinstance(token, str) and token, "DX task omitted its admitted token") @@ -181,7 +182,7 @@ def idle_fleet(fleet, binding): time.sleep(0.1) -def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, original): +def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, original, after_cancelled=None, force_new=False): path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/objects/" fleet.record("frozen-cancel-policy", POLICY) old = start(probe, fleet, path, active) @@ -190,7 +191,16 @@ def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, or require(stop_ack.get("httpStatus") == 200 and stop_ack.get("response", {}).get("summary") == "stopped", "active cancellation did not follow the frozen acknowledgement contract") cancelled = status(probe, fleet, path, old, "cancelled-outcome") + initial_cancellation = validate_cancelled(cancelled, bucket, expected, targets) + idle_fleet(fleet, binding) + cancelled = status(probe, fleet, path, old, "cancelled-after-drain") cancellation = validate_cancelled(cancelled, bucket, expected, targets) + require(cancellation["bucketIncarnationId"] == initial_cancellation["bucketIncarnationId"] + and all(cancellation["canonicalCounters"][name] >= initial_cancellation["canonicalCounters"][name] + for name in ("processed", "healed", "unchanged", "skipped")), + "in-flight cancellation refinement changed identity or moved counters backwards") + fleet.record("cancelled-settlement", {"initial": initial_cancellation, "afterActualDrain": cancellation, + "earlierObservationsPreserved": True, "reason": "candidate refines cancelled reports while retiring in-flight workers"}) # Keep the actual service alive. An ACK or stopping its process would mask # writes that escaped task cancellation. quiet = [] @@ -216,7 +226,8 @@ def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, or active.remove((path, old)) repeated = fleet.record("repeat-stop", probe.admin(path, [("clientToken", old), ("forceStop", "true")], b"")) repeated_proof = validate_retired_stop(repeated, old) - new = start(probe, fleet, path, active) + restart_proof = after_cancelled(path, old, cancelled, quiet[-1]) if after_cancelled is not None else None + new = start(probe, fleet, path, active, force=force_new) require(new != old, "new task reused a cancelled token") before = wait_partial(probe, fleet, path, new, remaining, repair_required=False) old_status = status(probe, fleet, path, old, "old-status-during-new") @@ -246,7 +257,7 @@ def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, or final_exit = idle_fleet(fleet, binding) return {"stop-active": {"verdict": "PASS", "activeWindow": first["outcome"]["counters"], "cancelled": cancellation, "physicalQuietProof": physical, "liveQuietSamples": 3, "betweenSamplesSeconds": QUIET_SECONDS, - "sameLiveProcesses": True, "allPeerIdleProofs": exit_proofs}, + "sameLiveProcesses": True, "allPeerIdleProofs": exit_proofs, "restartProof": restart_proof}, "repeat-stop": {"verdict": "PASS", **repeated_proof}, "old-status": {"verdict": "PASS", "oldTaskId": old, "originalPartialOutcomePreserved": True}, "old-stop-new-task": {"verdict": "PASS", "oldTaskId": old, "newTaskId": new, diff --git a/test_validation_heal.py b/test_validation_heal.py index 5e0eb920..1829ebe8 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -48,6 +48,8 @@ from scripts.validation_heal.fleet_node import resource_limits, verify_parts as node_verify_parts from scripts.validation_heal.stop import (partial_window, validate_retired_stop, validate_cancelled, cancelled_physical, idle_fleet as stop_idle_fleet, BRANCHES as STOP_BRANCHES) +from scripts.validation_heal.cancel_restart import validate_journal, BRANCHES as CANCEL_RESTART_BRANCHES +from scripts.validation_heal.fleet_node import task_records as node_task_records import errno import stat @@ -2041,5 +2043,71 @@ def test_task_exit_waits_for_actual_drain_and_never_hides_partial_idle(self): self.assertEqual(client.admin.call_count, 1) +class HealCancelledRestartOracleTest(unittest.TestCase): + token = "12345678-1234-1234-1234-123456789abc" + + def journal(self): + status, _ = HealStopOracleTest().cancelled() + terminal = {"schema": 1, "task_id": self.token, "status": "Cancelled", + "heal_type": {"type": "prefix", "bucket": HealCursorOracleTest.bucket, "prefix": "objects/"}} + report = {"schema": 1, "terminal": terminal, "outcome": status["outcome"], "min_seq": 1, "next_seq": 2, + "seqed_items": [[1, {"bucket": HealCursorOracleTest.bucket, "object": "objects/0000.bin"}]]} + records = [{"disk": disk, "kind": kind, "present": disk == 0 and kind != "intent", + **({"value": terminal if kind == "terminal" else report, "sha256": "original", "size": 100} + if disk == 0 and kind != "intent" else {})} for disk in range(4) for kind in ("intent", "terminal", "report")] + return {"node1": {"taskId": self.token, "readOnly": True, "records": records}}, status + + def test_actual_cancelled_journal_rejects_runnable_duplicate_or_wrong_epoch_records(self): + observed, status = self.journal() + self.assertTrue(validate_journal(observed, self.token, HealCursorOracleTest.bucket, status)["runnableIntentAbsentOnEveryDrive"]) + for kind in ("intent", "duplicate", "scope", "status", "schema", "identity", "outcome", "cursor", "owner", "missing"): + changed = copy.deepcopy(observed) + records = changed["node1"]["records"] + terminal, report = records[1]["value"], records[2]["value"] + if kind == "intent": records[0]["present"] = True + if kind == "duplicate": records[4].update(present=True, value=terminal) + if kind == "scope": terminal["heal_type"]["bucket"] = "unowned" + if kind == "status": terminal["status"] = "Completed" + if kind == "schema": terminal["schema"] = True + if kind == "identity": terminal["task_id"] = "another-task" + if kind == "outcome": report["outcome"]["counters"]["healed"] += 1 + if kind == "cursor": report["seqed_items"][0][0] = report["next_seq"] + if kind == "owner": records[2]["disk"] = 1 + if kind == "missing": records[2]["present"] = False + with self.subTest(kind=kind), self.assertRaises(ValueError): + validate_journal(changed, self.token, HealCursorOracleTest.bucket, status) + + def test_node_reader_is_read_only_and_only_accepts_exact_task_filenames(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder).resolve() + state = root / "state" + state.mkdir() + (state / "rustfs").write_bytes(b"candidate") + roots = [root / f"disk{disk}" for disk in range(4)] + for disk in roots: (disk / "data/.rustfs.sys").mkdir(parents=True) + system = roots[0] / "data/.rustfs.sys" + terminal = system / f"terminal-root-heal-{self.token}.json" + terminal.write_text('{"task_id":"' + self.token + '"}') + unrelated = system / "terminal-root-heal-unrelated.json" + unrelated.write_text('{"unrelated":"must not enter evidence"}') + owner = {"binarySha256": hashlib.sha256(b"candidate").hexdigest()} + with patch("scripts.validation_heal.fleet_node.owned", return_value=(state, roots, owner)), \ + patch("scripts.validation_heal.fleet_node.stopped"): + observed = node_task_records({"runId": "run", "taskId": self.token, "quiescent": True}) + self.assertTrue(observed["readOnly"]) + self.assertEqual(len(observed["records"]), 12) + self.assertEqual(sum(entry["present"] for entry in observed["records"]), 1) + self.assertNotIn("must not enter evidence", json.dumps(observed)) + self.assertEqual(terminal.read_text(), '{"task_id":"' + self.token + '"}') + for invalid in ("../other", self.token.upper(), "not-a-uuid"): + with self.assertRaises(ValueError): node_task_records({"runId": "run", "taskId": invalid, "quiescent": True}) + terminal.unlink() + terminal.symlink_to(unrelated) + with self.assertRaises(ValueError): node_task_records({"runId": "run", "taskId": self.token, "quiescent": True}) + + def test_restart_preserves_every_source_branch(self): + self.assertEqual(list(CANCEL_RESTART_BRANCHES), select("F06")[0]["variants"]) + + if __name__ == "__main__": unittest.main() From 5e650618aa8e05440932c6dca705f2e85c395236 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 02:31:31 +0800 Subject: [PATCH 060/343] test(validation): retain cancelled journal across replacement task completion --- scripts/validation_heal/cancel_restart.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/cancel_restart.py b/scripts/validation_heal/cancel_restart.py index ca046b8c..b6999768 100644 --- a/scripts/validation_heal/cancel_restart.py +++ b/scripts/validation_heal/cancel_restart.py @@ -72,11 +72,13 @@ def validate_journal(observed, token, bucket, cancelled): def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, original): fleet.record("journal-source-contract", REFERENCE) + archived = {} def restart(path, old, cancelled, quiet): fleet.stop() before = fleet.record("cancelled-journal-before-restart", {node: fleet.remote(node, "task-records", taskId=old, quiescent=True) for node in fleet.nodes}) journal = validate_journal(before, old, bucket, cancelled) + archived.update(records=before, cancelled=cancelled, path=path, old=old) stopped = fleet.snapshot("cancelled-before-restart", quiescent=True) require(physical_content(stopped) == physical_content(quiet), "graceful stop changed cancelled object files/formats") fleet.start() @@ -97,9 +99,15 @@ def restart(path, old, cancelled, quiet): "oldCancelledOutcomePreserved": True, "oldJournalBytesPreserved": True, "allPeerIdle": idle} proof = stop_task(probe, fleet, binding, bucket, expected, targets, active, original=original, after_cancelled=restart, force_new=True) + final_old = status(probe, fleet, archived["path"], archived["old"], "old-status-after-new-completion") + final_journal = fleet.record("old-journal-after-new-completion", {node: fleet.remote(node, "task-records", taskId=archived["old"]) + for node in fleet.nodes}) + validate_journal(final_journal, archived["old"], bucket, final_old) + require(canonical(final_journal) == canonical(archived["records"]), + "new forceStart task rewrote or removed the original cancelled task's journal") return {"cancel-restart": {"verdict": "PASS", "cancellation": proof["stop-active"]}, "old-token-after-restart": {"verdict": "PASS", "oldStatus": proof["old-status"], - "retiredStop": proof["old-stop-new-task"]["staleStop"]}, + "retiredStop": proof["old-stop-new-task"]["staleStop"], "oldJournalPreservedThroughNewCompletion": True}, "force-new-after-cancel": {"verdict": "PASS", "forceStart": True, **proof["old-stop-new-task"]}} From 8b8bb60d2d0a26bb0974b89d07ac753191aa95d6 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 02:36:36 +0800 Subject: [PATCH 061/343] test(validation): bind durable diagnostic items to original ACK identities --- scripts/validation_heal/cancel_restart.py | 19 ++++++++++++------- test_validation_heal.py | 12 ++++++++---- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/scripts/validation_heal/cancel_restart.py b/scripts/validation_heal/cancel_restart.py index b6999768..a7226af2 100644 --- a/scripts/validation_heal/cancel_restart.py +++ b/scripts/validation_heal/cancel_restart.py @@ -8,12 +8,14 @@ try: from .heal_probe import canonical, require from .healthy import physical_content - from .cursor import main as dx_main + from .cursor import main as dx_main, wanted_pairs + from .versions import public_version_id from .stop import observe_task as stop_task, status, idle_fleet except ImportError: from heal_probe import canonical, require from healthy import physical_content - from cursor import main as dx_main + from cursor import main as dx_main, wanted_pairs + from versions import public_version_id from stop import observe_task as stop_task, status, idle_fleet @@ -27,7 +29,8 @@ "role": "exact run-owned read-only journal witness; no synthesized checkpoint or record replay"} -def validate_journal(observed, token, bucket, cancelled): +def validate_journal(observed, token, bucket, cancelled, expected): + wanted = wanted_pairs(expected) present, all_records = {}, [] for node, value in observed.items(): require(value.get("readOnly") is True and value.get("taskId") == token @@ -62,7 +65,9 @@ def validate_journal(observed, token, bucket, cancelled): for item in report["seqed_items"]: require(isinstance(item, list) and len(item) == 2 and type(item[0]) is int and max(previous + 1, report["min_seq"]) <= item[0] < report["next_seq"] - and item[1].get("bucket") == bucket and item[1].get("object", "").startswith("objects/"), + and item[1].get("bucket") == bucket + and (item[1].get("object"), public_version_id(item[1].get("versionId"))) in wanted + and item[1].get("objectSize") == wanted[item[1]["object"], public_version_id(item[1].get("versionId"))]["payload"]["size"], "durable report cursor/item crossed the old task responsibility") previous = item[0] return {"ownerNode": terminal_node, "ownerDisk": terminal_entry["disk"], "cancelledTaskId": token, @@ -77,7 +82,7 @@ def restart(path, old, cancelled, quiet): fleet.stop() before = fleet.record("cancelled-journal-before-restart", {node: fleet.remote(node, "task-records", taskId=old, quiescent=True) for node in fleet.nodes}) - journal = validate_journal(before, old, bucket, cancelled) + journal = validate_journal(before, old, bucket, cancelled, expected) archived.update(records=before, cancelled=cancelled, path=path, old=old) stopped = fleet.snapshot("cancelled-before-restart", quiescent=True) require(physical_content(stopped) == physical_content(quiet), "graceful stop changed cancelled object files/formats") @@ -93,7 +98,7 @@ def restart(path, old, cancelled, quiet): for node in fleet.nodes), "restart replaced topology/candidate/roots or failed to start new processes") after = fleet.record("cancelled-journal-after-restart", {node: fleet.remote(node, "task-records", taskId=old) for node in fleet.nodes}) - validate_journal(after, old, bucket, old_after) + validate_journal(after, old, bucket, old_after, expected) require(canonical(before) == canonical(after), "restart rewrote the original cancelled task's journal") return {"journal": journal, "sameOriginalUnitsRootsAndCandidate": True, "newProcessOnEveryNode": True, "oldCancelledOutcomePreserved": True, "oldJournalBytesPreserved": True, "allPeerIdle": idle} @@ -102,7 +107,7 @@ def restart(path, old, cancelled, quiet): final_old = status(probe, fleet, archived["path"], archived["old"], "old-status-after-new-completion") final_journal = fleet.record("old-journal-after-new-completion", {node: fleet.remote(node, "task-records", taskId=archived["old"]) for node in fleet.nodes}) - validate_journal(final_journal, archived["old"], bucket, final_old) + validate_journal(final_journal, archived["old"], bucket, final_old, expected) require(canonical(final_journal) == canonical(archived["records"]), "new forceStart task rewrote or removed the original cancelled task's journal") return {"cancel-restart": {"verdict": "PASS", "cancellation": proof["stop-active"]}, diff --git a/test_validation_heal.py b/test_validation_heal.py index 1829ebe8..b2d06e0e 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -2051,7 +2051,7 @@ def journal(self): terminal = {"schema": 1, "task_id": self.token, "status": "Cancelled", "heal_type": {"type": "prefix", "bucket": HealCursorOracleTest.bucket, "prefix": "objects/"}} report = {"schema": 1, "terminal": terminal, "outcome": status["outcome"], "min_seq": 1, "next_seq": 2, - "seqed_items": [[1, {"bucket": HealCursorOracleTest.bucket, "object": "objects/0000.bin"}]]} + "seqed_items": [[1, {"bucket": HealCursorOracleTest.bucket, "object": "objects/0000.bin", "objectSize": CURSOR_SIZE}]]} records = [{"disk": disk, "kind": kind, "present": disk == 0 and kind != "intent", **({"value": terminal if kind == "terminal" else report, "sha256": "original", "size": 100} if disk == 0 and kind != "intent" else {})} for disk in range(4) for kind in ("intent", "terminal", "report")] @@ -2059,8 +2059,9 @@ def journal(self): def test_actual_cancelled_journal_rejects_runnable_duplicate_or_wrong_epoch_records(self): observed, status = self.journal() - self.assertTrue(validate_journal(observed, self.token, HealCursorOracleTest.bucket, status)["runnableIntentAbsentOnEveryDrive"]) - for kind in ("intent", "duplicate", "scope", "status", "schema", "identity", "outcome", "cursor", "owner", "missing"): + expected = HealCursorOracleTest().expected() + self.assertTrue(validate_journal(observed, self.token, HealCursorOracleTest.bucket, status, expected)["runnableIntentAbsentOnEveryDrive"]) + for kind in ("intent", "duplicate", "scope", "status", "schema", "identity", "outcome", "cursor", "owner", "missing", "item-key", "item-version", "item-size"): changed = copy.deepcopy(observed) records = changed["node1"]["records"] terminal, report = records[1]["value"], records[2]["value"] @@ -2074,8 +2075,11 @@ def test_actual_cancelled_journal_rejects_runnable_duplicate_or_wrong_epoch_reco if kind == "cursor": report["seqed_items"][0][0] = report["next_seq"] if kind == "owner": records[2]["disk"] = 1 if kind == "missing": records[2]["present"] = False + if kind == "item-key": report["seqed_items"][0][1]["object"] = "objects/unregistered.bin" + if kind == "item-version": report["seqed_items"][0][1]["versionId"] = "another-version" + if kind == "item-size": report["seqed_items"][0][1]["objectSize"] += 1 with self.subTest(kind=kind), self.assertRaises(ValueError): - validate_journal(changed, self.token, HealCursorOracleTest.bucket, status) + validate_journal(changed, self.token, HealCursorOracleTest.bucket, status, expected) def test_node_reader_is_read_only_and_only_accepts_exact_task_filenames(self): with tempfile.TemporaryDirectory() as folder: From 6b428e2fcae9b0a35dc212884fc3cb68d2092539 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 02:37:30 +0800 Subject: [PATCH 062/343] test(validation): reject duplicate or empty cancelled diagnostic journals --- scripts/validation_heal/cancel_restart.py | 7 +++++-- test_validation_heal.py | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/validation_heal/cancel_restart.py b/scripts/validation_heal/cancel_restart.py index a7226af2..2fce4d53 100644 --- a/scripts/validation_heal/cancel_restart.py +++ b/scripts/validation_heal/cancel_restart.py @@ -60,8 +60,8 @@ def validate_journal(observed, token, bucket, cancelled, expected): "durable report differs from the settled cancelled responsibility") require(type(report.get("next_seq")) is int and type(report.get("min_seq")) is int and 1 <= report["min_seq"] <= report["next_seq"] and isinstance(report.get("seqed_items"), list) - and len(report["seqed_items"]) <= 1024, "durable cancelled report has invalid sequence bounds") - previous = 0 + and 0 < len(report["seqed_items"]) <= 1024, "durable cancelled report has invalid sequence bounds") + previous, seen_items = 0, set() for item in report["seqed_items"]: require(isinstance(item, list) and len(item) == 2 and type(item[0]) is int and max(previous + 1, report["min_seq"]) <= item[0] < report["next_seq"] @@ -69,6 +69,9 @@ def validate_journal(observed, token, bucket, cancelled, expected): and (item[1].get("object"), public_version_id(item[1].get("versionId"))) in wanted and item[1].get("objectSize") == wanted[item[1]["object"], public_version_id(item[1].get("versionId"))]["payload"]["size"], "durable report cursor/item crossed the old task responsibility") + pair = item[1]["object"], public_version_id(item[1].get("versionId")) + require(pair not in seen_items, "durable cancelled report duplicated an ACK identity") + seen_items.add(pair) previous = item[0] return {"ownerNode": terminal_node, "ownerDisk": terminal_entry["disk"], "cancelledTaskId": token, "runnableIntentAbsentOnEveryDrive": True, "terminalAndReportExactlyMatch": True, diff --git a/test_validation_heal.py b/test_validation_heal.py index b2d06e0e..3619b699 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -2061,7 +2061,7 @@ def test_actual_cancelled_journal_rejects_runnable_duplicate_or_wrong_epoch_reco observed, status = self.journal() expected = HealCursorOracleTest().expected() self.assertTrue(validate_journal(observed, self.token, HealCursorOracleTest.bucket, status, expected)["runnableIntentAbsentOnEveryDrive"]) - for kind in ("intent", "duplicate", "scope", "status", "schema", "identity", "outcome", "cursor", "owner", "missing", "item-key", "item-version", "item-size"): + for kind in ("intent", "duplicate", "scope", "status", "schema", "identity", "outcome", "cursor", "owner", "missing", "item-key", "item-version", "item-size", "item-duplicate", "item-empty"): changed = copy.deepcopy(observed) records = changed["node1"]["records"] terminal, report = records[1]["value"], records[2]["value"] @@ -2078,6 +2078,10 @@ def test_actual_cancelled_journal_rejects_runnable_duplicate_or_wrong_epoch_reco if kind == "item-key": report["seqed_items"][0][1]["object"] = "objects/unregistered.bin" if kind == "item-version": report["seqed_items"][0][1]["versionId"] = "another-version" if kind == "item-size": report["seqed_items"][0][1]["objectSize"] += 1 + if kind == "item-duplicate": + report["next_seq"] = 3 + report["seqed_items"].append([2, copy.deepcopy(report["seqed_items"][0][1])]) + if kind == "item-empty": report["seqed_items"] = [] with self.subTest(kind=kind), self.assertRaises(ValueError): validate_journal(changed, self.token, HealCursorOracleTest.bucket, status, expected) From 6bbbc75aeb33e2e83c1689c0bcd354cd800862ae Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 02:58:07 +0800 Subject: [PATCH 063/343] test(validation): migrate Heal overlap and force-start controls --- docs/validation-heal-contract.md | 2 +- docs/validation-migration-map.md | 24 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/cursor.py | 65 ++++-- scripts/validation_heal/fleet_node.py | 5 +- scripts/validation_heal/healthy.py | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/overlap.py | 318 ++++++++++++++++++++++++++ scripts/validation_heal/run_case.sh | 1 + test_validation_heal.py | 120 ++++++++++ 10 files changed, 511 insertions(+), 30 deletions(-) create mode 100644 scripts/validation_heal/overlap.py diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index 8d4f283e..b09fb739 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -27 个用例已接入自动分发：A01、A03、A04、A07、A08、B02、B05、B07、B09、C01–C05、C08、D01–D03、D05–D07、D09、D10、E01–E03、J04。分片执行器使用实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash 校验。A03 的两个分支只执行 H2，比较协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome。A04 保留四个 DB 参数组合及两个 DV 分支，每个拓扑分别创建完整 DB/DV 环境；dry-run 不要求 incarnation，但必须有完整 dry_run_observed 集合且无对象修改。A08 保留五种主体和各自 start/status/stop 操作，使用真正重新计算的过期签名及仅授权 `admin:Heal` 的临时用户，清理后验证凭据失效。B09 使用明确启用 MRF 的专用服务；停止一个跨节点 set 成员节点，记录降级 ACK，并先验证逐盘补片，再执行 S3/手工验证请求。其余 57 项保留源 `operator-assisted` 方式。每项的实际执行状态见 [迁移映射](validation-migration-map.md)。方案、计划或完整文件归档均不能代替 live PASS。 +34 个用例已接入自动分发：A01、A03–A08、B02、B05、B07、B09、C01–C05、C08、D01–D07、D09、D10、E01–E04、F06、J01、J03、J04。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。其余 50 项保留源 operator-assisted 方式。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 67e4d45d..9ab5b23f 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 107 currently dispatchable IDs include all 74 non-Heal cases and thirty-three Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. +case evidence. The 108 currently dispatchable IDs include all 74 non-Heal cases and thirty-four Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 51 Heal cases remain +The other 50 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -587,3 +587,23 @@ and allowed monotonic in-flight report refinement before freezing the report after actual drain; later quiet/restart/new-task checks cannot rewrite it. Local journal-owner/schema/scope/cursor/read-boundary negatives passed; full live evidence remains pending. + +A05 `heal.admin-overlap-and-force-start` now dispatches all four source +branches on both H1 and H2, with an independent deployment/ACK ledger/fault +set for each instance. Two real prefix scopes contain 257 external objects +each; five missing shards per selected scope retain positive authority. +The actual process environment freezes `merge`: equivalent starts must return +the same token, parent/child requests must return exact 409/OperationAborted +with `overlapping_paths`, and disjoint requests must receive independent +tokens. Every control request needs a nonzero running bracket; a missed +window remains NOT_OBSERVED. forceStart must cancel the old task with partial +coverage, admit an independent replacement token, and resist old-token STOP. +Initial and settled cancellation reports are retained; exact old/new repair +counters, incarnations and diagnostic identities cannot leak or duplicate +responsibility. All 257 terminal item identities are read, including any +cropped HTTP suffix; this does not claim a 1024-item retention gap. Three +live physical samples after actual all-peer drain must remain unchanged, +with the same processes. Complete same-role shard bytes, streaming checksums +and all-peer full S3 manifests remain mandatory before PASS. Local +scope/error/window/duplicate-repair/item-omission/cleanup-stop negatives +passed. Full live evidence remains pending. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 397e7f11..486a0d4c 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/cursor.py b/scripts/validation_heal/cursor.py index 07839747..6b4578e3 100644 --- a/scripts/validation_heal/cursor.py +++ b/scripts/validation_heal/cursor.py @@ -50,11 +50,12 @@ "differentCandidates": "the measured terminal window must still actually exceed retention; no inferred gap"} -def wanted_pairs(expected): +def wanted_pairs(expected, *, count=OBJECTS): wanted = {(entry["key"], public_version_id(entry["versionId"])): entry for entry in expected} - require(len(wanted) == len(expected) == OBJECTS and all(not entry["deleteMarker"] + require(type(count) is int and 129 <= count <= OBJECTS + and len(wanted) == len(expected) == count and all(not entry["deleteMarker"] and pair[1] == "null" and entry["payload"]["size"] == SIZE for pair, entry in wanted.items()), - "cursor fixture must contain the pre-registered 1025 distinct external-size ACKs") + "DX fixture must contain every pre-registered distinct external-size ACK") return wanted @@ -161,9 +162,9 @@ def terminal_proof(status, bucket, expected, repaired, window): "retainedWindow": {name: value for name, value in retained.items() if name != "pairs"}} -def map_cohorts(snapshot, bucket, expected): +def map_cohorts(snapshot, bucket, expected, *, count=OBJECTS): """Index once, then use the existing complete-set oracle on each object.""" - wanted_pairs(expected) + wanted_pairs(expected, count=count) by_key = {entry["key"]: {node: {"formats": value["formats"], "files": []} for node, value in snapshot.items()} for entry in expected} for node, observation in snapshot.items(): @@ -177,7 +178,15 @@ def map_cohorts(snapshot, bucket, expected): for index, entry in enumerate(expected)] -def verify_all_parts(fleet, cohorts, label): +def verify_all_parts(fleet, cohorts, label, *, count=OBJECTS): + require(type(count) is int and 129 <= count <= OBJECTS and len(cohorts) == count, + "complete physical verification differs from its frozen object count") + width = 4 if len(fleet.nodes) == 1 else 16 + identities = [(entry["node"], entry["disk"], entry["relativePath"], entry["versionId"], entry["partNumber"]) + for target in cohorts for entry in target["cohort"]] + require(all(len(target["cohort"]) == width for target in cohorts) + and len(identities) == len(set(identities)) == count * width + and all(entry[0] in fleet.nodes for entry in identities), "complete part manifest duplicated or omitted a frozen shard") total = 0 for node in fleet.nodes: entries = [shard for target in cohorts for shard in target["cohort"] if shard["node"] == node] @@ -192,8 +201,8 @@ def verify_all_parts(fleet, cohorts, label): and all(proof.get(name) == entry[name] for name in ("disk", "relativePath", "versionId", "partNumber", "sha256")) and proof.get("physicalSize") == entry["size"], "external part differs from its frozen streaming/hash oracle") total += len(batch) - require(total == OBJECTS * (4 if len(fleet.nodes) == 1 else 16), "full external-part verifier missed a disk/object") - return {"allExternalObjects": OBJECTS, "allPartsStreamingVerified": total, + require(total == count * width, "full external-part verifier missed a disk/object") + return {"allExternalObjects": count, "allPartsStreamingVerified": total, "sameRoleOriginalShardBytesRestored": True} @@ -274,11 +283,17 @@ def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, or "completeManifestEvidenceRequired": True}} -def run_topology(args, topology, nodes, *, observer=observe_task, branches=BRANCHES): +def run_topology(args, topology, nodes, *, observer=observe_task, branches=BRANCHES, + count=OBJECTS, fixture_builder=None, missing_indices=None, + source_branches=None, runtime_profile="manual", policy_reference=LIMIT_REFERENCE): directory = args.evidence / topology directory.mkdir() run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" - fleet = Fleet(nodes, directory, run_id, args.tool_directory, resource_profile="cursor") + fleet = Fleet(nodes, directory, run_id, args.tool_directory, resource_profile="cursor", runtime_profile=runtime_profile) + require(type(count) is int and 129 <= count <= OBJECTS, "invalid pre-registered DX size") + missing_indices = list(range(0, count, 64)) if missing_indices is None else list(missing_indices) + require(len(missing_indices) > 1 and len(set(missing_indices)) == len(missing_indices) + and all(type(index) is int and 0 <= index < count for index in missing_indices), "invalid DX fault indices") bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) @@ -290,29 +305,31 @@ def run_topology(args, topology, nodes, *, observer=observe_task, branches=BRANC "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), - "maxInventoryObjects": OBJECTS, "maxInventoryBytes": OBJECTS * SIZE} + "maxInventoryObjects": count, "maxInventoryBytes": count * SIZE} fleet.record("binding", binding) - fleet.record("pre-registered-budget", {"fixture": "DX", "objects": OBJECTS, "objectBytes": SIZE, - "logicalBytes": OBJECTS * SIZE, "taskSeconds": TASK_SECONDS, "sourceLimitReference": LIMIT_REFERENCE, + fleet.record("pre-registered-budget", {"fixture": "DX", "objects": count, "objectBytes": SIZE, + "logicalBytes": count * SIZE, "taskSeconds": TASK_SECONDS, "sourceLimitReference": policy_reference, "maxSnapshotFilesPerNode": 12288, "snapshotSecondsPerNode": 600, "verificationBatchItems": 128, - "missingIndices": list(range(0, OBJECTS, 64)), "taskScope": "objects/ prefix; one result per ACK", + "missingIndices": missing_indices, "taskScope": "frozen prefix scopes; one result per ACK", "faultLeaseMinutes": 30, "serviceRuntimeSeconds": 2700, "topologyLeaseMinutes": 45}) recipe = select(args.case_id)[0] - require(recipe["variants"] == list(branches), "DX executor omitted a source branch") + require(recipe["variants"] == list(source_branches or branches) and set(branches) <= set(recipe["variants"]), + "DX executor omitted or invented a source branch") result = {"topology": topology, "fixture": "DX", "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} active, probe = [], Probe(binding) try: fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) - expected = seed(Recorder(probe, directory / "fixture"), bucket, "DX", OBJECTS, [SIZE]) - wanted_pairs(expected) + recorder = Recorder(probe, directory / "fixture") + expected = fixture_builder(recorder, bucket) if fixture_builder else seed(recorder, bucket, "DX", count, [SIZE]) + wanted_pairs(expected, count=count) baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=True, page_size=127)) fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) fleet.stop() original = fleet.snapshot("physical-baseline", quiescent=True) - cohorts = map_cohorts(original, bucket, expected) - fleet.record("all-external-baseline", verify_all_parts(fleet, cohorts, "baseline-part-batch")) - targets = [cohorts[index] for index in range(0, OBJECTS, 64)] + cohorts = map_cohorts(original, bucket, expected, count=count) + fleet.record("all-external-baseline", verify_all_parts(fleet, cohorts, "baseline-part-batch", count=count)) + targets = [cohorts[index] for index in missing_indices] receipts = [] for index, target in enumerate(targets): params = {"disk": target["disk"], "relativePath": target["relativePath"], "kind": "missing", @@ -322,12 +339,14 @@ def run_topology(args, topology, nodes, *, observer=observe_task, branches=BRANC faulted = fleet.snapshot("physical-fault", quiescent=True) fleet.record("fault-proof", validate_missing_batch(original, faulted, targets, receipts)) fleet.start() + require(physical_content(faulted) == physical_content(fleet.snapshot("physical-before-trigger")), + "ATTRIBUTION_NOT_OBSERVED: DX fault changed before the admitted manual task") result["branches"] = observer(probe, fleet, binding, bucket, expected, targets, active, original=original) require(set(result["branches"]) == set(branches), "DX observer omitted or invented a source branch") fleet.stop() repaired = fleet.snapshot("physical-repaired-before-get", quiescent=True) fleet.record("physical-repair-proof", validate_repaired_physical(original, repaired, targets[0], repair_targets=targets)) - fleet.record("all-external-repair-proof", verify_all_parts(fleet, cohorts, "repaired-part-batch")) + fleet.record("all-external-repair-proof", verify_all_parts(fleet, cohorts, "repaired-part-batch", count=count)) fleet.start() for endpoint in binding["endpoints"]: inventory = fleet.record("s3-complete-after", {"endpoint": endpoint, **Probe(binding, endpoint).inventory( @@ -360,7 +379,7 @@ def run_topology(args, topology, nodes, *, observer=observe_task, branches=BRANC return result -def main(*, case_id=CASE_ID, observer=observe_task, branches=BRANCHES): +def main(*, case_id=CASE_ID, observer=observe_task, branches=BRANCHES, runner=run_topology): def interrupted(signum, frame): raise KeyboardInterrupt(f"Heal workflow interrupted by signal {signum}") signal.signal(signal.SIGTERM, interrupted) @@ -381,7 +400,7 @@ def interrupted(signum, frame): results = [] try: for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): - results.append(run_topology(args, topology, selected, observer=observer, branches=branches)) + results.append(runner(args, topology, selected, observer=observer, branches=branches)) if results[-1]["cleanup"] != "PASS": break finally: diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index dfee2f7a..8885320f 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -30,7 +30,10 @@ def runtime_config(profile): - require(profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled"}, "unsupported Heal runtime profile") + require(profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap"}, "unsupported Heal runtime profile") + if profile == "overlap": + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_OVERLAP_POLICY": "merge", + "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false"} if profile == "disabled": return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_ENABLED": "false"} if profile == "scanner": diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 7c3ff91d..8afd5713 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -131,7 +131,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 323c2d18..804071ee 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/overlap.py b/scripts/validation_heal/overlap.py new file mode 100644 index 00000000..0cedf036 --- /dev/null +++ b/scripts/validation_heal/overlap.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""A05: frozen merge policy and force replacement in real active windows.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +from urllib.parse import quote +import xml.etree.ElementTree as ET + +try: + from .heal_probe import canonical, payload, require + from .healthy import BASE_OPTIONS, physical_content + from .cursor import SIZE, TASK_SECONDS, run_topology as dx_topology, main as dx_main, wanted_pairs + from .stop import start, status, idle_fleet, validate_retired_stop + from .versions import public_version_id +except ImportError: + from heal_probe import canonical, payload, require + from healthy import BASE_OPTIONS, physical_content + from cursor import SIZE, TASK_SECONDS, run_topology as dx_topology, main as dx_main, wanted_pairs + from stop import start, status, idle_fleet, validate_retired_stop + from versions import public_version_id + + +CASE_ID = "heal.admin-overlap-and-force-start" +BRANCHES = ("same-scope", "parent-child", "disjoint", "force-start") +SCOPE_OBJECTS = 257 +OBJECTS = 2 * SCOPE_OBJECTS +PREFIXES = ("objects/a/child/", "objects/b/") +PRIMARY = "objects/a/" +MISSING = tuple(range(0, SCOPE_OBJECTS, 64)) +POLICY = {"candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", + "runtime": {"RUSTFS_HEAL_OVERLAP_POLICY": "merge", "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false"}, + "same-scope": "HTTP 200 and the original admitted token", + "parent-child": {"httpStatus": 409, "code": "OperationAborted", + "message": "heal request not admitted: admission=dropped, reason=overlapping_paths"}, + "disjoint": "HTTP 200 and an independent token; original remains running", + "force-start": "HTTP 200 and independent token; old outcome aborted/cancelled with partial coverage", + "references": ["crates/config/src/constants/heal.rs", "crates/heal/src/heal/manager.rs:admin_overlap_rejection", + "rustfs/src/admin/handlers/heal.rs:reject_heal_admission"], + "itemRetention": 1024, "diagnosticObjects": 128, + "retentionGapClaimed": False, "nativeCommitRaceClaimed": False} + + +def seed_scopes(recorder, bucket): + recorder.call("create_bucket", {"Bucket": bucket}) + expected = [] + for prefix in PREFIXES: + for index in range(SCOPE_OBJECTS): + key = f"{prefix}{index:04d}.bin" + body = payload(f"{bucket}/{key}/0", SIZE) + ack = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": "DX", "scope": prefix}, + "Tagging": "suite=heal&fixture=DX"}, body) + require(ack.get("VersionId", "null") == "null", "DX fixture unexpectedly enabled versioning") + expected.append({"key": key, "versionId": "null", "deleteMarker": False, "isLatest": True, + "payload": {"size": SIZE, "sha256": hashlib.sha256(body).hexdigest()}}) + wanted_pairs(expected, count=OBJECTS) + return expected + + +def scope_expected(expected, prefix): + selected = [entry for entry in expected if entry["key"].startswith(prefix)] + require(len(selected) == SCOPE_OBJECTS, "scope escaped its frozen 257-object fixture") + return selected + + +def counters(current): + value = (current.get("outcome") or {}).get("counters", {}) + for name in ("processed", "healed", "unchanged", "skipped", "failed", "unknown", "attemptFailures"): + require(type(value.get(name)) is int and value[name] >= 0, "scope omitted typed canonical counters") + require(all(value[name] == 0 for name in ("skipped", "failed", "unknown", "attemptFailures")) + and value.get("overflowed") is False + and value["processed"] == value["healed"] + value["unchanged"], + "scope counters hide failed, unknown or unresolved responsibility") + return value + + +def active_window(current): + value = counters(current) + return (current.get("summary") == "running" and current.get("outcome", {}).get("execution", {}).get("state") == "running" + and 0 < value["processed"] < SCOPE_OBJECTS) + + +def wait_window(probe, fleet, path, token): + deadline = time.monotonic() + TASK_SECONDS + while True: + current = status(probe, fleet, path, token, "overlap-running-window") + if active_window(current): + return current + require(current.get("summary") == "running", "NOT_OBSERVED: scope finished before a nonzero active window") + require(time.monotonic() < deadline, "NOT_OBSERVED: active scope exceeded the frozen window budget") + time.sleep(0.05) + + +def monotonic(before, after): + left, right = counters(before), counters(after) + require(all(right[name] >= left[name] for name in ("processed", "healed", "unchanged")), + "same-token scope counters moved backwards") + + +def overlap_rejection(observed): + require(observed.get("httpStatus") == 409 and not observed.get("transportError"), + "parent-child overlap violated the frozen HTTP contract") + body = observed.get("response") + fields = {element.tag.rsplit("}", 1)[-1]: element.text for element in ET.fromstring(body).iter()} if isinstance(body, str) else body + require(isinstance(fields, dict) and (fields.get("Code") or fields.get("code")) == "OperationAborted" + and (fields.get("Message") or fields.get("message")) == POLICY["parent-child"]["message"], + "parent-child overlap lost its exact structured reason") + return POLICY["parent-child"] + + +def scope_proof(current, bucket, expected, damaged, *, cancelled=False, healed=None, replacement=False): + wanted = wanted_pairs(expected, count=SCOPE_OBJECTS) + require(damaged and set(damaged) <= set(wanted), "scope repair plan escaped its exact ACK identities") + value = counters(current) + outcome = current.get("outcome") or {} + if cancelled: + require(current.get("summary") == "stopped" and outcome.get("execution") == {"state": "aborted", "reason": "cancelled"} + and outcome.get("coverage") == "partial" and 0 < value["processed"] < SCOPE_OBJECTS + and value["healed"] <= len(damaged), "forceStart falsely completed or lost the old partial responsibility") + else: + require(current.get("summary") == "finished" and outcome.get("execution") == {"state": "completed"} + and outcome.get("coverage") == "complete" and value["processed"] == SCOPE_OBJECTS + and type(healed) is int and value["healed"] == healed, + "scope failed complete coverage or its exact planned repair count") + receipts, incarnation, seen, dispositions = outcome.get("objects"), None, set(), {} + require(isinstance(receipts, list) and 0 < len(receipts) <= 128 + and type(outcome.get("objectsTruncated")) is bool + and (cancelled or outcome["objectsTruncated"] is True), "scope omitted bounded diagnostic truth") + for receipt in receipts: + identity = receipt.get("identity", {}) + pair = identity.get("object"), public_version_id(identity.get("versionId")) + require(pair in wanted and pair not in seen and identity.get("bucket") == bucket + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0 + and isinstance(identity.get("bucketIncarnationId"), str) and identity["bucketIncarnationId"], + "scope receipt crossed a bucket/key/version/set or omitted incarnation") + incarnation = incarnation or identity["bucketIncarnationId"] + require(incarnation == identity["bucketIncarnationId"], "scope mixed bucket incarnations") + state = receipt.get("disposition", {}).get("state") + require(state in ({"repaired", "verified_healthy", "cancelled"} if cancelled else {"repaired", "verified_healthy"}) + and (state != "repaired" or pair in damaged) + and (state != "verified_healthy" or pair not in damaged or replacement), + "scope diagnostic concealed an unresolved or false repair") + dispositions[pair[0]] = state + seen.add(pair) + return {"canonicalCounters": value, "bucketIncarnationId": incarnation, "diagnosticDispositions": dispositions, + "diagnosticObjectsAreCompleteManifest": False, "coverage": outcome["coverage"]} + + +def completed_scope(probe, fleet, path, token, bucket, expected, damaged, *, healed, replacement=False): + deadline = time.monotonic() + TASK_SECONDS + while True: + current = status(probe, fleet, path, token, "overlap-terminal-wait") + if current.get("summary") != "running": + break + counters(current) + require(time.monotonic() < deadline, "scope exceeded its frozen completion budget") + time.sleep(0.05) + proof = scope_proof(current, bucket, expected, damaged, healed=healed, replacement=replacement) + # These scopes fit actual item retention. Read every retained suffix even + # when the HTTP serializer independently trims an individual response. + wanted = wanted_pairs(expected, count=SCOPE_OBJECTS) + cursor, seen = 0, set() + while cursor < SCOPE_OBJECTS: + observed = fleet.record("overlap-complete-items", probe.admin(path, + [("clientToken", token), ("sinceSeq", str(cursor))], b"")) + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), "scope cursor failed") + page = observed["response"] + require(page.get("summary") == "finished" and page.get("minSeq") == 1 and page.get("nextSeq") == SCOPE_OBJECTS + 1 + and canonical(page.get("outcome")) == canonical(current["outcome"]), "terminal scope changed, lost items or crossed tokens") + items = page.get("items", []) + require(isinstance(items, list) and 0 < len(items) <= SCOPE_OBJECTS - cursor + and (len(items) == SCOPE_OBJECTS - cursor or page.get("truncated") is True), "scope silently omitted a retained suffix") + for item in items: + pair = item.get("object"), public_version_id(item.get("versionId")) + require(item.get("bucket") == bucket and pair in wanted and pair not in seen + and item.get("objectSize") == wanted[pair]["payload"]["size"], "scope item is duplicate, missing or outside ACKs") + seen.add(pair) + cursor += len(items) + require(time.monotonic() < deadline, "scope item suffix exceeded completion budget") + require(seen == set(wanted), "terminal scope missed a real object/version") + return current, {**proof, "completeItemIdentities": len(seen), "retentionGapClaimed": False} + + +def force_accounting(old, new, damaged): + require(old["bucketIncarnationId"] == new["bucketIncarnationId"] + and old["canonicalCounters"]["healed"] + new["canonicalCounters"]["healed"] == len(damaged), + "forceStart leaked or duplicated repair responsibility between its task identities") + for key in set(old["diagnosticDispositions"]) & set(new["diagnosticDispositions"]): + require(not (old["diagnosticDispositions"][key] == new["diagnosticDispositions"][key] == "repaired"), + "old and new tasks both committed a repair receipt for the same missing shard") + return {"independentTokens": True, "sameBucketIncarnation": True, "exactTotalRepairs": len(damaged)} + + +def observe_branch(branch, probe, fleet, binding, bucket, expected, targets, active, *, original): + fleet.record("frozen-overlap-policy", POLICY) + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + PRIMARY + primary = scope_expected(expected, PRIMARY) + damaged = {(entry["key"], entry["versionId"]) for entry in targets if entry["key"].startswith(PRIMARY)} + old = start(probe, fleet, path, active) + before = wait_window(probe, fleet, path, old) + proof = {"oldTaskId": old, "runningBeforeRequest": counters(before), "policy": "merge"} + if branch in {"same-scope", "parent-child"}: + request_path = path if branch == "same-scope" else path + "child/" + observed = fleet.record("overlap-request", probe.admin(request_path, [], json.dumps({**BASE_OPTIONS, "recursive": True}).encode())) + if branch == "same-scope": + require(observed.get("httpStatus") == 200 and observed.get("response", {}).get("clientToken") == old, + "same-scope start did not reuse the active token under merge policy") + else: + proof["typedRejection"] = overlap_rejection(observed) + after = status(probe, fleet, path, old, "overlap-window-after-request") + require(active_window(after), "NOT_OBSERVED: overlap request was not bracketed by the same running task") + monotonic(before, after) + proof["runningAfterRequest"] = counters(after) + terminal, proof["completeResponsibility"] = completed_scope(probe, fleet, path, old, bucket, primary, damaged, healed=len(damaged)) + active.remove((path, old)) + elif branch == "disjoint": + other_path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + PREFIXES[1] + new = start(probe, fleet, other_path, active) + require(new != old, "disjoint scope reused another scope's token") + after = status(probe, fleet, path, old, "original-after-disjoint-start") + require(active_window(after), "NOT_OBSERVED: original task finished before the disjoint admission was bracketed") + monotonic(before, after) + wait_window(probe, fleet, other_path, new) + other = scope_expected(expected, PREFIXES[1]) + other_damaged = {(entry["key"], entry["versionId"]) for entry in targets if entry["key"].startswith(PREFIXES[1])} + terminal, proof["completeResponsibility"] = completed_scope(probe, fleet, path, old, bucket, primary, damaged, healed=len(damaged)) + other_terminal, proof["disjointResponsibility"] = completed_scope(probe, fleet, other_path, new, bucket, other, other_damaged, healed=len(other_damaged)) + require(proof["completeResponsibility"]["bucketIncarnationId"] == proof["disjointResponsibility"]["bucketIncarnationId"], + "disjoint tasks crossed bucket incarnation") + active.remove((path, old)); active.remove((other_path, new)) + proof.update(newTaskId=new, runningAfterRequest=counters(after)) + else: + require(branch == "force-start", "unregistered overlap branch") + new = start(probe, fleet, path, active, force=True) + require(new != old, "forceStart reused the original token") + first_old = status(probe, fleet, path, old, "force-old-initial-cancellation") + require(first_old.get("summary") != "finished", "NOT_OBSERVED: original task completed before forceStart could cancel it") + initial = scope_proof(first_old, bucket, primary, damaged, cancelled=True) + monotonic(before, first_old) + new_before = wait_window(probe, fleet, path, new) + retired = fleet.record("force-old-stop", probe.admin(path, [("clientToken", old), ("forceStop", "true")], b"")) + proof["oldStopIsolation"] = validate_retired_stop(retired, old) + new_after = status(probe, fleet, path, new, "force-new-after-old-stop") + require(new_after.get("summary") != "stopped", "old token stopped the replacement task") + require(active_window(new_after), "NOT_OBSERVED: old stop was not bracketed by the same running replacement") + monotonic(new_before, new_after) + # The old worker can refine its cancelled report while retiring. Wait + # for both actual workers to drain before freezing final responsibility. + deadline = time.monotonic() + TASK_SECONDS + while status(probe, fleet, path, new, "force-new-drain").get("summary") == "running": + require(time.monotonic() < deadline, "replacement task exceeded completion budget") + time.sleep(0.05) + idle_fleet(fleet, binding) + old_final = status(probe, fleet, path, old, "force-old-settled") + settled = scope_proof(old_final, bucket, primary, damaged, cancelled=True) + require(settled["bucketIncarnationId"] == initial["bucketIncarnationId"] + and all(settled["canonicalCounters"][name] >= initial["canonicalCounters"][name] + for name in ("processed", "healed", "unchanged")), "cancelled refinement changed identity or moved backwards") + terminal, new_proof = completed_scope(probe, fleet, path, new, bucket, primary, damaged, + healed=len(damaged) - settled["canonicalCounters"]["healed"], replacement=True) + proof.update(newTaskId=new, initialCancellation=initial, settledCancellation=settled, + newRunningBeforeOldStop=counters(new_before), newRunningAfterOldStop=counters(new_after), + completeResponsibility=new_proof, responsibilityAccounting=force_accounting(settled, new_proof, damaged)) + active.remove((path, old)); active.remove((path, new)) + proof["allPeerDrain"] = idle_fleet(fleet, binding) + quiet = [] + for index in range(3): + quiet.append(fleet.snapshot("overlap-live-quiet")) + stable = status(probe, fleet, path, old, "overlap-old-quiet-status") + require(canonical(stable.get("outcome")) == canonical((old_final if branch == "force-start" else terminal)["outcome"]), + "retired old task rewrote responsibility after actual drain") + if branch == "force-start": + require(canonical(status(probe, fleet, path, new, "overlap-new-quiet-status").get("outcome")) == canonical(terminal["outcome"]), + "old worker overwrote the replacement outcome after its completion") + elif branch == "disjoint": + require(canonical(status(probe, fleet, other_path, new, "overlap-disjoint-quiet-status").get("outcome")) == canonical(other_terminal["outcome"]), + "retired task rewrote a disjoint scope's responsibility") + if index < 2: + time.sleep(5) + require(all(physical_content(value) == physical_content(quiet[0]) + and all((value[node]["pid"], value[node]["unit"]) == (quiet[0][node]["pid"], quiet[0][node]["unit"]) + for node in fleet.nodes) for value in quiet[1:]), "retired task kept writing or service changed during live quiet proof") + return {branch: {"verdict": "PASS", **proof, "liveQuietSamples": 3, "betweenSamplesSeconds": 5, + "completePhysicalAndS3ProofRequired": True, "nativeCommitRaceClaimed": False}} + + +def run_topology(args, topology, nodes, *, observer=None, branches=BRANCHES): + require(tuple(branches) == BRANCHES, "overlap executor omitted source branches") + results = [] + for branch in branches: + branch_args = argparse.Namespace(**vars(args)) + branch_args.evidence = args.evidence / branch + branch_args.evidence.mkdir(exist_ok=True) + indices = list(MISSING) + ([SCOPE_OBJECTS + index for index in MISSING] if branch == "disjoint" else []) + result = dx_topology(branch_args, topology, nodes, + observer=lambda *values, **options: observe_branch(branch, *values, **options), branches=(branch,), + source_branches=BRANCHES, count=OBJECTS, fixture_builder=seed_scopes, missing_indices=indices, + runtime_profile="overlap", policy_reference=POLICY) + results.append({"branch": branch, **result}) + if result["cleanup"] != "PASS": + break + return {"topology": topology, "instances": results, "branches": {entry["branch"]: entry for entry in results}, + "verdict": "PASS" if len(results) == len(branches) and all(entry["verdict"] == "PASS" for entry in results) else "FAIL", + "cleanup": "PASS" if results and all(entry["cleanup"] == "PASS" for entry in results) else "FAIL"} + + +if __name__ == "__main__": + try: + dx_main(case_id=CASE_ID, branches=BRANCHES, runner=run_topology) + except Exception as exc: + print(f"Heal overlap validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index a7e7a798..0ead47a1 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -24,6 +24,7 @@ case "${case_id}" in heal.incremental-cursor-and-truncation) driver=cursor.py ;; heal.admin-stop-and-stale-token) driver=stop.py ;; heal.cancelled-task-restart) driver=cancel_restart.py ;; + heal.admin-overlap-and-force-start) driver=overlap.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; diff --git a/test_validation_heal.py b/test_validation_heal.py index 3619b699..e58bc2c2 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -50,6 +50,7 @@ cancelled_physical, idle_fleet as stop_idle_fleet, BRANCHES as STOP_BRANCHES) from scripts.validation_heal.cancel_restart import validate_journal, BRANCHES as CANCEL_RESTART_BRANCHES from scripts.validation_heal.fleet_node import task_records as node_task_records +from scripts.validation_heal import overlap import errno import stat @@ -2117,5 +2118,124 @@ def test_restart_preserves_every_source_branch(self): self.assertEqual(list(CANCEL_RESTART_BRANCHES), select("F06")[0]["variants"]) +class HealOverlapOracleTest(unittest.TestCase): + bucket = "rv-heal-overlap-unit" + + def expected(self): + return [{"key": f"objects/a/child/{index:04d}.bin", "versionId": "null", "deleteMarker": False, + "payload": {"size": CURSOR_SIZE}} for index in range(overlap.SCOPE_OBJECTS)] + + def damaged(self): + return {(self.expected()[index]["key"], "null") for index in overlap.MISSING} + + def state(self, *, cancelled=False, replacement=False, healed=None): + total = 100 if cancelled else overlap.SCOPE_OBJECTS + repaired = {0, 64} if cancelled else set(overlap.MISSING) + if replacement: repaired -= {0, 64} + return {"summary": "stopped" if cancelled else "finished", "outcome": { + "execution": {"state": "aborted", "reason": "cancelled"} if cancelled else {"state": "completed"}, + "coverage": "partial" if cancelled else "complete", "objectsTruncated": total > 128, + "objects": [{"identity": {"bucket": self.bucket, "object": self.expected()[index]["key"], + "versionId": None, "poolIndex": 0, "setIndex": 0, "bucketIncarnationId": "incarnation"}, + "disposition": {"state": "repaired" if index in repaired else "verified_healthy"}} + for index in range(min(total, 128))], + "counters": {"processed": total, "healed": len(repaired) if healed is None else healed, + "unchanged": total - (len(repaired) if healed is None else healed), + "skipped": 0, "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False}}} + + def proof(self, current, **kwargs): + return overlap.scope_proof(current, self.bucket, self.expected(), self.damaged(), **kwargs) + + def test_frozen_policy_rejects_arbitrary_http_and_reason(self): + good = {"httpStatus": 409, "response": {"Code": "OperationAborted", "Message": overlap.POLICY["parent-child"]["message"]}} + self.assertEqual(overlap.overlap_rejection(good)["code"], "OperationAborted") + for changed in ({**good, "httpStatus": 200}, {**good, "transportError": "timeout"}, + {**good, "response": {"Code": "OperationAborted", "Message": "already_running"}}): + with self.assertRaises(ValueError): overlap.overlap_rejection(changed) + + def test_incomplete_or_cross_scope_receipts_and_unknowns_never_pass(self): + self.proof(self.state(), healed=5) + for kind in ("count", "unknown", "failed", "attempt", "coverage", "diagnostic", "key", "version", "incarnation", "role", "duplicate"): + current = self.state() + value = current["outcome"] + if kind == "count": value["counters"]["healed"] = 4 + if kind in {"unknown", "failed"}: value["counters"][kind] = 1 + if kind == "attempt": value["counters"]["attemptFailures"] = 1 + if kind == "coverage": value["coverage"] = "partial" + if kind == "diagnostic": value["objectsTruncated"] = False + if kind == "key": value["objects"][0]["identity"]["object"] = "objects/b/0000.bin" + if kind == "version": value["objects"][0]["identity"]["versionId"] = "other-version" + if kind == "incarnation": value["objects"][0]["identity"]["bucketIncarnationId"] = "old-bucket" + if kind == "role": value["objects"][1]["disposition"]["state"] = "repaired" + if kind == "duplicate": value["objects"][1] = copy.deepcopy(value["objects"][0]) + with self.subTest(kind=kind), self.assertRaises(ValueError): self.proof(current, healed=5) + + def test_force_accounting_preserves_partial_truth_and_rejects_double_repair(self): + old = self.proof(self.state(cancelled=True), cancelled=True) + new = self.proof(self.state(replacement=True), healed=3, replacement=True) + self.assertEqual(overlap.force_accounting(old, new, self.damaged())["exactTotalRepairs"], 5) + for kind in ("count", "incarnation", "duplicate"): + changed = copy.deepcopy(new) + if kind == "count": changed["canonicalCounters"]["healed"] = 4 + if kind == "incarnation": changed["bucketIncarnationId"] = "new-bucket" + if kind == "duplicate": changed["diagnosticDispositions"][self.expected()[0]["key"]] = "repaired" + with self.subTest(kind=kind), self.assertRaises(ValueError): overlap.force_accounting(old, changed, self.damaged()) + with self.assertRaises(ValueError): self.proof(self.state(), cancelled=True) + with self.assertRaises(ValueError): self.proof(self.state(cancelled=True), healed=2) + + def test_terminal_suffix_checks_every_ack_and_never_infers_a_retention_gap(self): + current = self.state() + pages = [] + for left, right in ((0, 100), (100, 257)): + pages.append({**copy.deepcopy(current), "nextSeq": 258, "minSeq": 1, "truncated": right < 257, + "items": [{"bucket": self.bucket, "object": entry["key"], "versionId": "", "objectSize": CURSOR_SIZE} + for entry in self.expected()[left:right]]}) + client = types.SimpleNamespace(admin=lambda *args: {"httpStatus": 200, "response": pages.pop(0)}) + fleet = types.SimpleNamespace(record=lambda label, value: value) + with patch.object(overlap, "status", return_value=current): + _, proof = overlap.completed_scope(client, fleet, "/scope/", "task", self.bucket, self.expected(), self.damaged(), healed=5) + self.assertEqual(proof["completeItemIdentities"], 257) + self.assertFalse(proof["retentionGapClaimed"]) + for kind in ("duplicate", "gap", "empty", "changed-outcome"): + page = {**copy.deepcopy(current), "nextSeq": 258, "minSeq": 1, "items": [ + {"bucket": self.bucket, "object": entry["key"], "versionId": "", "objectSize": CURSOR_SIZE} for entry in self.expected()]} + if kind == "duplicate": page["items"][1] = copy.deepcopy(page["items"][0]) + if kind == "gap": page["minSeq"] = 2 + if kind == "empty": page["items"] = [] + if kind == "changed-outcome": page["outcome"]["counters"]["unknown"] = 1 + client.admin = lambda *args: {"httpStatus": 200, "response": page} + with self.subTest(kind=kind), patch.object(overlap, "status", return_value=current), self.assertRaises(ValueError): + overlap.completed_scope(client, fleet, "/scope/", "task", self.bucket, self.expected(), self.damaged(), healed=5) + + def test_finished_window_and_same_token_counter_rollback_cannot_pass(self): + self.assertFalse(overlap.active_window(self.state())) + before = self.state(cancelled=True) + after = copy.deepcopy(before) + after["outcome"]["counters"].update(processed=99, unchanged=97) + with self.assertRaisesRegex(ValueError, "backwards"): overlap.monotonic(before, after) + with patch.object(overlap, "status", return_value=self.state()), self.assertRaisesRegex(ValueError, "NOT_OBSERVED"): + overlap.wait_window(None, None, "/scope/", "task") + + def test_all_eight_source_instances_and_cleanup_failure_stop_are_real_dispatches(self): + with tempfile.TemporaryDirectory() as folder: + args = types.SimpleNamespace(evidence=Path(folder)) + calls = [] + def topology(params, topology, nodes, **options): + calls.append((topology, options["branches"][0], options["missing_indices"])) + return {"verdict": "PASS", "cleanup": "PASS"} + with patch.object(overlap, "dx_topology", side_effect=topology): + for name in ("H1", "H2"): + self.assertEqual(overlap.run_topology(args, name, ["node1"])["verdict"], "PASS") + self.assertEqual([(name, branch) for name, branch, _ in calls], [(name, branch) for name in ("H1", "H2") for branch in overlap.BRANCHES]) + self.assertTrue(all(len(indices) == (10 if branch == "disjoint" else 5) for _, branch, indices in calls)) + with patch.object(overlap, "dx_topology", return_value={"verdict": "FAIL", "cleanup": "FAIL"}) as operation: + self.assertEqual(overlap.run_topology(args, "H1", ["node1"])["cleanup"], "FAIL") + self.assertEqual(operation.call_count, 1) + self.assertEqual(list(overlap.BRANCHES), select("A05")[0]["variants"]) + self.assertEqual(runtime_config("overlap")["RUSTFS_HEAL_OVERLAP_POLICY"], "merge") + # Parameterizing A05 physical fixtures must not weaken J03 retention. + with self.assertRaises(ValueError): Window(self.bucket, self.expected(), "task", "node1", "epoch") + + if __name__ == "__main__": unittest.main() From 661362d32174f588e165bfe53832407885eca5e8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 03:10:52 +0800 Subject: [PATCH 064/343] test(validation): migrate original Heal coordinator restart recovery --- docs/validation-heal-contract.md | 2 +- docs/validation-migration-map.md | 24 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/cursor.py | 12 +- scripts/validation_heal/fleet_node.py | 87 +++++++- scripts/validation_heal/healthy.py | 12 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/root_restart.py | 268 ++++++++++++++++++++++++ scripts/validation_heal/run_case.sh | 1 + test_validation_heal.py | 186 +++++++++++++++- 10 files changed, 584 insertions(+), 12 deletions(-) create mode 100644 scripts/validation_heal/root_restart.py diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index b09fb739..ae4f4197 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -34 个用例已接入自动分发：A01、A03–A08、B02、B05、B07、B09、C01–C05、C08、D01–D07、D09、D10、E01–E04、F06、J01、J03、J04。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。其余 50 项保留源 operator-assisted 方式。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 +35 个用例已接入自动分发：A01、A03–A08、B02、B05、B07、B09、C01–C05、C08、D01–D07、D09、D10、E01–E04、F01、F06、J01、J03、J04。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。其余 49 项保留源 operator-assisted 方式。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 9ab5b23f..192a43dc 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 108 currently dispatchable IDs include all 74 non-Heal cases and thirty-four Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. +case evidence. The 109 currently dispatchable IDs include all 74 non-Heal cases and thirty-five Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 50 Heal cases remain +The other 49 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -607,3 +607,23 @@ with the same processes. Complete same-role shard bytes, streaming checksums and all-peer full S3 manifests remain mandatory before PASS. Local scope/error/window/duplicate-repair/item-omission/cleanup-stop negatives passed. Full live evidence remains pending. + +F01 `heal.root-graceful-coordinator-restart` now dispatches both source +branches on H1/H2 with the same fixed DX physical/S3 oracles. The coordinator +comes from the unique admitted physical root intent, rather than a guessed +node alias. Actual nonzero task progress and both restored/pending target +shards must precede interruption. Linux pidfd binds pause, SIGTERM and resume +to the original process birth; the original run-owned unit retains actual +normal-exit-zero evidence. All peers retain their original process except +that coordinator. The exact schema-3 root intent stays on the same drive, +with original task/scope/options/priority/admission time and a finite +remaining timeout that cannot increase. Pending peer writes must settle +before same-unit/environment/volumes/candidate restart. The original token +must recover, report complete exact responsibility and repair the remaining +physical defects; no new START compensates for lost responsibility. Runnable +intent must retire on every drive, while completed terminal/report remain on +the original owner. A second coordinator restart preserves their bytes, +completed outcome and all user files and remains idle. This proves durable +admission replay, and makes no cursor-resume or record-disk failover claim. +Local owner/budget/process-birth/exit/old-token/journal negatives passed; +full live evidence remains pending. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 486a0d4c..8da7ea33 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/cursor.py b/scripts/validation_heal/cursor.py index 6b4578e3..0f368fb3 100644 --- a/scripts/validation_heal/cursor.py +++ b/scripts/validation_heal/cursor.py @@ -125,7 +125,7 @@ def consume(self, status, cursor, *, token, endpoint, epoch): "exclusiveNextCursor": self.cursor, "taskId": token, "endpoint": endpoint, "deploymentEpoch": epoch} -def terminal_proof(status, bucket, expected, repaired, window): +def complete_outcome(status, bucket, expected, repaired): wanted = wanted_pairs(expected) require(repaired and set(repaired) <= set(wanted), "cursor repair plan is empty or escapes ACKs") outcome = status.get("outcome") or {} @@ -154,12 +154,16 @@ def terminal_proof(status, bucket, expected, repaired, window): require(receipt.get("disposition", {}).get("state") == ("repaired" if pair in repaired else "verified_healthy"), "bounded diagnostics claimed false repair, unknown responsibility or incomplete integrity") seen.add(pair) + return {"canonicalCounters": counters, "bucketIncarnationId": incarnation, + "diagnosticObjects": len(receipts), "diagnosticObjectsAreCompleteManifest": False} + + +def terminal_proof(status, bucket, expected, repaired, window): + proof = complete_outcome(status, bucket, expected, repaired) retained = window.check(status, 0) require(retained["retentionGap"] and retained["truncated"] and retained["nextSeq"] == OBJECTS + 1, "terminal task never actually exceeded its retained item window") - return {"canonicalCounters": counters, "bucketIncarnationId": incarnation, - "diagnosticObjects": len(receipts), "diagnosticObjectsAreCompleteManifest": False, - "retainedWindow": {name: value for name, value in retained.items() if name != "pairs"}} + return {**proof, "retainedWindow": {name: value for name, value in retained.items() if name != "pairs"}} def map_cohorts(snapshot, bucket, expected, *, count=OBJECTS): diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 8885320f..4d185cf2 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -13,10 +13,12 @@ from datetime import datetime, timedelta, timezone from pathlib import Path import re +import signal import shutil import socket import stat import subprocess +import time import uuid @@ -30,7 +32,9 @@ def runtime_config(profile): - require(profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap"}, "unsupported Heal runtime profile") + require(profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart"}, "unsupported Heal runtime profile") + if profile == "root-restart": + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false"} if profile == "overlap": return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_OVERLAP_POLICY": "merge", "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false"} @@ -208,6 +212,7 @@ def start(request): "--property=KillMode=control-group", "--property=TimeoutStopSec=30", "--property=RuntimeMaxSec=" + str(resource_limits(owner.get("resourceProfile", "standard"))["runtimeMaxSeconds"]), "--property=MemoryMax=4G", + *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") == "root-restart" else []), "--property=WorkingDirectory=" + str(state), "--property=EnvironmentFile=" + str(state / "service.env"), *gate_module().start_properties(gate_module().checked_gate(state, roots)), str(state / "rustfs"), *owner["volumes"]) @@ -236,6 +241,85 @@ def stop(request): return {"unit": owner["unit"], "quiescent": True, "service": stopped(owner)} +PROCESS_FIELDS = ("pid", "startTicks", "bootId", "unit", "executable", "binarySha256", "mountNamespace", + "commandSha256", "environmentFileSha256", "settings") + + +def process(request): + state, _, owner = owned(request["runId"]) + fields = ("MainPID", "ActiveState", "SubState", "ExecMainCode", "ExecMainStatus", "Result") + service = dict(line.split("=", 1) for line in command("systemctl", "show", owner["unit"], + *(part for name in fields for part in ("-p", name)), check=False).stdout.splitlines() if "=" in line) + pid = int(service.get("MainPID", "0")) + base = {"pid": pid, "unit": owner["unit"], "service": service, + "bootId": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "binarySha256": digest(state / "rustfs"), "environmentFileSha256": digest(state / "service.env")} + require(base["binarySha256"] == owner["binarySha256"], "process candidate changed") + if pid == 0: + return base + require(pid > 1, "invalid owned process PID") + root = Path(f"/proc/{pid}") + before = (root / "stat").read_text().rpartition(") ")[2].split() + require(len(before) >= 20 and before[19].isdigit(), "invalid process birth record") + executable = (root / "exe").resolve(strict=True) + require(executable == state / "rustfs" and digest(executable) == owner["binarySha256"], "signal target is not the owned candidate") + environment = dict(item.split("=", 1) for item in (root / "environ").read_bytes().decode().split("\0") if "=" in item) + settings = {**runtime_config(owner.get("runtimeProfile", "manual")), "RUSTFS_HEAL_TASK_TIMEOUT_SECS": "300"} + require(all(environment.get(key) == value for key, value in settings.items()), "owned process runtime changed") + result = {**base, "state": before[0], "startTicks": int(before[19]), "executable": str(executable), + "mountNamespace": os.readlink(root / "ns/mnt"), "commandSha256": hashlib.sha256((root / "cmdline").read_bytes()).hexdigest(), + "settings": settings} + after = (root / "stat").read_text().rpartition(") ")[2].split() + require(after[19] == before[19], "process identity changed while collecting evidence") + return result + + +def pidfd_signal(request, signum): + expected = request.get("expected") + require(isinstance(expected, dict) and all(name in expected for name in PROCESS_FIELDS) + and type(expected["pid"]) is int and expected["pid"] > 1, "signal lacks an exact process birth identity") + require(hasattr(os, "pidfd_open") and hasattr(signal, "pidfd_send_signal"), "safe process control requires Linux pidfd support") + descriptor = os.pidfd_open(expected["pid"], 0) + try: + current = process(request) + require(all(current.get(name) == expected[name] for name in PROCESS_FIELDS), "stale process identity; no signal sent") + signal.pidfd_send_signal(descriptor, signum, None, 0) + finally: + os.close(descriptor) + return current + + +def process_signal(request): + name = request.get("signal") + require(name in {"SIGSTOP", "SIGCONT"}, "only owned-process pause/resume signals are allowed") + before = pidfd_signal(request, getattr(signal, name)) + return {"signal": name, "before": before, "pidfdBound": True, "platformEventIssued": False} + + +def process_graceful(request): + require(owned(request["runId"])[2].get("runtimeProfile") == "root-restart" + and process(request).get("state") == "T", "graceful observation requires the paused run-owned root service") + before = pidfd_signal(request, signal.SIGTERM) + # A paused process must receive CONT to handle TERM. Use the same pidfd + # and exact birth identity again; a recycled PID never receives a signal. + pidfd_signal(request, signal.SIGCONT) + deadline = time.monotonic() + 60 + while True: + after = process(request) + if after["pid"] == 0: + break + require(all(after.get(name) == before[name] for name in PROCESS_FIELDS), "another process replaced the graceful target") + require(time.monotonic() < deadline, "owned coordinator did not exit gracefully in 60 seconds") + time.sleep(0.1) + service = after["service"] + require(after["bootId"] == before["bootId"] and after["binarySha256"] == before["binarySha256"] + and after["environmentFileSha256"] == before["environmentFileSha256"] + and service.get("ExecMainCode") == "1" and service.get("ExecMainStatus") == "0" + and service.get("Result") == "success", "process has no witnessed normal exit zero; signal/timeout exits are not graceful") + return {"before": before, "after": after, "signal": "SIGTERM", "actualExitCode": 0, + "pidfdBound": True, "platformEventIssued": False} + + def fault_module(): # SSH requests carry this exact repository module in memory; local imports # support contract tests without creating an importable file on a node. @@ -537,5 +621,6 @@ def main(request): "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part, "verify-parts": verify_parts, "verify-inline": verify_inline, "drive-gate": drive_gate, "drive-gate-probe": drive_gate_probe} actions["task-records"] = task_records + actions.update({"process": process, "process-signal": process_signal, "process-graceful": process_graceful}) require(request.get("action") in actions, "unsupported node operation") print(json.dumps(actions[request["action"]](request))) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 8afd5713..8810b5c4 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -131,7 +131,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -142,6 +142,7 @@ def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_prof self.faults = [] self.drive_gates = set() self.secret_values = set() + self.frozen_processes = {} def record(self, label, value): self.sequence += 1 @@ -250,6 +251,15 @@ def wait_ready(self): def cleanup(self): failures = [] + for node, identity in tuple(self.frozen_processes.items()): + try: + current = self.remote(node, "process") + if current["pid"]: + self.record("resume-for-finalization", {"node": node, **self.remote(node, "process-signal", + signal="SIGCONT", expected=identity)}) + del self.frozen_processes[node] + except Exception as exc: + failures.append({"node": node, "error": str(exc), "state": "DIRTY", "namespacesRetained": True}) if self.drive_gates: try: self.stop() diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 804071ee..dd76d8ef 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/root_restart.py b/scripts/validation_heal/root_restart.py new file mode 100644 index 00000000..6918a479 --- /dev/null +++ b/scripts/validation_heal/root_restart.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""F01: original root responsibility, owner and timeout across graceful restart.""" + +from __future__ import annotations + +import argparse +import copy +import json +import sys +import time +import uuid + +try: + from .heal_probe import Probe, canonical, require + from .healthy import BASE_OPTIONS, physical_content + from .cursor import OBJECTS, TASK_SECONDS, complete_outcome, wanted_pairs, run_topology as dx_topology, main as dx_main + from .shards import files, validate_repaired_physical + from .stop import status, wait_partial, idle_fleet, validate_retired_stop + from .cancel_restart import REFERENCE as JOURNAL_REFERENCE + from .fleet_node import PROCESS_FIELDS + from .versions import public_version_id +except ImportError: + from heal_probe import Probe, canonical, require + from healthy import BASE_OPTIONS, physical_content + from cursor import OBJECTS, TASK_SECONDS, complete_outcome, wanted_pairs, run_topology as dx_topology, main as dx_main + from shards import files, validate_repaired_physical + from stop import status, wait_partial, idle_fleet, validate_retired_stop + from cancel_restart import REFERENCE as JOURNAL_REFERENCE + from fleet_node import PROCESS_FIELDS + from versions import public_version_id + + +CASE_ID = "heal.root-graceful-coordinator-restart" +BRANCHES = ("root-partial-graceful", "same-record-disk-resume") +PATH = "/rustfs/admin/v3/heal/" +TIMEOUT_SECONDS = 300 +OPTIONS = {**BASE_OPTIONS, "recursive": True, "pool": None, "set": None} +REFERENCE = {**JOURNAL_REFERENCE, "intentSchema": 3, "rootHealType": {"type": "cluster"}, + "taskTimeoutSeconds": TIMEOUT_SECONDS, "recovery": "original admission replay; no cursor-resume or failover claim", + "processBoundary": "same unit/env/volumes/candidate and boot; coordinator only; pidfd SIGTERM, actual normal exit zero"} + + +def records(fleet, token, label, *, stopped_node=None): + return fleet.record(label, {node: fleet.remote(node, "task-records", taskId=token, + quiescent=node == stopped_node) for node in fleet.nodes}) + + +def owned_records(observed, token): + require(isinstance(token, str) and str(uuid.UUID(token)) == token, "root token is not a canonical task UUID") + present = {} + require(observed and len(observed) in {1, 4}, "root journal omitted bound physical nodes") + for node, value in observed.items(): + require(value.get("readOnly") is True and value.get("taskId") == token + and isinstance(value.get("records"), list) and len(value["records"]) == 12, "root journal omitted a disk/name witness") + seen = set() + for entry in value["records"]: + pair = entry.get("disk"), entry.get("kind") + require(type(pair[0]) is int and 0 <= pair[0] < 4 and pair[1] in {"intent", "terminal", "report"} + and pair not in seen and type(entry.get("present")) is bool, "root journal has duplicate or invalid physical identities") + seen.add(pair) + if entry["present"]: + require(pair[1] not in present, "root task has multiple durable owners") + present[pair[1]] = {"node": node, **entry} + return present + + +def root_intent(observed, token, coordinator): + present = owned_records(observed, token) + require(set(present) == {"intent"}, "NOT_OBSERVED: original runnable root intent is unavailable or already terminal") + entry = present["intent"] + value = entry.get("value", {}) + require(entry["node"] == coordinator and type(value.get("schema")) is int and value["schema"] == 3 + and value.get("task_id") == token and value.get("heal_type") == {"type": "cluster"} + and not value.get("bucket_incarnation_id") and value.get("priority") == "High" + and type(value.get("retry_attempts")) is int and value["retry_attempts"] == 0, + "root intent changed coordinator/scope/identity/priority or contains a retry") + options = value.get("options", {}) + required = {"scan_mode": 2, "remove_corrupted": False, "recreate_missing": False, + "update_parity": False, "recursive": True, "dry_run": False, "no_lock": False, + "pool_index": None, "set_index": None} + require(all(type(options.get(name)) is type(expect) and options[name] == expect for name, expect in required.items()), + "root intent lost its exact admitted options/selectors") + require(isinstance(value.get("created_at"), dict), "root intent omitted original admission time") + return entry + + +def duration_ns(value): + require(isinstance(value, dict) and set(value) == {"secs", "nanos"} + and type(value["secs"]) is int and 0 <= value["secs"] < 2**64 + and type(value["nanos"]) is int and 0 <= value["nanos"] < 1_000_000_000, "invalid persisted timeout duration") + return value["secs"] * 1_000_000_000 + value["nanos"] + + +def same_owner(before, after): + require(all(before[name] == after[name] for name in ("node", "disk", "relativePath")), "root record moved to a different physical owner") + left, right = copy.deepcopy(before["value"]), copy.deepcopy(after["value"]) + old_timeout, new_timeout = left["options"].pop("timeout", None), right["options"].pop("timeout", None) + require(canonical(left) == canonical(right), "restart changed the original root responsibility or non-timeout options") + remaining = duration_ns(new_timeout) + maximum = duration_ns(old_timeout) if old_timeout is not None else TIMEOUT_SECONDS * 1_000_000_000 + require(0 < remaining <= maximum <= TIMEOUT_SECONDS * 1_000_000_000, "graceful restart removed, refunded or exhausted the original task budget") + return {"samePhysicalOwner": True, "nonTimeoutIntentUnchanged": True, "beforeTimeoutNs": maximum, + "checkpointRemainingTimeoutNs": remaining, "budgetNotRefunded": True} + + +def partial_physical(original, current, targets): + actual, restored, remaining = files(current), [], [] + for target in targets: + entry = actual.get((target["node"], target["disk"], target["relativePath"])) + if entry is None: + remaining.append(target) + else: + require(entry["sha256"] == target["sha256"] and entry["size"] == target["size"], "partial root repair committed incorrect shard bytes") + restored.append(target) + require(restored and remaining, "NOT_OBSERVED: actual root repair was not partial at the interruption boundary") + expected = copy.deepcopy(original) + missing = {(entry["node"], entry["disk"], entry["relativePath"]) for entry in remaining} + for node, value in expected.items(): + value["files"] = [entry for entry in value["files"] if (node, entry["disk"], entry["path"]) not in missing] + proof = validate_repaired_physical(expected, current, restored[0], repair_targets=restored) + return remaining, {**proof, "actualRestoredParts": len(restored), "actualPendingParts": len(remaining), "backupRescueUsed": False} + + +def restart_identity(before, after, coordinator): + require(set(before) == set(after) and coordinator in before, "restart omitted an original peer") + for node, old in before.items(): + new = after[node] + require(all(new.get(name) == old.get(name) for name in PROCESS_FIELDS if name not in {"pid", "startTicks"}), + "restart changed boot, unit, namespace, candidate, command or environment") + require(type(new.get("pid")) is int and new["pid"] > 1 and type(new.get("startTicks")) is int, "restart has no actual new process") + if node == coordinator: + require((new["pid"], new["startTicks"]) != (old["pid"], old["startTicks"]), "coordinator was not restarted") + else: + require((new["pid"], new["startTicks"]) == (old["pid"], old["startTicks"]), "non-coordinator peer restarted or changed process") + return {"sameUnitEnvironmentVolumesAndCandidate": True, "sameBootAndNamespace": True, "onlyCoordinatorProcessChanged": True} + + +def recovery_status(probe, fleet, token): + deadline = time.monotonic() + 15 + while True: + observed = fleet.record("original-token-recovery-query", probe.admin(PATH, [("clientToken", token)], b"")) + if observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict): + return observed["response"] + # The exact absent-token error is an observation during admission + # replay, never completion evidence. All other failures remain errors. + validate_retired_stop(observed, token) + require(time.monotonic() < deadline, "original root token stayed unavailable beyond the 15-second recovery budget") + time.sleep(0.1) + + +def root_terminal(observed, token, owner, current, bucket, expected): + wanted = wanted_pairs(expected) + present = owned_records(observed, token) + require(set(present) == {"terminal", "report"}, "completed root task lost its terminal/report or retained runnable responsibility") + require(all((entry["node"], entry["disk"]) == (owner["node"], owner["disk"]) for entry in present.values()), + "root terminal/report changed original record disk") + terminal, report = present["terminal"]["value"], present["report"]["value"] + require(type(terminal.get("schema")) is int and terminal["schema"] == 1 and terminal.get("task_id") == token + and terminal.get("heal_type") == {"type": "cluster"} and terminal.get("status") == "Completed", + "root durable terminal changed task/scope/schema or is not completed") + require(canonical(terminal.get("options")) == canonical(owner["value"]["options"]), + "completed root task changed its checkpointed options or refunded the remaining budget") + require(type(report.get("schema")) is int and report["schema"] == 1 and report.get("terminal") == terminal + and canonical(report.get("outcome")) == canonical(current.get("outcome")), "root durable report lost the completed responsibility") + require(type(report.get("next_seq")) is int and type(report.get("min_seq")) is int + and 1 <= report["min_seq"] <= report["next_seq"] and isinstance(report.get("seqed_items"), list) + and 0 < len(report["seqed_items"]) <= 1024, "root report omitted valid bounded result items") + previous, seen = 0, set() + for sequence, item in report["seqed_items"]: + pair = item.get("object"), public_version_id(item.get("versionId")) + require(type(sequence) is int and max(previous + 1, report["min_seq"]) <= sequence < report["next_seq"] + and item.get("bucket") == bucket and pair in wanted and pair not in seen + and item.get("objectSize") == wanted[pair]["payload"]["size"], "root report items escaped complete ACK identities") + previous = sequence; seen.add(pair) + return {"runnableIntentRetiredOnEveryDrive": True, "sameOriginalRecordDisk": True, + "completedReportMatchesControlPlane": True, "retainedDiagnosticsAreFullManifest": False} + + +def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, original): + fleet.record("root-restart-contract", REFERENCE) + started = fleet.record("root-admission", probe.admin(PATH, [], json.dumps(OPTIONS).encode())) + require(started.get("httpStatus") == 200 and isinstance(started.get("response"), dict), "root admission failed") + token = started["response"].get("clientToken") + require(isinstance(token, str) and str(uuid.UUID(token)) == token, "root admission omitted canonical task ID") + active.append((PATH, token)) + first = wait_partial(probe, fleet, PATH, token, targets, repair_required=True) + admitted = records(fleet, token, "root-intent-admitted-owner") + owners = owned_records(admitted, token) + require(set(owners) == {"intent"}, "NOT_OBSERVED: original root owner finished before the interruption window") + coordinator = owners["intent"]["node"] + require(coordinator in fleet.nodes, "root coordinator is outside the actual bound fleet") + root_intent(admitted, token, coordinator) + fleet.record("actual-root-coordinator", {"node": coordinator, "source": "unique physical admitted intent owner", "taskId": token}) + processes = fleet.record("processes-before-interruption", {node: fleet.remote(node, "process") for node in fleet.nodes}) + identity = processes[coordinator] + fleet.frozen_processes[coordinator] = identity + fleet.record("root-freeze", fleet.remote(coordinator, "process-signal", signal="SIGSTOP", expected=identity)) + frozen = fleet.record("root-frozen-process", fleet.remote(coordinator, "process")) + require(frozen.get("state") == "T" and all(frozen.get(name) == identity.get(name) for name in PROCESS_FIELDS), "root freeze missed the original process") + scene = fleet.snapshot("root-frozen-physical") + pending, before_physical = partial_physical(original, scene, targets) + before_record = root_intent(records(fleet, token, "root-intent-before-stop"), token, coordinator) + fleet.record("root-graceful-exit", fleet.remote(coordinator, "process-graceful", expected=identity)) + del fleet.frozen_processes[coordinator] + fleet.record("release-exited-unit", fleet.remote(coordinator, "stop")) + after_record = root_intent(records(fleet, token, "root-intent-after-exit", stopped_node=coordinator), token, coordinator) + budget = same_owner(before_record, after_record) + exit_scenes = [] + for index in range(3): + exit_scenes.append(fleet.record("root-physical-after-exit", {node: fleet.remote(node, "snapshot", quiescent=node == coordinator) for node in fleet.nodes})) + if index < 2: + time.sleep(5) + require(all(physical_content(value) == physical_content(exit_scenes[0]) for value in exit_scenes[1:]), + "coordinator's retiring peer writes did not settle before restart") + after_scene = exit_scenes[-1] + remaining, after_physical = partial_physical(original, after_scene, targets) + require({entry["key"] for entry in remaining} <= {entry["key"] for entry in pending}, "interruption lost an already repaired shard") + fleet.record("root-original-unit-restart", fleet.remote(coordinator, "start")) + fleet.wait_ready() + restarted = fleet.record("processes-after-restart", {node: fleet.remote(node, "process") for node in fleet.nodes}) + continuity = restart_identity(processes, restarted, coordinator) + deadline = time.monotonic() + TASK_SECONDS + while True: + current = recovery_status(probe, fleet, token) + require(current.get("summary") in {"running", "finished"}, "original root task failed or became cancelled during recovery") + if current["summary"] == "finished": + break + require(time.monotonic() < deadline, "original root task did not recover within its pre-registered budget") + time.sleep(0.05) + terminal = complete_outcome(current, bucket, expected, {(entry["key"], entry["versionId"]) for entry in remaining}) + active.remove((PATH, token)) + idle = idle_fleet(fleet, binding) + journal_before = records(fleet, token, "root-completed-records") + journal = root_terminal(journal_before, token, after_record, current, bucket, expected) + complete_scene = fleet.snapshot("root-completed-live-scene") + # Repeat coordinator restart after completion. It must replay the durable + # terminal unchanged, never create a new task or write another user shard. + fleet.record("completed-coordinator-stop", fleet.remote(coordinator, "stop")) + fleet.record("completed-coordinator-start", fleet.remote(coordinator, "start")) + fleet.wait_ready() + repeated = fleet.record("processes-after-repeat-restart", {node: fleet.remote(node, "process") for node in fleet.nodes}) + restart_identity(restarted, repeated, coordinator) + repeat_idle = idle_fleet(fleet, binding) + repeat = recovery_status(probe, fleet, token) + require(repeat.get("summary") == "finished" and canonical(repeat.get("outcome")) == canonical(current["outcome"]), "repeat restart lost or changed original terminal responsibility") + journal_after = records(fleet, token, "root-completed-records-after-repeat") + root_terminal(journal_after, token, after_record, repeat, bucket, expected) + require(canonical(journal_before) == canonical(journal_after), "repeat restart rewrote the completed root journal") + require(physical_content(complete_scene) == physical_content(fleet.snapshot("root-repeat-live-scene")), "completed task revived and rewrote user files after repeat restart") + return {"root-partial-graceful": {"verdict": "PASS", "originalTaskId": token, "runningWindow": first["outcome"]["counters"], + "beforeInterruptionPhysical": before_physical, "afterActualExitPhysical": after_physical, + "sameTaskRecovered": True, "newStartRequestIssued": False, "processContinuity": continuity, + "completeResponsibility": terminal, "allPeerIdle": idle}, + "same-record-disk-resume": {"verdict": "PASS", "originalTaskId": token, "originalRecordOwner": before_record, + "budget": budget, "terminalJournal": journal, "repeatRestartIdle": repeat_idle, + "completedJournalBytesPreserved": True, "cursorResumeClaimed": False, "recordDiskFailoverClaimed": False}} + + +def run_topology(args, topology, nodes, *, observer=observe_task, branches=BRANCHES): + return dx_topology(args, topology, nodes, observer=observer, branches=branches, runtime_profile="root-restart", policy_reference=REFERENCE) + + +if __name__ == "__main__": + try: + dx_main(case_id=CASE_ID, observer=observe_task, branches=BRANCHES, runner=run_topology) + except Exception as exc: + print(f"Heal root-restart validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 0ead47a1..75a02762 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -25,6 +25,7 @@ case "${case_id}" in heal.admin-stop-and-stale-token) driver=stop.py ;; heal.cancelled-task-restart) driver=cancel_restart.py ;; heal.admin-overlap-and-force-start) driver=overlap.py ;; + heal.root-graceful-coordinator-restart) driver=root_restart.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; diff --git a/test_validation_heal.py b/test_validation_heal.py index e58bc2c2..2c57fa93 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -9,7 +9,7 @@ import gzip from datetime import UTC, datetime, timedelta import types -from unittest.mock import patch +from unittest.mock import patch, Mock from scripts.validation_heal.matrix import STAGES, TOPOLOGIES, audit_evidence, cases, compile_plan, select from scripts.validation_heal.healthy import validate_healthy, validate_topology, validate_rejection, Fleet @@ -51,6 +51,7 @@ from scripts.validation_heal.cancel_restart import validate_journal, BRANCHES as CANCEL_RESTART_BRANCHES from scripts.validation_heal.fleet_node import task_records as node_task_records from scripts.validation_heal import overlap +from scripts.validation_heal import root_restart, fleet_node import errno import stat @@ -2237,5 +2238,188 @@ def topology(params, topology, nodes, **options): with self.assertRaises(ValueError): Window(self.bucket, self.expected(), "task", "node1", "epoch") +class HealRootRestartOracleTest(unittest.TestCase): + token = "01234567-89ab-4def-8123-456789abcdef" + + def intent(self): + return {"schema": 3, "task_id": self.token, "heal_type": {"type": "cluster"}, "priority": "High", + "created_at": {"secs_since_epoch": 10, "nanos_since_epoch": 0}, "retry_attempts": 0, + "options": {"scan_mode": 2, "remove_corrupted": False, "recreate_missing": False, + "update_parity": False, "recursive": True, "dry_run": False, "no_lock": False, + "pool_index": None, "set_index": None, "timeout": None}} + + def records(self): + entries = [{"disk": disk, "kind": kind, "present": disk == 2 and kind == "intent", + "relativePath": ".rustfs.sys/root-heal-" + self.token + ".json", "value": self.intent()} + for disk in range(4) for kind in ("intent", "terminal", "report")] + return {"node1": {"readOnly": True, "taskId": self.token, "records": entries}} + + def process(self, pid=10, start=50): + return {name: value for name, value in zip(fleet_node.PROCESS_FIELDS, + (pid, start, "boot", "unit", "/state/rustfs", "sha", "mnt:[1]", "command", "env", {"budget": "300"}), strict=True)} | {"state": "T"} + + def test_root_owner_rejects_duplicate_missing_and_cross_scope_records(self): + self.assertEqual(root_restart.root_intent(self.records(), self.token, "node1")["disk"], 2) + for kind in ("missing", "duplicate", "scope", "schema", "selector", "read-repair", "retry", "coverage"): + observed = self.records() + entries = observed["node1"]["records"] + entry = next(row for row in entries if row["present"]) + if kind == "missing": entry["present"] = False + if kind == "duplicate": entries[0]["present"] = True + if kind == "scope": entry["value"]["heal_type"] = {"type": "prefix", "bucket": "other", "prefix": "objects/"} + if kind == "schema": entry["value"]["schema"] = 2 + if kind == "selector": entry["value"]["options"]["pool_index"] = 0 + if kind == "read-repair": entry["value"]["options"]["no_lock"] = True + if kind == "retry": entry["value"]["retry_attempts"] = 1 + if kind == "coverage": entries.pop() + with self.subTest(kind=kind), self.assertRaises(ValueError): root_restart.root_intent(observed, self.token, "node1") + + def test_checkpoint_budget_cannot_be_removed_refunded_or_move_owner(self): + before = root_restart.root_intent(self.records(), self.token, "node1") + after = copy.deepcopy(before) + after["value"]["options"]["timeout"] = {"secs": 280, "nanos": 7} + proof = root_restart.same_owner(before, after) + self.assertEqual(proof["checkpointRemainingTimeoutNs"], 280_000_000_007) + for kind in ("absent", "refund", "zero", "disk", "identity", "scope", "option", "bool", "nanos"): + current = copy.deepcopy(after) + if kind == "absent": current["value"]["options"]["timeout"] = None + if kind == "refund": current["value"]["options"]["timeout"]["secs"] = 301 + if kind == "zero": current["value"]["options"]["timeout"] = {"secs": 0, "nanos": 0} + if kind == "disk": current["disk"] = 3 + if kind == "identity": current["value"]["task_id"] = "new-task" + if kind == "scope": current["value"]["heal_type"] = {"type": "bucket", "bucket": "other"} + if kind == "option": current["value"]["options"]["remove_corrupted"] = True + if kind == "bool": current["value"]["options"]["timeout"]["secs"] = True + if kind == "nanos": current["value"]["options"]["timeout"]["nanos"] = 1_000_000_000 + with self.subTest(kind=kind), self.assertRaises(ValueError): root_restart.same_owner(before, current) + with self.assertRaises(ValueError): root_restart.same_owner(after, before) + + def test_only_original_coordinator_changes_birth_identity(self): + before = {"node1": self.process(), "node2": self.process(11, 51)} + after = {"node1": self.process(12, 60), "node2": self.process(11, 51)} + self.assertTrue(root_restart.restart_identity(before, after, "node1")["onlyCoordinatorProcessChanged"]) + for kind in ("no-restart", "peer-restart", "boot", "env", "namespace", "candidate", "missing"): + current = copy.deepcopy(after) + if kind == "no-restart": current["node1"] = copy.deepcopy(before["node1"]) + if kind == "peer-restart": current["node2"]["startTicks"] += 1 + if kind == "boot": current["node1"]["bootId"] = "new-boot" + if kind == "env": current["node1"]["environmentFileSha256"] = "new-env" + if kind == "namespace": current["node1"]["mountNamespace"] = "mnt:[2]" + if kind == "candidate": current["node1"]["binarySha256"] = "new-build" + if kind == "missing": del current["node2"] + with self.subTest(kind=kind), self.assertRaises(ValueError): root_restart.restart_identity(before, current, "node1") + + def test_pidfd_revalidates_birth_identity_before_any_signal(self): + expected = self.process() + with patch.object(fleet_node.os, "pidfd_open", return_value=17, create=True), \ + patch.object(fleet_node.signal, "pidfd_send_signal", create=True) as send, \ + patch.object(fleet_node.os, "close") as close, patch.object(fleet_node, "process", return_value=expected): + fleet_node.pidfd_signal({"expected": expected}, fleet_node.signal.SIGSTOP) + send.assert_called_once_with(17, fleet_node.signal.SIGSTOP, None, 0) + close.assert_called_once_with(17) + changed = {**expected, "startTicks": expected["startTicks"] + 1} + with patch.object(fleet_node.os, "pidfd_open", return_value=17, create=True), \ + patch.object(fleet_node.signal, "pidfd_send_signal", create=True) as send, \ + patch.object(fleet_node.os, "close") as close, patch.object(fleet_node, "process", return_value=changed), self.assertRaisesRegex(ValueError, "stale"): + fleet_node.pidfd_signal({"expected": expected}, fleet_node.signal.SIGSTOP) + send.assert_not_called(); close.assert_called_once_with(17) + with self.assertRaises(ValueError): fleet_node.process_signal({"signal": "SIGKILL", "expected": expected}) + + def test_graceful_exit_requires_actual_exit_zero_and_preserves_identity(self): + before = self.process() + exited = {"pid": 0, "bootId": "boot", "binarySha256": "sha", "environmentFileSha256": "env", + "service": {"ExecMainCode": "1", "ExecMainStatus": "0", "Result": "success"}} + for kind in ("good", "signal", "failed", "timeout", "boot", "candidate", "environment"): + current = copy.deepcopy(exited) + if kind == "signal": current["service"]["ExecMainCode"] = "2" + if kind == "failed": current["service"]["ExecMainStatus"] = "1" + if kind == "timeout": current["service"]["Result"] = "timeout" + if kind == "boot": current["bootId"] = "new-boot" + if kind == "candidate": current["binarySha256"] = "new-build" + if kind == "environment": current["environmentFileSha256"] = "new-env" + with self.subTest(kind=kind), patch.object(fleet_node, "owned", return_value=(None, None, {"runtimeProfile": "root-restart"})), \ + patch.object(fleet_node, "process", side_effect=[before, current]), patch.object(fleet_node, "pidfd_signal", return_value=before): + if kind == "good": + self.assertEqual(fleet_node.process_graceful({"runId": "run", "expected": before})["actualExitCode"], 0) + else: + with self.assertRaises(ValueError): fleet_node.process_graceful({"runId": "run", "expected": before}) + + def test_recovery_absence_is_bounded_observation_and_never_completion(self): + missing = {"httpStatus": 500, "response": {"Code": "InternalError", "Message": "Heal task not found: " + self.token}} + good = {"httpStatus": 200, "response": {"summary": "running"}} + client = types.SimpleNamespace(admin=Mock(side_effect=[missing, good])) + fleet = types.SimpleNamespace(record=lambda name, value: value) + with patch.object(root_restart.time, "sleep"): + self.assertEqual(root_restart.recovery_status(client, fleet, self.token)["summary"], "running") + client.admin = lambda *args: missing + with patch.object(root_restart.time, "monotonic", side_effect=[0, 16]), self.assertRaisesRegex(ValueError, "15-second"): + root_restart.recovery_status(client, fleet, self.token) + self.assertEqual(list(root_restart.BRANCHES), select("F01")[0]["variants"]) + + def test_completed_root_journal_retires_intent_and_preserves_checkpoint_options(self): + owner = root_restart.root_intent(self.records(), self.token, "node1") + owner["value"]["options"]["timeout"] = {"secs": 280, "nanos": 0} + expected = HealCursorOracleTest().expected() + bucket = HealCursorOracleTest.bucket + current = HealCursorOracleTest().terminal() + terminal = {"schema": 1, "task_id": self.token, "heal_type": {"type": "cluster"}, "status": "Completed", + "options": owner["value"]["options"]} + report = {"schema": 1, "terminal": copy.deepcopy(terminal), "outcome": current["outcome"], + "next_seq": 1027, "min_seq": 3, "seqed_items": [[index + 2, + {"bucket": bucket, "object": entry["key"], "versionId": "", "objectSize": CURSOR_SIZE}] + for index, entry in enumerate(expected[1:], 1)]} + observed = self.records() + for entry in observed["node1"]["records"]: + entry["present"] = entry["disk"] == 2 and entry["kind"] in {"terminal", "report"} + if entry["present"]: entry["value"] = terminal if entry["kind"] == "terminal" else report + self.assertTrue(root_restart.root_terminal(observed, self.token, owner, current, bucket, expected)["runnableIntentRetiredOnEveryDrive"]) + for kind in ("intent", "owner", "scope", "state", "refund", "outcome", "duplicate", "wrong-version", "wrong-size"): + changed = copy.deepcopy(observed) + rows = changed["node1"]["records"] + end = next(row for row in rows if row["present"] and row["kind"] == "terminal") + durable = next(row for row in rows if row["present"] and row["kind"] == "report") + if kind == "intent": rows[0]["present"] = True + if kind == "owner": end["disk"] = 3 + if kind == "scope": end["value"]["heal_type"] = {"type": "bucket", "bucket": bucket} + if kind == "state": end["value"]["status"] = "Cancelled" + if kind == "refund": end["value"]["options"]["timeout"]["secs"] = 300 + if kind == "outcome": durable["value"]["outcome"]["counters"]["unknown"] = 1 + if kind == "duplicate": durable["value"]["seqed_items"][1][1] = copy.deepcopy(durable["value"]["seqed_items"][0][1]) + if kind == "wrong-version": durable["value"]["seqed_items"][0][1]["versionId"] = "unacknowledged" + if kind == "wrong-size": durable["value"]["seqed_items"][0][1]["objectSize"] += 1 + with self.subTest(kind=kind), self.assertRaises(ValueError): root_restart.root_terminal(changed, self.token, owner, current, bucket, expected) + + def test_journal_reads_mark_the_actual_coordinator_as_stopped(self): + calls = [] + fleet = types.SimpleNamespace(nodes=["node1", "node2", "node3", "node4"], record=lambda name, value: value, + remote=lambda node, action, **params: calls.append((node, params["quiescent"])) or {}) + root_restart.records(fleet, self.token, "checkpoint", stopped_node="node3") + self.assertEqual(calls, [("node1", False), ("node2", False), ("node3", True), ("node4", False)]) + + def test_cleanup_resumes_only_the_proved_original_process_before_finalization(self): + fleet = Fleet(["node1"], Path("/tmp/unit-root-restart"), "run") + fleet.frozen_processes = {"node1": self.process()} + fleet.reserved = ["node1"] + calls = [] + def remote(node, action, **kwargs): + calls.append((action, kwargs)) + return {"pid": 10} if action == "process" else {} + fleet.remote = remote + fleet.record = lambda name, value: value + self.assertTrue(fleet.cleanup()) + self.assertEqual([action for action, _ in calls[:2]], ["process", "process-signal"]) + self.assertEqual(calls[1][1]["signal"], "SIGCONT") + self.assertEqual(calls[1][1]["expected"]["startTicks"], 50) + self.assertFalse(fleet.frozen_processes) + fleet.frozen_processes = {"node1": self.process()} + def stale(node, action, **kwargs): + calls.append((action, kwargs)) + if action == "process-signal": raise ValueError("stale process") + return {"pid": 10} if action == "process" else {} + fleet.remote = stale; calls.clear() + self.assertFalse(fleet.cleanup()) + self.assertFalse(any(action == "cleanup" for action, _ in calls)) + + if __name__ == "__main__": unittest.main() From a18f863d4a77a561a6be92848acc9076480f59cb Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 03:29:27 +0800 Subject: [PATCH 065/343] test(validation): migrate exact Heal read quorum and restored source boundaries --- docs/validation-migration-map.md | 27 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/quorum_boundary.py | 323 +++++++++++++++++++++ scripts/validation_heal/run_case.sh | 1 + test_validation_heal.py | 115 ++++++++ 6 files changed, 467 insertions(+), 3 deletions(-) create mode 100644 scripts/validation_heal/quorum_boundary.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 192a43dc..78a014a4 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 109 currently dispatchable IDs include all 74 non-Heal cases and thirty-five Heal cases. Forty-two have live PASS evidence; the other 116 have no live PASS claim. +case evidence. The 110 currently dispatchable IDs include all 74 non-Heal cases and thirty-six Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -627,3 +627,28 @@ completed outcome and all user files and remains idle. This proves durable admission replay, and makes no cursor-resume or record-disk failover claim. Local owner/budget/process-birth/exit/old-token/journal negatives passed; full live evidence remains pending. + + +B06 `heal.read-quorum-boundary` now dispatches all three source branches on +H1/H2 with three independent 5MiB+123-byte never-versioned ACK objects. Its +read and write quorums come from the recognized candidate artifact and actual +decoded geometry; the reviewed candidate lock policy counts pool host:port +clients, so four H2 hosts have exclusive quorum three, not nine disk locks. +The exact-R GET archives read-repair interference and never certifies manual +repair. The separate exact-R manual object must reconstruct every missing +original shard and return exact completed responsibility. F1 only isolates +payload files: all drives, metadata and locks remain online, so it cannot +represent insufficient writable-drive or lock quorum. That condition is +explicitly non-applicable here; B08 retains its independent real fault +requirements. The restored-W branch first proves the candidate's exact R-1 +read refusal with no physical changes, then restores only W-(R-1) original +backups. Those sources never count as product repairs. A distinct manual +token must reconstruct all N-W remaining targets, followed by full physical, +streaming checksum and all-peer S3 checks. Live evidence is pending. + +A08 `heal.admin-authorization` passed live in RustFS Actions run +`38069648003`, harness `0789743d439fd91b967c2d6fba85c06e29f844c5`. All five +source branches passed independently on H1 and H2; the canonical gate, +complete summary, cleanup and workflow all passed with the fixed candidate. +The prior A08 failures remain recorded. Forty-eight Heal executors remain +to be migrated; automatic dispatch and local tests do not claim live PASS. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 8da7ea33..54f02ab1 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index dd76d8ef..78fcefa1 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/quorum_boundary.py b/scripts/validation_heal/quorum_boundary.py new file mode 100644 index 00000000..ca781c43 --- /dev/null +++ b/scripts/validation_heal/quorum_boundary.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""B06: independent read, manual repair and restored-source quorum witnesses.""" + +from __future__ import annotations + +import argparse +from datetime import UTC, datetime, timedelta +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys + +try: + from .heal_probe import Probe, payload, require, save + from .heal_workload import Recorder, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .heal_fault import bounded_fault_id + from .shards import files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort + from .versions import (validate_version_receipts, validate_unrecoverable_get, + verify_quorum_contract, verify_version_s3) +except ImportError: + from heal_probe import Probe, payload, require, save + from heal_workload import Recorder, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from heal_fault import bounded_fault_id + from shards import files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort + from versions import (validate_version_receipts, validate_unrecoverable_get, + verify_quorum_contract, verify_version_s3) + + +CASE_ID = "heal.read-quorum-boundary" +BRANCHES = ("exact-r-read", "exact-r-heal-write-limited", "restored-w") +SIZE = 5 * 1024**2 + 123 +# These reviewed lock policies are deliberately candidate-specific. They do +# not require a service checkout on the jump host or a modified service binary. +LOCK_REFERENCE = {"candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", + "clientDomain": "distinct nonempty pool endpoint host:port; H1 uses its local manager", + "exclusiveQuorumRule": "floor(clientCount/2)+1; local manager=1", + "sharedQuorumRule": "ceil(clientCount/2); local manager=1", + "sourceFiles": { + "crates/ecstore/src/core/sets.rs": "06543b1d083f489346171ba489b1eb1e0b7899c844f0ebf15e16ad467cf5e2f1", + "crates/ecstore/src/runtime/sources.rs": "7d88f56e6d06fd31850bbbdb2fd9faf20fcffbaedf380888f5f9285c309794f2", + "crates/ecstore/src/set_disk/ops/locking.rs": "44d50605c5538adf8fc6f25034e426011d0087b37e895f8246c79cbd711b2a42", + "crates/ecstore/src/set_disk/ops/heal.rs": "767c468b780cf5f4e863b078f46c63440549e75d88b6b43bb59ed83c3b4a47b2", + "crates/lock/src/distributed_lock.rs": "657796708fdc5f826f896147b74a9366dae875fd6b69e4b253994966ed5f6e37"}} + + +def seed_fixtures(recorder, bucket): + recorder.call("create_bucket", {"Bucket": bucket}) + expected = [] + for branch in BRANCHES: + key = "boundary/" + branch + ".bin" + body = payload(f"{bucket}/{key}/0", SIZE) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "Metadata": {"heal-fixture": "DB", "branch": branch}, "Tagging": "suite=heal&fixture=DB"}, body) + require(response.get("VersionId") in {None, "null"}, "B06 requires never-versioned independent objects") + expected.append({"key": key, "versionId": "null", "deleteMarker": False, "isLatest": True, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + return expected + + +def boundary_plan(branch, snapshot, bucket, expected, contract): + require(branch in BRANCHES and contract.get("recognized") is True + and contract.get("writeQuorumRule") == "data-plus-one-if-equal-parity", + "B06 requires a recognized candidate read/write quorum contract") + selected = [entry for entry in expected if entry["key"] == "boundary/" + branch + ".bin"] + require(len(selected) == 1 and selected[0].get("versionId") == "null" + and selected[0].get("deleteMarker") is False, "B06 branch reused or lost its independent ACK") + target = map_target(snapshot, bucket, selected[0]["key"], "data", 1) + geometry = target["logical"]["erasure"] + read, parity = geometry["data_blocks"], geometry["parity_blocks"] + write, total = read + int(read == parity), read + parity + retained_count = read - 1 if branch == "restored-w" else read + ordered = sorted(target["cohort"], key=lambda item: item["index"]) + retained, missing = ordered[:retained_count], ordered[retained_count:] + faults = [{**target, **shard} for shard in missing] + restore_count = write - retained_count if branch == "restored-w" else 0 + require(read > 1 and len(ordered) == total and len({item["index"] for item in ordered}) == total + and len(retained) == retained_count and 0 <= restore_count < len(faults), + "B06 layout cannot leave real product repair targets after restoration to W") + hosts = len(snapshot) + require(hosts in {1, 4}, "B06 lock client domain requires actual H1/H2 topology") + return {"scope": {"bucket": bucket, "key": selected[0]["key"]}, "expected": selected, + "target": target, "faults": faults, "retained": retained, "restoreCount": restore_count, + "readQuorum": read, "writeQuorum": write, "sameVersionSourcesBefore": retained_count, + "writableTargets": total, "metadataAuthorities": total, + "lock": {"mode": "local" if hosts == 1 else "distributed", "clientCount": hosts, + "sharedQuorum": (hosts + 1) // 2, "exclusiveQuorum": hosts // 2 + 1, + "allEndpointsOnline": True, "successfulRepairAlsoProvesActualAdmission": True}, + "insufficientWriteOrLockApplicable": False, + "nonApplicableReason": "F1 isolates payload files only; all N drives, metadata and host lock endpoints remain online. " + "R counts readable payload authorities, not writable drives. B08 independently injects actual write/lock loss."} + + +def source_witness(snapshot, plan, missing): + indexed = files(snapshot) + identities = {(item["node"], item["disk"], item["relativePath"]) for item in missing} + cohort = plan["target"]["cohort"] + require(len(identities) == len(missing) and identities <= { + (item["node"], item["disk"], item["relativePath"]) for item in cohort}, "missing authority escaped its version/set") + for item in cohort: + identity = item["node"], item["disk"], item["relativePath"] + actual = indexed.get(identity) + if identity in identities: + require(actual is None, "ATTRIBUTION_NOT_OBSERVED: isolated source reappeared before manual repair") + else: + require(actual is not None and actual["sha256"] == item["sha256"] and actual["size"] == item["size"], + "retained same-version source differs from its full shard oracle") + metadata = indexed.get((item["node"], item["disk"], f"{plan['scope']['bucket']}/{plan['scope']['key']}/xl.meta")) + require(metadata and metadata["sha256"] == item["baselineMetadataSha256"], + "F1 changed metadata authority or used a different generation") + return {"sameVersionPayloadSources": len(cohort) - len(identities), "missingTargets": len(identities), + "allMetadataAuthoritiesOnlineAndUnchanged": True, "allRetainedShardBytesUnchanged": True} + + +def read_observation(observation, expected): + require(observation.get("acknowledged") is True and observation.get("resultUnknown") is False + and observation.get("response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode") == 200 + and observation.get("payload") == expected["payload"], "GET at exactly R did not return the acknowledged bytes") + + +def read_interference(before, after, plan): + """Permit only exact product reconstruction, and retain each observed target.""" + left, right = files(before), files(after) + targets = {(item["node"], item["disk"], item["relativePath"]): item for item in plan["faults"]} + reconstructed = [] + for identity, target in targets.items(): + require(identity not in left, "exact-R GET began with an already reconstructed source") + if identity in right: + require(right[identity]["sha256"] == target["sha256"] and right[identity]["size"] == target["size"], + "GET produced a target with incorrect shard bytes") + reconstructed.append({"node": identity[0], "disk": identity[1], "relativePath": identity[2]}) + return {"readRepairObserved": bool(reconstructed), "reconstructedTargets": reconstructed, + "manualRepairClaimed": False, "readRepairRuntimeEnabled": False, + "note": "Any unexpected reconstruction is retained as interference; this GET never certifies the separate manual branch."} + + +def recover_registered(fleet, registrations, label): + for registration in reversed(tuple(registrations)): + node, params = registration + receipt = fleet.record(label, fleet.remote(node, "recover-fault", **params)) + require(receipt.get("physicalRepairPassAllowed") is False, "backup restoration must not certify product repair") + fleet.faults.remove(registration) + registrations.remove(registration) + + +def run_topology(args, topology, nodes): + directory = args.evidence / "DB" / topology + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory) + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == LOCK_REFERENCE["candidateCommit"], "B06 lock policy is not reviewed for this candidate") + contract = verify_quorum_contract(fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)), candidate) + require(contract.get("writeQuorumRule") == "data-plus-one-if-equal-parity", "B06 write policy is not recognized") + require(list(BRANCHES) == select(CASE_ID)[0]["variants"], "B06 executor omitted a source branch") + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(UTC) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 512, "maxInventoryBytes": 512 * 1024**2} + fleet.record("binding", binding) + fleet.record("quorum-source-contract", {"payloadAndWrite": contract, "locks": LOCK_REFERENCE}) + probe, active, admissions = Probe(binding), [], [] + result = {"topology": topology, "fixture": "DB", "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) + expected = seed_fixtures(Recorder(probe, directory / "fixture"), bucket) + baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + for branch in BRANCHES: + registrations = [] + try: + fleet.stop() + before = fleet.snapshot("physical-before-" + branch, quiescent=True) + plan = boundary_plan(branch, before, bucket, expected, contract) + fleet.record("quorum-plan-" + branch, plan) + verify_cohort(fleet, plan["target"], "baseline-bitrot-" + branch) + prior = before + for index, target in enumerate(plan["faults"]): + params = {"disk": target["disk"], "faultId": bounded_fault_id("B06", branch, index)} + registration = target["node"], params + fleet.faults.append(registration) + registrations.append(registration) + receipt = fleet.record("fault-" + branch, fleet.remote(target["node"], "apply-fault", **params, + relativePath=target["relativePath"], expectedSha256=target["sha256"], kind="missing")) + faulted = fleet.snapshot("physical-fault-" + branch, quiescent=True) + validate_fault(prior, faulted, target, receipt) + prior = faulted + source_witness(prior, plan, plan["faults"]) + fleet.start() + witness = fleet.snapshot("physical-before-trigger-" + branch) + require(physical_content(prior) == physical_content(witness), "ATTRIBUTION_NOT_OBSERVED: B06 fault changed before trigger") + fleet.record("source-window-" + branch, source_witness(witness, plan, plan["faults"])) + proof = {"quorums": {name: plan[name] for name in ("readQuorum", "writeQuorum", "writableTargets", "lock", + "insufficientWriteOrLockApplicable", "nonApplicableReason")}} + if branch == "exact-r-read": + observed = fleet.record("exact-r-get", probe.s3("get_object", {"Bucket": bucket, "Key": plan["scope"]["key"]})) + read_observation(observed, plan["expected"][0]) + after_get = fleet.snapshot("physical-after-exact-r-get") + proof["read"] = read_interference(witness, after_get, plan) + fleet.stop() + recover_registered(fleet, registrations, "get-fixture-backup-rescue") + repaired = fleet.snapshot("physical-after-get-rescue", quiescent=True) + validate_repaired_physical(before, repaired, plan["target"], repair_targets=plan["faults"]) + proof["backupRescueNotManualRepair"] = True + else: + remaining = plan["faults"] + if branch == "restored-w": + denied = fleet.record("below-r-get", probe.s3("get_object", {"Bucket": bucket, "Key": plan["scope"]["key"]})) + validate_unrecoverable_get(denied, contract) + require(physical_content(prior) == physical_content(fleet.snapshot("physical-after-below-r-get")), + "below-R GET changed good shards or repaired without authority") + previous_token = admissions[-1]["clientToken"] if admissions else None + fleet.stop() + stopped = fleet.snapshot("physical-before-restore", quiescent=True) + require(physical_content(prior) == physical_content(stopped), "below-R read window changed original authority") + proof["belowReadWindow"] = {"frozenUnrecoverableGetMatched": True, + "sourceWitness": source_witness(stopped, plan, plan["faults"]), "backupRestoreHasNotRun": True} + count = plan["restoreCount"] + restored_registrations = registrations[:count] + recover_registered(fleet, restored_registrations, "restore-original-sources-to-w") + del registrations[:count] + remaining = plan["faults"][count:] + prior = fleet.snapshot("physical-at-w", quiescent=True) + proof["atW"] = source_witness(prior, plan, remaining) + require(proof["atW"]["sameVersionPayloadSources"] == plan["writeQuorum"], "restore did not reach exactly W") + proof["restoredOriginalSources"] = count + fleet.start() + witness = fleet.snapshot("physical-before-restored-w-trigger") + require(physical_content(prior) == physical_content(witness), "ATTRIBUTION_NOT_OBSERVED: restored-W fault changed before its token") + source_witness(witness, plan, remaining) + status = run_task(probe, fleet, plan["scope"], 2, active, admissions=admissions) + if branch == "restored-w": + require(previous_token is None or admissions[-1]["clientToken"] != previous_token, + "restored-W Heal reused another branch's manual token") + proof["previousIndependentToken"], proof["restoredToken"] = previous_token, admissions[-1]["clientToken"] + fleet.stop() + repaired = fleet.snapshot("physical-after-manual-" + branch, quiescent=True) + proof["responsibility"] = validate_version_receipts(status, bucket, plan["expected"], 2, + required_repairs={(plan["scope"]["key"], "null")}) + proof["physical"] = validate_repaired_physical(before, repaired, plan["target"], repair_targets=remaining) + proof["productReconstructedTargets"] = len(remaining) + for registration in registrations: + fleet.faults.remove(registration) + registrations.clear() + proof["bitrot"] = verify_cohort(fleet, plan["target"], "repaired-bitrot-" + branch) + fleet.start() + proof["s3"] = verify_version_s3(probe, fleet, bucket, expected, baseline, "s3-after-" + branch, fixture="DB") + result["branches"][branch] = {"verdict": "PASS", **proof} + except (Exception, KeyboardInterrupt) as exc: + result["branches"][branch] = {"verdict": "FAIL", "failure": {"type": type(exc).__name__, "message": str(exc)}} + fleet.stop() + fleet.snapshot("physical-failure-" + branch, quiescent=True) + recover_registered(fleet, registrations, "error-backup-rescue-" + branch) + if isinstance(exc, KeyboardInterrupt) or active: + raise + result["verdict"] = "PASS" if len(result["branches"]) == len(BRANCHES) and all( + item["verdict"] == "PASS" for item in result["branches"].values()) else "FAIL" + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + finally: + for path, token in active: + try: + fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: + fleet.record("stop-error", {"error": str(exc)}) + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("topology-result", result) + return result + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f"Heal workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path, required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(set(nodes)) == len(nodes) and all( + re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") + require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") + require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") + results = [] + try: + for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): + results.append(run_topology(args, topology, selected)) + if results[-1]["cleanup"] != "PASS": + break + finally: + complete = len(results) == 2 and all(item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, + "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", "complete": complete}) + require(complete, "B06 requires all independent source branches on H1/H2 and complete cleanup") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"Heal read quorum validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 75a02762..9c8e3a0e 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -26,6 +26,7 @@ case "${case_id}" in heal.cancelled-task-restart) driver=cancel_restart.py ;; heal.admin-overlap-and-force-start) driver=overlap.py ;; heal.root-graceful-coordinator-restart) driver=root_restart.py ;; + heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; diff --git a/test_validation_heal.py b/test_validation_heal.py index 2c57fa93..e81307d5 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -52,6 +52,7 @@ from scripts.validation_heal.fleet_node import task_records as node_task_records from scripts.validation_heal import overlap from scripts.validation_heal import root_restart, fleet_node +from scripts.validation_heal import quorum_boundary import errno import stat @@ -2421,5 +2422,119 @@ def stale(node, action, **kwargs): self.assertFalse(any(action == "cleanup" for action, _ in calls)) +class HealReadQuorumBoundaryTest(unittest.TestCase): + contract = {"recognized": True, "writeQuorumRule": "data-plus-one-if-equal-parity"} + bucket = "rv-heal-0123456789abcdef" + + def scene(self, branch="exact-r-read", *, total=4, data=2): + original = HealShardOracleTest().snapshot()["node1"] + snapshot = {} + disk_ids = [f"disk-{index}" for index in range(total)] + key = "boundary/" + branch + ".bin" + for slot in range(total): + node, disk = "node" + str(slot // 4 + 1), slot % 4 + value = snapshot.setdefault(node, {"formats": [], "files": []}) + value["formats"].append({"disk": disk, "value": {"format": "xl", "id": "pool", + "xl": {"sets": [disk_ids], "this": disk_ids[slot]}}}) + metadata = copy.deepcopy(original["files"][0]["metadata"]) + metadata["key"] = key + version = metadata["versions"][0] + version["role"] = "data" if slot < data else "parity" + version["erasure"].update(index=slot + 1, data_blocks=data, parity_blocks=total - data, + distribution=list(range(1, total + 1))) + value["files"].extend([ + {"disk": disk, "path": f"{self.bucket}/{key}/xl.meta", "size": 600, + "sha256": "metadata-" + str(slot), "metadata": metadata}, + {"disk": disk, "path": f"{self.bucket}/{key}/{version['dataDir']}/part.1", "size": 82, + "sha256": "shard-" + str(slot)}]) + expected = [{"key": key, "versionId": "null", "deleteMarker": False, "isLatest": True, + "payload": {"size": 100, "sha256": "acknowledged"}}] + return snapshot, expected + + def plan(self, branch="exact-r-read", **kwargs): + snapshot, expected = self.scene(branch, **kwargs) + return snapshot, quorum_boundary.boundary_plan(branch, snapshot, self.bucket, expected, self.contract) + + def missing(self, snapshot, targets): + changed = copy.deepcopy(snapshot) + for target in targets: + changed[target["node"]]["files"] = [entry for entry in changed[target["node"]]["files"] + if (entry["disk"], entry["path"]) != (target["disk"], target["relativePath"])] + return changed + + def test_source_quorum_is_distinct_from_writable_drives_and_host_locks(self): + for total, data, write, clients in ((4, 2, 3, 1), (16, 12, 12, 4), (16, 8, 9, 4)): + snapshot, plan = self.plan("exact-r-heal-write-limited", total=total, data=data) + self.assertEqual((plan["readQuorum"], plan["writeQuorum"], plan["writableTargets"]), (data, write, total)) + self.assertEqual(plan["lock"]["clientCount"], clients) + self.assertEqual(plan["lock"]["exclusiveQuorum"], clients // 2 + 1) + self.assertFalse(plan["insufficientWriteOrLockApplicable"]) + witness = quorum_boundary.source_witness(self.missing(snapshot, plan["faults"]), plan, plan["faults"]) + self.assertEqual(witness["sameVersionPayloadSources"], data) + + def test_restore_to_w_leaves_real_targets_and_does_not_count_restored_sources_as_heal(self): + for total, data, restored, remaining in ((4, 2, 2, 1), (16, 12, 1, 4), (16, 8, 2, 7)): + snapshot, plan = self.plan("restored-w", total=total, data=data) + self.assertEqual(len(plan["retained"]), data - 1) + self.assertEqual(plan["restoreCount"], restored) + targets = plan["faults"][restored:] + self.assertEqual(len(targets), remaining) + after = self.missing(snapshot, targets) + self.assertEqual(quorum_boundary.source_witness(after, plan, targets)["sameVersionPayloadSources"], plan["writeQuorum"]) + + def test_source_witness_rejects_changed_generation_extra_sources_and_good_shard_mutation(self): + snapshot, plan = self.plan() + faulted = self.missing(snapshot, plan["faults"]) + for kind in ("reappeared", "retained-bytes", "metadata", "foreign-target", "duplicate-target"): + changed, targets = copy.deepcopy(faulted), copy.deepcopy(plan["faults"]) + if kind == "reappeared": changed = self.missing(snapshot, targets[1:]) + elif kind == "retained-bytes": next(entry for entry in changed["node1"]["files"] if entry["path"].endswith("/part.1"))["sha256"] = "corrupt" + elif kind == "metadata": changed["node1"]["files"][0]["sha256"] = "another-generation" + elif kind == "foreign-target": targets[0]["relativePath"] += ".wrong-version" + else: targets.append(copy.deepcopy(targets[0])) + with self.subTest(kind=kind), self.assertRaises(ValueError): + quorum_boundary.source_witness(changed, plan, targets) + + def test_read_boundary_requires_complete_acknowledged_body_and_records_interference(self): + snapshot, plan = self.plan() + expected = plan["expected"][0] + good = {"acknowledged": True, "resultUnknown": False, "payload": expected["payload"], + "response": {"ResponseMetadata": {"HTTPStatusCode": 200}}} + quorum_boundary.read_observation(good, expected) + for field, value in (("acknowledged", False), ("resultUnknown", True), ("payload", {"size": 0, "sha256": "empty"})): + with self.subTest(field=field), self.assertRaises(ValueError): + quorum_boundary.read_observation({**good, field: value}, expected) + faulted = self.missing(snapshot, plan["faults"]) + self.assertFalse(quorum_boundary.read_interference(faulted, faulted, plan)["readRepairObserved"]) + proof = quorum_boundary.read_interference(faulted, snapshot, plan) + self.assertTrue(proof["readRepairObserved"]) + self.assertFalse(proof["manualRepairClaimed"]) + changed = copy.deepcopy(snapshot) + target = plan["faults"][0] + next(entry for entry in changed[target["node"]]["files"] if entry["disk"] == target["disk"] + and entry["path"] == target["relativePath"])["sha256"] = "bad-reconstruction" + with self.assertRaises(ValueError): quorum_boundary.read_interference(faulted, changed, plan) + + def test_recovery_keeps_unresolved_registration_when_backup_oracle_is_rejected(self): + registration = ("node1", {"disk": 0, "faultId": "B06-exact-r-read-0"}) + fleet = Mock(faults=[registration]) + fleet.remote.return_value = {"physicalRepairPassAllowed": False, "restoredFromBackup": True} + fleet.record.side_effect = lambda label, value: value + registrations = [registration] + quorum_boundary.recover_registered(fleet, registrations, "rescue") + self.assertFalse(fleet.faults or registrations) + fleet.faults, registrations = [registration], [registration] + fleet.remote.return_value = {"physicalRepairPassAllowed": True} + with self.assertRaises(ValueError): quorum_boundary.recover_registered(fleet, registrations, "rescue") + self.assertEqual(fleet.faults, registrations) + + def test_branch_and_geometry_contract_cannot_be_weakened(self): + self.assertEqual(list(quorum_boundary.BRANCHES), select("B06")[0]["variants"]) + snapshot, expected = self.scene() + for contract in ({}, {"recognized": True, "writeQuorumRule": "unknown"}): + with self.assertRaises(ValueError): quorum_boundary.boundary_plan("exact-r-read", snapshot, self.bucket, expected, contract) + with self.assertRaises(ValueError): quorum_boundary.boundary_plan("exact-r-read", snapshot, self.bucket, expected * 2, self.contract) + + if __name__ == "__main__": unittest.main() From 568f48f1074aba6601c4f7172cb5c85d7c73e192 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 03:30:17 +0800 Subject: [PATCH 066/343] test(validation): report observed quorum read interference without inferred runtime state --- scripts/validation_heal/quorum_boundary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validation_heal/quorum_boundary.py b/scripts/validation_heal/quorum_boundary.py index ca781c43..38ea6d30 100644 --- a/scripts/validation_heal/quorum_boundary.py +++ b/scripts/validation_heal/quorum_boundary.py @@ -139,7 +139,7 @@ def read_interference(before, after, plan): "GET produced a target with incorrect shard bytes") reconstructed.append({"node": identity[0], "disk": identity[1], "relativePath": identity[2]}) return {"readRepairObserved": bool(reconstructed), "reconstructedTargets": reconstructed, - "manualRepairClaimed": False, "readRepairRuntimeEnabled": False, + "manualRepairClaimed": False, "manualStartIssuedForReadBranch": False, "note": "Any unexpected reconstruction is retained as interference; this GET never certifies the separate manual branch."} From 613a0a99141fd4c32aa1bf1b76566eadeba2fa21 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 03:44:26 +0800 Subject: [PATCH 067/343] test(validation): migrate stale member rejoin with live permission faults --- docs/validation-migration-map.md | 35 +++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/drive_gate.py | 34 +++- scripts/validation_heal/fleet_node.py | 23 ++- scripts/validation_heal/healthy.py | 8 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/stale_rejoin.py | 254 ++++++++++++++++++++++++ scripts/validation_heal/versions.py | 27 ++- test_validation_heal.py | 199 ++++++++++++++++++- 10 files changed, 567 insertions(+), 19 deletions(-) create mode 100644 scripts/validation_heal/stale_rejoin.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 78a014a4..0b169c28 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 110 currently dispatchable IDs include all 74 non-Heal cases and thirty-six Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. +case evidence. The 111 currently dispatchable IDs include all 74 non-Heal cases and thirty-seven Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -650,5 +650,36 @@ A08 `heal.admin-authorization` passed live in RustFS Actions run `38069648003`, harness `0789743d439fd91b967c2d6fba85c06e29f844c5`. All five source branches passed independently on H1 and H2; the canonical gate, complete summary, cleanup and workflow all passed with the fixed candidate. -The prior A08 failures remain recorded. Forty-eight Heal executors remain +The prior A08 failures remain recorded. Forty-seven Heal executors remain to be migrated; automatic dispatch and local tests do not claim live PASS. + + +C06 `heal.stale-version-rejoin` now dispatches all three source branches on +H1/H2, with never-versioned overwrite and independent versioned historical +delete/marker fixtures. A private permission gate removes both DAC bypass +capabilities and activates only after the original candidate has started. +Same-process read/write EACCES proves that the unchanged member cannot +participate in the subsequent ACK transition. Every online metadata replica +must agree on the complete acknowledged namespace, the isolated member's +entire physical disk remains exact, and controls retain their original bytes. +The same disk UUID and set slot must rejoin without harness metadata writes +or restoring old-generation backups. An independent deep token must correct +the stale namespace; exact outcome counters, retained same-drive shards, +streaming checksums, deleted-version absence, current marker headers and +all-peer S3 manifests are required. Original payload nonces remain attached +to ACK versions after deletion so Range checks cannot silently renumber +history. The candidate's dangling grace is explicitly zero for this rejoin +profile; C10 retains the independent grace-policy requirements. Complete +cursor-bounded generation-selection logs are archived. Live evidence is pending. + +B02 run `38079787495` retained FAIL and cleanup PASS on both H1/H2. The +readonly window produced `completed_with_errors` with one failed receipt, +zero repaired targets and `summary=stopped`, but omitted bucket incarnation. +The stopped summary is now interpreted only with explicit failed execution, +complete coverage and exact counters; missing incarnation still fails. The +permission setup previously chmod'ed the data root before service startup, +causing the candidate to exit with EACCES before injection could be witnessed. +Permission gates now retain an armed recovery marker during initial startup +and inject chmod only into the same live candidate with DAC bypass already +removed. This fixes the injection window without certifying the old run or +relaxing physical/refusal/responsibility assertions. A fresh run is required. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 54f02ab1..e1bd39fc 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/drive_gate.py b/scripts/validation_heal/drive_gate.py index 90ed8016..0255e0ba 100644 --- a/scripts/validation_heal/drive_gate.py +++ b/scripts/validation_heal/drive_gate.py @@ -47,6 +47,7 @@ def checked_gate(state, roots): and info.st_size < 4096 and info.st_mode & 0o077 == 0, "invalid drive gate ownership") value = json.loads(path.read_text()) require(value.get("schemaVersion") == 1 and value.get("mode") in {"read-only", "permission-denied"} + and value.get("phase", "active") in {"armed", "active"} and type(value.get("disk")) is int and 0 <= value["disk"] < len(roots) and value.get("dataPath") == str(roots[value["disk"]] / "data") and type(value.get("originalMode")) is int and value["originalMode"] == 0o700, @@ -62,17 +63,41 @@ def apply_gate(state, roots, disk, mode): require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and stat.S_IMODE(info.st_mode) == 0o700, "expected private run-owned root with original mode 0700") value = {"schemaVersion": 1, "disk": disk, "mode": mode, "dataPath": str(folder), - "originalMode": 0o700, "device": info.st_dev, "inode": info.st_ino} + "originalMode": 0o700, "device": info.st_dev, "inode": info.st_ino, + "phase": "armed" if mode == "permission-denied" else "active"} # Intent precedes chmod/start. Cleanup can restore a partially applied gate. fd = os.open(state / "drive-gate.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) with os.fdopen(fd, "w") as stream: json.dump(value, stream) stream.flush() os.fsync(stream.fileno()) - if mode == "permission-denied": folder.chmod(0) return {**value, "state": "ARMED", "hostMountRemounted": False, "sharedPermissionsChanged": False} +def activate_gate(state, roots): + value = checked_gate(state, roots) + require(value and value["mode"] == "permission-denied" and value.get("phase") == "armed", + "permission activation requires a newly armed run-owned gate") + folder = Path(value["dataPath"]) + info = folder.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and info.st_dev == value["device"] + and info.st_ino == value["inode"] and stat.S_IMODE(info.st_mode) == 0o700, + "armed permission root changed before live injection") + # Keep the recovery intent before chmod. A partially activated gate remains + # recoverable from the armed marker and original inode/mode. + folder.chmod(0) + temporary = state / "drive-gate-active.json" + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + value["phase"] = "active" + with os.fdopen(fd, "w") as stream: + json.dump(value, stream) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, state / "drive-gate.json") + return {**value, "state": "ACTIVE", "injectedAfterActualServiceStart": True, + "hostMountRemounted": False, "sharedPermissionsChanged": False} + + def release_gate(state, roots): value = checked_gate(state, roots) if value is None: return {"state": "CLEAR", "gateWasAbsent": True} @@ -117,6 +142,7 @@ def validate_probe(observed, mode, identity): def probe_gate(state, roots, pid): value = checked_gate(state, roots) require(value is not None and type(pid) is int and pid > 0, "active gated service required") + require(value.get("phase", "active") == "active", "permission gate is still armed; no refusal claim") identity = process_identity(pid) require(Path(f"/proc/{pid}/ns/mnt").readlink() != Path("/proc/self/ns/mnt").readlink(), "gate requires an independent service mount namespace") @@ -144,7 +170,9 @@ def probe_gate(state, roots, pid): def validate_unwritable(status, bucket, entry): outcome = status.get("outcome") or {} objects = outcome.get("objects", []) - require(status.get("summary") == "finished" and status.get("truncated", False) is False + terminal = status.get("summary") == "finished" or status.get("summary") == "stopped" \ + and outcome.get("execution", {}).get("state") == "completed_with_errors" + require(terminal and status.get("truncated", False) is False and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False and len(objects) == 1, "unwritable task did not preserve complete exact responsibility") receipt = objects[0] diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 4d185cf2..4c5f9304 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -32,7 +32,9 @@ def runtime_config(profile): - require(profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart"}, "unsupported Heal runtime profile") + require(profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin"}, "unsupported Heal runtime profile") + if profile == "stale-rejoin": + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS": "0", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} if profile == "root-restart": return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false"} if profile == "overlap": @@ -350,6 +352,22 @@ def drive_gate_probe(request): return gate_module().probe_gate(state, roots, pid) +def drive_gate_activate(request): + state, roots, _ = owned(request["runId"]) + before = process(request) + require(before["pid"] > 1, "permission fault requires the running owned candidate") + receipt = gate_module().activate_gate(state, roots) + after = process(request) + require(all(before.get(name) == after.get(name) for name in PROCESS_FIELDS), + "permission injection crossed the original process identity") + return {**receipt, "processBefore": before, "processAfter": after, "sameOriginalProcess": True} + + +def drive_gate_state(request): + state, roots, _ = owned(request["runId"]) + return gate_module().checked_gate(state, roots) or {"state": "CLEAR"} + + def fault_root(request): _, roots, owner = owned(request["runId"]) stopped(owner) @@ -619,7 +637,8 @@ def main(request): actions = {"preflight": preflight, "reserve": reserve, "start": start, "stop": stop, "snapshot": snapshot, "logs": logs, "cleanup": cleanup, "audit-finalize": audit_finalize, "install-inspector": install_inspector, "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part, "verify-parts": verify_parts, - "verify-inline": verify_inline, "drive-gate": drive_gate, "drive-gate-probe": drive_gate_probe} + "verify-inline": verify_inline, "drive-gate": drive_gate, "drive-gate-probe": drive_gate_probe, + "drive-gate-activate": drive_gate_activate, "gate-state": drive_gate_state} actions["task-records"] = task_records actions.update({"process": process, "process-signal": process_signal, "process-graceful": process_graceful}) require(request.get("action") in actions, "unsupported node operation") diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 8810b5c4..a92bc6ad 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -131,7 +131,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -170,7 +170,7 @@ def remote(self, node, action, **params): if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline"}: source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" - if action in {"start", "cleanup", "audit-finalize", "drive-gate", "drive-gate-probe"}: + if action in {"start", "cleanup", "audit-finalize", "drive-gate", "drive-gate-probe", "drive-gate-activate", "gate-state"}: source += "\nimport types\ngate = types.ModuleType('drive_gate')\nexec(" + repr( NODE_SOURCE.with_name("drive_gate.py").read_text()) + ", gate.__dict__)\n" source += "\nmain(" + repr(value) + ")\n" @@ -194,6 +194,10 @@ def start(self): results = list(executor.map(lambda node: self.remote(node, "start"), self.nodes)) self.record("started", dict(zip(self.nodes, results, strict=True))) self.wait_ready() + for node in sorted(self.drive_gates): + gate = self.remote(node, "gate-state") + if gate.get("mode") == "permission-denied" and gate.get("phase") == "armed": + self.record("live-permission-injection", {"node": node, **self.remote(node, "drive-gate-activate")}) def deploy(self, binary_sha, bucket, access, secret, *, before_start=None): self.secret_values.update({access, secret}) diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 78fcefa1..1daed17e 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 9c8e3a0e..fca71e5a 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -27,7 +27,7 @@ case "${case_id}" in heal.admin-overlap-and-force-start) driver=overlap.py ;; heal.root-graceful-coordinator-restart) driver=root_restart.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/stale_rejoin.py b/scripts/validation_heal/stale_rejoin.py new file mode 100644 index 00000000..7a2cf74c --- /dev/null +++ b/scripts/validation_heal/stale_rejoin.py @@ -0,0 +1,254 @@ +"""C06: real offline member, acknowledged transitions and corrected authority.""" + +import copy +import hashlib + +try: + from .heal_probe import Probe, canonical, payload, require + from .heal_workload import verify_seed + from .healthy import physical_content + from .shards import files, map_target, run_task, verify_cohort + from .markers import add_marker, verify_markers + from .scanner_repair import JournalWindow +except ImportError: + from heal_probe import Probe, canonical, payload, require + from heal_workload import verify_seed + from healthy import physical_content + from shards import files, map_target, run_task, verify_cohort + from markers import add_marker, verify_markers + from scanner_repair import JournalWindow + + +BRANCHES = ("unversioned-overwrite", "delete-historical-version", "delete-marker") + + +def version_id(value): + return "null" if value in {None, "", "null", "00000000-0000-0000-0000-000000000000"} else value + + +def logical_metadata(metadata): + result = copy.deepcopy(metadata) + for version in result.get("versions", []): + version["versionId"] = version_id(version.get("versionId")) + version.pop("role", None) + for name in ("index", "checksums"): + version.get("erasure", {}).pop(name, None) + return result + + +def identities(metadata): + return {(version_id(entry.get("versionId")), entry.get("deleted"), entry.get("isLatest")) + for entry in metadata.get("versions", [])} + + +def transition(recorder, bucket, branch, key, expected): + manifest = copy.deepcopy(expected) + data = [entry for entry in manifest if entry["key"] == key and not entry["deleteMarker"]] + require(data and not any(entry["deleteMarker"] for entry in manifest if entry["key"] == key), + "stale transition must begin from acknowledged data without a current marker") + if branch == "unversioned-overwrite": + require(len(data) == 1 and data[0]["versionId"] == "null", "overwrite requires a never-versioned object") + nonce, size = 1, data[0]["payload"]["size"] + body = payload(f"{bucket}/{key}/{nonce}", size) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": "DB", "nonce": "offline-overwrite"}, + "Tagging": "suite=heal&fixture=DB"}, body) + require(response.get("VersionId") in {None, "null"}, "unversioned overwrite created an unexpected version") + data[0].update(payload={"size": size, "sha256": hashlib.sha256(body).hexdigest()}, payloadNonce=nonce) + return manifest, {"operation": "put_object", "newPayload": data[0]["payload"], "backupRestorationAllowed": False} + if branch == "delete-historical-version": + require(len(data) == 3 and not data[0]["isLatest"] and data[0]["versionId"] != "null", "delete requires a precise historical version") + removed = data[0] + response = recorder.call("delete_object", {"Bucket": bucket, "Key": key, "VersionId": removed["versionId"]}) + require(response.get("VersionId") == removed["versionId"] and response.get("DeleteMarker") in {None, False} + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 204, "historical DELETE lost its exact data-version ACK") + return [entry for entry in manifest if entry != removed], {"operation": "delete_object_version", + "deletedVersionId": removed["versionId"], "deletedDataDirFromOriginalRequired": True, "backupRestorationAllowed": False} + require(branch == "delete-marker" and all(entry["versionId"] != "null" for entry in data), "marker branch requires enabled versions") + response = recorder.call("delete_object", {"Bucket": bucket, "Key": key}) + return add_marker(manifest, key, response), {"operation": "delete_object", "deleteMarkerId": response.get("VersionId"), + "backupRestorationAllowed": False} + + +def authority_witness(before, authority, bucket, key, expected, target): + left, right = files(before), files(authority) + member = target["node"], target["disk"] + path, prefix = f"{bucket}/{key}/xl.meta", f"{bucket}/{key}/" + require({node: value["formats"] for node, value in before.items()} == + {node: value["formats"] for node, value in authority.items()}, "stale transition changed disk UUID/slot/format") + require({identity: entry for identity, entry in left.items() if identity[:2] == member} == + {identity: entry for identity, entry in right.items() if identity[:2] == member}, "offline member changed during the acknowledged transition") + require({identity: entry for identity, entry in left.items() if not identity[2].startswith(prefix)} == + {identity: entry for identity, entry in right.items() if not identity[2].startswith(prefix)}, "transition changed a control or another key") + selected = [entry for entry in expected if entry["key"] == key] + wanted = {(entry["versionId"], entry["deleteMarker"], entry["isLatest"]) for entry in selected} + donors, normalized = [], None + for shard in target["cohort"]: + identity = shard["node"], shard["disk"], path + if identity[:2] == member: continue + metadata = right.get(identity, {}).get("metadata") + require(metadata and metadata.get("bucket") == bucket and metadata.get("key") == key + and identities(metadata) == wanted and len(metadata["versions"]) == len(wanted), + "online member lacks the full acknowledged version/marker authority") + logical = logical_metadata(metadata) + require(normalized is None or canonical(logical) == canonical(normalized), "online same-version metadata does not form one full authority") + normalized = logical + donors.append(identity[:2]) + geometry = target["logical"]["erasure"] + read = geometry["data_blocks"] + write = read + int(read == geometry["parity_blocks"]) + require(len(donors) == len(target["cohort"]) - 1 >= max(read, write), "one member loss cannot admit this layout's read/write quorum") + stale = right.get((*member, path), {}).get("metadata") + require(stale and canonical(logical_metadata(stale)) != canonical(normalized), "NOT_OBSERVED: rejoin member was not genuinely stale") + return {"readQuorum": read, "writeQuorum": write, "onlineAuthorities": len(donors), "member": list(member), + "staleMetadataSha256": right[(*member, path)]["sha256"], "logicalAuthority": normalized, + "offlineMemberAllOriginalFilesPreserved": True, "originalFormatsAndControlFilesPreserved": True} + + +def physical_rejoin(authority, after, bucket, key, expected, target, witness): + left, right = files(authority), files(after) + path, prefix, member = f"{bucket}/{key}/xl.meta", f"{bucket}/{key}/", (target["node"], target["disk"]) + require({node: value["formats"] for node, value in authority.items()} == + {node: value["formats"] for node, value in after.items()}, "rejoin changed original format/disk/set identities") + # Only the stale member's selected key may gain/remove payload files. All + # existing quorum shards and every control remain exact independent oracles. + for identity, entry in left.items(): + if identity[:2] == member and identity[2].startswith(prefix): continue + actual = right.get(identity) + if identity[2] == path: + require(actual and canonical(logical_metadata(actual.get("metadata", {}))) == canonical(witness["logicalAuthority"]), + "repair changed the committed online metadata authority") + else: + require(actual == entry, "stale repair lost or changed a quorum shard or control") + for identity in set(right) - set(left): + require(identity[:2] == member and identity[2].startswith(prefix), "stale repair created a file outside its exact member/key") + current = right.get((*member, path), {}) + require(current.get("metadata") and current.get("sha256") != witness["staleMetadataSha256"] + and canonical(logical_metadata(current["metadata"])) == canonical(witness["logicalAuthority"]), + "stale member did not converge to the frozen ACK authority") + live_directories = {entry.get("dataDir") for entry in witness["logicalAuthority"]["versions"] if not entry.get("deleted")} + for identity, entry in left.items(): + if identity[:2] == member and any(identity[2].startswith(prefix + data_dir + "/") + for data_dir in live_directories if data_dir) and identity[2].rsplit("/", 1)[-1].startswith("part."): + require(right.get(identity) == entry, "stale repair changed an original retained same-drive data shard") + cohorts = [] + for entry in expected: + if entry["key"] != key or entry["deleteMarker"]: continue + cohort = map_target(after, bucket, key, "data", 1, entry["versionId"], latest_required=False) + require(cohort["logical"]["size"] == entry["payload"]["size"], "rejoined payload metadata differs from its ACK length") + cohorts.append(cohort) + require(cohorts, "rejoined key lost every retained acknowledged data version") + return cohorts, {"staleMemberMatchesFrozenAuthority": True, "sameDiskUUIDAndSetSlot": True, + "controlsAndExistingQuorumShardBytesUnchanged": True, "backupRestoreUsed": False, + "newGenerationProof": "frozen quorum metadata, streaming protected-shard checks and independent ACK plaintext"} + + +def responsibility(status, bucket, key, expected, *, deleted=None): + outcome = status.get("outcome") or {} + wanted = {(entry["key"], entry["versionId"]): entry for entry in expected if entry["key"] == key} + require(wanted and status.get("summary") == "finished" and status.get("truncated", False) is False + and outcome.get("execution", {}).get("state") == "completed" and outcome.get("coverage") == "complete" + and outcome.get("objectsTruncated") is False, "stale task did not preserve complete terminal responsibility") + allowed = set(wanted) | ({(key, deleted)} if deleted else set()) + seen, repairs, incarnation = set(), set(), None + objects = outcome.get("objects", []) + for receipt in objects: + identity = receipt.get("identity", {}) + pair = identity.get("object"), version_id(identity.get("versionId")) + require(pair in allowed and pair not in seen and identity.get("bucket") == bucket + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0 + and isinstance(identity.get("bucketIncarnationId"), str) and identity["bucketIncarnationId"], + "stale task mixed wrong/duplicate version, incarnation or pool/set responsibility") + incarnation = incarnation or identity["bucketIncarnationId"] + require(incarnation == identity["bucketIncarnationId"], "stale receipts mixed bucket incarnations") + state = receipt.get("disposition", {}).get("state") + if pair == (key, deleted): + require(state in {"repaired", "authoritatively_absent"}, "deleted historical version was falsely reported live or unknown") + else: + require(state in {"repaired", "metadata_healthy" if wanted[pair]["deleteMarker"] else "verified_healthy"}, + "stale version lacks an explicit correct repair/healthy disposition") + if state == "repaired": repairs.add(pair) + seen.add(pair) + require(set(wanted) <= seen and repairs, "NOT_OBSERVED: original task did not cover and repair the stale ACK namespace") + counters = outcome.get("counters", {}) + counts = {"processed": len(seen), "healed": len(repairs), "unchanged": len(seen) - len(repairs), + "failed": 0, "skipped": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False} + require(all(type(counters.get(name)) is type(value) and counters[name] == value for name, value in counts.items()), + "stale outcome counters hid errors or misrepresented actual responsibility") + return {"completeRetainedAndOptionalDeletedVersionReceipts": True, "repairedIdentities": sorted(repairs), + "bucketIncarnationId": incarnation, "counters": counters} + + +def run_branch(recorder, fleet, bucket, branch, expected, active, admissions, contract): + # Lazy imports keep the shared version executor's dependency graph acyclic. + try: + from .versions import verify_version_s3, public_version_id + except ImportError: + from versions import verify_version_s3, public_version_id + require(contract.get("recognized") is True and contract.get("writeQuorumRule") == "data-plus-one-if-equal-parity", + "stale rejoin requires a recognized candidate read/write policy") + key = "objects/0003.bin" if branch == "delete-marker" else "objects/0001.bin" + selected = [entry for entry in expected if entry["key"] == key and not entry["deleteMarker"]] + require(selected, "stale branch lost its target ACK") + before = fleet.snapshot("stale-original-" + branch, quiescent=True) + target = map_target(before, bucket, key, "data", 1, public_version_id(selected[0]["versionId"]), latest_required=False) + # Pick an actual member from decoded geometry, keeping coordinator node1 + # separate on H2. Four disks are one lock host, not four independent peers. + shard = next(item for item in reversed(target["cohort"]) if item["node"] == fleet.nodes[-1]) + target = {**target, **shard} + verify_cohort(fleet, target, "stale-baseline-bitrot-" + branch) + journal = JournalWindow(fleet, "stale-authority-" + branch) + node = target["node"] + fleet.drive_gates.add(node) + fleet.record("stale-member-isolated", fleet.remote(node, "drive-gate", disk=target["disk"], mode="permission-denied")) + fleet.start() + denial = fleet.record("stale-member-service-denial", fleet.remote(node, "drive-gate-probe")) + require(denial.get("readProbe", {}).get("errno") == 13 and denial.get("errno") == 13 + and denial.get("dacBypassRemoved") is True, "offline member lacks actual service-namespace read and write refusal") + manifest, ack = transition(recorder, bucket, branch, key, expected) + fleet.record("stale-transition-contract", {"branch": branch, **ack}) + baseline = fleet.record("stale-online-ack-inventory", recorder.probe.inventory(bucket, unversioned_current_tags=branch == "unversioned-overwrite")) + fleet.record("stale-online-ack-proof", {"expected": manifest, **verify_seed(manifest, baseline)}) + fleet.stop() + authority = fleet.snapshot("stale-online-authorities", quiescent=True) + witness = fleet.record("stale-generation-authority-proof", authority_witness(before, authority, bucket, key, manifest, target)) + fleet.record("stale-original-member-rejoin", fleet.remote(node, "drive-gate", mode="clear")) + fleet.drive_gates.remove(node) + fleet.start() + require(physical_content(authority) == physical_content(fleet.snapshot("stale-before-manual")), + "ATTRIBUTION_NOT_OBSERVED: stale namespace changed before its manual token") + status = run_task(recorder.probe, fleet, {"bucket": bucket, "key": key}, 2, active, + option_overrides={"recursive": True}, admissions=admissions) + fleet.stop() + after = fleet.snapshot("stale-after-manual", quiescent=True) + cohorts, physical = physical_rejoin(authority, after, bucket, key, manifest, target, witness) + streaming = [verify_cohort(fleet, cohort, "stale-repaired-streaming") for cohort in cohorts] + receipts = responsibility(status, bucket, key, manifest, deleted=ack.get("deletedVersionId")) + journal.capture() + fleet.start() + s3 = verify_version_s3(recorder.probe, fleet, bucket, manifest, baseline, "stale-s3-after", fixture="DB" if branch == "unversioned-overwrite" else "DV") + deleted_probes = [] + for endpoint in recorder.probe.binding["endpoints"]: + peer = Probe(recorder.probe.binding, endpoint) + inventory = fleet.record("stale-all-peer-inventory", {"endpoint": endpoint, **peer.inventory(bucket, + unversioned_current_tags=branch == "unversioned-overwrite")}) + verify_seed(manifest, inventory) + if ack.get("deletedVersionId"): + denied = fleet.record("stale-deleted-version-check", peer.s3("get_object", {"Bucket": bucket, "Key": key, "VersionId": ack["deletedVersionId"]})) + response = denied.get("response") or {} + require(denied.get("acknowledged") is False and denied.get("resultUnknown") is False and "payload" not in denied + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404 + and response.get("Error", {}).get("Code") in {"NoSuchKey", "NoSuchVersion"}, "deleted historical version was resurrected or its absence is unproven") + deleted_probes.append({"endpoint": endpoint, "deletedVersionId": ack["deletedVersionId"], "absent": True}) + if branch == "unversioned-overwrite": + current = fleet.record("stale-current-get", peer.s3("get_object", {"Bucket": bucket, "Key": key})) + wanted = next(entry for entry in manifest if entry["key"] == key) + require(current.get("acknowledged") is True and current.get("payload") == wanted["payload"], "latest overwrite regressed to the old plaintext") + if branch == "delete-marker": s3["markerIdentity"] = verify_markers(recorder.probe, fleet, bucket, key, manifest) + fleet.stop() + require(physical_content(after) == physical_content(fleet.snapshot("stale-after-s3", quiescent=True)), "S3 reads caused repair after manual convergence proof") + fleet.start() + return manifest, baseline, {"verdict": "PASS", "transition": ack, "authority": witness, "physical": physical, + "streaming": streaming, "responsibility": receipts, "s3": s3, "deletedVersionProbes": deleted_probes, + "generationSelectionJournalRetained": True, "manualTaskId": admissions[-1]["clientToken"]} diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 89753279..ef023e6d 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -32,6 +32,7 @@ from .scanner_repair import BRANCHES as SCANNER_BRANCHES, JournalWindow, observe_task as scanner_task from .markers import (BRANCHES as MARKER_BRANCHES, INSTANCES as MARKER_INSTANCES, seed_markers, marker_plan, prepare_stale, verify_markers, reveal_current) + from .stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch from .shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) except ImportError: @@ -49,6 +50,7 @@ from scanner_repair import BRANCHES as SCANNER_BRANCHES, JournalWindow, observe_task as scanner_task from markers import (BRANCHES as MARKER_BRANCHES, INSTANCES as MARKER_INSTANCES, seed_markers, marker_plan, prepare_stale, verify_markers, reveal_current) + from stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch from shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) @@ -70,6 +72,7 @@ "heal.disk-readonly-and-permission": ("read-only", "permission-denied"), "heal.scanner-cold-object-repair": SCANNER_BRANCHES, "heal.delete-marker-semantics": MARKER_BRANCHES, + "heal.stale-version-rejoin": STALE_BRANCHES, } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") @@ -689,6 +692,7 @@ def verify_version_s3(probe, fleet, bucket, expected, baseline, label, *, fixtur nonce = next(index for index, item in enumerate( item for item in expected if item["key"] == entry["key"] and not item["deleteMarker"]) if item["versionId"] == entry["versionId"]) + nonce = entry.get("payloadNonce", nonce) body = semantic_body(bucket, entry["key"], nonce) if fixture == "DS" else payload( f"{bucket}/{entry['key']}/{nonce}", entry["payload"]["size"]) ranges.append({"key": entry["key"], "versionId": entry["versionId"], "proof": verify_ranges( @@ -703,11 +707,12 @@ def run_topology(args, topology, nodes, fixture): run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="scanner" if args.case_id == "heal.scanner-cold-object-repair" else - "read-repair" if args.case_id == "heal.read-repair-attribution" else "manual") + "read-repair" if args.case_id == "heal.read-repair-attribution" else + "stale-rejoin" if args.case_id == "heal.stale-version-rejoin" else "manual") candidate = json.loads(args.candidate.read_text()) tool_manifest = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) contract = verify_quorum_contract(tool_manifest, candidate) if args.case_id in { - "heal.below-read-quorum", "heal.metadata-payload-mismatch"} else None + "heal.below-read-quorum", "heal.metadata-payload-mismatch", "heal.stale-version-rejoin"} else None size_rule = verify_size_contract(tool_manifest, candidate) if args.case_id == "heal.object-size-boundaries" else None bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) @@ -727,6 +732,8 @@ def run_topology(args, topology, nodes, fixture): and (args.case_id != "heal.below-read-quorum" or (branch == "r-minus-1-historical") == (fixture == "DV")) and (args.case_id != "heal.read-repair-attribution" or (branch == "historical-version-read") == (fixture == "DV")) and (args.case_id != "heal.missing-or-corrupt-xlmeta" or (branch == "versioned-xlmeta") == (fixture == "DV"))] + if args.case_id == "heal.stale-version-rejoin": + branches = [branch for branch in STALE_BRANCHES if (branch == "unversioned-overwrite") == (fixture == "DB")] result = {"topology": topology, "fixture": fixture, "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} active, admissions = [], [] probe = Probe(binding) @@ -764,10 +771,14 @@ def run_topology(args, topology, nodes, fixture): expected, public = seed_semantics(recorder, bucket) else: expected = seed(recorder, bucket, fixture, - 16 if args.case_id == "heal.multiple-shard-loss" else 4 if fixture == "DV" else 128, - [VERSION_SIZE] if args.case_id == "heal.multiple-shard-loss" or fixture == "DV" else SIZES) + 16 if args.case_id == "heal.multiple-shard-loss" else 4 if fixture == "DV" or args.case_id == "heal.stale-version-rejoin" else 128, + [VERSION_SIZE] if args.case_id in {"heal.multiple-shard-loss", "heal.stale-version-rejoin"} or fixture == "DV" else SIZES) if args.case_id == "heal.versioning-suspended-null": expected = suspend_fixture(recorder, bucket, expected) + if args.case_id == "heal.stale-version-rejoin": + for key in {entry["key"] for entry in expected}: + for nonce, entry in enumerate(entry for entry in expected if entry["key"] == key and not entry["deleteMarker"]): + entry["payloadNonce"] = nonce baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=fixture in {"DB", "DK"}, page_size=17 if fixture == "DK" else None)) fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) @@ -807,6 +818,12 @@ def run_topology(args, topology, nodes, fixture): fleet.record("lock-before-fault", prove_locked(probe, fleet, bucket, lock_context, key=LOCK_KEYS[LOCK_BRANCHES.index(branch)], label="lock-before-" + branch)) fleet.stop() + if args.case_id == "heal.stale-version-rejoin": + stale_marker_started = True + expected, baseline, proof = stale_branch(recorder, fleet, bucket, branch, expected, active, admissions, contract) + result["branches"][branch] = proof + stale_marker_started = False + continue before = fleet.snapshot("physical-before-" + branch, quiescent=True) scanner_window = JournalWindow(fleet, "scanner-fault-" + branch) if scanner else None if size_rule: @@ -1037,7 +1054,7 @@ def interrupted(signum, frame): and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") - fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.multiple-shard-loss", "heal.read-repair-attribution"} else ("DV",) + fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.multiple-shard-loss", "heal.read-repair-attribution", "heal.stale-version-rejoin"} else ("DV",) if args.case_id == "heal.object-metadata-and-tags": fixtures = ("DS",) if args.case_id == "heal.special-keys-and-pagination": diff --git a/test_validation_heal.py b/test_validation_heal.py index e81307d5..310099f6 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -34,7 +34,7 @@ from scripts.validation_heal.versions import mismatch_plan, seed_mismatch from scripts.validation_heal import heal_fault from scripts.validation_heal.read_repair import source_tasks, validate_read, tracked_read_task, ADMISSION_MESSAGE, ADMISSION_TARGET -from scripts.validation_heal.drive_gate import apply_gate, release_gate, checked_gate, start_properties, validate_probe, validate_unwritable +from scripts.validation_heal.drive_gate import apply_gate, activate_gate, release_gate, checked_gate, start_properties, validate_probe, validate_unwritable from scripts.validation_heal.scanner_repair import (source_tasks as scanner_sources, cycle_pairs, observe_task as observe_scanner, JournalWindow) from scripts.validation_heal.fleet_node import logs as node_logs @@ -53,6 +53,7 @@ from scripts.validation_heal import overlap from scripts.validation_heal import root_restart, fleet_node from scripts.validation_heal import quorum_boundary +from scripts.validation_heal import stale_rejoin import errno import stat @@ -1013,6 +1014,43 @@ def test_deleted_identity_requires_exact_s3_absence_without_payload_or_transport class HealDriveGateTest(unittest.TestCase): + def test_live_permission_activation_follows_actual_start_and_initial_readiness(self): + with tempfile.TemporaryDirectory() as directory: + fleet = Fleet(["node1"], Path(directory), "gh-12345-1-h1") + fleet.drive_gates.add("node1") + calls = [] + def remote(node, action, **kwargs): + calls.append(action) + return {"mode": "permission-denied", "phase": "armed"} if action == "gate-state" else {} + with patch.object(fleet, "remote", side_effect=remote), \ + patch.object(fleet, "wait_ready", side_effect=lambda: calls.append("ready")): + fleet.start() + self.assertEqual(calls, ["start", "ready", "gate-state", "drive-gate-activate"]) + calls.clear() + with patch.object(fleet, "remote", side_effect=remote), \ + patch.object(fleet, "wait_ready", side_effect=ValueError("startup failed")), self.assertRaises(ValueError): + fleet.start() + self.assertNotIn("drive-gate-activate", calls) + + def test_partial_live_activation_restores_from_original_armed_marker(self): + with tempfile.TemporaryDirectory() as directory: + state, root = Path(directory) / "state", Path(directory) / "root" + state.mkdir() + (root / "data").mkdir(parents=True, mode=0o700) + original = Path.lstat + def root_owned(path): + info = original(path) + return types.SimpleNamespace(st_mode=info.st_mode, st_uid=0, st_nlink=info.st_nlink, + st_size=info.st_size, st_dev=info.st_dev, st_ino=info.st_ino) + with patch.object(Path, "lstat", root_owned): + apply_gate(state, (root,), 0, "permission-denied") + with patch("scripts.validation_heal.drive_gate.os.replace", side_effect=OSError("interrupted")), self.assertRaises(OSError): + activate_gate(state, (root,)) + self.assertEqual(checked_gate(state, (root,))["phase"], "armed") + self.assertEqual(stat.S_IMODE((root / "data").stat().st_mode), 0) + self.assertTrue(release_gate(state, (root,))["originalPermissionsRestored"]) + self.assertEqual(stat.S_IMODE((root / "data").stat().st_mode), 0o700) + def test_gate_only_changes_owned_permissions_and_restores_partial_intent(self): for mode in ("read-only", "permission-denied"): with self.subTest(mode=mode), tempfile.TemporaryDirectory() as directory: @@ -1032,7 +1070,14 @@ def root_owned(path): gate = apply_gate(state, (root,), 0, mode) self.assertEqual(checked_gate(state, (root,))["mode"], mode) self.assertFalse(gate["hostMountRemounted"]) - self.assertEqual(stat.S_IMODE((root / "data").stat().st_mode), 0 if mode == "permission-denied" else 0o700) + self.assertEqual(stat.S_IMODE((root / "data").stat().st_mode), 0o700) + if mode == "permission-denied": + self.assertEqual(gate["phase"], "armed") + activated = activate_gate(state, (root,)) + self.assertTrue(activated["injectedAfterActualServiceStart"]) + self.assertEqual(stat.S_IMODE((root / "data").stat().st_mode), 0) + self.assertEqual(checked_gate(state, (root,))["phase"], "active") + with self.assertRaises(ValueError): activate_gate(state, (root,)) self.assertEqual(stat.S_IMODE(other.stat().st_mode), 0o700) with self.assertRaisesRegex(ValueError, "already active"): apply_gate(state, (root,), 0, mode) @@ -1065,6 +1110,10 @@ def test_unwritable_receipt_never_accepts_repaired_or_hidden_failures(self): status["outcome"]["execution"]["state"] = "completed_with_errors" status["outcome"]["counters"].update(healed=0, unchanged=0, failed=1, attemptFailures=1) self.assertFalse(validate_unwritable(status, bucket, entry)["repairClaimedWhileUnwritable"]) + stopped = {**copy.deepcopy(status), "summary": "stopped"} + validate_unwritable(stopped, bucket, entry) + stopped["outcome"]["execution"]["state"] = "aborted" + with self.assertRaises(ValueError): validate_unwritable(stopped, bucket, entry) for invalid in ("repaired", "metadata_healthy", "unknown", "wrong-version", "healed-count", "hidden-failure", "incarnation"): changed = copy.deepcopy(status) if invalid in {"repaired", "metadata_healthy", "unknown"}: changed["outcome"]["objects"][0]["disposition"]["state"] = invalid @@ -2536,5 +2585,151 @@ def test_branch_and_geometry_contract_cannot_be_weakened(self): with self.assertRaises(ValueError): quorum_boundary.boundary_plan("exact-r-read", snapshot, self.bucket, expected * 2, self.contract) +class HealStaleRejoinTest(unittest.TestCase): + def scene(self): + before, bucket, expected = versioned_scene(["objects/0001.bin", "control/object.bin"]) + for key in {entry["key"] for entry in expected}: + for nonce, entry in enumerate(entry for entry in expected if entry["key"] == key): entry["payloadNonce"] = nonce + target = map_target(before, bucket, "objects/0001.bin", "data", 1, "v0", latest_required=False) + manifest = [entry for entry in expected if (entry["key"], entry["versionId"]) != (target["key"], "v0")] + authority = copy.deepcopy(before) + member = target["node"], target["disk"] + for node, value in authority.items(): + for entry in value["files"]: + if (node, entry["disk"]) == member: continue + metadata = entry.get("metadata") + if metadata and metadata["key"] == target["key"]: + metadata["versions"] = [version for version in metadata["versions"] if version["versionId"] != "v0"] + entry["sha256"] += "-deleted-v0" + value["files"] = [entry for entry in value["files"] if (node, entry["disk"]) == member + or not entry["path"].startswith(f"{bucket}/{target['key']}/12345678-1234-1234-1234-000000000000/")] + witness = stale_rejoin.authority_witness(before, authority, bucket, target["key"], manifest, target) + after = copy.deepcopy(authority) + for entry in after[member[0]]["files"]: + if entry["disk"] == member[1] and entry.get("metadata", {}).get("key") == target["key"]: + entry["metadata"]["versions"] = [version for version in entry["metadata"]["versions"] if version["versionId"] != "v0"] + entry["sha256"] += "-deleted-v0" + return before, authority, after, bucket, manifest, target, witness + + def test_same_disk_stale_generation_requires_full_online_ack_authority(self): + before, authority, after, bucket, expected, target, witness = self.scene() + self.assertEqual((witness["readQuorum"], witness["writeQuorum"], witness["onlineAuthorities"]), (2, 3, 3)) + cohorts, proof = stale_rejoin.physical_rejoin(authority, after, bucket, target["key"], expected, target, witness) + self.assertEqual({cohort["versionId"] for cohort in cohorts}, {"v1", "v2"}) + self.assertFalse(proof["backupRestoreUsed"]) + for kind in ("offline-write", "lost-online-version", "no-stale-generation", "control", "format"): + changed = copy.deepcopy(authority) + if kind == "offline-write": next(entry for entry in changed[target["node"]]["files"] if entry["disk"] == target["disk"])["sha256"] = "changed" + elif kind == "lost-online-version": + next(entry for entry in changed["node1"]["files"] if entry["disk"] != target["disk"] + and entry.get("metadata", {}).get("key") == target["key"])["metadata"]["versions"].pop() + elif kind == "no-stale-generation": changed = copy.deepcopy(after) + elif kind == "control": next(entry for entry in changed["node1"]["files"] if "/control/" in entry["path"])["sha256"] = "changed" + else: changed["node1"]["formats"][0]["value"]["id"] = "foreign" + with self.subTest(kind=kind), self.assertRaises(ValueError): + stale_rejoin.authority_witness(before, changed, bucket, target["key"], expected, target) + + def test_physical_convergence_cannot_change_retained_shards_or_quorum_controls(self): + _, authority, after, bucket, expected, target, witness = self.scene() + for kind in ("retained-target-shard", "quorum-shard", "control", "resurrected-metadata", "extra-file"): + changed = copy.deepcopy(after) + if kind in {"retained-target-shard", "quorum-shard"}: + entry = next(entry for entry in changed["node1"]["files"] if entry["path"].endswith("/part.1") + and target["key"] in entry["path"] and "000000000001" in entry["path"] + and (entry["disk"] == target["disk"]) == (kind == "retained-target-shard")) + entry["sha256"] = "corrupt" + elif kind == "control": next(entry for entry in changed["node1"]["files"] if "/control/" in entry["path"])["sha256"] = "changed" + elif kind == "resurrected-metadata": + entry = next(entry for entry in changed["node1"]["files"] if entry["disk"] == target["disk"] and entry.get("metadata", {}).get("key") == target["key"]) + entry["metadata"] = copy.deepcopy(next(entry for entry in authority["node1"]["files"] if entry["disk"] == target["disk"] and entry.get("metadata", {}).get("key") == target["key"])["metadata"]) + else: changed["node1"]["files"].append({"disk": 0, "path": f"{bucket}/unrelated/new.bin", "size": 1, "sha256": "extra"}) + with self.subTest(kind=kind), self.assertRaises(ValueError): + stale_rejoin.physical_rejoin(authority, changed, bucket, target["key"], expected, target, witness) + + def test_historical_delete_keeps_original_payload_nonce_and_exact_ack(self): + _, _, _, bucket, expected_after, target, _ = self.scene() + _, _, expected = versioned_scene([target["key"]]) + for nonce, entry in enumerate(expected): entry["payloadNonce"] = nonce + recorder = Mock() + recorder.call.return_value = {"VersionId": "v0", "ResponseMetadata": {"HTTPStatusCode": 204}} + manifest, ack = stale_rejoin.transition(recorder, bucket, "delete-historical-version", target["key"], expected) + self.assertEqual([(entry["versionId"], entry["payloadNonce"]) for entry in manifest], [("v1", 1), ("v2", 2)]) + self.assertEqual(ack["deletedVersionId"], "v0") + self.assertFalse(ack["backupRestorationAllowed"]) + for changed in ({"VersionId": "v1", "ResponseMetadata": {"HTTPStatusCode": 204}}, + {"VersionId": "v0", "DeleteMarker": True, "ResponseMetadata": {"HTTPStatusCode": 204}}): + recorder.call.return_value = changed + with self.assertRaises(ValueError): stale_rejoin.transition(recorder, bucket, "delete-historical-version", target["key"], expected) + + def test_overwrite_uses_new_nonce_and_marker_preserves_all_acknowledged_versions(self): + key, bucket = "objects/0001.bin", "rv-heal-0123456789abcdef" + old = [{"key": key, "versionId": "null", "deleteMarker": False, "isLatest": True, + "payload": {"size": 100, "sha256": "old"}, "payloadNonce": 0}] + recorder = Mock() + recorder.call.return_value = {"ResponseMetadata": {"HTTPStatusCode": 200}} + manifest, ack = stale_rejoin.transition(recorder, bucket, "unversioned-overwrite", key, old) + self.assertEqual(manifest[0]["payloadNonce"], 1) + self.assertEqual(manifest[0]["payload"]["sha256"], hashlib.sha256(stale_rejoin.payload(f"{bucket}/{key}/1", 100)).hexdigest()) + self.assertEqual(old[0]["payload"]["sha256"], "old") + before, _, expected = versioned_scene([key]) + recorder.call.return_value = {"VersionId": "new-marker", "DeleteMarker": True, "ResponseMetadata": {"HTTPStatusCode": 204}} + manifest, ack = stale_rejoin.transition(recorder, bucket, "delete-marker", key, expected) + self.assertEqual(len(manifest), 4) + self.assertTrue(manifest[-1]["isLatest"] and manifest[-1]["deleteMarker"]) + self.assertTrue(all(not entry["isLatest"] for entry in manifest[:-1])) + + def test_stale_receipts_reject_unknown_resurrection_and_false_convergence(self): + _, _, _, bucket, expected, target, _ = self.scene() + selected = [entry for entry in expected if entry["key"] == target["key"]] + state = HealHealthyOracleTest().receipt("repaired") + template = state["outcome"]["objects"][0] + receipts = [] + for entry in selected: + receipt = copy.deepcopy(template) + receipt["identity"].update(bucket=bucket, object=target["key"], versionId=entry["versionId"]) + receipt["disposition"] = {"state": "repaired" if entry["versionId"] == "v1" else "verified_healthy"} + receipts.append(receipt) + absent = copy.deepcopy(template) + absent["identity"].update(bucket=bucket, object=target["key"], versionId="v0") + absent["disposition"] = {"state": "authoritatively_absent"} + state["outcome"]["objects"] = [*receipts, absent] + state["outcome"]["counters"].update(processed=3, healed=1, unchanged=2) + self.assertTrue(stale_rejoin.responsibility(state, bucket, target["key"], expected, deleted="v0")["completeRetainedAndOptionalDeletedVersionReceipts"]) + for kind in ("unknown", "live-deleted", "no-repair", "missing-retained", "incarnation", "wrong-count"): + changed = copy.deepcopy(state) + if kind == "unknown": changed["outcome"]["objects"][0]["disposition"]["state"] = "unknown" + elif kind == "live-deleted": changed["outcome"]["objects"][-1]["disposition"]["state"] = "verified_healthy" + elif kind == "no-repair": changed["outcome"]["objects"][0]["disposition"]["state"] = "verified_healthy" + elif kind == "missing-retained": changed["outcome"]["objects"].pop(1) + elif kind == "incarnation": changed["outcome"]["objects"][1]["identity"]["bucketIncarnationId"] = "foreign" + else: changed["outcome"]["counters"]["healed"] += 1 + with self.subTest(kind=kind), self.assertRaises(ValueError): + stale_rejoin.responsibility(changed, bucket, target["key"], expected, deleted="v0") + + def test_stale_runtime_and_source_branches_are_explicit(self): + self.assertEqual(list(stale_rejoin.BRANCHES), select("C06")[0]["variants"]) + config = runtime_config("stale-rejoin") + self.assertEqual(config["RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS"], "0") + self.assertEqual(config["RUSTFS_HEAL_MRF_ENABLE"], "false") + + def test_range_oracle_uses_ack_nonce_after_historical_deletion(self): + from scripts.validation_heal.versions import verify_version_s3 + bucket, key = "rv-heal-0123456789abcdef", "objects/0001.bin" + expected = [{"key": key, "versionId": f"v{nonce}", "deleteMarker": False, "isLatest": nonce == 2, + "payload": {"size": 1024**2 + 47}, "payloadNonce": nonce} for nonce in (1, 2)] + baseline = {"entries": [{**entry, "properties": {}} for entry in expected]} + probe, fleet = Mock(), Mock() + probe.inventory.return_value = baseline + probe.s3.return_value = {"acknowledged": True, "response": {}} + fleet.record.side_effect = lambda label, value: value + with patch("scripts.validation_heal.versions.compare_inventories", return_value={}), \ + patch("scripts.validation_heal.versions.payload", side_effect=lambda label, size: label.encode()) as generator, \ + patch("scripts.validation_heal.versions.verify_ranges", return_value=[]) as ranges: + verify_version_s3(probe, fleet, bucket, expected, baseline, "after-deletion") + self.assertEqual([call.args[0] for call in generator.call_args_list], [f"{bucket}/{key}/1", f"{bucket}/{key}/2"]) + self.assertEqual([call.kwargs["expected_body"] for call in ranges.call_args_list], + [f"{bucket}/{key}/1".encode(), f"{bucket}/{key}/2".encode()]) + + if __name__ == "__main__": unittest.main() From dec4115134d29a1dd57c149f967b3648295b781c Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 03:56:51 +0800 Subject: [PATCH 068/343] test(validation): migrate original disk return with degraded ACK repair --- docs/validation-heal-contract.md | 2 +- docs/validation-migration-map.md | 39 +++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/disk_loss.py | 259 +++++++++++++++++++++++ scripts/validation_heal/drive_gate.py | 3 +- scripts/validation_heal/heal_probe.py | 9 +- scripts/validation_heal/heal_workload.py | 8 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 14 +- test_validation_heal.py | 130 ++++++++++++ test_validation_heal_authorization.py | 28 +++ 12 files changed, 484 insertions(+), 14 deletions(-) create mode 100644 scripts/validation_heal/disk_loss.py diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index ae4f4197..cfe6ebd2 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -35 个用例已接入自动分发：A01、A03–A08、B02、B05、B07、B09、C01–C05、C08、D01–D07、D09、D10、E01–E04、F01、F06、J01、J03、J04。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。其余 49 项保留源 operator-assisted 方式。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 +38 个用例已接入自动分发：A01、A03–A08、B01、B02、B05–B07、B09、C01–C06、C08、D01–D07、D09、D10、E01–E04、F01、F06、J01、J03、J04。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。其余 46 项保留源 operator-assisted 方式。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 0b169c28..16a5f6f5 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 111 currently dispatchable IDs include all 74 non-Heal cases and thirty-seven Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. +case evidence. The 112 currently dispatchable IDs include all 74 non-Heal cases and thirty-eight Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 49 Heal cases remain +The other 46 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -650,7 +650,7 @@ A08 `heal.admin-authorization` passed live in RustFS Actions run `38069648003`, harness `0789743d439fd91b967c2d6fba85c06e29f844c5`. All five source branches passed independently on H1 and H2; the canonical gate, complete summary, cleanup and workflow all passed with the fixed candidate. -The prior A08 failures remain recorded. Forty-seven Heal executors remain +The prior A08 failures remain recorded. Forty-six Heal executors remain to be migrated; automatic dispatch and local tests do not claim live PASS. @@ -683,3 +683,36 @@ Permission gates now retain an armed recovery marker during initial startup and inject chmod only into the same live candidate with DAC bypass already removed. This fixes the injection window without certifying the old run or relaxing physical/refusal/responsibility assertions. A fresh run is required. + + +B01 `heal.disk-loss-and-repair` now dispatches both source branches on H1/H2 +with the full 128-object DB manifest, including zero/inline/external objects. +The same-service permission gate must deny reads and writes on one exact +original disk, and signed read-only Admin info must independently identify +that exact formatted UUID/set slot/endpoint as offline while all other +members remain online. The double witness brackets every unique-key PUT ACK +and degraded GET. Original member files, controls and formats remain exact +through the offline window; all N-1 online new-key metadata replicas and +payload shards must form one protected ACK authority. Original-drive-return +admits one new key and offline-ack-repair admits six independent new keys. +The same disk returns without restoring backups; each ACK needs its own +manual task, complete repaired receipt, actual new target metadata/part, +unchanged donor shards, protected streaming checks and full all-peer S3 +manifests. Ready/GET alone never prove repair. Live evidence is pending. + +B06 run `38081025407` retained FAIL and cleanup PASS. Its exact-R GETs on +both H1/H2 returned correct full payloads, but the collector omitted +`resultUnknown=false` on successful responses and the boundary check rejected +them. Successful complete S3 responses now explicitly expose known results; +partial bodies still fail. Both exact-R manual branches and H2 restored-W +passed within that failed run. H1's R-1 read returned HTTP 500/InternalError +instead of the frozen 503/SlowDownRead contract; that failure remains visible. +No failed run is promoted to PASS. + +C06 run `38081269532` retained FAIL and cleanup PASS before injection. Its +original ACK nonce was incorrectly compared as if it were an S3 property. +Inventory checks now validate the nonce as private collector state while +retaining exact version/marker/currentness/length/digest comparisons. A +regression check exercises nonce-bearing ACKs against real public inventory +shape, including wrong content and version negatives. Fresh live evidence +is required for both corrected collectors. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index e1bd39fc..a6262396 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/disk_loss.py b/scripts/validation_heal/disk_loss.py new file mode 100644 index 00000000..81b6820a --- /dev/null +++ b/scripts/validation_heal/disk_loss.py @@ -0,0 +1,259 @@ +"""B01: original member return and physical repair of degraded write ACKs.""" + +import copy +import hashlib +import time + +try: + from .heal_probe import Probe, canonical, compare_inventories, payload, require + from .heal_workload import verify_seed + from .healthy import physical_content + from .shards import files, map_target, run_task + from .stale_rejoin import logical_metadata, identities +except ImportError: + from heal_probe import Probe, canonical, compare_inventories, payload, require + from heal_workload import verify_seed + from healthy import physical_content + from shards import files, map_target, run_task + from stale_rejoin import logical_metadata, identities + + +BRANCHES = ("original-drive-return", "offline-ack-repair") +SIZE = 5 * 1024**2 + 123 + + +def quorum_plan(target, contract): + require(contract.get("recognized") is True + and contract.get("writeQuorumRule") == "data-plus-one-if-equal-parity", + "disk loss requires the recognized candidate quorum policy") + geometry = target["logical"]["erasure"] + read, parity = geometry["data_blocks"], geometry["parity_blocks"] + total, write = read + parity, read + int(read == parity) + require(total == len(target["cohort"]) and total in {4, 16} and total - 1 >= max(read, write), + "one member loss cannot admit the actual layout's degraded read/write ACKs") + return {"totalDrives": total, "readQuorum": read, "writeQuorum": write, + "onlineSameSetDrives": total - 1, "diskUUID": target["diskUUID"], + "setSlot": target["setSlot"], "node": target["node"], "disk": target["disk"], + "lockAdmission": "each write's actual ACK; no disk-count lock inference"} + + +def info_disks(observed, total): + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), + "member-state Admin response is unavailable") + servers = observed["response"].get("info", {}).get("servers") + require(isinstance(servers, list) and servers, "Admin info omitted its actual server members") + disks = [disk for server in servers for disk in server.get("drives", [])] + require(len(disks) == total, "Admin info omitted or duplicated a set member") + indexed = {} + for disk in disks: + slot = tuple(disk.get(name) for name in ("pool_index", "set_index", "disk_index")) + require(all(type(value) is int for value in slot) and slot[:2] == (0, 0) + and 0 <= slot[2] < total and slot not in indexed + and isinstance(disk.get("endpoint"), str) and disk["endpoint"], + "Admin info lacks distinct exact endpoint/pool/set/disk identities") + indexed[slot] = disk + return indexed + + +def member_state(observed, target, total, baseline=None): + indexed = info_disks(observed, total) + slot = (0, 0, target["setSlot"]) + selected = indexed[slot] + expected_uuids = {(0, 0, shard["setSlot"]): shard["diskUUID"] for shard in target["cohort"]} + if baseline is None: + require(all(disk.get("state") == "ok" and disk.get("runtimeState") in {None, "online"} + and isinstance(disk.get("uuid"), str) and disk["uuid"] for disk in indexed.values()), + "baseline Admin members were not positively online") + require(selected["uuid"] == target["diskUUID"], "Admin slot differs from the original physical disk UUID") + require(all(indexed[member_slot]["uuid"] == expected_uuid for member_slot, expected_uuid in expected_uuids.items()), + "Admin topology differs from the actual formatted full-set membership") + return {"endpoint": selected["endpoint"], "slot": list(slot), "uuid": selected["uuid"]} + require(selected["endpoint"] == baseline["endpoint"] and selected.get("uuid") in {"", baseline["uuid"]}, + "offline Admin witness changed the original member's endpoint/UUID") + for other_slot, disk in indexed.items(): + if other_slot == slot: continue + require(disk.get("state") == "ok" and disk.get("runtimeState") in {None, "online"} + and disk.get("uuid") == expected_uuids[other_slot], + "another Admin member is unavailable; single-drive admission is unproven") + # The pinned candidate counts these explicit disk errors as offline. An + # unknown/initializing server or aggregate count never witnesses this disk. + require(selected.get("state") in {"disk not found", "disk access denied", "file access denied", + "volume access denied", "offline"} or selected.get("runtimeState") == "offline", + "NOT_OBSERVED: exact gated member is not explicitly offline in Admin info") + return {**baseline, "reportedUUID": selected.get("uuid"), "state": selected.get("state"), + "runtimeState": selected.get("runtimeState"), "exactOriginalMemberOffline": True} + + +def offline_ack(recorder, fleet, bucket, branch, expected): + require(branch in BRANCHES, "unknown disk loss branch") + manifest, new = copy.deepcopy(expected), [] + # Each branch admits new keys while the original member is genuinely + # inaccessible; neither overwrites controls nor reuses another branch ACK. + for index in range(1 if branch == "original-drive-return" else 6): + key = f"disk-loss/{branch}/new-{index:02d}.bin" + require(not any(entry["key"] == key for entry in manifest), "offline key already has a fixture authority") + body = payload(f"{bucket}/{key}/0", SIZE + index) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": "DB", "branch": branch}, + "Tagging": "suite=heal&fixture=DB"}, body) + require(response.get("VersionId") in {None, "null"}, "degraded unique PUT created a foreign version") + entry = {"key": key, "versionId": "null", "deleteMarker": False, "isLatest": True, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}} + manifest.append(entry); new.append(entry) + # GET every new ACK plus the complete original manifest below. The gate + # remains active and its final witness brackets all operations. + observed = fleet.record("disk-loss-degraded-get", recorder.probe.s3("get_object", {"Bucket": bucket, "Key": key})) + require(observed.get("acknowledged") is True and observed.get("payload") == entry["payload"], + "degraded GET differs from the exact unique-key ACK") + return manifest, new + + +def authority_witness(before, authority, bucket, target, new): + left, right = files(before), files(authority) + member = target["node"], target["disk"] + require({node: value["formats"] for node, value in before.items()} == + {node: value["formats"] for node, value in authority.items()}, "offline ACKs changed original disk UUID/slot") + require(all(right.get(identity) == entry for identity, entry in left.items()), + "offline ACKs changed an existing original object/shard/control") + require({identity: entry for identity, entry in left.items() if identity[:2] == member} == + {identity: entry for identity, entry in right.items() if identity[:2] == member}, + "offline member received writes during degraded ACKs") + additions = [identity for identity in right if identity not in left] + prefixes = [f"{bucket}/{entry['key']}/" for entry in new] + require(new and len({entry["key"] for entry in new}) == len(new) + and all(identity[:2] != member and any(identity[2].startswith(prefix) for prefix in prefixes) + for identity in additions), "degraded ACKs wrote outside their exact new-key responsibility") + frozen = {} + for entry in new: + logical = None + for shard in target["cohort"]: + peer = shard["node"], shard["disk"] + if peer == member: continue + path = f"{bucket}/{entry['key']}/xl.meta" + metadata = right.get((*peer, path), {}).get("metadata") + require(metadata and metadata.get("bucket") == bucket and metadata.get("key") == entry["key"] + and identities(metadata) == {("null", False, True)} and len(metadata["versions"]) == 1, + "online member lacks the exact unique-key ACK authority") + normalized = logical_metadata(metadata) + require(logical is None or canonical(logical) == canonical(normalized), + "degraded new-key metadata lacks one full online authority") + logical = normalized + version = metadata["versions"][0] + require(version.get("inline") is None and version.get("size") == entry["payload"]["size"] + and version.get("dataDir") and version.get("usesLegacyChecksum") is False, + "degraded new-key ACK lacks a protected external part") + part = right.get((*peer, f"{bucket}/{entry['key']}/{version['dataDir']}/part.1")) + require(part and part["size"] > 32, "online ACK authority omitted its actual payload shard") + frozen[entry["key"]] = logical + return {"logicalAuthorities": frozen, "originalMemberUnchangedDuringAllACKs": True, + "originalControlsAndFormatsUnchanged": True, "onlineAuthorities": len(target["cohort"]) - 1} + + +def repaired_physical(authority, after, bucket, target, new, witness): + left, right = files(authority), files(after) + member = target["node"], target["disk"] + require({node: value["formats"] for node, value in authority.items()} == + {node: value["formats"] for node, value in after.items()}, "return changed original disk UUID/slot") + metadata_paths = {f"{bucket}/{entry['key']}/xl.meta": entry["key"] for entry in new} + for identity, entry in left.items(): + if identity[2] in metadata_paths: + metadata = right.get(identity, {}).get("metadata") + require(metadata and canonical(logical_metadata(metadata)) == canonical( + witness["logicalAuthorities"][metadata_paths[identity[2]]]), "repair changed online ACK metadata authority") + else: + require(right.get(identity) == entry, "repair changed an original control or surviving payload shard") + cohorts, allowed = [], set() + for entry in new: + cohort = map_target(after, bucket, entry["key"], "data", 1) + require(canonical(logical_metadata(cohort["metadata"])) == canonical(witness["logicalAuthorities"][entry["key"]]), + "rejoined key differs from its frozen degraded ACK authority") + shard = next(item for item in cohort["cohort"] if (item["node"], item["disk"]) == member) + part = (*member, shard["relativePath"]) + metadata = (*member, f"{bucket}/{entry['key']}/xl.meta") + require(part not in left and metadata not in left and part in right and metadata in right, + "NOT_OBSERVED: original member lacks newly reconstructed ACK metadata/payload") + require(shard["diskUUID"] == target["diskUUID"] and shard["setSlot"] == target["setSlot"], + "repair used a replacement disk or different member slot") + allowed.update((part, metadata)); cohorts.append(cohort) + require(set(right) - set(left) == allowed, "repair created unexpected or missing files outside the original member's ACKs") + return cohorts, {"sameDiskUUIDAndSetSlot": True, "allDegradedACKTargetPartsCreated": True, + "originalAndOnlineShardBytesUnchanged": True, "backupRestorationUsed": False, + "newShardOracle": "frozen protected online metadata, actual streaming checksums, independent ACK plaintext"} + + +def run_branch(recorder, fleet, bucket, branch, expected, active, admissions, contract): + try: + from .versions import validate_version_receipts, verify_version_s3 + except ImportError: + from versions import validate_version_receipts, verify_version_s3 + before = fleet.snapshot("disk-loss-original-" + branch, quiescent=True) + target = map_target(before, bucket, "objects/0008.bin", "data", 1) + target = {**target, **next(shard for shard in reversed(target["cohort"]) if shard["node"] == fleet.nodes[-1])} + plan = fleet.record("disk-loss-quorum-plan", quorum_plan(target, contract)) + fleet.start() + baseline = member_state(fleet.record("disk-loss-admin-baseline", recorder.probe.admin( + "/rustfs/admin/v3/info", [], b"", method="GET")), target, plan["totalDrives"]) + fleet.stop() + node = target["node"] + fleet.drive_gates.add(node) + gate = fleet.record("disk-loss-gate-armed", fleet.remote(node, "drive-gate", disk=target["disk"], mode="permission-denied")) + fleet.start() + def witness_offline(label): + denial = fleet.record("disk-loss-service-read-refusal-" + label, fleet.remote(node, "drive-gate-probe")) + require(denial.get("readProbe", {}).get("errno") == 13 and denial.get("errno") == 13 + and denial.get("dacBypassRemoved") is True, "member gate lacks actual same-service read/write refusal") + deadline = time.monotonic() + 90 + while True: + observed = fleet.record("disk-loss-admin-offline-" + label, recorder.probe.admin( + "/rustfs/admin/v3/info", [], b"", method="GET")) + try: + proof = member_state(observed, target, plan["totalDrives"], baseline) + break + except ValueError as exc: + if not str(exc).startswith("NOT_OBSERVED:"): raise + require(time.monotonic() < deadline, str(exc)) + time.sleep(2) + require(gate["dataPath"] == denial["dataPath"], "service denial moved to a different logical root") + return fleet.record("disk-loss-double-witness-" + label, {"service": denial, "admin": proof}) + first = witness_offline("before-acks") + manifest, new = offline_ack(recorder, fleet, bucket, branch, expected) + baseline_s3 = fleet.record("disk-loss-degraded-full-inventory", recorder.probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("disk-loss-degraded-full-proof", {"expected": manifest, **verify_seed(manifest, baseline_s3)}) + last = witness_offline("after-acks") + fleet.stop() + authority = fleet.snapshot("disk-loss-online-authorities", quiescent=True) + witness = fleet.record("disk-loss-ack-authority", authority_witness(before, authority, bucket, target, new)) + fleet.record("disk-loss-original-member-return", fleet.remote(node, "drive-gate", mode="clear")) + fleet.drive_gates.remove(node) + fleet.start() + require(physical_content(authority) == physical_content(fleet.snapshot("disk-loss-before-manual")), + "ATTRIBUTION_NOT_OBSERVED: original member gained ACK shards before the manual token") + receipts = [] + for entry in new: + status = run_task(recorder.probe, fleet, {"bucket": bucket, "key": entry["key"]}, 2, active, + option_overrides={"recursive": True}, admissions=admissions) + receipts.append(validate_version_receipts(status, bucket, [entry], 2, + required_repairs=[(entry["key"], "null")])) + fleet.stop() + after = fleet.snapshot("disk-loss-repaired", quiescent=True) + cohorts, physical = repaired_physical(authority, after, bucket, target, new, witness) + streaming = [] + for cohort in cohorts: + for shard in cohort["cohort"]: + proof = fleet.record("disk-loss-protected-streaming-check", {"node": shard["node"], + **fleet.remote(shard["node"], "verify-part", disk=shard["disk"], relativePath=shard["relativePath"], + versionId="null", partNumber=1)}) + require(proof.get("physicalBitrotVerified") is True and proof.get("sha256") == shard["sha256"] + and proof.get("physicalSize") == shard["size"], "rejoined shard lacks a matching actual protected checksum") + streaming.append(proof) + fleet.start() + s3 = verify_version_s3(recorder.probe, fleet, bucket, manifest, baseline_s3, "disk-loss-full-s3-after", fixture="DB") + peers = [] + for endpoint in recorder.probe.binding["endpoints"]: + inventory = fleet.record("disk-loss-all-peer-s3", {"endpoint": endpoint, + **Probe(recorder.probe.binding, endpoint).inventory(bucket, unversioned_current_tags=True)}) + peers.append({"endpoint": endpoint, **verify_seed(manifest, inventory), + "fullPublicManifest": compare_inventories(baseline_s3, inventory)}) + return manifest, baseline_s3, {"verdict": "PASS", "quorums": plan, "beforeACKs": first, "afterACKs": last, + "degradedUniqueACKs": len(new), "responsibility": receipts, "physical": physical, + "streaming": streaming, "s3": s3, "allPeerS3": peers} diff --git a/scripts/validation_heal/drive_gate.py b/scripts/validation_heal/drive_gate.py index 0255e0ba..fc2bc94e 100644 --- a/scripts/validation_heal/drive_gate.py +++ b/scripts/validation_heal/drive_gate.py @@ -163,7 +163,8 @@ def probe_gate(state, roots, pid): require(result.returncode == 0, "service-namespace read probe did not complete") read = json.loads(result.stdout) validate_probe(read, value["mode"], identity) - return {**validate_probe(observed, value["mode"], identity), "disk": value["disk"], "readProbe": read, + return {**validate_probe(observed, value["mode"], identity), "disk": value["disk"], + "dataPath": value["dataPath"], "readProbe": read, "privateMountNamespace": True, "hostMountRemounted": False, "observed": observed} diff --git a/scripts/validation_heal/heal_probe.py b/scripts/validation_heal/heal_probe.py index 3c52550e..20f7cd09 100644 --- a/scripts/validation_heal/heal_probe.py +++ b/scripts/validation_heal/heal_probe.py @@ -229,7 +229,7 @@ def s3(self, operation: str, params: dict, body: bytes | None = None, *, allow_g stream.close() evidence["payload"] = {"size": count, "sha256": digest.hexdigest()} require(response.get("ContentLength") == count, "GET length differs from response ContentLength") - evidence.update(response=response, acknowledged=True) + evidence.update(response=response, acknowledged=True, resultUnknown=False) except Exception as exc: response = getattr(exc, "response", None) evidence.update(response=response, acknowledged=False, @@ -255,8 +255,11 @@ def admin(self, path: str, query: list[tuple[str, str]], body: bytes, with observation_phase(observation, 'requestPreparation'): validate_binding(self.binding) require(path.startswith("/rustfs/admin/v3/heal/") or path in { - "/rustfs/admin/v3/background-heal/status", "/rustfs/admin/v4/heal/replacement-recovery"}, - "only Heal Admin routes are allowed") + "/rustfs/admin/v3/background-heal/status", "/rustfs/admin/v4/heal/replacement-recovery", + "/rustfs/admin/v3/info"}, "only bounded Heal evidence routes are allowed") + if path == "/rustfs/admin/v3/info": + require(method == "GET" and not query and body == b"" and auth == "signed", + "server-info evidence requires an exact signed read-only GET") require("?" not in path and "#" not in path and len(body) <= 2 * 1024**2, "invalid Admin path or body budget") require(not any(part in {".", ".."} for part in unquote(path).split("/")), diff --git a/scripts/validation_heal/heal_workload.py b/scripts/validation_heal/heal_workload.py index f24a28bf..61e89cbf 100644 --- a/scripts/validation_heal/heal_workload.py +++ b/scripts/validation_heal/heal_workload.py @@ -185,7 +185,13 @@ def verify_seed(expected: list[dict], inventory: dict) -> dict: if len(actual) != len(inventory["entries"]) or set(wanted) != set(actual): raise FixtureStateMismatch("fixture version/marker set differs from ACKs") for identity, entry in wanted.items(): - if not all(actual[identity].get(key) == value for key, value in entry.items()): + # payloadNonce binds a deterministic Range oracle to the original ACK. + # It is collector state, not an S3 response property; payload length, + # digest and every public identity/assertion remain exact comparisons. + if "payloadNonce" in entry: + require(type(entry["payloadNonce"]) is int and entry["payloadNonce"] >= 0, + "invalid acknowledged payload nonce") + if not all(actual[identity].get(key) == value for key, value in entry.items() if key != "payloadNonce"): raise FixtureStateMismatch("baseline content/version/marker differs from deterministic fixture ACK") return {"expectedEntries": len(expected), "seedAckAndFullGetAgree": True, "casePassClaimed": False} diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 1daed17e..b09afc3a 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index fca71e5a..6d533a5f 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -27,7 +27,7 @@ case "${case_id}" in heal.admin-overlap-and-force-start) driver=overlap.py ;; heal.root-graceful-coordinator-restart) driver=root_restart.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index ef023e6d..b753f886 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -33,6 +33,7 @@ from .markers import (BRANCHES as MARKER_BRANCHES, INSTANCES as MARKER_INSTANCES, seed_markers, marker_plan, prepare_stale, verify_markers, reveal_current) from .stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch + from .disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch from .shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) except ImportError: @@ -51,6 +52,7 @@ from markers import (BRANCHES as MARKER_BRANCHES, INSTANCES as MARKER_INSTANCES, seed_markers, marker_plan, prepare_stale, verify_markers, reveal_current) from stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch + from disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch from shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) @@ -73,6 +75,7 @@ "heal.scanner-cold-object-repair": SCANNER_BRANCHES, "heal.delete-marker-semantics": MARKER_BRANCHES, "heal.stale-version-rejoin": STALE_BRANCHES, + "heal.disk-loss-and-repair": DISK_LOSS_BRANCHES, } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") @@ -712,7 +715,8 @@ def run_topology(args, topology, nodes, fixture): candidate = json.loads(args.candidate.read_text()) tool_manifest = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) contract = verify_quorum_contract(tool_manifest, candidate) if args.case_id in { - "heal.below-read-quorum", "heal.metadata-payload-mismatch", "heal.stale-version-rejoin"} else None + "heal.below-read-quorum", "heal.metadata-payload-mismatch", "heal.stale-version-rejoin", + "heal.disk-loss-and-repair"} else None size_rule = verify_size_contract(tool_manifest, candidate) if args.case_id == "heal.object-size-boundaries" else None bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) @@ -818,6 +822,12 @@ def run_topology(args, topology, nodes, fixture): fleet.record("lock-before-fault", prove_locked(probe, fleet, bucket, lock_context, key=LOCK_KEYS[LOCK_BRANCHES.index(branch)], label="lock-before-" + branch)) fleet.stop() + if args.case_id == "heal.disk-loss-and-repair": + stale_marker_started = True # ACKs may create a new authority; never restore an older fixture. + expected, baseline, proof = disk_loss_branch(recorder, fleet, bucket, branch, expected, active, admissions, contract) + result["branches"][branch] = proof + stale_marker_started = False + continue if args.case_id == "heal.stale-version-rejoin": stale_marker_started = True expected, baseline, proof = stale_branch(recorder, fleet, bucket, branch, expected, active, admissions, contract) @@ -1063,7 +1073,7 @@ def interrupted(signum, frame): fixtures = ("DB",) if args.case_id == "heal.object-lock-retention": fixtures = ("DO",) - if args.case_id in {"heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair"}: + if args.case_id in {"heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.disk-loss-and-repair"}: fixtures = ("DB",) results = [] try: diff --git a/test_validation_heal.py b/test_validation_heal.py index 310099f6..c37f5a7a 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -54,6 +54,7 @@ from scripts.validation_heal import root_restart, fleet_node from scripts.validation_heal import quorum_boundary from scripts.validation_heal import stale_rejoin +from scripts.validation_heal import disk_loss import errno import stat @@ -2730,6 +2731,135 @@ def test_range_oracle_uses_ack_nonce_after_historical_deletion(self): self.assertEqual([call.kwargs["expected_body"] for call in ranges.call_args_list], [f"{bucket}/{key}/1".encode(), f"{bucket}/{key}/2".encode()]) + def test_nonce_bound_ack_reaches_inventory_checks_without_weakening_public_manifest(self): + from scripts.validation_heal.heal_workload import verify_seed, FixtureStateMismatch + entry = {"key": "objects/0001.bin", "versionId": "v1", "deleteMarker": False, "isLatest": True, + "payload": {"size": 100, "sha256": "original-ack"}} + expected, inventory = [{**entry, "payloadNonce": 1}], {"entries": [entry]} + self.assertTrue(verify_seed(expected, inventory)["seedAckAndFullGetAgree"]) + for field, value in (("payload", {"size": 100, "sha256": "wrong"}), ("versionId", "v2"), + ("deleteMarker", True), ("isLatest", False)): + with self.subTest(field=field), self.assertRaises(FixtureStateMismatch): + verify_seed(expected, {"entries": [{**entry, field: value}]}) + for nonce in (-1, True, "1"): + with self.subTest(nonce=nonce), self.assertRaises(ValueError): + verify_seed([{**entry, "payloadNonce": nonce}], inventory) + + +class HealDiskLossTest(unittest.TestCase): + def scene(self, total=4, data=2): + original, old = HealReadQuorumBoundaryTest().scene(total=total, data=data) + bucket, old_key, new_key = HealReadQuorumBoundaryTest.bucket, old[0]["key"], "disk-loss/offline-ack-repair/new-00.bin" + repaired = copy.deepcopy(original) + for node, value in repaired.items(): + additions = [] + for entry in value["files"]: + item = copy.deepcopy(entry) + item["path"] = item["path"].replace(old_key, new_key) + if item.get("metadata"): + item["metadata"]["key"] = new_key + item["metadata"]["versions"][0]["usesLegacyChecksum"] = False + item["sha256"] += "-new-ack" + additions.append(item) + value["files"] += additions + new = [{**old[0], "key": new_key}] + target = map_target(original, bucket, old_key, "data", 1) + target = {**target, **target["cohort"][-1]} + member = target["node"], target["disk"] + authority = copy.deepcopy(repaired) + authority[member[0]]["files"] = [entry for entry in authority[member[0]]["files"] + if entry["disk"] != member[1] or not entry["path"].startswith(f"{bucket}/{new_key}/")] + witness = disk_loss.authority_witness(original, authority, bucket, target, new) + return original, authority, repaired, bucket, target, new, witness + + def admin_info(self, target): + return {"httpStatus": 200, "response": {"info": {"servers": [{"drives": [ + {"pool_index": 0, "set_index": 0, "disk_index": shard["setSlot"], "uuid": shard["diskUUID"], + "endpoint": f"http://{shard['node']}:9000/data/{shard['disk']}", "state": "ok", "runtimeState": "online"} + for shard in target["cohort"]]}]}}} + + def test_actual_single_member_quorum_and_repair_preserve_all_original_bytes(self): + for total, data in ((4, 2), (16, 12), (16, 8)): + original, authority, repaired, bucket, target, new, witness = self.scene(total, data) + plan = disk_loss.quorum_plan(target, HealReadQuorumBoundaryTest.contract) + self.assertEqual(plan["onlineSameSetDrives"], total - 1) + self.assertEqual(plan["writeQuorum"], data + int(data == total - data)) + cohorts, proof = disk_loss.repaired_physical(authority, repaired, bucket, target, new, witness) + self.assertEqual(len(cohorts), 1) + self.assertTrue(proof["allDegradedACKTargetPartsCreated"]) + self.assertFalse(proof["backupRestorationUsed"]) + + def test_offline_ack_authority_rejects_target_writes_loss_of_controls_and_donor_versions(self): + original, authority, repaired, bucket, target, new, witness = self.scene() + for kind in ("offline-write", "control", "donor-version", "format", "outside"): + changed = copy.deepcopy(authority) + if kind == "offline-write": changed = copy.deepcopy(repaired) + elif kind == "control": changed["node1"]["files"][0]["sha256"] = "changed-original" + elif kind == "donor-version": + next(entry for entry in changed["node1"]["files"] if entry.get("metadata", {}).get("key") == new[0]["key"])["metadata"]["versions"][0]["versionId"] = "foreign" + elif kind == "format": changed["node1"]["formats"][0]["value"]["xl"]["this"] = "new-disk" + else: changed["node1"]["files"].append({"disk": 0, "path": "other/key/part.1", "size": 82, "sha256": "outside"}) + with self.subTest(kind=kind), self.assertRaises(ValueError): + disk_loss.authority_witness(original, changed, bucket, target, new) + + def test_ready_or_get_only_and_changed_surviving_shards_cannot_prove_repair(self): + original, authority, repaired, bucket, target, new, witness = self.scene() + for kind in ("no-repair", "donor", "missing-part", "foreign-generation", "extra"): + changed = copy.deepcopy(repaired) + if kind == "no-repair": changed = copy.deepcopy(authority) + elif kind == "donor": + next(entry for entry in changed["node1"]["files"] if entry["disk"] != target["disk"] + and entry["path"].startswith(f"{bucket}/{new[0]['key']}/") and entry["path"].endswith("part.1"))["sha256"] = "changed" + elif kind == "missing-part": changed["node1"]["files"] = [entry for entry in changed["node1"]["files"] + if not (entry["disk"] == target["disk"] and entry["path"].startswith(f"{bucket}/{new[0]['key']}/") and entry["path"].endswith("part.1"))] + elif kind == "foreign-generation": + next(entry for entry in changed["node1"]["files"] if entry["disk"] == target["disk"] + and entry.get("metadata", {}).get("key") == new[0]["key"])["metadata"]["versions"][0]["dataDir"] = "foreign" + else: changed["node1"]["files"].append({"disk": target["disk"], "path": f"{bucket}/{new[0]['key']}/orphan", "size": 82, "sha256": "extra"}) + with self.subTest(kind=kind), self.assertRaises(ValueError): + disk_loss.repaired_physical(authority, changed, bucket, target, new, witness) + + def test_offline_admin_requires_exact_slot_endpoint_uuid_and_complete_healthy_peers(self): + _, _, _, _, target, _, _ = self.scene() + online = self.admin_info(target) + baseline = disk_loss.member_state(online, target, 4) + offline = copy.deepcopy(online) + selected = offline["response"]["info"]["servers"][0]["drives"][target["setSlot"]] + selected.update(state="disk access denied", runtimeState="offline") + self.assertTrue(disk_loss.member_state(offline, target, 4, baseline)["exactOriginalMemberOffline"]) + for kind in ("wrong-slot", "unknown", "new-uuid", "new-endpoint", "peer-offline", "omitted", "duplicate"): + changed = copy.deepcopy(offline) + drives = changed["response"]["info"]["servers"][0]["drives"] + if kind == "wrong-slot": drives[target["setSlot"]]["disk_index"] = 0 + elif kind == "unknown": drives[target["setSlot"]].update(state="unknown", runtimeState="unknown") + elif kind == "new-uuid": drives[target["setSlot"]]["uuid"] = "another-disk" + elif kind == "new-endpoint": drives[target["setSlot"]]["endpoint"] = "http://another-node:9000/data" + elif kind == "peer-offline": drives[0].update(state="offline", runtimeState="offline") + elif kind == "omitted": drives.pop() + else: drives.append(copy.deepcopy(drives[0])) + with self.subTest(kind=kind), self.assertRaises(ValueError): + disk_loss.member_state(changed, target, 4, baseline) + + def test_degraded_unique_puts_keep_each_ack_and_branches_match_source(self): + self.assertEqual(list(disk_loss.BRANCHES), select("B01")[0]["variants"]) + bucket = HealReadQuorumBoundaryTest.bucket + observed = [] + def invoke(operation, params, body): + observed.append((params["Key"], body)) + return {"ResponseMetadata": {"HTTPStatusCode": 200}} + def get(operation, params): + body = next(body for key, body in observed if key == params["Key"]) + return {"acknowledged": True, "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}} + recorder = types.SimpleNamespace(call=invoke, probe=types.SimpleNamespace(s3=get)) + fleet = types.SimpleNamespace(record=lambda label, value: value) + expected = [] + for branch, count in zip(disk_loss.BRANCHES, (1, 6)): + expected, new = disk_loss.offline_ack(recorder, fleet, bucket, branch, expected) + self.assertEqual(len(new), count) + self.assertEqual(len(expected), 7) + with self.assertRaisesRegex(ValueError, "already has"): + disk_loss.offline_ack(recorder, fleet, bucket, disk_loss.BRANCHES[0], expected) + if __name__ == "__main__": unittest.main() diff --git a/test_validation_heal_authorization.py b/test_validation_heal_authorization.py index 72674fb7..5d6d885b 100644 --- a/test_validation_heal_authorization.py +++ b/test_validation_heal_authorization.py @@ -34,6 +34,34 @@ class Response(io.BytesIO): class AuthorizationBoundaryTest(unittest.TestCase): + def test_s3_complete_get_sets_known_result_and_partial_body_cannot_pass(self): + from unittest.mock import Mock + from scripts.validation_heal.quorum_boundary import read_observation + probe, client = Probe(binding(), credential_pair=("test-access", "test-secret")), Mock() + body = b"complete-acknowledged-plaintext" + expected = {"payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}} + def response(length): + return {"Body": io.BytesIO(body), "ContentLength": length, + "ResponseMetadata": {"HTTPStatusCode": 200}} + client.get_object.side_effect = [response(len(body)), response(len(body) + 1)] + with patch.object(probe, "s3_client", return_value=client): + observed = probe.s3("get_object", {"Bucket": binding()["bucketPrefix"], "Key": "unique.bin"}) + self.assertIs(observed["resultUnknown"], False) + read_observation(observed, expected) + partial = probe.s3("get_object", {"Bucket": binding()["bucketPrefix"], "Key": "unique.bin"}) + self.assertIs(partial["acknowledged"], False) + with self.assertRaises(ValueError): read_observation(partial, expected) + + def test_server_info_is_exact_read_only_evidence_route(self): + probe = Probe(binding(), credential_pair=("test-access", "test-secret")) + with patch.object(probe, "_open_admin", return_value=Response(b'{"info":{"servers":[]}}')) as opened: + self.assertEqual(probe.admin("/rustfs/admin/v3/info", [], b"", method="GET")["httpStatus"], 200) + self.assertEqual(opened.call_args.args[0].get_method(), "GET") + for method, query, body, auth in (("POST", [], b"", "signed"), ("GET", [("x", "y")], b"", "signed"), + ("GET", [], b"{}", "signed"), ("GET", [], b"", "anonymous")): + with self.subTest(method=method, query=query, body=body, auth=auth), self.assertRaises(ValueError): + probe.admin("/rustfs/admin/v3/info", query, body, method=method, auth=auth) + def test_old_timestamp_recomputes_valid_signature_and_credential_scope(self): timestamp, access, secret = "20000102T030405Z", "isolated-access", "isolated-secret" body = b'{"dryRun":true}' From d68964e13ed8dc10021bef3294d18f9e12afa9e8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 04:12:49 +0800 Subject: [PATCH 069/343] test(validation): migrate formatted disk replacement and preserve live rejoin attribution --- docs/validation-heal-contract.md | 2 +- docs/validation-heal-operations.md | 2 +- docs/validation-migration-map.md | 46 ++++++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 60 ++++++++- scripts/validation_heal/healthy.py | 6 +- scripts/validation_heal/matrix.py | 5 +- scripts/validation_heal/replace_drive.py | 130 ++++++++++++++++++ scripts/validation_heal/replacement.py | 149 +++++++++++++++++++++ scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/stale_rejoin.py | 20 ++- scripts/validation_heal/versions.py | 16 ++- test_validation_heal.py | 159 +++++++++++++++++++++++ 13 files changed, 582 insertions(+), 17 deletions(-) create mode 100644 scripts/validation_heal/replace_drive.py create mode 100644 scripts/validation_heal/replacement.py diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index cfe6ebd2..4e44a114 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -38 个用例已接入自动分发：A01、A03–A08、B01、B02、B05–B07、B09、C01–C06、C08、D01–D07、D09、D10、E01–E04、F01、F06、J01、J03、J04。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。其余 46 项保留源 operator-assisted 方式。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 +39 个用例已接入自动分发：A01、A03–A08、B01–B03、B05–B07、B09、C01–C06、C08、D01–D07、D09、D10、E01–E04、F01、F06、J01、J03、J04。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。其余 45 项保留源 operator-assisted 方式。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 66842609..454b5bce 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -2,7 +2,7 @@ 此手册和 [84 项逐例操作卡](validation-heal-cases.md) 一起构成可执行的**人工操作方案 + 脚本辅助**。不是 84 个无人值守自动测试，也不是通过报告。操作卡列出 301 个分支模板；真实盘角色、阈值、损坏数量和拓扑仍需在每批 `parameterBindings` 展开，不能把模板数量当实测次数。 -公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 支持 A01、A03、A04、A07、C01–C04、D02、D07、E01、E02，使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 +公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 已接入 39 个 Heal 用例，完整清单通过 `matrix.py list` 查看，以 `automaticExecutor` 为准。入口使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 ## 1. 使用入口与交付物 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 16a5f6f5..00f64340 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 112 currently dispatchable IDs include all 74 non-Heal cases and thirty-eight Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. +case evidence. The 113 currently dispatchable IDs include all 74 non-Heal cases and thirty-nine Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 46 Heal cases remain +The other 45 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -650,7 +650,7 @@ A08 `heal.admin-authorization` passed live in RustFS Actions run `38069648003`, harness `0789743d439fd91b967c2d6fba85c06e29f844c5`. All five source branches passed independently on H1 and H2; the canonical gate, complete summary, cleanup and workflow all passed with the fixed candidate. -The prior A08 failures remain recorded. Forty-six Heal executors remain +The prior A08 failures remain recorded. Forty-five Heal executors remain to be migrated; automatic dispatch and local tests do not claim live PASS. @@ -716,3 +716,43 @@ retaining exact version/marker/currentness/length/digest comparisons. A regression check exercises nonce-bearing ACKs against real public inventory shape, including wrong content and version negatives. Fresh live evidence is required for both corrected collectors. + + +B03 `heal.disk-replacement-formatted` now dispatches both source branches on +H1/H2, with full DB and historical/marker DV fixtures. A stopped, owned adapter +moves exactly one complete private data root into same-mount quarantine and +creates a genuinely empty new root. Before/after whole-tree digest and original +inode/device preserve every old file, including system records; no shared +mount is detached. The actual service uses a private namespace with quarantine +masked and both DAC bypass capabilities removed. Same-process EACCES proves +that the original cannot supply rebuild bytes. The candidate must initialize +a distinct new disk UUID in the same original pool/set slot without replacing +another member or changing geometry. Every original target version, marker, +inline value and external part must rebuild through one exact deep task, with +truthful complete receipts/counters, original same-role hashes, protected +streaming checks and full all-peer S3 manifests. Missing/unchanged UUID or early +automatic repair cannot satisfy this manual case. Partial rename/mkdir intent +remains recoverable; archive drift retains the namespace. Live proof is pending. + +B01 run `38081858808` retained FAIL and cleanup PASS. Both H1/H2 had real +same-service read/write EACCES, but the exact Admin disk never became explicitly +offline within the registered 90-second observation window. Degraded ACKs were +therefore not started. The missing double witness remains NOT_OBSERVED in the +failure evidence; ready, cached online state and an inaccessible directory do +not substitute for the required Admin offline proof. + + +B06 corrected run `38082447868` retained FAIL and cleanup PASS. Both exact-R +GET and independent manual branches now pass on H1/H2; all three H2 branches +pass. H1's R-1 GET still returns 500/InternalError rather than the frozen +503/SlowDownRead, so no complete canonical PASS is claimed. + +C06 corrected run `38082227039` retained FAIL and cleanup PASS after proving +ACK authorities: full fleet restart allowed physical changes before the manual +token and the attribution check stopped execution. Rejoin now restores only +the exact active permission gate in the same live process. Two identical live +physical observations freeze the settled ACK authority; every peer's actual +process birth/unit/candidate/environment must remain unchanged, and the fault +must still exist immediately before the independent manual token. Startup +reconciliation cannot contribute to the manual proof. Fresh live evidence is +pending; the original attribution failures remain unchanged. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index a6262396..c5d8991c 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 4c5f9304..4f5ee2c3 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -217,7 +217,8 @@ def start(request): *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") == "root-restart" else []), "--property=WorkingDirectory=" + str(state), "--property=EnvironmentFile=" + str(state / "service.env"), - *gate_module().start_properties(gate_module().checked_gate(state, roots)), str(state / "rustfs"), *owner["volumes"]) + *gate_module().start_properties(gate_module().checked_gate(state, roots)), + *replacement_module().start_properties(state, roots), str(state / "rustfs"), *owner["volumes"]) return {"unit": owner["unit"], "started": True} @@ -337,6 +338,43 @@ def gate_module(): return drive_gate +def replacement_module(): + if "replacement" in globals(): return globals()["replacement"] + from . import replace_drive + return replace_drive + + +def replace_drive(request): + state, roots, owner = owned(request["runId"]) + stopped(owner) + require(gate_module().checked_gate(state, roots) is None, "replacement cannot share an active drive gate") + return replacement_module().begin(state, roots, request.get("disk"), request.get("expectedFormatSha256")) + + +def replacement_state(request): + state, roots, owner = owned(request["runId"]) + require(digest(state / "rustfs") == owner["binarySha256"], "replacement candidate changed") + evidence = replacement_module().attest(state, roots) + formats = [] + for disk, root in enumerate(roots): + path = checked(root / "data/.rustfs.sys/format.json", root) + if path.exists(): + require(path.stat().st_size < 65536, "replacement format exceeds its bound") + formats.append({"disk": disk, "present": True, "sha256": digest(path), "value": json.loads(path.read_text())}) + else: formats.append({"disk": disk, "present": False}) + return {**evidence, "formats": formats, "binarySha256": owner["binarySha256"], "readOnly": True} + + +def replacement_probe(request): + state, roots, _ = owned(request["runId"]) + before = process(request) + result = replacement_module().probe(state, roots, before["pid"]) + after = process(request) + require(all(before.get(name) == after.get(name) for name in PROCESS_FIELDS), + "replacement backup probe crossed a service process change") + return {**result, "sameOriginalServiceProcess": True} + + def drive_gate(request): state, roots, owner = owned(request["runId"]) stopped(owner) @@ -368,6 +406,21 @@ def drive_gate_state(request): return gate_module().checked_gate(state, roots) or {"state": "CLEAR"} +def drive_gate_release_live(request): + state, roots, _ = owned(request["runId"]) + gate = gate_module().checked_gate(state, roots) + require(gate and gate.get("mode") == "permission-denied" and gate.get("phase") == "active", + "live rejoin only releases an actual active permission fault") + before = process(request) + require(before["pid"] > 1, "live rejoin requires the original running candidate") + receipt = gate_module().release_gate(state, roots) + after = process(request) + require(all(before.get(name) == after.get(name) for name in PROCESS_FIELDS), + "live rejoin crossed the original process identity") + return {**receipt, "processBefore": before, "processAfter": after, + "sameOriginalProcess": True, "serviceRestarted": False} + + def fault_root(request): _, roots, owner = owned(request["runId"]) stopped(owner) @@ -560,6 +613,7 @@ def cleanup(request): pid = command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"], check=False).stdout.strip() require(pid in {"", "0"}, "run-owned service remains active") gate_module().release_gate(state, roots) + replacement_module().attest(state, roots) command("systemctl", "reset-failed", owner["unit"], check=False) for root in (*roots, state): for directory, children, names in os.walk(root, followlinks=False): @@ -638,8 +692,10 @@ def main(request): "logs": logs, "cleanup": cleanup, "audit-finalize": audit_finalize, "install-inspector": install_inspector, "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part, "verify-parts": verify_parts, "verify-inline": verify_inline, "drive-gate": drive_gate, "drive-gate-probe": drive_gate_probe, - "drive-gate-activate": drive_gate_activate, "gate-state": drive_gate_state} + "drive-gate-activate": drive_gate_activate, "gate-state": drive_gate_state, + "drive-gate-release-live": drive_gate_release_live} actions["task-records"] = task_records + actions.update({"replace-drive": replace_drive, "replacement-state": replacement_state, "replacement-probe": replacement_probe}) actions.update({"process": process, "process-signal": process_signal, "process-graceful": process_graceful}) require(request.get("action") in actions, "unsupported node operation") print(json.dumps(actions[request["action"]](request))) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index a92bc6ad..a8eabfec 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -170,9 +170,13 @@ def remote(self, node, action, **params): if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline"}: source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" - if action in {"start", "cleanup", "audit-finalize", "drive-gate", "drive-gate-probe", "drive-gate-activate", "gate-state"}: + replacement_actions = {"start", "cleanup", "audit-finalize", "replace-drive", "replacement-state", "replacement-probe"} + if action in {"drive-gate", "drive-gate-probe", "drive-gate-activate", "gate-state", "drive-gate-release-live"} | replacement_actions: source += "\nimport types\ngate = types.ModuleType('drive_gate')\nexec(" + repr( NODE_SOURCE.with_name("drive_gate.py").read_text()) + ", gate.__dict__)\n" + if action in replacement_actions: + source += "\nimport sys\nsys.modules['drive_gate'] = gate\nreplacement = types.ModuleType('replace_drive')\nexec(" + repr( + NODE_SOURCE.with_name("replace_drive.py").read_text()) + ", replacement.__dict__)\n" source += "\nmain(" + repr(value) + ")\n" timeout = 600 if self.resource_profile == "cursor" and action == "snapshot" else 180 result = subprocess.run([*self.ssh(node), "sudo -n python3 -"], diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index b09afc3a..24e77530 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted"} def require(condition: bool, message: str) -> None: @@ -237,7 +237,8 @@ def main() -> int: result = {"baseCases": len(inventory), "operatorBranches": sum(len(item["variants"]) for item in inventory), "topologyBranchTemplates": len(full["instances"]), "automaticCaseCount": len(AUTOMATIC_CASES), "livePassClaimed": False} if args.command == "list": - result["cases"] = [{"code": item["code"], "id": item["id"], "variants": item["variants"]} for item in inventory] + result["cases"] = [{"code": item["code"], "id": item["id"], "variants": item["variants"], + "automaticExecutor": item["id"] in AUTOMATIC_CASES} for item in inventory] print(json.dumps(result, ensure_ascii=False)) elif args.command == "show": print(json.dumps(select(args.selection), ensure_ascii=False, indent=2)) diff --git a/scripts/validation_heal/replace_drive.py b/scripts/validation_heal/replace_drive.py new file mode 100644 index 00000000..99739d97 --- /dev/null +++ b/scripts/validation_heal/replace_drive.py @@ -0,0 +1,130 @@ +"""Whole private logical-root replacement; the shared mount stays attached.""" + +import hashlib +import json +import os +from pathlib import Path +import stat +import subprocess + +if __package__: + from .drive_gate import DENY_DAC_CAPS, READ_PROBE_SOURCE, process_identity +else: + from drive_gate import DENY_DAC_CAPS, READ_PROBE_SOURCE, process_identity + + +def require(value, message): + if not value: raise ValueError(message) + + +def digest(path): + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size <= 64 * 1024**2, + "replacement archive contains a link or unsupported file") + value = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024**2): value.update(chunk) + return value.hexdigest() + + +def tree_identity(root): + info = root.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and stat.S_IMODE(info.st_mode) == 0o700, + "replacement original root is not private/root-owned") + rows = [] + for folder, children, names in os.walk(root, followlinks=False): + for name in (*children, *names): + path = Path(folder) / name + item = path.lstat() + require(item.st_dev == info.st_dev and not path.is_symlink(), "replacement archive crossed a mount or link") + if stat.S_ISDIR(item.st_mode): continue + require(len(rows) < 16384, "replacement archive exceeds the registered file budget") + rows.append({"path": path.relative_to(root).as_posix(), "size": item.st_size, "sha256": digest(path)}) + value = json.dumps(sorted(rows, key=lambda item: item["path"]), sort_keys=True, separators=(",", ":")).encode() + return {"sha256": hashlib.sha256(value).hexdigest(), "files": len(rows), "device": info.st_dev, "inode": info.st_ino} + + +def checked_replacement(state, roots): + path = state / "disk-replacement.json" + if not path.exists(): return None + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_uid == 0 and info.st_nlink == 1 + and stat.S_IMODE(info.st_mode) == 0o600 and info.st_size < 4096, "invalid replacement ownership marker") + value = json.loads(path.read_text()) + disk = value.get("disk") + require(value.get("schemaVersion") == 1 and type(disk) is int and 0 <= disk < len(roots) + and value.get("dataPath") == str(roots[disk] / "data") + and value.get("backupPath") == str(roots[disk] / "quarantine" / "old-disk") + and value.get("phase") == "armed" and isinstance(value.get("original"), dict), + "replacement marker moved outside its exact private member") + quarantine = roots[disk] / "quarantine" + info = quarantine.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and stat.S_IMODE(info.st_mode) == 0o700 + and info.st_dev == roots[disk].lstat().st_dev, "replacement quarantine crossed a link, mount or owner") + return value + + +def begin(state, roots, disk, expected_format_sha256): + require(type(disk) is int and 0 <= disk < len(roots) and checked_replacement(state, roots) is None, + "invalid or repeated logical disk replacement") + data, backup = roots[disk] / "data", roots[disk] / "quarantine" / "old-disk" + quarantine = backup.parent.lstat() + require(stat.S_ISDIR(quarantine.st_mode) and quarantine.st_uid == 0 and stat.S_IMODE(quarantine.st_mode) == 0o700 + and quarantine.st_dev == roots[disk].lstat().st_dev, "replacement quarantine is not an exact private same-mount directory") + require(not backup.exists() and not backup.is_symlink(), "replacement original backup already exists") + original = tree_identity(data) + format_path = data / ".rustfs.sys" / "format.json" + require(digest(format_path) == expected_format_sha256, "original disk format differs before whole-root replacement") + value = {"schemaVersion": 1, "phase": "armed", "disk": disk, "dataPath": str(data), + "backupPath": str(backup), "original": original, "originalFormatSha256": expected_format_sha256} + fd = os.open(state / "disk-replacement.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(value, stream); stream.flush(); os.fsync(stream.fileno()) + os.rename(data, backup) + data.mkdir(mode=0o700) + for folder in (roots[disk], backup.parent): + fd = os.open(folder, os.O_RDONLY | os.O_DIRECTORY) + try: os.fsync(fd) + finally: os.close(fd) + require(tree_identity(backup) == original and not any(data.iterdir()), "replacement did not retain exact original bytes and an empty new root") + return {**value, "oldRootMovedIntact": True, "newDataRootEmpty": True, + "sharedMountDetached": False, "backupRestorationUsed": False} + + +def attest(state, roots): + value = checked_replacement(state, roots) + if value is None: return {"replacementAbsent": True} + backup, data = Path(value["backupPath"]), Path(value["dataPath"]) + # An interrupted begin remains recoverable only when the original tree is + # still at one of the two exact intended roots; otherwise retain the run. + original_path = backup if backup.exists() else data + require(tree_identity(original_path) == value["original"], "original replacement archive drifted; retain namespace") + return {**value, "originalArchiveUnchanged": True, "originalStillAtDataRoot": original_path == data} + + +def start_properties(state, roots): + value = checked_replacement(state, roots) + if value is None: return [] + require(Path(value["backupPath"]).is_dir(), "replacement original has not moved to quarantine") + return ["--property=PrivateMounts=true", "--property=InaccessiblePaths=" + str(Path(value["backupPath"]).parent), + "--property=CapabilityBoundingSet=~CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH"] + + +def probe(state, roots, pid): + value = attest(state, roots) + require(not value.get("replacementAbsent") and not value["originalStillAtDataRoot"] + and type(pid) is int and pid > 1, "live replacement namespace required") + identity = process_identity(pid) + require(identity["uid"] == 0 and identity["gid"] == 0 and identity["capEff"] & DENY_DAC_CAPS == 0 + and Path(f"/proc/{pid}/ns/mnt").readlink() != Path("/proc/self/ns/mnt").readlink(), + "replacement backup is not isolated in the actual service identity/namespace") + result = subprocess.run(["nsenter", "--mount=/proc/" + str(pid) + "/ns/mnt", "setpriv", + "--bounding-set=-dac_override,-dac_read_search", "python3", "-c", READ_PROBE_SOURCE, + value["backupPath"] + "/.rustfs.sys/format.json"], capture_output=True, text=True, timeout=15) + require(result.returncode == 0, "replacement service read probe did not finish") + observed = json.loads(result.stdout) + require(observed.get("errno") == 13 and observed.get("uid") == identity["uid"] and observed.get("gid") == identity["gid"], + "service can read old replacement backup or lacks exact EACCES") + return {"disk": value["disk"], "backupPath": value["backupPath"], "serviceReadDenied": True, + "sameServiceIdentity": True, "privateMountNamespace": True, "readProbe": observed, + "originalArchiveUnchanged": True, "backupRestorationUsed": False} diff --git a/scripts/validation_heal/replacement.py b/scripts/validation_heal/replacement.py new file mode 100644 index 00000000..edec9234 --- /dev/null +++ b/scripts/validation_heal/replacement.py @@ -0,0 +1,149 @@ +"""B03: full private-drive replacement with original same-slot shard oracles.""" + +import copy + +try: + from .heal_probe import Probe, canonical, compare_inventories, require + from .heal_workload import verify_seed + from .healthy import validate_topology + from .shards import files, map_target, run_task +except ImportError: + from heal_probe import Probe, canonical, compare_inventories, require + from heal_workload import verify_seed + from healthy import validate_topology + from shards import files, map_target, run_task + + +BRANCHES = ("new-disk-data", "new-disk-versioned") + + +def format_identity(before, after, member): + topology = "H1" if len(before) == 1 else "H2" + validate_topology(before, topology); validate_topology(after, topology) + old = next(entry for entry in before[member[0]]["formats"] if entry["disk"] == member[1]) + new = next(entry for entry in after[member[0]]["formats"] if entry["disk"] == member[1]) + old_uuid, new_uuid = old["value"]["xl"]["this"], new["value"]["xl"]["this"] + sets = old["value"]["xl"]["sets"] + slot = sets[0].index(old_uuid) + require(new_uuid != old_uuid and new_uuid not in sets[0], "replacement did not expose a distinct new disk UUID") + require(set(before) == set(after), "replacement changed the original node fleet") + for node, observation in before.items(): + current = {entry["disk"]: entry for entry in after[node]["formats"]} + require(set(current) == {entry["disk"] for entry in observation["formats"]}, "replacement omitted a physical member") + for entry in observation["formats"]: + expected = copy.deepcopy(entry["value"]) + expected["xl"]["sets"][0][slot] = new_uuid + if (node, entry["disk"]) == member: expected["xl"]["this"] = new_uuid + require(current[entry["disk"]]["value"] == expected, + "replacement changed pool/set geometry, another disk identity or an unrelated format field") + return {"oldDiskUUID": old_uuid, "newDiskUUID": new_uuid, "sameOriginalSetSlot": slot, + "originalPoolAndAllOtherMemberIdentitiesPreserved": True} + + +def metadata_semantics(metadata): + value = copy.deepcopy(metadata) + for version in value.get("versions", []): + inline = version.get("inline") + if inline: version["inline"] = {name: inline.get(name) for name in ("length", "sha256")} + version.get("erasure", {}).pop("checksums", None) + return value + + +def physical_rebuild(before, after, member): + identity = format_identity(before, after, member) + left, right = files(before), files(after) + require(left and set(left) <= set(right), "replacement lost an original object/version/part file") + original_target = {key: entry for key, entry in left.items() if key[:2] == member} + require(original_target, "replacement has no original target file oracle") + for key, entry in left.items(): + actual = right[key] + if key[:2] != member: + require(actual == entry, "replacement changed a surviving member's original shard/metadata") + elif entry.get("metadata"): + require(actual.get("metadata") and canonical(metadata_semantics(actual["metadata"])) == + canonical(metadata_semantics(entry["metadata"])), "new disk changed an original version/marker/inline/role authority") + else: + require(actual == entry, "replacement did not reconstruct exact original same-role part bytes") + for key in set(right) - set(left): + # A target-side rollback backup is bounded to the exact original xl.meta + # bytes for that same member/key. No orphan or system record is allowed. + original = left.get((key[0], key[1], key[2].removesuffix(".bkp"))) if key[2].endswith("/xl.meta.bkp") else None + require(key[:2] == member and original and original.get("metadata") + and all(right[key].get(name) == original.get(name) for name in ("sha256", "size")), + "replacement created an unexpected file or foreign rollback backup") + return {**identity, "allOriginalTargetFilesRebuilt": len(original_target), + "originalSameRolePartAndInlineDigestsRestored": True, "survivingFilesUnchanged": True, + "backupRestorationUsed": False} + + +def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admissions): + try: + from .versions import map_inline_target, validate_version_receipts, verify_version_s3 + except ImportError: + from versions import map_inline_target, validate_version_receipts, verify_version_s3 + require(branch in BRANCHES, "unknown formatted replacement branch") + before = fleet.snapshot("replacement-original-" + branch, quiescent=True) + node, disk = fleet.nodes[-1], 3 + member = node, disk + physical_format = next(entry for entry in before[node]["formats"] if entry["disk"] == disk) + proofs = [] + indexed = files(before) + for entry in expected: + if entry["deleteMarker"]: continue + metadata = indexed[(node, disk, f"{bucket}/{entry['key']}/xl.meta")]["metadata"] + version = next(version for version in metadata["versions"] if (version.get("versionId") or "null") == entry["versionId"]) + target = map_inline_target(before, bucket, entry, allow_empty=True) if version.get("inline") or entry["payload"]["size"] == 0 \ + else map_target(before, bucket, entry["key"], "data", 1, entry["versionId"], latest_required=False) + shard = next(value for value in target["cohort"] if (value["node"], value["disk"]) == member) + proofs.append({**target, **shard}) + replaced = fleet.record("replacement-original-root-isolated", fleet.remote(node, "replace-drive", disk=disk, + expectedFormatSha256=physical_format["sha256"])) + require(replaced.get("oldRootMovedIntact") is True and replaced.get("newDataRootEmpty") is True + and replaced.get("sharedMountDetached") is False, "replacement lacks exact original and empty-root evidence") + fleet.start() + denied = fleet.record("replacement-backup-service-refusal", fleet.remote(node, "replacement-probe")) + require(denied.get("serviceReadDenied") is True and denied.get("sameOriginalServiceProcess") is True, + "replacement candidate can access its old physical backup") + initialized = fleet.snapshot("replacement-candidate-initialization") + fleet.record("replacement-new-format-proof", format_identity(before, initialized, member)) + require(not any(item["disk"] == disk for item in initialized[node]["files"]), + "ATTRIBUTION_NOT_OBSERVED: new drive received object files before its manual repair token") + require({key: value for key, value in files(initialized).items() if key[:2] != member} == + {key: value for key, value in indexed.items() if key[:2] != member}, "replacement initialization changed surviving objects") + status = run_task(recorder.probe, fleet, {"bucket": bucket, "key": ""}, 2, active, + option_overrides={"recursive": True}, admissions=admissions) + receipts = validate_version_receipts(status, bucket, expected, 2, + required_repairs=[(entry["key"], entry["versionId"]) for entry in expected]) + fleet.stop() + after = fleet.snapshot("replacement-after-manual", quiescent=True) + physical = fleet.record("replacement-full-physical-proof", physical_rebuild(before, after, member)) + fleet.record("replacement-original-archive-after-rebuild", fleet.remote(node, "replacement-state")) + streaming = [] + for target in proofs: + if target.get("zeroProof"): + streaming.append({"key": target["key"], "zeroLengthMetadataVerified": True, "noPartInvented": True}) + continue + operation = "verify-inline" if target.get("inlineProof") else "verify-part" + params = {name: target[name] for name in ("disk", "relativePath", "versionId")} + if operation == "verify-part": params["partNumber"] = target["partNumber"] + proof = fleet.record("replacement-target-streaming-check", fleet.remote(node, operation, **params)) + require(proof.get("physicalBitrotVerified") is True, "reconstructed target has no verified streaming checksum") + if operation == "verify-inline": + require(proof.get("inlineLength") == target["inlineValue"]["length"] + and proof.get("inlineSha256") == target["inlineValue"]["sha256"], "new target inline bytes differ from the original same-role oracle") + else: + require(proof.get("sha256") == target["sha256"] and proof.get("physicalSize") == target["size"], + "new target external shard differs from the original same-role oracle") + streaming.append({"key": target["key"], "versionId": target["versionId"], "proof": proof}) + fleet.start() + fleet.record("replacement-backup-service-refusal-after", fleet.remote(node, "replacement-probe")) + fixture = "DB" if branch == "new-disk-data" else "DV" + s3 = verify_version_s3(recorder.probe, fleet, bucket, expected, baseline, "replacement-s3-after", fixture=fixture) + peers = [] + for endpoint in recorder.probe.binding["endpoints"]: + inventory = fleet.record("replacement-all-peer-s3", {"endpoint": endpoint, + **Probe(recorder.probe.binding, endpoint).inventory(bucket, unversioned_current_tags=fixture == "DB")}) + peers.append({"endpoint": endpoint, **verify_seed(expected, inventory), + "manifest": compare_inventories(baseline, inventory)}) + return {"verdict": "PASS", "responsibility": receipts, "physical": physical, + "allTargetVersionStreamingProofs": streaming, "s3": s3, "allPeerS3": peers} diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 6d533a5f..91e77157 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -27,7 +27,7 @@ case "${case_id}" in heal.admin-overlap-and-force-start) driver=overlap.py ;; heal.root-graceful-coordinator-restart) driver=root_restart.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/stale_rejoin.py b/scripts/validation_heal/stale_rejoin.py index 7a2cf74c..754e6d5b 100644 --- a/scripts/validation_heal/stale_rejoin.py +++ b/scripts/validation_heal/stale_rejoin.py @@ -7,6 +7,7 @@ from .heal_probe import Probe, canonical, payload, require from .heal_workload import verify_seed from .healthy import physical_content + from .fleet_node import PROCESS_FIELDS from .shards import files, map_target, run_task, verify_cohort from .markers import add_marker, verify_markers from .scanner_repair import JournalWindow @@ -14,6 +15,7 @@ from heal_probe import Probe, canonical, payload, require from heal_workload import verify_seed from healthy import physical_content + from fleet_node import PROCESS_FIELDS from shards import files, map_target, run_task, verify_cohort from markers import add_marker, verify_markers from scanner_repair import JournalWindow @@ -210,12 +212,22 @@ def run_branch(recorder, fleet, bucket, branch, expected, active, admissions, co fleet.record("stale-transition-contract", {"branch": branch, **ack}) baseline = fleet.record("stale-online-ack-inventory", recorder.probe.inventory(bucket, unversioned_current_tags=branch == "unversioned-overwrite")) fleet.record("stale-online-ack-proof", {"expected": manifest, **verify_seed(manifest, baseline)}) - fleet.stop() - authority = fleet.snapshot("stale-online-authorities", quiescent=True) + # Restarting the fleet here can perform startup reconciliation before the + # manual token. ACKs have settled; freeze two unchanged live observations, + # then release only the exact permission gate in the original process. + authority = fleet.snapshot("stale-online-authorities") + require(physical_content(authority) == physical_content(fleet.snapshot("stale-online-authorities-stable")), + "NOT_OBSERVED: offline ACK authority is not physically stable before live rejoin") witness = fleet.record("stale-generation-authority-proof", authority_witness(before, authority, bucket, key, manifest, target)) - fleet.record("stale-original-member-rejoin", fleet.remote(node, "drive-gate", mode="clear")) + processes = {peer: fleet.remote(peer, "process") for peer in fleet.nodes} + release = fleet.record("stale-original-member-rejoin", fleet.remote(node, "drive-gate-release-live")) + require(release.get("sameOriginalProcess") is True and release.get("serviceRestarted") is False, + "stale member rejoin restarted or replaced the original process") fleet.drive_gates.remove(node) - fleet.start() + rejoined = {peer: fleet.remote(peer, "process") for peer in fleet.nodes} + require(all(all(processes[peer].get(name) == rejoined[peer].get(name) for name in PROCESS_FIELDS) for peer in fleet.nodes), + "stale live rejoin changed an original peer process/unit/candidate/environment") + fleet.record("stale-live-rejoin-process-proof", {"before": processes, "after": rejoined, "allOriginalProcessesPreserved": True}) require(physical_content(authority) == physical_content(fleet.snapshot("stale-before-manual")), "ATTRIBUTION_NOT_OBSERVED: stale namespace changed before its manual token") status = run_task(recorder.probe, fleet, {"bucket": bucket, "key": key}, 2, active, diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index b753f886..d451db6d 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -34,6 +34,7 @@ marker_plan, prepare_stale, verify_markers, reveal_current) from .stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch from .disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch + from .replacement import BRANCHES as REPLACEMENT_BRANCHES, run_branch as replacement_branch from .shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) except ImportError: @@ -53,6 +54,7 @@ marker_plan, prepare_stale, verify_markers, reveal_current) from stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch from disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch + from replacement import BRANCHES as REPLACEMENT_BRANCHES, run_branch as replacement_branch from shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) @@ -76,6 +78,7 @@ "heal.delete-marker-semantics": MARKER_BRANCHES, "heal.stale-version-rejoin": STALE_BRANCHES, "heal.disk-loss-and-repair": DISK_LOSS_BRANCHES, + "heal.disk-replacement-formatted": REPLACEMENT_BRANCHES, } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") @@ -738,6 +741,8 @@ def run_topology(args, topology, nodes, fixture): and (args.case_id != "heal.missing-or-corrupt-xlmeta" or (branch == "versioned-xlmeta") == (fixture == "DV"))] if args.case_id == "heal.stale-version-rejoin": branches = [branch for branch in STALE_BRANCHES if (branch == "unversioned-overwrite") == (fixture == "DB")] + if args.case_id == "heal.disk-replacement-formatted": + branches = [branch for branch in REPLACEMENT_BRANCHES if (branch == "new-disk-data") == (fixture == "DB")] result = {"topology": topology, "fixture": fixture, "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} active, admissions = [], [] probe = Probe(binding) @@ -822,6 +827,11 @@ def run_topology(args, topology, nodes, fixture): fleet.record("lock-before-fault", prove_locked(probe, fleet, bucket, lock_context, key=LOCK_KEYS[LOCK_BRANCHES.index(branch)], label="lock-before-" + branch)) fleet.stop() + if args.case_id == "heal.disk-replacement-formatted": + stale_marker_started = True + result["branches"][branch] = replacement_branch(recorder, fleet, bucket, branch, expected, baseline, active, admissions) + stale_marker_started = False + continue if args.case_id == "heal.disk-loss-and-repair": stale_marker_started = True # ACKs may create a new authority; never restore an older fixture. expected, baseline, proof = disk_loss_branch(recorder, fleet, bucket, branch, expected, active, admissions, contract) @@ -1012,6 +1022,10 @@ def run_topology(args, topology, nodes, fixture): result["branches"][branch] = {"verdict": "FAIL", "failure": {"type": type(exc).__name__, "message": str(exc)}} try: fleet.stop() + if args.case_id == "heal.disk-replacement-formatted": + for node in fleet.nodes: + fleet.record("replacement-failure-format-and-original", {"node": node, + **fleet.remote(node, "replacement-state")}) if gate_node is not None: fleet.record("gate-error-rescue", fleet.remote(gate_node, "drive-gate", mode="clear")) fleet.drive_gates.remove(gate_node) @@ -1064,7 +1078,7 @@ def interrupted(signum, frame): and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") - fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.multiple-shard-loss", "heal.read-repair-attribution", "heal.stale-version-rejoin"} else ("DV",) + fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.multiple-shard-loss", "heal.read-repair-attribution", "heal.stale-version-rejoin", "heal.disk-replacement-formatted"} else ("DV",) if args.case_id == "heal.object-metadata-and-tags": fixtures = ("DS",) if args.case_id == "heal.special-keys-and-pagination": diff --git a/test_validation_heal.py b/test_validation_heal.py index c37f5a7a..d7348230 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -55,6 +55,7 @@ from scripts.validation_heal import quorum_boundary from scripts.validation_heal import stale_rejoin from scripts.validation_heal import disk_loss +from scripts.validation_heal import replace_drive, replacement import errno import stat @@ -2861,5 +2862,163 @@ def get(operation, params): disk_loss.offline_ack(recorder, fleet, bucket, disk_loss.BRANCHES[0], expected) +class HealWholeDriveReplacementTest(unittest.TestCase): + def test_live_permission_rejoin_restores_same_inode_and_requires_same_process(self): + for changed_process in (False, True): + with self.subTest(changed_process=changed_process), tempfile.TemporaryDirectory() as folder: + state, root = self.fixture(folder) + identity = HealRootRestartOracleTest().process() + after = {**identity, "startTicks": identity["startTicks"] + int(changed_process)} + with self.root_owned(): + apply_gate(state, (root,), 0, "permission-denied") + activate_gate(state, (root,)) + original_inode = (root / "data").stat().st_ino + with patch.object(fleet_node, "owned", return_value=(state, (root,), {})), \ + patch.object(fleet_node, "process", side_effect=[identity, after]), \ + patch.object(fleet_node, "stopped") as stopped_service: + if changed_process: + with self.assertRaisesRegex(ValueError, "process identity"): + fleet_node.drive_gate_release_live({"runId": "owned-run"}) + else: + proof = fleet_node.drive_gate_release_live({"runId": "owned-run"}) + self.assertTrue(proof["sameOriginalProcess"]) + self.assertFalse(proof["serviceRestarted"]) + stopped_service.assert_not_called() + self.assertEqual((root / "data").stat().st_ino, original_inode) + self.assertEqual(stat.S_IMODE((root / "data").stat().st_mode), 0o700) + self.assertIsNone(checked_gate(state, (root,))) + + def test_live_rejoin_rejects_unactivated_gate_without_mutating_root(self): + with tempfile.TemporaryDirectory() as folder: + state, root = self.fixture(folder) + with self.root_owned(): + apply_gate(state, (root,), 0, "permission-denied") + with patch.object(fleet_node, "owned", return_value=(state, (root,), {})), \ + patch.object(fleet_node, "process") as process, self.assertRaisesRegex(ValueError, "active permission"): + fleet_node.drive_gate_release_live({"runId": "owned-run"}) + process.assert_not_called() + self.assertEqual(checked_gate(state, (root,))["phase"], "armed") + self.assertEqual(stat.S_IMODE((root / "data").stat().st_mode), 0o700) + + def replaced_scene(self): + before = HealShardOracleTest().snapshot() + member = "node1", 3 + after = copy.deepcopy(before) + for entry in after["node1"]["formats"]: + entry["value"]["xl"]["sets"][0][3] = "new-member" + if entry["disk"] == 3: entry["value"]["xl"]["this"] = "new-member" + return before, after, member + + def test_new_disk_requires_same_pool_set_slot_and_distinct_uuid(self): + before, after, member = self.replaced_scene() + self.assertEqual(replacement.format_identity(before, after, member)["newDiskUUID"], "new-member") + for kind in ("same-uuid", "new-cluster", "wrong-slot", "changed-survivor"): + changed = copy.deepcopy(after) + if kind == "same-uuid": changed = copy.deepcopy(before) + elif kind == "new-cluster": + for entry in changed["node1"]["formats"]: entry["value"]["id"] = "another-cluster" + elif kind == "wrong-slot": changed["node1"]["formats"][3]["value"]["xl"]["this"] = "0" + else: changed["node1"]["formats"][0]["value"]["version"] = "future" + with self.subTest(kind=kind), self.assertRaises(ValueError): replacement.format_identity(before, changed, member) + + def test_rebuild_requires_all_same_role_original_parts_and_unchanged_survivors(self): + before, after, member = self.replaced_scene() + self.assertFalse(replacement.physical_rebuild(before, after, member)["backupRestorationUsed"]) + for kind in ("missing-target", "bad-target", "donor", "version", "orphan"): + changed = copy.deepcopy(after) + target = next(entry for entry in changed["node1"]["files"] if entry["disk"] == member[1] + and entry["path"].endswith("part.1")) + if kind == "missing-target": changed["node1"]["files"].remove(target) + elif kind == "bad-target": target["sha256"] = "wrong-shard" + elif kind == "donor": changed["node1"]["files"][0]["sha256"] = "changed-donor" + elif kind == "version": + next(entry for entry in changed["node1"]["files"] if entry["disk"] == member[1] + and entry.get("metadata"))["metadata"]["versions"][0]["versionId"] = "foreign-version" + else: changed["node1"]["files"].append({"disk": member[1], "path": "run/key/orphan", "size": 82, "sha256": "orphan"}) + with self.subTest(kind=kind), self.assertRaises(ValueError): replacement.physical_rebuild(before, changed, member) + + def fixture(self, folder): + state, root = Path(folder) / "state", Path(folder) / "root" + state.mkdir(mode=0o700) + (root / "data/.rustfs.sys").mkdir(parents=True, mode=0o700) + (root / "data").chmod(0o700) + (root / "quarantine").mkdir(mode=0o700) + (root / "data/.rustfs.sys/format.json").write_bytes(b'{"original":"format"}') + (root / "data/ack-part").write_bytes(b"original-ack-shard") + return state, root + + def root_owned(self): + original = Path.lstat + def observed(path): + value = original(path) + return types.SimpleNamespace(st_mode=value.st_mode, st_uid=0, st_nlink=value.st_nlink, + st_size=value.st_size, st_dev=value.st_dev, st_ino=value.st_ino) + return patch.object(Path, "lstat", observed) + + def test_replacement_keeps_whole_original_and_rejects_drift_or_second_begin(self): + with tempfile.TemporaryDirectory() as folder: + state, root = self.fixture(folder) + with self.root_owned(): + before = replace_drive.tree_identity(root / "data") + proof = replace_drive.begin(state, (root,), 0, replace_drive.digest(root / "data/.rustfs.sys/format.json")) + self.assertTrue(proof["newDataRootEmpty"]) + self.assertEqual(replace_drive.tree_identity(root / "quarantine/old-disk"), before) + self.assertEqual((root / "quarantine/old-disk/ack-part").read_bytes(), b"original-ack-shard") + self.assertTrue(replace_drive.attest(state, (root,))["originalArchiveUnchanged"]) + self.assertIn("--property=InaccessiblePaths=" + str(root / "quarantine"), replace_drive.start_properties(state, (root,))) + with self.assertRaises(ValueError): replace_drive.begin(state, (root,), 0, proof["originalFormatSha256"]) + (root / "quarantine/old-disk/ack-part").write_bytes(b"changed-old-backup") + with self.assertRaisesRegex(ValueError, "drifted"): replace_drive.attest(state, (root,)) + + def test_partial_begin_retains_recovery_intent_and_never_copies_original_back(self): + for boundary in ("rename", "mkdir"): + with self.subTest(boundary=boundary), tempfile.TemporaryDirectory() as folder: + state, root = self.fixture(folder) + with self.root_owned(): + digest = replace_drive.digest(root / "data/.rustfs.sys/format.json") + if boundary == "rename": failure = patch.object(replace_drive.os, "rename", side_effect=OSError("interrupted rename")) + else: failure = patch.object(Path, "mkdir", side_effect=OSError("interrupted mkdir")) + with failure, self.assertRaises(OSError): replace_drive.begin(state, (root,), 0, digest) + proof = replace_drive.attest(state, (root,)) + self.assertTrue(proof["originalArchiveUnchanged"]) + self.assertEqual(proof["originalStillAtDataRoot"], boundary == "rename") + original = root / ("data" if boundary == "rename" else "quarantine/old-disk") + self.assertEqual((original / "ack-part").read_bytes(), b"original-ack-shard") + + def test_archive_rejects_links_and_wrong_format_before_mutation(self): + for kind in ("symlink", "hardlink", "format"): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as folder: + state, root = self.fixture(folder) + if kind == "symlink": (root / "data/link").symlink_to(root / "data/ack-part") + elif kind == "hardlink": __import__("os").link(root / "data/ack-part", root / "data/another") + with self.root_owned(), self.assertRaises(ValueError): replace_drive.begin(state, (root,), 0, "0" * 64) + self.assertFalse((state / "disk-replacement.json").exists()) + self.assertFalse((root / "quarantine/old-disk").exists()) + self.assertEqual((root / "data/ack-part").read_bytes(), b"original-ack-shard") + + def test_remote_replacement_module_initializes_and_all_source_branches_remain(self): + import sys + self.assertEqual(list(replacement.BRANCHES), select("B03")[0]["variants"]) + prior = sys.modules.get("drive_gate") + observed = [] + def invoke(args, **kwargs): + source = kwargs["input"].rsplit("\nmain(", 1)[0] + scope = {"__name__": "private-node-script"} + exec(source, scope) + self.assertEqual(scope["replacement_module"]().DENY_DAC_CAPS, (1 << 1) | (1 << 2)) + self.assertTrue(callable(scope["replacement_module"]().begin)) + observed.append(source) + return types.SimpleNamespace(returncode=0, stdout='{"compiled":true}', stderr="") + try: + with tempfile.TemporaryDirectory() as folder, patch("scripts.validation_heal.healthy.subprocess.run", side_effect=invoke): + fleet = Fleet(["node1"], Path(folder), "gh-12345-1-h1") + for action in ("start", "cleanup", "replace-drive", "replacement-state", "replacement-probe"): + self.assertTrue(fleet.remote("node1", action)["compiled"]) + finally: + if prior is None: sys.modules.pop("drive_gate", None) + else: sys.modules["drive_gate"] = prior + self.assertEqual(len(observed), 5) + + if __name__ == "__main__": unittest.main() From fa14d8ec7dfdb03abdb6d1c6ace78c92220fb1b2 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 04:19:52 +0800 Subject: [PATCH 070/343] fix(validation): preserve complete scanner journal pages --- docs/validation-migration-map.md | 9 +++++++ scripts/validation_heal/fleet_node.py | 27 ++++++++++++------- scripts/validation_heal/scanner_repair.py | 33 ++++++++++++++++------- scripts/validation_heal/versions.py | 8 +++--- test_validation_heal.py | 31 +++++++++++++++++++-- 5 files changed, 83 insertions(+), 25 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 00f64340..32590b82 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -756,3 +756,12 @@ process birth/unit/candidate/environment must remain unchanged, and the fault must still exist immediately before the independent manual token. Startup reconciliation cannot contribute to the manual proof. Fresh live evidence is pending; the original attribution failures remain unchanged. + +E04 journal evidence now reads the first bounded page after each exact cursor +using `journalctl -n +10000` and per-entry journal cursors. Full pages are drained +in order, with six pages per capture and a hard failure on incomplete or +nonadvancing windows. Raw pages remain evidence; only source-attribution targets +are retained in the scanner's in-memory index. The healthy checkpoint starts +after seed, baseline and the initial physical snapshot, so a fresh complete +scanner cycle must actually occur in the registered observation window. Earlier +log-budget failures remain FAIL; this harness change has no live PASS claim. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 4f5ee2c3..213cc376 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -590,20 +590,29 @@ def logs(request): cursor = request.get("afterCursor") require(cursor is None or isinstance(cursor, str) and re.fullmatch(r"[A-Za-z0-9=;_-]{1,512}", cursor), "invalid journal cursor") - args = ["journalctl", "-u", owner["unit"], "--no-pager", "-o", "cat", "--show-cursor"] - args += ["--after-cursor", cursor, "-n", "10001"] if cursor else ["-n", "1"] + args = ["journalctl", "-u", owner["unit"], "--no-pager", "-o", "json", + "--output-fields=__CURSOR,MESSAGE"] + # +N selects the FIRST N entries after the cursor. A plain -n N + # selects the tail and could silently drop the beginning of a window. + args += ["--after-cursor", cursor, "-n", "+10000"] if cursor else ["-n", "1"] output = command(*args).stdout - lines, observed_cursor = [], None + lines, observed_cursor, entries = [], None, 0 for line in output.splitlines(): - if line.startswith("-- cursor: "): - observed_cursor = line[len("-- cursor: "):].strip() - else: - lines.append(line) - require(len(lines) < 10000, "journal window exceeds its bound; source attribution is incomplete") + row = json.loads(line) + entry_cursor = row.get("__CURSOR") + require(isinstance(entry_cursor, str) and re.fullmatch(r"[A-Za-z0-9=;_-]{1,512}", entry_cursor) + and entry_cursor not in {cursor, observed_cursor}, "journal page repeated or omitted its exact entry cursor") + message = row.get("MESSAGE") + require(isinstance(message, str), "journal message is not complete text") + entries += 1 + require(entries <= 10000, "journal page exceeds its bound; source attribution is incomplete") + observed_cursor = entry_cursor + lines.append(message) require(observed_cursor is not None or cursor is not None and not lines, "journal did not expose a complete cursor window") return {"unit": owner["unit"], "journal": "\n".join(lines), "beforeCursor": cursor, - "afterCursor": observed_cursor or cursor, "complete": True} + "afterCursor": observed_cursor or cursor, "complete": True, + "entries": entries, "hasMore": cursor is not None and entries == 10000} return {"unit": owner["unit"], "journal": command("journalctl", "-u", owner["unit"], "--no-pager", "-n", "2000").stdout} diff --git a/scripts/validation_heal/scanner_repair.py b/scripts/validation_heal/scanner_repair.py index a9e02ff2..9987ae75 100644 --- a/scripts/validation_heal/scanner_repair.py +++ b/scripts/validation_heal/scanner_repair.py @@ -13,6 +13,8 @@ BRANCHES = ("cold-missing", "cold-bitrot", "scanner-cycle-boundary") BUDGET_SECONDS = 240 +SOURCE_TARGETS = {"rustfs::scanner", "rustfs::scanner::folder", "rustfs::heal::channel", + "rustfs::heal::manager", "rustfs::heal::task"} def event_time(event): @@ -28,8 +30,9 @@ def null_version(value): class JournalWindow: - def __init__(self, fleet, label): + def __init__(self, fleet, label, *, event_targets=None): self.fleet, self.label = fleet, label + self.event_targets = event_targets self.events = {node: [] for node in fleet.nodes} self.cursors = {} for node in fleet.nodes: @@ -39,15 +42,25 @@ def __init__(self, fleet, label): def capture(self): for node in self.fleet.nodes: - observed = self.fleet.record(self.label + "-window", {"node": node, - **self.fleet.remote(node, "logs", afterCursor=self.cursors[node])}) - require(observed.get("complete") is True and observed.get("beforeCursor") == self.cursors[node] - and observed.get("afterCursor"), "journal window lost its checkpoint or completeness") - self.cursors[node] = observed["afterCursor"] - for line in observed.get("journal", "").splitlines(): - try: event = json.loads(line) - except json.JSONDecodeError: continue - if isinstance(event, dict): self.events[node].append(event) + for page in range(6): + observed = self.fleet.record(self.label + "-window", {"node": node, + **self.fleet.remote(node, "logs", afterCursor=self.cursors[node])}) + require(observed.get("complete") is True and observed.get("beforeCursor") == self.cursors[node] + and observed.get("afterCursor"), "journal window lost its checkpoint or completeness") + require(observed.get("hasMore") is not True or observed["afterCursor"] != self.cursors[node], + "journal pagination did not advance") + self.cursors[node] = observed["afterCursor"] + for line in observed.get("journal", "").splitlines(): + try: event = json.loads(line) + except json.JSONDecodeError: continue + # Raw pages remain in immutable evidence; keep only source + # attribution events in memory, not unrelated debug noise. + if isinstance(event, dict) and (self.event_targets is None + or event.get("target") in self.event_targets): + self.events[node].append(event) + if observed.get("hasMore") is not True: break + else: + raise ValueError("journal window exceeds its page budget; source attribution is incomplete") require(len(self.events[node]) <= 60000, "scanner evidence exceeded its registered event budget") return self.events diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index d451db6d..067a5fe4 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -29,7 +29,7 @@ from .object_lock import BRANCHES as LOCK_BRANCHES, KEYS as LOCK_KEYS, seed_locked, prove_locked, cleanup_locked_fleet from .read_repair import BRANCHES as READ_BRANCHES, tracked_read_task from .drive_gate import validate_unwritable - from .scanner_repair import BRANCHES as SCANNER_BRANCHES, JournalWindow, observe_task as scanner_task + from .scanner_repair import BRANCHES as SCANNER_BRANCHES, SOURCE_TARGETS, JournalWindow, observe_task as scanner_task from .markers import (BRANCHES as MARKER_BRANCHES, INSTANCES as MARKER_INSTANCES, seed_markers, marker_plan, prepare_stale, verify_markers, reveal_current) from .stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch @@ -49,7 +49,7 @@ from object_lock import BRANCHES as LOCK_BRANCHES, KEYS as LOCK_KEYS, seed_locked, prove_locked, cleanup_locked_fleet from read_repair import BRANCHES as READ_BRANCHES, tracked_read_task from drive_gate import validate_unwritable - from scanner_repair import BRANCHES as SCANNER_BRANCHES, JournalWindow, observe_task as scanner_task + from scanner_repair import BRANCHES as SCANNER_BRANCHES, SOURCE_TARGETS, JournalWindow, observe_task as scanner_task from markers import (BRANCHES as MARKER_BRANCHES, INSTANCES as MARKER_INSTANCES, seed_markers, marker_plan, prepare_stale, verify_markers, reveal_current) from stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch @@ -752,7 +752,6 @@ def run_topology(args, topology, nodes, fixture): processes = fleet.snapshot("deployed-topology") fleet.record("topology-proof", validate_topology(processes, topology)) scanner = args.case_id == "heal.scanner-cold-object-repair" - healthy_window = JournalWindow(fleet, "scanner-healthy") if scanner else None if scanner: fleet.record("scanner-registered-budget", {"perBranchSeconds": 240, "sourceBranches": list(SCANNER_BRANCHES), "actualProcessConfiguration": {node: value["backgroundConfig"] for node, value in processes.items()}, @@ -799,6 +798,7 @@ def run_topology(args, topology, nodes, fixture): fleet.record("semantic-baseline", verify_semantics(public, baseline)) if scanner: healthy_before = fleet.snapshot("scanner-healthy-physical-before") + healthy_window = JournalWindow(fleet, "scanner-healthy", event_targets=SOURCE_TARGETS) for branch in branches: plan = branch_plan(args.case_id, branch, healthy_before, bucket, expected, fixture) status = scanner_task(probe, fleet, healthy_window, bucket, plan["expected"][0], branch, healthy=True) @@ -845,7 +845,7 @@ def run_topology(args, topology, nodes, fixture): stale_marker_started = False continue before = fleet.snapshot("physical-before-" + branch, quiescent=True) - scanner_window = JournalWindow(fleet, "scanner-fault-" + branch) if scanner else None + scanner_window = JournalWindow(fleet, "scanner-fault-" + branch, event_targets=SOURCE_TARGETS) if scanner else None if size_rule: plan = boundary_plan(next(entry for entry in instances if entry["name"] == branch), before, bucket, expected) elif args.case_id == "heal.delete-marker-semantics": diff --git a/test_validation_heal.py b/test_validation_heal.py index d7348230..96d0ae04 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -1342,17 +1342,44 @@ def test_remote_log_window_rejects_truncation_and_invalid_cursor(self): owner = {"unit": "private.service"} with patch("scripts.validation_heal.fleet_node.owned", return_value=(None, None, owner)), \ patch("scripts.validation_heal.fleet_node.command") as command: - command.return_value = types.SimpleNamespace(stdout='{"message":"fresh"}\n-- cursor: s=abc;i=def\n') + command.return_value = types.SimpleNamespace(stdout=json.dumps({"MESSAGE": '{"message":"fresh"}', + "__CURSOR": "s=abc;i=def"}) + "\n") value = node_logs({"runId": "unit", "afterCursor": "s=previous;i=1"}) self.assertTrue(value["complete"]) self.assertEqual(value["afterCursor"], "s=abc;i=def") self.assertIn("--after-cursor", command.call_args.args) + self.assertIn("+10000", command.call_args.args) + self.assertFalse(value["hasMore"]) with self.assertRaisesRegex(ValueError, "invalid journal cursor"): node_logs({"runId": "unit", "afterCursor": "invalid\n--help"}) - command.return_value = types.SimpleNamespace(stdout="row\n" * 10000 + "-- cursor: s=abc;i=def\n") + rows = [json.dumps({"MESSAGE": "row", "__CURSOR": f"s=abc;i={index}"}) for index in range(10000)] + command.return_value = types.SimpleNamespace(stdout="\n".join(rows)) + page = node_logs({"runId": "unit", "afterCursor": "s=previous;i=1"}) + self.assertTrue(page["hasMore"]) + self.assertEqual(page["afterCursor"], "s=abc;i=9999") + command.return_value = types.SimpleNamespace(stdout="\n".join(rows + [json.dumps({"MESSAGE": "row", "__CURSOR": "s=abc;i=10000"})])) with self.assertRaisesRegex(ValueError, "incomplete"): node_logs({"runId": "unit", "afterCursor": "s=previous;i=1"}) + def test_cursor_pagination_keeps_early_attribution_and_does_not_skip_pages(self): + events = self.events() + observations = [{"complete": True, "afterCursor": "checkpoint"}, + {"complete": True, "beforeCursor": "checkpoint", "afterCursor": "page-one", "hasMore": True, + "journal": json.dumps(events[0])}, + {"complete": True, "beforeCursor": "page-one", "afterCursor": "page-two", "hasMore": False, + "journal": json.dumps(events[1])}] + calls = [] + def remote(node, action, **params): + calls.append(params) + return observations.pop(0) + fleet = types.SimpleNamespace(nodes=["host"], record=lambda label, value: value, remote=remote) + window = JournalWindow(fleet, "unit") + self.assertEqual(window.capture(), {"host": events[:2]}) + self.assertEqual(calls[1:], [{"afterCursor": "checkpoint"}, {"afterCursor": "page-one"}]) + window.fleet.remote = lambda *args, **kwargs: {"complete": True, "beforeCursor": "page-two", + "afterCursor": "page-two", "hasMore": True} + with self.assertRaisesRegex(ValueError, "did not advance"): window.capture() + def test_scanner_profile_enables_only_scanner_source_and_fixed_deep_timing(self): config = runtime_config("scanner") self.assertEqual(config["RUSTFS_SCANNER_ENABLED"], "true") From 3ac26660b1a693b2309fbdf7b990a530060dd212 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 04:27:52 +0800 Subject: [PATCH 071/343] feat(validation): migrate full-cluster heal restart --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 31 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 55 +-- scripts/validation_heal/full_restart.py | 347 ++++++++++++++++++ scripts/validation_heal/healthy.py | 4 + scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/replacement.py | 20 + scripts/validation_heal/run_case.sh | 1 + test_validation_heal.py | 14 + test_validation_heal_full_restart.py | 157 ++++++++ 11 files changed, 611 insertions(+), 26 deletions(-) create mode 100644 scripts/validation_heal/full_restart.py create mode 100644 test_validation_heal_full_restart.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index a478a607..ec954fdb 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -18,6 +18,7 @@ on: - test_validation_heal.py - test_validation_heal_recovery.py - test_validation_heal_authorization.py + - test_validation_heal_full_restart.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -49,6 +50,7 @@ on: - test_validation_heal.py - test_validation_heal_recovery.py - test_validation_heal_authorization.py + - test_validation_heal_full_restart.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -80,7 +82,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 32590b82..b4296d50 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 113 currently dispatchable IDs include all 74 non-Heal cases and thirty-nine Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. +case evidence. The 114 currently dispatchable IDs include all 74 non-Heal cases and forty Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 45 Heal cases remain +The other 44 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -765,3 +765,30 @@ are retained in the scanner's in-memory index. The healthy checkpoint starts after seed, baseline and the initial physical snapshot, so a fresh complete scanner cycle must actually occur in the registered observation window. Earlier log-budget failures remain FAIL; this harness change has no live PASS claim. + +F05 `heal.full-cluster-restart` now has an H2 executor for all three source +branches. It seeds DB, historical DV/markers, completed multipart objects, +exact pending UploadIds/parts and a 1,025-object DX observation fixture on the +current four Heal peers. A root task must have physically repaired some but not +all injected shards before the original coordinator is paused. Every original +peer then exits with witnessed SIGTERM/normal code zero in a registered order. +The same task ID, physical intent owner, non-timeout options and remaining +budget must survive all units being stopped. Recovery uses no new root start. +Complete canonical counters, bounded exact diagnostics, durable terminal/report, +full same-role physical manifests, streaming shard checks and all-peer S3 +inventories must agree. All four processes restart again after completion; +terminal records and user files must remain unchanged. Pending MPU listings +remain exact across both restarts and their original parts are then completed +and checked by full payload hash on every peer. Multi-bucket snapshots and +streaming verification bind each run-owned bucket explicitly. F05 has no live +PASS claim until that complete sequence succeeds. + +C06 run `38083035055` retained FAIL and cleanup PASS: all three H1 branches +passed with the same live service rejoin, but H2 failed the older 10,000-line +journal bound. The paginated journal implementation must be validated in a fresh +run. B03 run `38083328049` retained FAIL and cleanup PASS on all H1/H2 DB/DV +instances: the candidate had no replacement `format.json` at the first physical +snapshot. The executor now observes a registered 90-second, read-only format +initialization window before requiring distinct new UUID/set-slot evidence; +service health still cannot certify replacement acceptance. Neither result +increases the complete live PASS count. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index c5d8991c..b2d1cd80 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 213cc376..e8a4aad0 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -471,10 +471,11 @@ def verify_part_owned(request, roots, owner, tool): data = root / "data" part = checked(data / relative, data) metadata = checked(part.parent.parent / "xl.meta", data) - key = metadata.parent.relative_to(data / owner["bucket"]).as_posix() - result = json.loads(command(str(tool), "--verify-part", str(metadata), owner["bucket"], key, + bucket = parts[0] + key = metadata.parent.relative_to(data / bucket).as_posix() + result = json.loads(command(str(tool), "--verify-part", str(metadata), bucket, key, str(part), request["versionId"], str(request["partNumber"])).stdout) - require(result.get("physicalBitrotVerified") is True and result.get("key") == key + require(result.get("physicalBitrotVerified") is True and result.get("bucket") == bucket and result.get("key") == key and result.get("versionId") == request["versionId"] and result.get("partNumber") == request["partNumber"], "bitrot proof identity differs") return {**result, "sha256": digest(part), "relativePath": relative, "disk": request["disk"]} @@ -505,8 +506,8 @@ def verify_inline(request): key = "/".join(parts[1:-1]) tool = checked(state / "heal-meta-inspect", state) require(digest(tool) == owner.get("inspectorSha256"), "inline verifier changed") - result = json.loads(command(str(tool), "--verify-inline", str(path), owner["bucket"], key, request["versionId"]).stdout) - require(result.get("physicalBitrotVerified") is True and result.get("key") == key + result = json.loads(command(str(tool), "--verify-inline", str(path), parts[0], key, request["versionId"]).stdout) + require(result.get("physicalBitrotVerified") is True and result.get("bucket") == parts[0] and result.get("key") == key and result.get("versionId") == request["versionId"], "inline proof identity differs") return {**result, "relativePath": request["relativePath"], "disk": request["disk"]} @@ -534,6 +535,17 @@ def inspect_metadata(tool, path, bucket, key, entry, metadata_faults): return {"metadata": decoded} +def snapshot_buckets(owner, request): + buckets = request.get("buckets", [owner["bucket"]]) + require(isinstance(buckets, list) and 1 <= len(buckets) <= 8 + and all(isinstance(bucket, str) and 3 <= len(bucket) <= 63 + and re.fullmatch(r"[a-z0-9][a-z0-9.-]+[a-z0-9]", bucket) + and (bucket == owner["bucket"] or bucket.startswith(owner["bucket"] + "-")) for bucket in buckets) + and len(set(buckets)) == len(buckets) and owner["bucket"] in buckets, + "physical bucket scope escaped the original run-owned prefix or omitted its primary bucket") + return buckets + + def snapshot(request): state, roots, owner = owned(request["runId"]) pid = int(command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"], check=False).stdout.strip() or "0") @@ -557,21 +569,22 @@ def snapshot(request): require(digest(checked(tool, state)) == owner["inspectorSha256"], "metadata inspector changed") for index, root in enumerate(roots): data = root / "data" - folder = checked(data / owner["bucket"], data) - if folder.exists(): - for directory, children, names in os.walk(folder, followlinks=False): - for name in (*children, *names): - checked(Path(directory) / name, data) - for name in sorted(names): - path = Path(directory) / name - require(len(files) < limits["maxSnapshotFiles"] and path.stat().st_size <= 64 * 1024**2, - "fixture exceeds physical evidence budget") - entry = {"disk": index, "path": str(path.relative_to(data)), - "size": path.stat().st_size, "sha256": digest(path)} - if path.name == "xl.meta" and owner.get("inspectorSha256"): - key = path.parent.relative_to(folder).as_posix() - entry.update(inspect_metadata(tool, path, owner["bucket"], key, entry, request.get("metadataFaults", {}))) - files.append(entry) + for bucket in snapshot_buckets(owner, request): + folder = checked(data / bucket, data) + if folder.exists(): + for directory, children, names in os.walk(folder, followlinks=False): + for name in (*children, *names): + checked(Path(directory) / name, data) + for name in sorted(names): + path = Path(directory) / name + require(len(files) < limits["maxSnapshotFiles"] and path.stat().st_size <= 64 * 1024**2, + "fixture exceeds physical evidence budget") + entry = {"disk": index, "path": str(path.relative_to(data)), + "size": path.stat().st_size, "sha256": digest(path)} + if path.name == "xl.meta" and owner.get("inspectorSha256"): + key = path.parent.relative_to(folder).as_posix() + entry.update(inspect_metadata(tool, path, bucket, key, entry, request.get("metadataFaults", {}))) + files.append(entry) path = checked(data / ".rustfs.sys/format.json", data) require(path.stat().st_size < 65536, "disk format exceeds budget") formats.append({"disk": index, "sha256": digest(path), "value": json.loads(path.read_text())}) @@ -590,7 +603,7 @@ def logs(request): cursor = request.get("afterCursor") require(cursor is None or isinstance(cursor, str) and re.fullmatch(r"[A-Za-z0-9=;_-]{1,512}", cursor), "invalid journal cursor") - args = ["journalctl", "-u", owner["unit"], "--no-pager", "-o", "json", + args = ["journalctl", "-u", owner["unit"], "--quiet", "--no-pager", "-o", "json", "--output-fields=__CURSOR,MESSAGE"] # +N selects the FIRST N entries after the cursor. A plain -n N # selects the tail and could silently drop the beginning of a window. diff --git a/scripts/validation_heal/full_restart.py b/scripts/validation_heal/full_restart.py new file mode 100644 index 00000000..426553eb --- /dev/null +++ b/scripts/validation_heal/full_restart.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +"""F05: all owned peers restart with original durable root responsibility.""" + +from __future__ import annotations + +import argparse +import copy +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +import uuid + +try: + from .heal_probe import Probe, canonical, compare_inventories, payload, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .heal_fault import bounded_fault_id + from .shards import pending_inventory, validate_repaired_physical + from .cursor import OBJECTS, SIZE, TASK_SECONDS, map_cohorts, verify_all_parts, validate_missing_batch + from .root_restart import (PATH, OPTIONS, records, owned_records, root_intent, same_owner, + partial_physical, recovery_status) + from .stop import wait_partial, idle_fleet + from .fleet_node import PROCESS_FIELDS + from .versions import public_version_id +except ImportError: + from heal_probe import Probe, canonical, compare_inventories, payload, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from heal_fault import bounded_fault_id + from shards import pending_inventory, validate_repaired_physical + from cursor import OBJECTS, SIZE, TASK_SECONDS, map_cohorts, verify_all_parts, validate_missing_batch + from root_restart import PATH, OPTIONS, records, owned_records, root_intent, same_owner, partial_physical, recovery_status + from stop import wait_partial, idle_fleet + from fleet_node import PROCESS_FIELDS + from versions import public_version_id + + +CASE_ID = "heal.full-cluster-restart" +BRANCHES = ("all-graceful-restart", "all-restart-with-mpu", "repeat-full-restart") + + +def manifest(fixtures): + wanted = {(bucket, entry["key"], entry["versionId"]): entry + for bucket, fixture in fixtures.items() for entry in fixture["expected"]} + require(len(wanted) == sum(len(fixture["expected"]) for fixture in fixtures.values()) + and len(wanted) > 1024, "full restart lacks its frozen complete ACK manifest") + return wanted + + +def responsibility(current, wanted, repaired): + outcome = current.get("outcome") or {} + require(current.get("summary") == "finished" and outcome.get("execution", {}).get("state") == "completed" + and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is True, + "root recovery has no complete responsibility or bounded diagnostic receipts") + require(repaired and set(repaired) <= set(wanted), "root recovery repair set escaped original ACKs") + counts = {"processed": len(wanted), "healed": len(repaired), "unchanged": len(wanted) - len(repaired), + "skipped": 0, "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False} + counters = outcome.get("counters", {}) + require(all(type(counters.get(name)) is type(value) and counters[name] == value for name, value in counts.items()), + "root recovery counters omit ACKs, lost responsibility or claim false repair") + objects = outcome.get("objects") + require(isinstance(objects, list) and 0 < len(objects) <= 128, "root diagnostics exceed their bounded contract") + seen, incarnations = set(), {} + for receipt in objects: + identity = receipt.get("identity", {}) + pair = identity.get("bucket"), identity.get("object"), public_version_id(identity.get("versionId")) + incarnation = identity.get("bucketIncarnationId") + require(pair in wanted and pair not in seen and isinstance(incarnation, str) and incarnation + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0, + "root diagnostics contain an unknown, duplicate or incomplete ACK identity") + require(incarnations.setdefault(pair[0], incarnation) == incarnation, "root diagnostics mixed bucket incarnations") + state = "repaired" if pair in repaired else "metadata_healthy" if wanted[pair]["deleteMarker"] else "verified_healthy" + require(receipt.get("disposition", {}).get("state") == state, "root diagnostic falsely attributed a repair or integrity check") + seen.add(pair) + return {"canonicalCounters": counters, "diagnosticObjects": len(objects), + "diagnosticsAreFullManifest": False, "originalACKs": len(wanted)} + + +def terminal_records(observed, token, owner, current, wanted): + present = owned_records(observed, token) + require(set(present) == {"terminal", "report"} + and all((item["node"], item["disk"]) == (owner["node"], owner["disk"]) for item in present.values()), + "all-peer restart lost the original record owner or retained runnable responsibility") + terminal, report = present["terminal"]["value"], present["report"]["value"] + require(type(terminal.get("schema")) is int and terminal["schema"] == 1 + and terminal.get("task_id") == token and terminal.get("status") == "Completed" + and terminal.get("heal_type") == {"type": "cluster"} + and canonical(terminal.get("options")) == canonical(owner["value"]["options"]), + "root completion changed task, scope, checkpointed options or original budget") + require(type(report.get("schema")) is int and report["schema"] == 1 and report.get("terminal") == terminal + and canonical(report.get("outcome")) == canonical(current.get("outcome")), + "root durable report differs from the original recovered task") + require(type(report.get("next_seq")) is int and report["next_seq"] == len(wanted) + 1 + and type(report.get("min_seq")) is int and 1 <= report["min_seq"] <= report["next_seq"] + and isinstance(report.get("seqed_items"), list) and 0 < len(report["seqed_items"]) <= 1024, + "root durable report has incomplete retained sequence bounds") + seen, previous = set(), report["min_seq"] - 1 + for sequence, item in report["seqed_items"]: + pair = item.get("bucket"), item.get("object"), public_version_id(item.get("versionId")) + require(type(sequence) is int and sequence == previous + 1 and sequence < report["next_seq"] + and pair in wanted and pair not in seen + and item.get("objectSize") == wanted[pair].get("payload", {}).get("size", 0), + "root durable report skipped a sequence or changed an ACK version/marker/size") + seen.add(pair); previous = sequence + require(previous == report["next_seq"] - 1, "root durable report omitted its retained suffix") + return {"sameRecordDisk": True, "runnableIntentRetiredOnEveryDisk": True, + "terminalAndReportMatchOriginalToken": True, "retainedItems": len(seen)} + + +def all_processes(fleet, label): + return fleet.record(label, {node: fleet.remote(node, "process") for node in fleet.nodes}) + + +def continuity(before, after): + require(set(before) == set(after) and len(before) == 4, "full restart omitted an original peer") + for node, old in before.items(): + new = after[node] + require(all(new.get(name) == old.get(name) for name in PROCESS_FIELDS if name not in {"pid", "startTicks"}) + and type(new.get("pid")) is int and new["pid"] > 1 and type(new.get("startTicks")) is int + and (new["pid"], new["startTicks"]) != (old["pid"], old["startTicks"]), + "all-peer restart changed candidate/boot/unit/environment/namespace or did not restart a peer") + return {"allFourOriginalProcessesRestarted": True, "sameBootCandidateUnitsEnvironmentAndNamespace": True} + + +def graceful_all(fleet, processes, first): + require(first in processes and set(processes) == set(fleet.nodes), "graceful stop omitted original peer identities") + order = [first, *(node for node in fleet.nodes if node != first)] + fleet.record("registered-all-peer-stop-order", {"nodes": order, "onlyOwnedUnits": True}) + exits = {} + for node in order: + identity = processes[node] + if node not in fleet.frozen_processes: + fleet.frozen_processes[node] = identity + fleet.record("full-restart-freeze", fleet.remote(node, "process-signal", signal="SIGSTOP", expected=identity)) + exits[node] = fleet.record("full-restart-graceful-exit", {"node": node, + **fleet.remote(node, "process-graceful", expected=identity)}) + require(exits[node].get("actualExitCode") == 0 and exits[node].get("signal") == "SIGTERM" + and exits[node].get("pidfdBound") is True, "a peer lacks a witnessed normal graceful exit") + del fleet.frozen_processes[node] + fleet.record("release-exited-unit", {"node": node, **fleet.remote(node, "stop")}) + return exits + + +def verify_public(probe, fleet, fixtures, pending, label): + for endpoint in probe.binding["endpoints"]: + peer = Probe(probe.binding, endpoint) + for bucket, fixture in fixtures.items(): + actual = fleet.record(label, {"endpoint": endpoint, "bucket": bucket, + **peer.inventory(bucket, unversioned_current_tags=fixture["fixture"] != "DV", page_size=127)}) + verify_seed(fixture["expected"], actual) + compare_inventories(fixture["baseline"], actual) + actual_pending = fleet.record(label + "-pending-mpu", {"endpoint": endpoint, + "uploads": pending_inventory(peer, pending["bucket"])}) + require(actual_pending["uploads"] == pending["uploads"], "full restart lost an exact pending UploadId/part/ETag") + + +def run_topology(args, nodes): + directory = args.evidence / "H2" + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-h2" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, resource_profile="cursor", runtime_profile="root-restart") + candidate = json.loads(args.candidate.read_text()) + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + fleet.snapshot_buckets = [bucket, bucket + "-dx", bucket + "-versions"] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, 5)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 2048, "maxInventoryBytes": 2 * 1024**3} + fleet.record("binding", binding) + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "full restart omitted a source branch") + result = {"topology": "H2", "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} + active, probe = [], Probe(binding) + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), "H2")) + fleet.record("registered-full-restart-budget", {"dxObjects": OBJECTS, "dxBytes": SIZE, + "taskSeconds": 300, "recoveryObservationSeconds": TASK_SECONDS, "serviceMinutes": 45, + "sourceFixtures": ["DB", "DV", "DM"], "allOriginalUnitsRestartedTwice": True}) + recorder = Recorder(probe, directory / "fixture") + db = seed(recorder, bucket, "DB", 128, [0, 4096, SIZE, 5 * 1024**2 + 123]) + db += seed(recorder, bucket, "DM", 1, [SIZE], existing_bucket=True) + dx_bucket, dv_bucket = bucket + "-dx", bucket + "-versions" + dx = seed(recorder, dx_bucket, "DX", OBJECTS, [SIZE]) + dv = seed(recorder, dv_bucket, "DV", 4, [5 * 1024**2 + 123]) + fixtures = {name: {"fixture": kind, "expected": expected} for name, kind, expected in + ((bucket, "DB", db), (dx_bucket, "DX", dx), (dv_bucket, "DV", dv))} + for name, fixture in fixtures.items(): + fixture["baseline"] = fleet.record("s3-baseline", probe.inventory(name, unversioned_current_tags=fixture["fixture"] != "DV", page_size=127)) + fleet.record("fixture-manifest", {"bucket": name, "expected": fixture["expected"], **verify_seed(fixture["expected"], fixture["baseline"])}) + wanted = manifest(fixtures) + pending = {"bucket": bucket, "uploads": fleet.record("mpu-baseline", pending_inventory(probe, bucket))} + fleet.stop() + original = fleet.snapshot("physical-baseline", quiescent=True) + dx_scene = copy.deepcopy(original) + for value in dx_scene.values(): value["files"] = [item for item in value["files"] if item["path"].startswith(dx_bucket + "/")] + cohorts = map_cohorts(dx_scene, dx_bucket, dx) + verify_all_parts(fleet, cohorts, "full-restart-baseline-streaming") + targets, receipts = [cohorts[index] for index in range(0, OBJECTS, 32)], [] + for index, target in enumerate(targets): + params = {"disk": target["disk"], "relativePath": target["relativePath"], "kind": "missing", + "faultId": bounded_fault_id("F05", "full-restart", index), "expectedSha256": target["sha256"]} + fleet.faults.append((target["node"], params)) + receipts.append(fleet.record("fault", {"node": target["node"], **fleet.remote(target["node"], "apply-fault", **params)})) + faulted = fleet.snapshot("physical-fault", quiescent=True) + fleet.record("fault-proof", validate_missing_batch(original, faulted, targets, receipts)) + fleet.start() + require(physical_content(faulted) == physical_content(fleet.snapshot("physical-before-trigger")), + "ATTRIBUTION_NOT_OBSERVED: restart fault changed before root admission") + admission = fleet.record("root-admission", probe.admin(PATH, [], json.dumps(OPTIONS).encode())) + token = admission.get("response", {}).get("clientToken") + require(admission.get("httpStatus") == 200 and isinstance(token, str) and str(uuid.UUID(token)) == token, "root admission failed") + active.append((PATH, token)) + first = wait_partial(probe, fleet, PATH, token, targets, repair_required=True) + owners = owned_records(records(fleet, token, "root-intent-owner"), token) + require(set(owners) == {"intent"}, "NOT_OBSERVED: root completed before full-cluster interruption") + coordinator = owners["intent"]["node"] + processes = all_processes(fleet, "all-peers-before-restart") + fleet.frozen_processes[coordinator] = processes[coordinator] + fleet.record("root-freeze", fleet.remote(coordinator, "process-signal", signal="SIGSTOP", expected=processes[coordinator])) + _, before_partial = partial_physical(original, fleet.snapshot("root-frozen-physical"), targets) + before_record = root_intent(records(fleet, token, "root-intent-before-all-stop"), token, coordinator) + exits = graceful_all(fleet, processes, coordinator) + after_observed = fleet.record("root-intent-after-all-stop", {node: fleet.remote(node, "task-records", taskId=token, quiescent=True) for node in nodes}) + after_record = root_intent(after_observed, token, coordinator) + budget = same_owner(before_record, after_record) + stopped = fleet.snapshot("all-peers-stopped-physical", quiescent=True) + remaining, after_partial = partial_physical(original, stopped, targets) + fleet.start() + restarted = all_processes(fleet, "all-peers-after-restart") + changed = continuity(processes, restarted) + deadline = time.monotonic() + TASK_SECONDS + while True: + current = recovery_status(probe, fleet, token) + require(current.get("summary") in {"running", "finished"}, "original root task failed, stopped or disappeared after full restart") + if current["summary"] == "finished": break + require(time.monotonic() < deadline, "original root task exceeded its full-restart recovery budget") + time.sleep(0.05) + repaired = {(item["bucket"], item["key"], item["versionId"]) for item in remaining} + proof = responsibility(current, wanted, repaired) + active.remove((PATH, token)) + idle_fleet(fleet, binding) + journal = records(fleet, token, "root-terminal-before-repeat") + durable = terminal_records(journal, token, after_record, current, wanted) + fleet.stop() + rebuilt = fleet.snapshot("physical-repaired-before-get", quiescent=True) + validate_repaired_physical(original, rebuilt, targets[0], repair_targets=targets) + verify_all_parts(fleet, cohorts, "full-restart-repaired-streaming") + fleet.start() + verify_public(probe, fleet, fixtures, pending, "all-peer-s3-after-first-restart") + before_repeat = all_processes(fleet, "all-peers-before-repeat") + graceful_all(fleet, before_repeat, coordinator) + require(physical_content(rebuilt) == physical_content(fleet.snapshot("physical-before-repeat", quiescent=True)), "first recovery GET repaired a previously unproven file") + fleet.start() + repeated = all_processes(fleet, "all-peers-after-repeat") + repeat_processes = continuity(before_repeat, repeated) + idle_fleet(fleet, binding) + repeat = recovery_status(probe, fleet, token) + require(repeat.get("summary") == "finished" and canonical(repeat.get("outcome")) == canonical(current["outcome"]), "repeat full restart revived or changed original responsibility") + repeated_journal = records(fleet, token, "root-terminal-after-repeat") + terminal_records(repeated_journal, token, after_record, repeat, wanted) + require(canonical(journal) == canonical(repeated_journal), "repeat full restart changed completed journal bytes") + verify_public(probe, fleet, fixtures, pending, "all-peer-s3-after-repeat") + fleet.stop() + require(physical_content(rebuilt) == physical_content(fleet.snapshot("physical-after-repeat-get", quiescent=True)), "repeat full restart or verification changed a user file") + fleet.start() + for upload in pending["uploads"]: + recorder.call("complete_multipart_upload", {"Bucket": bucket, "Key": upload["key"], "UploadId": upload["uploadId"], + "MultipartUpload": {"Parts": [{"PartNumber": item["PartNumber"], "ETag": item["ETag"]} for item in upload["parts"]]}}) + expected_body = payload(f"{bucket}/{upload['key']}", 5 * 1024**2 + 3) + for endpoint in binding["endpoints"]: + observed = fleet.record("pending-mpu-content-after-repeat", Probe(binding, endpoint).s3("get_object", {"Bucket": bucket, "Key": upload["key"]})) + require(observed.get("acknowledged") is True and observed.get("resultUnknown") is False + and observed.get("response", {}).get("payload") == {"size": len(expected_body), "sha256": hashlib.sha256(expected_body).hexdigest()}, "restarted pending MPU parts lost their actual acknowledged bytes") + result["branches"] = { + BRANCHES[0]: {"verdict": "PASS", "originalTaskId": token, "runningWindow": first["outcome"]["counters"], + "beforeInterruptionPhysical": before_partial, "afterAllActualExitsPhysical": after_partial, + "actualPeerExits": exits, "processContinuity": changed, "budget": budget, "responsibility": proof, + "durableTerminal": durable, "newRootAdmissionIssued": False}, + BRANCHES[1]: {"verdict": "PASS", "exactPendingUploadsAndPartsPreserved": True, + "pendingPartsCompletedAndFullContentCheckedOnAllPeers": True, "completedMPUAndHistoricalVersionsAndMarkersVerified": True}, + BRANCHES[2]: {"verdict": "PASS", "processContinuity": repeat_processes, + "completedJournalBytesUnchanged": True, "originalTerminalUnchanged": True, "allUserFilesUnchanged": True}} + result["verdict"] = "PASS" + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + try: + # Fleet.cleanup resumes a frozen original process before stopping it. + if not fleet.frozen_processes: + fleet.stop(); fleet.snapshot("physical-failure", quiescent=True) + except Exception as evidence_error: result["failureEvidenceError"] = str(evidence_error) + finally: + for path, token in active: + try: fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: fleet.record("stop-error", {"error": str(exc)}) + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("topology-result", result) + return result + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f"Heal full-restart interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path, required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split()[:4] + require(len(nodes) == len(set(nodes)) == 4 and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal peers required") + require(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")) + and re.fullmatch(r"[0-9a-f]{64}", args.binary_sha), "immutable candidate and workflow identity required") + results = [] + try: results.append(run_topology(args, nodes)) + finally: + complete = len(results) == 1 and results[0]["verdict"] == "PASS" and results[0]["cleanup"] == "PASS" + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, + "cleanup": results[0]["cleanup"] if results else "FAIL", "complete": complete}) + require(complete, "full-cluster restart requires every H2 source branch and clean finalization") + + +if __name__ == "__main__": + try: main() + except Exception as exc: + print(f"Heal full-restart validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index a8eabfec..45302a01 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -143,6 +143,7 @@ def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_prof self.drive_gates = set() self.secret_values = set() self.frozen_processes = {} + self.snapshot_buckets = None def record(self, label, value): self.sequence += 1 @@ -165,6 +166,9 @@ def ssh(self, node): os.environ.get("RUSTFS_SSH_USER", "azureuser") + "@" + node] def remote(self, node, action, **params): + if action == "snapshot" and self.snapshot_buckets is not None: + require("buckets" not in params, "physical bucket scope must use the frozen fleet binding") + params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline"}: diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 24e77530..62d2e3d2 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted", "heal.full-cluster-restart"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/replacement.py b/scripts/validation_heal/replacement.py index edec9234..bb46dc50 100644 --- a/scripts/validation_heal/replacement.py +++ b/scripts/validation_heal/replacement.py @@ -1,6 +1,7 @@ """B03: full private-drive replacement with original same-slot shard oracles.""" import copy +import time try: from .heal_probe import Probe, canonical, compare_inventories, require @@ -15,6 +16,24 @@ BRANCHES = ("new-disk-data", "new-disk-versioned") +FORMAT_SECONDS = 90 + + +def wait_format(fleet, node, disk): + deadline = time.monotonic() + FORMAT_SECONDS + while True: + observed = fleet.record("replacement-format-initialization-window", fleet.remote(node, "replacement-state")) + require(observed.get("readOnly") is True and observed.get("originalArchiveUnchanged") is True, + "replacement initialization lost its read-only original-archive witness") + formats = observed.get("formats") + require(isinstance(formats, list) and len(formats) == 4 + and {item.get("disk") for item in formats} == set(range(4)), "replacement format probe omitted a member") + target = next(item for item in formats if item["disk"] == disk) + require(type(target.get("present")) is bool, "replacement format presence is unknown") + if target["present"]: return observed + require(time.monotonic() < deadline, + "candidate did not initialize the replacement format within its registered 90-second budget") + time.sleep(2) def format_identity(before, after, member): @@ -104,6 +123,7 @@ def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admi denied = fleet.record("replacement-backup-service-refusal", fleet.remote(node, "replacement-probe")) require(denied.get("serviceReadDenied") is True and denied.get("sameOriginalServiceProcess") is True, "replacement candidate can access its old physical backup") + wait_format(fleet, node, disk) initialized = fleet.snapshot("replacement-candidate-initialization") fleet.record("replacement-new-format-proof", format_identity(before, initialized, member)) require(not any(item["disk"] == disk for item in initialized[node]["files"]), diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 91e77157..358ba763 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -26,6 +26,7 @@ case "${case_id}" in heal.cancelled-task-restart) driver=cancel_restart.py ;; heal.admin-overlap-and-force-start) driver=overlap.py ;; heal.root-graceful-coordinator-restart) driver=root_restart.py ;; + heal.full-cluster-restart) driver=full_restart.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal.py b/test_validation_heal.py index 96d0ae04..5f901518 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -3023,6 +3023,20 @@ def test_archive_rejects_links_and_wrong_format_before_mutation(self): self.assertFalse((root / "quarantine/old-disk").exists()) self.assertEqual((root / "data/ack-part").read_bytes(), b"original-ack-shard") + def test_replacement_format_wait_never_substitutes_service_ready_for_member_readiness(self): + absent = {"readOnly": True, "originalArchiveUnchanged": True, + "formats": [{"disk": index, "present": index != 3} for index in range(4)]} + present = copy.deepcopy(absent); present["formats"][3]["present"] = True + fleet = types.SimpleNamespace(record=lambda label, value: value, remote=lambda *args: absent) + with patch("scripts.validation_heal.replacement.time.monotonic", side_effect=[0, 91]), self.assertRaisesRegex(ValueError, "90-second"): + replacement.wait_format(fleet, "node1", 3) + rows = [absent, present] + fleet.remote = lambda *args: rows.pop(0) + with patch("scripts.validation_heal.replacement.time.sleep"), patch("scripts.validation_heal.replacement.time.monotonic", return_value=0): + self.assertTrue(replacement.wait_format(fleet, "node1", 3)["formats"][3]["present"]) + fleet.remote = lambda *args: {**present, "originalArchiveUnchanged": False} + with self.assertRaisesRegex(ValueError, "archive witness"): replacement.wait_format(fleet, "node1", 3) + def test_remote_replacement_module_initializes_and_all_source_branches_remain(self): import sys self.assertEqual(list(replacement.BRANCHES), select("B03")[0]["variants"]) diff --git a/test_validation_heal_full_restart.py b/test_validation_heal_full_restart.py new file mode 100644 index 00000000..197efb0d --- /dev/null +++ b/test_validation_heal_full_restart.py @@ -0,0 +1,157 @@ +import copy +import json +from pathlib import Path +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import full_restart as restart +from scripts.validation_heal.matrix import select, AUTOMATIC_CASES +from scripts.validation_heal.fleet_node import PROCESS_FIELDS, snapshot_buckets, verify_part_owned + + +class FullRestartTests(unittest.TestCase): + token = "12345678-1234-4234-8234-123456789abc" + + def wanted(self): + wanted = {("db", f"objects/{index:04d}", "null"): {"deleteMarker": False, "payload": {"size": 17}} + for index in range(1025)} + wanted[("versions", "history", "old-version")] = {"deleteMarker": False, "payload": {"size": 19}} + wanted[("versions", "history", "marker")] = {"deleteMarker": True} + return wanted + + def receipt(self): + repaired = {("db", "objects/0000", "null")} + objects = [] + for pair in (next(iter(repaired)), ("versions", "history", "old-version"), ("versions", "history", "marker")): + state = "repaired" if pair in repaired else "metadata_healthy" if pair[2] == "marker" else "verified_healthy" + objects.append({"identity": {"bucket": pair[0], "object": pair[1], "versionId": pair[2], + "poolIndex": 0, "setIndex": 0, "bucketIncarnationId": "incarnation-" + pair[0]}, "disposition": {"state": state}}) + return {"summary": "finished", "outcome": {"execution": {"state": "completed"}, "coverage": "complete", + "objectsTruncated": True, "objects": objects, + "counters": {"processed": 1027, "healed": 1, "unchanged": 1026, "skipped": 0, + "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False}}}, repaired + + def test_complete_counters_do_not_make_unknown_or_cross_bucket_receipts_pass(self): + current, repaired = self.receipt() + proof = restart.responsibility(current, self.wanted(), repaired) + self.assertFalse(proof["diagnosticsAreFullManifest"]) + for kind in ("unknown", "false-repair", "duplicate", "bucket", "version", "incarnation", "counter", "bool", "coverage", "truncation"): + with self.subTest(kind=kind): + bad = copy.deepcopy(current) + if kind == "unknown": bad["outcome"]["objects"][0]["disposition"]["state"] = "unknown" + elif kind == "false-repair": bad["outcome"]["objects"][1]["disposition"]["state"] = "repaired" + elif kind == "duplicate": bad["outcome"]["objects"].append(copy.deepcopy(bad["outcome"]["objects"][0])) + elif kind == "bucket": bad["outcome"]["objects"][0]["identity"]["bucket"] = "versions" + elif kind == "version": bad["outcome"]["objects"][1]["identity"]["versionId"] = "current-version" + elif kind == "incarnation": bad["outcome"]["objects"][2]["identity"]["bucketIncarnationId"] = "other" + elif kind == "counter": bad["outcome"]["counters"]["processed"] -= 1 + elif kind == "bool": bad["outcome"]["counters"]["healed"] = True + elif kind == "coverage": bad["outcome"]["coverage"] = "partial" + elif kind == "truncation": bad["outcome"]["objectsTruncated"] = False + with self.assertRaises(ValueError): restart.responsibility(bad, self.wanted(), repaired) + + def journal(self): + current, _ = self.receipt() + owner = {"node": "n0", "disk": 2, "value": {"options": {"timeout": {"secs": 149, "nanos": 0}}}} + terminal = {"schema": 1, "task_id": self.token, "status": "Completed", "heal_type": {"type": "cluster"}, + "options": copy.deepcopy(owner["value"]["options"])} + item = {"bucket": "versions", "object": "history", "versionId": "marker", "objectSize": 0} + report = {"schema": 1, "terminal": terminal, "outcome": current["outcome"], "min_seq": 1027, + "next_seq": 1028, "seqed_items": [[1027, item]]} + observed = {node: {"taskId": self.token, "readOnly": True, "records": [ + {"disk": disk, "kind": kind, "present": node == "n0" and disk == 2 and kind != "intent", + **({"value": terminal if kind == "terminal" else report} if node == "n0" and disk == 2 and kind != "intent" else {})} + for disk in range(4) for kind in ("intent", "terminal", "report")]} for node in ("n0", "n1", "n2", "n3")} + return observed, owner, current + + def test_original_record_and_retained_suffix_are_required_after_all_peer_restart(self): + observed, owner, current = self.journal() + self.assertTrue(restart.terminal_records(observed, self.token, owner, current, self.wanted())["sameRecordDisk"]) + for kind in ("missing", "owner", "intent", "token", "scope", "budget", "outcome", "sequence", "suffix", "marker-size"): + with self.subTest(kind=kind): + bad = copy.deepcopy(observed) + terminal = next(row for row in bad["n0"]["records"] if row["kind"] == "terminal" and row["present"]) + report = next(row for row in bad["n0"]["records"] if row["kind"] == "report" and row["present"]) + if kind == "missing": terminal["present"] = False + elif kind == "owner": report["disk"] = 1 + elif kind == "intent": next(row for row in bad["n0"]["records"] if row["disk"] == 2 and row["kind"] == "intent")["present"] = True + elif kind == "token": terminal["value"]["task_id"] = "new-task" + elif kind == "scope": terminal["value"]["heal_type"] = {"type": "bucket", "bucket": "db"} + elif kind == "budget": terminal["value"]["options"]["timeout"]["secs"] = 300 + elif kind == "outcome": report["value"]["outcome"]["coverage"] = "partial" + elif kind == "sequence": report["value"]["seqed_items"][0][0] = 1026 + elif kind == "suffix": report["value"]["next_seq"] += 1 + elif kind == "marker-size": report["value"]["seqed_items"][0][1]["objectSize"] = 17 + with self.assertRaises(ValueError): restart.terminal_records(bad, self.token, owner, current, self.wanted()) + + def processes(self): + return {node: {**{name: name + "-" + node for name in PROCESS_FIELDS}, "pid": 100 + index, "startTicks": 1000 + index} + for index, node in enumerate(("n0", "n1", "n2", "n3"))} + + def test_each_original_peer_must_really_restart_without_changing_environment(self): + before = self.processes() + after = copy.deepcopy(before) + for item in after.values(): item["pid"] += 100; item["startTicks"] += 100 + self.assertTrue(restart.continuity(before, after)["allFourOriginalProcessesRestarted"]) + for field in ("pid", "environmentFileSha256", "binarySha256", "bootId", "mountNamespace", "unit"): + with self.subTest(field=field): + bad = copy.deepcopy(after) + if field == "pid": bad["n2"] = before["n2"] + else: bad["n2"][field] = "changed" + with self.assertRaises(ValueError): restart.continuity(before, bad) + + def test_all_peer_stop_is_bound_to_original_birth_identity_and_real_zero_exit(self): + processes, calls = self.processes(), [] + def remote(node, action, **params): + calls.append((node, action, params)) + if action == "process-graceful": return {"actualExitCode": 0, "signal": "SIGTERM", "pidfdBound": True} + return {} + fleet = types.SimpleNamespace(nodes=list(processes), frozen_processes={"n2": processes["n2"]}, + remote=remote, record=lambda label, value: value) + exits = restart.graceful_all(fleet, processes, "n2") + self.assertEqual(list(exits), ["n2", "n0", "n1", "n3"]) + self.assertFalse(fleet.frozen_processes) + for node, action, params in calls: + if action != "stop": self.assertEqual(params["expected"], processes[node]) + fleet.remote = lambda *args, **kwargs: {"actualExitCode": 9, "signal": "SIGTERM", "pidfdBound": True} + with self.assertRaisesRegex(ValueError, "normal graceful exit"): restart.graceful_all(fleet, processes, "n0") + self.assertIn("n0", fleet.frozen_processes) + + def test_every_source_branch_has_an_automatic_h2_entry(self): + recipe = select("F05")[0] + self.assertEqual(recipe["variants"], list(restart.BRANCHES)) + self.assertEqual(recipe["contract"]["probes"]["topology"], "H2") + self.assertIn(restart.CASE_ID, AUTOMATIC_CASES) + + def test_multi_bucket_physical_scope_preserves_the_original_run_boundary(self): + owner = {"bucket": "rv-heal-0123456789abcdef"} + prefix = owner["bucket"] + self.assertEqual(snapshot_buckets(owner, {}), [prefix]) + self.assertEqual(snapshot_buckets(owner, {"buckets": [prefix, prefix + "-dx", prefix + "-versions"]}), + [prefix, prefix + "-dx", prefix + "-versions"]) + for buckets in ([prefix, "other"], [prefix, prefix + "/../outside"], [prefix, prefix], [prefix + "-dx"], [prefix, 7], []): + with self.subTest(buckets=buckets), self.assertRaises(ValueError): snapshot_buckets(owner, {"buckets": buckets}) + + def test_secondary_bucket_streaming_proof_cannot_bind_to_primary_bucket_with_same_key(self): + prefix = "rv-heal-0123456789abcdef" + bucket = prefix + "-dx" + relative = bucket + "/objects/object.bin/generation/part.1" + request = {"disk": 0, "relativePath": relative, "versionId": "null", "partNumber": 1} + result = {"physicalBitrotVerified": True, "bucket": bucket, "key": "objects/object.bin", "versionId": "null", "partNumber": 1} + with tempfile.TemporaryDirectory() as folder: + root = Path(folder).resolve() + part = root / "data" / relative + part.parent.mkdir(parents=True) + part.write_bytes(b"original-protected-shard") + with patch("scripts.validation_heal.fleet_node.command", return_value=types.SimpleNamespace(stdout=json.dumps(result))) as command: + proof = verify_part_owned(request, [root], {"bucket": prefix}, root / "tool") + self.assertEqual(proof["bucket"], bucket) + self.assertEqual(command.call_args.args[3:5], (bucket, "objects/object.bin")) + result["bucket"] = prefix + with patch("scripts.validation_heal.fleet_node.command", return_value=types.SimpleNamespace(stdout=json.dumps(result))), self.assertRaisesRegex(ValueError, "identity differs"): + verify_part_owned(request, [root], {"bucket": prefix}, root / "tool") + + +if __name__ == "__main__": unittest.main() From 2da814da90db2d9cf64fa2ac211dae9dc048168d Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 04:31:58 +0800 Subject: [PATCH 072/343] fix(validation): collect full journal message fields --- scripts/validation_heal/fleet_node.py | 4 +++- test_validation_heal.py | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index e8a4aad0..b6ea3f01 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -603,7 +603,7 @@ def logs(request): cursor = request.get("afterCursor") require(cursor is None or isinstance(cursor, str) and re.fullmatch(r"[A-Za-z0-9=;_-]{1,512}", cursor), "invalid journal cursor") - args = ["journalctl", "-u", owner["unit"], "--quiet", "--no-pager", "-o", "json", + args = ["journalctl", "-u", owner["unit"], "--quiet", "--no-pager", "--all", "-o", "json", "--output-fields=__CURSOR,MESSAGE"] # +N selects the FIRST N entries after the cursor. A plain -n N # selects the tail and could silently drop the beginning of a window. @@ -616,6 +616,8 @@ def logs(request): require(isinstance(entry_cursor, str) and re.fullmatch(r"[A-Za-z0-9=;_-]{1,512}", entry_cursor) and entry_cursor not in {cursor, observed_cursor}, "journal page repeated or omitted its exact entry cursor") message = row.get("MESSAGE") + if isinstance(message, list) and all(type(value) is int and 0 <= value <= 255 for value in message): + message = bytes(message).decode("utf-8") require(isinstance(message, str), "journal message is not complete text") entries += 1 require(entries <= 10000, "journal page exceeds its bound; source attribution is incomplete") diff --git a/test_validation_heal.py b/test_validation_heal.py index 5f901518..b7da123d 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -1349,9 +1349,15 @@ def test_remote_log_window_rejects_truncation_and_invalid_cursor(self): self.assertEqual(value["afterCursor"], "s=abc;i=def") self.assertIn("--after-cursor", command.call_args.args) self.assertIn("+10000", command.call_args.args) + self.assertIn("--all", command.call_args.args) self.assertFalse(value["hasMore"]) with self.assertRaisesRegex(ValueError, "invalid journal cursor"): node_logs({"runId": "unit", "afterCursor": "invalid\n--help"}) + command.return_value = types.SimpleNamespace(stdout=json.dumps({"MESSAGE": list('完整消息'.encode()), + "__CURSOR": "s=abc;i=bytes"}) + "\n") + self.assertEqual(node_logs({"runId": "unit", "afterCursor": "s=previous;i=1"})["journal"], '完整消息') + command.return_value = types.SimpleNamespace(stdout=json.dumps({"MESSAGE": None, "__CURSOR": "s=abc;i=omitted"}) + "\n") + with self.assertRaisesRegex(ValueError, "complete text"): node_logs({"runId": "unit", "afterCursor": "s=previous;i=1"}) rows = [json.dumps({"MESSAGE": "row", "__CURSOR": f"s=abc;i={index}"}) for index in range(10000)] command.return_value = types.SimpleNamespace(stdout="\n".join(rows)) page = node_logs({"runId": "unit", "afterCursor": "s=previous;i=1"}) From 849454f2b12e01ef06545a0c4bf018aaae687d77 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 04:37:06 +0800 Subject: [PATCH 073/343] feat(validation): migrate automatic replacement heal --- docs/validation-migration-map.md | 23 +++++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 5 +- scripts/validation_heal/healthy.py | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/replacement.py | 98 +++++++++++++++++++++++--- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 14 ++-- test_validation_heal.py | 50 +++++++++++++ 9 files changed, 178 insertions(+), 20 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index b4296d50..2cf6a10b 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 114 currently dispatchable IDs include all 74 non-Heal cases and forty Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. +case evidence. The 115 currently dispatchable IDs include all 74 non-Heal cases and forty-one Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 44 Heal cases remain +The other 43 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -792,3 +792,22 @@ snapshot. The executor now observes a registered 90-second, read-only format initialization window before requiring distinct new UUID/set-slot evidence; service health still cannot certify replacement acceptance. Neither result increases the complete live PASS count. + +E06 `heal.replacement-auto-rebuild` now dispatches both H1/H2 DB/DV branches +using the same whole-root replacement isolation and complete shard oracles as +B03. Its isolated runtime enables only automatic disk replacement (10-second +interval), with scanner and MRF disabled. The current lab's private directory +roots explicitly use the candidate's directory-backed replacement fallback; +physical-device replacement readiness is not claimed. Every peer must expose +one fresh, stable generation bound to the exact target endpoint and set, with +completed state and a fresh verification timestamp. Missing/partial/unknown +status, a changed generation or divergent terminal records fail. No Admin heal +mutation or target S3 read occurs before full same-role physical reconstruction +and streaming verification. The old archive remains inaccessible to the service +and unchanged. A healthy service or empty queue cannot finish the case. This +executor still requires complete live evidence. + +E04 `38083897141` and C06 `38084060749` retained FAIL/cleanup PASS because the +first paginated collector used journald's default JSON field-size limit, which +can omit a long MESSAGE. The collector now requests `--all`, decodes complete +UTF-8 byte-array messages and rejects missing fields; original failures remain. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index b2d1cd80..eae03e5b 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index b6ea3f01..a57dda87 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -32,7 +32,10 @@ def runtime_config(profile): - require(profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin"}, "unsupported Heal runtime profile") + require(profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement"}, "unsupported Heal runtime profile") + if profile == "replacement": + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_AUTO_HEAL_ENABLE": "true", "RUSTFS_HEAL_INTERVAL_SECS": "10", + "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} if profile == "stale-rejoin": return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS": "0", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} if profile == "root-restart": diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 45302a01..56776fba 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -131,7 +131,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 62d2e3d2..9b08e0f3 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted", "heal.full-cluster-restart"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted", "heal.full-cluster-restart", "heal.replacement-auto-rebuild"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/replacement.py b/scripts/validation_heal/replacement.py index bb46dc50..c4a94418 100644 --- a/scripts/validation_heal/replacement.py +++ b/scripts/validation_heal/replacement.py @@ -2,6 +2,7 @@ import copy import time +import uuid try: from .heal_probe import Probe, canonical, compare_inventories, require @@ -16,7 +17,73 @@ BRANCHES = ("new-disk-data", "new-disk-versioned") +AUTOMATIC_BRANCHES = ("replacement-unversioned", "replacement-versioned-marker") FORMAT_SECONDS = 90 +REBUILD_SECONDS = 300 +STATUS_PATH = "/rustfs/admin/v4/heal/replacement-recovery" + + +def replacement_records(probe, fleet, label): + observed = {} + for endpoint in probe.binding["endpoints"]: + value = fleet.record(label, {"endpoint": endpoint, + **Probe(probe.binding, endpoint).admin(STATUS_PATH, [], b"", method="GET")}) + body = value.get("response", {}) + cluster = body.get("cluster", {}) + require(value.get("httpStatus") == 200 and type(body.get("contractVersion")) is int + and body["contractVersion"] == 2 and body.get("scope") == "cluster_survivor_disks" + and cluster.get("definitive") is True and cluster.get("reason") is None + and isinstance(cluster.get("records"), list), "replacement source status is partial, unknown or incompatible") + observed[endpoint] = cluster["records"] + return observed + + +def generation(records, endpoint, earliest, *, original=None, completed=False): + require(isinstance(records, list) and len(records) == 1 and isinstance(records[0], dict), + "replacement did not expose exactly one owned target generation") + record = records[0] + token = record.get("taskId") + require(isinstance(token, str) and str(uuid.UUID(token)) == token and record.get("generation") == token + and record.get("setDiskId") == "pool_0_set_0" and record.get("targetSlots") == [endpoint], + "replacement generation is not bound to the exact target endpoint/set") + binding = {name: record[name] for name in ("taskId", "generation", "setDiskId", "targetSlots")} + require(original is None or binding == original, "automatic repair crossed replacement generations or target members") + require(record.get("state") in {"waiting_for_replacement", "running", "cleanup_pending", "completed"} + and record.get("reason") is None, "replacement generation is incomplete, unrecoverable or unknown") + if record["state"] == "completed": + require(type(record.get("verifiedAt")) is int and earliest <= record["verifiedAt"] <= int(time.time()) + 5, + "replacement completion is stale or has no actual verification timestamp") + else: require(record.get("verifiedAt") is None, "unfinished replacement claimed a completion timestamp") + require(not completed or record["state"] == "completed", "replacement generation has not completed its responsibility") + return binding + + +def automatic_completion(probe, fleet, endpoint, earliest): + deadline, original, timeline = time.monotonic() + REBUILD_SECONDS, None, [] + while True: + observed = replacement_records(probe, fleet, "automatic-replacement-generation-window") + ready = True + for peer, records in observed.items(): + if not records: + ready = False + continue + bound = generation(records, endpoint, earliest, original=original) + original = original or bound + ready = ready and records[0]["state"] == "completed" + timeline.append(observed) + if ready and original is not None: + require(len({canonical(records[0]) for records in observed.values()}) == 1, + "completed replacement responsibility differs across peers") + break + require(time.monotonic() < deadline, "ATTRIBUTION_NOT_OBSERVED: original automatic replacement generation did not complete on every peer") + time.sleep(2) + for index in range(2): + repeated = replacement_records(probe, fleet, "automatic-replacement-completion-repeat") + for records in repeated.values(): generation(records, endpoint, earliest, original=original, completed=True) + require(canonical(repeated) == canonical(observed), "completed replacement generation changed after its terminal observation") + return {"source": "automatic-replacement", "generation": original, "observations": len(timeline), + "completionObservedOnEveryPeer": True, "budgetSeconds": REBUILD_SECONDS, + "adminHealMutations": 0, "targetS3ReadsBeforePhysicalProof": 0} def wait_format(fleet, node, disk): @@ -100,8 +167,17 @@ def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admi from .versions import map_inline_target, validate_version_receipts, verify_version_s3 except ImportError: from versions import map_inline_target, validate_version_receipts, verify_version_s3 - require(branch in BRANCHES, "unknown formatted replacement branch") + automatic = branch in AUTOMATIC_BRANCHES + require(branch in (*BRANCHES, *AUTOMATIC_BRANCHES), "unknown replacement branch") + require(not automatic or fleet.runtime_profile == "replacement", "automatic replacement lacks its isolated source configuration") before = fleet.snapshot("replacement-original-" + branch, quiescent=True) + if automatic: + fleet.start() + healthy = replacement_records(recorder.probe, fleet, "automatic-replacement-empty-baseline") + require(all(not records for records in healthy.values()), "replacement baseline retained an older generation") + fleet.stop() + require(files(before) == files(fleet.snapshot("replacement-original-after-source-preflight", quiescent=True)), + "replacement source preflight changed an acknowledged original file") node, disk = fleet.nodes[-1], 3 member = node, disk physical_format = next(entry for entry in before[node]["formats"] if entry["disk"] == disk) @@ -115,6 +191,7 @@ def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admi else map_target(before, bucket, entry["key"], "data", 1, entry["versionId"], latest_required=False) shard = next(value for value in target["cohort"] if (value["node"], value["disk"]) == member) proofs.append({**target, **shard}) + earliest = int(time.time()) replaced = fleet.record("replacement-original-root-isolated", fleet.remote(node, "replace-drive", disk=disk, expectedFormatSha256=physical_format["sha256"])) require(replaced.get("oldRootMovedIntact") is True and replaced.get("newDataRootEmpty") is True @@ -126,14 +203,19 @@ def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admi wait_format(fleet, node, disk) initialized = fleet.snapshot("replacement-candidate-initialization") fleet.record("replacement-new-format-proof", format_identity(before, initialized, member)) - require(not any(item["disk"] == disk for item in initialized[node]["files"]), - "ATTRIBUTION_NOT_OBSERVED: new drive received object files before its manual repair token") + if not automatic: + require(not any(item["disk"] == disk for item in initialized[node]["files"]), + "ATTRIBUTION_NOT_OBSERVED: new drive received object files before its manual repair token") require({key: value for key, value in files(initialized).items() if key[:2] != member} == {key: value for key, value in indexed.items() if key[:2] != member}, "replacement initialization changed surviving objects") - status = run_task(recorder.probe, fleet, {"bucket": bucket, "key": ""}, 2, active, - option_overrides={"recursive": True}, admissions=admissions) - receipts = validate_version_receipts(status, bucket, expected, 2, - required_repairs=[(entry["key"], entry["versionId"]) for entry in expected]) + if automatic: + endpoint = replaced["dataPath"] if len(fleet.nodes) == 1 else f"http://rustfs-node{fleet.nodes.index(node) + 1}:9000{replaced['dataPath']}" + receipts = automatic_completion(recorder.probe, fleet, endpoint, earliest) + else: + status = run_task(recorder.probe, fleet, {"bucket": bucket, "key": ""}, 2, active, + option_overrides={"recursive": True}, admissions=admissions) + receipts = validate_version_receipts(status, bucket, expected, 2, + required_repairs=[(entry["key"], entry["versionId"]) for entry in expected]) fleet.stop() after = fleet.snapshot("replacement-after-manual", quiescent=True) physical = fleet.record("replacement-full-physical-proof", physical_rebuild(before, after, member)) @@ -157,7 +239,7 @@ def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admi streaming.append({"key": target["key"], "versionId": target["versionId"], "proof": proof}) fleet.start() fleet.record("replacement-backup-service-refusal-after", fleet.remote(node, "replacement-probe")) - fixture = "DB" if branch == "new-disk-data" else "DV" + fixture = "DB" if branch in {"new-disk-data", "replacement-unversioned"} else "DV" s3 = verify_version_s3(recorder.probe, fleet, bucket, expected, baseline, "replacement-s3-after", fixture=fixture) peers = [] for endpoint in recorder.probe.binding["endpoints"]: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 358ba763..0c2ff4ce 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -28,7 +28,7 @@ case "${case_id}" in heal.root-graceful-coordinator-restart) driver=root_restart.py ;; heal.full-cluster-restart) driver=full_restart.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 067a5fe4..89427e57 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -34,7 +34,7 @@ marker_plan, prepare_stale, verify_markers, reveal_current) from .stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch from .disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch - from .replacement import BRANCHES as REPLACEMENT_BRANCHES, run_branch as replacement_branch + from .replacement import BRANCHES as REPLACEMENT_BRANCHES, AUTOMATIC_BRANCHES as AUTO_REPLACEMENT_BRANCHES, run_branch as replacement_branch from .shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) except ImportError: @@ -54,7 +54,7 @@ marker_plan, prepare_stale, verify_markers, reveal_current) from stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch from disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch - from replacement import BRANCHES as REPLACEMENT_BRANCHES, run_branch as replacement_branch + from replacement import BRANCHES as REPLACEMENT_BRANCHES, AUTOMATIC_BRANCHES as AUTO_REPLACEMENT_BRANCHES, run_branch as replacement_branch from shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) @@ -79,6 +79,7 @@ "heal.stale-version-rejoin": STALE_BRANCHES, "heal.disk-loss-and-repair": DISK_LOSS_BRANCHES, "heal.disk-replacement-formatted": REPLACEMENT_BRANCHES, + "heal.replacement-auto-rebuild": AUTO_REPLACEMENT_BRANCHES, } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") @@ -714,6 +715,7 @@ def run_topology(args, topology, nodes, fixture): fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="scanner" if args.case_id == "heal.scanner-cold-object-repair" else "read-repair" if args.case_id == "heal.read-repair-attribution" else + "replacement" if args.case_id == "heal.replacement-auto-rebuild" else "stale-rejoin" if args.case_id == "heal.stale-version-rejoin" else "manual") candidate = json.loads(args.candidate.read_text()) tool_manifest = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) @@ -743,6 +745,8 @@ def run_topology(args, topology, nodes, fixture): branches = [branch for branch in STALE_BRANCHES if (branch == "unversioned-overwrite") == (fixture == "DB")] if args.case_id == "heal.disk-replacement-formatted": branches = [branch for branch in REPLACEMENT_BRANCHES if (branch == "new-disk-data") == (fixture == "DB")] + if args.case_id == "heal.replacement-auto-rebuild": + branches = [branch for branch in AUTO_REPLACEMENT_BRANCHES if (branch == "replacement-unversioned") == (fixture == "DB")] result = {"topology": topology, "fixture": fixture, "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} active, admissions = [], [] probe = Probe(binding) @@ -827,7 +831,7 @@ def run_topology(args, topology, nodes, fixture): fleet.record("lock-before-fault", prove_locked(probe, fleet, bucket, lock_context, key=LOCK_KEYS[LOCK_BRANCHES.index(branch)], label="lock-before-" + branch)) fleet.stop() - if args.case_id == "heal.disk-replacement-formatted": + if args.case_id in {"heal.disk-replacement-formatted", "heal.replacement-auto-rebuild"}: stale_marker_started = True result["branches"][branch] = replacement_branch(recorder, fleet, bucket, branch, expected, baseline, active, admissions) stale_marker_started = False @@ -1022,7 +1026,7 @@ def run_topology(args, topology, nodes, fixture): result["branches"][branch] = {"verdict": "FAIL", "failure": {"type": type(exc).__name__, "message": str(exc)}} try: fleet.stop() - if args.case_id == "heal.disk-replacement-formatted": + if args.case_id in {"heal.disk-replacement-formatted", "heal.replacement-auto-rebuild"}: for node in fleet.nodes: fleet.record("replacement-failure-format-and-original", {"node": node, **fleet.remote(node, "replacement-state")}) @@ -1078,7 +1082,7 @@ def interrupted(signum, frame): and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") - fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.multiple-shard-loss", "heal.read-repair-attribution", "heal.stale-version-rejoin", "heal.disk-replacement-formatted"} else ("DV",) + fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.multiple-shard-loss", "heal.read-repair-attribution", "heal.stale-version-rejoin", "heal.disk-replacement-formatted", "heal.replacement-auto-rebuild"} else ("DV",) if args.case_id == "heal.object-metadata-and-tags": fixtures = ("DS",) if args.case_id == "heal.special-keys-and-pagination": diff --git a/test_validation_heal.py b/test_validation_heal.py index b7da123d..022d41aa 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -3043,6 +3043,56 @@ def test_replacement_format_wait_never_substitutes_service_ready_for_member_read fleet.remote = lambda *args: {**present, "originalArchiveUnchanged": False} with self.assertRaisesRegex(ValueError, "archive witness"): replacement.wait_format(fleet, "node1", 3) + def test_automatic_replacement_generation_rejects_unknown_wrong_target_and_stale_completion(self): + endpoint = "http://rustfs-node4:9000/private/new/data" + row = {"taskId": "12345678-1234-4234-8234-123456789abc", "generation": "12345678-1234-4234-8234-123456789abc", + "setDiskId": "pool_0_set_0", "targetSlots": [endpoint], "state": "completed", "reason": None, "verifiedAt": 101} + with patch("scripts.validation_heal.replacement.time.time", return_value=102): + binding = replacement.generation([row], endpoint, 100, completed=True) + for kind in ("state", "reason", "stale", "target", "set", "generation", "task", "missing", "duplicate"): + with self.subTest(kind=kind): + bad = copy.deepcopy(row) + rows = [bad] + if kind == "state": bad["state"] = "unknown" + elif kind == "reason": bad["reason"] = "incomplete" + elif kind == "stale": bad["verifiedAt"] = 99 + elif kind == "target": bad["targetSlots"] = [endpoint + "-different"] + elif kind == "set": bad["setDiskId"] = "pool_1_set_0" + elif kind == "generation": bad["generation"] = "other" + elif kind == "task": bad["taskId"] = bad["generation"] = "ffffffff-ffff-4fff-8fff-ffffffffffff" + elif kind == "missing": rows = [] + else: rows *= 2 + with self.assertRaises(ValueError): replacement.generation(rows, endpoint, 100, original=binding, completed=True) + running = {**row, "state": "running", "verifiedAt": None} + replacement.generation([running], endpoint, 100, original=binding) + with self.assertRaisesRegex(ValueError, "not completed"): replacement.generation([running], endpoint, 100, completed=True) + + def test_automatic_replacement_requires_every_peer_and_stable_completed_generation(self): + endpoint = "/private/new/data" + row = {"taskId": "12345678-1234-4234-8234-123456789abc", "generation": "12345678-1234-4234-8234-123456789abc", + "setDiskId": "pool_0_set_0", "targetSlots": [endpoint], "state": "completed", "reason": None, "verifiedAt": 101} + complete = {"node1": [row], "node2": [copy.deepcopy(row)]} + with patch("scripts.validation_heal.replacement.replacement_records", side_effect=[{"node1": [row], "node2": []}, complete, complete, complete]), \ + patch("scripts.validation_heal.replacement.time.sleep"), \ + patch("scripts.validation_heal.replacement.time.monotonic", return_value=0), \ + patch("scripts.validation_heal.replacement.time.time", return_value=102): + proof = replacement.automatic_completion(None, None, endpoint, 100) + self.assertTrue(proof["completionObservedOnEveryPeer"]) + self.assertEqual(proof["adminHealMutations"], 0) + self.assertEqual(proof["observations"], 2) + changed = copy.deepcopy(complete); changed["node2"][0]["verifiedAt"] = 102 + with patch("scripts.validation_heal.replacement.replacement_records", return_value=changed), \ + patch("scripts.validation_heal.replacement.time.time", return_value=102), self.assertRaisesRegex(ValueError, "differs across peers"): + replacement.automatic_completion(None, None, endpoint, 100) + + def test_automatic_replacement_profile_enables_only_its_registered_source(self): + config = runtime_config("replacement") + self.assertEqual(config["RUSTFS_HEAL_AUTO_HEAL_ENABLE"], "true") + self.assertEqual(config["RUSTFS_HEAL_MRF_ENABLE"], "false") + self.assertEqual(config["RUSTFS_SCANNER_ENABLED"], "false") + self.assertEqual(config["RUSTFS_HEAL_INTERVAL_SECS"], "10") + self.assertEqual(list(replacement.AUTOMATIC_BRANCHES), select("E06")[0]["variants"]) + def test_remote_replacement_module_initializes_and_all_source_branches_remain(self): import sys self.assertEqual(list(replacement.BRANCHES), select("B03")[0]["variants"]) From 8d78ff32188412644a55776be7a794024a4f5614 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 04:45:43 +0800 Subject: [PATCH 074/343] feat(validation): migrate partial replacement target restarts --- docs/validation-migration-map.md | 27 ++++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 32 ++++- scripts/validation_heal/healthy.py | 3 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/replacement.py | 157 +++++++++++++++++++++++-- scripts/validation_heal/run_case.sh | 2 +- scripts/validation_heal/versions.py | 35 ++++-- test_validation_heal.py | 90 ++++++++++++++ 9 files changed, 320 insertions(+), 30 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 2cf6a10b..f697052a 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 115 currently dispatchable IDs include all 74 non-Heal cases and forty-one Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. +case evidence. The 117 currently dispatchable IDs include all 74 non-Heal cases and forty-three Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 43 Heal cases remain +The other 41 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -811,3 +811,26 @@ E04 `38083897141` and C06 `38084060749` retained FAIL/cleanup PASS because the first paginated collector used journald's default JSON field-size limit, which can omit a long MESSAGE. The collector now requests `--all`, decodes complete UTF-8 byte-array messages and rejects missing fields; original failures remain. + +F02/F03 now execute both H2 source branches using automatic replacement. The +unversioned observation fixture has 1,025 external objects; the versioned +fixture has 64 keys, 192 data versions and their delete markers, with a 45-minute +private-service lease. Only node1 enables automatic disk discovery; the other +three nodes retain enabled Heal with discovery disabled. Before interrupting +node4, fresh cursor-bounded logs must bind the original generation to accepted +`auto_heal` admission and exact erasure-set task start on one distinct +coordinator. Physical census must show both committed original same-role +external shards and pending data versions. The target is paused through its +exact MainPID/birth identity and the partial window is checked again. F02 then +requires witnessed SIGTERM/normal exit zero; F03 requires pidfd-bound SIGKILL, +signal 9 and a non-graceful systemd exit, with no platform power event. No other +peer process may change. Restart uses the same unit, environment, roots and new +disk UUID, reproves old-archive refusal and finishes the same automatic +generation. Complete physical/streaming and all-peer S3 proof remain mandatory. +A fast completed task, missing distinct coordinator, changed generation or +missed physical window cannot PASS. External target verification uses bounded +128-part batches to keep the DX run within its registered resource budget. + +B03 bounded run `38084312163` retained FAIL/cleanup PASS in all four instances: +the candidate still did not initialize the replacement format within 90 seconds. +That explicit acceptance failure is not relabeled as a completed reconstruction. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index eae03e5b..5b1dc68d 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index a57dda87..6c771ff0 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -32,9 +32,9 @@ def runtime_config(profile): - require(profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement"}, "unsupported Heal runtime profile") - if profile == "replacement": - return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_AUTO_HEAL_ENABLE": "true", "RUSTFS_HEAL_INTERVAL_SECS": "10", + require(profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target"}, "unsupported Heal runtime profile") + if profile in {"replacement", "replacement-target"}: + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_AUTO_HEAL_ENABLE": "true" if profile == "replacement" else "false", "RUSTFS_HEAL_INTERVAL_SECS": "10", "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} if profile == "stale-rejoin": return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS": "0", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} @@ -217,7 +217,7 @@ def start(request): "--property=KillMode=control-group", "--property=TimeoutStopSec=30", "--property=RuntimeMaxSec=" + str(resource_limits(owner.get("resourceProfile", "standard"))["runtimeMaxSeconds"]), "--property=MemoryMax=4G", - *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") == "root-restart" else []), + *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") in {"root-restart", "replacement", "replacement-target"} else []), "--property=WorkingDirectory=" + str(state), "--property=EnvironmentFile=" + str(state / "service.env"), *gate_module().start_properties(gate_module().checked_gate(state, roots)), @@ -303,7 +303,7 @@ def process_signal(request): def process_graceful(request): - require(owned(request["runId"])[2].get("runtimeProfile") == "root-restart" + require(owned(request["runId"])[2].get("runtimeProfile") in {"root-restart", "replacement", "replacement-target"} and process(request).get("state") == "T", "graceful observation requires the paused run-owned root service") before = pidfd_signal(request, signal.SIGTERM) # A paused process must receive CONT to handle TERM. Use the same pidfd @@ -326,6 +326,26 @@ def process_graceful(request): "pidfdBound": True, "platformEventIssued": False} +def process_crash(request): + require(owned(request["runId"])[2].get("runtimeProfile") in {"replacement", "replacement-target"} + and process(request).get("state") == "T", "crash observation requires the paused owned replacement service") + before = pidfd_signal(request, signal.SIGKILL) + deadline = time.monotonic() + 30 + while True: + after = process(request) + if after["pid"] == 0: break + require(all(after.get(name) == before[name] for name in PROCESS_FIELDS), "another process replaced the crash target") + require(time.monotonic() < deadline, "owned replacement process did not exit after SIGKILL") + time.sleep(0.1) + service = after["service"] + require(after["bootId"] == before["bootId"] and after["binarySha256"] == before["binarySha256"] + and after["environmentFileSha256"] == before["environmentFileSha256"] + and service.get("ExecMainCode") == "2" and service.get("ExecMainStatus") == "9" + and service.get("Result") == "signal", "process has no exact non-graceful SIGKILL exit witness") + return {"before": before, "after": after, "signal": "SIGKILL", "actualSignal": 9, + "pidfdBound": True, "graceful": False, "platformEventIssued": False} + + def fault_module(): # SSH requests carry this exact repository module in memory; local imports # support contract tests without creating an importable file on a node. @@ -723,6 +743,6 @@ def main(request): "drive-gate-release-live": drive_gate_release_live} actions["task-records"] = task_records actions.update({"replace-drive": replace_drive, "replacement-state": replacement_state, "replacement-probe": replacement_probe}) - actions.update({"process": process, "process-signal": process_signal, "process-graceful": process_graceful}) + actions.update({"process": process, "process-signal": process_signal, "process-graceful": process_graceful, "process-crash": process_crash}) require(request.get("action") in actions, "unsupported node operation") print(json.dumps(actions[request["action"]](request))) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 56776fba..d6e0fdf5 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -136,6 +136,7 @@ def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_prof self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory self.runtime_profile = runtime_profile + self.node_runtime_profiles = {} self.resource_profile = resource_profile self.sequence = 0 self.reserved = [] @@ -230,7 +231,7 @@ def deploy(self, binary_sha, bucket, access, secret, *, before_start=None): if self.tool_directory else None value = self.remote(node, "reserve", hostname=checked[node]["hostname"], mounts=checked[node]["mounts"], binarySha256=binary_sha, bucket=bucket, volumes=volumes, access=access, secret=secret, - inspectorSha256=inspector_sha, runtimeProfile=self.runtime_profile, + inspectorSha256=inspector_sha, runtimeProfile=self.node_runtime_profiles.get(node, self.runtime_profile), resourceProfile=self.resource_profile) self.record("reserved", {"node": node, **value}) if self.tool_directory: diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 9b08e0f3..51c2a3f2 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted", "heal.full-cluster-restart", "heal.replacement-auto-rebuild"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted", "heal.full-cluster-restart", "heal.replacement-auto-rebuild", "heal.target-graceful-restart", "heal.target-process-crash"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/replacement.py b/scripts/validation_heal/replacement.py index c4a94418..fa6ece07 100644 --- a/scripts/validation_heal/replacement.py +++ b/scripts/validation_heal/replacement.py @@ -9,15 +9,21 @@ from .heal_workload import verify_seed from .healthy import validate_topology from .shards import files, map_target, run_task + from .fleet_node import PROCESS_FIELDS + from .scanner_repair import JournalWindow, SOURCE_TARGETS, event_time except ImportError: from heal_probe import Probe, canonical, compare_inventories, require from heal_workload import verify_seed from healthy import validate_topology from shards import files, map_target, run_task + from fleet_node import PROCESS_FIELDS + from scanner_repair import JournalWindow, SOURCE_TARGETS, event_time BRANCHES = ("new-disk-data", "new-disk-versioned") AUTOMATIC_BRANCHES = ("replacement-unversioned", "replacement-versioned-marker") +GRACEFUL_BRANCHES = ("target-partial-graceful", "versioned-target-partial") +CRASH_BRANCHES = ("target-partial-sigkill", "target-partial-sigkill-versioned") FORMAT_SECONDS = 90 REBUILD_SECONDS = 300 STATUS_PATH = "/rustfs/admin/v4/heal/replacement-recovery" @@ -58,8 +64,115 @@ def generation(records, endpoint, earliest, *, original=None, completed=False): return binding -def automatic_completion(probe, fleet, endpoint, earliest): +def target_progress(snapshot, proofs): + indexed, complete, external_complete, external_pending = files(snapshot), [], 0, 0 + for target in proofs: + identity = target["node"], target["disk"], f"{target['bucket']}/{target['key']}/xl.meta" + metadata = indexed.get(identity, {}).get("metadata") + matched = None if not metadata else next((version for version in metadata["versions"] + if (version.get("versionId") or "null") == target["versionId"]), None) + valid = False + if matched is not None: + logical = {name: matched.get(name) for name in target["logical"] if name != "erasure"} + logical["erasure"] = {name: matched.get("erasure", {}).get(name) for name in target["logical"]["erasure"]} + require(logical == target["logical"] and matched.get("role") == target["role"] + and matched.get("erasure", {}).get("index") == target["index"], + "partial replacement changed the acknowledged version authority or original shard role") + if target.get("zeroProof"): + require(matched.get("size") == 0 and not matched.get("inline"), "replacement invented data for a zero-length ACK") + valid = True + elif target.get("inlineProof"): + inline = matched.get("inline") or {} + require(inline.get("length") == target["inlineValue"]["length"] + and inline.get("sha256") == target["inlineValue"]["sha256"], "partial replacement committed incorrect inline bytes") + valid = True + else: + entry = indexed.get((target["node"], target["disk"], target["relativePath"])) + if entry: + require(entry["sha256"] == target["sha256"] and entry["size"] == target["size"], + "partial replacement committed incorrect original same-role shard bytes") + valid = True + if valid: complete.append((target["key"], target["versionId"])) + if not target.get("zeroProof") and not target.get("inlineProof"): + if valid: external_complete += 1 + else: external_pending += 1 + return {"completeDataVersions": complete, "dataVersions": len(proofs), + "externalComplete": external_complete, "externalPending": external_pending, + "partial": 0 < len(complete) < len(proofs) and external_complete > 0 and external_pending > 0} + + +def target_restart_identity(before, after, target): + require(set(before) == set(after) and target in before and len(before) == 4, "target restart omitted an actual H2 peer") + for node, old in before.items(): + new = after[node] + ignored = {"pid", "startTicks", "mountNamespace"} if node == target else set() + require(all(new.get(name) == old.get(name) for name in PROCESS_FIELDS if name not in ignored), + "target restart changed candidate, boot, unit, settings or a non-target peer") + if node == target: + require(type(new.get("pid")) is int and new["pid"] > 1 and type(new.get("startTicks")) is int + and isinstance(new.get("mountNamespace"), str) and new["mountNamespace"] + and (new["pid"], new["startTicks"]) != (old["pid"], old["startTicks"]), "replacement target was not actually restarted") + return {"onlyOriginalTargetProcessRestarted": True, "sameBootCandidateUnitSettingsAndRoots": True, + "newPrivateMountNamespaceMustProveArchiveRefusal": True} + + +def interrupt_target(fleet, node, proofs, mode, original_format): + before = fleet.record("replacement-processes-before-interruption", {peer: fleet.remote(peer, "process") for peer in fleet.nodes}) + identity = before[node] + fleet.frozen_processes[node] = identity + fleet.record("replacement-target-freeze", fleet.remote(node, "process-signal", signal="SIGSTOP", expected=identity)) + frozen = fleet.snapshot("replacement-target-frozen-physical") + progress = target_progress(frozen, proofs) + require(progress["partial"], "NOT_OBSERVED: replacement was not physically partial at its frozen interruption boundary") + require(all(frozen[peer]["formats"] == original_format[peer]["formats"] for peer in fleet.nodes), + "partial replacement changed its new target UUID or another disk identity") + exit_proof = fleet.record("replacement-target-actual-exit", fleet.remote(node, + "process-graceful" if mode == "graceful" else "process-crash", expected=identity)) + if mode == "graceful": + require(exit_proof.get("actualExitCode") == 0 and exit_proof.get("signal") == "SIGTERM", "target has no real graceful exit") + else: + require(exit_proof.get("actualSignal") == 9 and exit_proof.get("graceful") is False + and exit_proof.get("platformEventIssued") is False, "target crash lacks an exact SIGKILL witness") + del fleet.frozen_processes[node] + fleet.record("replacement-target-release-unit", fleet.remote(node, "stop")) + stopped = fleet.record("replacement-target-after-exit-physical", {peer: fleet.remote(peer, "snapshot", quiescent=peer == node) for peer in fleet.nodes}) + after_progress = target_progress(stopped, proofs) + require(after_progress["partial"] and set(progress["completeDataVersions"]) <= set(after_progress["completeDataVersions"]), + "target exit lost a committed shard or missed the partial interruption window") + require(all(stopped[peer]["formats"] == original_format[peer]["formats"] for peer in fleet.nodes), + "target exit changed the original replacement generation's physical identity") + fleet.record("replacement-original-target-start", fleet.remote(node, "start")) + fleet.wait_ready() + after = fleet.record("replacement-processes-after-interruption", {peer: fleet.remote(peer, "process") for peer in fleet.nodes}) + continuity = target_restart_identity(before, after, node) + archive = fleet.record("replacement-backup-refusal-after-interruption", fleet.remote(node, "replacement-probe")) + require(archive.get("serviceReadDenied") is True and archive.get("sameOriginalServiceProcess") is True, + "restarted target can access its original backup") + return {"interruption": mode, "before": progress, "afterActualExit": after_progress, + "actualExit": exit_proof, "processContinuity": continuity, "sameNewDiskUUIDAndSetSlot": True} + + +def coordinator(events, token, target): + owners = [] + for node, rows in events.items(): + admitted = [event for event in rows if event.get("target") == "rustfs::heal::manager" + and event.get("request_id") == token and event.get("source") == "auto_heal" + and event.get("result") == "accepted" and event.get("message") == "Heal queue request accepted" + and event_time(event) is not None] + started = [event for event in rows if event.get("target") == "rustfs::heal::task" + and event.get("task_id") == token and event.get("set_disk_id") == "pool_0_set_0" + and event.get("stage") == "start" and event.get("message") == "Heal erasure set started" + and event_time(event) is not None] + if any(event_time(start) >= event_time(admission) for start in started for admission in admitted): + owners.append(node) + require(len(owners) == 1 and owners[0] != target, + "NOT_OBSERVED: replacement has no uniquely witnessed coordinator distinct from the interrupted target") + return owners[0] + + +def automatic_completion(probe, fleet, endpoint, earliest, *, interruption=None, proofs=(), original_format=None, journal=None): deadline, original, timeline = time.monotonic() + REBUILD_SECONDS, None, [] + interrupted = None while True: observed = replacement_records(probe, fleet, "automatic-replacement-generation-window") ready = True @@ -71,6 +184,17 @@ def automatic_completion(probe, fleet, endpoint, earliest): original = original or bound ready = ready and records[0]["state"] == "completed" timeline.append(observed) + if interruption is not None and interrupted is None: + require(not ready, "NOT_OBSERVED: replacement completed before its required target interruption") + if any(records and records[0]["state"] == "running" for records in observed.values()): + progress = target_progress(fleet.snapshot("replacement-running-physical-window"), proofs) + if progress["partial"]: + owner = coordinator(journal.capture(), original["taskId"], proofs[0]["node"]) + fleet.record("replacement-distinct-coordinator-proof", {"node": owner, + "target": proofs[0]["node"], "originalGeneration": original, "source": "fresh accepted auto_heal and exact erasure task start"}) + interrupted = interrupt_target(fleet, proofs[0]["node"], proofs, interruption, original_format) + interrupted["coordinatorKeptRunning"] = owner + continue if ready and original is not None: require(len({canonical(records[0]) for records in observed.values()}) == 1, "completed replacement responsibility differs across peers") @@ -81,9 +205,10 @@ def automatic_completion(probe, fleet, endpoint, earliest): repeated = replacement_records(probe, fleet, "automatic-replacement-completion-repeat") for records in repeated.values(): generation(records, endpoint, earliest, original=original, completed=True) require(canonical(repeated) == canonical(observed), "completed replacement generation changed after its terminal observation") + require(interruption is None or interrupted is not None, "NOT_OBSERVED: no actual partial target interruption occurred") return {"source": "automatic-replacement", "generation": original, "observations": len(timeline), "completionObservedOnEveryPeer": True, "budgetSeconds": REBUILD_SECONDS, - "adminHealMutations": 0, "targetS3ReadsBeforePhysicalProof": 0} + "adminHealMutations": 0, "targetS3ReadsBeforePhysicalProof": 0, "targetInterruption": interrupted} def wait_format(fleet, node, disk): @@ -167,8 +292,8 @@ def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admi from .versions import map_inline_target, validate_version_receipts, verify_version_s3 except ImportError: from versions import map_inline_target, validate_version_receipts, verify_version_s3 - automatic = branch in AUTOMATIC_BRANCHES - require(branch in (*BRANCHES, *AUTOMATIC_BRANCHES), "unknown replacement branch") + automatic = branch in (*AUTOMATIC_BRANCHES, *GRACEFUL_BRANCHES, *CRASH_BRANCHES) + require(branch in (*BRANCHES, *AUTOMATIC_BRANCHES, *GRACEFUL_BRANCHES, *CRASH_BRANCHES), "unknown replacement branch") require(not automatic or fleet.runtime_profile == "replacement", "automatic replacement lacks its isolated source configuration") before = fleet.snapshot("replacement-original-" + branch, quiescent=True) if automatic: @@ -192,6 +317,8 @@ def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admi shard = next(value for value in target["cohort"] if (value["node"], value["disk"]) == member) proofs.append({**target, **shard}) earliest = int(time.time()) + journal = JournalWindow(fleet, "replacement-interruption-source", event_targets=SOURCE_TARGETS) \ + if branch in (*GRACEFUL_BRANCHES, *CRASH_BRANCHES) else None replaced = fleet.record("replacement-original-root-isolated", fleet.remote(node, "replace-drive", disk=disk, expectedFormatSha256=physical_format["sha256"])) require(replaced.get("oldRootMovedIntact") is True and replaced.get("newDataRootEmpty") is True @@ -210,7 +337,9 @@ def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admi {key: value for key, value in indexed.items() if key[:2] != member}, "replacement initialization changed surviving objects") if automatic: endpoint = replaced["dataPath"] if len(fleet.nodes) == 1 else f"http://rustfs-node{fleet.nodes.index(node) + 1}:9000{replaced['dataPath']}" - receipts = automatic_completion(recorder.probe, fleet, endpoint, earliest) + receipts = automatic_completion(recorder.probe, fleet, endpoint, earliest, + interruption="graceful" if branch in GRACEFUL_BRANCHES else "sigkill" if branch in CRASH_BRANCHES else None, + proofs=proofs, original_format=initialized, journal=journal) else: status = run_task(recorder.probe, fleet, {"bucket": bucket, "key": ""}, 2, active, option_overrides={"recursive": True}, admissions=admissions) @@ -220,12 +349,15 @@ def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admi after = fleet.snapshot("replacement-after-manual", quiescent=True) physical = fleet.record("replacement-full-physical-proof", physical_rebuild(before, after, member)) fleet.record("replacement-original-archive-after-rebuild", fleet.remote(node, "replacement-state")) - streaming = [] + streaming, external = [], [] for target in proofs: if target.get("zeroProof"): streaming.append({"key": target["key"], "zeroLengthMetadataVerified": True, "noPartInvented": True}) continue operation = "verify-inline" if target.get("inlineProof") else "verify-part" + if operation == "verify-part": + external.append(target) + continue params = {name: target[name] for name in ("disk", "relativePath", "versionId")} if operation == "verify-part": params["partNumber"] = target["partNumber"] proof = fleet.record("replacement-target-streaming-check", fleet.remote(node, operation, **params)) @@ -237,9 +369,20 @@ def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admi require(proof.get("sha256") == target["sha256"] and proof.get("physicalSize") == target["size"], "new target external shard differs from the original same-role oracle") streaming.append({"key": target["key"], "versionId": target["versionId"], "proof": proof}) + for offset in range(0, len(external), 128): + batch = external[offset:offset + 128] + observed = fleet.record("replacement-target-streaming-batch", fleet.remote(node, "verify-parts", parts=[ + {name: target[name] for name in ("disk", "relativePath", "versionId", "partNumber")} for target in batch])) + require(observed.get("complete") is True and isinstance(observed.get("parts"), list) + and len(observed["parts"]) == len(batch), "replacement streaming batch omitted an exact target part") + for target, proof in zip(batch, observed["parts"], strict=True): + require(proof.get("physicalBitrotVerified") is True + and all(proof.get(name) == target[name] for name in ("disk", "relativePath", "versionId", "partNumber", "sha256")) + and proof.get("physicalSize") == target["size"], "replacement target streaming batch changed an acknowledged same-role part") + streaming.append({"key": target["key"], "versionId": target["versionId"], "proof": proof}) fleet.start() fleet.record("replacement-backup-service-refusal-after", fleet.remote(node, "replacement-probe")) - fixture = "DB" if branch in {"new-disk-data", "replacement-unversioned"} else "DV" + fixture = "DB" if branch in {"new-disk-data", "replacement-unversioned", "target-partial-graceful", "target-partial-sigkill"} else "DV" s3 = verify_version_s3(recorder.probe, fleet, bucket, expected, baseline, "replacement-s3-after", fixture=fixture) peers = [] for endpoint in recorder.probe.binding["endpoints"]: diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 0c2ff4ce..b95a4013 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -28,7 +28,7 @@ case "${case_id}" in heal.root-graceful-coordinator-restart) driver=root_restart.py ;; heal.full-cluster-restart) driver=full_restart.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 89427e57..a48eaeb2 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -34,7 +34,7 @@ marker_plan, prepare_stale, verify_markers, reveal_current) from .stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch from .disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch - from .replacement import BRANCHES as REPLACEMENT_BRANCHES, AUTOMATIC_BRANCHES as AUTO_REPLACEMENT_BRANCHES, run_branch as replacement_branch + from .replacement import BRANCHES as REPLACEMENT_BRANCHES, AUTOMATIC_BRANCHES as AUTO_REPLACEMENT_BRANCHES, GRACEFUL_BRANCHES, CRASH_BRANCHES, run_branch as replacement_branch from .shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) except ImportError: @@ -54,7 +54,7 @@ marker_plan, prepare_stale, verify_markers, reveal_current) from stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch from disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch - from replacement import BRANCHES as REPLACEMENT_BRANCHES, AUTOMATIC_BRANCHES as AUTO_REPLACEMENT_BRANCHES, run_branch as replacement_branch + from replacement import BRANCHES as REPLACEMENT_BRANCHES, AUTOMATIC_BRANCHES as AUTO_REPLACEMENT_BRANCHES, GRACEFUL_BRANCHES, CRASH_BRANCHES, run_branch as replacement_branch from shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) @@ -80,6 +80,8 @@ "heal.disk-loss-and-repair": DISK_LOSS_BRANCHES, "heal.disk-replacement-formatted": REPLACEMENT_BRANCHES, "heal.replacement-auto-rebuild": AUTO_REPLACEMENT_BRANCHES, + "heal.target-graceful-restart": GRACEFUL_BRANCHES, + "heal.target-process-crash": CRASH_BRANCHES, } VERSION_SIZE = 5 * 1024**2 + 123 SEMANTIC_KEYS = ("中文/文件/metadata.bin", "space key/headers.bin", "literal%2Fslash/versioned.bin") @@ -712,11 +714,14 @@ def run_topology(args, topology, nodes, fixture): directory = args.evidence / fixture / topology directory.mkdir(parents=True) run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" - fleet = Fleet(nodes, directory, run_id, args.tool_directory, + target_restart = args.case_id in {"heal.target-graceful-restart", "heal.target-process-crash"} + fleet = Fleet(nodes, directory, run_id, args.tool_directory, resource_profile="cursor" if target_restart else "standard", runtime_profile="scanner" if args.case_id == "heal.scanner-cold-object-repair" else "read-repair" if args.case_id == "heal.read-repair-attribution" else - "replacement" if args.case_id == "heal.replacement-auto-rebuild" else + "replacement" if args.case_id == "heal.replacement-auto-rebuild" or target_restart else "stale-rejoin" if args.case_id == "heal.stale-version-rejoin" else "manual") + if target_restart: + fleet.node_runtime_profiles = {node: "replacement-target" for node in nodes[1:]} candidate = json.loads(args.candidate.read_text()) tool_manifest = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) contract = verify_quorum_contract(tool_manifest, candidate) if args.case_id in { @@ -727,12 +732,12 @@ def run_topology(args, topology, nodes, fixture): access, secret = secrets.token_hex(16), secrets.token_hex(24) os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, - "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45 if size_rule else 30)).isoformat(), "bucketPrefix": bucket, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45 if size_rule or target_restart else 30)).isoformat(), "bucketPrefix": bucket, "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), - "maxInventoryObjects": 512, "maxInventoryBytes": 512 * 1024**2} + "maxInventoryObjects": 2048 if target_restart else 512, "maxInventoryBytes": 2 * 1024**3 if target_restart else 512 * 1024**2} fleet.record("binding", binding) recipe = select(args.case_id)[0] require(list(BRANCHES[args.case_id]) == recipe["variants"], "executor omitted a source version branch") @@ -747,6 +752,8 @@ def run_topology(args, topology, nodes, fixture): branches = [branch for branch in REPLACEMENT_BRANCHES if (branch == "new-disk-data") == (fixture == "DB")] if args.case_id == "heal.replacement-auto-rebuild": branches = [branch for branch in AUTO_REPLACEMENT_BRANCHES if (branch == "replacement-unversioned") == (fixture == "DB")] + if target_restart: + branches = [branch for index, branch in enumerate(BRANCHES[args.case_id]) if (index == 0) == (fixture == "DB")] result = {"topology": topology, "fixture": fixture, "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} active, admissions = [], [] probe = Probe(binding) @@ -777,6 +784,9 @@ def run_topology(args, topology, nodes, fixture): elif args.case_id == "heal.delete-marker-semantics": expected = seed_markers(recorder, bucket, VERSION_SIZE) instances = list(MARKER_INSTANCES) + elif target_restart: + expected = seed(recorder, bucket, fixture, 1025 if fixture == "DB" else 64, [1024**2 + 73]) + fleet.record("replacement-interruption-budget", {"dataObjects": 1025 if fixture == "DB" else 192, "objectBytes": 1024**2 + 73, "requiresActualPartialPhysicalProgress": True, "runtimeMinutes": 45}) elif fixture == "DK": expected = seed_special_keys(recorder, bucket) elif fixture == "DS": @@ -831,7 +841,7 @@ def run_topology(args, topology, nodes, fixture): fleet.record("lock-before-fault", prove_locked(probe, fleet, bucket, lock_context, key=LOCK_KEYS[LOCK_BRANCHES.index(branch)], label="lock-before-" + branch)) fleet.stop() - if args.case_id in {"heal.disk-replacement-formatted", "heal.replacement-auto-rebuild"}: + if args.case_id in {"heal.disk-replacement-formatted", "heal.replacement-auto-rebuild", "heal.target-graceful-restart", "heal.target-process-crash"}: stale_marker_started = True result["branches"][branch] = replacement_branch(recorder, fleet, bucket, branch, expected, baseline, active, admissions) stale_marker_started = False @@ -1026,7 +1036,7 @@ def run_topology(args, topology, nodes, fixture): result["branches"][branch] = {"verdict": "FAIL", "failure": {"type": type(exc).__name__, "message": str(exc)}} try: fleet.stop() - if args.case_id in {"heal.disk-replacement-formatted", "heal.replacement-auto-rebuild"}: + if args.case_id in {"heal.disk-replacement-formatted", "heal.replacement-auto-rebuild", "heal.target-graceful-restart", "heal.target-process-crash"}: for node in fleet.nodes: fleet.record("replacement-failure-format-and-original", {"node": node, **fleet.remote(node, "replacement-state")}) @@ -1082,7 +1092,7 @@ def interrupted(signum, frame): and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") - fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.multiple-shard-loss", "heal.read-repair-attribution", "heal.stale-version-rejoin", "heal.disk-replacement-formatted", "heal.replacement-auto-rebuild"} else ("DV",) + fixtures = ("DB", "DV") if args.case_id in {"heal.repeat-idempotency", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.multiple-shard-loss", "heal.read-repair-attribution", "heal.stale-version-rejoin", "heal.disk-replacement-formatted", "heal.replacement-auto-rebuild", "heal.target-graceful-restart", "heal.target-process-crash"} else ("DV",) if args.case_id == "heal.object-metadata-and-tags": fixtures = ("DS",) if args.case_id == "heal.special-keys-and-pagination": @@ -1093,9 +1103,12 @@ def interrupted(signum, frame): fixtures = ("DO",) if args.case_id in {"heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.disk-loss-and-repair"}: fixtures = ("DB",) + allowed_topologies = set(select(args.case_id)[0]["contract"]["probes"]["topology"].split(",")) + require(allowed_topologies <= {"H1", "H2"}, "version executor lacks a required topology") + topologies = [(name, selected) for name, selected in (("H1", nodes[:1]), ("H2", nodes[:4])) if name in allowed_topologies] results = [] try: - for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): + for topology, selected in topologies: for fixture in fixtures: results.append(run_topology(args, topology, selected, fixture)) if results[-1]["cleanup"] != "PASS": @@ -1103,7 +1116,7 @@ def interrupted(signum, frame): if results[-1]["cleanup"] != "PASS": break finally: - complete = len(results) == 2 * len(fixtures) and all( + complete = len(results) == len(topologies) * len(fixtures) and all( item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) save(args.evidence / "healthy-summary.json", {"caseId": args.case_id, "results": results, "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", "complete": complete}) diff --git a/test_validation_heal.py b/test_validation_heal.py index 022d41aa..a8737794 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -2896,6 +2896,96 @@ def get(operation, params): class HealWholeDriveReplacementTest(unittest.TestCase): + def test_partial_target_window_requires_actual_committed_shards_and_pending_versions(self): + scene, bucket, expected = versioned_scene(["first", "second"]) + proofs = [map_target(scene, bucket, key, "data", 1, "v0", latest_required=False) for key in ("first", "second")] + partial = copy.deepcopy(scene) + missing = proofs[1] + partial[missing["node"]]["files"] = [item for item in partial[missing["node"]]["files"] + if not (item["disk"] == missing["disk"] and item["path"] == missing["relativePath"])] + progress = replacement.target_progress(partial, proofs) + self.assertTrue(progress["partial"]) + self.assertEqual(progress["completeDataVersions"], [("first", "v0")]) + self.assertFalse(replacement.target_progress(scene, proofs)["partial"]) + for kind in ("bytes", "version-authority", "role"): + changed = copy.deepcopy(partial) + target = proofs[0] + if kind == "bytes": + next(item for item in changed[target["node"]]["files"] if item["disk"] == target["disk"] + and item["path"] == target["relativePath"])["sha256"] = "wrong" + else: + version = next(item for item in changed[target["node"]]["files"] if item["disk"] == target["disk"] + and item["path"] == f"{bucket}/first/xl.meta")["metadata"]["versions"][0] + if kind == "version-authority": version["dataDir"] = "foreign-generation" + else: version["role"] = "wrong" + with self.subTest(kind=kind), self.assertRaises(ValueError): replacement.target_progress(changed, proofs) + + def test_target_restart_keeps_other_peers_and_rechecks_private_namespace(self): + before = {f"node{index}": HealRootRestartOracleTest().process(10 + index, 50 + index) for index in range(1, 5)} + after = copy.deepcopy(before) + after["node4"].update(pid=29, startTicks=99, mountNamespace="mnt:[new-private]") + self.assertTrue(replacement.target_restart_identity(before, after, "node4")["onlyOriginalTargetProcessRestarted"]) + for node, field in (("node1", "pid"), ("node4", "binarySha256"), ("node4", "bootId"), ("node4", "settings")): + changed = copy.deepcopy(after); changed[node][field] = "changed" + with self.subTest(node=node, field=field), self.assertRaises(ValueError): replacement.target_restart_identity(before, changed, "node4") + + def test_target_crash_requires_exact_signal_nine_and_original_birth_identity(self): + before = HealRootRestartOracleTest().process() + exited = {"pid": 0, "bootId": "boot", "binarySha256": "sha", "environmentFileSha256": "env", + "service": {"ExecMainCode": "2", "ExecMainStatus": "9", "Result": "signal"}} + for kind in ("good", "graceful", "other-signal", "timeout", "boot", "candidate", "environment"): + current = copy.deepcopy(exited) + if kind == "graceful": current["service"]["ExecMainCode"] = "1" + elif kind == "other-signal": current["service"]["ExecMainStatus"] = "15" + elif kind == "timeout": current["service"]["Result"] = "timeout" + elif kind == "boot": current["bootId"] = "new-boot" + elif kind == "candidate": current["binarySha256"] = "new-build" + elif kind == "environment": current["environmentFileSha256"] = "new-env" + with self.subTest(kind=kind), patch.object(fleet_node, "owned", return_value=(None, None, {"runtimeProfile": "replacement"})), \ + patch.object(fleet_node, "process", side_effect=[before, current]), patch.object(fleet_node, "pidfd_signal", return_value=before) as send: + if kind == "good": + proof = fleet_node.process_crash({"runId": "run", "expected": before}) + self.assertEqual(proof["actualSignal"], 9) + self.assertFalse(proof["platformEventIssued"]) + self.assertEqual(send.call_args.args[1], fleet_node.signal.SIGKILL) + else: + with self.assertRaises(ValueError): fleet_node.process_crash({"runId": "run", "expected": before}) + with patch.object(fleet_node, "owned", return_value=(None, None, {"runtimeProfile": "manual"})), \ + patch.object(fleet_node, "pidfd_signal") as send, self.assertRaises(ValueError): + fleet_node.process_crash({"runId": "run", "expected": before}) + send.assert_not_called() + + def test_fast_completed_replacement_cannot_certify_an_unobserved_interruption(self): + endpoint = "/private/new/data" + row = {"taskId": "12345678-1234-4234-8234-123456789abc", "generation": "12345678-1234-4234-8234-123456789abc", + "setDiskId": "pool_0_set_0", "targetSlots": [endpoint], "state": "completed", "reason": None, "verifiedAt": 101} + with patch("scripts.validation_heal.replacement.replacement_records", return_value={"node1": [row]}), \ + patch("scripts.validation_heal.replacement.time.time", return_value=102), self.assertRaisesRegex(ValueError, "NOT_OBSERVED"): + replacement.automatic_completion(None, None, endpoint, 100, interruption="sigkill") + self.assertEqual(list(replacement.GRACEFUL_BRANCHES), select("F02")[0]["variants"]) + self.assertEqual(list(replacement.CRASH_BRANCHES), select("F03")[0]["variants"]) + + def test_target_interruption_needs_fresh_source_and_a_distinct_actual_coordinator(self): + token = "12345678-1234-4234-8234-123456789abc" + rows = [{"timestamp": "2026-10-10T10:00:01Z", "target": "rustfs::heal::manager", + "request_id": token, "source": "auto_heal", "result": "accepted", "message": "Heal queue request accepted"}, + {"timestamp": "2026-10-10T10:00:02Z", "target": "rustfs::heal::task", "task_id": token, + "set_disk_id": "pool_0_set_0", "stage": "start", "message": "Heal erasure set started"}] + self.assertEqual(replacement.coordinator({"node1": rows, "node4": []}, token, "node4"), "node1") + for kind in ("same-target", "duplicate", "source", "task", "order", "missing"): + events = {"node1": copy.deepcopy(rows), "node4": []} + if kind == "same-target": events = {"node4": rows} + elif kind == "duplicate": events["node2"] = rows + elif kind == "source": events["node1"][0]["source"] = "admin" + elif kind == "task": events["node1"][1]["task_id"] = "other-task" + elif kind == "order": events["node1"][1]["timestamp"] = "2026-10-10T10:00:00Z" + else: events["node1"].pop() + with self.subTest(kind=kind), self.assertRaisesRegex(ValueError, "NOT_OBSERVED"): + replacement.coordinator(events, token, "node4") + config = runtime_config("replacement-target") + self.assertEqual(config["RUSTFS_HEAL_AUTO_HEAL_ENABLE"], "false") + self.assertEqual(config["RUSTFS_HEAL_ENABLED"], "true") + def test_live_permission_rejoin_restores_same_inode_and_requires_same_process(self): for changed_process in (False, True): with self.subTest(changed_process=changed_process), tempfile.TemporaryDirectory() as folder: From 54775f39cad615aeab9cfc89851bc8c630739583 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 05:00:10 +0800 Subject: [PATCH 075/343] feat(validation): migrate durable MRF ACK replay and retirement --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-heal-contract.md | 2 +- docs/validation-migration-map.md | 35 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 29 +- scripts/validation_heal/full_restart.py | 2 +- scripts/validation_heal/healthy.py | 9 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/mrf_records.py | 228 +++++++++++ scripts/validation_heal/mrf_replay.py | 374 ++++++++++++++++++ scripts/validation_heal/run_case.sh | 1 + test_validation_heal_full_restart.py | 13 +- test_validation_heal_mrf.py | 196 +++++++++ 13 files changed, 881 insertions(+), 16 deletions(-) create mode 100644 scripts/validation_heal/mrf_records.py create mode 100644 scripts/validation_heal/mrf_replay.py create mode 100644 test_validation_heal_mrf.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index ec954fdb..1911d78a 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -19,6 +19,7 @@ on: - test_validation_heal_recovery.py - test_validation_heal_authorization.py - test_validation_heal_full_restart.py + - test_validation_heal_mrf.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -51,6 +52,7 @@ on: - test_validation_heal_recovery.py - test_validation_heal_authorization.py - test_validation_heal_full_restart.py + - test_validation_heal_mrf.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -82,7 +84,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index 4e44a114..13992199 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -39 个用例已接入自动分发：A01、A03–A08、B01–B03、B05–B07、B09、C01–C06、C08、D01–D07、D09、D10、E01–E04、F01、F06、J01、J03、J04。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。其余 45 项保留源 operator-assisted 方式。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 +44 个用例已接入自动分发：A01、A03–A08、B01–B03、B05–B07、B09、C01–C06、C08、D01–D07、D09、D10、E01–E06、F01–F03、F05、F06、J01、J03、J04。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。其余 40 项保留源 operator-assisted 方式。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index f697052a..b35da616 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 117 currently dispatchable IDs include all 74 non-Heal cases and forty-three Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. +case evidence. The 118 currently dispatchable IDs include all 74 non-Heal cases and forty-four Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 41 Heal cases remain +The other 40 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -834,3 +834,34 @@ missed physical window cannot PASS. External target verification uses bounded B03 bounded run `38084312163` retained FAIL/cleanup PASS in all four instances: the candidate still did not initialize the replacement format within 90 seconds. That explicit acceptance failure is not relabeled as a completed reconstruction. + + +E05 `heal.mrf-offline-write-replay` now has a dedicated H2 executor for all +three source branches: unique version PUT, exact historical-version DELETE, +and marker creation while the original fourth peer is stopped and its endpoint +refuses connections. Every ACK must have a fresh active durable responsibility +in the candidate's committed MRF snapshot and matching lifecycle companion. +The read-only observer opens only six exact snapshot/commit/lifecycle filenames +per private root. It validates full journal CRCs, both commit SHA-256 bindings, +owner/sequence, responsibility ID and bucket incarnation; it never enumerates +IAM or credential records, changes journals, or accepts operator risk. + +After the same node and disk identities rejoin, fresh manager admission with +source `mrf`, an exact-version automatic object task, complete successful +responsibility and `repaired`/`verified_healthy`/`authoritatively_absent` must +agree with committed responsibility retirement. Missing/held/unknown results, +queue emptiness, copied backup bytes or a manual verifier cannot certify this +case. All 16 replicas, retained same-role bytes, deleted data directories, +streaming checks and complete all-peer S3 ACK inventories precede PASS. The +record/proof decoder is explicitly bound to candidate source +`072b57b37559fbe626aeda8abf61617fdcbdd926` and reviewed source file digests; +ordinary execution consumes package/tools without a RustFS source checkout. +E05 has no live PASS claim yet. + +F05 run `38084904969` retained FAIL/cleanup PASS before any restart because its +combined DB/DV/DM/DX physical census exceeded the cursor resource profile. It +now registers a distinct bounded 24,576-file census with the original 2 GiB +per-drive free-space requirement and 45-minute service lease. E04 +`38085241447` verified complete journal collection on the corrected `--all` +reader, then failed its healthy task's missing public pool/set responsibility +identity; cleanup passed. The original failed runs remain unchanged. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 5b1dc68d..59254e81 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 6c771ff0..4d3920e7 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -32,7 +32,9 @@ def runtime_config(profile): - require(profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target"}, "unsupported Heal runtime profile") + require(profile in {"manual", "offline-replay", "durable-mrf", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target"}, "unsupported Heal runtime profile") + if profile == "durable-mrf": + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} if profile in {"replacement", "replacement-target"}: return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_AUTO_HEAL_ENABLE": "true" if profile == "replacement" else "false", "RUSTFS_HEAL_INTERVAL_SECS": "10", "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} @@ -122,10 +124,13 @@ def mounts(): def resource_limits(profile): - require(profile in {"standard", "cursor"}, "unsupported Heal resource profile") - return {"maxSnapshotFiles": 12288 if profile == "cursor" else 4096, - "minimumDriveFreeBytes": (2 if profile == "cursor" else 1) * 1024**3, - "runtimeMaxSeconds": 2700 if profile == "cursor" else 1800} + require(profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") + # F05 combines the 1,025-object cursor fixture with DB, DV and completed + # MPU objects. Each of four local roots also retains metadata backups. + # Its combined census needs a distinct, still bounded file allowance. + return {"maxSnapshotFiles": {"standard": 4096, "cursor": 12288, "full-restart": 24576}[profile], + "minimumDriveFreeBytes": (1 if profile == "standard" else 2) * 1024**3, + "runtimeMaxSeconds": 1800 if profile == "standard" else 2700} def preflight(request): @@ -733,9 +738,23 @@ def task_records(request): return {"taskId": token, "binarySha256": owner["binarySha256"], "readOnly": True, "records": records} +def mrf_records(request): + require(set(request) <= {"runId", "action", "quiescent"}, "MRF observer does not accept paths or mutation options") + state, roots, owner = owned(request["runId"]) + if request.get("quiescent") is True: + stopped(owner) + else: + pid = int(command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"]).stdout.strip() or "0") + require(pid > 0 and Path(f"/proc/{pid}/exe").resolve(strict=True) == state / "rustfs", "MRF observer requires its owned process") + module = globals().get("mrf") + require(module is not None, "MRF observer source missing") + return module.read_records(state, roots, owner, checked) + + def main(request): require(os.geteuid() == 0, "node adapter requires passwordless sudo") actions = {"preflight": preflight, "reserve": reserve, "start": start, "stop": stop, "snapshot": snapshot, + "mrf-records": mrf_records, "logs": logs, "cleanup": cleanup, "audit-finalize": audit_finalize, "install-inspector": install_inspector, "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part, "verify-parts": verify_parts, "verify-inline": verify_inline, "drive-gate": drive_gate, "drive-gate-probe": drive_gate_probe, diff --git a/scripts/validation_heal/full_restart.py b/scripts/validation_heal/full_restart.py index 426553eb..73445858 100644 --- a/scripts/validation_heal/full_restart.py +++ b/scripts/validation_heal/full_restart.py @@ -170,7 +170,7 @@ def run_topology(args, nodes): directory = args.evidence / "H2" directory.mkdir() run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-h2" - fleet = Fleet(nodes, directory, run_id, args.tool_directory, resource_profile="cursor", runtime_profile="root-restart") + fleet = Fleet(nodes, directory, run_id, args.tool_directory, resource_profile="full-restart", runtime_profile="root-restart") candidate = json.loads(args.candidate.read_text()) fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index d6e0fdf5..26691373 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -131,8 +131,8 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"manual", "offline-replay", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement"}, "unsupported Heal runtime profile") - require(resource_profile in {"standard", "cursor"}, "unsupported Heal resource profile") + require(runtime_profile in {"manual", "offline-replay", "durable-mrf", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement"}, "unsupported Heal runtime profile") + require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory self.runtime_profile = runtime_profile @@ -172,6 +172,9 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() + if action == "mrf-records": + source += "\nimport types\nmrf = types.ModuleType('mrf_records')\nexec(" + repr( + NODE_SOURCE.with_name("mrf_records.py").read_text()) + ", mrf.__dict__)\n" if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline"}: source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" @@ -183,7 +186,7 @@ def remote(self, node, action, **params): source += "\nimport sys\nsys.modules['drive_gate'] = gate\nreplacement = types.ModuleType('replace_drive')\nexec(" + repr( NODE_SOURCE.with_name("replace_drive.py").read_text()) + ", replacement.__dict__)\n" source += "\nmain(" + repr(value) + ")\n" - timeout = 600 if self.resource_profile == "cursor" and action == "snapshot" else 180 + timeout = 600 if self.resource_profile in {"cursor", "full-restart"} and action == "snapshot" else 180 result = subprocess.run([*self.ssh(node), "sudo -n python3 -"], input=source, capture_output=True, text=True, timeout=timeout) require(result.returncode == 0, f"Heal node {node} {action} failed: {result.stderr[-1200:]}") diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 51c2a3f2..4826ff37 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted", "heal.full-cluster-restart", "heal.replacement-auto-rebuild", "heal.target-graceful-restart", "heal.target-process-crash"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted", "heal.full-cluster-restart", "heal.replacement-auto-rebuild", "heal.target-graceful-restart", "heal.target-process-crash", "heal.mrf-offline-write-replay"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/mrf_records.py b/scripts/validation_heal/mrf_records.py new file mode 100644 index 00000000..30e39e64 --- /dev/null +++ b/scripts/validation_heal/mrf_records.py @@ -0,0 +1,228 @@ +"""Read only the candidate's exact committed MRF files and decode whole records. + +This is a source-bound observer. It never searches system metadata, repairs a +checkpoint, accepts operator risk, or treats a missing journal as repair proof. +""" + +import base64 +import hashlib +import json +import os +from pathlib import Path +import stat +import struct +import uuid +import zlib + + +REFERENCE = {"candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", "sourceFiles": { + "crates/heal/src/heal/mrf_queue.rs": "44fc2616024392ceef1e28816cb998d7e385618792545b26af3a3d7337ef9ca3", + "crates/heal/src/heal/mrf_queue/snapshot.rs": "ecfcb5ee63edef5c5d3d6713edc127e16f489522afc5fc02dc1044ccca241d14", + "crates/heal/src/heal/mrf_queue/partial_write.rs": "20415e0e985daea8fc62c5e152ed438e75008c33a2a592275e1dd9d1d4fee771", + "crates/common/src/mrf_channel.rs": "075bb84bcbbe7d76674f730c64c8b69fe37d0790ec75da46365ba2e0cab34e0f", + "crates/heal/src/heal/manager/scheduler.rs": "79ce1f9c278c664231b6a1eca7c521f8e98742fe6d17d8f852368da4e77cc33b"}, + "journal": "complete CRC32/ISO-HDLC records; format 1, layouts 1/2/3", + "commit": "RFMRFC01 version 1; owner, sequence, payload length and two SHA-256 digests", + "lifecycle": "RFMFLC01 version 1; checksummed JSON, same committed owner and sequence", + "retirement": "exact kind/bucket/object/version/pool/set/incarnation and successful verified disposition"} +LIMIT = 4 * 1024**2 +FILES = {f".rustfs.sys/.heal-mrf-{kind}.{slot}.bin": LIMIT * (4 if kind == "lifecycle" else 1) + for slot in (0, 1) for kind in ("snapshot", "commit", "lifecycle")} + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def canonical_uuid(value): + require(isinstance(value, str) and str(uuid.UUID(value)) == value and uuid.UUID(value).int != 0, + "MRF identity is not a canonical nonzero UUID") + return value + + +def verify_reference(candidate): + require(candidate.get("source_sha") == REFERENCE["candidateCommit"], "unsupported candidate MRF record/proof contract") + return REFERENCE + + +def read_records(state, roots, owner, checked): + """Exact allowlist, no directory enumeration, no write descriptor.""" + require(owner.get("runtimeProfile") == "durable-mrf", "MRF records require the dedicated run-owned runtime") + require(hashlib.sha256((state / "rustfs").read_bytes()).hexdigest() == owner["binarySha256"], "MRF candidate changed") + + def collect(): + rows = [] + for disk, root in enumerate(roots): + data = root / "data" + for relative, limit in FILES.items(): + path = checked(data / relative, data) + row = {"disk": disk, "relativePath": relative, "present": False} + try: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + except FileNotFoundError: + rows.append(row) + continue + with os.fdopen(fd, "rb") as stream: + before = os.fstat(stream.fileno()) + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_size <= limit, + "MRF record exceeds its exact regular-file limit") + raw = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + try: + current = path.lstat() + except FileNotFoundError: + return None + fields = ("st_dev", "st_ino", "st_nlink", "st_size", "st_mtime_ns", "st_ctime_ns") + if len(raw) != before.st_size or any(getattr(before, field) != getattr(other, field) + for other in (after, current) for field in fields): + return None + row.update(present=True, size=len(raw), sha256=hashlib.sha256(raw).hexdigest(), + rawBase64=base64.b64encode(raw).decode()) + rows.append(row) + return rows + + for _ in range(3): + first, second = collect(), collect() + if first is not None and first == second: + return {"readOnly": True, "stable": True, "binarySha256": owner["binarySha256"], "records": second} + return {"readOnly": True, "stable": False, "binarySha256": owner["binarySha256"], "records": []} + + +def decode_journal(raw): + require(isinstance(raw, bytes) and len(raw) <= LIMIT, "MRF journal exceeds the decoder bound") + records, cursor = [], 0 + while cursor < len(raw): + start = cursor + require(len(records) < 2048 and len(raw) - cursor >= 25, "MRF journal is torn or has too many records") + format_tag, layout, kind, attempts, stamp, has_version = struct.unpack_from("= owner["sequence"], + "MRF retirement changed owner or rolled back the committed sequence") + require(not any(identity(intent) == wanted or intent.get("lifecycle", {}).get("responsibility_id") + == owner["responsibilityId"] for intent in value["intents"]), + "NOT_OBSERVED: exact MRF ACK responsibility has not retired") + else: + require(value.get("allAllowlistedFilesAbsent") is True, "MRF retirement has an unknown record state") + require(not any(identity(intent) == wanted for value in checkpoints.values() for intent in value["intents"]), + "MRF ACK responsibility migrated to another owner instead of retiring") + return {"allExactACKResponsibilitiesRetired": True, "queueDepthUsedAsProof": False, + "operatorAcceptanceUsed": False, "responsibilityCount": len(obligations)} + + +def source_tasks(events, obligation, owners, after): + tasks = [] + for node in {owner["node"] for owner in owners}: + rows = events.get(node, []) + for start in rows: + stamp, token = event_time(start), start.get("task_id") + if not (start.get("target") == "rustfs::heal::task" and start.get("message") == "Heal object started" + and start.get("stage") == "start" and start.get("bucket") == obligation["bucket"] + and start.get("object") == obligation["key"] and start.get("version_id") == obligation["versionId"] + and isinstance(token, str) and token and stamp is not None and stamp >= after): + continue + admission = next((event for event in rows if event.get("target") == "rustfs::heal::manager" + and event.get("message") == "Heal queue request accepted" and event.get("request_id") == token + and event.get("source") == "mrf" and event.get("result") == "accepted" + and event_time(event) is not None and after <= event_time(event) <= stamp), None) + completed = next((event for event in rows if event.get("target") == "rustfs::heal::manager" + and event.get("state") == "task_completed" and event.get("task_id") == token + and event_time(event) is not None and event_time(event) >= stamp), None) + if admission is not None and completed is not None: + tasks.append({"node": node, "taskId": token, "admission": admission, "started": start, "completed": completed}) + return tasks + + +def verified_receipt(status, obligation, owners): + outcome = status.get("outcome") or {} + require(status.get("summary") == "finished" and status.get("truncated", False) is False + and outcome.get("execution", {}).get("state") == "completed" and outcome.get("coverage") == "complete" + and outcome.get("objectsTruncated") is False, "MRF task cannot discharge incomplete or failed responsibility") + objects = outcome.get("objects") + require(isinstance(objects, list) and len(objects) == 1, "MRF task did not preserve the exact ACK version scope") + receipt = objects[0] + item = receipt.get("identity", {}) + incarnation = {owner["bucketIncarnationId"] for owner in owners} + require(item.get("kind") == "object" and item.get("bucket") == obligation["bucket"] + and item.get("object") == obligation["key"] and item.get("versionId") == obligation["versionId"] + and type(item.get("poolIndex")) is int and item["poolIndex"] == obligation["poolIndex"] + and type(item.get("setIndex")) is int and item["setIndex"] == obligation["setIndex"] + and item.get("bucketIncarnationId") in incarnation, "MRF proof belongs to another ACK, set or bucket generation") + state = receipt.get("disposition", {}).get("state") + allowed = {"authoritatively_absent"} if obligation["authoritativelyAbsentRequired"] else {"repaired", "verified_healthy"} + require(state in allowed, "MRF retirement has no storage-verified disposition for this exact responsibility") + counters = outcome.get("counters", {}) + expected = {"processed": 1, "healed": int(state == "repaired"), "unchanged": int(state != "repaired"), + "skipped": 0, "failed": 0, "unknown": 0, "attemptFailures": 0} + require(all(type(counters.get(name)) is int and counters[name] == count for name, count in expected.items()) + and counters.get("overflowed") is False, "MRF proof counters hide unknown, failed or overflowed responsibility") + return {"identity": item, "disposition": state, "completeSuccessfulTask": True} + + +def read_checkpoints(fleet, nodes, label, *, quiescent=False): + observed = {} + for node in nodes: + raw = fleet.record(label + "-raw", {"node": node, **fleet.remote(node, "mrf-records", quiescent=quiescent)}) + observed[node] = checkpoint(raw) + return fleet.record(label + "-decoded", observed) + + +def deleted_parts_absent(snapshot, targets): + indexed = files(snapshot) + for target in targets: + for shard in target["cohort"]: + require((shard["node"], shard["disk"], shard["relativePath"]) not in indexed, + "MRF retired an explicit deletion with its original data shard still present") + + +def run_branch(args, branch, nodes): + directory = args.evidence / branch + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-h2" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="durable-mrf") + candidate = json.loads(args.candidate.read_text()) + tool = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + contract = verify_quorum_contract(tool, candidate) + fleet.record("candidate-MRF-record-and-proof-contract", verify_reference(candidate)) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, 5)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 128, "maxInventoryBytes": 128 * 1024**2} + fleet.record("binding", binding) + probe = Probe(binding) + result = {"topology": "H2", "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + recorder = Recorder(probe, directory / "fixture") + expected = seed(recorder, bucket, "DV", 4, [SIZE]) + baseline = fleet.record("s3-baseline", probe.inventory(bucket)) + fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + before = fleet.snapshot("physical-baseline") + fleet.record("topology-proof", validate_topology(before, "H2")) + initial_records = read_checkpoints(fleet, nodes, "MRF-baseline") + require(all(not value["intents"] for value in initial_records.values()), "fresh MRF fixture has preexisting responsibilities") + target = map_target(before, bucket, "objects/0001.bin", "data", 1, + next(entry["versionId"] for entry in expected if entry["key"] == "objects/0001.bin" and entry["isLatest"])) + offline_node, online_nodes = nodes[-1], nodes[:-1] + fleet.record("offline-quorum-plan", quorum_plan(target, offline_node, contract)) + stopped = fleet.record("offline-original-node", fleet.remote(offline_node, "stop")) + require(stopped.get("quiescent") is True and stopped.get("service", {}).get("MainPID") == "0", "MRF target did not actually stop") + try: + with request.build_opener(request.ProxyHandler({})).open(binding["endpoints"]["node4"] + "/health/ready", timeout=3): + raise ValueError("MRF offline endpoint still accepts requests") + except URLError as exc: + require(isinstance(exc.reason, ConnectionRefusedError), "MRF target outage has no exact connection refusal") + fleet.record("offline-refusal", {"node": offline_node, "connectionRefused": True}) + frozen = fleet.record("original-offline-scene", fleet.remote(offline_node, "snapshot", quiescent=True)) + earliest_ms = int(time.time() * 1000) - 5000 + expected, obligations, deleted = acknowledge(recorder, bucket, branch, expected) + fleet.record("offline-ACK-responsibility-manifest", {"expected": expected, "obligations": obligations, "deleted": deleted}) + acknowledged = fleet.record("offline-S3-ACK-proof", probe.inventory(bucket)) + fleet.record("offline-S3-full-manifest", verify_seed(expected, acknowledged)) + deadline = time.monotonic() + PERSIST_SECONDS + while True: + try: + checkpoints = read_checkpoints(fleet, online_nodes, "offline-committed-MRF") + bindings = pending_bindings(checkpoints, obligations, earliest_ms) + break + except ValueError as exc: + if not str(exc).startswith("NOT_OBSERVED:"): raise + require(time.monotonic() < deadline, "MRF persistence deadline exceeded: " + str(exc)) + time.sleep(1) + fleet.record("durable-ACK-binding", [{"identity": list(key), "owners": value} for key, value in bindings.items()]) + still_offline = fleet.record("offline-after-ACKs", fleet.remote(offline_node, "snapshot", quiescent=True)) + require(physical_content({offline_node: frozen}) == physical_content({offline_node: still_offline}), "MRF ACKs reached the stopped original node") + authorities = fleet.record("offline-physical-authority", {node: fleet.remote(node, "snapshot", quiescent=node == offline_node) for node in nodes}) + deleted_targets = [map_target(before, bucket, entry["key"], "data", 1, entry["versionId"], latest_required=False) for entry in deleted] + window = JournalWindow(fleet, "MRF-rejoin") + rejoin_time = datetime.now(timezone.utc) + fleet.record("rejoin-original-node", fleet.remote(offline_node, "start")) + fleet.wait_ready() + fleet.record("registered-automatic-replay-budget", {"persistenceSeconds": PERSIST_SECONDS, + "replaySeconds": REPLAY_SECONDS, "adminMutations": 0, "targetReadsBeforePhysicalProof": 0, + "source": "MRF", "originalNodeAndDiskIdentitiesRequired": True}) + deadline, proofs = time.monotonic() + REPLAY_SECONDS, {} + while True: + events = window.capture("MRF-replay-source") + for obligation in obligations: + key = identity(obligation) + if key in proofs: continue + for task in source_tasks(events, obligation, bindings[key], rejoin_time): + endpoint = "node" + str(nodes.index(task["node"]) + 1) + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + quote(obligation["key"], safe="/") + observed = fleet.record("MRF-exact-task-status", {"source": task, + **Probe(binding, endpoint).admin(path, [("clientToken", task["taskId"])], b"")}) + require(observed.get("httpStatus") == 200, "MRF source task status is unavailable") + try: + proof = verified_receipt(observed.get("response", {}), obligation, bindings[key]) + except ValueError as exc: + fleet.record("MRF-undischarged-task-result", {"task": task, "error": str(exc)}) + continue + proofs[key] = {"source": task, "proof": proof} + break + scene = fleet.snapshot("MRF-replay-physical-progress") + try: + require({node: value["formats"] for node, value in scene.items()} == {node: value["formats"] for node, value in before.items()}, + "MRF rejoin changed an original disk UUID or slot") + targets = replica_census(scene, bucket, expected, authorities) + deleted_parts_absent(scene, deleted_targets) + require(set(proofs) == set(bindings), "NOT_OBSERVED: MRF replay lacks an exact verified source task for every ACK") + retired = retirement(read_checkpoints(fleet, nodes, "MRF-retirement"), obligations, bindings) + break + except ValueError as exc: + fleet.record("MRF-replay-incomplete", {"error": str(exc), "verifiedACKs": len(proofs)}) + require(time.monotonic() < deadline, "MRF automatic replay deadline exceeded: " + str(exc)) + time.sleep(2) + fleet.record("MRF-verified-responsibility-retirement", {"retirement": retired, + "proofs": [{"identity": list(key), **value} for key, value in proofs.items()]}) + retirement(read_checkpoints(fleet, nodes, "MRF-repeat-retirement"), obligations, bindings) + fleet.stop() + repaired = fleet.snapshot("MRF-quiescent-repaired", quiescent=True) + targets = replica_census(repaired, bucket, expected, authorities) + deleted_parts_absent(repaired, deleted_targets) + streaming = [verify_cohort(fleet, target, "MRF-streaming-proof") for target in targets] + retirement(read_checkpoints(fleet, nodes, "MRF-quiescent-retirement", quiescent=True), obligations, bindings) + fleet.start() + for endpoint in binding["endpoints"]: + peer = Probe(binding, endpoint) + inventory = fleet.record("MRF-all-peer-S3", {"endpoint": endpoint, **peer.inventory(bucket)}) + fleet.record("MRF-all-peer-manifest", {**verify_seed(expected, inventory), **compare_inventories(acknowledged, inventory)}) + for entry in deleted: + validate_deleted_read(fleet.record("MRF-deleted-version-refusal", peer.s3("get_object", + {"Bucket": bucket, "Key": entry["key"], "VersionId": entry["versionId"]})), entry["versionId"]) + fleet.stop() + require(physical_content(repaired) == physical_content(fleet.snapshot("MRF-after-verification", quiescent=True)), + "S3 verification changed the physical MRF result") + result.update(verdict="PASS", durableACKs=len(obligations), sourceTasks=len(proofs), retirement=retired, + streamingVersions=len(streaming), allPeerS3ManifestEqual=True, adminMutationCount=0, backupRestorationUsed=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + if fleet.reserved: + try: + fleet.stop() + fleet.snapshot("MRF-failure-physical", quiescent=True) + read_checkpoints(fleet, nodes, "MRF-failure-records", quiescent=True) + except Exception as evidence_error: + result["failureEvidenceError"] = str(evidence_error) + finally: + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("branch-result", result) + return result + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f"Heal workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path, required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(set(nodes)) == len(nodes) and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), + "four current Heal nodes required") + require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)), "invalid candidate binary digest") + require(bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "workflow fleet identity required") + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "MRF executor omitted a source branch") + results = [] + try: + for branch in BRANCHES: + results.append(run_branch(args, branch, nodes[:4])) + if results[-1]["cleanup"] != "PASS": break + finally: + complete = len(results) == len(BRANCHES) and all(item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, + "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", "complete": complete}) + require(complete, "E05 requires all three H2 durable MRF branches and complete cleanup") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"Heal durable MRF validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index b95a4013..f71c7052 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -27,6 +27,7 @@ case "${case_id}" in heal.admin-overlap-and-force-start) driver=overlap.py ;; heal.root-graceful-coordinator-restart) driver=root_restart.py ;; heal.full-cluster-restart) driver=full_restart.py ;; + heal.mrf-offline-write-replay) driver=mrf_replay.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal_full_restart.py b/test_validation_heal_full_restart.py index 197efb0d..ba5f27df 100644 --- a/test_validation_heal_full_restart.py +++ b/test_validation_heal_full_restart.py @@ -8,12 +8,23 @@ from scripts.validation_heal import full_restart as restart from scripts.validation_heal.matrix import select, AUTOMATIC_CASES -from scripts.validation_heal.fleet_node import PROCESS_FIELDS, snapshot_buckets, verify_part_owned +from scripts.validation_heal.fleet_node import PROCESS_FIELDS, resource_limits, snapshot_buckets, verify_part_owned class FullRestartTests(unittest.TestCase): token = "12345678-1234-4234-8234-123456789abc" + def test_combined_fixture_has_explicit_bounded_physical_census_budget(self): + # Four roots, each with an xl.meta, its backup and an external part + # for every DX/DB/DV object. MPU adds up to three independent parts. + conservative_files = 4 * (3 * (1025 + 128 + 16) + 5) + self.assertGreater(conservative_files, resource_limits("cursor")["maxSnapshotFiles"]) + limits = resource_limits("full-restart") + self.assertLess(conservative_files, limits["maxSnapshotFiles"]) + self.assertEqual(limits["runtimeMaxSeconds"], 2700) + self.assertEqual(limits["minimumDriveFreeBytes"], 2 * 1024**3) + with self.assertRaises(ValueError): resource_limits("unbounded") + def wanted(self): wanted = {("db", f"objects/{index:04d}", "null"): {"deleteMarker": False, "payload": {"size": 17}} for index in range(1025)} diff --git a/test_validation_heal_mrf.py b/test_validation_heal_mrf.py new file mode 100644 index 00000000..c0b9e1dc --- /dev/null +++ b/test_validation_heal_mrf.py @@ -0,0 +1,196 @@ +import base64 +import copy +import hashlib +import json +from pathlib import Path +import struct +import tempfile +import unittest +import uuid +from unittest.mock import patch +import zlib + +from scripts.validation_heal import mrf_records as records, mrf_replay as replay +from scripts.validation_heal.fleet_node import runtime_config +from scripts.validation_heal.matrix import select, AUTOMATIC_CASES + + +class DurableMrfTests(unittest.TestCase): + owner = "12345678-1234-4234-8234-123456789abc" + version = "22345678-1234-4234-8234-123456789abc" + incarnation = "32345678-1234-4234-8234-123456789abc" + responsibility = "42345678-1234-4234-8234-123456789abc" + + def journal(self, *, layout=2, kind=3, key="offline/new.bin"): + bucket = b"rv-heal-0123456789abcdef" + raw = struct.pack(" Date: Sun, 11 Oct 2026 05:03:00 +0800 Subject: [PATCH 076/343] docs(validation): record complete stale-version rejoin live pass --- docs/validation-migration-map.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index b35da616..06ec6931 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 118 currently dispatchable IDs include all 74 non-Heal cases and forty-four Heal cases. Forty-three have live PASS evidence; the other 115 have no live PASS claim. +case evidence. The 118 currently dispatchable IDs include all 74 non-Heal cases and forty-four Heal cases. Forty-four have live PASS evidence; the other 114 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -865,3 +865,11 @@ per-drive free-space requirement and 45-minute service lease. E04 `38085241447` verified complete journal collection on the corrected `--all` reader, then failed its healthy task's missing public pool/set responsibility identity; cleanup passed. The original failed runs remain unchanged. + + +C06 `heal.stale-version-rejoin` passed complete H1/H2 unversioned overwrite, +historical-version deletion and marker creation in run `38085554143` at harness +`8d78ff32188412644a55776be7a794024a4f5614`. All six applicable instances have +PASS and cleanup PASS; downloaded canonical evidence passed the independent +selected-case gate. This raises the full live PASS total to 44. Older incomplete +journal-window runs remain FAIL. From 91c20d2fa6c069a6a8f9703436a12268b5cd6c07 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 05:09:24 +0800 Subject: [PATCH 077/343] feat(validation): migrate blank and foreign-format replacement safety --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-heal-contract.md | 2 +- docs/validation-migration-map.md | 30 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 39 +++ scripts/validation_heal/healthy.py | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/replace_drive.py | 53 ++++ scripts/validation_heal/run_case.sh | 1 + scripts/validation_heal/unformatted.py | 284 ++++++++++++++++++ test_validation_heal_unformatted.py | 114 +++++++ 11 files changed, 526 insertions(+), 7 deletions(-) create mode 100644 scripts/validation_heal/unformatted.py create mode 100644 test_validation_heal_unformatted.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 1911d78a..6e1d54a0 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -20,6 +20,7 @@ on: - test_validation_heal_authorization.py - test_validation_heal_full_restart.py - test_validation_heal_mrf.py + - test_validation_heal_unformatted.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -53,6 +54,7 @@ on: - test_validation_heal_authorization.py - test_validation_heal_full_restart.py - test_validation_heal_mrf.py + - test_validation_heal_unformatted.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -84,7 +86,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index 13992199..599a7cfa 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -44 个用例已接入自动分发：A01、A03–A08、B01–B03、B05–B07、B09、C01–C06、C08、D01–D07、D09、D10、E01–E06、F01–F03、F05、F06、J01、J03、J04。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。其余 40 项保留源 operator-assisted 方式。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 +45 个用例已接入自动分发：A01、A03–A08、B01–B07、B09、C01–C06、C08、D01–D07、D09、D10、E01–E06、F01–F03、F05、F06、J01、J03、J04。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。其余 39 项保留源 operator-assisted 方式。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 06ec6931..7f397a3f 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 118 currently dispatchable IDs include all 74 non-Heal cases and forty-four Heal cases. Forty-four have live PASS evidence; the other 114 have no live PASS claim. +case evidence. The 119 currently dispatchable IDs include all 74 non-Heal cases and forty-five Heal cases. Forty-four have live PASS evidence; the other 114 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 40 Heal cases remain +The other 39 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -873,3 +873,29 @@ historical-version deletion and marker creation in run `38085554143` at harness PASS and cleanup PASS; downloaded canonical evidence passed the independent selected-case gate. This raises the full live PASS total to 44. Older incomplete journal-window runs remain FAIL. + + +B04 `heal.disk-replacement-unformatted` now dispatches both H1/H2 source +branches with independent 128-object ACK fixtures. A second released-process +H1 service generates the actual foreign format; the read-only exporter retains +its exact bytes, digest, candidate binary and cluster/member identities before +complete donor cleanup. The replacement adapter copies only those verified +format bytes into the witnessed empty run-owned root, rejects reused original +identities, and never manufactures a disk UUID or edits the old archive. + +Blank initialization must expose the candidate's new UUID in its exact original +slot, complete the same automatic replacement generation, reconstruct every +original same-role shard, pass streaming verification and preserve all peer +ACK inventories. Refusal instead needs the exact Admin slot to remain explicitly +offline/unformatted/corrupt throughout a registered 90-second window, with all +other UUIDs/formats/user files unchanged and no target user writes. Foreign +format bytes must stay exact. The negative branch then retains the rejected +scene and restores the original archive only for recovery; it explicitly makes +no product-rebuild claim. Complete all-peer S3 and physical checks are still +required. B04 has no live PASS claim yet. + +E06 `38085937937` retained FAIL/cleanup PASS in every H1/H2 DB/DV instance. +Three instances generated a format without a distinct replacement UUID; the +H2 unversioned instance instead exposed partial/unknown/incompatible cluster +replacement status. Neither directory-root fallback nor a format file alone +certifies actual replacement readiness or responsibility. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 59254e81..d116af02 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 4d3920e7..9368edb3 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -8,6 +8,7 @@ from __future__ import annotations import hashlib +import base64 import json import os from datetime import datetime, timedelta, timezone @@ -403,6 +404,37 @@ def replacement_probe(request): return {**result, "sameOriginalServiceProcess": True} +def format_export(request): + require(set(request) == {"runId", "action", "disk"}, "format export accepts one owned member only") + state, roots, owner = owned(request["runId"]) + stopped(owner) + disk = request["disk"] + require(type(disk) is int and 0 <= disk < len(roots), "invalid exported disk") + path = checked(roots[disk] / "data/.rustfs.sys/format.json", roots[disk]) + require(path.stat().st_size < 65536, "exported format exceeds its bound") + sha = digest(path) + raw = path.read_bytes() + require(hashlib.sha256(raw).hexdigest() == sha == digest(path) and digest(state / "rustfs") == owner["binarySha256"], + "released donor format or candidate changed during export") + return {"runId": owner["runId"], "disk": disk, "binarySha256": owner["binarySha256"], + "sha256": sha, "rawBase64": base64.b64encode(raw).decode(), "value": json.loads(raw), "readOnly": True} + + +def foreign_format(request): + require(set(request) == {"runId", "action", "rawBase64", "sha256"}, "foreign format accepts exact exported bytes only") + state, roots, owner = owned(request["runId"]) + stopped(owner) + require(owner.get("runtimeProfile") == "replacement", "foreign format requires the isolated replacement runtime") + return replacement_module().install_foreign(state, roots, request["rawBase64"], request["sha256"]) + + +def restore_replacement(request): + require(set(request) == {"runId", "action"}, "replacement recovery accepts no paths") + state, roots, owner = owned(request["runId"]) + stopped(owner) + return replacement_module().restore_original(state, roots) + + def drive_gate(request): state, roots, owner = owned(request["runId"]) stopped(owner) @@ -614,6 +646,12 @@ def snapshot(request): entry.update(inspect_metadata(tool, path, bucket, key, entry, request.get("metadataFaults", {}))) files.append(entry) path = checked(data / ".rustfs.sys/format.json", data) + if not path.exists() and request.get("allowUnformattedReplacement") is True: + replacement = replacement_module().attest(state, roots) + require(not replacement.get("replacementAbsent") and replacement["disk"] == index + and not replacement["originalStillAtDataRoot"], "missing format is outside the exact isolated replacement member") + formats.append({"disk": index, "present": False, "unformattedReplacement": True}) + continue require(path.stat().st_size < 65536, "disk format exceeds budget") formats.append({"disk": index, "sha256": digest(path), "value": json.loads(path.read_text())}) return {"hostname": owner["hostname"], "unit": owner["unit"], "pid": pid, @@ -762,6 +800,7 @@ def main(request): "drive-gate-release-live": drive_gate_release_live} actions["task-records"] = task_records actions.update({"replace-drive": replace_drive, "replacement-state": replacement_state, "replacement-probe": replacement_probe}) + actions.update({"format-export": format_export, "foreign-format": foreign_format, "restore-replacement": restore_replacement}) actions.update({"process": process, "process-signal": process_signal, "process-graceful": process_graceful, "process-crash": process_crash}) require(request.get("action") in actions, "unsupported node operation") print(json.dumps(actions[request["action"]](request))) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 26691373..c9c68b26 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -178,7 +178,7 @@ def remote(self, node, action, **params): if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline"}: source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" - replacement_actions = {"start", "cleanup", "audit-finalize", "replace-drive", "replacement-state", "replacement-probe"} + replacement_actions = {"start", "cleanup", "audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot"} if action in {"drive-gate", "drive-gate-probe", "drive-gate-activate", "gate-state", "drive-gate-release-live"} | replacement_actions: source += "\nimport types\ngate = types.ModuleType('drive_gate')\nexec(" + repr( NODE_SOURCE.with_name("drive_gate.py").read_text()) + ", gate.__dict__)\n" diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 4826ff37..1ffc5841 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -25,7 +25,7 @@ "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted", "heal.full-cluster-restart", "heal.replacement-auto-rebuild", "heal.target-graceful-restart", "heal.target-process-crash", "heal.mrf-offline-write-replay"} + "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted", "heal.full-cluster-restart", "heal.replacement-auto-rebuild", "heal.target-graceful-restart", "heal.target-process-crash", "heal.mrf-offline-write-replay", "heal.disk-replacement-unformatted"} def require(condition: bool, message: str) -> None: diff --git a/scripts/validation_heal/replace_drive.py b/scripts/validation_heal/replace_drive.py index 99739d97..ea57e41c 100644 --- a/scripts/validation_heal/replace_drive.py +++ b/scripts/validation_heal/replace_drive.py @@ -1,11 +1,13 @@ """Whole private logical-root replacement; the shared mount stays attached.""" +import base64 import hashlib import json import os from pathlib import Path import stat import subprocess +import uuid if __package__: from .drive_gate import DENY_DAC_CAPS, READ_PROBE_SOURCE, process_identity @@ -102,6 +104,57 @@ def attest(state, roots): return {**value, "originalArchiveUnchanged": True, "originalStillAtDataRoot": original_path == data} +def install_foreign(state, roots, raw_base64, expected_sha256): + value = attest(state, roots) + require(not value.get("replacementAbsent") and not value["originalStillAtDataRoot"], "foreign format requires an isolated original member") + data = Path(value["dataPath"]) + require(not any(data.iterdir()), "foreign format requires the witnessed empty replacement root") + raw = base64.b64decode(raw_base64, validate=True) + require(len(raw) < 65536 and hashlib.sha256(raw).hexdigest() == expected_sha256, "foreign donor bytes differ from their exported digest") + donor = json.loads(raw) + original = json.loads((Path(value["backupPath"]) / ".rustfs.sys/format.json").read_bytes()) + require(donor.get("version") == "1" and donor.get("format") == "xl" and donor.get("xl", {}).get("version") == "3", + "foreign donor is not a supported released candidate format") + donor_id, original_id = uuid.UUID(donor["id"]), uuid.UUID(original["id"]) + members = [member for group in donor["xl"]["sets"] for member in group] + original_members = {member for group in original["xl"]["sets"] for member in group} + require(donor_id.int != 0 and donor_id != original_id and donor["xl"]["this"] in members + and len(members) == len(set(members)) and all(uuid.UUID(member).int != 0 for member in members) + and not set(members) & original_members, "foreign donor reused the original cluster or a member identity") + folder = data / ".rustfs.sys" + folder.mkdir(mode=0o700) + descriptor = os.open(folder / "format.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(raw); stream.flush(); os.fsync(stream.fileno()) + descriptor = os.open(folder, os.O_RDONLY | os.O_DIRECTORY) + try: os.fsync(descriptor) + finally: os.close(descriptor) + return {"disk": value["disk"], "foreignFormatSha256": expected_sha256, "foreignClusterId": str(donor_id), + "foreignDiskUUID": donor["xl"]["this"], "actualDonorBytesCopiedUnchanged": True, + "originalArchiveUnchanged": True, "syntheticFormatOrUUIDUsed": False} + + +def restore_original(state, roots): + value = attest(state, roots) + require(not value.get("replacementAbsent") and not value["originalStillAtDataRoot"], "original member is not at its witnessed archive") + backup, data = Path(value["backupPath"]), Path(value["dataPath"]) + rejected = backup.parent / "rejected-root" + require(not rejected.exists() and not rejected.is_symlink(), "rejected replacement archive already exists") + rejected_identity = tree_identity(data) + # Retain the rejected scene; this recovery is explicitly not a product + # rebuild. The existing intent still makes a partial rename recoverable. + os.rename(data, rejected) + os.rename(backup, data) + require(tree_identity(data) == value["original"] and tree_identity(rejected) == rejected_identity, + "original-member recovery changed its frozen bytes or rejected scene") + descriptor = os.open(data.parent, os.O_RDONLY | os.O_DIRECTORY) + try: os.fsync(descriptor) + finally: os.close(descriptor) + (state / "disk-replacement.json").unlink() + return {"disk": value["disk"], "originalRestoredExactly": True, "rejectedScene": rejected_identity, + "backupRestorationUsed": True, "productRepairClaimed": False, "sharedMountDetached": False} + + def start_properties(state, roots): value = checked_replacement(state, roots) if value is None: return [] diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index f71c7052..8e8a4137 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -28,6 +28,7 @@ case "${case_id}" in heal.root-graceful-coordinator-restart) driver=root_restart.py ;; heal.full-cluster-restart) driver=full_restart.py ;; heal.mrf-offline-write-replay) driver=mrf_replay.py ;; + heal.disk-replacement-unformatted) driver=unformatted.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/scripts/validation_heal/unformatted.py b/scripts/validation_heal/unformatted.py new file mode 100644 index 00000000..8627f4d9 --- /dev/null +++ b/scripts/validation_heal/unformatted.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""B04: candidate-created blank-member initialization or exact safe refusal. + +The foreign format is exported unchanged from a second released-process H1 +deployment. Restoring the original archive is recovery for a negative branch, +and never earns a product-rebuild claim. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time + +try: + from .heal_probe import Probe, compare_inventories, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .disk_loss import info_disks, member_state + from .replacement import format_identity, physical_rebuild, automatic_completion + from .shards import files, map_target +except ImportError: + from heal_probe import Probe, compare_inventories, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from disk_loss import info_disks, member_state + from replacement import format_identity, physical_rebuild, automatic_completion + from shards import files, map_target + + +CASE_ID = "heal.disk-replacement-unformatted" +BRANCHES = ("blank-unformatted", "foreign-format") +OBSERVATION_SECONDS = 90 +SIZE = 1024**2 + 73 +REJECTED_STATES = {"disk not found", "disk access denied", "file access denied", "volume access denied", + "offline", "unformatted disk", "corrupted format", "inconsistent disk", "corrupted backend"} + + +def binding(args, candidate, nodes, run_id, bucket): + return {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 512, "maxInventoryBytes": 512 * 1024**2} + + +def foreign_donor(args, node): + directory = args.evidence / "foreign-donor-H1" + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-h1" + fleet = Fleet([node], directory, run_id, args.tool_directory) + candidate = json.loads(args.candidate.read_text()) + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + exported, error = None, None + try: + fleet.deploy(args.binary_sha, bucket, secrets.token_hex(16), secrets.token_hex(24)) + fleet.record("actual-foreign-donor-topology", validate_topology(fleet.snapshot("foreign-donor-formats"), "H1")) + fleet.stop() + exported = fleet.record("released-process-foreign-format", fleet.remote(node, "format-export", disk=0)) + require(exported.get("readOnly") is True and exported.get("binarySha256") == args.binary_sha, + "foreign format was not exported from the exact stopped released candidate") + except (Exception, KeyboardInterrupt) as exc: + error = exc + finally: + clean = fleet.cleanup() + fleet.record("foreign-donor-finalization", {"cleanup": "PASS" if clean else "FAIL", "productRepairClaimed": False}) + require(clean, "foreign donor cleanup failed; primary cluster must not start") + if error is not None: raise error + return exported + + +def refusal(observed, target, original_endpoint, foreign): + total = len(target["cohort"]) + indexed = info_disks(observed, total) + selected = indexed[(0, 0, target["setSlot"])] + require(selected["endpoint"] == original_endpoint and selected.get("state") in REJECTED_STATES, + "unready/foreign member lacks a structured exact-slot refusal or was marked healthy") + permitted_ids = {"", target["diskUUID"]} | ({foreign["value"]["xl"]["this"]} if foreign else set()) + require(selected.get("uuid") in permitted_ids and selected.get("runtimeState") not in {"online", "ready"}, + "unready/foreign member became writable or acquired an invented disk identity") + expected = {shard["setSlot"]: shard["diskUUID"] for shard in target["cohort"]} + for slot, disk in indexed.items(): + if slot[2] == target["setSlot"]: continue + require(disk.get("state") == "ok" and disk.get("uuid") == expected[slot[2]] + and disk.get("runtimeState") in {None, "online"}, "replacement refusal changed another original member") + return {"exactSlot": target["setSlot"], "endpoint": selected["endpoint"], "state": selected["state"], + "reportedUUID": selected.get("uuid"), "unreadyMemberHealthyClaimed": False} + + +def negative_scene(before, scene, member, branch, foreign): + left, right = files(before), files(scene) + require({key: value for key, value in right.items() if key[:2] != member} + == {key: value for key, value in left.items() if key[:2] != member}, "refusal changed a surviving user file") + require(not any(key[:2] == member for key in right), "unready/foreign member received user data") + for node, value in before.items(): + for old in value["formats"]: + current = next(item for item in scene[node]["formats"] if item["disk"] == old["disk"]) + if (node, old["disk"]) != member: + require(current == old, "refusal changed an unrelated original disk format") + elif branch == "foreign-format": + require(current.get("sha256") == foreign["sha256"] and current.get("value") == foreign["value"], + "candidate overwrote the exact foreign cluster/member format") + else: + require(current.get("present") is False and current.get("unformattedReplacement") is True, + "blank refusal created an unexplained format") + return {"allSurvivingUserFilesAndFormatsUnchanged": True, "targetUserFilesAbsent": True, + "foreignIdentityUnchanged": branch == "foreign-format", "productRepairClaimed": False} + + +def run_branch(args, topology, nodes, branch, foreign): + directory = args.evidence / f"{topology}-{branch}" + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="replacement") + candidate = json.loads(args.candidate.read_text()) + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + lab = binding(args, candidate, nodes, run_id, bucket) + fleet.record("binding", lab) + probe = Probe(lab) + result = {"topology": topology, "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} + try: + require(candidate["source_sha"] == "072b57b37559fbe626aeda8abf61617fdcbdd926", "unsupported candidate initialization/refusal contract") + fleet.deploy(args.binary_sha, bucket, access, secret) + recorder = Recorder(probe, directory / "fixture") + expected = seed(recorder, bucket, "DB", 128, [SIZE]) + baseline = fleet.record("S3-baseline", probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("fixture-ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + fleet.stop() + before = fleet.snapshot("original-full-physical", quiescent=True) + fleet.record("actual-original-topology", validate_topology(before, topology)) + target = map_target(before, bucket, expected[0]["key"], "data", 1) + node = nodes[-1] + selected = next(shard for shard in target["cohort"] if shard["node"] == node) + target = {**target, **selected} + member, disk = (node, target["disk"]), target["disk"] + physical_format = next(item for item in before[node]["formats"] if item["disk"] == disk) + fleet.start() + original = member_state(fleet.record("original-member-Admin", probe.admin("/rustfs/admin/v3/info", [], b"", method="GET")), + target, len(target["cohort"])) + fleet.stop() + replaced = fleet.record("whole-original-root-quarantined", fleet.remote(node, "replace-drive", disk=disk, + expectedFormatSha256=physical_format["sha256"])) + require(replaced.get("newDataRootEmpty") is True and replaced.get("oldRootMovedIntact") is True, + "B04 replacement lacks its actual empty root and exact original archive") + if branch == "foreign-format": + require(foreign["binarySha256"] == args.binary_sha, "foreign donor used another released candidate") + fleet.record("foreign-source-provenance", foreign) + fleet.record("exact-foreign-format-injection", fleet.remote(node, "foreign-format", rawBase64=foreign["rawBase64"], sha256=foreign["sha256"])) + earliest = int(time.time()) + fleet.record("registered-initialization-refusal-window", {"seconds": OBSERVATION_SECONDS, + "branch": branch, "directoryTestRootFallback": True, "actualDeviceReplacementClaimed": False, + "manualHealMutations": 0, "refusalRequiresExactAdminSlotAndUnchangedScene": True}) + fleet.start() + fleet.record("actual-old-archive-refusal", fleet.remote(node, "replacement-probe")) + deadline, samples, initialized = time.monotonic() + OBSERVATION_SECONDS, [], False + while True: + state = fleet.record("initialization-physical-format-window", fleet.remote(node, "replacement-state")) + require(state.get("originalArchiveUnchanged") is True, "candidate accessed or changed the original replacement archive") + current = next(item for item in state["formats"] if item["disk"] == disk) + if branch == "blank-unformatted" and current["present"]: + initialized = True + break + if branch == "foreign-format": + require(current.get("sha256") == foreign["sha256"] and current.get("value") == foreign["value"], + "candidate overwrote or reidentified a foreign formatted member") + samples.append(refusal(fleet.record("exact-unready-member-Admin", probe.admin("/rustfs/admin/v3/info", [], b"", method="GET")), + target, original["endpoint"], foreign if branch == "foreign-format" else None)) + if time.monotonic() >= deadline: break + time.sleep(5) + if initialized: + current = fleet.snapshot("candidate-generated-initialized-format") + fleet.record("actual-new-disk-identity", format_identity(before, current, member)) + endpoint = replaced["dataPath"] if topology == "H1" else f"http://rustfs-node{nodes.index(node) + 1}:9000{replaced['dataPath']}" + responsibility = automatic_completion(probe, fleet, endpoint, earliest) + fleet.stop() + rebuilt = fleet.snapshot("automatic-blank-rebuild", quiescent=True) + physical = fleet.record("complete-original-role-rebuild", physical_rebuild(before, rebuilt, member)) + parts = [] + for entry in expected: + cohort = map_target(rebuilt, bucket, entry["key"], "data", 1) + shard = next(shard for shard in cohort["cohort"] if (shard["node"], shard["disk"]) == member) + parts.append({"disk": disk, "relativePath": shard["relativePath"], "versionId": "null", "partNumber": 1, + "sha256": shard["sha256"], "physicalSize": shard["size"]}) + streamed = fleet.record("all-new-member-streaming-checks", fleet.remote(node, "verify-parts", parts=[ + {name: part[name] for name in ("disk", "relativePath", "versionId", "partNumber")} for part in parts])) + require(streamed.get("complete") is True and len(streamed.get("parts", [])) == len(parts), "blank rebuild omitted a streaming part") + for wanted, proof in zip(parts, streamed["parts"], strict=True): + require(proof.get("physicalBitrotVerified") is True and all(proof.get(name) == value for name, value in wanted.items()), + "blank rebuild streaming proof differs from the original same-role ACK shard") + result.update(initialization="candidate-generated-and-completely-rebuilt", responsibility=responsibility, + physical=physical, backupRestorationUsed=False) + else: + require(len(samples) >= 3, "unformatted/foreign refusal did not span its registered observation window") + fleet.stop() + scene = fleet.record("refused-complete-physical", {peer: fleet.remote(peer, "snapshot", quiescent=True, + allowUnformattedReplacement=peer == node) for peer in nodes}) + physical = negative_scene(before, scene, member, branch, foreign) + fleet.record("safe-refusal-negative-proof", {"samples": samples, "physical": physical}) + recovery = fleet.record("negative-original-member-recovery", fleet.remote(node, "restore-replacement")) + require(recovery.get("originalRestoredExactly") is True and recovery.get("productRepairClaimed") is False, + "negative recovery was not exact or falsely claimed product reconstruction") + rebuilt = fleet.snapshot("negative-original-restored", quiescent=True) + require(physical_content(before) == physical_content(rebuilt), "refusal recovery changed the original physical scene") + result.update(initialization="structured-safe-refusal", physical=physical, refusalSamples=len(samples), + backupRestorationUsed=True, productRepairClaimed=False) + fleet.start() + for endpoint in lab["endpoints"]: + inventory = fleet.record("all-peer-final-S3", {"endpoint": endpoint, + **Probe(lab, endpoint).inventory(bucket, unversioned_current_tags=True)}) + fleet.record("all-peer-exact-ACK-manifest", {**verify_seed(expected, inventory), **compare_inventories(baseline, inventory)}) + fleet.stop() + require(physical_content(rebuilt) == physical_content(fleet.snapshot("after-S3-physical", quiescent=True)), "S3 verification changed the proven physical result") + result.update(verdict="PASS", all128ACKsPreserved=True, allPeerS3ManifestEqual=True, syntheticFormatOrUUIDUsed=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + if fleet.reserved: + try: + fleet.stop() + for peer in nodes: + fleet.record("B04-failure-format-and-original", {"node": peer, **fleet.remote(peer, "replacement-state")}) + except Exception as evidence_error: + result["failureEvidenceError"] = str(evidence_error) + finally: + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("branch-result", result) + return result + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f"Heal workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path, required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(set(nodes)) == len(nodes) and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") + require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)) and bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "verified workflow candidate required") + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "B04 executor omitted a source branch") + results, donor_error = [], None + try: + donor = foreign_donor(args, nodes[0]) + for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): + for branch in BRANCHES: + results.append(run_branch(args, topology, selected, branch, donor)) + if results[-1]["cleanup"] != "PASS": break + if results[-1]["cleanup"] != "PASS": break + except (Exception, KeyboardInterrupt) as exc: + donor_error = {"type": type(exc).__name__, "message": str(exc)} + finally: + complete = len(results) == 4 and all(item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "donorFailure": donor_error, + "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", "complete": complete}) + require(complete, "B04 requires both source branches on H1/H2, released foreign donor and complete cleanup") + + +if __name__ == "__main__": + try: main() + except Exception as exc: + print(f"Heal unformatted replacement validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/test_validation_heal_unformatted.py b/test_validation_heal_unformatted.py new file mode 100644 index 00000000..2fad69e4 --- /dev/null +++ b/test_validation_heal_unformatted.py @@ -0,0 +1,114 @@ +import base64 +import copy +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from scripts.validation_heal import replace_drive, unformatted +from scripts.validation_heal.matrix import select, AUTOMATIC_CASES + + +class UnformattedReplacementTests(unittest.TestCase): + original_id = "12345678-1234-4234-8234-123456789abc" + foreign_id = "22345678-1234-4234-8234-123456789abc" + original_members = [f"32345678-1234-4234-8234-123456789ab{index}" for index in range(4)] + foreign_members = [f"42345678-1234-4234-8234-123456789ab{index}" for index in range(4)] + + def format(self, foreign=False): + members = self.foreign_members if foreign else self.original_members + return {"version": "1", "format": "xl", "id": self.foreign_id if foreign else self.original_id, + "xl": {"version": "3", "this": members[0], "sets": [members], "distributionAlgo": "SIPMOD+PARITY"}} + + def scene(self): + before = {"node": {"formats": [{"disk": index, "sha256": "original-" + str(index), "value": {"id": self.original_id}} + for index in range(4)], + "files": [{"disk": index, "path": "bucket/object/part.1", "sha256": "part-" + str(index), "size": 17} + for index in range(4)]}} + after = copy.deepcopy(before) + after["node"]["files"] = after["node"]["files"][1:] + after["node"]["formats"][0] = {"disk": 0, "present": False, "unformattedReplacement": True} + return before, after + + def test_negative_scene_cannot_hide_survivor_write_or_foreign_reidentification(self): + before, after = self.scene() + self.assertFalse(unformatted.negative_scene(before, after, ("node", 0), "blank-unformatted", None)["productRepairClaimed"]) + foreign = {"sha256": "foreign-digest", "value": self.format(True)} + after["node"]["formats"][0] = {"disk": 0, "sha256": foreign["sha256"], "value": foreign["value"]} + unformatted.negative_scene(before, after, ("node", 0), "foreign-format", foreign) + for kind in ("survivor", "target", "format", "foreign"): + bad = copy.deepcopy(after) + if kind == "survivor": bad["node"]["files"][0]["sha256"] = "changed" + elif kind == "target": bad["node"]["files"].append(before["node"]["files"][0]) + elif kind == "format": bad["node"]["formats"][1]["value"]["id"] = "changed" + else: bad["node"]["formats"][0]["value"]["id"] = self.original_id + with self.subTest(kind=kind), self.assertRaises(ValueError): + unformatted.negative_scene(before, bad, ("node", 0), "foreign-format", foreign) + + def admin(self): + return {"httpStatus": 200, "response": {"info": {"servers": [{"drives": [ + {"pool_index": 0, "set_index": 0, "disk_index": index, "endpoint": "endpoint-" + str(index), + "uuid": self.original_members[index] if index else "", "state": "ok" if index else "unformatted disk", + "runtimeState": "online" if index else "offline"} for index in range(4)]}]}}} + + def test_structured_refusal_requires_exact_slot_and_each_other_original_member(self): + target = {"setSlot": 0, "diskUUID": self.original_members[0], + "cohort": [{"setSlot": index, "diskUUID": value} for index, value in enumerate(self.original_members)]} + value = self.admin() + self.assertFalse(unformatted.refusal(value, target, "endpoint-0", None)["unreadyMemberHealthyClaimed"]) + for field, changed in (("state", "ok"), ("state", "unknown"), ("runtimeState", "online"), ("endpoint", "other"), ("uuid", "invented")): + bad = self.admin(); bad["response"]["info"]["servers"][0]["drives"][0][field] = changed + with self.subTest(field=field), self.assertRaises(ValueError): unformatted.refusal(bad, target, "endpoint-0", None) + bad = self.admin(); bad["response"]["info"]["servers"][0]["drives"][2]["uuid"] = self.foreign_members[2] + with self.assertRaises(ValueError): unformatted.refusal(bad, target, "endpoint-0", None) + + def archive(self, root): + state = root / "state"; state.mkdir() + drive = root / "drive"; (drive / "quarantine" / "old-disk" / ".rustfs.sys").mkdir(parents=True) + data = drive / "data"; data.mkdir() + backup = drive / "quarantine" / "old-disk" + (backup / ".rustfs.sys" / "format.json").write_text(json.dumps(self.format())) + value = {"disk": 0, "dataPath": str(data), "backupPath": str(backup), "originalStillAtDataRoot": False, + "originalArchiveUnchanged": True, "original": {"kind": "original"}} + return state, drive, data, backup, value + + def test_import_copies_released_donor_bytes_and_rejects_original_or_shared_members(self): + with tempfile.TemporaryDirectory() as folder: + state, drive, data, backup, value = self.archive(Path(folder)) + with patch.object(replace_drive, "attest", return_value=value): + for donor in (self.format(), {**self.format(True), "xl": self.format()["xl"]}, + {**self.format(True), "version": "future"}): + raw = json.dumps(donor).encode() + with self.assertRaises(ValueError): replace_drive.install_foreign(state, [drive], base64.b64encode(raw).decode(), hashlib.sha256(raw).hexdigest()) + self.assertFalse(list(data.iterdir())) + raw = b' ' + json.dumps(self.format(True), indent=2).encode() + b'\n' + proof = replace_drive.install_foreign(state, [drive], base64.b64encode(raw).decode(), hashlib.sha256(raw).hexdigest()) + self.assertEqual((data / ".rustfs.sys/format.json").read_bytes(), raw) + self.assertFalse(proof["syntheticFormatOrUUIDUsed"]) + self.assertEqual(json.loads((backup / ".rustfs.sys/format.json").read_text()), self.format()) + with self.assertRaises(ValueError): replace_drive.install_foreign(state, [drive], base64.b64encode(raw).decode(), hashlib.sha256(raw).hexdigest()) + + def test_negative_recovery_preserves_rejected_scene_and_cannot_claim_reconstruction(self): + with tempfile.TemporaryDirectory() as folder: + state, drive, data, backup, value = self.archive(Path(folder)) + (state / "disk-replacement.json").write_text("intent") + (data / "foreign.bin").write_bytes(b"exact-rejected-scene") + def tree(path): + return {"kind": "rejected" if (path / "foreign.bin").exists() else "original"} + with patch.object(replace_drive, "attest", return_value=value), patch.object(replace_drive, "tree_identity", side_effect=tree): + proof = replace_drive.restore_original(state, [drive]) + self.assertEqual(json.loads((data / ".rustfs.sys/format.json").read_text()), self.format()) + self.assertEqual((drive / "quarantine/rejected-root/foreign.bin").read_bytes(), b"exact-rejected-scene") + self.assertFalse(proof["productRepairClaimed"]) + self.assertTrue(proof["backupRestorationUsed"]) + self.assertFalse((state / "disk-replacement.json").exists()) + self.assertFalse(backup.exists()) + + def test_both_source_branches_have_an_automatic_entry(self): + self.assertEqual(select("B04")[0]["variants"], list(unformatted.BRANCHES)) + self.assertIn(unformatted.CASE_ID, AUTOMATIC_CASES) + + +if __name__ == "__main__": unittest.main() From 437f558b8720b35ebef1128b5bd8291fa178e663 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 05:16:18 +0800 Subject: [PATCH 078/343] fix(validation): count candidate integrity sidecars in DX budget --- docs/validation-heal-operations.md | 2 +- docs/validation-migration-map.md | 2 +- scripts/validation_heal/cursor.py | 4 +++- scripts/validation_heal/fleet_node.py | 5 ++++- test_validation_heal.py | 4 +++- test_validation_heal_full_restart.py | 6 +++--- 6 files changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 454b5bce..ac0ee5bd 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -287,7 +287,7 @@ WINDOW 的必要证据是 running 且 `0 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 7f397a3f..c7bf122a 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -544,7 +544,7 @@ counters must cover all 1025 ACKs with the exact 17 real missing-shard repairs, no failures/unknowns, and complete traversal. Bounded diagnostic receipts remain explicitly incomplete. Full physical manifests, every original same-role shard and streaming checksum, and all-peer S3 manifests independently cover every -object before PASS. The case has a fixed 12288-file snapshot budget, 128-part +object before PASS. The case has a fixed 16384-file snapshot budget, 128-part verification batches, a 45-minute service/lease budget and 2GiB free per drive. Local cursor/gap/counter/scope/part-omission negatives passed; live evidence remains pending. diff --git a/scripts/validation_heal/cursor.py b/scripts/validation_heal/cursor.py index 0f368fb3..f6c84a41 100644 --- a/scripts/validation_heal/cursor.py +++ b/scripts/validation_heal/cursor.py @@ -25,6 +25,7 @@ from .heal_fault import bounded_fault_id from .shards import files, map_target, validate_repaired_physical from .versions import public_version_id + from .fleet_node import resource_limits except ImportError: from heal_probe import Probe, canonical, compare_inventories, require, save from heal_workload import Recorder, seed, verify_seed @@ -34,6 +35,7 @@ from heal_fault import bounded_fault_id from shards import files, map_target, validate_repaired_physical from versions import public_version_id + from fleet_node import resource_limits CASE_ID = "heal.incremental-cursor-and-truncation" @@ -313,7 +315,7 @@ def run_topology(args, topology, nodes, *, observer=observe_task, branches=BRANC fleet.record("binding", binding) fleet.record("pre-registered-budget", {"fixture": "DX", "objects": count, "objectBytes": SIZE, "logicalBytes": count * SIZE, "taskSeconds": TASK_SECONDS, "sourceLimitReference": policy_reference, - "maxSnapshotFilesPerNode": 12288, "snapshotSecondsPerNode": 600, "verificationBatchItems": 128, + "maxSnapshotFilesPerNode": resource_limits("cursor")["maxSnapshotFiles"], "snapshotSecondsPerNode": 600, "verificationBatchItems": 128, "missingIndices": missing_indices, "taskScope": "frozen prefix scopes; one result per ACK", "faultLeaseMinutes": 30, "serviceRuntimeSeconds": 2700, "topologyLeaseMinutes": 45}) recipe = select(args.case_id)[0] diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 9368edb3..246f0dd4 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -126,10 +126,13 @@ def mounts(): def resource_limits(profile): require(profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") + # DX has 1,025 external objects on four local roots. The candidate keeps + # an xl.meta, part and integrity sidecar for each: 12,300 files even + # without backups. Retain every sidecar within an explicit allowance. # F05 combines the 1,025-object cursor fixture with DB, DV and completed # MPU objects. Each of four local roots also retains metadata backups. # Its combined census needs a distinct, still bounded file allowance. - return {"maxSnapshotFiles": {"standard": 4096, "cursor": 12288, "full-restart": 24576}[profile], + return {"maxSnapshotFiles": {"standard": 4096, "cursor": 16384, "full-restart": 24576}[profile], "minimumDriveFreeBytes": (1 if profile == "standard" else 2) * 1024**3, "runtimeMaxSeconds": 1800 if profile == "standard" else 2700} diff --git a/test_validation_heal.py b/test_validation_heal.py index a8737794..56730e09 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -1994,7 +1994,9 @@ def test_batch_bitrot_verifier_requires_every_exact_part_and_no_duplicates(self) def test_cursor_resources_are_explicit_bounded_and_source_branches_preserved(self): self.assertEqual(resource_limits("standard")["maxSnapshotFiles"], 4096) - self.assertEqual(resource_limits("cursor")["maxSnapshotFiles"], 12288) + # The live candidate writes a part integrity sidecar on every root. + self.assertGreater(resource_limits("cursor")["maxSnapshotFiles"], 1025 * 4 * 3) + self.assertEqual(resource_limits("cursor")["maxSnapshotFiles"], 16384) self.assertEqual(resource_limits("cursor")["minimumDriveFreeBytes"], 2 * 1024**3) self.assertEqual(list(CURSOR_BRANCHES), select("J03")[0]["variants"]) with self.assertRaises(ValueError): resource_limits("unbounded") diff --git a/test_validation_heal_full_restart.py b/test_validation_heal_full_restart.py index ba5f27df..16cf0950 100644 --- a/test_validation_heal_full_restart.py +++ b/test_validation_heal_full_restart.py @@ -15,9 +15,9 @@ class FullRestartTests(unittest.TestCase): token = "12345678-1234-4234-8234-123456789abc" def test_combined_fixture_has_explicit_bounded_physical_census_budget(self): - # Four roots, each with an xl.meta, its backup and an external part - # for every DX/DB/DV object. MPU adds up to three independent parts. - conservative_files = 4 * (3 * (1025 + 128 + 16) + 5) + # Four roots retain xl.meta, an external part and its integrity + # sidecar, plus a possible metadata backup. MPU adds three parts. + conservative_files = 4 * (4 * (1025 + 128 + 16) + 7) self.assertGreater(conservative_files, resource_limits("cursor")["maxSnapshotFiles"]) limits = resource_limits("full-restart") self.assertLess(conservative_files, limits["maxSnapshotFiles"]) From 39992b3a23c57789065e8cafe35e05ff30135e3d Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 05:27:03 +0800 Subject: [PATCH 079/343] feat(validation): migrate fixed ABBA repair performance acceptance --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-heal-operations.md | 5 +- docs/validation-migration-map.md | 46 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 24 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/mrf_replay.py | 70 +-- scripts/validation_heal/performance.py | 439 ++++++++++++++++++ scripts/validation_heal/run_case.sh | 1 + test_validation_heal_mrf.py | 34 ++ test_validation_heal_performance.py | 192 ++++++++ 11 files changed, 779 insertions(+), 40 deletions(-) create mode 100644 scripts/validation_heal/performance.py create mode 100644 test_validation_heal_performance.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 6e1d54a0..6caf8f4d 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -21,6 +21,7 @@ on: - test_validation_heal_full_restart.py - test_validation_heal_mrf.py - test_validation_heal_unformatted.py + - test_validation_heal_performance.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -55,6 +56,7 @@ on: - test_validation_heal_full_restart.py - test_validation_heal_mrf.py - test_validation_heal_unformatted.py + - test_validation_heal_performance.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -86,7 +88,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index ac0ee5bd..572354d6 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -2,7 +2,7 @@ 此手册和 [84 项逐例操作卡](validation-heal-cases.md) 一起构成可执行的**人工操作方案 + 脚本辅助**。不是 84 个无人值守自动测试，也不是通过报告。操作卡列出 301 个分支模板；真实盘角色、阈值、损坏数量和拓扑仍需在每批 `parameterBindings` 展开，不能把模板数量当实测次数。 -公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 已接入 39 个 Heal 用例，完整清单通过 `matrix.py list` 查看，以 `automaticExecutor` 为准。入口使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 +公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 已接入 46 个 Heal 用例，完整清单通过 `matrix.py list` 查看，以 `automaticExecutor` 为准。入口使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 ## 1. 使用入口与交付物 @@ -637,3 +637,6 @@ audit检查完整性/路径/摘要，输出`EVIDENCE_COMPLETE_REVIEW_REQUIRED` 自动正向夹具固定使用独立分片承诺配置；`integrity-fleet-attestation` 记录本例 endpoint/节点/候选摘要，启动前逐节点 reserve 校验候选，启动后实际进程环境读回两项开关。此配置只作用于 run-owned 服务。默认 legacy 配置的失败批次独立保留；保护模式不代表旧版本混部或默认环境能力。D07 的条件请求以服务器 HEAD 的 LastModified 和基线 ETag 冻结预期，不改主机时间；304/412/416 不能包含成功 payload 或未知传输结果。 + + +J06 的自动入口为 `heal.performance-abba`。执行器在部署前冻结三组 ABBA、每阶段 30 秒、2 并发、4 请求/秒、GET/PUT 各半与完整阈值；每阶段重启同一服务并按固定次数预热。每个 B 阶段的真实部分修复窗口必须包围 GET 和 PUT，否则整例不能 PASS。A 阶段持续检查完整 idle 与物理不变。所有排定请求（包括失败、未知和错过 admission 的请求）均计入分母；GET/PUT 独立报告 P95/P99，三组配对差异、实际修复分片吞吐与服务 CPU/IO/RSS/FD 分别留证。阈值保存在 `scripts/validation_heal/performance.py` 的 `PROTOCOL`，修改阈值必须产生新的 harness commit 和新验证记录；此入口不缩短 J05 的 24h/72h 要求。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index c7bf122a..e15be37f 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 119 currently dispatchable IDs include all 74 non-Heal cases and forty-five Heal cases. Forty-four have live PASS evidence; the other 114 have no live PASS claim. +case evidence. The 120 currently dispatchable IDs include all 74 non-Heal cases and forty-six Heal cases. Forty-four have live PASS evidence; the other 114 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 39 Heal cases remain +The other 38 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -650,7 +650,7 @@ A08 `heal.admin-authorization` passed live in RustFS Actions run `38069648003`, harness `0789743d439fd91b967c2d6fba85c06e29f844c5`. All five source branches passed independently on H1 and H2; the canonical gate, complete summary, cleanup and workflow all passed with the fixed candidate. -The prior A08 failures remain recorded. Forty-five Heal executors remain +The prior A08 failures remain recorded. Thirty-eight Heal executors remain to be migrated; automatic dispatch and local tests do not claim live PASS. @@ -899,3 +899,43 @@ Three instances generated a format without a distinct replacement UUID; the H2 unversioned instance instead exposed partial/unknown/incompatible cluster replacement status. Neither directory-root fallback nor a format file alone certifies actual replacement readiness or responsibility. + + +The cursor resource budget now includes the candidate's `part.N.integrity.*` +sidecars: 1,025 external objects on four roots already require at least 12,300 +files. The explicit limit is 16,384 and the registered DX budget reads the same +configuration. F02 `38086220066` and F03 `38086565751` retain FAIL/cleanup PASS: +their DB branches exceeded the old 12,288-file allowance; the DV branches +failed to expose a distinct replacement UUID. Budget correction permits fresh +verification and does not change these original results. + +E05 `38086919036` retained FAIL/cleanup PASS on all three branches. PUT/marker +reached durable ACK binding but the replay caller passed an unsupported label +to `JournalWindow.capture`; the actual shared interface and replay observer +are now covered together by a regression test. Exact historical deletion +failed to expose committed durable responsibility within the registered +30-second persistence deadline. That failure remains; no substitute manual +mutation or inferred queue record is accepted. + +J06 `heal.performance-abba` now dispatches three complete H2 ABBA groups on +the same candidate, rc, four nodes, original roots and disk UUIDs. A frozen +protocol sets 30-second phases, two workers, four requests per second, a 1:1 +GET/PUT ratio and the same 16 logical 64KiB load objects. Every scheduled request +has durable intent/receipt evidence; unknowns, failed samples and missed +admission deadlines remain in both error and latency denominators. Each phase +restarts the same services and uses the same fixed warmup without changing host +caches. Every A phase proves idle throughout and unchanged repair files. Each +B phase injects the same 17 missing data/parity shards in an independent +1,025-object repair bucket and binds one actual task's partial running progress +to both foreground operations. No observed overlap means NOT_OBSERVED and a +complete-case failure. + +Exact owned-process birth, boot and settings bind read-only CPU, logical/storage +IO, RSS and FD samples. Candidate completion, actual reconstructed shard bytes, +streaming verification and full peer S3 manifests must agree. Repair throughput +uses physical target bytes, separately from logical S3 throughput. Frozen gates +require zero errors, GET/PUT P95 <= 5s and P99 <= 10s, paired B/A ratios <= 3, +positive verified repair throughput and bounded resources. All twelve phases +and every paired difference are reported; no phase may be omitted or reordered +and functional success cannot substitute for performance acceptance. J06 has +no live PASS claim yet. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index d116af02..3fcb9471 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 246f0dd4..77248fa7 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -304,6 +304,28 @@ def pidfd_signal(request, signum): return current +def process_metrics(request): + """Read only the owned service's resource counters, with process birth binding.""" + require(set(request) == {"runId", "action"}, "process metrics accept no arbitrary PID or path") + before = process(request) + require(before["pid"] > 1, "resource sample requires the original live service") + root = Path(f"/proc/{before['pid']}") + fields = (root / "stat").read_text().rpartition(") ")[2].split() + require(len(fields) >= 22 and int(fields[19]) == before["startTicks"], "resource sample crossed process birth") + io = {name: int(value.strip()) for name, value in (line.split(":", 1) for line in (root / "io").read_text().splitlines())} + required_io = {"rchar", "wchar", "syscr", "syscw", "read_bytes", "write_bytes", "cancelled_write_bytes"} + require(required_io <= set(io) and all(value >= 0 for value in io.values()), "owned process IO counters are incomplete") + result = {"identity": before, "clockDomain": before["bootId"], "monotonicNs": time.monotonic_ns(), + "cpuTicks": int(fields[11]) + int(fields[12]), "clockTicksPerSecond": os.sysconf("SC_CLK_TCK"), + "rssBytes": int(fields[21]) * os.sysconf("SC_PAGE_SIZE"), "virtualBytes": int(fields[20]), + "threads": int(fields[17]), "fileDescriptors": sum(1 for _ in (root / "fd").iterdir()), + "io": {name: io[name] for name in sorted(required_io)}, "readOnly": True, + "ioScope": "owned process counters; cached logical IO and storage IO reported separately"} + after = process(request) + require(all(after.get(name) == before[name] for name in PROCESS_FIELDS), "resource sample crossed a service identity change") + return result + + def process_signal(request): name = request.get("signal") require(name in {"SIGSTOP", "SIGCONT"}, "only owned-process pause/resume signals are allowed") @@ -804,6 +826,6 @@ def main(request): actions["task-records"] = task_records actions.update({"replace-drive": replace_drive, "replacement-state": replacement_state, "replacement-probe": replacement_probe}) actions.update({"format-export": format_export, "foreign-format": foreign_format, "restore-replacement": restore_replacement}) - actions.update({"process": process, "process-signal": process_signal, "process-graceful": process_graceful, "process-crash": process_crash}) + actions.update({"process": process, "process-metrics": process_metrics, "process-signal": process_signal, "process-graceful": process_graceful, "process-crash": process_crash}) require(request.get("action") in actions, "unsupported node operation") print(json.dumps(actions[request["action"]](request))) diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 1ffc5841..3e039d6f 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -24,7 +24,7 @@ AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", - "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", + "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted", "heal.full-cluster-restart", "heal.replacement-auto-rebuild", "heal.target-graceful-restart", "heal.target-process-crash", "heal.mrf-offline-write-replay", "heal.disk-replacement-unformatted"} diff --git a/scripts/validation_heal/mrf_replay.py b/scripts/validation_heal/mrf_replay.py index 4d63d38e..c85f3b6a 100644 --- a/scripts/validation_heal/mrf_replay.py +++ b/scripts/validation_heal/mrf_replay.py @@ -194,6 +194,42 @@ def deleted_parts_absent(snapshot, targets): "MRF retired an explicit deletion with its original data shard still present") +def wait_replay(fleet, window, binding, nodes, bucket, expected, obligations, bindings, rejoin_time, + before, authorities, deleted_targets): + deadline, proofs = time.monotonic() + REPLAY_SECONDS, {} + while True: + events = window.capture() + for obligation in obligations: + key = identity(obligation) + if key in proofs: continue + for task in source_tasks(events, obligation, bindings[key], rejoin_time): + endpoint = "node" + str(nodes.index(task["node"]) + 1) + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + quote(obligation["key"], safe="/") + observed = fleet.record("MRF-exact-task-status", {"source": task, + **Probe(binding, endpoint).admin(path, [("clientToken", task["taskId"])], b"")}) + require(observed.get("httpStatus") == 200, "MRF source task status is unavailable") + try: + proof = verified_receipt(observed.get("response", {}), obligation, bindings[key]) + except ValueError as exc: + fleet.record("MRF-undischarged-task-result", {"task": task, "error": str(exc)}) + continue + proofs[key] = {"source": task, "proof": proof} + break + scene = fleet.snapshot("MRF-replay-physical-progress") + try: + require({node: value["formats"] for node, value in scene.items()} == {node: value["formats"] for node, value in before.items()}, + "MRF rejoin changed an original disk UUID or slot") + replica_census(scene, bucket, expected, authorities) + deleted_parts_absent(scene, deleted_targets) + require(set(proofs) == set(bindings), "NOT_OBSERVED: MRF replay lacks an exact verified source task for every ACK") + retired = retirement(read_checkpoints(fleet, nodes, "MRF-retirement"), obligations, bindings) + return proofs, retired + except ValueError as exc: + fleet.record("MRF-replay-incomplete", {"error": str(exc), "verifiedACKs": len(proofs)}) + require(time.monotonic() < deadline, "MRF automatic replay deadline exceeded: " + str(exc)) + time.sleep(2) + + def run_branch(args, branch, nodes): directory = args.evidence / branch directory.mkdir() @@ -266,38 +302,8 @@ def run_branch(args, branch, nodes): fleet.record("registered-automatic-replay-budget", {"persistenceSeconds": PERSIST_SECONDS, "replaySeconds": REPLAY_SECONDS, "adminMutations": 0, "targetReadsBeforePhysicalProof": 0, "source": "MRF", "originalNodeAndDiskIdentitiesRequired": True}) - deadline, proofs = time.monotonic() + REPLAY_SECONDS, {} - while True: - events = window.capture("MRF-replay-source") - for obligation in obligations: - key = identity(obligation) - if key in proofs: continue - for task in source_tasks(events, obligation, bindings[key], rejoin_time): - endpoint = "node" + str(nodes.index(task["node"]) + 1) - path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + quote(obligation["key"], safe="/") - observed = fleet.record("MRF-exact-task-status", {"source": task, - **Probe(binding, endpoint).admin(path, [("clientToken", task["taskId"])], b"")}) - require(observed.get("httpStatus") == 200, "MRF source task status is unavailable") - try: - proof = verified_receipt(observed.get("response", {}), obligation, bindings[key]) - except ValueError as exc: - fleet.record("MRF-undischarged-task-result", {"task": task, "error": str(exc)}) - continue - proofs[key] = {"source": task, "proof": proof} - break - scene = fleet.snapshot("MRF-replay-physical-progress") - try: - require({node: value["formats"] for node, value in scene.items()} == {node: value["formats"] for node, value in before.items()}, - "MRF rejoin changed an original disk UUID or slot") - targets = replica_census(scene, bucket, expected, authorities) - deleted_parts_absent(scene, deleted_targets) - require(set(proofs) == set(bindings), "NOT_OBSERVED: MRF replay lacks an exact verified source task for every ACK") - retired = retirement(read_checkpoints(fleet, nodes, "MRF-retirement"), obligations, bindings) - break - except ValueError as exc: - fleet.record("MRF-replay-incomplete", {"error": str(exc), "verifiedACKs": len(proofs)}) - require(time.monotonic() < deadline, "MRF automatic replay deadline exceeded: " + str(exc)) - time.sleep(2) + proofs, retired = wait_replay(fleet, window, binding, nodes, bucket, expected, obligations, bindings, + rejoin_time, before, authorities, deleted_targets) fleet.record("MRF-verified-responsibility-retirement", {"retirement": retired, "proofs": [{"identity": list(key), **value} for key, value in proofs.items()]}) retirement(read_checkpoints(fleet, nodes, "MRF-repeat-retirement"), obligations, bindings) diff --git a/scripts/validation_heal/performance.py b/scripts/validation_heal/performance.py new file mode 100644 index 00000000..80f4f53e --- /dev/null +++ b/scripts/validation_heal/performance.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python3 +"""J06: three fixed ABBA groups with concurrent ACK traffic and real repair.""" + +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor +import copy +from datetime import datetime, timedelta, timezone +import hashlib +import json +import math +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +from urllib.parse import quote + +try: + from .heal_probe import Probe, compare_inventories, payload, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .heal_fault import bounded_fault_id + from .cursor import OBJECTS, SIZE, map_cohorts, verify_all_parts, validate_missing_batch, complete_outcome + from .shards import validate_repaired_physical + from .stop import idle_fleet, status + from .fleet_node import PROCESS_FIELDS +except ImportError: + from heal_probe import Probe, compare_inventories, payload, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from heal_fault import bounded_fault_id + from cursor import OBJECTS, SIZE, map_cohorts, verify_all_parts, validate_missing_batch, complete_outcome + from shards import validate_repaired_physical + from stop import idle_fleet, status + from fleet_node import PROCESS_FIELDS + + +CASE_ID = "heal.performance-abba" +BRANCHES = ("abba-repeat-1", "abba-repeat-2", "abba-repeat-3") +# These are acceptance gates chosen before deployment, not results derived +# from the measured candidate. Changing one requires a new immutable run. +PROTOCOL = {"groups": 3, "order": ["A", "B", "B", "A"], "secondsPerPhase": 30, + "requestsPerSecond": 4, "concurrency": 2, "readWriteRatio": "1:1", + "loadObjects": 16, "loadObjectBytes": 65536, "warmupRequests": 16, + "repairObjects": OBJECTS, "repairObjectBytes": SIZE, "missingIndices": list(range(0, OBJECTS, 64)), + "phaseAdmissionGraceSeconds": 1, "minimumSamplesPerOperation": 50, + "maxErrorRate": 0, "getP95Ms": 5000, "getP99Ms": 10000, + "putP95Ms": 5000, "putP99Ms": 10000, "maxPairedLatencyRatio": 3, + "minimumRunningOverlapSeconds": 1, "minimumBracketedCalls": 4, + "minimumRepairMiBPerSecond": .001, "maxProcessCpuCores": 2.5, + "maxProcessRssBytes": 2800 * 1024**2, "maxProcessFDs": 4096, + "metricsIntervalSeconds": 2, "repairDeadlineSeconds": 300, + "cachePolicy": "same original four roots; restart every phase; fixed warmup; no host cache eviction", + "latencyScope": "collector scheduled admission through complete SDK response/body; includes queue delay and errors", + "retryPolicy": "no retries; every scheduled GET and PUT remains in the denominator", + "hardwareScope": "same candidate, rc, hosts, mounts and original disk UUIDs for all twelve phases"} + + +def schedule(protocol=PROTOCOL): + require(protocol == PROTOCOL, "J06 requires the immutable preregistered protocol") + return [{"group": group, "phase": index, "mode": mode, "branch": BRANCHES[group - 1]} + for group in range(1, 4) for index, mode in enumerate(protocol["order"], 1)] + + +def percentile(values, fraction): + require(values and all(type(value) in {int, float} and math.isfinite(value) and value >= 0 for value in values) + and fraction in {.95, .99}, "invalid performance sample or percentile") + return sorted(values)[math.ceil(len(values) * fraction) - 1] + + +def load_body(bucket, index): + return payload(f"{bucket}/load/{index:02d}/0", PROTOCOL["loadObjectBytes"]) + + +def load_manifest(bucket): + return [{"key": f"load/{index:02d}.bin", "versionId": "null", "deleteMarker": False, + "isLatest": True, "payload": {"size": PROTOCOL["loadObjectBytes"], + "sha256": hashlib.sha256(load_body(bucket, index)).hexdigest()}} + for index in range(PROTOCOL["loadObjects"])] + + +def prepare_load(recorder, bucket): + recorder.call("create_bucket", {"Bucket": bucket}) + for index, entry in enumerate(load_manifest(bucket)): + response = recorder.call("put_object", {"Bucket": bucket, "Key": entry["key"]}, load_body(bucket, index)) + require(response.get("VersionId") in {None, "null"}, "performance load bucket unexpectedly versioned") + + +def repair_view(snapshot, bucket): + return {node: {**value, "files": [entry for entry in value["files"] if entry["path"].startswith(bucket + "/")]} + for node, value in snapshot.items()} + + +def same_hardware(original, actual): + require(set(original) == set(actual), "ABBA hardware omitted or added a node") + for node in original: + require(original[node]["formats"] == actual[node]["formats"] + and original[node]["hostname"] == actual[node]["hostname"] + and original[node]["binarySha256"] == actual[node]["binarySha256"], + "ABBA candidate, host or disk identities changed") + return {"sameHostsCandidateAndDiskUUIDs": True} + + +def request(probe, bucket, sequence, epoch, output): + operation = "get_object" if sequence % 2 == 0 else "put_object" + index = (sequence // 2) % PROTOCOL["loadObjects"] + body = load_body(bucket, index) + scheduled = epoch + sequence / PROTOCOL["requestsPerSecond"] + remaining = scheduled - time.monotonic() + if remaining > 0: time.sleep(remaining) + params = {"Bucket": bucket, "Key": f"load/{index:02d}.bin"} + prepared = {"sequence": sequence, "operation": operation, "params": params, + "scheduledMonotonicNs": int(scheduled * 1e9), + "expectedPayload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}} + path = output / f"{sequence:05d}-request.json" + save(path, prepared) + if time.monotonic() > epoch + PROTOCOL["secondsPerPhase"] + PROTOCOL["phaseAdmissionGraceSeconds"]: + observed = {"acknowledged": False, "resultUnknown": False, "clientCallInvoked": False, + "error": {"type": "AdmissionDeadline", "message": "scheduled sample missed the phase admission deadline"}} + else: + observed = probe.s3(operation, params, body if operation == "put_object" else None) + finished = time.monotonic_ns() + code = (observed.get("response") or {}).get("ResponseMetadata", {}).get("HTTPStatusCode") + success = observed.get("acknowledged") is True and observed.get("resultUnknown") is False and code == 200 \ + and not observed.get("clientCleanupError") + if operation == "get_object": success = success and observed.get("payload") == prepared["expectedPayload"] + else: success = success and (observed.get("response") or {}).get("VersionId") in {None, "null"} + result = {**prepared, "requestSha256": hashlib.sha256(path.read_bytes()).hexdigest(), "observation": observed, + "finishedMonotonicNs": finished, "latencyMs": (finished - prepared["scheduledMonotonicNs"]) / 1e6, + "verdict": "PASS" if success else "FAIL"} + save(output / f"{sequence:05d}-ack.json", result) + return result + + +def summarize(rows): + count = PROTOCOL["secondsPerPhase"] * PROTOCOL["requestsPerSecond"] + require(len(rows) == count and {row["sequence"] for row in rows} == set(range(count)), + "scheduled requests were omitted, duplicated or filtered") + statistics = {"scheduledRequests": count, "failedRequests": sum(row["verdict"] != "PASS" for row in rows)} + statistics["errorRate"] = statistics["failedRequests"] / count + for operation, label in (("get_object", "get"), ("put_object", "put")): + samples = [row["latencyMs"] for row in rows if row["operation"] == operation] + require(len(samples) == count // 2 >= PROTOCOL["minimumSamplesPerOperation"], "GET/PUT sample count is incomplete") + statistics[label] = {"requests": len(samples), "p95Ms": percentile(samples, .95), "p99Ms": percentile(samples, .99)} + return statistics + + +def resource_delta(before, after): + require(set(before) == set(after) and len(before) == 4, "performance metrics require all four actual peers") + result = {} + for node, start in before.items(): + end = after[node] + require(start.get("readOnly") is True and end.get("readOnly") is True + and all(start["identity"].get(name) == end["identity"].get(name) for name in PROCESS_FIELDS) + and start["clockDomain"] == end["clockDomain"] == start["identity"]["bootId"] + and type(start["clockTicksPerSecond"]) is int and start["clockTicksPerSecond"] > 0 + and start["clockTicksPerSecond"] == end["clockTicksPerSecond"], "resource counters crossed a process or clock identity") + elapsed = (end["monotonicNs"] - start["monotonicNs"]) / 1e9 + require(elapsed > 0 and end["cpuTicks"] >= start["cpuTicks"], "resource clock/CPU counters went backwards") + io = {} + for name in ("rchar", "wchar", "syscr", "syscw", "read_bytes", "write_bytes"): + require(type(start["io"].get(name)) is int and type(end["io"].get(name)) is int + and end["io"][name] >= start["io"][name], "process IO counters went backwards or were omitted") + io[name] = end["io"][name] - start["io"][name] + result[node] = {"elapsedSeconds": elapsed, + "cpuCores": (end["cpuTicks"] - start["cpuTicks"]) / start["clockTicksPerSecond"] / elapsed, + "maxSampledRssBytes": max(start["rssBytes"], end["rssBytes"]), + "maxSampledFDs": max(start["fileDescriptors"], end["fileDescriptors"]), "ioDelta": io} + return result + + +def overlap(rows, windows): + intervals = [] + for left, right in zip(windows, windows[1:]): + lc, rc = left["response"].get("outcome", {}).get("counters", {}), right["response"].get("outcome", {}).get("counters", {}) + for counter in (lc, rc): + require(all(type(counter.get(name)) is int and counter[name] >= 0 for name in + ("processed", "healed", "unchanged", "skipped", "failed", "unknown", "attemptFailures")), + "active performance task lacks typed counters") + require(all(counter[name] == 0 for name in ("failed", "unknown", "attemptFailures")) + and counter.get("overflowed") is False, "performance task has failed or unknown responsibility") + require(rc["processed"] >= lc["processed"] and rc["healed"] >= lc["healed"], "same performance task progress went backwards") + if all(row["response"].get("summary") == "running" and row["response"].get("outcome", {}).get("execution", {}).get("state") == "running" + for row in (left, right)) and 0 < lc["processed"] < rc["processed"] < OBJECTS: + start, end = left["callFinishedMonotonicNs"], right["callStartedMonotonicNs"] + if end > start: intervals.append((start, end)) + calls = [row for row in rows if row["observation"].get("clientCallInvoked") is True and any( + start <= row["observation"]["callStartedMonotonicNs"] <= row["observation"]["callFinishedMonotonicNs"] <= end + for start, end in intervals)] + duration = sum((end - start) / 1e9 for start, end in intervals) + require(duration >= PROTOCOL["minimumRunningOverlapSeconds"] and len(calls) >= PROTOCOL["minimumBracketedCalls"] + and {row["operation"] for row in calls} == {"get_object", "put_object"}, + "NOT_OBSERVED: workload GET/PUT did not overlap independently witnessed nonzero partial repair") + return {"bracketedRunningSeconds": duration, "bracketedCalls": len(calls), "bothOperationsDuringRealRepair": True} + + +def acceptance(phases): + require([{name: phase[name] for name in ("group", "phase", "mode", "branch")} for phase in phases] == schedule(), + "ABBA phases were omitted, reordered, repeated or selected after measurement") + ratios = [] + for phase in phases: + require(phase["statistics"]["errorRate"] <= PROTOCOL["maxErrorRate"], "performance request error-rate gate failed") + for operation in ("get", "put"): + for quantile in ("95", "99"): + value = phase["statistics"][operation]["p" + quantile + "Ms"] + require(math.isfinite(value) and 0 <= value <= PROTOCOL[operation + "P" + quantile + "Ms"], + "absolute performance latency gate failed") + for node in phase["resources"].values(): + require(node["cpuCores"] <= PROTOCOL["maxProcessCpuCores"] + and node["maxSampledRssBytes"] <= PROTOCOL["maxProcessRssBytes"] + and node["maxSampledFDs"] <= PROTOCOL["maxProcessFDs"], "performance resource gate failed") + if phase["mode"] == "B": + require(phase["overlap"]["bothOperationsDuringRealRepair"] is True and phase["repair"]["physicalVerified"] is True + and phase["repair"]["MiBPerSecond"] >= PROTOCOL["minimumRepairMiBPerSecond"], + "performance phase did not prove actual physical repair throughput") + else: + require(phase["noHeal"] is True and phase["repair"] is None, "baseline phase contains a heal task") + for group in range(1, 4): + selected = [phase for phase in phases if phase["group"] == group] + for operation in ("get", "put"): + for quantile in ("95", "99"): + name = "p" + quantile + "Ms" + a = sum(phase["statistics"][operation][name] for phase in selected if phase["mode"] == "A") / 2 + b = sum(phase["statistics"][operation][name] for phase in selected if phase["mode"] == "B") / 2 + require(a > 0 and b / a <= PROTOCOL["maxPairedLatencyRatio"], "paired ABBA latency regression gate failed") + ratios.append({"group": group, "operation": operation, "percentile": name, "aMeanMs": a, + "bMeanMs": b, "differenceMs": b - a, "ratio": b / a}) + return {"allTwelvePhasesAccepted": True, "pairedComparisons": ratios, + "thresholdsFrozenBeforeDeployment": True, "functionalChecksAreSeparate": True} + + +def sample_metrics(fleet, label): + return fleet.record(label, {node: fleet.remote(node, "process-metrics") for node in fleet.nodes}) + + +def run_phase(spec, fleet, probe, binding, bucket, expected, load_bucket, hardware, cohorts, active, directory): + label = f"g{spec['group']}-p{spec['phase']}-{spec['mode']}" + output = directory / label + output.mkdir() + fleet.record("phase-protocol-" + label, {**spec, "protocol": PROTOCOL}) + idle_fleet(fleet, binding) + fleet.stop() + original = repair_view(fleet.snapshot("physical-phase-baseline-" + label, quiescent=True), bucket) + same_hardware(hardware, original) + targets = [cohorts[index] for index in PROTOCOL["missingIndices"]] + registrations, receipts = [], [] + if spec["mode"] == "B": + for index, target in enumerate(targets): + params = {"disk": target["disk"], "relativePath": target["relativePath"], "kind": "missing", + "faultId": bounded_fault_id("J06", label, index), "expectedSha256": target["sha256"]} + fleet.faults.append((target["node"], params)) + registrations.append((target["node"], params)) + receipts.append(fleet.record("fault-" + label, {"node": target["node"], **fleet.remote(target["node"], "apply-fault", **params)})) + faulted = repair_view(fleet.snapshot("physical-phase-fault-" + label, quiescent=True), bucket) + fleet.record("fault-proof-" + label, validate_missing_batch(original, faulted, targets, receipts)) + else: faulted = original + fleet.start() + before_trigger = repair_view(fleet.snapshot("physical-phase-before-trigger-" + label), bucket) + require(physical_content(faulted) == physical_content(before_trigger), "ATTRIBUTION_NOT_OBSERVED: ABBA fault changed before trigger") + for index in range(PROTOCOL["warmupRequests"]): + observed = fleet.record("warmup-" + label, probe.s3("get_object", {"Bucket": load_bucket, "Key": f"load/{index:02d}.bin"})) + require(observed.get("acknowledged") is True and observed.get("payload") == load_manifest(load_bucket)[index]["payload"], "fixed warmup failed") + initial_metrics = sample_metrics(fleet, "metrics-before-" + label) + path, token, windows, terminal = "/rustfs/admin/v3/heal/" + quote(bucket, safe=""), None, [], None + repair_started = None + if spec["mode"] == "B": + repair_started = time.monotonic() + start = fleet.record("trigger-" + label, probe.admin(path, [], json.dumps({**BASE_OPTIONS, "recursive": True}).encode())) + require(start.get("httpStatus") == 200 and isinstance(start.get("response"), dict), "ABBA repair admission failed") + token = start["response"].get("clientToken") + require(isinstance(token, str) and token, "ABBA repair omitted task identity") + active.append((path, token)) + else: idle_fleet(fleet, binding) + epoch = time.monotonic() + samples, metric_rows = [], [initial_metrics] + next_metric = epoch + PROTOCOL["metricsIntervalSeconds"] + repair_finished = None + with ThreadPoolExecutor(max_workers=PROTOCOL["concurrency"]) as workers: + futures = [workers.submit(request, probe, load_bucket, sequence, epoch, output) + for sequence in range(PROTOCOL["secondsPerPhase"] * PROTOCOL["requestsPerSecond"])] + while not all(future.done() for future in futures) or token and terminal is None: + if token and terminal is None: + observed = fleet.record("performance-running-" + label, probe.admin(path, [("clientToken", token)], b"")) + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), "ABBA task status failed") + windows.append(observed) + if observed["response"].get("summary") != "running": + terminal, repair_finished = observed["response"], time.monotonic() + active.remove((path, token)) + require(time.monotonic() - repair_started <= PROTOCOL["repairDeadlineSeconds"], "ABBA task exceeded frozen deadline") + if time.monotonic() >= next_metric: + metric_rows.append(sample_metrics(fleet, "metrics-during-" + label)) + if spec["mode"] == "A": idle_fleet(fleet, binding) + next_metric = time.monotonic() + PROTOCOL["metricsIntervalSeconds"] + time.sleep(.2) + samples = [future.result() for future in futures] + remaining = epoch + PROTOCOL["secondsPerPhase"] - time.monotonic() + if remaining > 0: time.sleep(remaining) + metric_rows.append(sample_metrics(fleet, "metrics-after-" + label)) + resources = resource_delta(metric_rows[0], metric_rows[-1]) + for left, right in zip(metric_rows, metric_rows[1:]): + delta = resource_delta(left, right) + for node in resources: + for name in ("cpuCores", "maxSampledRssBytes", "maxSampledFDs"): + resources[node][name] = max(resources[node][name], delta[node][name]) + statistics = summarize(samples) + proof = {**spec, "statistics": statistics, "resources": resources, "taskId": token, "repair": None, "noHeal": token is None} + if token: + repaired_pairs = {(target["key"], "null") for target in targets} + proof["responsibility"] = complete_outcome(terminal, bucket, expected, repaired_pairs) + proof["overlap"] = overlap(samples, windows) + require(repair_finished > repair_started, "repair throughput has no measured positive duration") + idle_fleet(fleet, binding) + fleet.stop() + repaired = repair_view(fleet.snapshot("physical-phase-after-" + label, quiescent=True), bucket) + same_hardware(hardware, repaired) + if token: + fleet.record("physical-proof-" + label, validate_repaired_physical(original, repaired, targets[0], repair_targets=targets)) + fleet.record("streaming-proof-" + label, verify_all_parts(fleet, cohorts, "part-batch-" + label)) + byte_count = sum(target["size"] for target in targets) + proof["repair"] = {"physicalVerified": True, "originalShardBytes": byte_count, + "seconds": repair_finished - repair_started, "MiBPerSecond": byte_count / 1024**2 / (repair_finished - repair_started), + "scope": "actual reconstructed target shard bytes; not logical S3 bytes"} + for registration in registrations: fleet.faults.remove(registration) + else: + require(physical_content(original) == physical_content(repaired), "no-heal baseline changed the repair fixture") + fleet.start() + fleet.record("load-functional-after-" + label, verify_seed(load_manifest(load_bucket), probe.inventory(load_bucket, unversioned_current_tags=True))) + if token: + # A complete outcome was already tied to the original token before + # physical verification. Later GETs cannot create its repair evidence. + require(status(probe, fleet, path, token, "retained-terminal-" + label).get("outcome") == terminal.get("outcome"), + "physical/workload checks changed the retained original outcome") + fleet.record("phase-result-" + label, proof) + return proof + + +def run(args, nodes): + directory = args.evidence / "H2" + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-h2" + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + load_bucket = bucket + "-load" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, resource_profile="cursor") + fleet.snapshot_buckets = [bucket, load_bucket] + candidate = json.loads(args.candidate.read_text()) + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, 5)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 1100, "maxInventoryBytes": 2 * 1024**3} + fleet.record("pre-registered-performance-protocol", copy.deepcopy(PROTOCOL)) + fleet.record("binding", binding) + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "performance executor omitted a source repetition") + probe, active = Probe(binding), [] + result = {"topology": "H2", "verdict": "RUNNING", "branches": {}, "phases": [], "cleanup": "PENDING"} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), "H2")) + expected = seed(Recorder(probe, directory / "repair-fixture"), bucket, "DX", OBJECTS, [SIZE]) + prepare_load(Recorder(probe, directory / "load-fixture"), load_bucket) + baseline = fleet.record("s3-repair-baseline", probe.inventory(bucket, unversioned_current_tags=True, page_size=127)) + fleet.record("repair-ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + load_baseline = fleet.record("s3-load-baseline", probe.inventory(load_bucket, unversioned_current_tags=True)) + fleet.record("load-ACK-manifest", verify_seed(load_manifest(load_bucket), load_baseline)) + fleet.stop() + hardware = repair_view(fleet.snapshot("physical-baseline", quiescent=True), bucket) + cohorts = map_cohorts(hardware, bucket, expected) + fleet.record("complete-streaming-baseline", verify_all_parts(fleet, cohorts, "baseline-part-batch")) + fleet.start() + for spec in schedule(): + result["phases"].append(run_phase(spec, fleet, probe, binding, bucket, expected, load_bucket, hardware, cohorts, active, directory)) + result["performanceAcceptance"] = acceptance(result["phases"]) + for endpoint in binding["endpoints"]: + peer = Probe(binding, endpoint) + actual = fleet.record("complete-repair-S3-after", {"endpoint": endpoint, **peer.inventory(bucket, unversioned_current_tags=True, page_size=127)}) + fleet.record("complete-repair-S3-proof", {"endpoint": endpoint, **verify_seed(expected, actual), **compare_inventories(baseline, actual)}) + load_actual = fleet.record("complete-load-S3-after", {"endpoint": endpoint, **peer.inventory(load_bucket, unversioned_current_tags=True)}) + fleet.record("complete-load-S3-proof", {"endpoint": endpoint, **verify_seed(load_manifest(load_bucket), load_actual), + **compare_inventories(load_baseline, load_actual)}) + for branch in BRANCHES: + result["branches"][branch] = {"verdict": "PASS", "phases": [row for row in result["phases"] if row["branch"] == branch], + "independentPerformanceAndFunctionalGates": True} + result["verdict"] = "PASS" + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + try: + fleet.stop() + fleet.snapshot("physical-failure", quiescent=True) + except Exception as error: result["failureEvidenceError"] = str(error) + finally: + for path, token in active: + try: fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: fleet.record("stop-error", {"error": str(exc)}) + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("topology-result", result) + return result + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f"Heal performance interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path, required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(nodes) == len(set(nodes)) and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) + for node in nodes), "four current Heal nodes required") + require(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")) and re.fullmatch(r"[0-9a-f]{64}", args.binary_sha), + "immutable workflow and candidate identities required") + result = run(args, nodes[:4]) + complete = result["verdict"] == "PASS" and result["cleanup"] == "PASS" + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": [result], "cleanup": result["cleanup"], "complete": complete}) + require(complete, "J06 requires all three full ABBA groups, independent performance/functional gates and cleanup") + + +if __name__ == "__main__": + try: main() + except Exception as exc: + print(f"Heal performance validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 8e8a4137..1674ebfc 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -28,6 +28,7 @@ case "${case_id}" in heal.root-graceful-coordinator-restart) driver=root_restart.py ;; heal.full-cluster-restart) driver=full_restart.py ;; heal.mrf-offline-write-replay) driver=mrf_replay.py ;; + heal.performance-abba) driver=performance.py ;; heal.disk-replacement-unformatted) driver=unformatted.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; diff --git a/test_validation_heal_mrf.py b/test_validation_heal_mrf.py index c0b9e1dc..75d5b507 100644 --- a/test_validation_heal_mrf.py +++ b/test_validation_heal_mrf.py @@ -5,6 +5,7 @@ from pathlib import Path import struct import tempfile +import types import unittest import uuid from unittest.mock import patch @@ -192,5 +193,38 @@ def test_candidate_contract_and_runtime_keep_other_repair_sources_disabled(self) self.assertEqual(select("E05")[0]["variants"], list(replay.BRANCHES)) self.assertIn(replay.CASE_ID, AUTOMATIC_CASES) + def test_replay_observer_uses_real_journal_interface_and_requires_exact_task_before_retirement(self): + obligation = self.obligation() + bindings = replay.pending_bindings({"node1": records.checkpoint(self.observed())}, [obligation], 1) + scene = {"node1": {"formats": ["original-format"], "files": []}} + queries = [] + def remote(node, action, **params): + self.assertEqual(action, "logs") + cursor = params.get("afterCursor", "baseline") + return {"complete": True, "beforeCursor": params.get("afterCursor"), "afterCursor": cursor, + "journal": "", "hasMore": False} + fleet = types.SimpleNamespace(nodes=["node1"], record=lambda label, value: value, remote=remote, + snapshot=lambda label: scene) + window = replay.JournalWindow(fleet, "MRF-rejoin") + def admin(path, query, body): + queries.append((path, query, body)) + self.assertEqual(query, [("clientToken", self.responsibility)]) + self.assertEqual(body, b"") + return {"httpStatus": 200, "response": self.receipt()} + task = {"node": "node1", "taskId": self.responsibility} + absent = {"node1": {"present": False, "intents": [], "allAllowlistedFilesAbsent": True}} + with patch.object(replay, "source_tasks", return_value=[task]), \ + patch.object(replay, "Probe", return_value=types.SimpleNamespace(admin=admin)), \ + patch.object(replay, "replica_census", return_value=[]), patch.object(replay, "read_checkpoints", return_value=absent): + proofs, retired = replay.wait_replay(fleet, window, {}, ["node1"], obligation["bucket"], [], [obligation], + bindings, None, scene, {}, []) + self.assertEqual(len(proofs), 1) + self.assertFalse(retired["queueDepthUsedAsProof"]) + self.assertEqual(len(queries), 1) + with patch.object(replay, "source_tasks", return_value=[]), patch.object(replay, "replica_census", return_value=[]), \ + patch.object(replay, "REPLAY_SECONDS", 0), patch.object(replay, "read_checkpoints", side_effect=AssertionError("no source proof")): + with self.assertRaisesRegex(ValueError, "lacks an exact verified source task"): + replay.wait_replay(fleet, window, {}, ["node1"], obligation["bucket"], [], [obligation], bindings, None, scene, {}, []) + if __name__ == "__main__": unittest.main() diff --git a/test_validation_heal_performance.py b/test_validation_heal_performance.py new file mode 100644 index 00000000..d4c6ec40 --- /dev/null +++ b/test_validation_heal_performance.py @@ -0,0 +1,192 @@ +import copy +import hashlib +import json +from pathlib import Path +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import performance as perf +from scripts.validation_heal import fleet_node as node +from scripts.validation_heal.matrix import select, AUTOMATIC_CASES + + +class PerformanceTests(unittest.TestCase): + def metrics(self, epoch=1): + return {name: {"identity": {field: name + field for field in node.PROCESS_FIELDS}, + "clockDomain": name + "bootId", "monotonicNs": epoch * 10**9, + "cpuTicks": epoch * 10, "clockTicksPerSecond": 100, "rssBytes": epoch * 100, + "fileDescriptors": 100, "io": {key: epoch * 100 for key in + ("rchar", "wchar", "syscr", "syscw", "read_bytes", "write_bytes")}, "readOnly": True} + for name in ("n0", "n1", "n2", "n3")} + + def rows(self): + return [{"sequence": index, "operation": "get_object" if index % 2 == 0 else "put_object", + "latencyMs": 10, "verdict": "PASS"} for index in range(120)] + + def phases(self): + return [{**spec, "statistics": perf.summarize(self.rows()), + "resources": perf.resource_delta(self.metrics(), self.metrics(2)), + "repair": {"physicalVerified": True, "MiBPerSecond": 1} if spec["mode"] == "B" else None, + "overlap": {"bothOperationsDuringRealRepair": True}, "noHeal": spec["mode"] == "A"} + for spec in perf.schedule()] + + def test_frozen_order_contains_every_source_repetition(self): + self.assertEqual(select(perf.CASE_ID)[0]["variants"], list(perf.BRANCHES)) + self.assertIn(perf.CASE_ID, AUTOMATIC_CASES) + self.assertEqual([phase["mode"] for phase in perf.schedule()], list("ABBAABBAABBA")) + with self.assertRaises(ValueError): perf.schedule({**perf.PROTOCOL, "maxErrorRate": 1}) + + def test_nearest_rank_percentiles_keep_the_slowest_request(self): + samples = list(range(1, 101)) + self.assertEqual(perf.percentile(samples, .95), 95) + self.assertEqual(perf.percentile(samples, .99), 99) + for bad in ([], [float("nan")], [-1], [float("inf")]): + with self.assertRaises(ValueError): perf.percentile(bad, .95) + + def test_failed_samples_stay_in_latency_and_error_denominators(self): + rows = self.rows() + for index in (1, 3): rows[index].update(verdict="FAIL", latencyMs=30000) + value = perf.summarize(rows) + self.assertEqual(value["put"]["requests"], 60) + self.assertEqual(value["failedRequests"], 2) + self.assertEqual(value["errorRate"], 2 / 120) + self.assertEqual(value["put"]["p99Ms"], 30000) + for bad in (rows[:-1], rows + [rows[0]], rows[:-1] + [rows[0]]): + with self.assertRaises(ValueError): perf.summarize(bad) + + def test_transport_unknown_is_durable_before_summary_and_cannot_pass(self): + with tempfile.TemporaryDirectory() as name: + root = Path(name) + probe = types.SimpleNamespace(s3=lambda *args: {"acknowledged": False, "resultUnknown": True, + "clientCallInvoked": True, "error": {"type": "Timeout"}}) + with patch.object(perf.time, "monotonic", return_value=1), patch.object(perf.time, "monotonic_ns", return_value=10**9): + row = perf.request(probe, "rv-heal-abcdef-load", 0, 1, root) + self.assertEqual(row["verdict"], "FAIL") + self.assertEqual(row["requestSha256"], hashlib.sha256((root / "00000-request.json").read_bytes()).hexdigest()) + self.assertTrue(json.loads((root / "00000-ack.json").read_text())["observation"]["resultUnknown"]) + + def test_missed_admission_is_a_failed_sample_without_a_late_network_write(self): + probe = types.SimpleNamespace(s3=lambda *args: self.fail("late request was sent")) + with tempfile.TemporaryDirectory() as name, patch.object(perf.time, "monotonic", return_value=33), \ + patch.object(perf.time, "monotonic_ns", return_value=33 * 10**9): + value = perf.request(probe, "rv-heal-abcdef-load", 1, 1, Path(name)) + self.assertEqual(value["verdict"], "FAIL") + self.assertFalse(value["observation"]["clientCallInvoked"]) + + def test_get_success_with_the_wrong_bytes_is_a_failure(self): + observed = {"acknowledged": True, "resultUnknown": False, "response": {"ResponseMetadata": {"HTTPStatusCode": 200}}, + "payload": {"size": 65536, "sha256": "wrong"}} + with tempfile.TemporaryDirectory() as name, patch.object(perf.time, "monotonic", return_value=1), \ + patch.object(perf.time, "monotonic_ns", return_value=10**9): + value = perf.request(types.SimpleNamespace(s3=lambda *args: observed), "rv-heal-abcdef-load", 0, 1, Path(name)) + self.assertEqual(value["verdict"], "FAIL") + + def test_resource_counters_require_all_peers_and_the_same_process_birth(self): + start, end = self.metrics(), self.metrics(3) + delta = perf.resource_delta(start, end) + self.assertEqual(delta["n0"]["cpuCores"], .1) + self.assertEqual(delta["n0"]["ioDelta"]["read_bytes"], 200) + for kind in ("missing", "pid", "boot", "binary", "mount", "clock", "backwards", "io", "mutable"): + with self.subTest(kind=kind): + bad = copy.deepcopy(end) + if kind == "missing": del bad["n1"] + elif kind in {"pid", "binary", "mount"}: + key = {"pid": "pid", "binary": "binarySha256", "mount": "mountNamespace"}[kind] + bad["n1"]["identity"][key] = "other" + elif kind == "boot": bad["n1"]["clockDomain"] = "another-boot" + elif kind == "clock": bad["n1"]["clockTicksPerSecond"] = 1000 + elif kind == "backwards": bad["n1"]["monotonicNs"] = 0 + elif kind == "io": bad["n1"]["io"]["write_bytes"] = 99 + else: bad["n1"]["readOnly"] = False + with self.assertRaises(ValueError): perf.resource_delta(start, bad) + + def windows(self): + return [{"callStartedMonotonicNs": index * 3 * 10**9, + "callFinishedMonotonicNs": (index * 3 + .1) * 10**9, + "response": {"summary": "running", "outcome": {"execution": {"state": "running"}, + "counters": {"processed": (index + 1) * 10, "healed": index, "unchanged": 10, + "skipped": 0, "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False}}}} + for index in range(3)] + + def bracketed(self): + return [{"operation": "get_object" if index % 2 == 0 else "put_object", "observation": { + "clientCallInvoked": True, "callStartedMonotonicNs": (index * .3 + .2) * 10**9, + "callFinishedMonotonicNs": (index * .3 + .3) * 10**9}} for index in range(4)] + + def test_true_partial_running_progress_must_bracket_both_workload_operations(self): + self.assertEqual(perf.overlap(self.bracketed(), self.windows())["bracketedCalls"], 4) + for kind in ("finished", "zero", "stalled", "regressed", "unknown", "put-only", "outside", "not-invoked", "few"): + with self.subTest(kind=kind): + rows, windows = self.bracketed(), self.windows() + if kind == "finished": + for row in windows: row["response"]["summary"] = "finished" + elif kind in {"zero", "stalled"}: + for row in windows: row["response"]["outcome"]["counters"]["processed"] = 0 if kind == "zero" else 10 + elif kind == "regressed": windows[-1]["response"]["outcome"]["counters"]["processed"] = 1 + elif kind == "unknown": windows[0]["response"]["outcome"]["counters"]["unknown"] = 1 + elif kind == "put-only": + for row in rows: row["operation"] = "put_object" + elif kind == "outside": + for row in rows: row["observation"]["callFinishedMonotonicNs"] = 99 * 10**9 + elif kind == "not-invoked": + for row in rows: row["observation"]["clientCallInvoked"] = False + else: rows.pop() + with self.assertRaises(ValueError): perf.overlap(rows, windows) + + def test_independent_performance_gates_reject_functional_success_with_slow_or_failed_calls(self): + self.assertEqual(len(perf.acceptance(self.phases())["pairedComparisons"]), 12) + for kind in ("omit", "reorder", "error", "absolute", "paired", "no-physical", "no-overlap", "cpu", "rss", "fd", "baseline-heal"): + with self.subTest(kind=kind): + phases = self.phases() + if kind == "omit": phases.pop() + elif kind == "reorder": phases[0], phases[1] = phases[1], phases[0] + elif kind == "error": phases[1]["statistics"]["errorRate"] = 1 / 120 + elif kind == "absolute": phases[1]["statistics"]["get"]["p99Ms"] = 11000 + elif kind == "paired": phases[1]["statistics"]["get"]["p99Ms"] = 100 + elif kind == "no-physical": phases[1]["repair"]["physicalVerified"] = False + elif kind == "no-overlap": phases[1]["overlap"]["bothOperationsDuringRealRepair"] = False + elif kind in {"cpu", "rss", "fd"}: + key = {"cpu": "cpuCores", "rss": "maxSampledRssBytes", "fd": "maxSampledFDs"}[kind] + phases[1]["resources"]["n2"][key] = 10**12 + else: phases[0]["noHeal"] = False + with self.assertRaises(ValueError): perf.acceptance(phases) + + def test_phase_hardware_cannot_drift(self): + snapshot = {node: {"formats": [{"value": "disk-" + node}], "hostname": node, "binarySha256": "candidate"} + for node in ("n0", "n1", "n2", "n3")} + self.assertTrue(perf.same_hardware(snapshot, copy.deepcopy(snapshot))) + for key in ("formats", "hostname", "binarySha256"): + changed = copy.deepcopy(snapshot) + changed["n2"][key] = "new" + with self.assertRaises(ValueError): perf.same_hardware(snapshot, changed) + + def test_node_metrics_reject_path_or_pid_injection_before_reading_process(self): + for field in ("pid", "path", "command", "expected"): + with patch.object(node, "process", side_effect=AssertionError("must reject before reading")): + with self.assertRaises(ValueError): node.process_metrics({"runId": "gh-1-1-h2", "action": "process-metrics", field: "/etc/shadow"}) + + def test_node_metrics_attest_resources_and_reject_a_recycled_pid(self): + identity = {name: name for name in node.PROCESS_FIELDS} + identity.update(pid=123, startTicks=900) + fields = ["0"] * 22 + fields[0], fields[11], fields[12], fields[17], fields[19], fields[20], fields[21] = "S", "100", "50", "12", "900", "9000", "5" + proc = {"stat": "123 (rustfs worker) " + " ".join(fields), + "io": "\n".join(name + ": 100" for name in ("rchar", "wchar", "syscr", "syscw", "read_bytes", "write_bytes", "cancelled_write_bytes"))} + class Proc: + def __init__(self, name=""): self.name = name + def __truediv__(self, name): return Proc(name) + def read_text(self): return proc[self.name] + def iterdir(self): return iter(range(7)) + with patch.object(node, "Path", return_value=Proc()), patch.object(node, "process", return_value=identity), \ + patch.object(node.os, "sysconf", side_effect=lambda name: 100 if name == "SC_CLK_TCK" else 4096): + value = node.process_metrics({"runId": "gh-1-1-h2", "action": "process-metrics"}) + self.assertEqual(value["cpuTicks"], 150) + self.assertEqual(value["rssBytes"], 5 * 4096) + self.assertEqual(value["fileDescriptors"], 7) + with patch.object(node, "process", side_effect=[identity, {**identity, "startTicks": 901}]): + with self.assertRaises(ValueError): node.process_metrics({"runId": "gh-1-1-h2", "action": "process-metrics"}) + + +if __name__ == "__main__": unittest.main() From 38858c40f8976697e0d6bf8b52f13484dc5b25a3 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 05:29:05 +0800 Subject: [PATCH 080/343] fix(validation): refresh service identity across process exit races --- docs/validation-migration-map.md | 9 +++++++++ scripts/validation_heal/fleet_node.py | 17 ++++++++++++++++- test_validation_heal_performance.py | 14 ++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index e15be37f..f2837e38 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -939,3 +939,12 @@ positive verified repair throughput and bounded resources. All twelve phases and every paired difference are reported; no phase may be omitted or reordered and functional success cannot substitute for performance acceptance. J06 has no live PASS claim yet. + + +F05 `38087158034` passed the expanded physical census and reached the real +all-peer graceful exit, then retained FAIL/cleanup PASS when the original +process exited between systemd MainPID and `/proc/PID/stat` reads. The node +observer now refreshes the complete service observation up to three times +only for disappearing `/proc/PID` entries. It never infers PID zero from +absence, accepts arbitrary file loss, or relaxes exact normal exit code zero, +service identity and pidfd birth checks. A fresh run is required. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 77248fa7..ebf0ab37 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -260,7 +260,7 @@ def stop(request): "commandSha256", "environmentFileSha256", "settings") -def process(request): +def process_once(request): state, _, owner = owned(request["runId"]) fields = ("MainPID", "ActiveState", "SubState", "ExecMainCode", "ExecMainStatus", "Result") service = dict(line.split("=", 1) for line in command("systemctl", "show", owner["unit"], @@ -289,6 +289,21 @@ def process(request): return result +def process(request): + # systemd's MainPID can remain visible briefly after /proc disappears. + # Refresh the whole service snapshot; absence alone never proves exit or + # permits an invented PID=0. Exact exit code/result checks remain in the + # graceful/crash observer, and pidfd callers still compare process birth. + for attempt in range(3): + try: + return process_once(request) + except FileNotFoundError as exc: + if not isinstance(exc.filename, str) or not re.match(r"^/proc/[1-9][0-9]*/", exc.filename): + raise + require(attempt < 2, "owned process/service state did not settle after process exit") + time.sleep(.05) + + def pidfd_signal(request, signum): expected = request.get("expected") require(isinstance(expected, dict) and all(name in expected for name in PROCESS_FIELDS) diff --git a/test_validation_heal_performance.py b/test_validation_heal_performance.py index d4c6ec40..c0bfddee 100644 --- a/test_validation_heal_performance.py +++ b/test_validation_heal_performance.py @@ -188,5 +188,19 @@ def iterdir(self): return iter(range(7)) with patch.object(node, "process", side_effect=[identity, {**identity, "startTicks": 901}]): with self.assertRaises(ValueError): node.process_metrics({"runId": "gh-1-1-h2", "action": "process-metrics"}) + def test_exit_race_refreshes_systemd_and_never_infers_an_exit_from_proc_absence(self): + request = {"runId": "gh-1-1-h2", "action": "process"} + missing = FileNotFoundError(2, "process exited", "/proc/123/stat") + terminal = {"pid": 0, "service": {"ExecMainCode": "1", "ExecMainStatus": "0", "Result": "success"}} + with patch.object(node, "process_once", side_effect=[missing, terminal]) as reader, patch.object(node.time, "sleep"): + self.assertEqual(node.process(request), terminal) + self.assertEqual(reader.call_count, 2) + with patch.object(node, "process_once", side_effect=missing), patch.object(node.time, "sleep"): + with self.assertRaisesRegex(ValueError, "did not settle"): node.process(request) + wrong_path = FileNotFoundError(2, "candidate missing", "/var/lib/rustfs-auto-testing/heal/rustfs") + with patch.object(node, "process_once", side_effect=wrong_path) as reader: + with self.assertRaises(FileNotFoundError): node.process(request) + self.assertEqual(reader.call_count, 1) + if __name__ == "__main__": unittest.main() From e64136f40a922b434c96729eb538e01d0528d912 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 05:37:47 +0800 Subject: [PATCH 081/343] feat(validation): migrate selectors with genuine cross-node set isolation --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-heal-operations.md | 5 +- docs/validation-migration-map.md | 43 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 7 +- scripts/validation_heal/healthy.py | 25 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 1 + scripts/validation_heal/scopes.py | 349 ++++++++++++++++++ scripts/validation_heal/shards.py | 20 +- test_validation_heal_scopes.py | 187 ++++++++++ 11 files changed, 623 insertions(+), 22 deletions(-) create mode 100644 scripts/validation_heal/scopes.py create mode 100644 test_validation_heal_scopes.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 6caf8f4d..0c6cb8ba 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -22,6 +22,7 @@ on: - test_validation_heal_mrf.py - test_validation_heal_unformatted.py - test_validation_heal_performance.py + - test_validation_heal_scopes.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -57,6 +58,7 @@ on: - test_validation_heal_mrf.py - test_validation_heal_unformatted.py - test_validation_heal_performance.py + - test_validation_heal_scopes.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -88,7 +90,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 572354d6..9ca0b042 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -2,7 +2,7 @@ 此手册和 [84 项逐例操作卡](validation-heal-cases.md) 一起构成可执行的**人工操作方案 + 脚本辅助**。不是 84 个无人值守自动测试，也不是通过报告。操作卡列出 301 个分支模板；真实盘角色、阈值、损坏数量和拓扑仍需在每批 `parameterBindings` 展开，不能把模板数量当实测次数。 -公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 已接入 46 个 Heal 用例，完整清单通过 `matrix.py list` 查看，以 `automaticExecutor` 为准。入口使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 +公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 已接入 47 个 Heal 用例，完整清单通过 `matrix.py list` 查看，以 `automaticExecutor` 为准。入口使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 ## 1. 使用入口与交付物 @@ -640,3 +640,6 @@ audit检查完整性/路径/摘要，输出`EVIDENCE_COMPLETE_REVIEW_REQUIRED` J06 的自动入口为 `heal.performance-abba`。执行器在部署前冻结三组 ABBA、每阶段 30 秒、2 并发、4 请求/秒、GET/PUT 各半与完整阈值；每阶段重启同一服务并按固定次数预热。每个 B 阶段的真实部分修复窗口必须包围 GET 和 PUT，否则整例不能 PASS。A 阶段持续检查完整 idle 与物理不变。所有排定请求（包括失败、未知和错过 admission 的请求）均计入分母；GET/PUT 独立报告 P95/P99，三组配对差异、实际修复分片吞吐与服务 CPU/IO/RSS/FD 分别留证。阈值保存在 `scripts/validation_heal/performance.py` 的 `PROTOCOL`，修改阈值必须产生新的 harness commit 和新验证记录；此入口不缩短 J05 的 24h/72h 要求。 + + +A02 的自动入口为 `heal.admin-scope-isolation`。H2 的专用 `scope-isolation` runtime 使用当前四节点/十六盘，按盘序交织节点并设置 `RUSTFS_ERASURE_SET_DRIVE_COUNT=8`，要求真实两个 set 各跨全部四个节点。普通执行器仍只接受原单 set。两个桶的完整 ACK/实际物理 set 清单冻结后，root/bucket/prefix/object/pool-set 各自新部署、注入选中与控制故障。完整物理和 streaming 证明必须先于 GET 和控制备份恢复。H1 保留原四盘单 set，明确记录不存在未选 set 的适用边界；不能拿同盘目录冒充第二个 set。未限制选择器的原始 receipt 在该固定候选中允许 null，单独保存实际物理映射，显式 pool/set 则必须严格匹配原始字段。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index f2837e38..a613727b 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 120 currently dispatchable IDs include all 74 non-Heal cases and forty-six Heal cases. Forty-four have live PASS evidence; the other 114 have no live PASS claim. +case evidence. The 121 currently dispatchable IDs include all 74 non-Heal cases and forty-seven Heal cases. Forty-four have live PASS evidence; the other 114 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 38 Heal cases remain +The other 37 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -650,7 +650,7 @@ A08 `heal.admin-authorization` passed live in RustFS Actions run `38069648003`, harness `0789743d439fd91b967c2d6fba85c06e29f844c5`. All five source branches passed independently on H1 and H2; the canonical gate, complete summary, cleanup and workflow all passed with the fixed candidate. -The prior A08 failures remain recorded. Thirty-eight Heal executors remain +The prior A08 failures remain recorded. Thirty-seven Heal executors remain to be migrated; automatic dispatch and local tests do not claim live PASS. @@ -948,3 +948,40 @@ observer now refreshes the complete service observation up to three times only for disappearing `/proc/PID` entries. It never infers PID zero from absence, accepts arbitrary file loss, or relaxes exact normal exit code zero, service identity and pidfd birth checks. A fresh run is required. + + +B04 `38087657754` retained overall FAIL/cleanup PASS. H1 foreign-format passed +its complete structured refusal window, unchanged original format/controls, +rejected foreign bytes and all 128 ACK/S3 checks, with archive restoration +explicitly excluded from product repair. Both blank branches lacked a distinct +new candidate-generated disk UUID. H2 foreign-format did not make every bound +peer ready; a startup failure alone is not relabeled as structured safe refusal. + +A02 `heal.admin-scope-isolation` now dispatches all five selectors with fresh +H1/H2 fixtures, two owned buckets and exact `a/`, `ab/`, `b/`, nested, Unicode +and literal encoded-slash keys. The candidate-bound H2 runtime preserves the +current four nodes and sixteen independent mounts, explicitly configures two +8-drive sets and interleaves original endpoints so each set spans all four +hosts. Actual format UUIDs, complete set membership, metadata distribution and +same-role shard identities bind every ACK to its physical set. Existing +single-set executors retain their strict default and reject this layout unless +it is explicitly requested. No format, UUID or second control set is invented. +H1 retains its original four-drive set and records the physical applicability +boundary for a nonexistent unselected set. + +Root covers the entire owned service manifest; bucket, exact object and `a/` +prefix selectors preserve other buckets, adjacent keys/prefixes and control +faults; pool/set selectors require real selected-set reconstruction and an +unchanged missing shard in the other actual set. The complete responsibility +set and canonical counters must match all selected ACKs, including healthy +objects. On this exact candidate, unrestricted request selectors may remain +null in raw receipts; independent complete physical mapping resolves their +scope in separate evidence, without rewriting public fields or weakening +shared validators. Explicit selectors require exact integer receipt scope; +partial nulls, wrong sets, mixed incarnations, duplicates and unknowns fail. + +Full physical comparison and selected streaming integrity precede any target +GET or unselected-control backup rescue. Rescue cannot certify product repair. +Both complete S3 manifests through every peer and a final unchanged physical +scene are required. A02 has no live PASS claim yet. Ten scope/topology/receipt/ +control negatives join the focused local checks; 226 checks now pass. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 3fcb9471..3af723cd 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index ebf0ab37..5a6c0934 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -33,7 +33,9 @@ def runtime_config(profile): - require(profile in {"manual", "offline-replay", "durable-mrf", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target"}, "unsupported Heal runtime profile") + require(profile in {"manual", "offline-replay", "durable-mrf", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "scope-isolation"}, "unsupported Heal runtime profile") + if profile == "scope-isolation": + return {**BACKGROUND_CONFIG, "RUSTFS_ERASURE_SET_DRIVE_COUNT": "8"} if profile == "durable-mrf": return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} if profile in {"replacement", "replacement-target"}: @@ -170,6 +172,9 @@ def reserve(request): h1 = run_id.endswith("-h1") expected = local_paths if h1 else [f"http://rustfs-node{node}:9000{path}" for node in range(1, 5) for path in local_paths] + if profile == "scope-isolation": + require(not h1, "two-set runtime requires the four-node topology") + expected = [f"http://rustfs-node{node}:9000{path}" for path in local_paths for node in range(1, 5)] require(volumes == expected, "endpoint sequence differs from H1/H2 deployment contract") state, roots = paths(run_id) owner = {"runId": run_id, "unit": unit(run_id), "hostname": observed["hostname"], diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index c9c68b26..422ea595 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -87,24 +87,30 @@ def physical_content(snapshot): return {node: {key: value[key] for key in ("files", "formats")} for node, value in snapshot.items()} -def validate_topology(snapshot, topology): +def validate_topology(snapshot, topology, *, expected_sets=1): require(topology in {"H1", "H2"} and len(snapshot) == (1 if topology == "H1" else 4), "wrong node count for Heal topology") records = [(node, value) for node, entry in snapshot.items() for value in entry["formats"]] require(len(records) == (4 if topology == "H1" else 16), "wrong physical disk count") values = [item["value"] for _, item in records] sets = values[0]["xl"]["sets"] - require(len(sets) == 1 and len(sets[0]) == len(records) and len(set(sets[0])) == len(records), - "A01 requires one complete independently formatted erasure set") + require(type(expected_sets) is int and (expected_sets == 1 or topology == "H2" and expected_sets == 2), + "unsupported explicit set layout") + identities = [identity for group in sets for identity in group] + require(len(sets) == expected_sets and all(len(group) == len(records) // expected_sets for group in sets) + and len(identities) == len(set(identities)) == len(records), + "explicit topology lacks complete independently formatted erasure sets") require(all(value["format"] == "xl" and value["xl"]["sets"] == sets and value["id"] == values[0]["id"] for value in values) - and {value["xl"]["this"] for value in values} == set(sets[0]), + and {value["xl"]["this"] for value in values} == set(identities), "physical disk UUID, pool and set mapping differs") if topology == "H2": - require(all(len({item["value"]["xl"]["this"] for owner, item in records if owner == node}) == 4 - for node in snapshot), "H2 set does not span every node's four drives") + require(all(len({item["value"]["xl"]["this"] for owner, item in records if owner == node + and item["value"]["xl"]["this"] in group}) == 4 // expected_sets + for node in snapshot for group in sets), "each H2 set must span every original physical node") return {"topology": topology, "nodes": len(snapshot), "independentDrives": len(records), - "poolIndex": 0, "setIndex": 0, "diskUUIDs": sets[0]} + "poolIndex": 0, "setIndex": 0 if expected_sets == 1 else None, "diskUUIDs": identities, + "setCount": expected_sets, "actualSets": sets} def validate_healthy(status, bucket, key, disposition): @@ -131,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"manual", "offline-replay", "durable-mrf", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "offline-replay", "durable-mrf", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "scope-isolation"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -220,6 +226,9 @@ def deploy(self, binary_sha, bucket, access, secret, *, before_start=None): paths = checked[self.nodes[0]]["dataRoots"] volumes = paths if len(self.nodes) == 1 else [f"http://rustfs-node{index}:9000{path}" for index in range(1, 5) for path in paths] + if self.runtime_profile == "scope-isolation": + require(len(self.nodes) == 4, "two-set isolation requires the four-node topology") + volumes = [f"http://rustfs-node{index}:9000{path}" for path in paths for index in range(1, 5)] self.record("integrity-fleet-attestation", {"scope": "this run's private H1/H2 endpoint set", "allParticipatingNodesUseCandidate": True, "binarySha256": binary_sha, "nodes": self.nodes, "volumes": volumes, "mixedVersionsAllowed": False, diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 3e039d6f..b6b64266 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 1674ebfc..a64b1cef 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -29,6 +29,7 @@ case "${case_id}" in heal.full-cluster-restart) driver=full_restart.py ;; heal.mrf-offline-write-replay) driver=mrf_replay.py ;; heal.performance-abba) driver=performance.py ;; + heal.admin-scope-isolation) driver=scopes.py ;; heal.disk-replacement-unformatted) driver=unformatted.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; diff --git a/scripts/validation_heal/scopes.py b/scripts/validation_heal/scopes.py new file mode 100644 index 00000000..13d16dc3 --- /dev/null +++ b/scripts/validation_heal/scopes.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +"""A02: exact root/bucket/prefix/object and genuine erasure-set isolation.""" + +from __future__ import annotations + +import argparse +import copy +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +from urllib.parse import quote + +try: + from .heal_probe import Probe, compare_inventories, payload, require, save + from .heal_workload import Recorder, verify_seed + from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .heal_fault import bounded_fault_id + from .shards import files, map_target, validate_fault, validate_repaired_physical + from .versions import public_version_id + from .stop import idle_fleet +except ImportError: + from heal_probe import Probe, compare_inventories, payload, require, save + from heal_workload import Recorder, verify_seed + from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from heal_fault import bounded_fault_id + from shards import files, map_target, validate_fault, validate_repaired_physical + from versions import public_version_id + from stop import idle_fleet + + +CASE_ID = "heal.admin-scope-isolation" +BRANCHES = ("root", "bucket", "prefix", "object", "pool-set") +SIZE = 1024**2 + 73 +KEYS = ("a/target.bin", "a/target.bin-suffix", "a/child/neighbor.bin", "a/中文 空%2F.bin", + "a/literal/slash.bin", "a/literal%2Fslash.bin", "a/06.bin", "a/07.bin", + "ab/control.bin", "ab/09.bin", "ab/10.bin", "ab/11.bin", + "b/control.bin", "b/13.bin", "b/14.bin", "b/15.bin") +REFERENCE = {"candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", + "sourceFiles": {"crates/heal/src/heal/channel.rs": "7f669ca43b9559db9b97fba6b419a6c76a94d76acd8526db132d74b7c534d6b0", + "crates/heal/src/heal/task.rs": "81e8e2ec8807d1094095e444d0aaf8913f94114836bc1bc9f7953cbeff98d855", + "crates/ecstore/src/core/sets.rs": "06543b1d083f489346171ba489b1eb1e0b7899c844f0ebf15e16ad467cf5e2f1", + "crates/ecstore/src/layout/disks_layout.rs": "b143aa9fa97b2cd32f8a2cd548779e0656cab0f197988483ee8a92b5ab3f17f8"}, + "nullableIdentityScope": "unrestricted admitted selectors; exact ACK and physical set binding still required", + "h2": "four original nodes, two 8-drive sets, two original mounts per node per set, disk-major endpoints", + "h1": "original single-node four-drive set; no invented unselected erasure set"} + + +def seed_fixture(recorder, buckets, branch): + expected = {} + for bucket in buckets: + recorder.call("create_bucket", {"Bucket": bucket}) + expected[bucket] = [] + for key in KEYS: + body = payload(f"A02/{branch}/{bucket}/{key}", SIZE) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "Metadata": {"suite": "A02", "branch": branch}, "Tagging": "suite=heal&fixture=DS"}, body) + require(response.get("VersionId") in {None, "null"}, "scope fixture must remain never-versioned") + expected[bucket].append({"key": key, "versionId": "null", "deleteMarker": False, "isLatest": True, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + return expected + + +def object_mapping(snapshot, expected, topology): + validate_topology(snapshot, topology, expected_sets=2 if topology == "H2" else 1) + indexed = files(snapshot) + mapped = {} + for bucket, entries in expected.items(): + for index, entry in enumerate(entries): + identity = bucket, entry["key"] + scopes = set() + for node, observation in snapshot.items(): + for record in observation["formats"]: + if (node, record["disk"], f"{bucket}/{entry['key']}/xl.meta") not in indexed: continue + layout = record["value"]["xl"] + members = [number for number, group in enumerate(layout["sets"]) if layout["this"] in group] + require(len(members) == 1, "object member belongs to ambiguous erasure sets") + scopes.add(members[0]) + require(len(scopes) == 1, "object metadata lacks one exact physical set authority") + set_index = next(iter(scopes)) + mapped[identity] = map_target(snapshot, bucket, entry["key"], "data" if index % 2 == 0 else "parity", 1, + multi_set=topology == "H2", set_index=set_index) + require(len(mapped) == sum(len(entries) for entries in expected.values()), "physical scope manifest omitted an ACK") + if topology == "H2": require({target["setIndex"] for target in mapped.values()} == {0, 1}, "fixture did not populate both actual sets") + return mapped + + +def selection(branch, primary, mapped): + require(branch in BRANCHES and mapped, "invalid scope case or manifest") + selected = {pair for pair, target in mapped.items() if branch == "root" + or branch == "bucket" and pair[0] == primary + or branch == "prefix" and pair[0] == primary and pair[1].startswith("a/") + or branch == "object" and pair == (primary, "a/target.bin") + or branch == "pool-set" and target["poolIndex"] == 0 and target["setIndex"] == 0} + require(selected, "selector has no acknowledged object scope") + return selected + + +def request_spec(branch, primary): + options = {**BASE_OPTIONS, "recursive": branch != "object", "pool": None, "set": None} + path = "/rustfs/admin/v3/heal/" + if branch not in {"root", "pool-set"}: + path += quote(primary, safe="") + if branch == "prefix": path += "/" + quote("a/", safe="/") + elif branch == "object": path += "/" + quote("a/target.bin", safe="/") + if branch == "pool-set": options.update(pool=0, set=0) + require(branch in BRANCHES, "unknown selector branch") + return path, options + + +def fault_targets(primary, secondary, mapped, topology): + pairs = [(primary, key) for key in ("a/target.bin", "a/child/neighbor.bin", "ab/control.bin", "b/control.bin")] + pairs.append((secondary, "a/target.bin")) + if topology == "H2": + for number in (0, 1): + if not any(mapped[pair]["setIndex"] == number for pair in pairs): + pair = next(pair for pair, target in mapped.items() if target["setIndex"] == number) + pairs.append(pair) + require(len(pairs) == len(set(pairs)), "scope fault list reused an object") + return [mapped[pair] for pair in pairs] + + +def responsibility(current, mapped, selected, repaired, options): + outcome = current.get("outcome") or {} + require(current.get("summary") == "finished" and current.get("truncated", False) is False + and outcome.get("execution", {}).get("state") == "completed" and outcome.get("coverage") == "complete" + and outcome.get("objectsTruncated") is False, "scope task lacks complete traversal and diagnostics") + require(repaired <= selected <= set(mapped) and repaired, "scope repair plan lacks selected faults or escaped ACKs") + objects = outcome.get("objects") + require(isinstance(objects, list) and len(objects) == len(selected), "scope receipts omitted or added an object") + seen, incarnations, resolved = set(), {}, [] + for receipt in objects: + item = receipt.get("identity", {}) + pair = item.get("bucket"), item.get("object") + require(pair in selected and pair not in seen and item.get("kind") == "object" + and public_version_id(item.get("versionId")) == "null", "scope receipt has a wrong/duplicate ACK identity") + target = mapped[pair] + scope = item.get("poolIndex"), item.get("setIndex") + requested = options["pool"], options["set"] + if requested != (None, None): + require(all(type(value) is int for value in scope) and scope == requested == (target["poolIndex"], target["setIndex"]), + "selected pool/set receipt lost or changed explicit scope") + elif scope != (None, None): + require(all(type(value) is int for value in scope) and scope == (target["poolIndex"], target["setIndex"]), + "unrestricted receipt contradicts independently observed physical set") + incarnation = item.get("bucketIncarnationId") + require(isinstance(incarnation, str) and incarnation and incarnations.setdefault(pair[0], incarnation) == incarnation, + "scope receipts lack or mix bucket incarnations") + state = "repaired" if pair in repaired else "verified_healthy" + require(receipt.get("disposition", {}).get("state") == state, "scope receipt claimed unknown, false repair or incomplete integrity") + seen.add(pair) + resolved.append({"bucket": pair[0], "key": pair[1], "rawSelectorScope": list(scope), + "physicalPoolIndex": target["poolIndex"], "physicalSetIndex": target["setIndex"], + "originalSetDiskUUIDs": [member["diskUUID"] for member in target["cohort"]]}) + counts = {"processed": len(selected), "healed": len(repaired), "unchanged": len(selected) - len(repaired), + "failed": 0, "skipped": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False} + require(all(type(outcome.get("counters", {}).get(name)) is type(value) and outcome["counters"][name] == value + for name, value in counts.items()), "scope counters differ from exact physical/ACK selection") + return {"exactSelectedResponsibility": True, "rawIdentitiesNotRewritten": True, "physicalScopeBindings": resolved} + + +def physical_selection(original, repaired, targets, selected): + repairs = [target for target in targets if (target["bucket"], target["key"]) in selected] + controls = [target for target in targets if (target["bucket"], target["key"]) not in selected] + expected = copy.deepcopy(original) + missing = {(target["node"], target["disk"], target["relativePath"]) for target in controls} + actual = files(repaired) + for identity in missing: require(identity not in actual, "unselected control was repaired before the physical proof") + for node, value in expected.items(): + value["files"] = [entry for entry in value["files"] if (node, entry["disk"], entry["path"]) not in missing] + require(repairs, "physical isolation has no selected repair target") + proof = validate_repaired_physical(expected, repaired, repairs[0], repair_targets=repairs) + return {**proof, "selectedReconstructedParts": len(repairs), "unselectedStillMissingParts": len(controls), + "beforeAnyTargetGETOrBackupRescue": True} + + +def verify_parts(fleet, mapped, label): + members = [member for target in mapped.values() for member in target["cohort"]] + identities = {(member["node"], member["disk"], member["relativePath"]) for member in members} + require(len(identities) == len(members) and members, "part manifest duplicated or omitted a physical identity") + for node in fleet.nodes: + local = [member for member in members if member["node"] == node] + for offset in range(0, len(local), 128): + batch = local[offset:offset + 128] + raw = fleet.record(label, {"node": node, **fleet.remote(node, "verify-parts", parts=[{ + name: member[name] for name in ("disk", "relativePath", "versionId", "partNumber")} for member in batch])}) + require(raw.get("complete") is True and len(raw.get("parts", [])) == len(batch), "scope streaming verifier omitted a shard") + for member, proof in zip(batch, raw["parts"], strict=True): + require(proof.get("physicalBitrotVerified") is True and proof.get("physicalSize") == member["size"] + and all(proof.get(name) == member[name] for name in ("disk", "relativePath", "versionId", "partNumber", "sha256")), + "scope part failed exact identity/hash/streaming verification") + return {"exactStreamingParts": len(members), "exactObjects": len(mapped)} + + +def run_branch(args, topology, nodes, branch): + directory = args.evidence / topology / branch + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="scope-isolation" if topology == "H2" else "manual") + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + buckets = [bucket, bucket + "-other"] + fleet.snapshot_buckets = buckets + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == REFERENCE["candidateCommit"], "scope/nullable selector contract is not reviewed for this candidate") + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), "maxInventoryObjects": 128, + "maxInventoryBytes": 256 * 1024**2} + fleet.record("source-bound-scope-contract", REFERENCE) + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + fleet.record("binding", binding) + probe, active = Probe(binding), [] + result = {"topology": topology, "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + expected = seed_fixture(Recorder(probe, directory / "fixture"), buckets, branch) + baselines = {name: fleet.record("s3-baseline", probe.inventory(name, unversioned_current_tags=True)) for name in buckets} + for name in buckets: fleet.record("ACK-manifest", {"bucket": name, "expected": expected[name], **verify_seed(expected[name], baselines[name])}) + fleet.stop() + original = fleet.snapshot("physical-baseline", quiescent=True) + mapped = object_mapping(original, expected, topology) + fleet.record("complete-physical-object-mapping", [{"bucket": pair[0], "key": pair[1], "pool": target["poolIndex"], + "set": target["setIndex"], "members": [{key: member[key] for key in ("node", "disk", "diskUUID", "setSlot", "index")} + for member in target["cohort"]]} for pair, target in mapped.items()]) + verify_parts(fleet, mapped, "streaming-baseline") + selected = selection(branch, bucket, mapped) + targets = fault_targets(*buckets, mapped, topology) + selected_faults = {(target["bucket"], target["key"]) for target in targets} & selected + path, options = request_spec(branch, bucket) + fleet.record("exact-selector-plan", {"branch": branch, "path": path, "options": options, + "selectedACKs": [list(pair) for pair in sorted(selected)], "selectedFaults": [list(pair) for pair in sorted(selected_faults)], + "unselectedSetControl": "actual set 1" if topology == "H2" else "NOT_APPLICABLE: source H1 has one original four-drive set"}) + prior, registrations = original, [] + for index, target in enumerate(targets): + params = {"disk": target["disk"], "faultId": bounded_fault_id("A02", branch, index)} + fleet.faults.append((target["node"], params)) + registrations.append((target["node"], params, target)) + raw = fleet.record("fault", fleet.remote(target["node"], "apply-fault", **params, + relativePath=target["relativePath"], expectedSha256=target["sha256"], kind="missing")) + scene = fleet.snapshot("physical-after-fault", quiescent=True) + validate_fault(prior, scene, target, raw) + prior = scene + fleet.start() + require(physical_content(prior) == physical_content(fleet.snapshot("physical-before-trigger")), + "ATTRIBUTION_NOT_OBSERVED: selector fault changed before admission") + start = fleet.record("selector-admission", probe.admin(path, [], json.dumps(options).encode())) + require(start.get("httpStatus") == 200 and isinstance(start.get("response"), dict), "selector admission failed") + token = start["response"].get("clientToken") + require(isinstance(token, str) and token, "selector admission omitted original token") + active.append((path, token)) + deadline = time.monotonic() + 180 + while True: + raw = fleet.record("selector-status", probe.admin(path, [("clientToken", token)], b"")) + require(raw.get("httpStatus") == 200 and isinstance(raw.get("response"), dict), "original selector task status failed") + current = raw["response"] + if current.get("summary") != "running": break + require(time.monotonic() < deadline, "scope task exceeded registered deadline") + time.sleep(.2) + active.remove((path, token)) + outcome_proof = responsibility(current, mapped, selected, selected_faults, options) + idle_fleet(fleet, binding) + fleet.stop() + repaired = fleet.snapshot("physical-after-selector-before-GET", quiescent=True) + physical_proof = physical_selection(original, repaired, targets, selected) + streaming_proof = verify_parts(fleet, {pair: mapped[pair] for pair in selected}, "selected-streaming-proof") + # The unselected control backups are recovered only after preserving + # the complete scope isolation proof, and never count as task repair. + for node, params, target in registrations: + if (target["bucket"], target["key"]) not in selected: + raw = fleet.record("unselected-control-backup-rescue", fleet.remote(node, "recover-fault", **params)) + require(raw.get("physicalRepairPassAllowed") is False, "control restoration claimed product repair") + fleet.faults.remove((node, params)) + recovered = fleet.snapshot("physical-after-control-rescue", quiescent=True) + validate_repaired_physical(original, recovered, targets[0], repair_targets=targets) + verify_parts(fleet, mapped, "complete-streaming-after") + fleet.start() + for endpoint in binding["endpoints"]: + for name in buckets: + actual = fleet.record("all-peer-S3-after", {"endpoint": endpoint, **Probe(binding, endpoint).inventory(name, unversioned_current_tags=True)}) + fleet.record("full-ACK-S3-proof", {"endpoint": endpoint, **verify_seed(expected[name], actual), **compare_inventories(baselines[name], actual)}) + fleet.stop() + require(physical_content(recovered) == physical_content(fleet.snapshot("physical-after-GET", quiescent=True)), "S3 verification changed the proven selector result") + result.update(verdict="PASS", taskId=token, responsibility=outcome_proof, physical=physical_proof, streaming=streaming_proof, + controlBackupRescueExcludedFromRepair=True) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + try: + fleet.stop() + fleet.snapshot("physical-failure", quiescent=True) + except Exception as error: result["failureEvidenceError"] = str(error) + finally: + for path, token in active: + try: fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: fleet.record("stop-error", {"error": str(exc)}) + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("branch-result", result) + return result + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f"Heal scope interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path, required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(nodes) == len(set(nodes)) and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") + require(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")) and re.fullmatch(r"[0-9a-f]{64}", args.binary_sha), "immutable workflow/candidate identities required") + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "scope executor omitted a source selector") + results = [] + try: + for topology, selected_nodes in (("H1", nodes[:1]), ("H2", nodes[:4])): + for branch in BRANCHES: + results.append(run_branch(args, topology, selected_nodes, branch)) + if results[-1]["cleanup"] != "PASS": break + if results[-1]["cleanup"] != "PASS": break + finally: + complete = len(results) == 10 and all(row["verdict"] == "PASS" and row["cleanup"] == "PASS" for row in results) + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, + "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL", "complete": complete}) + require(complete, "A02 requires every real selector on H1/H2, exact physical scope isolation and cleanup") + + +if __name__ == "__main__": + try: main() + except Exception as exc: + print(f"Heal scope validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index eee17e2e..41498711 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -96,14 +96,22 @@ def files(snapshot): return entries -def map_target(snapshot, bucket, key, role, part_number, version_id="null", *, preferred_nodes=None, latest_required=True): - validate_topology(snapshot, "H1" if len(snapshot) == 1 else "H2") +def map_target(snapshot, bucket, key, role, part_number, version_id="null", *, preferred_nodes=None, latest_required=True, + multi_set=False, set_index=0): + require(type(multi_set) is bool and type(set_index) is int and 0 <= set_index < (2 if multi_set else 1), + "invalid explicit object set mapping") + validate_topology(snapshot, "H1" if len(snapshot) == 1 else "H2", expected_sets=2 if multi_set else 1) indexed = files(snapshot) cohort, logical = [], None for node, observation in snapshot.items(): for format_entry in observation["formats"]: disk = format_entry["disk"] identity = (node, disk, f"{bucket}/{key}/xl.meta") + layout = format_entry["value"]["xl"] + if layout["this"] not in layout["sets"][set_index]: + require(not any(owner == node and drive == disk and path.startswith(f"{bucket}/{key}/") + for owner, drive, path in indexed), "object files span the selected and unselected erasure sets") + continue metadata = indexed.get(identity, {}).get("metadata", {}) require(metadata.get("bucket") == bucket and metadata.get("key") == key, "missing authoritative metadata identity on a baseline drive") @@ -120,9 +128,9 @@ def map_target(snapshot, bucket, key, role, part_number, version_id="null", *, p distribution = erasure["distribution"] require(type(data) is int and type(parity) is int and data > 0 and parity > 0 and data + parity == len(distribution) + and data + parity == len(layout["sets"][set_index]) and sorted(distribution) == list(range(1, data + parity + 1)), "invalid erasure geometry") - layout = format_entry["value"]["xl"] - slot = layout["sets"][0].index(layout["this"]) + slot = layout["sets"][set_index].index(layout["this"]) require(erasure["index"] == distribution[slot], "shard role does not match the disk's actual set slot") actual_role = "data" if erasure["index"] <= data else "parity" require(version.get("role") == actual_role, "decoded shard role differs from erasure index") @@ -143,14 +151,14 @@ def map_target(snapshot, bucket, key, role, part_number, version_id="null", *, p "dataDir": data_dir, "diskUUID": layout["this"], "setSlot": slot, "sha256": entry["sha256"], "size": entry["size"], "metadata": metadata, "baselineMetadataSha256": indexed[identity]["sha256"]}) - require(len(cohort) == logical["erasure"]["data_blocks"] + logical["erasure"]["parity_blocks"] + require(logical is not None and len(cohort) == logical["erasure"]["data_blocks"] + logical["erasure"]["parity_blocks"] and len({entry["index"] for entry in cohort}) == len(cohort) and len({entry["sha256"] for entry in cohort}) == len(cohort), "independent full-set shard identity or nonrepeating content is missing") target = next((entry for entry in cohort if entry["role"] == role and (preferred_nodes is None or entry["node"] in preferred_nodes)), None) require(target is not None, "requested shard role is not present") - return {**target, "bucket": bucket, "key": key, "poolIndex": 0, "setIndex": 0, + return {**target, "bucket": bucket, "key": key, "poolIndex": 0, "setIndex": set_index, "cohort": cohort, "logical": logical} diff --git a/test_validation_heal_scopes.py b/test_validation_heal_scopes.py new file mode 100644 index 00000000..d55d42ce --- /dev/null +++ b/test_validation_heal_scopes.py @@ -0,0 +1,187 @@ +import copy +from pathlib import Path +import tempfile +import unittest +import uuid +from unittest.mock import patch + +from scripts.validation_heal import scopes +from scripts.validation_heal.healthy import validate_topology, Fleet +from scripts.validation_heal.fleet_node import runtime_config +from scripts.validation_heal.shards import map_target, files +from scripts.validation_heal.matrix import select, AUTOMATIC_CASES + + +class ScopeTests(unittest.TestCase): + primary, secondary = "rv-heal-0123456789abcdef", "rv-heal-0123456789abcdef-other" + + def snapshot(self): + sets = [[f"disk-{group}-{slot}" for slot in range(8)] for group in range(2)] + snapshot = {f"node{node + 1}": {"formats": [], "files": []} for node in range(4)} + objects = [(self.primary, "a/target.bin", 0), (self.primary, "a/child/neighbor.bin", 1), + (self.primary, "ab/control.bin", 1), (self.primary, "b/control.bin", 0), + (self.secondary, "a/target.bin", 1)] + for node, observation in enumerate(snapshot.values()): + for disk in range(4): + group, slot = disk // 2, (disk % 2) * 4 + node + observation["formats"].append({"disk": disk, "value": {"format": "xl", "id": "pool", + "xl": {"sets": sets, "this": sets[group][slot]}}}) + for index, (bucket, key, selected_set) in enumerate(objects): + if selected_set != group: continue + data_dir = str(uuid.UUID(int=index + 1)) + metadata = {"bucket": bucket, "key": key, "versions": [{"versionId": None, "isLatest": True, + "deleted": False, "dataDir": data_dir, "size": scopes.SIZE, "inline": None, "usesLegacyChecksum": False, + "role": "data" if slot < 4 else "parity", "parts": [{"number": 1, "size": scopes.SIZE, "actualSize": scopes.SIZE}], + "erasure": {"index": slot + 1, "data_blocks": 4, "parity_blocks": 4, "block_size": 1048576, + "distribution": list(range(1, 9)), "algorithm": "rs-vandermonde"}}]} + observation["files"].extend([ + {"disk": disk, "path": f"{bucket}/{key}/xl.meta", "size": 600, "sha256": f"meta-{index}-{slot}", "metadata": metadata}, + {"disk": disk, "path": f"{bucket}/{key}/{data_dir}/part.1", "size": 400, "sha256": f"part-{index}-{slot}"}]) + return snapshot + + def expected(self): + return {self.primary: [{"key": key} for key in ("a/target.bin", "a/child/neighbor.bin", "ab/control.bin", "b/control.bin")], + self.secondary: [{"key": "a/target.bin"}]} + + def mapping(self): + return scopes.object_mapping(self.snapshot(), self.expected(), "H2") + + def receipt(self, branch): + mapped = self.mapping() + selected = scopes.selection(branch, self.primary, mapped) + _, options = scopes.request_spec(branch, self.primary) + objects = [{"identity": {"kind": "object", "bucket": bucket, "object": key, "versionId": "null", + "poolIndex": options["pool"], "setIndex": options["set"], "bucketIncarnationId": "incarnation-" + bucket}, + "disposition": {"state": "repaired"}} for bucket, key in sorted(selected)] + counts = {"processed": len(selected), "healed": len(selected), "unchanged": 0, + "failed": 0, "skipped": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False} + current = {"summary": "finished", "outcome": {"execution": {"state": "completed"}, "coverage": "complete", + "objectsTruncated": False, "objects": objects, "counters": counts}} + return current, mapped, selected, options + + def test_five_source_branches_and_real_two_set_runtime_are_retained(self): + self.assertEqual(select(scopes.CASE_ID)[0]["variants"], list(scopes.BRANCHES)) + self.assertIn(scopes.CASE_ID, AUTOMATIC_CASES) + config = runtime_config("scope-isolation") + self.assertEqual(config["RUSTFS_ERASURE_SET_DRIVE_COUNT"], "8") + self.assertEqual(config["RUSTFS_SCANNER_ENABLED"], "false") + self.assertEqual(config["RUSTFS_HEAL_AUTO_HEAL_ENABLE"], "false") + self.assertEqual(config["RUSTFS_HEAL_MRF_ENABLE"], "false") + + def test_existing_single_set_oracle_rejects_a_multi_set_baseline_without_explicit_binding(self): + snapshot = self.snapshot() + with self.assertRaises(ValueError): validate_topology(snapshot, "H2") + with self.assertRaises(ValueError): map_target(snapshot, self.primary, "a/target.bin", "data", 1) + proof = validate_topology(snapshot, "H2", expected_sets=2) + self.assertEqual(proof["setCount"], 2) + self.assertEqual(len(proof["diskUUIDs"]), 16) + for group in proof["actualSets"]: self.assertEqual(len(group), 8) + + def test_every_set_must_span_all_four_original_hosts(self): + snapshot = self.snapshot() + records = [record for observation in snapshot.values() for record in observation["formats"]] + sets = [[record["value"]["xl"]["this"] for record in records[:8]], + [record["value"]["xl"]["this"] for record in records[8:]]] + for record in records: record["value"]["xl"]["sets"] = sets + with self.assertRaisesRegex(ValueError, "span every"): validate_topology(snapshot, "H2", expected_sets=2) + + def test_mapping_uses_actual_members_and_role_distribution_without_projecting_format_values(self): + snapshot = self.snapshot() + before = copy.deepcopy(snapshot) + mapped = scopes.object_mapping(snapshot, self.expected(), "H2") + self.assertEqual(snapshot, before) + self.assertEqual(mapped[(self.primary, "a/target.bin")]["setIndex"], 0) + other = mapped[(self.primary, "ab/control.bin")] + self.assertEqual(other["setIndex"], 1) + self.assertEqual({member["node"] for member in other["cohort"]}, set(snapshot)) + self.assertEqual(len(other["cohort"]), 8) + self.assertTrue(all(member["setSlot"] + 1 == member["index"] for member in other["cohort"])) + self.assertEqual(len(scopes.fault_targets(self.primary, self.secondary, mapped, "H2")), 5) + with self.assertRaises(ValueError): map_target(snapshot, self.primary, "ab/control.bin", "data", 1, multi_set=True, set_index=0) + + def test_orphan_payload_on_an_unselected_set_is_not_a_complete_baseline(self): + snapshot = self.snapshot() + part = next(entry for entry in snapshot["node1"]["files"] if "a/target.bin/" in entry["path"] and entry["path"].endswith("part.1")) + snapshot["node1"]["files"].append({**part, "disk": 2}) + with self.assertRaisesRegex(ValueError, "selected and unselected"): + scopes.object_mapping(snapshot, self.expected(), "H2") + + def test_each_selector_uses_exact_bucket_key_prefix_and_physical_set(self): + mapped = self.mapping() + self.assertEqual(len(scopes.selection("root", self.primary, mapped)), 5) + self.assertEqual(len(scopes.selection("bucket", self.primary, mapped)), 4) + prefix = scopes.selection("prefix", self.primary, mapped) + self.assertEqual(prefix, {(self.primary, "a/target.bin"), (self.primary, "a/child/neighbor.bin")}) + self.assertEqual(scopes.selection("object", self.primary, mapped), {(self.primary, "a/target.bin")}) + self.assertEqual(scopes.selection("pool-set", self.primary, mapped), {(self.primary, "a/target.bin"), (self.primary, "b/control.bin")}) + self.assertFalse(scopes.request_spec("object", self.primary)[1]["recursive"]) + self.assertIsNone(scopes.request_spec("root", self.primary)[1]["pool"]) + self.assertEqual(scopes.request_spec("pool-set", self.primary)[1]["set"], 0) + + def test_nullable_unrestricted_receipts_keep_raw_fields_and_independent_physical_bindings(self): + current, mapped, selected, options = self.receipt("root") + original = copy.deepcopy(current) + proof = scopes.responsibility(current, mapped, selected, selected, options) + self.assertEqual(current, original) + self.assertTrue(proof["rawIdentitiesNotRewritten"]) + self.assertEqual({row["physicalSetIndex"] for row in proof["physicalScopeBindings"]}, {0, 1}) + self.assertTrue(all(row["rawSelectorScope"] == [None, None] for row in proof["physicalScopeBindings"])) + item = current["outcome"]["objects"][0]["identity"] + item.update(poolIndex=0, setIndex=mapped[(item["bucket"], item["object"])]["setIndex"]) + scopes.responsibility(current, mapped, selected, selected, options) + + def test_explicit_scope_nulls_wrong_sets_extra_objects_unknowns_and_false_counts_fail(self): + for kind in ("null", "set", "bool", "missing", "duplicate", "kind", "version", "incarnation", "unknown", "counter", "coverage"): + with self.subTest(kind=kind): + current, mapped, selected, options = self.receipt("pool-set") + item = current["outcome"]["objects"][0] + if kind == "null": item["identity"].update(poolIndex=None, setIndex=None) + elif kind == "set": item["identity"]["setIndex"] = 1 + elif kind == "bool": item["identity"]["setIndex"] = False + elif kind == "missing": current["outcome"]["objects"].pop() + elif kind == "duplicate": current["outcome"]["objects"][-1] = copy.deepcopy(item) + elif kind in {"kind", "version", "incarnation"}: + item["identity"][{"kind": "kind", "version": "versionId", "incarnation": "bucketIncarnationId"}[kind]] = "" if kind == "incarnation" else "foreign" + elif kind == "unknown": item["disposition"]["state"] = "unknown" + elif kind == "counter": current["outcome"]["counters"]["healed"] += 1 + else: current["outcome"]["coverage"] = "partial" + with self.assertRaises(ValueError): scopes.responsibility(current, mapped, selected, selected, options) + current, mapped, selected, options = self.receipt("root") + current["outcome"]["objects"][0]["identity"].update(poolIndex=None, setIndex=0) + with self.assertRaises(ValueError): scopes.responsibility(current, mapped, selected, selected, options) + + def test_outside_control_faults_must_still_be_missing_before_backup_rescue(self): + original = self.snapshot() + mapped = self.mapping() + targets = list(mapped.values()) + selected = scopes.selection("prefix", self.primary, mapped) + after = copy.deepcopy(original) + removed = {(target["node"], target["disk"], target["relativePath"]) for target in targets if (target["bucket"], target["key"]) not in selected} + for name, observation in after.items(): + observation["files"] = [entry for entry in observation["files"] if (name, entry["disk"], entry["path"]) not in removed] + proof = scopes.physical_selection(original, after, targets, selected) + self.assertEqual(proof["selectedReconstructedParts"], 2) + self.assertEqual(proof["unselectedStillMissingParts"], 3) + self.assertTrue(proof["beforeAnyTargetGETOrBackupRescue"]) + wrong = copy.deepcopy(after) + identity = next(iter(removed)) + wrong[identity[0]]["files"].append(files(original)[identity]) + with self.assertRaisesRegex(ValueError, "unselected control was repaired"): + scopes.physical_selection(original, wrong, targets, selected) + bad = copy.deepcopy(after) + metadata = next(entry for entry in bad["node1"]["files"] if "ab/control.bin/xl.meta" in entry["path"]) + metadata["sha256"] = "changed" + with self.assertRaises(ValueError): scopes.physical_selection(original, bad, targets, selected) + + def test_scope_adapter_uploads_a_compilable_complete_node_program(self): + import types + def invoke(argv, **params): + compile(params["input"], "remote-scoped-node", "exec") + return types.SimpleNamespace(returncode=0, stdout='{"checked":true}', stderr="") + with tempfile.TemporaryDirectory() as name, patch("scripts.validation_heal.healthy.subprocess.run", side_effect=invoke): + fleet = Fleet(["n0", "n1", "n2", "n3"], Path(name), "gh-1234-1-h2", runtime_profile="scope-isolation") + self.assertTrue(fleet.remote("n0", "process-metrics")["checked"]) + self.assertTrue(fleet.remote("n0", "start")["checked"]) + + +if __name__ == "__main__": unittest.main() From b1c7c1a764fff85675eac7ba727f2d4f5a77d95f Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 05:54:25 +0800 Subject: [PATCH 082/343] feat(validation): migrate isolated bucket metadata recovery checks --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 37 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/bucket_metadata.py | 337 ++++++++++++++++++ scripts/validation_heal/bucket_records.py | 184 ++++++++++ scripts/validation_heal/fleet_node.py | 26 ++ scripts/validation_heal/healthy.py | 11 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 1 + test_validation_heal_bucket_metadata.py | 153 ++++++++ 10 files changed, 750 insertions(+), 7 deletions(-) create mode 100644 scripts/validation_heal/bucket_metadata.py create mode 100644 scripts/validation_heal/bucket_records.py create mode 100644 test_validation_heal_bucket_metadata.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 0c6cb8ba..b9bb1fab 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -23,6 +23,7 @@ on: - test_validation_heal_unformatted.py - test_validation_heal_performance.py - test_validation_heal_scopes.py + - test_validation_heal_bucket_metadata.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -59,6 +60,7 @@ on: - test_validation_heal_unformatted.py - test_validation_heal_performance.py - test_validation_heal_scopes.py + - test_validation_heal_bucket_metadata.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -90,7 +92,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index a613727b..4da2144e 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 121 currently dispatchable IDs include all 74 non-Heal cases and forty-seven Heal cases. Forty-four have live PASS evidence; the other 114 have no live PASS claim. +case evidence. The 122 currently dispatchable IDs include all 74 non-Heal cases and forty-eight Heal cases. Forty-four have live PASS evidence; the other 114 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 37 Heal cases remain +The other 36 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -985,3 +985,36 @@ GET or unselected-control backup rescue. Rescue cannot certify product repair. Both complete S3 manifests through every peer and a final unchanged physical scene are required. A02 has no live PASS claim yet. Ten scope/topology/receipt/ control negatives join the focused local checks; 226 checks now pass. + + +D11 now dispatches all four source branches separately on H1/H2 (eight fresh +instances). Its dedicated node backend derives only two owned bucket-config +object paths and their incarnation controls. It never widens the general +user-object fault allowlist, walks IAM records, or prints bucket-policy/inline +payload bytes from physical metadata. A fresh namespace with no persisted IAM +subtree is required before and after execution. Exact system object geometry +binds each shard index to the original disk UUID/set slot; current inline or +external bytes require candidate-bound streaming verification. + +The fixture preserves Enabled versioning, bucket-only policy, tags, an Object +Lock default set after fixture writes, and all acknowledged historical business +versions. One config-envelope replica is missing or truncated for positive +recovery; the candidate's startup/config reads and explicit bucket Heal are +recorded as allowed mechanisms. Every original system file must be restored +byte-for-byte before backup rescue or business GET, while incarnation/control +records, business files and formats remain unchanged. The unknown-record +negative changes only outer XL2 major 1 to 2 on the exact primary config +replicas; it claims no inner msgpack schema coverage. It requires structured +public refusal, zero successful repair responsibility, preserved unknown +bytes and healthy control-bucket APIs. A startup/readiness failure alone is +not safe-refusal evidence. Rescue archives remain separate from product +repair, and an unrecognized config rewrite retains the stopped namespace. +D11 still requires live evidence. + +F02 corrected-budget run `38088240693` retained FAIL/cleanup PASS in both H2 +DB/DV branches: neither reached partial-rebuild interruption because the +candidate did not expose a distinct new replacement UUID. F03 corrected-budget +run `38088777413` also retained FAIL/cleanup PASS in both H2 DB/DV branches: +replacement source status was partial, unknown or incompatible. The bounded +file census no longer prevented either run; these product/evidence failures +are not converted into successful restart coverage. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 3af723cd..2120a3f5 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/bucket_metadata.py b/scripts/validation_heal/bucket_metadata.py new file mode 100644 index 00000000..f63fcace --- /dev/null +++ b/scripts/validation_heal/bucket_metadata.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +"""D11: bucket-config recovery and unknown-envelope refusal on H1/H2.""" + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +from urllib.parse import quote + +try: + from .heal_probe import Probe, compare_inventories, payload, require, save + from .heal_workload import Recorder, verify_seed + from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .bucket_records import bucket_path + from .versions import validate_version_receipts + from .stop import idle_fleet +except ImportError: + from heal_probe import Probe, compare_inventories, payload, require, save + from heal_workload import Recorder, verify_seed + from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from bucket_records import bucket_path + from versions import validate_version_receipts + from stop import idle_fleet + + +CASE_ID = "heal.bucket-metadata-integrity" +BRANCHES = ("versioning-config", "bucket-policy-tags", "lock-config", "unknown-record-negative") +REFERENCE = {"candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", "sourceFiles": { + "crates/ecstore/src/bucket/metadata.rs": "3b4f9ed4dba259f259653ba4bf1568c54a10b71ff0f6111135394e225a38d990", + "crates/filemeta/src/filemeta/codec.rs": "ef82b3390b0a377e423f99fb384c6a8a96beaf5722b0c907852cc0c904c4f04d"}, + "physicalRecord": ".rustfs.sys/buckets//.metadata.bin/xl.meta", + "representation": "system object XL2 envelope, not a raw JSON or raw inner bucket-msgpack file", + "unknownNegative": "outer XL2 major 1 -> 2; all other bytes retained; decoder rejects before checksum", + "repairSourcesAllowed": ["candidate startup/config read", "explicit bucket Heal"], + "credentialBoundary": "fresh owned service; IAM subtree must remain absent; never enumerated"} +GETTERS = ("get_bucket_versioning", "get_bucket_policy", "get_bucket_tagging", "get_object_lock_configuration") + + +def seed_fixture(recorder, buckets): + expected = {} + for bucket in buckets: + recorder.call("create_bucket", {"Bucket": bucket, "ObjectLockEnabledForBucket": True}) + recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) + expected[bucket] = [] + for key in ("objects/target.bin", "objects/control.bin"): + for nonce in range(2): + body = payload(f"D11/{bucket}/{key}/{nonce}", 1024**2 + 97) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "Metadata": {"suite": "D11", "nonce": str(nonce)}, "Tagging": "suite=heal&fixture=DS"}, body) + version = response.get("VersionId") + require(isinstance(version, str) and version not in {"", "null"}, "bucket fixture lacks a version ACK") + expected[bucket].append({"key": key, "versionId": version, "isLatest": nonce == 1, "deleteMarker": False, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + document = {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": "*", + "Action": "s3:GetObject", "Resource": f"arn:aws:s3:::{bucket}/objects/*"}]} + recorder.call("put_bucket_policy", {"Bucket": bucket, "Policy": json.dumps(document)}) + recorder.call("put_bucket_tagging", {"Bucket": bucket, "Tagging": {"TagSet": [ + {"Key": "suite", "Value": "D11"}, {"Key": "bucket", "Value": bucket}]}}) + # Configure defaults AFTER writes. The historical fixture has no + # retained objects, so namespace finalization cannot bypass retention. + recorder.call("put_object_lock_configuration", {"Bucket": bucket, "ObjectLockConfiguration": { + "ObjectLockEnabled": "Enabled", "Rule": {"DefaultRetention": {"Mode": "GOVERNANCE", "Days": 1}}}}) + return expected + + +def configuration(observed, operation): + require(observed.get("acknowledged") is True and observed.get("resultUnknown") is False, + "bucket configuration read failed or has an unknown outcome") + response = {key: value for key, value in observed["response"].items() if key != "ResponseMetadata"} + if operation == "get_bucket_policy": + response["Policy"] = json.loads(response["Policy"]) + if operation == "get_bucket_tagging": + response["TagSet"] = sorted(response["TagSet"], key=lambda tag: (tag["Key"], tag["Value"])) + require(bool(response), "bucket configuration silently became an empty default") + return response + + +def configs(probe, fleet, bucket, label): + return {op: configuration(fleet.record(label, {"bucket": bucket, **probe.s3(op, {"Bucket": bucket})}), op) + for op in GETTERS} + + +def system_snapshot(fleet, buckets, label, faults=None): + return fleet.record(label, {node: fleet.remote(node, "bucket-record-snapshot", buckets=buckets, + metadataFaults=(faults or {}).get(node, {})) for node in fleet.nodes}) + + +def system_files(value): + result = {(node, row["disk"], row["path"]): {name: row[name] for name in ("size", "sha256")} + for node, sample in value.items() for row in sample["files"]} + require(len(result) == sum(len(sample["files"]) for sample in value.values()), "duplicate system file identity") + require(all(sample["credentialsRead"] is False and all(row["present"] is False and row["traversed"] is False + for row in sample["iamNamespace"]) for sample in value.values()), "credential boundary changed") + return result + + +def record_plan(snapshot, bucket, topology, physical): + path = bucket_path(bucket, bucket) + "/xl.meta" + targets = [(node, row) for node, sample in snapshot.items() for row in sample["files"] if row["path"] == path] + require(len(targets) == (4 if topology == "H1" else 16), "bucket config lacks every original member") + geometry, indices, latest = None, set(), None + for node, row in targets: + metadata = row.get("metadata") or {} + require(metadata.get("bucket") == ".rustfs.sys" and metadata.get("key") == f"buckets/{bucket}/.metadata.bin", + "system object decoder identity differs") + versions = metadata.get("versions", []) + require(len(versions) == 1 and versions[0].get("deleted") is False and versions[0].get("isLatest") is True, + "bucket record must have one authoritative live version") + version = versions[0] + erasure = version["erasure"] + current = {key: erasure[key] for key in ("data_blocks", "parity_blocks", "distribution", "block_size", "algorithm")} + require(geometry is None or current == geometry, "bucket config lacks one erasure authority") + geometry = current + layouts = [record["value"]["xl"] for record in physical[node]["formats"] if record["disk"] == row["disk"]] + require(len(layouts) == 1 and len(layouts[0]["sets"]) == 1, + "system object member lacks its actual disk/set format") + layout = layouts[0] + require(erasure["index"] == erasure["distribution"][layout["sets"][0].index(layout["this"])], + "system shard index differs from its physical disk UUID/set slot") + require(erasure["index"] not in indices and version.get("size", 0) > 4, "duplicate or empty config shard") + indices.add(erasure["index"]) + identity = version.get("versionId"), version.get("dataDir"), version["size"], version["parts"] + require(latest is None or identity == latest, "bucket-config version/dataDir differs across members") + latest = identity + require(geometry["data_blocks"] + geometry["parity_blocks"] == len(targets) + and indices == set(range(1, len(targets) + 1)) and geometry["parity_blocks"] >= 1, + "system record geometry lacks one recoverable-member loss") + return targets + + +def refusal(observed): + response = observed.get("response") or {} + require(observed.get("acknowledged") is False and observed.get("resultUnknown") is False + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") in {500, 503} + and response.get("Error", {}).get("Code") in {"InternalError", "XMinioReadQuorum", "ServiceUnavailable"}, + "unknown bucket-config envelope was served as a default or lacks a structured server refusal") + + +def bucket_heal(fleet, probe, bucket, active, *, negative=False): + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + admitted = fleet.record("bucket-heal-admission", probe.admin(path, [], json.dumps({**BASE_OPTIONS, "recursive": True}).encode())) + require(admitted.get("httpStatus") == 200 and isinstance(admitted.get("response"), dict), "bucket Heal was not admitted") + token = admitted["response"].get("clientToken") + require(isinstance(token, str) and token, "bucket Heal omitted task identity") + active.append((path, token)) + deadline = time.monotonic() + 180 + while True: + raw = fleet.record("bucket-heal-status", probe.admin(path, [("clientToken", token)], b"")) + require(raw.get("httpStatus") == 200 and isinstance(raw.get("response"), dict), "bucket Heal lost its task") + current = raw["response"] + if current.get("summary") != "running": + break + require(time.monotonic() < deadline, "bucket Heal exceeded its fixed deadline") + time.sleep(.2) + active.remove((path, token)) + if negative: + outcome = current.get("outcome", {}) + require(outcome.get("execution", {}).get("state") in {"failed", "completed_with_errors"} + and outcome.get("counters", {}).get("healed") == 0, + "unknown record became successful bucket Heal responsibility") + return current + + +def run_branch(args, topology, nodes, branch): + directory = args.evidence / topology / branch + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + buckets = [bucket, bucket + "-control"] + fleet.snapshot_buckets = buckets + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == REFERENCE["candidateCommit"], "bucket-config schema contract is unreviewed for this candidate") + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), "maxInventoryObjects": 32, + "maxInventoryBytes": 64 * 1024**2} + fleet.record("source-bound-system-record-contract", REFERENCE) + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + fleet.record("binding", binding) + probe, active = Probe(binding), [] + result = {"topology": topology, "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + expected = seed_fixture(Recorder(probe, directory / "fixture"), buckets) + baseline_config = {name: configs(probe, fleet, name, "config-baseline") for name in buckets} + baseline_s3 = {name: fleet.record("S3-baseline", probe.inventory(name)) for name in buckets} + for name in buckets: + fleet.record("ACK-manifest", {"bucket": name, "expected": expected[name], **verify_seed(expected[name], baseline_s3[name])}) + fleet.stop() + original = fleet.snapshot("physical-baseline", quiescent=True) + validate_topology(original, topology) + records = system_snapshot(fleet, buckets, "system-record-baseline") + targets = record_plan(records, bucket, topology, original) + record_plan(records, buckets[1], topology, original) + negative = branch == "unknown-record-negative" + selected = targets if negative else targets[:1] + kind = "future-xl-schema" if negative else "truncate" if branch == "bucket-policy-tags" else "missing" + fault_map, registered = {}, [] + wanted_fault = system_files(records).copy() + for index, (node, target) in enumerate(selected): + params = {"disk": target["disk"], "faultId": f"D11-{branch}-{index}"} + fleet.bucket_record_faults.append((node, params)) + registered.append((node, params)) + receipt = fleet.record("exact-system-record-fault", fleet.remote(node, "bucket-record-apply", **params, + bucket=bucket, relativePath=target["path"], expectedSha256=target["sha256"], kind=kind)) + pair = node, target["disk"], target["path"] + if receipt["fault"] is None: + wanted_fault.pop(pair) + else: + wanted_fault[pair] = receipt["fault"] + fault_map.setdefault(node, {})[f"{target['disk']}:{target['path']}"] = receipt["fault"] + fault_scene = system_snapshot(fleet, buckets, "system-record-fault-witness", fault_map) + require(system_files(fault_scene) == wanted_fault, "system fault changed an unselected record") + require(physical_content(original) == physical_content(fleet.snapshot("business-fault-control", quiescent=True)), + "system fault changed business objects or disk formats") + fleet.start() + if negative: + for endpoint in binding["endpoints"]: + observer = Probe(binding, endpoint) + for operation in GETTERS: + refusal(fleet.record("unknown-record-config-refusal", observer.s3(operation, {"Bucket": bucket}))) + require(configs(observer, fleet, buckets[1], "healthy-config-control") == baseline_config[buckets[1]], + "unknown primary record changed the control bucket") + # Public refusal and untouched raw unknown records are the oracle; + # startup failure alone cannot satisfy this branch. + task = bucket_heal(fleet, probe, bucket, active, negative=True) + else: + task = bucket_heal(fleet, probe, bucket, active) + validate_version_receipts(task, bucket, expected[bucket], 2) + for endpoint in binding["endpoints"]: + for name in buckets: + require(configs(Probe(binding, endpoint), fleet, name, "post-repair-config") == baseline_config[name], + "versioning/policy/tags/lock config changed after record repair") + idle_fleet(fleet, binding) + fleet.stop() + after = system_snapshot(fleet, buckets, "system-record-before-rescue-and-business-GET", fault_map if negative else None) + require(system_files(after) == (wanted_fault if negative else system_files(records)), + "unknown record was rewritten or recovery failed to restore complete original config bytes") + require(physical_content(original) == physical_content(fleet.snapshot("physical-before-GET", quiescent=True)), + "bucket metadata recovery changed business versions or disk formats") + # Positive repair is already witnessed. Negative rescue is separately + # recorded and is never counted as candidate repair. + for node, params in registered: + if negative: + rescue = fleet.record("unknown-record-backup-rescue", fleet.remote(node, "bucket-record-recover", **params)) + require(rescue["physicalRepairPassAllowed"] is False, "backup rescue claimed repair") + fleet.bucket_record_faults.remove((node, params)) + fleet.start() + for endpoint in binding["endpoints"]: + observer = Probe(binding, endpoint) + for name in buckets: + require(configs(observer, fleet, name, "final-config") == baseline_config[name], "final config differs") + actual = fleet.record("all-peer-S3-after", {"endpoint": endpoint, **observer.inventory(name)}) + fleet.record("full-S3-proof", {**verify_seed(expected[name], actual), **compare_inventories(baseline_s3[name], actual)}) + fleet.stop() + require(system_files(system_snapshot(fleet, buckets, "final-system-records")) == system_files(records), + "final system records differ from same-run baseline") + require(physical_content(original) == physical_content(fleet.snapshot("final-business-physical", quiescent=True)), + "business reads changed the proven result") + result.update(verdict="PASS", configAPIs=list(GETTERS), completeSystemBytes=True, taskId=task.get("clientToken"), + unknownSchemaRefused=negative, backupRescueExcludedFromRepair=negative, credentialsRead=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + try: + fleet.stop() + fleet.snapshot("physical-failure", quiescent=True) + except Exception as error: + result["failureEvidenceError"] = str(error) + finally: + for path, token in active: + try: + fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: + fleet.record("stop-error", {"error": str(exc)}) + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("branch-result", result) + return result + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f"bucket metadata validation interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path, required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(nodes) == len(set(nodes)) and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), + "four original Heal nodes required") + require(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")) + and re.fullmatch(r"[0-9a-f]{64}", args.binary_sha), "immutable workflow/candidate identities required") + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "bucket metadata executor omitted a source branch") + results = [] + try: + for topology, selected_nodes in (("H1", nodes[:1]), ("H2", nodes[:4])): + for branch in BRANCHES: + results.append(run_branch(args, topology, selected_nodes, branch)) + if results[-1]["cleanup"] != "PASS": + break + if results[-1]["cleanup"] != "PASS": + break + finally: + complete = len(results) == 8 and all(row["verdict"] == "PASS" and row["cleanup"] == "PASS" for row in results) + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, + "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL", "complete": complete}) + require(complete, "D11 requires all eight H1/H2 record recovery/refusal instances and clean finalization") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"Heal bucket metadata validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/bucket_records.py b/scripts/validation_heal/bucket_records.py new file mode 100644 index 00000000..76e39d89 --- /dev/null +++ b/scripts/validation_heal/bucket_records.py @@ -0,0 +1,184 @@ +"""D11 node backend: exact owned bucket-config records, never an IAM walk.""" + +import json +import os +import re +import uuid + + +def require(value, message): + if not value: + raise ValueError(message) + + +def bucket_path(bucket, prefix): + require(isinstance(bucket, str) and (bucket == prefix or bucket.startswith(prefix + "-")) + and re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket), + "system record is outside the exact owned bucket prefix") + return f".rustfs.sys/buckets/{bucket}/.metadata.bin" + + +def record_parts(relative, bucket, prefix): + base = bucket_path(bucket, prefix) + require(relative == base + "/xl.meta", "only the owned bucket-config xl.meta is mutable") + return relative.split("/") + + +def changed_bytes(before, kind): + require(kind in {"missing", "truncate", "future-xl-schema"}, "unsupported bucket-record fault") + require(len(before) >= 8 and before[:4] == b"XL2 " and int.from_bytes(before[4:6], "little") == 1, + "bucket record is not the reviewed candidate XL2 major 1 envelope") + if kind == "missing": + return None + if kind == "truncate": + return before[:len(before) // 2] + # The candidate rejects an outer major above 1 before decoding/checksums. + # Preserve every other byte; this is not a fabricated inner msgpack record. + return before[:4] + (2).to_bytes(2, "little") + before[6:] + + +def apply(root_fd, owner, fault, request): + bucket, identity = request["bucket"], request["faultId"] + parts = record_parts(request["relativePath"], bucket, owner["bucketPrefix"]) + require(re.fullmatch(r"D11-[a-z0-9-]{1,48}", identity), "invalid bucket-record fault ID") + require(re.fullmatch(r"[0-9a-f]{64}", request["expectedSha256"]), "missing frozen record digest") + with fault.parent(root_fd, ["data", *parts[:-1]]) as target, fault.parent(root_fd, ["quarantine"]) as backup: + before = fault.read_file(target, parts[-1], 4 * 1024**2) + require(fault.digest(before) == request["expectedSha256"], "bucket record changed before injection") + changed = changed_bytes(before, request["kind"]) + receipt = {"schemaVersion": 1, "runId": owner["runId"], "bucket": bucket, "faultId": identity, + "relativePath": request["relativePath"], "kind": request["kind"], "backup": identity + ".original", + "before": {"sha256": fault.digest(before), "size": len(before)}, + "fault": None if changed is None else {"sha256": fault.digest(changed), "size": len(changed)}, + "backend": "exact-owned-bucket-config-envelope", "requiresQuiescentService": True, + "credentialsRead": False, "innerMetadataPayloadRewritten": False} + fault.new_file(backup, identity + ".intent.json", fault.encode(receipt)) + fault.new_file(backup, receipt["backup"], before) + require(fault.read_file(target, parts[-1]) == before, "record changed after its intent") + if changed is None: + os.unlink(parts[-1], dir_fd=target) + else: + temporary = ".bucket-fault-" + identity + fault.new_file(target, temporary, changed) + os.rename(temporary, parts[-1], src_dir_fd=target, dst_dir_fd=target) + os.fsync(target) + if changed is None: + try: + os.stat(parts[-1], dir_fd=target, follow_symlinks=False) + except FileNotFoundError: + pass + else: + raise ValueError("bucket record removal was not witnessed") + else: + require(fault.read_file(target, parts[-1]) == changed, "bucket record mutation was not witnessed") + fault.new_file(backup, identity + ".applied.json", fault.encode(receipt)) + return receipt + + +def recover(root_fd, owner, fault, request): + identity = request["faultId"] + require(re.fullmatch(r"D11-[a-z0-9-]{1,48}", identity), "invalid bucket-record recovery ID") + with fault.parent(root_fd, ["quarantine"]) as backup: + receipt = json.loads(fault.read_file(backup, identity + ".intent.json", 16384)) + require(receipt["runId"] == owner["runId"] and receipt["faultId"] == identity + and receipt["backup"] == identity + ".original", "bucket-record recovery identity differs") + parts = record_parts(receipt["relativePath"], receipt["bucket"], owner["bucketPrefix"]) + before = fault.read_file(backup, receipt["backup"]) + require({"sha256": fault.digest(before), "size": len(before)} == receipt["before"], "record backup changed") + with fault.parent(root_fd, ["data", *parts[:-1]]) as target: + try: + current = fault.read_file(target, parts[-1]) + except FileNotFoundError: + current = None + restored = current != before + if restored: + require(current is None or {"sha256": fault.digest(current), "size": len(current)} == receipt["fault"], + "bucket record has an unrecognized post-fault rewrite; retain namespace") + if current is not None: + name = identity + ".failed-" + fault.digest(current) + try: + fault.new_file(backup, name, current) + except FileExistsError: + require(fault.read_file(backup, name) == current, "preserved record differs") + temporary = ".bucket-rescue-" + identity + try: + fault.new_file(target, temporary, before) + except FileExistsError: + require(fault.read_file(target, temporary) == before, "interrupted rescue bytes differ") + os.rename(temporary, parts[-1], src_dir_fd=target, dst_dir_fd=target) + os.fsync(target) + require(fault.read_file(target, parts[-1]) == before, "record rescue did not restore exact baseline") + return {"faultId": identity, "restored": restored, "physicalRepairPassAllowed": False, + "restorationIsProductRepair": False} + + +def snapshot(roots, owner, request, checked, digest, tool, command): + buckets = request["buckets"] + require(isinstance(buckets, list) and len(buckets) == 2 and len(set(buckets)) == 2 + and owner["bucketPrefix"] in buckets, "exact primary/control bucket pair required") + entries, iam = [], [] + for disk, root in enumerate(roots): + data = root / "data" + # Root credentials live in the owned service environment. An IAM + # subtree is a precondition failure: never enumerate credential files. + iam_root = checked(data / ".rustfs.sys/config/iam", data) + iam.append({"disk": disk, "present": iam_root.exists(), "traversed": False}) + require(not iam_root.exists(), "D11 requires the fresh owned namespace without persisted IAM records") + for bucket in buckets: + for record_name in (".metadata.bin", ".bucket-incarnation"): + base = bucket_path(bucket, owner["bucketPrefix"]).rsplit("/", 1)[0] + "/" + record_name + folder = checked(data / base, data) + if not folder.exists(): + continue + for directory, children, names in os.walk(folder, followlinks=False): + for name in (*children, *names): + checked(type(folder)(directory) / name, data) + for name in sorted(names): + path = type(folder)(directory) / name + relative = path.relative_to(folder).as_posix() + segments = relative.split("/") + valid = relative == "xl.meta" or len(segments) == 2 and bool(re.fullmatch( + r"part\.[1-9][0-9]*(?:\.integrity\.[a-z0-9-]+)?", segments[1])) + if len(segments) == 2: + require(str(uuid.UUID(segments[0])) == segments[0], "noncanonical bucket-config dataDir") + require(valid and len(entries) < 128 and path.stat().st_size <= 4 * 1024**2, + "bucket-config record exceeds the exact file/byte allowlist") + entry = {"disk": disk, "bucket": bucket, "path": str(path.relative_to(data)), + "size": path.stat().st_size, "sha256": digest(path)} + if relative == "xl.meta": + decoded = command(str(tool), str(path), ".rustfs.sys", f"buckets/{bucket}/{record_name}", check=False) + if decoded.returncode == 0: + entry["metadata"] = json.loads(decoded.stdout) + versions = entry["metadata"].get("versions", []) + require(len(versions) == 1 and not versions[0].get("deleted"), + "system streaming proof requires one live record") + version = versions[0] + version_id = version.get("versionId") or "null" + args = [str(tool), "--verify-inline", str(path), ".rustfs.sys", + f"buckets/{bucket}/{record_name}", version_id] + if version.get("inline") is not None: + entry["streaming"] = [json.loads(command(*args).stdout)] + else: + data_dir = version.get("dataDir") + require(isinstance(data_dir, str) and str(uuid.UUID(data_dir)) == data_dir, + "system record lacks a canonical external dataDir") + proofs = [] + for part in version["parts"]: + number = part["number"] + require(type(number) is int and 0 < number <= 8, "unbounded system record part") + part_path = checked(folder / data_dir / f"part.{number}", data) + proofs.append(json.loads(command(str(tool), "--verify-part", str(path), ".rustfs.sys", + f"buckets/{bucket}/{record_name}", str(part_path), version_id, str(number)).stdout)) + require(proofs, "system record has no streaming part proof") + entry["streaming"] = proofs + require(all(proof.get("physicalBitrotVerified") is True and proof.get("bucket") == ".rustfs.sys" + and proof.get("key") == f"buckets/{bucket}/{record_name}" for proof in entry["streaming"]), + "system record streaming proof differs from its exact identity") + else: + expected = request.get("metadataFaults", {}).get(f"{disk}:{entry['path']}") + require(expected == {name: entry[name] for name in ("size", "sha256")}, + "bucket record decoding failed outside its exact witnessed mutation") + entry["decodeFailure"] = True + entries.append(entry) + return {"files": entries, "iamNamespace": iam, "credentialsRead": False, + "scope": "two exact owned bucket metadata object directories; no system-directory traversal"} diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 5a6c0934..1006e1de 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -742,6 +742,30 @@ def logs(request): return {"unit": owner["unit"], "journal": command("journalctl", "-u", owner["unit"], "--no-pager", "-n", "2000").stdout} +def bucket_record_operation(request): + state, roots, owner = owned(request["runId"]) + stopped(owner) + module, fault = globals()["bucket_records"], fault_module() + if request["action"] == "bucket-record-snapshot": + tool = checked(state / "heal-meta-inspect", state) + require(owner.get("inspectorSha256") and digest(tool) == owner["inspectorSha256"], + "bucket-config inspector identity differs") + return module.snapshot(roots, {**owner, "bucketPrefix": owner["bucket"]}, request, checked, digest, tool, command) + root, owner = fault_root(request) + with fault.directory(root) as fd: + marker = root / "fault-owner.json" + if request["action"] == "bucket-record-apply" and not marker.exists(): + fault.new_file(fd, marker.name, fault.encode({"schemaVersion": 1, "runId": owner["runId"], + "bucketPrefix": owner["bucket"], "mountId": fault.mount_id(fd), "device": os.fstat(fd).st_dev, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat()})) + # Retain the common owner/mount/lease checks without widening its + # generic target allowlist to include any system path. + binding = fault.checked_owner(fd, owner["runId"], recovery=request["action"] == "bucket-record-recover") + if request["action"] == "bucket-record-apply": + return module.apply(fd, binding, fault, request) + return module.recover(fd, binding, fault, request) + + def cleanup(request): state, roots, owner = owned(request["runId"]) command("systemctl", "stop", owner["unit"], check=False) @@ -844,6 +868,8 @@ def main(request): "drive-gate-activate": drive_gate_activate, "gate-state": drive_gate_state, "drive-gate-release-live": drive_gate_release_live} actions["task-records"] = task_records + for action in ("bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover"): + actions[action] = bucket_record_operation actions.update({"replace-drive": replace_drive, "replacement-state": replacement_state, "replacement-probe": replacement_probe}) actions.update({"format-export": format_export, "foreign-format": foreign_format, "restore-replacement": restore_replacement}) actions.update({"process": process, "process-metrics": process_metrics, "process-signal": process_signal, "process-graceful": process_graceful, "process-crash": process_crash}) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 422ea595..415dcdac 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -147,6 +147,7 @@ def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_prof self.sequence = 0 self.reserved = [] self.faults = [] + self.bucket_record_faults = [] self.drive_gates = set() self.secret_values = set() self.frozen_processes = {} @@ -181,9 +182,13 @@ def remote(self, node, action, **params): if action == "mrf-records": source += "\nimport types\nmrf = types.ModuleType('mrf_records')\nexec(" + repr( NODE_SOURCE.with_name("mrf_records.py").read_text()) + ", mrf.__dict__)\n" - if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline"}: + if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline", + "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover"}: source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" + if action.startswith("bucket-record-"): + source += "\nimport types\nbucket_records = types.ModuleType('bucket_records')\nexec(" + repr( + NODE_SOURCE.with_name("bucket_records.py").read_text()) + ", bucket_records.__dict__)\n" replacement_actions = {"start", "cleanup", "audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot"} if action in {"drive-gate", "drive-gate-probe", "drive-gate-activate", "gate-state", "drive-gate-release-live"} | replacement_actions: source += "\nimport types\ngate = types.ModuleType('drive_gate')\nexec(" + repr( @@ -298,11 +303,13 @@ def cleanup(self): self.record("logs", {"node": node, **self.remote(node, "logs")}) except Exception as exc: self.record("logs-error", {"node": node, "error": str(exc)}) - if self.faults and not failures: + if (self.faults or self.bucket_record_faults) and not failures: try: self.stop() for node, params in self.faults: self.record("fault-recovery", {"node": node, **self.remote(node, "recover-fault", **params)}) + for node, params in self.bucket_record_faults: + self.record("bucket-record-rescue", {"node": node, **self.remote(node, "bucket-record-recover", **params)}) except Exception as exc: failures.append({"error": str(exc), "state": "DIRTY", "namespacesRetained": True}) if not failures: diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index b6b64266..a8a1aee7 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index a64b1cef..52391d6a 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -30,6 +30,7 @@ case "${case_id}" in heal.mrf-offline-write-replay) driver=mrf_replay.py ;; heal.performance-abba) driver=performance.py ;; heal.admin-scope-isolation) driver=scopes.py ;; + heal.bucket-metadata-integrity) driver=bucket_metadata.py ;; heal.disk-replacement-unformatted) driver=unformatted.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; diff --git a/test_validation_heal_bucket_metadata.py b/test_validation_heal_bucket_metadata.py new file mode 100644 index 00000000..07ae9b97 --- /dev/null +++ b/test_validation_heal_bucket_metadata.py @@ -0,0 +1,153 @@ +import copy +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from scripts.validation_heal import bucket_metadata as executor, bucket_records as backend, heal_fault as fault +from scripts.validation_heal import fleet_node as node +from scripts.validation_heal.matrix import AUTOMATIC_CASES, select + + +class BucketMetadataTests(unittest.TestCase): + bucket = "rv-heal-0123456789abcdef" + value = b"XL2 " + b"\x01\x00\x03\x00" + b"same-run-config-envelope" * 10 + + def test_system_allowlist_cannot_escape_or_widen_generic_faults(self): + owned = backend.bucket_path(self.bucket, self.bucket) + "/xl.meta" + self.assertEqual(backend.record_parts(owned, self.bucket, self.bucket)[-1], "xl.meta") + for path, bucket in ((owned.replace("buckets", "config/iam"), self.bucket), + (owned + "/../xl.meta", self.bucket), + (owned.replace(".metadata.bin", ".bucket-incarnation"), self.bucket), + (owned.replace("xl.meta", "part.1"), self.bucket), (owned, "foreign")): + with self.subTest(path=path, bucket=bucket), self.assertRaises(ValueError): + backend.record_parts(path, bucket, self.bucket) + with self.assertRaises(ValueError): + fault.target_parts(owned, self.bucket) + + def test_unknown_envelope_preserves_every_byte_besides_reviewed_major(self): + changed = backend.changed_bytes(self.value, "future-xl-schema") + self.assertEqual(changed[:4], b"XL2 ") + self.assertEqual(changed[4:6], b"\x02\x00") + self.assertEqual(changed[6:], self.value[6:]) + for original in (b"", b"not-XL2!", changed): + with self.assertRaises(ValueError): + backend.changed_bytes(original, "future-xl-schema") + + def scene(self, root, kind): + relative = backend.bucket_path(self.bucket, self.bucket) + "/xl.meta" + path = root / "data" / relative + path.parent.mkdir(parents=True) + (root / "quarantine").mkdir() + path.write_bytes(self.value) + owner = {"runId": "test-run", "bucketPrefix": self.bucket} + request = {"bucket": self.bucket, "faultId": "D11-negative-0", "relativePath": relative, + "expectedSha256": fault.digest(self.value), "kind": kind} + return path, owner, request + + @unittest.skipUnless(Path('/proc/self/fdinfo').is_dir(), 'Linux fd/mount guard required') + def test_faults_keep_originals_and_reject_unrecognized_post_fault_writes(self): + for kind in ("missing", "truncate", "future-xl-schema"): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as temp: + root = Path(temp) + path, owner, request = self.scene(root, kind) + with fault.directory(root) as fd: + receipt = backend.apply(fd, owner, fault, request) + self.assertEqual((root / "quarantine" / receipt["backup"]).read_bytes(), self.value) + self.assertTrue((root / "quarantine" / (request["faultId"] + ".intent.json")).exists()) + path.write_bytes(b"new acknowledged config generation") + with self.assertRaisesRegex(ValueError, "post-fault rewrite"): + backend.recover(fd, owner, fault, request) + self.assertEqual(path.read_bytes(), b"new acknowledged config generation") + changed = backend.changed_bytes(self.value, kind) + if changed is None: + path.unlink() + else: + path.write_bytes(changed) + restored = backend.recover(fd, owner, fault, request) + self.assertEqual(path.read_bytes(), self.value) + self.assertFalse(restored["physicalRepairPassAllowed"]) + + @unittest.skipUnless(Path('/proc/self/fdinfo').is_dir(), 'Linux fd/mount guard required') + def test_symlink_and_hardlink_record_targets_are_rejected_before_mutation(self): + for kind in ("symlink", "hardlink"): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as temp: + root = Path(temp) + path, owner, request = self.scene(root, "truncate") + other = root / "unrelated" + other.write_bytes(self.value) + path.unlink() + if kind == "symlink": + path.symlink_to(other) + else: + path.hardlink_to(other) + with fault.directory(root) as fd, self.assertRaises((ValueError, OSError)): + backend.apply(fd, owner, fault, request) + self.assertEqual(other.read_bytes(), self.value) + self.assertFalse(list((root / "quarantine").iterdir())) + + def test_empty_default_success_and_transport_error_are_not_unknown_schema_refusal(self): + good = {"acknowledged": False, "resultUnknown": False, "response": { + "ResponseMetadata": {"HTTPStatusCode": 500}, "Error": {"Code": "InternalError"}}} + executor.refusal(good) + for observed in ({"acknowledged": True, "resultUnknown": False, "response": {}}, + {"acknowledged": False, "resultUnknown": True, "response": None}, + {**good, "response": {"ResponseMetadata": {"HTTPStatusCode": 404}, "Error": {"Code": "NoSuchBucket"}}}): + with self.assertRaises(ValueError): + executor.refusal(observed) + with self.assertRaises(ValueError): + executor.configuration({"acknowledged": True, "resultUnknown": False, "response": {"ResponseMetadata": {}}}, + "get_bucket_versioning") + + def test_duplicate_files_or_persisted_iam_block_the_oracle(self): + row = {"disk": 0, "path": "exact-record", "size": 5, "sha256": "a" * 64} + sample = {"node": {"credentialsRead": False, "files": [row], + "iamNamespace": [{"disk": 0, "present": False, "traversed": False}]}} + executor.system_files(sample) + for kind in ("duplicate", "iam", "traversal"): + broken = copy.deepcopy(sample) + if kind == "duplicate": + broken["node"]["files"].append(row) + elif kind == "iam": + broken["node"]["iamNamespace"][0]["present"] = True + else: + broken["node"]["iamNamespace"][0]["traversed"] = True + with self.assertRaises(ValueError): + executor.system_files(broken) + + def test_remote_node_script_compiles_with_dedicated_backend(self): + directory = Path(__file__).parent / "scripts/validation_heal" + source = (directory / "fleet_node.py").read_text() + for name in ("heal_fault", "bucket_records"): + source += f"\nimport types\n{name} = types.ModuleType('{name}')\nexec(" + repr((directory / (name + ".py")).read_text()) + f", {name}.__dict__)\n" + compile(source, "remote-D11", "exec") + self.assertIn(executor.CASE_ID, AUTOMATIC_CASES) + self.assertEqual(select(executor.CASE_ID)[0]["variants"], list(executor.BRANCHES)) + + def test_system_snapshot_receives_the_real_service_bucket_prefix(self): + owner = {"bucket": self.bucket, "inspectorSha256": "a" * 64} + with patch.object(node, "owned", return_value=(Path('/owned'), (), owner)), \ + patch.object(node, "stopped"), patch.object(node, "checked", side_effect=lambda path, _: path), \ + patch.object(node, "digest", return_value="a" * 64), patch.object(node, "fault_module"), \ + patch.object(node, "bucket_records", backend, create=True), patch.object(backend, "snapshot", return_value={}) as observe: + node.bucket_record_operation({"runId": "test", "action": "bucket-record-snapshot"}) + self.assertEqual(observe.call_args.args[1]["bucketPrefix"], self.bucket) + + @unittest.skipUnless(Path('/proc/self/fdinfo').is_dir(), 'Linux fd/mount guard required') + def test_first_system_fault_creates_a_checked_mount_and_lease_marker(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + path, _, request = self.scene(root, "missing") + owner = {"runId": "test-run", "bucket": self.bucket} + with patch.object(node, "owned", return_value=(root, (root,), owner)), patch.object(node, "stopped"), \ + patch.object(node, "fault_root", return_value=(root, owner)), patch.object(node, "fault_module", return_value=fault), \ + patch.object(node, "bucket_records", backend, create=True): + node.bucket_record_operation({**request, "runId": "test-run", "action": "bucket-record-apply"}) + self.assertFalse(path.exists()) + with fault.directory(root) as fd: + marker = fault.checked_owner(fd, "test-run") + self.assertEqual(marker["bucketPrefix"], self.bucket) + + +if __name__ == "__main__": + unittest.main() From 0fc5b01e3c86ff4f4206302541ac24fd1a7067fc Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 06:03:27 +0800 Subject: [PATCH 083/343] test(validation): collect exact-source native Heal regressions --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 16 ++ scripts/validation_heal/native_regressions.py | 230 ++++++++++++++++++ test_validation_heal_native.py | 52 ++++ 4 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 scripts/validation_heal/native_regressions.py create mode 100644 test_validation_heal_native.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index b9bb1fab..9ea98e14 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -24,6 +24,7 @@ on: - test_validation_heal_performance.py - test_validation_heal_scopes.py - test_validation_heal_bucket_metadata.py + - test_validation_heal_native.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -61,6 +62,7 @@ on: - test_validation_heal_performance.py - test_validation_heal_scopes.py - test_validation_heal_bucket_metadata.py + - test_validation_heal_native.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -92,7 +94,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 4da2144e..e715c9bc 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1018,3 +1018,19 @@ run `38088777413` also retained FAIL/cleanup PASS in both H2 DB/DV branches: replacement source status was partial, unknown or incompatible. The bounded file census no longer prevented either run; these product/evidence failures are not converted into successful restart coverage. + + +The optional native-regression tools job builds the unmodified candidate's +optimized ecstore test executable on hosted Linux, retains Cargo/lock/toolchain +identities, enumerates actual full test names, and runs each selected name with +`--exact` in a fresh process and a fixed 120-second deadline. It currently +collects two real read-repair/namespace barrier regressions and two uncontrolled +overwrite/delete concurrency regressions. Zero-test filters, ignored tests, +foreign source/collector identities and other-name success are rejected. The +four regression verdicts have their own archive and never increment a fleet or +canonical-case PASS count. Existing tests assert captured lock events internally; +the collector does not fabricate emitted barrier logs. Missing critical-point +fixtures, Admin nolock semantics and required fleet overlap remain explicit gaps. +The current implementation has 239 focused Heal/catalog checks, including the +Linux-only D11 ownership/mount/recovery checks. Native compilation and execution +still require the separate hosted job; local parser checks are not native PASS. diff --git a/scripts/validation_heal/native_regressions.py b/scripts/validation_heal/native_regressions.py new file mode 100644 index 00000000..686d93b7 --- /dev/null +++ b/scripts/validation_heal/native_regressions.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Build/list/run exact candidate native regressions; never certify fleet cases.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import time + + +CANDIDATE = "072b57b37559fbe626aeda8abf61617fdcbdd926" +SOURCE = "crates/ecstore/src/set_disk/ops/heal.rs" +SOURCE_SHA = "767c468b780cf5f4e863b078f46c63440549e75d88b6b43bb59ed83c3b4a47b2" +PREFIX = "set_disk::ops::heal::heal_result_report_tests::" +REGRESSIONS = { + "read-repair-final-rename": { + "test": PREFIX + "read_repair_commit_aborts_when_object_changes_before_final_rename", + "sourceAssertions": ["arrived.notified() reaches the real final commit pause within five seconds", + "overwrite ACK precedes release.notify_one()", "stale repair returns typed DiskError::Timeout", + "current ETag remains the overwrite ACK"], + "fenceKind": "deterministic assertion in the exact unmodified candidate test", + "relatedCases": ["heal.race-unversioned-overwrite", "heal.lock-timeout-and-caller-abort"], + "limits": "does not prove PUT cleanup-tail, Admin nolock compatibility or a fleet overlap window"}, + "read-repair-lock-owner": { + "test": PREFIX + "object_lock_diagnostics_distinguish_read_repair_from_queued_put", + "sourceAssertions": ["read-repair owns its actual shared data-phase lock before PUT queues", + "PUT namespace remains pending while a historical GET times out", + "captured acquired lock event has no premature guard_dropped event", + "release permits acknowledged PUT and historical repair; all three versions retain bytes"], + "fenceKind": "deterministic assertions and in-test captured lock events", + "relatedCases": ["heal.race-versioned-put", "heal.lock-timeout-and-caller-abort"], + "limits": "captured event rows are asserted internally; this runner cannot invent emitted barrier logs"}, + "overwrite-concurrency": { + "test": PREFIX + "heal_racing_unversioned_overwrites_preserves_the_last_commit", + "sourceAssertions": ["eight joined overwrite/Heal rounds preserve the last ACK ETag"], + "fenceKind": "uncontrolled concurrency, not deterministic fence evidence", + "relatedCases": ["heal.race-unversioned-overwrite"], + "limits": "best-effort damage and tokio::join! do not prove a commit/cleanup critical point"}, + "version-delete-concurrency": { + "test": PREFIX + "heal_racing_version_delete_never_resurrects_the_deleted_version", + "sourceAssertions": ["confirmed deleted version remains VersionNotFound", "surviving version remains readable"], + "fenceKind": "uncontrolled concurrency, not deterministic fence evidence", + "relatedCases": ["heal.race-version-delete"], + "limits": "does not prove a deterministic delete/repair commit fence"}, +} + + +def require(value, message): + if not value: + raise ValueError(message) + + +def sha(path): + result = hashlib.sha256() + with path.open('rb') as stream: + while chunk := stream.read(1024**2): + result.update(chunk) + return result.hexdigest() + + +def save(path, value): + with path.open('x') as stream: + json.dump(value, stream, indent=2) + stream.write('\n') + + +def exact_listing(output, name): + tests = [line.removesuffix(': test') for line in output.splitlines() if line.endswith(': test')] + require(tests.count(name) == 1, "native test inventory must contain exactly one complete requested name") + require(len(tests) == len(set(tests)), "native inventory repeats a test identity") + return len(tests) + + +def test_result(output, name, code): + # A successful zero-test filter or an ignored test is not a regression run. + require(code == 0 and len(re.findall(r"(?m)^running 1 test\s*$", output)) == 1 + and re.search(r"(?m)^test " + re.escape(name) + r" \.\.\. ok\s*$", output) + and re.search(r"test result: ok\. 1 passed; 0 failed; 0 ignored;", output), + "exact native execution did not report one real passing test") + return {"exactTestExecuted": True, "nativeRegressionPassed": True, + "fleetPassClaimed": False, "emittedBarrierLogsClaimed": False} + + +def build(source, commit, out): + require(commit == CANDIDATE and subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip() == commit, + "native regression contract requires the reviewed immutable candidate checkout") + require(subprocess.run(['git', '-C', str(source), 'diff', '--quiet']).returncode == 0 + and subprocess.run(['git', '-C', str(source), 'diff', '--cached', '--quiet']).returncode == 0, + "native source has tracked modifications") + require(not subprocess.check_output(['git', '-C', str(source), 'ls-files', '--others', '--exclude-standard'], text=True).strip(), + "native candidate checkout has untracked source/configuration") + require(sha(source / SOURCE) == SOURCE_SHA, "native test source hash differs from its reviewed fence assertions") + require(not out.exists() and shutil.disk_usage(source).free >= 12 * 1024**3, + "fresh output and at least 12 GiB build space required") + out.mkdir(parents=True) + save(out / 'native-build-request.json', {'candidateCommit': commit, 'sourceFiles': {SOURCE: SOURCE_SHA}, + 'candidateCargoLockSha256': sha(source / 'Cargo.lock'), 'regressions': REGRESSIONS, + 'runId': os.environ.get('GITHUB_RUN_ID'), 'runAttempt': os.environ.get('GITHUB_RUN_ATTEMPT'), + 'fleetPassClaimed': False}) + target = out / 'target' + build_env = {'CARGO_BUILD_JOBS': '2', 'CARGO_INCREMENTAL': '0', 'CARGO_PROFILE_RELEASE_DEBUG': '0', + 'CARGO_PROFILE_RELEASE_STRIP': 'debuginfo'} + env = {**os.environ, **build_env, 'CARGO_TARGET_DIR': str(target)} + cargo = ['cargo', 'test', '--locked', '--release', '-p', 'rustfs-ecstore', '--lib', '--no-run', '--message-format=json'] + # Preserve diagnostics in files; native artifacts never require modifying + # candidate source or adding a fake fence/event to an existing test. + with (out / 'cargo.jsonl').open('x') as stdout, (out / 'cargo.stderr').open('x') as stderr: + result = subprocess.run(cargo, cwd=source, env=env, stdout=stdout, stderr=stderr, timeout=150 * 60) + require(result.returncode == 0, "native regression build failed; retained Cargo diagnostics") + require(subprocess.run(['git', '-C', str(source), 'diff', '--quiet']).returncode == 0 + and subprocess.run(['git', '-C', str(source), 'diff', '--cached', '--quiet']).returncode == 0, + "native build changed tracked candidate source") + executables = [] + for line in (out / 'cargo.jsonl').read_text().splitlines(): + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if row.get('reason') == 'compiler-artifact' and row.get('profile', {}).get('test') is True \ + and row.get('target', {}).get('name') == 'rustfs_ecstore' and row.get('executable'): + executables.append(Path(row['executable'])) + require(len(executables) == 1 and executables[0].resolve().is_relative_to(target.resolve()), + "Cargo did not expose one owned candidate ecstore test executable") + binary = out / 'ecstore-native-tests' + shutil.copyfile(executables[0], binary) + binary.chmod(0o500) + listing = subprocess.run([str(binary), '--list'], capture_output=True, text=True, timeout=60) + (out / 'test-inventory.log').write_text(listing.stdout + listing.stderr) + require(listing.returncode == 0, "candidate native inventory failed") + for spec in REGRESSIONS.values(): + exact_listing(listing.stdout, spec['test']) + manifest = {'schemaVersion': 1, 'candidateCommit': commit, 'candidateSourceTree': subprocess.check_output( + ['git', '-C', str(source), 'rev-parse', 'HEAD^{tree}'], text=True).strip(), + 'sourceFiles': {SOURCE: SOURCE_SHA}, 'candidateCargoLockSha256': sha(source / 'Cargo.lock'), + 'binarySha256': sha(binary), 'binaryName': binary.name, 'binaryKind': 'native-regression-not-released-service', + 'collectorSha256': sha(Path(__file__)), 'testingCommit': os.environ.get('AUTO_TESTING_SHA'), + 'sourceUnmodified': True, 'buildCommand': cargo, 'buildEnvironment': build_env, 'optimized': True, 'buildJobs': 2, + 'toolchain': subprocess.check_output(['rustc', '-vV'], cwd=source, text=True), + 'runId': os.environ.get('GITHUB_RUN_ID'), 'runAttempt': os.environ.get('GITHUB_RUN_ATTEMPT'), + 'regressions': REGRESSIONS, 'fleetPassClaimed': False, + 'limits': 'native and fleet evidence are independently scored; emitted fence logs and unmapped critical points remain gaps'} + save(out / 'native-regressions.json', manifest) + # Remove only the build cache created under this fresh task output after + # copying and identifying the executable and every build diagnostic. + shutil.rmtree(target) + return manifest + + +def run(directory, selected, out): + manifest = json.loads((directory / 'native-regressions.json').read_text()) + require(manifest.get('schemaVersion') == 1 and manifest.get('candidateCommit') == CANDIDATE + and manifest.get('sourceFiles') == {SOURCE: SOURCE_SHA} and manifest.get('sourceUnmodified') is True + and manifest.get('regressions') == REGRESSIONS and manifest.get('optimized') is True + and manifest.get('collectorSha256') == sha(Path(__file__)), + "native artifact identity or reviewed contract differs") + require(manifest.get('binaryName') == 'ecstore-native-tests', "native binary path is not allowlisted") + binary = directory / manifest['binaryName'] + require(binary.is_file() and not binary.is_symlink() and sha(binary) == manifest['binarySha256'], "native binary changed") + require(selected and len(selected) == len(set(selected)) and set(selected) <= set(REGRESSIONS), "invalid native selection") + require(not out.exists(), "native evidence directory must be fresh") + out.mkdir(parents=True) + listing = subprocess.run([str(binary), '--list'], capture_output=True, text=True, timeout=60) + (out / 'runtime-inventory.log').write_text(listing.stdout + listing.stderr) + require(listing.returncode == 0, "native runtime inventory failed") + results = [] + for key in selected: + spec = REGRESSIONS[key] + exact_listing(listing.stdout, spec['test']) + require(not binary.is_symlink() and sha(binary) == manifest['binarySha256'], "native executable changed before exact execution") + args = [str(binary), spec['test'], '--exact', '--nocapture', '--test-threads=1'] + started = time.monotonic_ns() + record = {'regression': key, 'test': spec['test'], 'command': args, 'sourceAssertions': spec['sourceAssertions'], + 'binarySha256': manifest['binarySha256'], 'candidateCommit': CANDIDATE, 'fleetPassClaimed': False, + 'emittedBarrierLogsClaimed': False, 'fenceKind': spec['fenceKind'], 'limitations': spec['limits']} + try: + child = subprocess.run(args, capture_output=True, text=True, timeout=120) + output = child.stdout + child.stderr + (out / (key + '.log')).write_text(output) + record['exitCode'] = child.returncode + test_result(output, spec['test'], child.returncode) + record['verdict'] = 'PASS' + except subprocess.TimeoutExpired as error: + def text_part(value): + return value.decode(errors='replace') if isinstance(value, bytes) else value or '' + output = text_part(error.stdout) + text_part(error.stderr) + (out / (key + '.log')).write_text(output) + record.update(verdict='FAIL', reason='native test exceeded its fixed 120-second deadline') + except ValueError as error: + record.update(verdict='FAIL', reason=str(error)) + except OSError as error: + (out / (key + '.log')).write_text(str(error)) + record.update(verdict='INFRA_ERROR', reason='native test executable could not start: ' + str(error)) + record['elapsedNanoseconds'] = time.monotonic_ns() - started + save(out / (key + '.json'), record) + results.append(record) + save(out / 'native-summary.json', {'results': results, 'fleetPassClaimed': False, + 'complete': len(results) == len(selected) and all(row['verdict'] == 'PASS' for row in results), + 'canonicalCasePassClaimed': False, 'sourceUnmodified': True}) + require(all(row['verdict'] == 'PASS' for row in results), 'native regression failure remains in separate evidence') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + compiling = commands.add_parser('build') + compiling.add_argument('--source', type=Path, required=True) + compiling.add_argument('--commit', required=True) + compiling.add_argument('--out', type=Path, required=True) + running = commands.add_parser('run') + running.add_argument('--directory', type=Path, required=True) + running.add_argument('--selection', default=','.join(REGRESSIONS)) + running.add_argument('--out', type=Path, required=True) + args = parser.parse_args() + if args.command == 'build': + build(args.source.resolve(strict=True), args.commit, args.out.resolve()) + else: + run(args.directory.resolve(strict=True), args.selection.split(','), args.out.resolve()) + + +if __name__ == '__main__': + try: + main() + except Exception as error: + print(f'native validation failed: {error}', file=sys.stderr) + sys.exit(1) diff --git a/test_validation_heal_native.py b/test_validation_heal_native.py new file mode 100644 index 00000000..b3d1b0c9 --- /dev/null +++ b/test_validation_heal_native.py @@ -0,0 +1,52 @@ +import json +from pathlib import Path +import tempfile +import unittest + +from scripts.validation_heal import native_regressions as native + + +class NativeRegressionTests(unittest.TestCase): + name = native.PREFIX + 'read_repair_commit_aborts_when_object_changes_before_final_rename' + + def test_full_test_name_must_appear_once_in_actual_inventory(self): + native.exact_listing(self.name + ': test\n', self.name) + for listing in ('0 tests, 0 benchmarks\n', self.name.split('::')[-1] + ': test\n', + (self.name + ': test\n') * 2, self.name + '-other: test\n'): + with self.subTest(listing=listing), self.assertRaises(ValueError): + native.exact_listing(listing, self.name) + + def test_zero_filtered_ignored_failed_and_other_test_success_are_not_pass(self): + passed = f'running 1 test\ntest {self.name} ... ok\ntest result: ok. 1 passed; 0 failed; 0 ignored; 200 filtered out;\n' + proof = native.test_result(passed, self.name, 0) + self.assertTrue(proof['exactTestExecuted']) + self.assertFalse(proof['fleetPassClaimed']) + self.assertFalse(proof['emittedBarrierLogsClaimed']) + for output, code in ((passed, 1), (passed.replace('running 1 test', 'running 0 tests'), 0), + (passed.replace('1 passed; 0 failed; 0 ignored', '0 passed; 0 failed; 1 ignored'), 0), + (passed.replace(self.name, self.name + '-other'), 0), + (passed.replace('... ok', '... FAILED'), 0)): + with self.subTest(output=output, code=code), self.assertRaises(ValueError): + native.test_result(output, self.name, code) + + def test_uncontrolled_concurrency_is_explicitly_not_a_fence(self): + for key in ('overwrite-concurrency', 'version-delete-concurrency'): + self.assertIn('not deterministic fence', native.REGRESSIONS[key]['fenceKind']) + self.assertIn('cannot invent emitted barrier logs', native.REGRESSIONS['read-repair-lock-owner']['limits']) + + def test_wrong_source_modified_contract_or_collector_fail_before_execution(self): + base = {'schemaVersion': 1, 'candidateCommit': native.CANDIDATE, 'sourceFiles': {native.SOURCE: native.SOURCE_SHA}, + 'sourceUnmodified': True, 'regressions': native.REGRESSIONS, 'optimized': True, + 'collectorSha256': native.sha(Path(native.__file__)), 'binaryName': 'ecstore-native-tests', 'binarySha256': '0' * 64} + for field, value in (('candidateCommit', '0' * 40), ('sourceUnmodified', False), + ('regressions', {}), ('collectorSha256', '0' * 64), ('binaryName', '../foreign')): + with self.subTest(field=field), tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + (directory / 'native-regressions.json').write_text(json.dumps({**base, field: value})) + with self.assertRaises(ValueError): + native.run(directory, ['read-repair-final-rename'], directory / 'evidence') + self.assertFalse((directory / 'evidence').exists()) + + +if __name__ == '__main__': + unittest.main() From 7ef1b0ef3b65e043a601fc4f84bbbe11c45a0af1 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 06:08:58 +0800 Subject: [PATCH 084/343] docs(validation): record complete live cursor acceptance --- docs/validation-migration-map.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index e715c9bc..da1701b1 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 122 currently dispatchable IDs include all 74 non-Heal cases and forty-eight Heal cases. Forty-four have live PASS evidence; the other 114 have no live PASS claim. +case evidence. The 122 currently dispatchable IDs include all 74 non-Heal cases and forty-eight Heal cases. Forty-five have live PASS evidence; the other 113 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1034,3 +1034,16 @@ fixtures, Admin nolock semantics and required fleet overlap remain explicit gaps The current implementation has 239 focused Heal/catalog checks, including the Linux-only D11 ownership/mount/recovery checks. Native compilation and execution still require the separate hosted job; local parser checks are not native PASS. + + +J03 corrected-budget run `38089238767` passed every within-window, +past-retention-gap, cross-node and terminal-truncation branch on both H1 and +H2. It retained complete DX responsibility, real running-status windows, +physical/streaming integrity, all-peer S3 evidence, cleanup PASS and the +independent canonical selected-case gate. This increases complete live PASS +to 45/158; branch-only successes and native regressions remain outside that +count. A06/F06/F01 and A02/E05/F05/J06/D11 queues continue serially on the +current auto-testing fleet. The optional native build waits for those queues +and uses the existing Functional Chain tools-only driver, because the reusable +workflow is still in a draft PR and cannot be dispatched directly from the +default branch. From 537c7c0bb4799b1f423f985b2675e1d3a41494fb Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 06:15:50 +0800 Subject: [PATCH 085/343] fix(validation): allow immutable startup IAM format controls --- docs/validation-migration-map.md | 16 ++- scripts/validation_heal/bucket_metadata.py | 15 ++- scripts/validation_heal/bucket_records.py | 134 ++++++++++++--------- test_validation_heal_bucket_metadata.py | 83 ++++++++++++- 4 files changed, 170 insertions(+), 78 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index da1701b1..868966f4 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -991,8 +991,12 @@ D11 now dispatches all four source branches separately on H1/H2 (eight fresh instances). Its dedicated node backend derives only two owned bucket-config object paths and their incarnation controls. It never widens the general user-object fault allowlist, walks IAM records, or prints bucket-policy/inline -payload bytes from physical metadata. A fresh namespace with no persisted IAM -subtree is required before and after execution. Exact system object geometry +payload bytes from physical metadata. The candidate startup persists a non-sensitive +`config/iam/format.json` object even in a fresh root-only service. Its exact +physical files and streaming proofs are read-only controls before and after +execution; mutation never targets that object. The six exact sensitive IAM +subtrees are checked for absence without traversal or payload reads. The reviewed +IAM manager source hash is bound alongside the bucket/XL2 contracts. Exact system object geometry binds each shard index to the original disk UUID/set slot; current inline or external bytes require candidate-bound streaming verification. @@ -1031,14 +1035,14 @@ four regression verdicts have their own archive and never increment a fleet or canonical-case PASS count. Existing tests assert captured lock events internally; the collector does not fabricate emitted barrier logs. Missing critical-point fixtures, Admin nolock semantics and required fleet overlap remain explicit gaps. -The current implementation has 239 focused Heal/catalog checks, including the +The current implementation has 242 focused Heal/catalog checks, including the Linux-only D11 ownership/mount/recovery checks. Native compilation and execution still require the separate hosted job; local parser checks are not native PASS. -J03 corrected-budget run `38089238767` passed every within-window, -past-retention-gap, cross-node and terminal-truncation branch on both H1 and -H2. It retained complete DX responsibility, real running-status windows, +J03 corrected-budget run `38089238767` passed every applicable within-window, +past-retention-gap and terminal-truncation branch on H1/H2. The cross-node +branch is explicitly NOT_APPLICABLE on single-node H1 and passed on H2. It retained complete DX responsibility, real running-status windows, physical/streaming integrity, all-peer S3 evidence, cleanup PASS and the independent canonical selected-case gate. This increases complete live PASS to 45/158; branch-only successes and native regressions remain outside that diff --git a/scripts/validation_heal/bucket_metadata.py b/scripts/validation_heal/bucket_metadata.py index f63fcace..0f8ec4c6 100644 --- a/scripts/validation_heal/bucket_metadata.py +++ b/scripts/validation_heal/bucket_metadata.py @@ -20,7 +20,7 @@ from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology from .inspector import verify as verify_inspector from .matrix import select - from .bucket_records import bucket_path + from .bucket_records import bucket_path, IAM_SENSITIVE_SUBTREES from .versions import validate_version_receipts from .stop import idle_fleet except ImportError: @@ -29,7 +29,7 @@ from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology from inspector import verify as verify_inspector from matrix import select - from bucket_records import bucket_path + from bucket_records import bucket_path, IAM_SENSITIVE_SUBTREES from versions import validate_version_receipts from stop import idle_fleet @@ -38,12 +38,13 @@ BRANCHES = ("versioning-config", "bucket-policy-tags", "lock-config", "unknown-record-negative") REFERENCE = {"candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", "sourceFiles": { "crates/ecstore/src/bucket/metadata.rs": "3b4f9ed4dba259f259653ba4bf1568c54a10b71ff0f6111135394e225a38d990", - "crates/filemeta/src/filemeta/codec.rs": "ef82b3390b0a377e423f99fb384c6a8a96beaf5722b0c907852cc0c904c4f04d"}, + "crates/filemeta/src/filemeta/codec.rs": "ef82b3390b0a377e423f99fb384c6a8a96beaf5722b0c907852cc0c904c4f04d", + "crates/iam/src/manager.rs": "c58d60946204973f8c5cfc81f08130614abb51102f31de0494ba4e22b88aa97c"}, "physicalRecord": ".rustfs.sys/buckets//.metadata.bin/xl.meta", "representation": "system object XL2 envelope, not a raw JSON or raw inner bucket-msgpack file", "unknownNegative": "outer XL2 major 1 -> 2; all other bytes retained; decoder rejects before checksum", "repairSourcesAllowed": ["candidate startup/config read", "explicit bucket Heal"], - "credentialBoundary": "fresh owned service; IAM subtree must remain absent; never enumerated"} + "credentialBoundary": "fresh owned root-only service; exact non-sensitive IAM format read-only; sensitive subtrees absent and never traversed"} GETTERS = ("get_bucket_versioning", "get_bucket_policy", "get_bucket_tagging", "get_object_lock_configuration") @@ -100,8 +101,10 @@ def system_files(value): result = {(node, row["disk"], row["path"]): {name: row[name] for name in ("size", "sha256")} for node, sample in value.items() for row in sample["files"]} require(len(result) == sum(len(sample["files"]) for sample in value.values()), "duplicate system file identity") - require(all(sample["credentialsRead"] is False and all(row["present"] is False and row["traversed"] is False - for row in sample["iamNamespace"]) for sample in value.values()), "credential boundary changed") + require(all(sample["credentialsRead"] is False and bool(sample["iamNamespace"]) + and all(row["traversed"] is False and row["formatOnlyRead"] is True + and row["sensitiveSubtrees"] == dict.fromkeys(IAM_SENSITIVE_SUBTREES, False) + for row in sample["iamNamespace"]) for sample in value.values()), "credential boundary changed") return result diff --git a/scripts/validation_heal/bucket_records.py b/scripts/validation_heal/bucket_records.py index 76e39d89..c69ce467 100644 --- a/scripts/validation_heal/bucket_records.py +++ b/scripts/validation_heal/bucket_records.py @@ -6,6 +6,10 @@ import uuid +IAM_SENSITIVE_SUBTREES = ("users", "service-accounts", "sts", "groups", "policies", "policydb") +IAM_FORMAT_KEY = "config/iam/format.json" + + def require(value, message): if not value: raise ValueError(message) @@ -116,69 +120,79 @@ def snapshot(roots, owner, request, checked, digest, tool, command): buckets = request["buckets"] require(isinstance(buckets, list) and len(buckets) == 2 and len(set(buckets)) == 2 and owner["bucketPrefix"] in buckets, "exact primary/control bucket pair required") + for bucket in buckets: + bucket_path(bucket, owner["bucketPrefix"]) entries, iam = [], [] for disk, root in enumerate(roots): data = root / "data" - # Root credentials live in the owned service environment. An IAM - # subtree is a precondition failure: never enumerate credential files. + # Startup writes the non-sensitive IAM format object even for a fresh + # root-only service. Check sensitive subtrees by exact path only; never + # enumerate the IAM namespace or read any credential/config payload. iam_root = checked(data / ".rustfs.sys/config/iam", data) - iam.append({"disk": disk, "present": iam_root.exists(), "traversed": False}) - require(not iam_root.exists(), "D11 requires the fresh owned namespace without persisted IAM records") - for bucket in buckets: - for record_name in (".metadata.bin", ".bucket-incarnation"): - base = bucket_path(bucket, owner["bucketPrefix"]).rsplit("/", 1)[0] + "/" + record_name - folder = checked(data / base, data) - if not folder.exists(): - continue - for directory, children, names in os.walk(folder, followlinks=False): - for name in (*children, *names): - checked(type(folder)(directory) / name, data) - for name in sorted(names): - path = type(folder)(directory) / name - relative = path.relative_to(folder).as_posix() - segments = relative.split("/") - valid = relative == "xl.meta" or len(segments) == 2 and bool(re.fullmatch( - r"part\.[1-9][0-9]*(?:\.integrity\.[a-z0-9-]+)?", segments[1])) - if len(segments) == 2: - require(str(uuid.UUID(segments[0])) == segments[0], "noncanonical bucket-config dataDir") - require(valid and len(entries) < 128 and path.stat().st_size <= 4 * 1024**2, - "bucket-config record exceeds the exact file/byte allowlist") - entry = {"disk": disk, "bucket": bucket, "path": str(path.relative_to(data)), - "size": path.stat().st_size, "sha256": digest(path)} - if relative == "xl.meta": - decoded = command(str(tool), str(path), ".rustfs.sys", f"buckets/{bucket}/{record_name}", check=False) - if decoded.returncode == 0: - entry["metadata"] = json.loads(decoded.stdout) - versions = entry["metadata"].get("versions", []) - require(len(versions) == 1 and not versions[0].get("deleted"), - "system streaming proof requires one live record") - version = versions[0] - version_id = version.get("versionId") or "null" - args = [str(tool), "--verify-inline", str(path), ".rustfs.sys", - f"buckets/{bucket}/{record_name}", version_id] - if version.get("inline") is not None: - entry["streaming"] = [json.loads(command(*args).stdout)] - else: - data_dir = version.get("dataDir") - require(isinstance(data_dir, str) and str(uuid.UUID(data_dir)) == data_dir, - "system record lacks a canonical external dataDir") - proofs = [] - for part in version["parts"]: - number = part["number"] - require(type(number) is int and 0 < number <= 8, "unbounded system record part") - part_path = checked(folder / data_dir / f"part.{number}", data) - proofs.append(json.loads(command(str(tool), "--verify-part", str(path), ".rustfs.sys", - f"buckets/{bucket}/{record_name}", str(part_path), version_id, str(number)).stdout)) - require(proofs, "system record has no streaming part proof") - entry["streaming"] = proofs - require(all(proof.get("physicalBitrotVerified") is True and proof.get("bucket") == ".rustfs.sys" - and proof.get("key") == f"buckets/{bucket}/{record_name}" for proof in entry["streaming"]), - "system record streaming proof differs from its exact identity") + sensitive = {name: checked(iam_root / name, data).exists() for name in IAM_SENSITIVE_SUBTREES} + require(not any(sensitive.values()), "D11 requires no persisted sensitive IAM subtree") + iam.append({"disk": disk, "present": iam_root.exists(), "traversed": False, + "sensitiveSubtrees": sensitive, "formatOnlyRead": True}) + objects = [(bucket, f"buckets/{bucket}/{name}", "bucket-record") for bucket in buckets + for name in (".metadata.bin", ".bucket-incarnation")] + objects.append((None, IAM_FORMAT_KEY, "iam-format")) + for bucket, key, record_class in objects: + base = ".rustfs.sys/" + key + folder = checked(data / base, data) + if not folder.exists(): + continue + for directory, children, names in os.walk(folder, followlinks=False): + for name in (*children, *names): + checked(type(folder)(directory) / name, data) + for name in sorted(names): + path = type(folder)(directory) / name + relative = path.relative_to(folder).as_posix() + segments = relative.split("/") + valid = relative == "xl.meta" or len(segments) == 2 and bool(re.fullmatch( + r"part\.[1-9][0-9]*(?:\.integrity\.[a-z0-9-]+)?", segments[1])) + if len(segments) == 2: + require(str(uuid.UUID(segments[0])) == segments[0], "noncanonical bucket-config dataDir") + require(valid and len(entries) < 128 and path.stat().st_size <= 4 * 1024**2, + "bucket-config record exceeds the exact file/byte allowlist") + entry = {"disk": disk, "bucket": bucket, "recordClass": record_class, "path": str(path.relative_to(data)), + "size": path.stat().st_size, "sha256": digest(path)} + if relative == "xl.meta": + decoded = command(str(tool), str(path), ".rustfs.sys", key, check=False) + if decoded.returncode == 0: + entry["metadata"] = json.loads(decoded.stdout) + require(entry["metadata"].get("bucket") == ".rustfs.sys" + and entry["metadata"].get("key") == key, "system decoder identity differs") + versions = entry["metadata"].get("versions", []) + require(len(versions) == 1 and not versions[0].get("deleted"), + "system streaming proof requires one live record") + version = versions[0] + version_id = version.get("versionId") or "null" + args = [str(tool), "--verify-inline", str(path), ".rustfs.sys", + key, version_id] + if version.get("inline") is not None: + entry["streaming"] = [json.loads(command(*args).stdout)] else: - expected = request.get("metadataFaults", {}).get(f"{disk}:{entry['path']}") - require(expected == {name: entry[name] for name in ("size", "sha256")}, - "bucket record decoding failed outside its exact witnessed mutation") - entry["decodeFailure"] = True - entries.append(entry) + data_dir = version.get("dataDir") + require(isinstance(data_dir, str) and str(uuid.UUID(data_dir)) == data_dir, + "system record lacks a canonical external dataDir") + proofs = [] + for part in version["parts"]: + number = part["number"] + require(type(number) is int and 0 < number <= 8, "unbounded system record part") + part_path = checked(folder / data_dir / f"part.{number}", data) + proofs.append(json.loads(command(str(tool), "--verify-part", str(path), ".rustfs.sys", + key, str(part_path), version_id, str(number)).stdout)) + require(proofs, "system record has no streaming part proof") + entry["streaming"] = proofs + require(all(proof.get("physicalBitrotVerified") is True and proof.get("bucket") == ".rustfs.sys" + and proof.get("key") == key for proof in entry["streaming"]), + "system record streaming proof differs from its exact identity") + else: + expected = request.get("metadataFaults", {}).get(f"{disk}:{entry['path']}") + require(record_class == "bucket-record" + and expected == {name: entry[name] for name in ("size", "sha256")}, + "bucket record decoding failed outside its exact witnessed mutation") + entry["decodeFailure"] = True + entries.append(entry) return {"files": entries, "iamNamespace": iam, "credentialsRead": False, - "scope": "two exact owned bucket metadata object directories; no system-directory traversal"} + "scope": "two exact owned bucket metadata object directories plus read-only IAM format; no IAM namespace walk"} diff --git a/test_validation_heal_bucket_metadata.py b/test_validation_heal_bucket_metadata.py index 07ae9b97..a8e9347d 100644 --- a/test_validation_heal_bucket_metadata.py +++ b/test_validation_heal_bucket_metadata.py @@ -1,6 +1,9 @@ import copy +import hashlib +import json from pathlib import Path import tempfile +from types import SimpleNamespace import unittest from unittest.mock import patch @@ -49,7 +52,7 @@ def scene(self, root, kind): def test_faults_keep_originals_and_reject_unrecognized_post_fault_writes(self): for kind in ("missing", "truncate", "future-xl-schema"): with self.subTest(kind=kind), tempfile.TemporaryDirectory() as temp: - root = Path(temp) + root = Path(temp).resolve() path, owner, request = self.scene(root, kind) with fault.directory(root) as fd: receipt = backend.apply(fd, owner, fault, request) @@ -72,7 +75,7 @@ def test_faults_keep_originals_and_reject_unrecognized_post_fault_writes(self): def test_symlink_and_hardlink_record_targets_are_rejected_before_mutation(self): for kind in ("symlink", "hardlink"): with self.subTest(kind=kind), tempfile.TemporaryDirectory() as temp: - root = Path(temp) + root = Path(temp).resolve() path, owner, request = self.scene(root, "truncate") other = root / "unrelated" other.write_bytes(self.value) @@ -99,22 +102,90 @@ def test_empty_default_success_and_transport_error_are_not_unknown_schema_refusa executor.configuration({"acknowledged": True, "resultUnknown": False, "response": {"ResponseMetadata": {}}}, "get_bucket_versioning") - def test_duplicate_files_or_persisted_iam_block_the_oracle(self): + def test_duplicate_files_or_sensitive_iam_block_the_oracle(self): row = {"disk": 0, "path": "exact-record", "size": 5, "sha256": "a" * 64} sample = {"node": {"credentialsRead": False, "files": [row], - "iamNamespace": [{"disk": 0, "present": False, "traversed": False}]}} + "iamNamespace": [{"disk": 0, "present": True, "traversed": False, "formatOnlyRead": True, + "sensitiveSubtrees": dict.fromkeys(backend.IAM_SENSITIVE_SUBTREES, False)}]}} executor.system_files(sample) for kind in ("duplicate", "iam", "traversal"): broken = copy.deepcopy(sample) if kind == "duplicate": broken["node"]["files"].append(row) elif kind == "iam": - broken["node"]["iamNamespace"][0]["present"] = True + broken["node"]["iamNamespace"][0]["sensitiveSubtrees"]["users"] = True else: broken["node"]["iamNamespace"][0]["traversed"] = True with self.assertRaises(ValueError): executor.system_files(broken) + def observe_format(self, root, *, corrupt=False): + path = root / "data/.rustfs.sys" / backend.IAM_FORMAT_KEY / "xl.meta" + calls, reads = [], [] + + def digest(file): + reads.append(file) + return hashlib.sha256(file.read_bytes()).hexdigest() + + def command(*args, **kwargs): + calls.append(args) + if corrupt: + return SimpleNamespace(returncode=1, stdout="") + if args[1] == "--verify-inline": + value = {"physicalBitrotVerified": True, "bucket": ".rustfs.sys", "key": backend.IAM_FORMAT_KEY} + else: + value = {"bucket": ".rustfs.sys", "key": backend.IAM_FORMAT_KEY, + "versions": [{"deleted": False, "versionId": "null", "inline": {"length": 42}}]} + return SimpleNamespace(returncode=0, stdout=json.dumps(value)) + + result = backend.snapshot((root,), {"bucketPrefix": self.bucket}, + {"buckets": [self.bucket, self.bucket + "-control"], + "metadataFaults": {"0:" + str(path.relative_to(root / "data")): + {"size": path.stat().st_size, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}}}, + node.checked, digest, Path('/inspector'), command) + return result, reads, calls + + def test_startup_format_is_read_only_frozen_and_verified_without_payload_output(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + path = root / "data/.rustfs.sys" / backend.IAM_FORMAT_KEY / "xl.meta" + path.parent.mkdir(parents=True) + path.write_bytes(self.value) + observed, reads, calls = self.observe_format(root) + self.assertEqual(reads, [path]) + self.assertEqual([call[-1] for call in calls], [backend.IAM_FORMAT_KEY, "null"]) + self.assertEqual(observed["files"][0]["recordClass"], "iam-format") + self.assertTrue(observed["iamNamespace"][0]["present"]) + self.assertNotIn("same-run-config-envelope", json.dumps(observed)) + before = executor.system_files({"node": observed}) + path.write_bytes(self.value + b"changed") + after, _, _ = self.observe_format(root) + self.assertNotEqual(before, executor.system_files({"node": after})) + with self.assertRaises(ValueError): + backend.record_parts(str(path.relative_to(root / "data")), self.bucket, self.bucket) + + def test_sensitive_subtree_presence_blocks_before_any_walk_or_file_read(self): + for subtree in backend.IAM_SENSITIVE_SUBTREES: + with self.subTest(subtree=subtree), tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + folder = root / "data/.rustfs.sys/config/iam" / subtree + folder.mkdir(parents=True) + (folder / "never-read").write_bytes(b"sensitive") + with patch.object(backend.os, "walk", side_effect=AssertionError("IAM walk")), \ + self.assertRaisesRegex(ValueError, "sensitive IAM subtree"): + backend.snapshot((root,), {"bucketPrefix": self.bucket}, + {"buckets": [self.bucket, self.bucket + "-control"]}, node.checked, + lambda _: self.fail("credential read"), Path('/inspector'), lambda *_: self.fail("decode")) + + def test_iam_format_decode_failure_cannot_use_bucket_fault_exception(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp).resolve() + path = root / "data/.rustfs.sys" / backend.IAM_FORMAT_KEY / "xl.meta" + path.parent.mkdir(parents=True) + path.write_bytes(self.value) + with self.assertRaisesRegex(ValueError, "decoding failed"): + self.observe_format(root, corrupt=True) + def test_remote_node_script_compiles_with_dedicated_backend(self): directory = Path(__file__).parent / "scripts/validation_heal" source = (directory / "fleet_node.py").read_text() @@ -136,7 +207,7 @@ def test_system_snapshot_receives_the_real_service_bucket_prefix(self): @unittest.skipUnless(Path('/proc/self/fdinfo').is_dir(), 'Linux fd/mount guard required') def test_first_system_fault_creates_a_checked_mount_and_lease_marker(self): with tempfile.TemporaryDirectory() as temp: - root = Path(temp) + root = Path(temp).resolve() path, _, request = self.scene(root, "missing") owner = {"runId": "test-run", "bucket": self.bucket} with patch.object(node, "owned", return_value=(root, (root,), owner)), patch.object(node, "stopped"), \ From 03e885b9b13cfed8f28ab18e16812d29fc16a1a9 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 06:27:56 +0800 Subject: [PATCH 086/343] feat(validation): migrate independent write and lock quorum negatives --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 39 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 31 ++ scripts/validation_heal/healthy.py | 12 + scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/quorum_gate.py | 143 +++++++ scripts/validation_heal/run_case.sh | 1 + scripts/validation_heal/write_quorum.py | 350 ++++++++++++++++++ test_validation_heal_write_quorum.py | 174 +++++++++ 10 files changed, 752 insertions(+), 6 deletions(-) create mode 100644 scripts/validation_heal/quorum_gate.py create mode 100644 scripts/validation_heal/write_quorum.py create mode 100644 test_validation_heal_write_quorum.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 9ea98e14..005a4f1e 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -24,6 +24,7 @@ on: - test_validation_heal_performance.py - test_validation_heal_scopes.py - test_validation_heal_bucket_metadata.py + - test_validation_heal_write_quorum.py - test_validation_heal_native.py - scripts/validation_heal/** - docs/validation-heal-*.md @@ -62,6 +63,7 @@ on: - test_validation_heal_performance.py - test_validation_heal_scopes.py - test_validation_heal_bucket_metadata.py + - test_validation_heal_write_quorum.py - test_validation_heal_native.py - scripts/validation_heal/** - docs/validation-heal-*.md @@ -94,7 +96,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 868966f4..3ae01908 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 122 currently dispatchable IDs include all 74 non-Heal cases and forty-eight Heal cases. Forty-five have live PASS evidence; the other 113 have no live PASS claim. +case evidence. The 123 currently dispatchable IDs include all 74 non-Heal cases and forty-nine Heal cases. Forty-five have live PASS evidence; the other 113 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 36 Heal cases remain +The other 35 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -1035,7 +1035,7 @@ four regression verdicts have their own archive and never increment a fleet or canonical-case PASS count. Existing tests assert captured lock events internally; the collector does not fabricate emitted barrier logs. Missing critical-point fixtures, Admin nolock semantics and required fleet overlap remain explicit gaps. -The current implementation has 242 focused Heal/catalog checks, including the +The current implementation has 252 focused Heal/catalog checks, including the Linux-only D11 ownership/mount/recovery checks. Native compilation and execution still require the separate hosted job; local parser checks are not native PASS. @@ -1051,3 +1051,36 @@ current auto-testing fleet. The optional native build waits for those queues and uses the existing Functional Chain tools-only driver, because the reusable workflow is still in a draft PR and cannot be dispatched directly from the default branch. + + +B08 now has an automatic current-fleet executor for both source branches. +The write-negative branch measures every original metadata read and root write +inside the exact candidate process mount namespace. Precisely N-W+1 private +roots return EROFS, including the missing payload target; all N metadata +members and all lock clients remain online. The H2 lock-negative branch +isolates two exact run-owned peer processes, witnesses their exited MainPIDs +and refused endpoints, and keeps R metadata/payload sources and shared-lock +quorum while losing exclusive-lock and object-write quorum. It does not claim +host network partition or physical disk loss. H1 uses an in-process local +manager; its distributed peer-loss branch is explicitly NOT_APPLICABLE with +the reviewed candidate locking policy, while its write-negative branch remains +mandatory. Both original branch IDs and H1/H2 contracts are retained. + +Every negative scene is frozen before removing the fault. A single unique-key +PUT has no automatic retry; all unknown outcomes and success ACKs are queried +on every recovered peer. Unresolved outcomes retain the stopped namespace and +fail cleanup. The negative Heal must preserve complete failed/deferred exact +responsibility with zero repaired counters and no changed physical control. +After gate removal/source return, an independent task must reconstruct the +original missing target before any target GET or backup rescue. Streaming +verification, all-peer complete S3 manifests and clean finalization remain +required. Ten boundary, identity, mount, cleanup and unknown-ACK negatives join +the focused checks. B08 has no live PASS claim yet. + +A06 run `38090151509` retained FAIL/cleanup PASS on H1/H2 after its repeat-stop +assertion. It exposed a harness contract defect: the reviewed manager's +`root_recovery::cancel_pending` treats a retained terminal record as an +idempotent stop, so the public response is HTTP 200/stopped, rather than the +unknown-token HTTP 500 assumed by the harness. The original result remains +unchanged. Fixing the candidate-bound assertion and repeating the complete +canonical run are required; the observed acknowledgement alone is not PASS. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 2120a3f5..b4bc4357 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 1006e1de..f45a44fc 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -235,6 +235,7 @@ def start(request): "--property=WorkingDirectory=" + str(state), "--property=EnvironmentFile=" + str(state / "service.env"), *gate_module().start_properties(gate_module().checked_gate(state, roots)), + *quorum_gate_module().properties(state, roots), *replacement_module().start_properties(state, roots), str(state / "rustfs"), *owner["volumes"]) return {"unit": owner["unit"], "started": True} @@ -412,6 +413,31 @@ def gate_module(): return drive_gate +def quorum_gate_module(): + if "quorum_gate_backend" in globals(): return globals()["quorum_gate_backend"] + from . import quorum_gate + return quorum_gate + + +def quorum_gate_operation(request): + state, roots, owner = owned(request["runId"]) + module = quorum_gate_module() + if request["action"] == "quorum-gate": + stopped(owner) + return module.release(state, roots) if request.get("mode") == "clear" else module.apply(state, roots, request["disks"]) + before = process(request) + require(before["pid"] > 1 and digest(state / "rustfs") == owner["binarySha256"], "quorum probe candidate differs") + relative = request["relativePath"] + require(isinstance(relative, str) and relative.startswith(owner["bucket"] + "/") + and relative.endswith("/xl.meta"), "quorum read probe escaped its owned business bucket") + for root in roots: + checked(root / "data" / relative, root / "data") + observed = module.probe(state, roots, before["pid"], relative, request["expectedReads"]) + after = process(request) + require(all(before.get(name) == after.get(name) for name in PROCESS_FIELDS), "quorum probe crossed candidate process identity") + return {**observed, "processBefore": before, "processAfter": after} + + def replacement_module(): if "replacement" in globals(): return globals()["replacement"] from . import replace_drive @@ -422,6 +448,7 @@ def replace_drive(request): state, roots, owner = owned(request["runId"]) stopped(owner) require(gate_module().checked_gate(state, roots) is None, "replacement cannot share an active drive gate") + require(quorum_gate_module().checked(state, roots) is None, "replacement cannot share an active quorum gate") return replacement_module().begin(state, roots, request.get("disk"), request.get("expectedFormatSha256")) @@ -484,6 +511,7 @@ def drive_gate(request): state, roots, owner = owned(request["runId"]) stopped(owner) if request.get("mode") == "clear": return gate_module().release_gate(state, roots) + require(quorum_gate_module().checked(state, roots) is None, "single drive gate cannot share a quorum gate") return gate_module().apply_gate(state, roots, request.get("disk"), request.get("mode")) @@ -772,6 +800,7 @@ def cleanup(request): pid = command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"], check=False).stdout.strip() require(pid in {"", "0"}, "run-owned service remains active") gate_module().release_gate(state, roots) + quorum_gate_module().release(state, roots) replacement_module().attest(state, roots) command("systemctl", "reset-failed", owner["unit"], check=False) for root in (*roots, state): @@ -870,6 +899,8 @@ def main(request): actions["task-records"] = task_records for action in ("bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover"): actions[action] = bucket_record_operation + for action in ("quorum-gate", "quorum-gate-probe"): + actions[action] = quorum_gate_operation actions.update({"replace-drive": replace_drive, "replacement-state": replacement_state, "replacement-probe": replacement_probe}) actions.update({"format-export": format_export, "foreign-format": foreign_format, "restore-replacement": restore_replacement}) actions.update({"process": process, "process-metrics": process_metrics, "process-signal": process_signal, "process-graceful": process_graceful, "process-crash": process_crash}) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 415dcdac..3c6dc504 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -149,6 +149,7 @@ def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_prof self.faults = [] self.bucket_record_faults = [] self.drive_gates = set() + self.quorum_gates = set() self.secret_values = set() self.frozen_processes = {} self.snapshot_buckets = None @@ -190,6 +191,9 @@ def remote(self, node, action, **params): source += "\nimport types\nbucket_records = types.ModuleType('bucket_records')\nexec(" + repr( NODE_SOURCE.with_name("bucket_records.py").read_text()) + ", bucket_records.__dict__)\n" replacement_actions = {"start", "cleanup", "audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot"} + if action in {"start", "cleanup", "audit-finalize", "drive-gate", "replace-drive", "quorum-gate", "quorum-gate-probe"}: + source += "\nimport types\nquorum_gate_backend = types.ModuleType('quorum_gate')\nexec(" + repr( + NODE_SOURCE.with_name("quorum_gate.py").read_text()) + ", quorum_gate_backend.__dict__)\n" if action in {"drive-gate", "drive-gate-probe", "drive-gate-activate", "gate-state", "drive-gate-release-live"} | replacement_actions: source += "\nimport types\ngate = types.ModuleType('drive_gate')\nexec(" + repr( NODE_SOURCE.with_name("drive_gate.py").read_text()) + ", gate.__dict__)\n" @@ -298,6 +302,14 @@ def cleanup(self): self.drive_gates.remove(node) except Exception as exc: failures.append({"error": str(exc), "state": "DIRTY", "namespacesRetained": True}) + if self.quorum_gates: + try: + self.stop() + for node in tuple(self.quorum_gates): + self.record("quorum-gate-recovery", {"node": node, **self.remote(node, "quorum-gate", mode="clear")}) + self.quorum_gates.remove(node) + except Exception as exc: + failures.append({"error": str(exc), "state": "DIRTY", "namespacesRetained": True}) for node in self.reserved: try: self.record("logs", {"node": node, **self.remote(node, "logs")}) diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index a8a1aee7..68ea6e19 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/quorum_gate.py b/scripts/validation_heal/quorum_gate.py new file mode 100644 index 00000000..97a5212b --- /dev/null +++ b/scripts/validation_heal/quorum_gate.py @@ -0,0 +1,143 @@ +"""Read-only gates for exact run-owned roots inside the candidate's mount namespace.""" + +import errno +import json +import os +from pathlib import Path +import stat +import subprocess + + +MARKER = "write-quorum-gate.json" +PROBE_SOURCE = '''import errno,hashlib,json,os,stat,sys +folder,metadata = sys.argv[1:] +result = {"uid": os.geteuid(), "gid": os.getegid()} +try: + fd = os.open(metadata, os.O_RDONLY | os.O_NOFOLLOW) + digest = hashlib.sha256() + count = 0 + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_size > 64*1024**2: + raise ValueError("expected bounded single-link metadata") + while chunk := os.read(fd, 1024**2): + count += len(chunk) + if count > 64*1024**2: raise ValueError("metadata grew") + digest.update(chunk) + finally: + os.close(fd) + result["read"] = {"errno": 0, "size": count, "sha256": digest.hexdigest()} +except OSError as exc: + result["read"] = {"errno": exc.errno} +name = folder + "/.validation-quorum-write-probe" +try: + fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) +except OSError as exc: + result["writeErrno"] = exc.errno +else: + try: + os.write(fd, b"quorum-probe") + os.fsync(fd) + finally: + os.close(fd) + os.unlink(name) + result["writeErrno"] = 0 +print(json.dumps(result)) +''' + + +def require(value, message): + if not value: + raise ValueError(message) + + +def checked(state, roots): + path = state / MARKER + if not path.exists(): + return None + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_uid == 0 and info.st_nlink == 1 + and stat.S_IMODE(info.st_mode) == 0o600 and info.st_size < 4096, "invalid quorum gate marker") + value = json.loads(path.read_text()) + entries = value.get("roots") + require(value.get("schemaVersion") == 1 and value.get("mode") == "read-only" + and isinstance(entries, list) and 0 < len(entries) <= len(roots), "invalid quorum gate contract") + disks = [row.get("disk") for row in entries] + require(all(type(disk) is int and 0 <= disk < len(roots) for disk in disks) + and disks == sorted(set(disks)), "quorum gate disks differ") + for row in entries: + folder = roots[row["disk"]] / "data" + info = folder.lstat() + require(row["path"] == str(folder) and not folder.is_symlink() and stat.S_ISDIR(info.st_mode) + and info.st_uid == 0 and stat.S_IMODE(info.st_mode) == 0o700 + and row["device"] == info.st_dev and row["inode"] == info.st_ino, + "quorum gate root ownership/device/inode drifted; retain namespace") + return value + + +def apply(state, roots, disks): + require(isinstance(disks, list) and bool(disks) and disks == sorted(set(disks)) + and all(type(disk) is int and 0 <= disk < len(roots) for disk in disks), "invalid quorum gate disk list") + require(checked(state, roots) is None and not (state / "drive-gate.json").exists() + and not (state / "disk-replacement.json").exists(), "quorum gate cannot share another drive mutation") + rows = [] + for disk in disks: + folder = roots[disk] / "data" + info = folder.lstat() + require(not folder.is_symlink() and stat.S_ISDIR(info.st_mode) and info.st_uid == 0 + and stat.S_IMODE(info.st_mode) == 0o700, "expected private owned data root") + rows.append({"disk": disk, "path": str(folder), "device": info.st_dev, "inode": info.st_ino}) + value = {"schemaVersion": 1, "mode": "read-only", "roots": rows} + fd = os.open(state / MARKER, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(value, stream) + stream.flush() + os.fsync(stream.fileno()) + require(checked(state, roots) == value, "quorum gate intent differs") + return {**value, "state": "ARMED", "hostMountRemounted": False, "sharedPermissionsChanged": False} + + +def properties(state, roots): + value = checked(state, roots) + return [] if value is None else ["--property=PrivateMounts=true", + "--property=ReadOnlyPaths=" + " ".join(row["path"] for row in value["roots"])] + + +def release(state, roots): + value = checked(state, roots) + if value is not None: + (state / MARKER).unlink() + return {"state": "CLEAR", "gateWasAbsent": value is None, "hostMountRemounted": False, + "privateMountRemovedByStoppedService": True} + + +def validate_probe(observed, expected_read, read_only): + require(observed.get("uid") == observed.get("gid") == 0, "quorum probe differs from root service identity") + require(observed.get("read") == {"errno": 0, **expected_read}, "quorum probe cannot read exact original metadata") + require(observed.get("writeErrno") == (errno.EROFS if read_only else 0), + "quorum probe did not witness the expected actual writable/refused root") + return {"readableMetadata": True, "writable": not read_only, "writeErrno": observed["writeErrno"], + "sameServiceUidAndMountNamespace": True} + + +def probe(state, roots, pid, relative, expected_reads): + value = checked(state, roots) + gated = {row["disk"] for row in value["roots"]} if value else set() + require(type(pid) is int and pid > 1 and len(expected_reads) == len(roots), "live service and all disk expectations required") + namespace = Path(f"/proc/{pid}/ns/mnt") + if gated: + require(namespace.readlink() != Path('/proc/self/ns/mnt').readlink(), "read-only gate lacks private mount namespace") + fields = dict(line.split(":", 1) for line in Path(f"/proc/{pid}/status").read_text().splitlines() if ":" in line) + require(list(map(int, fields["Uid"].split()))[1] == list(map(int, fields["Gid"].split()))[1] == 0, + "quorum candidate effective identity changed") + rows = [] + for disk, root in enumerate(roots): + result = subprocess.run(["nsenter", "--mount=" + str(namespace), "python3", "-c", PROBE_SOURCE, + str(root / "data"), str(root / "data" / relative)], + capture_output=True, text=True, timeout=15) + require(result.returncode == 0, "candidate-namespace quorum probe did not complete") + observed = json.loads(result.stdout) + rows.append({"disk": disk, "observed": observed, + **validate_probe(observed, expected_reads[disk], disk in gated)}) + return {"disks": rows, "backend": "candidate-service-private-read-only-mounts", + "hostMountRemounted": False, "sharedPermissionsChanged": False} diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 52391d6a..f9c3c2cf 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -32,6 +32,7 @@ case "${case_id}" in heal.admin-scope-isolation) driver=scopes.py ;; heal.bucket-metadata-integrity) driver=bucket_metadata.py ;; heal.disk-replacement-unformatted) driver=unformatted.py ;; + heal.below-write-quorum) driver=write_quorum.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/scripts/validation_heal/write_quorum.py b/scripts/validation_heal/write_quorum.py new file mode 100644 index 00000000..1c44298a --- /dev/null +++ b/scripts/validation_heal/write_quorum.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""B08: independently witnessed write and distributed-lock quorum loss.""" + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +from urllib import request +from urllib.error import URLError + +try: + from .heal_probe import Probe, payload, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .shards import SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort + from .versions import validate_version_receipts, verify_quorum_contract, verify_version_s3 + from .drive_gate import validate_unwritable + from .quorum_boundary import LOCK_REFERENCE + from .stop import idle_fleet +except ImportError: + from heal_probe import Probe, payload, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from shards import SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort + from versions import validate_version_receipts, verify_quorum_contract, verify_version_s3 + from drive_gate import validate_unwritable + from quorum_boundary import LOCK_REFERENCE + from stop import idle_fleet + + +CASE_ID = "heal.below-write-quorum" +BRANCHES = ("read-present-write-absent", "lock-quorum-absent") +REFUSALS = {"XMinioWriteQuorum", "XMinioLockTimeout", "InternalError", "ServiceUnavailable", "SlowDown", "RequestTimeout"} +LOCAL_LOCK_REASON = ("H1 uses the candidate's in-process local manager, not a distributed client quorum. " + "Losing its only process also removes the read endpoint; distributed peer-loss is applicable on H2 only.") + + +def plan_for(snapshot, target, branch, contract): + require(branch in BRANCHES and contract.get("recognized") is True + and contract.get("writeQuorumRule") == "data-plus-one-if-equal-parity", "unrecognized B08 contract") + cohort = target["cohort"] + geometry = target["logical"]["erasure"] + read, parity = geometry["data_blocks"], geometry["parity_blocks"] + write, total = read + int(read == parity), read + parity + hosts = list(snapshot) + require(len(hosts) in {1, 4} and total == len(cohort) == 4 * len(hosts) + and len({(row["node"], row["disk"]) for row in cohort}) == total + and all({row["disk"] for row in cohort if row["node"] == node} == set(range(4)) for node in hosts), + "B08 requires actual full H1/H2 member identities") + require(read < write and total - 1 >= read, "layout cannot preserve R while removing W") + lock = {"mode": "local" if len(hosts) == 1 else "distributed", "clients": len(hosts), + "sharedQuorum": (len(hosts) + 1) // 2, "exclusiveQuorum": len(hosts) // 2 + 1} + if branch == "lock-quorum-absent" and len(hosts) == 1: + return {"applicable": False, "reason": LOCAL_LOCK_REASON, "readQuorum": read, + "writeQuorum": write, "lock": lock} + ordered = sorted(cohort, key=lambda row: (hosts.index(row["node"]), row["disk"])) + if branch == "read-present-write-absent": + # Include the precise missing target in the read-only set. All N + # metadata authorities and lock clients remain readable/online. + selected = [next(row for row in ordered if (row["node"], row["disk"]) == (target["node"], target["disk"]))] + selected += [row for row in ordered if (row["node"], row["disk"]) != (target["node"], target["disk"])][:total - write] + gates = {node: sorted(row["disk"] for row in selected if row["node"] == node) for node in hosts} + gates = {node: disks for node, disks in gates.items() if disks} + offline = [] + sources, metadata, writable, clients = total - 1, total, write - 1, len(hosts) + else: + offline = hosts[lock["exclusiveQuorum"] - 1:] + require(len(offline) == 2, "H2 must lose exactly two distinct lock clients") + # The missing target belongs to an offline member, preserving all R + # payload sources on the survivors. Do not subtract it twice. + require(target["node"] in offline, "lock-negative missing target must be on an unavailable member") + gates = {} + sources = metadata = writable = total - 4 * len(offline) + clients = len(hosts) - len(offline) + require(sources >= read and writable < write and metadata >= read + and (clients >= lock["exclusiveQuorum"] if gates else lock["sharedQuorum"] <= clients < lock["exclusiveQuorum"]), + "B08 fault does not preserve R or independently remove W/lock admission") + return {"applicable": True, "readQuorum": read, "writeQuorum": write, "totalDrives": total, + "readablePayloadSources": sources, "readableMetadataAuthorities": metadata, + "writableDrives": writable, "lock": {**lock, "availableClients": clients}, + "gates": gates, "offlineNodes": offline, + "backend": "service-private-read-only-mounts" if gates else "exact-owned-peer-process-stop", + "hostNetworkRulesChanged": False, "hostMountRemounted": False} + + +def expected_reads(snapshot, node, relative): + indexed = files(snapshot) + result = [] + for disk in range(4): + entry = indexed.get((node, disk, relative)) + require(entry is not None, "quorum witness lacks original metadata on a selected disk") + result.append({name: entry[name] for name in ("size", "sha256")}) + return result + + +def validate_gate_witness(observed, plan, online): + require(set(observed) == set(online), "quorum probe omitted an available candidate") + rows = [(node, row) for node, sample in observed.items() for row in sample["disks"]] + require(len(rows) == 4 * len(online) and len({(node, row["disk"]) for node, row in rows}) == len(rows) + and all(row["readableMetadata"] is True and row["sameServiceUidAndMountNamespace"] is True for _, row in rows), + "quorum gate witness omitted or duplicated real metadata authorities") + readonly = {(node, disk) for node, disks in plan["gates"].items() for disk in disks} + require(all(type(row["writable"]) is bool and row["writable"] is ((node, row["disk"]) not in readonly) + and row["writeErrno"] == (30 if (node, row["disk"]) in readonly else 0) for node, row in rows) + and sum(row["writable"] for _, row in rows) == plan["writableDrives"], + "measured service write refusals differ from the frozen W boundary") + return {"measuredReadableMetadataAuthorities": len(rows), "measuredWritableDrives": plan["writableDrives"], + "diskCountsAreNotLockAdmission": True} + + +def negative_put(observed): + require(observed.get("acknowledged") is False and observed.get("clientCallInvoked") is True, + "quorum-negative PUT returned a success ACK or was never invoked") + if observed.get("resultUnknown") is True: + return {"state": "UNKNOWN_PENDING_RESOLUTION", "retryAllowed": False} + response = observed.get("response") or {} + require(observed.get("resultUnknown") is False and response.get("ResponseMetadata", {}).get("HTTPStatusCode") in {500, 503} + and response.get("Error", {}).get("Code") in REFUSALS, + "quorum-negative PUT lacks a structured candidate refusal") + return {"state": "REFUSED", "retryAllowed": False, "code": response["Error"]["Code"]} + + +def resolve_put(observed, reads, expected_payload): + require(bool(reads), "unique PUT outcome was not queried after recovery") + states = [] + for read in reads: + if read.get("acknowledged") is True and read.get("resultUnknown") is False: + require(read.get("payload") == expected_payload, "uncertain PUT resolved to incorrect committed bytes") + states.append("COMMITTED") + else: + response = read.get("response") or {} + require(read.get("acknowledged") is False and read.get("resultUnknown") is False + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404 + and response.get("Error", {}).get("Code") == "NoSuchKey", "unique PUT outcome remains unresolved") + states.append("ABSENT") + require(len(set(states)) == 1, "peers disagree on the unique PUT commit outcome") + if observed.get("acknowledged") is True: + require(states[0] == "COMMITTED", "a success PUT ACK cannot be redeemed on the recovered peers") + else: + require(observed.get("resultUnknown") is True or states[0] == "ABSENT", "refused PUT nevertheless committed") + return {"state": states[0], "allPeersAgree": True, "retryIssued": False, "successAckClaimed": False} + + +def endpoint_refused(binding, node_index): + endpoint = binding["endpoints"][f"node{node_index + 1}"] + try: + with request.build_opener(request.ProxyHandler({})).open(endpoint + "/health/ready", timeout=2): + raise ValueError("stopped lock peer still accepts requests") + except URLError as exc: + require(isinstance(exc.reason, ConnectionRefusedError), "lock peer failure is not witnessed endpoint refusal") + return {"endpoint": endpoint, "connectionRefused": True, "lockAndDiskEndpointUnavailable": True} + + +def run_branch(args, topology, nodes, branch): + directory = args.evidence / topology / branch + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory) + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == LOCK_REFERENCE["candidateCommit"], "B08 lock client policy is unreviewed") + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), "maxInventoryObjects": 256, + "maxInventoryBytes": 512 * 1024**2} + fleet.record("lock-policy-source-contract", LOCK_REFERENCE) + contract = verify_quorum_contract(fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)), candidate) + fleet.record("binding", binding) + probe, active = Probe(binding), [] + result = {"topology": topology, "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} + put_observed, put_key, put_payload, original = None, None, None, None + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + expected = seed(Recorder(probe, directory / "fixture"), bucket, "DB", 128, SIZES) + baseline = fleet.record("S3-baseline", probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + fleet.stop() + original = fleet.snapshot("physical-baseline", quiescent=True) + validate_topology(original, topology) + selected = next(entry for entry in expected if entry["payload"]["size"] > 1024**2) + target = map_target(original, bucket, selected["key"], "data", 1) + if branch == "lock-quorum-absent" and topology == "H2": + shard = next(row for row in target["cohort"] if row["node"] == nodes[-1]) + target = {**target, **shard} + plan = fleet.record("frozen-quorum-plan", plan_for(original, target, branch, contract)) + if not plan["applicable"]: + result.update(verdict="NOT_APPLICABLE", reason=plan["reason"]) + return result + params = {"disk": target["disk"], "faultId": "B08-missing-target"} + registration = target["node"], params + fleet.faults.append(registration) + receipt = fleet.record("target-fault", fleet.remote(target["node"], "apply-fault", **params, + relativePath=target["relativePath"], kind="missing", expectedSha256=target["sha256"])) + faulted = fleet.snapshot("physical-fault", quiescent=True) + validate_fault(original, faulted, target, receipt) + for node, disks in plan["gates"].items(): + fleet.quorum_gates.add(node) + fleet.record("write-gate-intent", {"node": node, **fleet.remote(node, "quorum-gate", disks=disks)}) + online = [node for node in nodes if node not in plan["offlineNodes"]] + if plan["offlineNodes"]: + # Start the intact cluster first, then isolate its exact two owned + # peer processes. No host firewall, mount or VM is changed. + fleet.start() + for node in plan["offlineNodes"]: + stopped = fleet.record("lock-peer-stopped", {"node": node, **fleet.remote(node, "stop")}) + require(stopped.get("quiescent") is True and stopped.get("service", {}).get("MainPID") == "0", + "lock peer process did not actually exit") + fleet.record("lock-peer-endpoint-refused", endpoint_refused(binding, nodes.index(node))) + else: + fleet.start() + relative = f"{bucket}/{selected['key']}/xl.meta" + def witness(label): + raw = fleet.record(label, {node: fleet.remote(node, "quorum-gate-probe", relativePath=relative, + expectedReads=expected_reads(original, node, relative)) for node in online}) + return validate_gate_witness(raw, plan, online) + fleet.record("quorums-before-operations", witness("service-quorum-before")) + require(physical_content(faulted) == physical_content(fleet.snapshot("physical-before-operations")), + "quorum loss changed the fault before its negative operations") + read = fleet.record("read-without-write-admission", probe.s3("get_object", {"Bucket": bucket, "Key": selected["key"]})) + require(read.get("acknowledged") is True and read.get("resultUnknown") is False + and read.get("payload") == selected["payload"], "read quorum did not serve exact acknowledged bytes") + put_key = "quorum-negative/unique-put.bin" + put_body = payload(f"{bucket}/{put_key}/0", 5 * 1024**2 + 97) + put_payload = {"size": len(put_body), "sha256": hashlib.sha256(put_body).hexdigest()} + put_observed = fleet.record("single-negative-PUT-no-retry", probe.s3("put_object", {"Bucket": bucket, "Key": put_key, + "Metadata": {"heal-fixture": "DL"}, "Tagging": "suite=heal&fixture=DL"}, put_body)) + result["put"] = negative_put(put_observed) + status = run_task(probe, fleet, {"bucket": bucket, "key": selected["key"]}, 2, active) + result["negativeResponsibility"] = validate_unwritable(status, bucket, selected) + fleet.record("quorums-after-operations", witness("service-quorum-after")) + for node in plan["offlineNodes"]: + fleet.record("lock-peer-still-refused", endpoint_refused(binding, nodes.index(node))) + # Evidence must precede any source return, gate removal, backup rescue + # or target GET on the recovered cluster. + fleet.stop() + scene = fleet.snapshot("physical-negative-before-recovery", quiescent=True) + require(physical_content(faulted) == physical_content(scene), "negative write/Heal mutated target or intact controls") + for node in nodes: + fleet.record("negative-quorum-peer-logs", {"node": node, **fleet.remote(node, "logs")}) + for node in tuple(fleet.quorum_gates): + fleet.record("release-write-quorum-gate", fleet.remote(node, "quorum-gate", mode="clear")) + fleet.quorum_gates.remove(node) + fleet.start() + require(physical_content(faulted) == physical_content(fleet.snapshot("physical-source-return-before-repair")), + "source return repaired the target before the independent manual recovery task") + recovered = run_task(probe, fleet, {"bucket": bucket, "key": selected["key"]}, 2, active, + status_endpoints=tuple(binding["endpoints"])[1:]) + result["recoveryResponsibility"] = validate_version_receipts(recovered, bucket, [selected], 2, + required_repairs={(selected["key"], "null")}) + idle_fleet(fleet, binding) + fleet.stop() + after = fleet.snapshot("physical-reconstructed-before-GET", quiescent=True) + result["physical"] = validate_repaired_physical(original, after, target) + result["streaming"] = verify_cohort(fleet, target, "all-original-cohort-bitrot") + fleet.faults.remove(registration) + fleet.start() + reads = [fleet.record("unique-PUT-resolution", Probe(binding, endpoint).s3("get_object", { + "Bucket": bucket, "Key": put_key})) for endpoint in binding["endpoints"]] + result["putResolution"] = resolve_put(put_observed, reads, put_payload) + # The complete physical no-mutation oracle also proves this particular + # unknown call did not commit. No uncertain call is silently discarded. + require(result["putResolution"]["state"] == "ABSENT", "unique PUT committed despite the proven W/lock loss") + result["s3"] = {endpoint: verify_version_s3(Probe(binding, endpoint), fleet, bucket, expected, baseline, + "all-peer-S3-" + endpoint, fixture="DB") for endpoint in binding["endpoints"]} + result["verdict"] = "PASS" + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + finally: + for path, token in active: + try: fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: fleet.record("stop-unresolved-error", {"error": str(exc)}) + # An unknown unique PUT is resolved even if a later assertion failed. + # Recovery cannot convert that original failed branch to PASS. + if put_observed and put_key and "putResolution" not in result: + try: + fleet.stop() + for node in tuple(fleet.quorum_gates): + fleet.record("failed-branch-release-gate", fleet.remote(node, "quorum-gate", mode="clear")) + fleet.quorum_gates.remove(node) + fleet.start() + reads = [fleet.record("failed-branch-unique-PUT-resolution", Probe(binding, endpoint).s3("get_object", { + "Bucket": bucket, "Key": put_key})) for endpoint in binding["endpoints"]] + result["putResolution"] = resolve_put(put_observed, reads, put_payload) + except Exception as exc: + result["putResolution"] = {"state": "UNRESOLVED", "error": str(exc)} + if result.get("putResolution", {}).get("state") == "UNRESOLVED": + fleet.stop() + fleet.record("retained-unresolved-PUT", {"state": "DIRTY", "namespacesRetained": True, + "reason": "the original unique request has no complete all-peer resolution"}) + clean = False + else: + clean = fleet.cleanup() + result["cleanup"] = "PASS" if clean else "FAIL" + fleet.record("branch-result", result) + return result + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f"B08 workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ("candidate", "rc-bin", "evidence", "tool-directory"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--case-id", choices=[CASE_ID], required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + args = parser.parse_args() + require(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha) and re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")), + "immutable workflow candidate identity required") + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "B08 source branch omitted") + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(nodes) == len(set(nodes)) and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", n) for n in nodes), + "four current Heal nodes required") + results = [] + try: + for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): + for branch in BRANCHES: + results.append(run_branch(args, topology, selected, branch)) + if results[-1]["cleanup"] != "PASS": break + if results[-1]["cleanup"] != "PASS": break + finally: + complete = len(results) == 4 and all(row["cleanup"] == "PASS" and (row["verdict"] == "PASS" + or row["topology"] == "H1" and row["branch"] == "lock-quorum-absent" + and row["verdict"] == "NOT_APPLICABLE" and row.get("reason") == LOCAL_LOCK_REASON) for row in results) + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "complete": complete, + "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL"}) + require(complete, "B08 requires full independent write/lock negative evidence, product recovery and all-peer ACK verification") + + +if __name__ == "__main__": + try: main() + except Exception as exc: + print(f"B08 write/lock quorum validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/test_validation_heal_write_quorum.py b/test_validation_heal_write_quorum.py new file mode 100644 index 00000000..cccbe400 --- /dev/null +++ b/test_validation_heal_write_quorum.py @@ -0,0 +1,174 @@ +import copy +import errno +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from scripts.validation_heal import quorum_gate as gate, write_quorum as executor +from scripts.validation_heal import healthy, shards +from scripts.validation_heal.matrix import AUTOMATIC_CASES, select +import test_validation_heal as oracle + + +class WriteQuorumTests(unittest.TestCase): + contract = {"recognized": True, "writeQuorumRule": "data-plus-one-if-equal-parity"} + + def scene(self, total=4, branch=executor.BRANCHES[0]): + source = oracle.HealReadQuorumBoundaryTest() + snapshot, expected = source.scene(total=total, data=total // 2) + target = shards.map_target(snapshot, source.bucket, expected[0]["key"], "data", 1) + if branch == executor.BRANCHES[1] and total == 16: + target = {**target, **next(row for row in target["cohort"] if row["node"] == "node4")} + return snapshot, target, executor.plan_for(snapshot, target, branch, self.contract) + + def test_payload_loss_does_not_count_as_actual_write_or_lock_loss(self): + for total in (4, 16): + snapshot, target, plan = self.scene(total) + self.assertEqual(plan["writableDrives"], plan["writeQuorum"] - 1) + self.assertEqual(sum(len(disks) for disks in plan["gates"].values()), total - plan["writeQuorum"] + 1) + self.assertIn(target["disk"], plan["gates"][target["node"]]) + self.assertGreaterEqual(plan["readablePayloadSources"], plan["readQuorum"]) + self.assertEqual(plan["readableMetadataAuthorities"], total) + self.assertEqual(plan["lock"]["availableClients"], len(snapshot)) + + def test_lock_negative_preserves_read_quorum_and_loses_actual_distinct_clients(self): + _, _, plan = self.scene(16, executor.BRANCHES[1]) + self.assertEqual(plan["offlineNodes"], ["node3", "node4"]) + self.assertEqual(plan["readablePayloadSources"], plan["readQuorum"]) + self.assertEqual(plan["lock"]["availableClients"], plan["lock"]["sharedQuorum"]) + self.assertLess(plan["lock"]["availableClients"], plan["lock"]["exclusiveQuorum"]) + snapshot, target, _ = self.scene(16) + with self.assertRaisesRegex(ValueError, "offline member|unavailable member"): + executor.plan_for(snapshot, target, executor.BRANCHES[1], self.contract) + + def test_h1_distributed_loss_is_explicitly_inapplicable_and_retains_both_source_branches(self): + _, _, plan = self.scene(4, executor.BRANCHES[1]) + self.assertFalse(plan["applicable"]) + self.assertEqual(plan["reason"], executor.LOCAL_LOCK_REASON) + self.assertEqual(select(executor.CASE_ID)[0]["variants"], list(executor.BRANCHES)) + self.assertIn(executor.CASE_ID, AUTOMATIC_CASES) + + def test_layout_and_source_uncertainty_cannot_be_assumed_sufficient(self): + snapshot, target, _ = self.scene(16) + for kind in ("duplicate-member", "other-node", "unrecognized", "different-policy", "no-read-write-gap"): + broken, contract = copy.deepcopy(target), dict(self.contract) + if kind == "duplicate-member": broken["cohort"][-1] = copy.deepcopy(broken["cohort"][0]) + elif kind == "other-node": broken["cohort"][-1]["node"] = "foreign" + elif kind == "unrecognized": contract["recognized"] = False + elif kind == "different-policy": contract["writeQuorumRule"] = "assumed" + else: broken["logical"]["erasure"].update(data_blocks=10, parity_blocks=6) + with self.subTest(kind=kind), self.assertRaises(ValueError): + executor.plan_for(snapshot, broken, executor.BRANCHES[0], contract) + + def test_measured_read_and_write_refusals_must_match_every_root(self): + _, _, plan = self.scene(4) + observed = {"node1": {"disks": [{"disk": disk, "readableMetadata": True, + "sameServiceUidAndMountNamespace": True, "writable": disk not in plan["gates"]["node1"], + "writeErrno": errno.EROFS if disk in plan["gates"]["node1"] else 0} for disk in range(4)]}} + executor.validate_gate_witness(observed, plan, ["node1"]) + for kind in ("missing", "duplicate", "still-writable", "not-readable", "eacces"): + broken = copy.deepcopy(observed) + rows = broken["node1"]["disks"] + if kind == "missing": rows.pop() + elif kind == "duplicate": rows[-1] = copy.deepcopy(rows[0]) + elif kind == "still-writable": rows[0].update(writable=True, writeErrno=0) + elif kind == "not-readable": rows[0]["readableMetadata"] = False + else: rows[0]["writeErrno"] = errno.EACCES + with self.subTest(kind=kind), self.assertRaises(ValueError): + executor.validate_gate_witness(broken, plan, ["node1"]) + + def test_unknown_put_is_resolved_without_retries_and_false_acks_are_rejected(self): + unknown = {"acknowledged": False, "resultUnknown": True, "clientCallInvoked": True} + self.assertFalse(executor.negative_put(unknown)["retryAllowed"]) + absent = {"acknowledged": False, "resultUnknown": False, "response": { + "ResponseMetadata": {"HTTPStatusCode": 404}, "Error": {"Code": "NoSuchKey"}}} + body = {"size": 19, "sha256": "a" * 64} + committed = {"acknowledged": True, "resultUnknown": False, "payload": body} + self.assertEqual(executor.resolve_put(unknown, [absent, absent], body)["state"], "ABSENT") + self.assertEqual(executor.resolve_put(unknown, [committed, committed], body)["state"], "COMMITTED") + ack = {**unknown, "acknowledged": True, "resultUnknown": False} + with self.assertRaises(ValueError): executor.negative_put(ack) + with self.assertRaises(ValueError): executor.resolve_put(ack, [absent], body) + self.assertEqual(executor.resolve_put(ack, [committed], body)["state"], "COMMITTED") + for reads in ([], [absent, committed], [unknown], [{**committed, "payload": {**body, "size": 1}}]): + with self.subTest(reads=reads), self.assertRaises(ValueError): + executor.resolve_put(unknown, reads, body) + + def test_structured_quorum_refusal_cannot_resolve_to_a_committed_put(self): + observed = {"acknowledged": False, "resultUnknown": False, "clientCallInvoked": True, + "response": {"ResponseMetadata": {"HTTPStatusCode": 503}, "Error": {"Code": "XMinioWriteQuorum"}}} + self.assertEqual(executor.negative_put(observed)["state"], "REFUSED") + for code, http in (("AccessDenied", 403), ("NoSuchBucket", 404), ("InternalError", 200)): + broken = {**observed, "response": {"ResponseMetadata": {"HTTPStatusCode": http}, "Error": {"Code": code}}} + with self.assertRaises(ValueError): executor.negative_put(broken) + with self.assertRaises(ValueError): + executor.resolve_put(observed, [{"acknowledged": True, "resultUnknown": False, "payload": {}}], {}) + + def test_probe_rejects_wrong_bytes_and_other_filesystem_errors(self): + expected = {"size": 42, "sha256": "a" * 64} + observed = {"uid": 0, "gid": 0, "read": {"errno": 0, **expected}, "writeErrno": errno.EROFS} + gate.validate_probe(observed, expected, True) + gate.validate_probe({**observed, "writeErrno": 0}, expected, False) + for kind in ("uid", "read", "write", "enospc"): + broken = copy.deepcopy(observed) + if kind == "uid": broken["uid"] = 1000 + elif kind == "read": broken["read"]["sha256"] = "b" * 64 + else: broken["writeErrno"] = 0 if kind == "write" else errno.ENOSPC + with self.subTest(kind=kind), self.assertRaises(ValueError): gate.validate_probe(broken, expected, True) + + def root_owned_stat(self, original): + def result(path, *args, **kwargs): + value = list(original(path, *args, **kwargs)) + value[4] = value[5] = 0 + return os.stat_result(value) + return result + + def test_gate_preserves_host_permissions_and_refuses_inode_drift_and_other_mutations(self): + with tempfile.TemporaryDirectory() as temp: + state = Path(temp).resolve() + roots = tuple(state / f"disk-{disk}" for disk in range(4)) + for root in roots: (root / "data").mkdir(parents=True, mode=0o700) + with patch.object(Path, "lstat", self.root_owned_stat(Path.lstat)): + gate.apply(state, roots, [0, 2]) + properties = gate.properties(state, roots) + self.assertIn("--property=PrivateMounts=true", properties) + self.assertEqual(properties[-1], "--property=ReadOnlyPaths=" + " ".join(str(roots[disk] / "data") for disk in (0, 2))) + self.assertTrue(all((root / "data").stat().st_mode & 0o777 == 0o700 for root in roots)) + (roots[0] / "data").rename(roots[0] / "saved") + (roots[0] / "data").mkdir(mode=0o700) + with self.assertRaisesRegex(ValueError, "drifted"): gate.release(state, roots) + self.assertTrue((state / gate.MARKER).exists()) + (roots[0] / "data").rmdir() + (roots[0] / "saved").rename(roots[0] / "data") + gate.release(state, roots) + (state / "drive-gate.json").write_text('{}') + with self.assertRaises(ValueError): gate.apply(state, roots, [0]) + + def test_cleanup_failure_retains_namespace_and_remote_script_includes_new_backend(self): + with tempfile.TemporaryDirectory() as temp: + fleet = healthy.Fleet(["node1"], Path(temp), "test-run", None) + fleet.quorum_gates.add("node1") + calls = [] + def remote(node, action, **kwargs): + calls.append(action) + if action == "quorum-gate": raise ValueError("mount identity drift") + return {} + fleet.remote = remote + self.assertFalse(fleet.cleanup()) + self.assertNotIn("cleanup", calls) + fleet.quorum_gates.clear() + with patch.object(healthy.subprocess, "run") as execute: + execute.return_value.returncode = 0 + execute.return_value.stdout = '{}' + fleet.remote = healthy.Fleet.remote.__get__(fleet) + for action in ("start", "cleanup", "audit-finalize", "drive-gate", "replace-drive", "quorum-gate", "quorum-gate-probe"): + fleet.remote("node1", action) + script = execute.call_args.kwargs["input"] + self.assertIn("quorum_gate_backend = types.ModuleType", script) + compile(script, "remote-quorum", "exec") + + +if __name__ == "__main__": + unittest.main() From 6354298c4168374c88af78e832439bde74395671 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 06:30:45 +0800 Subject: [PATCH 087/343] fix(validation): distinguish durable retired and unknown Heal tokens --- docs/validation-migration-map.md | 20 ++++++++-- scripts/validation_heal/overlap.py | 6 +-- scripts/validation_heal/root_restart.py | 6 +-- scripts/validation_heal/stop.py | 49 +++++++++++++++++++++---- test_validation_heal.py | 25 ++++++++++--- 5 files changed, 84 insertions(+), 22 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 3ae01908..786574e1 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -554,8 +554,9 @@ A06 `heal.admin-stop-and-stale-token` now uses the same pre-registered 1025 external-object DX fixture on H1/H2. Cancellation requires actual running, nonzero partial canonical progress and both repaired and still-missing targets. The frozen candidate policy requires active stop acknowledgement, retained -aborted/cancelled partial responsibility, and exact 500/InternalError with the -retired token for repeat/stale stops. Three complete physical snapshots with +aborted/cancelled partial responsibility, and exact HTTP 200/stopped +idempotent acknowledgements for repeat/stale stops of a durable terminal token. +Unknown tokens remain a separate exact 500/InternalError contract. Three complete physical snapshots with five-second gaps keep the service running after cancellation; stopping the process cannot substitute for task write quiescence. Actual restored shard counts must equal the retained cancellation counters, and remaining shards @@ -1035,7 +1036,7 @@ four regression verdicts have their own archive and never increment a fleet or canonical-case PASS count. Existing tests assert captured lock events internally; the collector does not fabricate emitted barrier logs. Missing critical-point fixtures, Admin nolock semantics and required fleet overlap remain explicit gaps. -The current implementation has 252 focused Heal/catalog checks, including the +The current implementation has 253 focused Heal/catalog checks, including the Linux-only D11 ownership/mount/recovery checks. Native compilation and execution still require the separate hosted job; local parser checks are not native PASS. @@ -1084,3 +1085,16 @@ idempotent stop, so the public response is HTTP 200/stopped, rather than the unknown-token HTTP 500 assumed by the harness. The original result remains unchanged. Fixing the candidate-bound assertion and repeating the complete canonical run are required; the observed acknowledgement alone is not PASS. + + +The A06/F06 durable-retired-token assertion is corrected against all four +reviewed source hashes, including `root_recovery::cancel_pending`. It now +requires the exact signed old-token request and the candidate's HTTP 200, +stopped acknowledgement shape. Its default acknowledgement settings and new +response timestamp are not mistaken for task identity or original task options. +The old settled canonical outcome must remain byte-equivalent immediately +after both repeat and stale stops, and the new task must stay running across +the stale request. Unknown-token queries and forceStart-retired tokens without +a retained durable terminal keep their independent exact 500/InternalError +checks. Both archived A06 acknowledgement samples satisfy the corrected shape; +that replay is not fresh live acceptance. Complete A06/F06 reruns remain required. diff --git a/scripts/validation_heal/overlap.py b/scripts/validation_heal/overlap.py index 0cedf036..7fa964df 100644 --- a/scripts/validation_heal/overlap.py +++ b/scripts/validation_heal/overlap.py @@ -15,13 +15,13 @@ from .heal_probe import canonical, payload, require from .healthy import BASE_OPTIONS, physical_content from .cursor import SIZE, TASK_SECONDS, run_topology as dx_topology, main as dx_main, wanted_pairs - from .stop import start, status, idle_fleet, validate_retired_stop + from .stop import start, status, idle_fleet, validate_unknown_token from .versions import public_version_id except ImportError: from heal_probe import canonical, payload, require from healthy import BASE_OPTIONS, physical_content from cursor import SIZE, TASK_SECONDS, run_topology as dx_topology, main as dx_main, wanted_pairs - from stop import start, status, idle_fleet, validate_retired_stop + from stop import start, status, idle_fleet, validate_unknown_token from versions import public_version_id @@ -245,7 +245,7 @@ def observe_branch(branch, probe, fleet, binding, bucket, expected, targets, act monotonic(before, first_old) new_before = wait_window(probe, fleet, path, new) retired = fleet.record("force-old-stop", probe.admin(path, [("clientToken", old), ("forceStop", "true")], b"")) - proof["oldStopIsolation"] = validate_retired_stop(retired, old) + proof["oldStopIsolation"] = validate_unknown_token(retired, old) new_after = status(probe, fleet, path, new, "force-new-after-old-stop") require(new_after.get("summary") != "stopped", "old token stopped the replacement task") require(active_window(new_after), "NOT_OBSERVED: old stop was not bracketed by the same running replacement") diff --git a/scripts/validation_heal/root_restart.py b/scripts/validation_heal/root_restart.py index 6918a479..601c9d42 100644 --- a/scripts/validation_heal/root_restart.py +++ b/scripts/validation_heal/root_restart.py @@ -15,7 +15,7 @@ from .healthy import BASE_OPTIONS, physical_content from .cursor import OBJECTS, TASK_SECONDS, complete_outcome, wanted_pairs, run_topology as dx_topology, main as dx_main from .shards import files, validate_repaired_physical - from .stop import status, wait_partial, idle_fleet, validate_retired_stop + from .stop import status, wait_partial, idle_fleet, validate_unknown_token from .cancel_restart import REFERENCE as JOURNAL_REFERENCE from .fleet_node import PROCESS_FIELDS from .versions import public_version_id @@ -24,7 +24,7 @@ from healthy import BASE_OPTIONS, physical_content from cursor import OBJECTS, TASK_SECONDS, complete_outcome, wanted_pairs, run_topology as dx_topology, main as dx_main from shards import files, validate_repaired_physical - from stop import status, wait_partial, idle_fleet, validate_retired_stop + from stop import status, wait_partial, idle_fleet, validate_unknown_token from cancel_restart import REFERENCE as JOURNAL_REFERENCE from fleet_node import PROCESS_FIELDS from versions import public_version_id @@ -143,7 +143,7 @@ def recovery_status(probe, fleet, token): return observed["response"] # The exact absent-token error is an observation during admission # replay, never completion evidence. All other failures remain errors. - validate_retired_stop(observed, token) + validate_unknown_token(observed, token) require(time.monotonic() < deadline, "original root token stayed unavailable beyond the 15-second recovery budget") time.sleep(0.1) diff --git a/scripts/validation_heal/stop.py b/scripts/validation_heal/stop.py index cb2d8b61..c11bdf96 100644 --- a/scripts/validation_heal/stop.py +++ b/scripts/validation_heal/stop.py @@ -4,6 +4,7 @@ from __future__ import annotations import copy +from datetime import datetime import json import sys import time @@ -31,10 +32,18 @@ QUIET_SECONDS = 5 POLICY = {"referenceCandidate": "072b57b37559fbe626aeda8abf61617fdcbdd926", "cancelActive": {"httpStatus": 200, "summary": "stopped"}, - "repeatOrRetiredTokenStop": {"httpStatus": 500, "code": "InternalError", "messagePrefix": "Heal task not found: "}, + "repeatOrRetiredTokenStop": {"httpStatus": 200, "summary": "stopped", "detail": "", + "reason": "root_recovery::cancel_pending returns true for a retained terminal record"}, + "unknownTokenStop": {"httpStatus": 500, "code": "InternalError", "messagePrefix": "Heal task not found: "}, "oldStatus": "retained aborted/cancelled outcome, partial coverage; same path/token", "sources": ["crates/heal/src/heal/manager.rs:cancel_task", "crates/heal/src/heal/channel.rs:process_cancel_request", + "crates/heal/src/heal/manager/root_recovery.rs:cancel_pending retained-terminal branch", "rustfs/src/admin/handlers/heal.rs:force_stop handler"], + "sourceHashes": { + "crates/heal/src/heal/manager.rs": "ba9289942fbbe8ce2632686ae86b9d1be0ae9923420e3b091d67148b4ee02094", + "crates/heal/src/heal/channel.rs": "7f669ca43b9559db9b97fba6b419a6c76a94d76acd8526db132d74b7c534d6b0", + "crates/heal/src/heal/manager/root_recovery.rs": "ba6beba9d9d14985e4c5448d85727036388d3d72f28879ac5c67fc5f760048ba", + "rustfs/src/admin/handlers/heal.rs": "7e4007c29a8329ced8e48b88111400e79cffd845bd6ddc6243b79f3cd48e8199"}, "unsupportedCandidates": "fail the frozen HTTP/body contract; do not accept arbitrary alternatives"} @@ -77,17 +86,34 @@ def wait_partial(probe, fleet, path, token, targets, *, repair_required): def validate_retired_stop(observed, token): - require(observed.get("httpStatus") == 500 and not observed.get("transportError"), + require(observed.get("httpStatus") == 200 and not observed.get("transportError"), "retired token stop did not follow the frozen HTTP contract") + require(observed.get("query") == [["clientToken", token], ["forceStop", "true"]] + and observed.get("method") == "POST" and observed.get("auth") == "signed" + and observed.get("requestBody") == "" and isinstance(observed.get("path"), str) + and observed["path"].startswith("/rustfs/admin/v3/heal/"), "retired stop did not target the exact original token") + body = observed.get("response") + defaults = {**BASE_OPTIONS, "scanMode": 1, "pool": None, "set": None} + require(isinstance(body, dict) and set(body) == {"summary", "detail", "startTime", "settings"} + and body["summary"] == "stopped" and body["detail"] == "" and body["settings"] == defaults, + "retired stop lacks the exact idempotent acknowledgement; ACK defaults are not task settings") + require(isinstance(body["startTime"], str) and datetime.fromisoformat(body["startTime"].replace("Z", "+00:00")).tzinfo is not None, + "retired stop acknowledgement lacks a valid response timestamp") + return {"httpStatus": 200, "summary": "stopped", "exactRetiredTaskId": token, + "responseEchoesTaskIdentity": False, "ackDefaultsAreOriginalTaskSettings": False, + "terminalResponsibilityMustRemainUnchanged": True} + + +def validate_unknown_token(observed, token): + require(observed.get("httpStatus") == 500 and not observed.get("transportError"), + "unknown token did not follow the frozen HTTP contract") body = observed.get("response") - if isinstance(body, str): - fields = {element.tag.rsplit("}", 1)[-1]: element.text for element in ET.fromstring(body).iter()} - else: - fields = body if isinstance(body, dict) else {} + fields = {element.tag.rsplit("}", 1)[-1]: element.text for element in ET.fromstring(body).iter()} \ + if isinstance(body, str) else body if isinstance(body, dict) else {} require((fields.get("Code") or fields.get("code")) == "InternalError" and (fields.get("Message") or fields.get("message")) == "Heal task not found: " + token, - "retired-token response has the wrong exact structured error/task identity") - return {"httpStatus": 500, "code": "InternalError", "exactRetiredTaskId": token} + "unknown-token response has the wrong exact structured error/task identity") + return {"httpStatus": 500, "code": "InternalError", "exactUnknownTaskId": token} def validate_cancelled(current, bucket, expected, targets): @@ -183,6 +209,7 @@ def idle_fleet(fleet, binding): def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, original, after_cancelled=None, force_new=False): + require(binding["candidateCommit"] == POLICY["referenceCandidate"], "cancel policy is unreviewed for this candidate") path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/objects/" fleet.record("frozen-cancel-policy", POLICY) old = start(probe, fleet, path, active) @@ -226,6 +253,9 @@ def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, or active.remove((path, old)) repeated = fleet.record("repeat-stop", probe.admin(path, [("clientToken", old), ("forceStop", "true")], b"")) repeated_proof = validate_retired_stop(repeated, old) + repeated_status = status(probe, fleet, path, old, "old-status-after-idempotent-stop") + require(canonical(repeated_status.get("outcome")) == canonical(cancelled["outcome"]), + "idempotent stop changed the original settled cancellation responsibility") restart_proof = after_cancelled(path, old, cancelled, quiet[-1]) if after_cancelled is not None else None new = start(probe, fleet, path, active, force=force_new) require(new != old, "new task reused a cancelled token") @@ -235,6 +265,9 @@ def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, or require(canonical(old_status["outcome"]) == canonical(cancelled["outcome"]), "new task overwrote old partial responsibility") stale = fleet.record("old-stop-during-new", probe.admin(path, [("clientToken", old), ("forceStop", "true")], b"")) stale_proof = validate_retired_stop(stale, old) + stale_status = status(probe, fleet, path, old, "old-status-after-stale-stop") + require(canonical(stale_status.get("outcome")) == canonical(cancelled["outcome"]), + "stale stop changed the old task's terminal responsibility") after = status(probe, fleet, path, new, "new-after-stale-stop") require(after.get("summary") != "stopped", "old token actually stopped the new task") require(partial_window(after, remaining, repair_required=False), diff --git a/test_validation_heal.py b/test_validation_heal.py index 56730e09..980e8a4a 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -12,7 +12,7 @@ from unittest.mock import patch, Mock from scripts.validation_heal.matrix import STAGES, TOPOLOGIES, audit_evidence, cases, compile_plan, select -from scripts.validation_heal.healthy import validate_healthy, validate_topology, validate_rejection, Fleet +from scripts.validation_heal.healthy import validate_healthy, validate_topology, validate_rejection, Fleet, BASE_OPTIONS from scripts.validation_heal.fleet_node import paths, stopped, inspect_metadata from scripts.validation_heal.inspector import verify as verify_inspector from scripts.validation_heal.shards import BRANCHES, files, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal, validate_dry_run_status @@ -46,7 +46,7 @@ from scripts.validation_heal.cursor import (Window, OBJECTS as CURSOR_OBJECTS, SIZE as CURSOR_SIZE, BRANCHES as CURSOR_BRANCHES, terminal_proof as cursor_terminal, validate_missing_batch, verify_all_parts) from scripts.validation_heal.fleet_node import resource_limits, verify_parts as node_verify_parts -from scripts.validation_heal.stop import (partial_window, validate_retired_stop, validate_cancelled, +from scripts.validation_heal.stop import (partial_window, validate_retired_stop, validate_unknown_token, validate_cancelled, cancelled_physical, idle_fleet as stop_idle_fleet, BRANCHES as STOP_BRANCHES) from scripts.validation_heal.cancel_restart import validate_journal, BRANCHES as CANCEL_RESTART_BRANCHES from scripts.validation_heal.fleet_node import task_records as node_task_records @@ -2069,13 +2069,28 @@ def test_active_window_requires_actual_partial_canonical_progress_and_no_errors( changed["outcome"]["counters"]["unknown"] = 1 with self.assertRaises(ValueError): partial_window(changed, targets, repair_required=True) - def test_repeat_stop_freezes_http_structured_code_and_the_retired_token(self): - response = {"httpStatus": 500, "response": "InternalErrorHeal task not found: old-token"} + def test_repeat_stop_requires_idempotent_ack_for_the_exact_retained_token(self): + response = {"httpStatus": 200, "query": [["clientToken", "old-token"], ["forceStop", "true"]], + "method": "POST", "auth": "signed", "requestBody": "", "path": "/rustfs/admin/v3/heal/bucket/objects/", + "response": {"summary": "stopped", "detail": "", "startTime": "2026-10-10T22:16:17.150884876Z", + "settings": {**BASE_OPTIONS, "scanMode": 1, "pool": None, "set": None}}} self.assertEqual(validate_retired_stop(response, "old-token")["exactRetiredTaskId"], "old-token") + self.assertFalse(validate_retired_stop(response, "old-token")["responseEchoesTaskIdentity"]) + for changed in ({**response, "httpStatus": 500}, {**response, "transportError": "timeout"}, + {**response, "query": [["clientToken", "new-token"], ["forceStop", "true"]]}, + {**response, "response": {**response["response"], "summary": "finished"}}, + {**response, "response": {**response["response"], "outcome": {"execution": {"state": "completed"}}}}, + {**response, "response": {**response["response"], "settings": BASE_OPTIONS}}, + {**response, "response": "InternalErrorHeal task not found: old-token"}): + with self.assertRaises(ValueError): validate_retired_stop(changed, "old-token") + + def test_unknown_token_is_not_confused_with_retained_terminal_idempotency(self): + response = {"httpStatus": 500, "response": "InternalErrorHeal task not found: old-token"} + self.assertEqual(validate_unknown_token(response, "old-token")["exactUnknownTaskId"], "old-token") for changed in ({**response, "httpStatus": 200}, {**response, "transportError": "timeout"}, {**response, "response": response["response"].replace("old-token", "new-token")}, {**response, "response": response["response"].replace("InternalError", "AccessDenied")}): - with self.assertRaises(ValueError): validate_retired_stop(changed, "old-token") + with self.assertRaises(ValueError): validate_unknown_token(changed, "old-token") def test_cancelled_physical_count_must_match_actual_original_shards(self): before = HealShardOracleTest().snapshot() From 0c598c39525b28d34b4961f4730ff8f50616ca80 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 06:48:40 +0800 Subject: [PATCH 088/343] feat(validation): migrate foreign identity and future schema refusal cases --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 34 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/cursor.py | 7 + scripts/validation_heal/fleet_node.py | 25 +- scripts/validation_heal/heal_fault.py | 7 +- scripts/validation_heal/healthy.py | 9 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 1 + scripts/validation_heal/schema.py | 299 ++++++++++++++++++ scripts/validation_heal/schema_faults.py | 90 ++++++ test_validation_heal.py | 2 +- test_validation_heal_schema.py | 183 +++++++++++ 13 files changed, 651 insertions(+), 14 deletions(-) create mode 100644 scripts/validation_heal/schema.py create mode 100644 scripts/validation_heal/schema_faults.py create mode 100644 test_validation_heal_schema.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 005a4f1e..535e205f 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -25,6 +25,7 @@ on: - test_validation_heal_scopes.py - test_validation_heal_bucket_metadata.py - test_validation_heal_write_quorum.py + - test_validation_heal_schema.py - test_validation_heal_native.py - scripts/validation_heal/** - docs/validation-heal-*.md @@ -64,6 +65,7 @@ on: - test_validation_heal_scopes.py - test_validation_heal_bucket_metadata.py - test_validation_heal_write_quorum.py + - test_validation_heal_schema.py - test_validation_heal_native.py - scripts/validation_heal/** - docs/validation-heal-*.md @@ -96,7 +98,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 786574e1..09fc9713 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 123 currently dispatchable IDs include all 74 non-Heal cases and forty-nine Heal cases. Forty-five have live PASS evidence; the other 113 have no live PASS claim. +case evidence. The 124 currently dispatchable IDs include all 74 non-Heal cases and fifty Heal cases. Forty-five have live PASS evidence; the other 113 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 35 Heal cases remain +The other 34 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -1098,3 +1098,33 @@ the stale request. Unknown-token queries and forceStart-retired tokens without a retained durable terminal keep their independent exact 500/InternalError checks. Both archived A06 acknowledgement samples satisfy the corrected shape; that replay is not fresh live acceptance. Complete A06/F06 reruns remain required. + + +C09 `heal.foreign-format-and-unknown-schema` dispatches all four canonical +branches on H1/H2. An isolated second released-process deployment exports a +real foreign format. The disk branch replaces only `xl.this` with its witnessed +donor UUID; the set branch copies the exact donor bytes, including its real +four-member set when the target has sixteen members. This topology difference +is retained explicitly. The future-format branch changes only the top-level +V1 semantic version to `999`. Only one empty, owned replacement root is +modified, while its complete original root is quarantined and inaccessible to +the candidate. Shared formats and IAM are outside the mutation scope. + +A registered 90-second window requires exact Admin slot/endpoint refusal and +stable original peer identities, followed by deep Heal and target GET. Startup +failure alone cannot earn PASS. Manual Heal may verify the surviving object or +report its complete failure, but must never claim reconstruction of the refused +member. All surviving files/formats and the injected bytes stay frozen. + +The object branch changes only the reviewed XL2 envelope's little-endian major +from 1 to 2 on every member of one ACKed target. It does not fabricate a future +inner msgpack schema. Per-disk metadata hashes bind each actual decoder refusal; +all peer GETs must return a structured server refusal, and deep Heal must retain +complete failed responsibility without a missing/healthy/repaired claim. All +payloads, control objects and formats remain exact. Only after the negative +proof may original bytes/identity be rescued. All 128 ACKs, every external shard +with streaming checksums, and full S3 inventories through every peer are checked +again. Backup rescue never earns product-repair credit; failed recovery retains +the namespace and stops subsequent fleet use. C09 still requires fresh live +evidence. Eleven targeted oracle/guard tests are added; its actual Linux +FD/mount guard test runs in CI, not as a simulated macOS syscall claim. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index b4bc4357..f1e16d98 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/cursor.py b/scripts/validation_heal/cursor.py index f6c84a41..658b4a41 100644 --- a/scripts/validation_heal/cursor.py +++ b/scripts/validation_heal/cursor.py @@ -187,6 +187,13 @@ def map_cohorts(snapshot, bucket, expected, *, count=OBJECTS): def verify_all_parts(fleet, cohorts, label, *, count=OBJECTS): require(type(count) is int and 129 <= count <= OBJECTS and len(cohorts) == count, "complete physical verification differs from its frozen object count") + return verify_part_manifest(fleet, cohorts, label, count=count) + + +def verify_part_manifest(fleet, cohorts, label, *, count): + """Stream every member of an explicitly counted external-object manifest.""" + require(type(count) is int and 1 <= count <= OBJECTS and len(cohorts) == count, + "complete external-object manifest differs from its frozen count") width = 4 if len(fleet.nodes) == 1 else 16 identities = [(entry["node"], entry["disk"], entry["relativePath"], entry["versionId"], entry["partNumber"]) for target in cohorts for entry in target["cohort"]] diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index f45a44fc..5a61b2f1 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -33,12 +33,12 @@ def runtime_config(profile): - require(profile in {"manual", "offline-replay", "durable-mrf", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "scope-isolation"}, "unsupported Heal runtime profile") + require(profile in {"manual", "offline-replay", "durable-mrf", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") if profile == "scope-isolation": return {**BACKGROUND_CONFIG, "RUSTFS_ERASURE_SET_DRIVE_COUNT": "8"} if profile == "durable-mrf": return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} - if profile in {"replacement", "replacement-target"}: + if profile in {"replacement", "replacement-target", "schema-refusal"}: return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_AUTO_HEAL_ENABLE": "true" if profile == "replacement" else "false", "RUSTFS_HEAL_INTERVAL_SECS": "10", "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} if profile == "stale-rejoin": @@ -231,7 +231,7 @@ def start(request): "--property=KillMode=control-group", "--property=TimeoutStopSec=30", "--property=RuntimeMaxSec=" + str(resource_limits(owner.get("resourceProfile", "standard"))["runtimeMaxSeconds"]), "--property=MemoryMax=4G", - *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") in {"root-restart", "replacement", "replacement-target"} else []), + *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") in {"root-restart", "replacement", "replacement-target", "schema-refusal"} else []), "--property=WorkingDirectory=" + str(state), "--property=EnvironmentFile=" + str(state / "service.env"), *gate_module().start_properties(gate_module().checked_gate(state, roots)), @@ -656,7 +656,7 @@ def install_inspector(request): def inspect_metadata(tool, path, bucket, key, entry, metadata_faults): observed = command(str(tool), str(path), bucket, key, check=False) if observed.returncode: - expected_fault = metadata_faults.get(entry["path"]) + expected_fault = metadata_faults.get(f"{entry['disk']}:{entry['path']}", metadata_faults.get(entry["path"])) require(expected_fault == {name: entry[name] for name in ("sha256", "size")}, "metadata decoder failed outside an exact witnessed file fault") return {"metadataError": {"code": observed.returncode, "message": observed.stderr[-500:], @@ -794,6 +794,22 @@ def bucket_record_operation(request): return module.recover(fd, binding, fault, request) +def format_schema_fault(request): + state, roots, owner = owned(request["runId"]) + stopped(owner) + replacement = replacement_module().attest(state, roots) + root, _ = fault_root(request) + fault = fault_module() + with fault.directory(root) as fd: + marker = root / "fault-owner.json" + if not marker.exists(): + fault.new_file(fd, marker.name, fault.encode({"schemaVersion": 1, "runId": owner["runId"], + "bucketPrefix": owner["bucket"], "mountId": fault.mount_id(fd), "device": os.fstat(fd).st_dev, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat()})) + fault.checked_owner(fd, owner["runId"]) + return globals()["schema_faults"].install(fd, replacement, owner, request, fault) + + def cleanup(request): state, roots, owner = owned(request["runId"]) command("systemctl", "stop", owner["unit"], check=False) @@ -901,6 +917,7 @@ def main(request): actions[action] = bucket_record_operation for action in ("quorum-gate", "quorum-gate-probe"): actions[action] = quorum_gate_operation + actions["format-schema-fault"] = format_schema_fault actions.update({"replace-drive": replace_drive, "replacement-state": replacement_state, "replacement-probe": replacement_probe}) actions.update({"format-export": format_export, "foreign-format": foreign_format, "restore-replacement": restore_replacement}) actions.update({"process": process, "process-metrics": process_metrics, "process-signal": process_signal, "process-graceful": process_graceful, "process-crash": process_crash}) diff --git a/scripts/validation_heal/heal_fault.py b/scripts/validation_heal/heal_fault.py index 35e17aa6..42b7efc6 100644 --- a/scripts/validation_heal/heal_fault.py +++ b/scripts/validation_heal/heal_fault.py @@ -21,7 +21,7 @@ LIMIT = 64 * 1024**2 ROOT_PATTERN = re.compile(r"/data/rustfs[1-9][0-9]*/heal-validation/((?:gh|manual)-[a-z0-9-]{5,60})") -FAULTS = ("missing", "bitflip", "truncate", "zero", "header", "replace-part") +FAULTS = ("missing", "bitflip", "truncate", "zero", "header", "replace-part", "future-xl-schema") def bounded_fault_id(code: str, branch: str, index: int) -> str: @@ -122,6 +122,10 @@ def mutation(value: bytes, kind: str, offset: int | None = None) -> bytes | None require(kind != "replace-part", "replacement requires a verified owned donor") if kind == "missing": return None + if kind == "future-xl-schema": + require(offset is None and len(value) >= 8 and value[:4] == b"XL2 " + and int.from_bytes(value[4:6], "little") == 1, "future envelope requires exact reviewed XL2 major 1") + return value[:4] + (2).to_bytes(2, "little") + value[6:] require(bool(value), "empty file cannot witness this fault") if kind == "truncate": length = len(value) // 2 if offset is None else offset @@ -154,6 +158,7 @@ def apply_fault(root_fd: int, owner: dict, relative: str, fault_id: str, kind: s expected_sha: str, offset: int | None = None, *, donor_relative: str | None = None, donor_sha: str | None = None) -> dict: parts = target_parts(relative, owner["bucketPrefix"]) + require(kind != "future-xl-schema" or parts[-1] == "xl.meta", "future schema fault requires exact object metadata") require(bool(re.fullmatch(r"[A-J][0-9]{2}-[a-z0-9-]{1,48}", fault_id)), "invalid fault ID") require(bool(re.fullmatch(r"[0-9a-f]{64}", expected_sha)), "baseline SHA-256 required") with parent(root_fd, ["data", *parts[:-1]]) as target_fd, parent(root_fd, ["quarantine"]) as backup_fd: diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 3c6dc504..8020c017 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"manual", "offline-replay", "durable-mrf", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "scope-isolation"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "offline-replay", "durable-mrf", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -184,13 +184,16 @@ def remote(self, node, action, **params): source += "\nimport types\nmrf = types.ModuleType('mrf_records')\nexec(" + repr( NODE_SOURCE.with_name("mrf_records.py").read_text()) + ", mrf.__dict__)\n" if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline", - "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover"}: + "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault"}: source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" if action.startswith("bucket-record-"): source += "\nimport types\nbucket_records = types.ModuleType('bucket_records')\nexec(" + repr( NODE_SOURCE.with_name("bucket_records.py").read_text()) + ", bucket_records.__dict__)\n" - replacement_actions = {"start", "cleanup", "audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot"} + if action == "format-schema-fault": + source += "\nimport types\nschema_faults = types.ModuleType('schema_faults')\nexec(" + repr( + NODE_SOURCE.with_name("schema_faults.py").read_text()) + ", schema_faults.__dict__)\n" + replacement_actions = {"start", "cleanup", "audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot", "format-schema-fault"} if action in {"start", "cleanup", "audit-finalize", "drive-gate", "replace-drive", "quorum-gate", "quorum-gate-probe"}: source += "\nimport types\nquorum_gate_backend = types.ModuleType('quorum_gate')\nexec(" + repr( NODE_SOURCE.with_name("quorum_gate.py").read_text()) + ", quorum_gate_backend.__dict__)\n" diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 68ea6e19..69f645ef 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index f9c3c2cf..3c4d051e 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -33,6 +33,7 @@ case "${case_id}" in heal.bucket-metadata-integrity) driver=bucket_metadata.py ;; heal.disk-replacement-unformatted) driver=unformatted.py ;; heal.below-write-quorum) driver=write_quorum.py ;; + heal.foreign-format-and-unknown-schema) driver=schema.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/scripts/validation_heal/schema.py b/scripts/validation_heal/schema.py new file mode 100644 index 00000000..985299ac --- /dev/null +++ b/scripts/validation_heal/schema.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""C09: refuse witnessed foreign identities and future format/object envelopes.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time + +try: + from .heal_probe import Probe, compare_inventories, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .shards import files, map_target, run_task + from .versions import metadata_target, validate_failed_version, validate_version_receipts + from .unformatted import binding, foreign_donor, negative_scene, refusal, OBSERVATION_SECONDS, SIZE + from .disk_loss import member_state + from .cursor import verify_part_manifest + from .heal_fault import bounded_fault_id +except ImportError: + from heal_probe import Probe, compare_inventories, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from shards import files, map_target, run_task + from versions import metadata_target, validate_failed_version, validate_version_receipts + from unformatted import binding, foreign_donor, negative_scene, refusal, OBSERVATION_SECONDS, SIZE + from disk_loss import member_state + from cursor import verify_part_manifest + from heal_fault import bounded_fault_id + + +CASE_ID = "heal.foreign-format-and-unknown-schema" +BRANCHES = ("foreign-disk", "foreign-set", "future-format", "future-object-schema") +OBJECTS = 128 +CONTRACT = {"candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", + "sourceFiles": { + "crates/ecstore/src/layout/format.rs": "a730e6c897de6fca6071a806bcb9ad27a13e292b6c3975dfda6f436187f4784d", + "crates/ecstore/src/store/init_format.rs": "b6f5d009eccaeb28f9700f721bbf9a005b289716bfec939309193e06e098263c", + "crates/filemeta/src/filemeta/codec.rs": "ef82b3390b0a377e423f99fb384c6a8a96beaf5722b0c907852cc0c904c4f04d"}, + "futureFormat": {"field": "version", "original": "1", "injected": "999"}, + "futureObjectSchema": {"envelope": "XL2 ", "majorOffset": 4, "byteOrder": "little", "originalMajor": 1, + "injectedMajor": 2, "parserRefusal": "xl file version err", "innerMsgpackSchemaChanged": False}, + "formatFaultScope": "one witnessed empty owned replacement root; shared formats never mutated"} + + +def exact_get(observed, expected): + require(observed.get("acknowledged") is True and observed.get("resultUnknown") is False + and observed.get("response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode") == 200 + and observed.get("payload") == expected["payload"], "C09 surviving/recovered GET differs from its independent ACK") + return {"fullPayloadEqualsACK": True} + + +def unsupported_get(observed): + response = observed.get("response") or {} + require(observed.get("acknowledged") is False and observed.get("resultUnknown") is False + and "payload" not in observed and response.get("ResponseMetadata", {}).get("HTTPStatusCode") in {500, 503} + and response.get("Error", {}).get("Code") in {"InternalError", "SlowDownRead"}, + "future-schema GET returned bytes, absence, a transport error or an unreviewed refusal") + return {"structuredServerRefusal": response["Error"]["Code"], "unknownTreatedAsMissing": False} + + +def metadata_scene(before, after, targets, receipts): + """Every unknown record survives byte-for-byte; every control remains frozen.""" + left, right = files(before), files(after) + require(len(targets) == len(receipts) in {4, 16} and set(left) == set(right), "C09 unknown record was deleted or duplicated") + changed = {} + for target, receipt in zip(targets, receipts, strict=True): + identity = target["node"], target["disk"], target["relativePath"] + require(identity not in changed and identity in left + and receipt.get("kind") == "future-xl-schema" and receipt.get("relativePath") == identity[2] + and receipt.get("before") == {name: left[identity][name] for name in ("sha256", "size")} + and isinstance(receipt.get("fault"), dict) and receipt["fault"]["size"] == left[identity]["size"] + and receipt["fault"]["sha256"] != left[identity]["sha256"], "future envelope receipt differs from the exact original") + require({name: right[identity][name] for name in ("sha256", "size")} == receipt["fault"] + and "metadata" not in right[identity] + and right[identity].get("metadataError", {}).get("exactWitnessedFault") is True + and right[identity].get("metadataError", {}).get("validMetadataClaimed") is False + and CONTRACT["futureObjectSchema"]["parserRefusal"] in right[identity].get("metadataError", {}).get("message", ""), + "unknown metadata was rewritten or did not actually reach the reviewed parser refusal") + changed[identity] = receipt["fault"] + require({key: value for key, value in left.items() if key not in changed} + == {key: value for key, value in right.items() if key not in changed} + and all(before[node]["formats"] == after[node]["formats"] for node in before), + "future-schema test changed a payload, control object or disk format") + return {"unknownRecordsRetainedExactly": len(changed), "allPayloadsControlsAndFormatsUnchanged": True, + "innerMsgpackSchemaChanged": False, "productRepairClaimed": False} + + +def format_outcome(status, bucket, expected, target): + # A quarantined disk may leave the existing object verifiable on surviving + # authorities. Its exact disk refusal is proved independently, never inferred + # from a healthy object receipt or from a startup failure. + execution = (status.get("outcome") or {}).get("execution", {}).get("state") + if execution == "completed": + proof = validate_version_receipts(status, bucket, expected, 2) + else: + proof = validate_failed_version(status, bucket, expected, target) + require((status.get("outcome") or {}).get("counters", {}).get("healed") == 0, + "foreign/future format was reported reconstructed") + return {**proof, "refusedSlotValidatedSeparately": True, "productRepairClaimed": False} + + +def run_branch(args, topology, nodes, branch, donor): + directory = args.evidence / f"{topology}-{branch}" + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="schema-refusal") + candidate = json.loads(args.candidate.read_text()) + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + require(candidate["source_sha"] == CONTRACT["candidateCommit"], "unreviewed C09 parser/refusal contract") + fleet.record("candidate-schema-contract", CONTRACT) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + lab = binding(args, candidate, nodes, run_id, bucket) + fleet.record("binding", lab) + probe, active, metadata_faults = Probe(lab), [], {} + replacement_node, replacement_dirty = None, False + result = {"topology": topology, "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + expected = seed(Recorder(probe, directory / "fixture"), bucket, "DB", OBJECTS, [SIZE]) + baseline = fleet.record("S3-baseline", probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("fixture-ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + fleet.stop() + before = fleet.snapshot("original-complete-physical", quiescent=True) + fleet.record("actual-original-topology", validate_topology(before, topology)) + cohorts = [map_target(before, bucket, entry["key"], "data", 1) for entry in expected] + fleet.record("all-original-streaming-proof", verify_part_manifest(fleet, cohorts, "original-part", count=OBJECTS)) + target = cohorts[8] + selected = [expected[8]] + require(target["key"] == selected[0]["key"], "C09 target lost its registered ACK order") + if branch == "future-object-schema": + targets = [metadata_target(before, {**target, **shard}) for shard in target["cohort"]] + receipts = [] + for index, fault in enumerate(targets): + params = {"disk": fault["disk"], "faultId": bounded_fault_id("C09", branch, index)} + # Register before mutation so a lost SSH response still restores + # the exact intent and retains an unresolved namespace on failure. + fleet.faults.append((fault["node"], params)) + receipt = fleet.record("future-envelope-injection", {"node": fault["node"], **fleet.remote(fault["node"], + "apply-fault", **params, relativePath=fault["relativePath"], kind="future-xl-schema", expectedSha256=fault["sha256"])}) + receipts.append(receipt) + metadata_faults.setdefault(fault["node"], {})[f"{fault['disk']}:{fault['relativePath']}"] = receipt["fault"] + faulted = fleet.snapshot("actual-future-envelope-scene", quiescent=True, metadata_faults=metadata_faults) + fleet.record("future-envelope-witness", metadata_scene(before, faulted, targets, receipts)) + fleet.start() + for endpoint in lab["endpoints"]: + observed = fleet.record("unknown-target-GET", Probe(lab, endpoint).s3("get_object", {"Bucket": bucket, "Key": target["key"]})) + fleet.record("unknown-target-refusal", unsupported_get(observed)) + control = fleet.record("unaffected-control-GET", Probe(lab, endpoint).s3("get_object", {"Bucket": bucket, "Key": expected[0]["key"]})) + fleet.record("unaffected-control-ACK", exact_get(control, expected[0])) + status = run_task(probe, fleet, target, 2, active) + fleet.record("explicit-unknown-schema-responsibility", validate_failed_version(status, bucket, selected, target)) + fleet.stop() + scene = fleet.snapshot("refused-unknown-records-preserved", quiescent=True, metadata_faults=metadata_faults) + fleet.record("negative-full-physical-proof", metadata_scene(before, scene, targets, receipts)) + require(physical_content(faulted) == physical_content(scene), "GET/deep Heal changed an unknown record or control") + for node, params in tuple(fleet.faults): + fleet.record("negative-record-rescue", {"node": node, **fleet.remote(node, "recover-fault", **params)}) + fleet.faults.remove((node, params)) + metadata_faults.clear() + else: + member = next(shard for shard in target["cohort"] if shard["node"] == nodes[-1]) + target = {**target, **member} + node, disk = member["node"], member["disk"] + original_format = next(item for item in before[node]["formats"] if item["disk"] == disk) + fleet.start() + original = member_state(fleet.record("original-member-Admin", probe.admin("/rustfs/admin/v3/info", [], b"", method="GET")), + target, len(target["cohort"])) + fleet.stop() + replacement_node = node + fleet.record("one-owned-original-root-quarantined", fleet.remote(node, "replace-drive", disk=disk, + expectedFormatSha256=original_format["sha256"])) + if branch != "future-format": fleet.record("released-foreign-donor", donor) + injected = fleet.record("exact-schema-format-injection", fleet.remote(node, "format-schema-fault", disk=disk, + branch=branch, donor=donor if branch != "future-format" else None)) + foreign = {"sha256": injected["injectedSha256"], "value": injected["injectedValue"]} + fleet.record("registered-format-refusal-window", {"seconds": OBSERVATION_SECONDS, + "automaticHealDisabled": True, "directoryTestRootFallback": True, "actualDeviceReplacementClaimed": False, + "startupFailureAloneEarnsPass": False}) + fleet.start() + fleet.record("original-root-inaccessible-to-candidate", fleet.remote(node, "replacement-probe")) + deadline, samples = time.monotonic() + OBSERVATION_SECONDS, [] + while True: + state = fleet.record("injected-format-window", fleet.remote(node, "replacement-state")) + current = next(item for item in state["formats"] if item["disk"] == disk) + require(state.get("originalArchiveUnchanged") is True and current.get("sha256") == foreign["sha256"] + and current.get("value") == foreign["value"], "candidate altered or reidentified foreign/future format") + samples.append(refusal(fleet.record("structured-format-refusal", probe.admin("/rustfs/admin/v3/info", [], b"", method="GET")), + target, original["endpoint"], foreign)) + if time.monotonic() >= deadline: break + time.sleep(5) + require(len(samples) >= 3, "format refusal did not span its registered window") + status = run_task(probe, fleet, target, 2, active) + fleet.record("deep-Heal-format-responsibility", format_outcome(status, bucket, selected, target)) + for endpoint in lab["endpoints"]: + observed = fleet.record("surviving-authority-target-GET", Probe(lab, endpoint).s3("get_object", {"Bucket": bucket, "Key": target["key"]})) + fleet.record("surviving-authority-ACK-proof", exact_get(observed, selected[0])) + # GET and deep Heal must both preserve the injected identity and all + # survivors. Only then can the original root be used for rescue. + fleet.stop() + scene = fleet.snapshot("refused-format-complete-physical", quiescent=True) + fleet.record("format-negative-full-physical-proof", negative_scene(before, scene, (node, disk), "foreign-format", foreign)) + restored = fleet.record("original-identity-negative-rescue", fleet.remote(node, "restore-replacement")) + require(restored.get("originalRestoredExactly") is True and restored.get("productRepairClaimed") is False, + "C09 rescue differs from the original or claimed product repair") + replacement_node = None + result["refusalSamples"] = len(samples) + recovered = fleet.snapshot("exact-original-identity-recovered", quiescent=True) + require(physical_content(before) == physical_content(recovered), "C09 rescue changed any original record or payload") + fleet.record("all-recovered-streaming-proof", verify_part_manifest(fleet, cohorts, "recovered-part", count=OBJECTS)) + fleet.start() + for endpoint in lab["endpoints"]: + inventory = fleet.record("all-peer-complete-S3", {"endpoint": endpoint, + **Probe(lab, endpoint).inventory(bucket, unversioned_current_tags=True)}) + fleet.record("all-peer-independent-ACK-proof", {**verify_seed(expected, inventory), **compare_inventories(baseline, inventory)}) + fleet.stop() + require(physical_content(before) == physical_content(fleet.snapshot("after-recovery-S3-physical", quiescent=True)), + "recovery verification repaired records that the negative proof had not preserved") + result.update(verdict="PASS", all128ACKsPreserved=True, fullPhysicalAndStreamingAndS3Verified=True, + backupRestorationUsed=True, productRepairClaimed=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + if fleet.reserved: + try: + fleet.stop() + fleet.snapshot("C09-failure-physical", quiescent=True, metadata_faults=metadata_faults) + if replacement_node: + fleet.record("C09-failure-original-archive", fleet.remote(replacement_node, "replacement-state")) + except Exception as evidence_error: + result["failureEvidenceError"] = str(evidence_error) + finally: + for path, token in active: + try: fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: fleet.record("stop-error", {"error": str(exc)}) + if replacement_node: + try: + fleet.stop() + fleet.record("failed-format-original-rescue", fleet.remote(replacement_node, "restore-replacement")) + except Exception as exc: + replacement_dirty = True + fleet.record("failed-format-recovery-DIRTY", {"error": str(exc), "namespacesRetained": True}) + result["cleanup"] = "PASS" if not replacement_dirty and fleet.cleanup() else "FAIL" + fleet.record("branch-result", result) + return result + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f"Heal workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path, required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(nodes) == len(set(nodes)) + and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") + require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)) + and bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "verified workflow candidate required") + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "C09 executor omitted a canonical source branch") + results, donor_failure = [], None + try: + donor = foreign_donor(args, nodes[0]) + for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): + for branch in BRANCHES: + results.append(run_branch(args, topology, selected, branch, donor)) + if results[-1]["cleanup"] != "PASS": break + if results[-1]["cleanup"] != "PASS": break + except (Exception, KeyboardInterrupt) as exc: + donor_failure = {"type": type(exc).__name__, "message": str(exc)} + finally: + complete = len(results) == 8 and all(item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "donorFailure": donor_failure, + "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", "complete": complete}) + require(complete, "C09 requires all four branches on both H1/H2, a released donor and complete recovery") + + +if __name__ == "__main__": + try: main() + except Exception as exc: + print(f"Heal foreign/future schema validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/schema_faults.py b/scripts/validation_heal/schema_faults.py new file mode 100644 index 00000000..f6b64440 --- /dev/null +++ b/scripts/validation_heal/schema_faults.py @@ -0,0 +1,90 @@ +"""C09 format faults in one witnessed empty replacement root, never shared format.""" + +import base64 +import copy +import hashlib +import json +import os +import uuid + + +def require(value, message): + if not value: + raise ValueError(message) + + +def reviewed_format(value): + require(isinstance(value, dict) and value.get("version") == "1" and value.get("format") == "xl" + and value.get("xl", {}).get("version") == "3", "format is outside the reviewed V1/XL V3 contract") + require(str(uuid.UUID(value["id"])) == value["id"] and uuid.UUID(value["id"]).int != 0, "invalid deployment UUID") + erasure = value["xl"] + groups = erasure["sets"] + require(isinstance(groups, list) and len(groups) == 1 and len(groups[0]) in {4, 16} + and erasure.get("distributionAlgo") == "SIPMOD+PARITY", "unreviewed donor/set geometry") + members = groups[0] + require(len(members) == len(set(members)) and erasure["this"] in members + and all(str(uuid.UUID(member)) == member and uuid.UUID(member).int != 0 for member in members), + "format lacks distinct canonical member identities") + return set(members) + + +def derive(original_raw, branch, donor=None): + require(branch in {"foreign-disk", "foreign-set", "future-format"}, "unsupported C09 format branch") + original = json.loads(original_raw) + original_members = reviewed_format(original) + changed = copy.deepcopy(original) + if branch == "future-format": + changed["version"] = "999" + raw = json.dumps(changed, sort_keys=True, separators=(",", ":")).encode() + detail = {"futureField": "version", "before": "1", "after": "999", "otherSemanticFieldsUnchanged": True} + else: + require(isinstance(donor, dict) and donor.get("readOnly") is True, "released-process donor evidence required") + donor_raw = base64.b64decode(donor["rawBase64"], validate=True) + require(len(donor_raw) < 65536, "donor format exceeds its exact byte budget") + require(hashlib.sha256(donor_raw).hexdigest() == donor["sha256"], "donor raw format digest differs") + foreign = json.loads(donor_raw) + foreign_members = reviewed_format(foreign) + require(donor.get("value") == foreign and foreign["id"] != original["id"] + and not foreign_members & original_members, "donor reused the original deployment or member UUIDs") + if branch == "foreign-disk": + changed["xl"]["this"] = foreign["xl"]["this"] + raw = json.dumps(changed, sort_keys=True, separators=(",", ":")).encode() + detail = {"changedField": "xl.this", "actualForeignDiskUUID": foreign["xl"]["this"], + "originalDeploymentAndSetPreserved": True, "derivedIdentityFixture": True} + else: + # Preserve the real foreign cluster/set bytes without fabricating + # set membership, disk UUIDs, deployment IDs or format versions. + changed, raw = foreign, donor_raw + detail = {"actualForeignDeploymentId": foreign["id"], "actualForeignSet": foreign["xl"]["sets"], + "releasedDonorBytesCopiedUnchanged": True, "derivedIdentityFixture": False} + return raw, changed, detail + + +def install(root_fd, replacement, owner, request, fault): + require(request.get("disk") == replacement["disk"] and replacement.get("originalArchiveUnchanged") is True + and replacement.get("originalStillAtDataRoot") is False, "C09 requires its exact isolated original member") + branch = request["branch"] + donor = request.get("donor") + if donor is not None: + require(donor.get("binarySha256") == owner["binarySha256"], "donor candidate binary differs") + with fault.parent(root_fd, ["quarantine", "old-disk", ".rustfs.sys"]) as saved: + original = fault.read_file(saved, "format.json", 65536) + require(fault.digest(original) == replacement["originalFormatSha256"], "original format archive differs") + raw, changed, detail = derive(original, branch, donor) + require(len(raw) < 65536, "derived format exceeds its byte budget") + receipt = {"schemaVersion": 1, "runId": owner["runId"], "disk": replacement["disk"], "branch": branch, + "relativePath": ".rustfs.sys/format.json", "originalSha256": fault.digest(original), + "injectedSha256": fault.digest(raw), "injectedValue": changed, "detail": detail, + "backend": "one-exact-empty-owned-replacement-format", "credentialsRead": False, + "sharedFormatTouched": False, "backupRestorationIsProductRepair": False} + with fault.parent(root_fd, ["quarantine"]) as archive, fault.parent(root_fd, ["data"]) as data: + require(not os.listdir(data), "format mutation requires the witnessed empty replacement root") + fault.new_file(archive, "C09-format-intent.json", fault.encode(receipt)) + fault.new_file(archive, "C09-injected-format.original", raw) + os.mkdir(".rustfs.sys", 0o700, dir_fd=data) + with fault.parent(root_fd, ["data", ".rustfs.sys"]) as target: + fault.new_file(target, "format.json", raw) + require(fault.read_file(target, "format.json", 65536) == raw, "format injection was not witnessed") + os.fsync(data) + fault.new_file(archive, "C09-format-applied.json", fault.encode(receipt)) + return receipt diff --git a/test_validation_heal.py b/test_validation_heal.py index 980e8a4a..b1c5c76b 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -667,7 +667,7 @@ def call(operation, params, body=None): with self.subTest(kind=kind), self.assertRaises(ValueError): verify_semantics(public, changed) def test_decoder_failure_is_evidence_only_for_the_exact_witnessed_corruption(self): - entry = {"path": "fixture/key/xl.meta", "sha256": "changed-sha", "size": 9} + entry = {"disk": 0, "path": "fixture/key/xl.meta", "sha256": "changed-sha", "size": 9} fault = {entry["path"]: {"sha256": "changed-sha", "size": 9}} rejected = types.SimpleNamespace(returncode=1, stderr="invalid captured metadata", stdout="") with patch("scripts.validation_heal.fleet_node.command", return_value=rejected): diff --git a/test_validation_heal_schema.py b/test_validation_heal_schema.py new file mode 100644 index 00000000..43c4526d --- /dev/null +++ b/test_validation_heal_schema.py @@ -0,0 +1,183 @@ +import base64 +import copy +import hashlib +import json +from pathlib import Path +import sys +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import schema, schema_faults, heal_fault, fleet_node +from scripts.validation_heal.cursor import verify_all_parts, verify_part_manifest +from scripts.validation_heal.matrix import AUTOMATIC_CASES, select + + +class SchemaRefusalTests(unittest.TestCase): + def format(self, foreign=False, width=4): + prefix = "4" if foreign else "3" + members = [f"{prefix}2345678-1234-4234-8234-{index:012x}" for index in range(width)] + return {"version": "1", "format": "xl", "id": ("2" if foreign else "1") + "2345678-1234-4234-8234-123456789abc", + "xl": {"version": "3", "this": members[0], "sets": [members], "distributionAlgo": "SIPMOD+PARITY"}} + + def donor(self): + raw = b" \n" + json.dumps(self.format(True), indent=2).encode() + b"\n" + return {"readOnly": True, "binarySha256": "candidate", "rawBase64": base64.b64encode(raw).decode(), + "sha256": hashlib.sha256(raw).hexdigest(), "value": self.format(True)} + + def test_foreign_disk_uses_actual_donor_uuid_and_keeps_original_deployment_and_set(self): + original = self.format(width=16) + raw, changed, detail = schema_faults.derive(json.dumps(original).encode(), "foreign-disk", self.donor()) + self.assertEqual(json.loads(raw), changed) + self.assertEqual(changed["xl"]["this"], self.donor()["value"]["xl"]["this"]) + changed["xl"]["this"] = original["xl"]["this"] + self.assertEqual(changed, original) + self.assertTrue(detail["derivedIdentityFixture"]) + + def test_foreign_set_copies_exact_donor_bytes_even_when_topology_width_differs(self): + donor = self.donor() + raw, changed, detail = schema_faults.derive(json.dumps(self.format(width=16)).encode(), "foreign-set", donor) + self.assertEqual(raw, base64.b64decode(donor["rawBase64"])) + self.assertEqual(changed, donor["value"]) + self.assertFalse(detail["derivedIdentityFixture"]) + self.assertEqual(len(changed["xl"]["sets"][0]), 4) + + def test_future_format_changes_only_reviewed_semantic_version(self): + original = self.format() + raw, changed, detail = schema_faults.derive(json.dumps(original).encode(), "future-format") + self.assertEqual(json.loads(raw)["version"], "999") + self.assertTrue(detail["otherSemanticFieldsUnchanged"]) + changed["version"] = "1" + self.assertEqual(changed, original) + + def test_foreign_provenance_rejects_shared_ids_changed_bytes_and_unreviewed_schema(self): + for kind in ("sha", "value", "read-only", "shared-deployment", "shared-member", "schema", "geometry"): + donor = self.donor() + if kind == "sha": donor["sha256"] = "0" * 64 + elif kind == "value": donor["value"]["id"] = self.format()["id"] + elif kind == "read-only": donor["readOnly"] = False + else: + value = donor["value"] + if kind == "shared-deployment": value["id"] = self.format()["id"] + if kind == "shared-member": value["xl"]["sets"][0][1] = self.format()["xl"]["sets"][0][1] + if kind == "schema": value["xl"]["version"] = "4" + if kind == "geometry": value["xl"]["sets"][0].pop() + raw = json.dumps(value).encode() + donor.update(rawBase64=base64.b64encode(raw).decode(), sha256=hashlib.sha256(raw).hexdigest()) + with self.subTest(kind=kind), self.assertRaises(ValueError): + schema_faults.derive(json.dumps(self.format()).encode(), "foreign-set", donor) + + def test_future_object_schema_changes_exact_major_and_preserves_every_other_byte(self): + original = b"XL2 \x01\x00\x03\x00original-msgpack-and-payload" + changed = heal_fault.mutation(original, "future-xl-schema") + self.assertEqual(changed, original[:4] + b"\x02\x00" + original[6:]) + for value, offset in ((b"XL2 \x02\x00\x03\x00", None), (b"bad!\x01\x00\x03\x00", None), (original, 5), (b"XL2 ", None)): + with self.subTest(value=value, offset=offset), self.assertRaises(ValueError): + heal_fault.mutation(value, "future-xl-schema", offset) + with self.assertRaisesRegex(ValueError, "exact object metadata"): + heal_fault.apply_fault(0, {"bucketPrefix": "bucket"}, "bucket/key/data/part.1", "C09-future-0", "future-xl-schema", "a" * 64) + + @unittest.skipUnless(sys.platform == "linux", "actual Linux FD/mount guards run in CI") + def test_format_install_uses_one_empty_root_preserves_archive_and_rejects_other_disk(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + (root / "data").mkdir() + saved = root / "quarantine/old-disk/.rustfs.sys" + saved.mkdir(parents=True) + original = json.dumps(self.format()).encode() + (saved / "format.json").write_bytes(original) + replacement = {"disk": 0, "originalArchiveUnchanged": True, "originalStillAtDataRoot": False, + "originalFormatSha256": hashlib.sha256(original).hexdigest()} + owner = {"runId": "run", "binarySha256": "candidate"} + with heal_fault.directory(root) as fd: + for kind in ("wrong-disk", "occupied", "wrong-candidate", "archive"): + request = {"disk": 0, "branch": "foreign-set", "donor": self.donor()} + current = copy.deepcopy(replacement) + if kind == "wrong-disk": request["disk"] = 1 + if kind == "occupied": (root / "data/occupied").write_bytes(b"live") + if kind == "wrong-candidate": request["donor"]["binarySha256"] = "other" + if kind == "archive": current["originalArchiveUnchanged"] = False + with self.subTest(kind=kind), self.assertRaises(ValueError): schema_faults.install(fd, current, owner, request, heal_fault) + (root / "data/occupied").unlink(missing_ok=True) + proof = schema_faults.install(fd, replacement, owner, {"disk": 0, "branch": "future-format"}, heal_fault) + self.assertEqual((saved / "format.json").read_bytes(), original) + self.assertEqual(hashlib.sha256((root / "data/.rustfs.sys/format.json").read_bytes()).hexdigest(), proof["injectedSha256"]) + self.assertFalse(proof["sharedFormatTouched"]) + self.assertFalse(proof["credentialsRead"]) + + def scene(self): + before = {"node": {"formats": [{"disk": i, "sha256": f"format-{i}"} for i in range(4)], "files": []}} + after = copy.deepcopy(before) + targets, receipts = [], [] + for i in range(4): + relative = "bucket/key/xl.meta" + original = {"disk": i, "path": relative, "sha256": f"original-{i}", "size": 91, + "metadata": {"physicalBitrotVerified": False}} + before["node"]["files"].extend([original, {"disk": i, "path": "bucket/key/data/part.1", "sha256": f"part-{i}", "size": 100}]) + changed = {"sha256": f"future-{i}", "size": 91} + after["node"]["files"].extend([{**original, **changed, "metadataError": {"exactWitnessedFault": True, + "validMetadataClaimed": False, "message": "xl file version err"}}, copy.deepcopy(before["node"]["files"][-1])]) + del after["node"]["files"][-2]["metadata"] + targets.append({"node": "node", "disk": i, "relativePath": relative}) + receipts.append({"kind": "future-xl-schema", "relativePath": relative, + "before": {"sha256": original["sha256"], "size": 91}, "fault": changed}) + return before, after, targets, receipts + + def test_negative_scene_cannot_hide_unknown_record_rewrite_absence_or_payload_mutation(self): + before, after, targets, receipts = self.scene() + self.assertEqual(schema.metadata_scene(before, after, targets, receipts)["unknownRecordsRetainedExactly"], 4) + for kind in ("absent", "rewrite", "valid", "control", "format", "parser", "duplicate", "length"): + bad, bad_receipts = copy.deepcopy(after), copy.deepcopy(receipts) + if kind == "absent": bad["node"]["files"].pop(0) + if kind == "rewrite": bad["node"]["files"][0]["sha256"] = "rewritten" + if kind == "valid": bad["node"]["files"][0]["metadata"] = {} + if kind == "control": bad["node"]["files"][1]["sha256"] = "changed-payload" + if kind == "format": bad["node"]["formats"][0]["sha256"] = "changed" + if kind == "parser": bad["node"]["files"][0]["metadataError"]["message"] = "other corrupt metadata" + if kind == "duplicate": bad["node"]["files"].append(bad["node"]["files"][0]) + if kind == "length": bad_receipts[0]["fault"]["size"] += 1 + with self.subTest(kind=kind), self.assertRaises(ValueError): schema.metadata_scene(before, bad, targets, bad_receipts) + + def test_per_disk_fault_hash_does_not_accept_another_drives_metadata(self): + entry = {"disk": 1, "path": "bucket/key/xl.meta", "sha256": "second", "size": 8} + faults = {"0:bucket/key/xl.meta": {"sha256": "first", "size": 8}, "1:bucket/key/xl.meta": {"sha256": "second", "size": 8}} + with patch.object(fleet_node, "command", return_value=types.SimpleNamespace(returncode=1, stderr="xl file version err")): + observed = fleet_node.inspect_metadata(Path("tool"), Path("path"), "bucket", "key", entry, faults) + self.assertFalse(observed["metadataError"]["validMetadataClaimed"]) + entry["sha256"] = "first" + with self.assertRaises(ValueError): fleet_node.inspect_metadata(Path("tool"), Path("path"), "bucket", "key", entry, faults) + + def test_unknown_get_requires_explicit_server_refusal_without_payload_or_absence(self): + observed = {"acknowledged": False, "resultUnknown": False, + "response": {"ResponseMetadata": {"HTTPStatusCode": 500}, "Error": {"Code": "InternalError"}}} + self.assertFalse(schema.unsupported_get(observed)["unknownTreatedAsMissing"]) + for kind in ("transport", "payload", "absence", "success"): + bad = copy.deepcopy(observed) + if kind == "transport": bad["resultUnknown"] = True + if kind == "payload": bad["payload"] = {"sha256": "bytes"} + if kind == "absence": bad["response"] = {"ResponseMetadata": {"HTTPStatusCode": 404}, "Error": {"Code": "NoSuchKey"}} + if kind == "success": bad["acknowledged"] = True + with self.subTest(kind=kind), self.assertRaises(ValueError): schema.unsupported_get(bad) + + def test_full_128_object_streaming_manifest_keeps_cursor_gap_requirement_separate(self): + cohorts = [{"cohort": [{"node": "node", "disk": i, "relativePath": f"bucket/key{n}/part.1", + "versionId": "null", "partNumber": 1, "sha256": f"hash-{n}-{i}", "size": 81} for i in range(4)]} for n in range(128)] + authority = {(item["disk"], item["relativePath"]): item for cohort in cohorts for item in cohort["cohort"]} + def remote(node, action, *, parts): + self.assertLessEqual(len(parts), 128) + return {"complete": True, "parts": [{**part, "sha256": authority[part["disk"], part["relativePath"]]["sha256"], + "physicalBitrotVerified": True, "physicalSize": 81} for part in parts]} + fleet = types.SimpleNamespace(nodes=["node"], remote=remote, record=lambda label, value: value) + self.assertEqual(verify_part_manifest(fleet, cohorts, "proof", count=128)["allPartsStreamingVerified"], 512) + with self.assertRaises(ValueError): verify_all_parts(fleet, cohorts, "proof", count=128) + with self.assertRaises(ValueError): verify_part_manifest(fleet, cohorts[:-1], "proof", count=128) + + def test_preserves_four_source_branches_and_disables_unrelated_automatic_heal(self): + self.assertEqual(select("C09")[0]["variants"], list(schema.BRANCHES)) + self.assertIn(schema.CASE_ID, AUTOMATIC_CASES) + self.assertEqual(fleet_node.runtime_config("schema-refusal")["RUSTFS_HEAL_AUTO_HEAL_ENABLE"], "false") + self.assertEqual(schema.CONTRACT["candidateCommit"], "072b57b37559fbe626aeda8abf61617fdcbdd926") + + +if __name__ == "__main__": unittest.main() From 21b4ff9656710867dbcbaced08e1804110c6bdf7 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 06:50:38 +0800 Subject: [PATCH 089/343] fix(validation): collect the reviewed complete Heal response budget --- docs/validation-migration-map.md | 11 +++++++++++ scripts/validation_heal/heal_probe.py | 10 ++++++++-- test_validation_heal.py | 25 +++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 09fc9713..909ec55c 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1128,3 +1128,14 @@ again. Backup rescue never earns product-repair credit; failed recovery retains the namespace and stops subsequent fleet use. C09 still requires fresh live evidence. Eleven targeted oracle/guard tests are added; its actual Linux FD/mount guard test runs in CI, not as a simulated macOS syscall claim. + + +F01 `heal.root-graceful-coordinator-restart` run `38091619866` retained +canonical FAIL with cleanup PASS. H1 passed both branches; H2 exceeded the +collector's 2 MiB response limit during original-token recovery. The reviewed +candidate `crates/heal/src/heal/channel.rs` permits an 8 MiB status payload +(source SHA-256 `7f669ca43b9559db9b97fba6b419a6c76a94d76acd8526db132d74b7c534d6b0`). +The collector now uses that explicit 8 MiB bound, records complete byte counts +and still rejects overflow or Content-Length mismatch. A boundary regression +covers a complete 8 MiB JSON response, one-byte overflow and truncated bodies. +No F01 canonical PASS is claimed until a fresh complete rerun. diff --git a/scripts/validation_heal/heal_probe.py b/scripts/validation_heal/heal_probe.py index 20f7cd09..2eda949e 100644 --- a/scripts/validation_heal/heal_probe.py +++ b/scripts/validation_heal/heal_probe.py @@ -23,6 +23,10 @@ MAX_BODY = 64 * 1024**2 +# The reviewed candidate's channel::encode_heal_task_status_payload allows +# 8 MiB (channel.rs SHA-256 7f669ca43b9559db9b97fba6b419a6c76a94d76acd8526db132d74b7c534d6b0). +# Retained H2 diagnostics can exceed 2 MiB without exceeding that wire contract. +ADMIN_RESPONSE_LIMIT = 8 * 1024**2 # Only compare monotonic observations from this collector process. Other # processes/hosts have a different domain even if wall clocks look aligned. CLOCK_DOMAIN = str(uuid.uuid4()) @@ -302,13 +306,15 @@ def admin(self, path: str, query: list[tuple[str, str]], body: bytes, response = exc with observation_phase(observation, 'responseBody'): with response: - raw = response.read(2 * 1024**2 + 1) - require(len(raw) <= 2 * 1024**2, "Admin response exceeds evidence budget") + raw = response.read(ADMIN_RESPONSE_LIMIT + 1) + require(len(raw) <= ADMIN_RESPONSE_LIMIT, "Admin response exceeds evidence budget") length = response.headers.get('Content-Length') require(length is None or (isinstance(length, str) and length.isdigit() and int(length) == len(raw)), 'Admin response Content-Length differs from complete body') evidence["httpStatus"] = response.code evidence["responseHeaders"] = dict(response.headers.items()) + evidence["responseBytes"] = len(raw) + evidence["responseBudgetBytes"] = ADMIN_RESPONSE_LIMIT with observation_phase(observation, 'responseParsing'): try: evidence["response"] = json.loads(raw) diff --git a/test_validation_heal.py b/test_validation_heal.py index b1c5c76b..1d2607f6 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -3224,5 +3224,30 @@ def invoke(args, **kwargs): self.assertEqual(len(observed), 5) +class HealAdminResponseBudgetTest(unittest.TestCase): + def test_complete_reviewed_8mib_response_and_strict_oversize_and_content_length(self): + from io import BytesIO + from test_validation_heal_authorization import binding + from scripts.validation_heal.heal_probe import ADMIN_RESPONSE_LIMIT + self.assertEqual(ADMIN_RESPONSE_LIMIT, 8 * 1024**2) + class Response(BytesIO): + code = 200 + def __init__(self, raw, length=None): + super().__init__(raw) + self.headers = {"Content-Length": str(len(raw) if length is None else length)} + probe = Probe(binding()) + raw = b'{"complete":"' + b'x' * (ADMIN_RESPONSE_LIMIT - len(b'{"complete":""}')) + b'"}' + with patch.object(probe, "credentials", return_value=("test-access", "test-secret")), \ + patch.object(probe, "_open_admin", side_effect=lambda outgoing: Response(raw)): + observed = probe.admin("/rustfs/admin/v3/heal/", [], b"") + self.assertEqual(observed["responseBytes"], ADMIN_RESPONSE_LIMIT) + self.assertEqual(observed["responseBudgetBytes"], ADMIN_RESPONSE_LIMIT) + self.assertEqual(len(observed["response"]["complete"]), ADMIN_RESPONSE_LIMIT - len(b'{"complete":""}')) + for changed, length in ((raw + b" ", None), (b"{}", 3)): + with self.subTest(length=length), patch.object(probe, "credentials", return_value=("test-access", "test-secret")), \ + patch.object(probe, "_open_admin", side_effect=lambda outgoing: Response(changed, length)), self.assertRaises(ValueError): + probe.admin("/rustfs/admin/v3/heal/", [], b"") + + if __name__ == "__main__": unittest.main() From 62d0ca9f5f062a8dde30373993dcd12cfeeef33e Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 07:02:10 +0800 Subject: [PATCH 090/343] fix(validation): validate in-memory and serialized Heal query pairs --- scripts/validation_heal/stop.py | 5 ++++- test_validation_heal.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/stop.py b/scripts/validation_heal/stop.py index c11bdf96..05b001a0 100644 --- a/scripts/validation_heal/stop.py +++ b/scripts/validation_heal/stop.py @@ -88,7 +88,10 @@ def wait_partial(probe, fleet, path, token, targets, *, repair_required): def validate_retired_stop(observed, token): require(observed.get("httpStatus") == 200 and not observed.get("transportError"), "retired token stop did not follow the frozen HTTP contract") - require(observed.get("query") == [["clientToken", token], ["forceStop", "true"]] + query = observed.get("query") + require(isinstance(query, list) and len(query) == 2 + and all(isinstance(pair, (list, tuple)) and len(pair) == 2 for pair in query) + and [tuple(pair) for pair in query] == [("clientToken", token), ("forceStop", "true")] and observed.get("method") == "POST" and observed.get("auth") == "signed" and observed.get("requestBody") == "" and isinstance(observed.get("path"), str) and observed["path"].startswith("/rustfs/admin/v3/heal/"), "retired stop did not target the exact original token") diff --git a/test_validation_heal.py b/test_validation_heal.py index 1d2607f6..f4557020 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -2076,8 +2076,12 @@ def test_repeat_stop_requires_idempotent_ack_for_the_exact_retained_token(self): "settings": {**BASE_OPTIONS, "scanMode": 1, "pool": None, "set": None}}} self.assertEqual(validate_retired_stop(response, "old-token")["exactRetiredTaskId"], "old-token") self.assertFalse(validate_retired_stop(response, "old-token")["responseEchoesTaskIdentity"]) + in_memory = {**response, "query": [("clientToken", "old-token"), ("forceStop", "true")]} + self.assertEqual(validate_retired_stop(in_memory, "old-token"), validate_retired_stop(json.loads(json.dumps(in_memory)), "old-token")) for changed in ({**response, "httpStatus": 500}, {**response, "transportError": "timeout"}, {**response, "query": [["clientToken", "new-token"], ["forceStop", "true"]]}, + {**response, "query": [["clientToken", "old-token"], ["forceStop", "true"], ["clientToken", "other"]]}, + {**response, "query": {"clientToken": "old-token", "forceStop": "true"}}, {**response, "response": {**response["response"], "summary": "finished"}}, {**response, "response": {**response["response"], "outcome": {"execution": {"state": "completed"}}}}, {**response, "response": {**response["response"], "settings": BASE_OPTIONS}}, From f248cb63d032cfc1658d32c508bce745f96b885f Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 07:06:09 +0800 Subject: [PATCH 091/343] Migrate encrypted object Heal validation with ciphertext and key authority proofs --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 44 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/encrypted.py | 339 ++++++++++++++++++ scripts/validation_heal/encryption_runtime.py | 68 ++++ scripts/validation_heal/fleet_node.py | 18 +- scripts/validation_heal/heal_probe.py | 18 +- scripts/validation_heal/healthy.py | 5 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 1 + scripts/validation_heal/shards.py | 15 +- test_validation_heal_encrypted.py | 152 ++++++++ 12 files changed, 651 insertions(+), 17 deletions(-) create mode 100644 scripts/validation_heal/encrypted.py create mode 100644 scripts/validation_heal/encryption_runtime.py create mode 100644 test_validation_heal_encrypted.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 535e205f..ce238281 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -25,6 +25,7 @@ on: - test_validation_heal_scopes.py - test_validation_heal_bucket_metadata.py - test_validation_heal_write_quorum.py + - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py - scripts/validation_heal/** @@ -65,6 +66,7 @@ on: - test_validation_heal_scopes.py - test_validation_heal_bucket_metadata.py - test_validation_heal_write_quorum.py + - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py - scripts/validation_heal/** @@ -98,7 +100,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 909ec55c..5fd6eff2 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 124 currently dispatchable IDs include all 74 non-Heal cases and fifty Heal cases. Forty-five have live PASS evidence; the other 113 have no live PASS claim. +case evidence. The 125 currently dispatchable IDs include all 74 non-Heal cases and fifty-one Heal cases. Forty-five have live PASS evidence; the other 113 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 34 Heal cases remain +The other 33 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -1036,8 +1036,8 @@ four regression verdicts have their own archive and never increment a fleet or canonical-case PASS count. Existing tests assert captured lock events internally; the collector does not fabricate emitted barrier logs. Missing critical-point fixtures, Admin nolock semantics and required fleet overlap remain explicit gaps. -The current implementation has 253 focused Heal/catalog checks, including the -Linux-only D11 ownership/mount/recovery checks. Native compilation and execution +The current implementation has 274 focused Heal/catalog checks, including the +Linux-only D11, C09 and D08 ownership/mount/recovery checks. Native compilation and execution still require the separate hosted job; local parser checks are not native PASS. @@ -1139,3 +1139,39 @@ The collector now uses that explicit 8 MiB bound, records complete byte counts and still rejects overflow or Content-Length mismatch. A boundary regression covers a complete 8 MiB JSON response, one-byte overflow and truncated bodies. No F01 canonical PASS is claimed until a fresh complete rerun. + +A06 run `38092515125` retained FAIL/cleanup PASS on both H1/H2 because the +collector represents signed query pairs as Python tuples in memory and as +JSON arrays on disk. The exact durable-stop validator incorrectly accepted +only the latter. It now normalizes either pair representation while retaining +the exact pair count/order/token, signed POST, empty body, response shape and +old/new responsibility assertions. Tests reject duplicated tokens and mapping +queries. All six CI lanes passed at `62d0ca9`; fresh complete A06/F06 runs are +still required. Earlier FAIL results are preserved. + +D08 `heal.encrypted-objects` dispatches SSE-S3, SSE-KMS, SSE-C and encrypted +historical versions on both H1/H2. It configures one fresh AES-256 static KMS +key across the run-owned stopped services, passes material through SSH stdin +and retains it only in a private 0600 file inside each run namespace. Every +snapshot attests the file/configuration identity and the actual process +settings, including explicit per-key authorization enforcement. Shared KMS +resources are not modified and secret material is excluded from evidence. + +Each branch independently ACKs sixteen external encrypted objects; the +versioned branch ACKs three distinct generations per key. It removes a data +shard and corrupts a parity shard, including historical and current versions, +then requires a manual deep token with complete exact responsibility. Before +any decrypted target GET, the original same-role ciphertext hashes/lengths and +streaming integrity must be reconstructed. An encrypted-only mapper preserves +the distinct plaintext actualSize and ciphertext part size without relaxing +the ordinary plaintext mapper. Every peer then verifies the entire plaintext +ACK/version/tag/property inventory and re-freezes the ciphertext scene. + +SSE-C missing/wrong keys must produce the reviewed 400/InvalidRequest refusal. +SSE-KMS principals have the same working S3 GET rights to an independently +verified plaintext control, but no decrypt grant or only an unrelated key +grant; the encrypted GET must return 403/AccessDenied before and after repair. +SSE-S3 records that its service-managed key has no caller decryption-key +argument. Failed IAM cleanup retains a DIRTY stopped namespace. Nine targeted +geometry, authority, version, redaction and Linux file-ownership tests pass; +D08 still requires fresh complete live evidence and has no live PASS claim. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index f1e16d98..419bb7ea 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/encrypted.py b/scripts/validation_heal/encrypted.py new file mode 100644 index 00000000..2ab6e0f7 --- /dev/null +++ b/scripts/validation_heal/encrypted.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +"""D08: same-role ciphertext reconstruction and independently decrypted ACKs.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys + +try: + from .heal_probe import Probe, compare_inventories, payload, require, save + from .heal_workload import Recorder, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .shards import _map_external_target, files, run_task, validate_repaired_physical, validate_fault, verify_cohort + from .versions import validate_version_receipts + from .unformatted import binding + from .authorization import iam_exchange + from .schema import exact_get +except ImportError: + from heal_probe import Probe, compare_inventories, payload, require, save + from heal_workload import Recorder, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from shards import _map_external_target, files, run_task, validate_repaired_physical, validate_fault, verify_cohort + from versions import validate_version_receipts + from unformatted import binding + from authorization import iam_exchange + from schema import exact_get + + +CASE_ID = "heal.encrypted-objects" +BRANCHES = ("sse-s3", "sse-kms", "sse-c", "encrypted-versioned") +COUNT = 16 +SIZE = 5 * 1024**2 + 123 +CANDIDATE = "072b57b37559fbe626aeda8abf61617fdcbdd926" +REFERENCE = {"candidateCommit": CANDIDATE, "dependency": "fresh run-owned static AES-256 KMS key on every candidate node", + "sourceFiles": { + "crates/kms/src/config.rs": "7931bb4c6757e60ca3e7fdbff4ac56c5fab59d6d6df8756c9d0287ca692a26f9", + "crates/kms/src/backends/static_kms.rs": "a5cf1cbe4251f79221d171c58b3844b6aa7f073a4173da0783a6d9208c5b580a", + "rustfs/src/storage/sse.rs": "37ecffdb4dd18a0528014afd30237c8772c3fb7f27fe2c4cc018e6d475a29641", + "crates/e2e_test/src/kms/kms_authorization_negative_matrix_test.rs": "d64724dbec366fb54eabcfa80cd1b11448a11174f12c7c801107d37b10825643"}, + "ciphertextPolicy": "single uncompressed encrypted part; decoded ciphertext size and original plaintext actualSize are distinct", + "negativeKeyPolicy": "SSE-C missing/wrong customer key; SSE-KMS absent/other-key decrypt grant; explicit per-key enforcement", + "managedS3ReadKey": "service-managed; S3 caller does not supply a decryption key"} +SSEC = {"SSECustomerAlgorithm": "AES256", "SSECustomerKey": "$HEAL_SSE_C_KEY"} + + +def mode_for(branch): + require(branch in BRANCHES, "unrecognized D08 source branch") + return "sse-kms" if branch == "encrypted-versioned" else branch + + +def put_options(branch, key_id): + mode = mode_for(branch) + if mode == "sse-c": return dict(SSEC) + if mode == "sse-s3": return {"ServerSideEncryption": "AES256"} + return {"ServerSideEncryption": "aws:kms", "SSEKMSKeyId": key_id} + + +def encryption_headers(response, branch, key_id): + mode = mode_for(branch) + if mode == "sse-c": + key_md5 = base64.b64encode(hashlib.md5(os.environ["HEAL_SSE_C_KEY"].encode(), usedforsecurity=False).digest()).decode() + require(response.get("SSECustomerAlgorithm") == "AES256" and response.get("SSECustomerKeyMD5") == key_md5 + and not response.get("ServerSideEncryption"), "SSE-C ACK/GET did not preserve its actual encryption classification") + else: + require(response.get("ServerSideEncryption") == ("AES256" if mode == "sse-s3" else "aws:kms"), + "encrypted ACK/GET silently downgraded its server encryption mode") + if mode == "sse-kms": require(response.get("SSEKMSKeyId") == key_id, "KMS object used another key identity") + return {"encrypted": True, "mode": mode, "keyMaterialLogged": False} + + +def cipher_part_policy(expected): + plaintext = expected["payload"]["size"] + require(type(plaintext) is int and plaintext > 1024**2 and not expected["deleteMarker"], "D08 requires its full external plaintext ACK") + def checked(version, part): + require(len(version.get("parts", [])) == 1 and part.get("number") == 1 + and type(part.get("size")) is int and type(part.get("actualSize")) is int + and part["actualSize"] == plaintext and plaintext < part["size"] <= plaintext + 1024**2 + and version.get("size") == part["size"] and version.get("inline") is None, + "ciphertext geometry is not the reviewed uncompressed single-part encoding of this plaintext ACK") + return checked + + +def map_cipher_target(snapshot, bucket, expected, role): + target = _map_external_target(snapshot, bucket, expected["key"], role, 1, expected["versionId"], + latest_required=expected["isLatest"], part_policy=cipher_part_policy(expected)) + return {**target, "encryptedPhysicalLayer": True, "plaintextACK": expected["payload"], + "ciphertextPartSize": target["logical"]["parts"][0]["size"]} + + +def denied_key(observed, *, customer): + expected = (400, "InvalidRequest") if customer else (403, "AccessDenied") + response = observed.get("response") or {} + require(observed.get("acknowledged") is False and observed.get("resultUnknown") is False and "payload" not in observed + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == expected[0] + and response.get("Error", {}).get("Code") == expected[1], + "missing/wrong decryption authority returned data, absence or an unreviewed error") + return {"httpStatus": expected[0], "code": expected[1], "missingOrWrongKeyIsNotObjectAbsence": True} + + +def seed_encrypted(recorder, bucket, branch, key_id): + recorder.call("create_bucket", {"Bucket": bucket}) + versioned = branch == "encrypted-versioned" + if versioned: + recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) + require(recorder.call("get_bucket_versioning", {"Bucket": bucket}).get("Status") == "Enabled", "encrypted versioning was not enabled") + expected, identities = [], set() + for index in range(COUNT): + key = f"encrypted/{index:04d}.bin" + for generation in range(3 if versioned else 1): + body = payload(f"{bucket}/{key}/{generation}", SIZE + generation * 37) + observed = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": "DE", "generation": str(generation)}, + "Tagging": "suite=heal&fixture=DE", **put_options(branch, key_id)}, body) + encryption_headers(observed, branch, key_id) + version = observed.get("VersionId", "null") + require((versioned and isinstance(version, str) and version not in {"", "null"}) + or not versioned and version in {None, "null"}, "encrypted PUT omitted its exact expected version identity") + version = version or "null" + require((key, version) not in identities, "encrypted ACK reused a logical version") + identities.add((key, version)) + expected.append({"key": key, "versionId": version, "isLatest": not versioned or generation == 2, + "deleteMarker": False, "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + return expected + + +def principal_setup(probe, fleet, bucket, key_id, context): + suffix = hashlib.sha256(probe.binding["runId"].encode()).hexdigest()[:14] + for index, grant in enumerate((None, key_id + "-other")): + user, policy = "rvheal" + suffix + f"encrypted{index}", "rvheal" + suffix + f"keypolicy{index}" + secret = secrets.token_hex(24) + fleet.secret_values.update({user, secret}) + item = {"user": user, "policy": policy, "attached": False, "policyCreated": False, + "principal": Probe(probe.binding, credential_pair=(user, secret)), "grant": grant} + context.append(item) + iam_exchange(probe, fleet, "add-user", user, secret=secret) + statements = [{"Effect": "Allow", "Action": ["s3:GetObject", "s3:GetObjectVersion"], + "Resource": [f"arn:aws:s3:::{bucket}/*", f"arn:aws:s3:::{bucket}-control/*"]}] + if grant: + statements.append({"Effect": "Allow", "Action": ["kms:Decrypt"], "Resource": ["arn:aws:kms:::key/" + grant]}) + item["policyCreated"] = True + iam_exchange(probe, fleet, "add-policy", user, policy=policy, document={"Version": "2012-10-17", "Statement": statements}) + item["attached"] = True + iam_exchange(probe, fleet, "attach", user, policy=policy) + + +def key_negatives(probe, fleet, bucket, branch, entry, control, context, label): + params = {"Bucket": bucket, "Key": entry["key"], "VersionId": entry["versionId"]} + proofs = [] + if mode_for(branch) == "sse-c": + for name, options in (("missing", {}), ("wrong", {**SSEC, "SSECustomerKey": "$HEAL_SSE_C_WRONG_KEY"})): + observed = fleet.record(label + "-customer-key-" + name, probe.s3("get_object", {**params, **options})) + proofs.append({"variant": name, **denied_key(observed, customer=True)}) + elif mode_for(branch) == "sse-kms": + for item in context: + principal = item["principal"] + readable = fleet.record(label + "-same-principal-plaintext-control", principal.s3("get_object", { + "Bucket": bucket + "-control", "Key": control["key"]})) + exact_get(readable, control) + observed = fleet.record(label + "-KMS-grant-denied", principal.s3("get_object", params)) + proofs.append({"variant": "missing-grant" if item["grant"] is None else "other-key-grant", **denied_key(observed, customer=False), + "samePrincipalS3ReadAllowed": True}) + else: + # SSE-S3 does not take a customer key on GET. Test its positive public + # classification; inventing a missing client key would be false coverage. + proofs.append({"variant": "service-managed-key", "callerKeyArgumentApplicable": False}) + return proofs + + +def run_branch(args, topology, nodes, branch): + directory = args.evidence / f"{topology}-{branch}" + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory) + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == CANDIDATE, "unreviewed encrypted geometry/KMS policy contract") + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + fleet.record("encryption-contract", REFERENCE) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + key_id, material = "rv-heal-key-" + run_id, base64.b64encode(secrets.token_bytes(32)).decode() + access, secret = secrets.token_hex(16), secrets.token_hex(24) + customer, wrong = secrets.token_hex(16), secrets.token_hex(16) + require(customer != wrong, "negative customer key duplicated the valid key") + fleet.secret_values.update({material, key_id + ":" + material, customer, wrong, + base64.b64encode(customer.encode()).decode(), base64.b64encode(wrong.encode()).decode()}) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret, HEAL_SSE_C_KEY=customer, HEAL_SSE_C_WRONG_KEY=wrong) + lab = binding(args, candidate, nodes, run_id, bucket) + fleet.record("binding", lab) + probe, active, context = Probe(lab), [], [] + fleet.snapshot_buckets = [bucket, bucket + "-control"] + result = {"topology": topology, "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} + iam_dirty = False + try: + def configure(): + for node in nodes: + fleet.record("dedicated-KMS-configured", {"node": node, **fleet.remote(node, "encryption-configure", keyId=key_id, keyBase64=material)}) + fleet.deploy(args.binary_sha, bucket, access, secret, before_start=configure) + recorder = Recorder(probe, directory / "fixture") + expected = seed_encrypted(recorder, bucket, branch, key_id) + read_options = SSEC if mode_for(branch) == "sse-c" else {} + baseline = fleet.record("decrypted-S3-baseline", probe.inventory(bucket, + unversioned_current_tags=branch != "encrypted-versioned", object_read_options=read_options)) + fleet.record("independent-plaintext-ACKs", {"expected": expected, **verify_seed(expected, baseline)}) + for entry in baseline["entries"]: encryption_headers(entry["properties"], branch, key_id) + control_bucket = bucket + "-control" + recorder.call("create_bucket", {"Bucket": control_bucket}) + control_body = payload(control_bucket + "/unaffected", SIZE) + recorder.call("put_object", {"Bucket": control_bucket, "Key": "unaffected.bin"}, control_body) + control = {"key": "unaffected.bin", "versionId": "null", "deleteMarker": False, "isLatest": True, + "payload": {"size": len(control_body), "sha256": hashlib.sha256(control_body).hexdigest()}} + control_baseline = fleet.record("plaintext-control-baseline", probe.inventory(control_bucket, unversioned_current_tags=True)) + verify_seed([control], control_baseline) + require(all(not any(name in entry["properties"] for name in + ("ServerSideEncryption", "SSEKMSKeyId", "SSECustomerAlgorithm")) for entry in control_baseline["entries"]), + "key-authority control is not an independently readable plaintext object") + if mode_for(branch) == "sse-kms": principal_setup(probe, fleet, bucket, key_id, context) + selected = [next(entry for entry in expected if entry["key"] == key and (entry["isLatest"] if index else True)) + for index, key in enumerate(("encrypted/0002.bin", "encrypted/0004.bin"))] + result["baselineKeyNegatives"] = key_negatives(probe, fleet, bucket, branch, selected[0], control, context, "baseline") + # Each peer must already serve the correct decrypted version before a + # shard fault. This proves actual key propagation and service readiness. + for endpoint in lab["endpoints"]: + observed = fleet.record("all-peer-dependency-readiness", Probe(lab, endpoint).s3("get_object", { + "Bucket": bucket, "Key": selected[0]["key"], "VersionId": selected[0]["versionId"], **read_options})) + exact_get(observed, selected[0]); encryption_headers(observed["response"], branch, key_id) + fleet.stop() + before = fleet.snapshot("original-ciphertext-full-physical", quiescent=True) + fleet.record("actual-topology", validate_topology(before, topology)) + require(all(before[node]["encryptionConfigAttestation"]["runOwnedSecretFileUnchanged"] for node in nodes), "dedicated KMS identity changed") + targets = [map_cipher_target(before, bucket, entry, "data" if index == 0 else "parity") for index, entry in enumerate(selected)] + for target in targets: fleet.record("original-ciphertext-streaming", verify_cohort(fleet, target, "original-ciphertext-part")) + prior = before + registrations = [] + for index, target in enumerate(targets): + params = {"disk": target["disk"], "faultId": f"D08-{branch}-{index}"} + registrations.append((target["node"], params)); fleet.faults.append(registrations[-1]) + receipt = fleet.record("ciphertext-fault", fleet.remote(target["node"], "apply-fault", **params, + relativePath=target["relativePath"], kind="missing" if index == 0 else "bitflip", expectedSha256=target["sha256"], + offset=None if index == 0 else target["size"] // 2)) + changed = fleet.snapshot("witnessed-ciphertext-fault", quiescent=True) + validate_fault(prior, changed, target, receipt); prior = changed + fleet.start() + require(physical_content(prior) == physical_content(fleet.snapshot("ciphertext-before-manual-token")), + "ciphertext repaired before its attributed manual token") + status = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, option_overrides={"recursive": True}, + status_endpoints=tuple(lab["endpoints"])[1:]) + result["responsibility"] = validate_version_receipts(status, bucket, expected, 2, + required_repairs={(entry["key"], entry["versionId"]) for entry in selected}) + fleet.stop() + repaired = fleet.snapshot("ciphertext-repaired-before-plaintext-GET", quiescent=True) + result["physical"] = validate_repaired_physical(before, repaired, targets[0], repair_targets=targets) + for target in targets: fleet.record("reconstructed-ciphertext-streaming", verify_cohort(fleet, target, "repaired-ciphertext-part")) + for registration in registrations: fleet.faults.remove(registration) + fleet.start() + result["postRepairKeyNegatives"] = key_negatives(probe, fleet, bucket, branch, selected[0], control, context, "post-repair") + for endpoint in lab["endpoints"]: + entry_probe = Probe(lab, endpoint) + inventory = fleet.record("all-peer-full-decrypted-manifest", {"endpoint": endpoint, **entry_probe.inventory(bucket, + unversioned_current_tags=branch != "encrypted-versioned", object_read_options=read_options)}) + fleet.record("all-peer-plaintext-and-properties", {**verify_seed(expected, inventory), **compare_inventories(baseline, inventory)}) + for entry in inventory["entries"]: encryption_headers(entry["properties"], branch, key_id) + controls = fleet.record("all-peer-plaintext-controls", entry_probe.inventory(control_bucket, unversioned_current_tags=True)) + compare_inventories(control_baseline, controls) + fleet.stop() + require(physical_content(repaired) == physical_content(fleet.snapshot("ciphertext-after-decryption-and-negatives", quiescent=True)), + "decryption verification changed or downgraded the proven ciphertext repair") + result.update(verdict="PASS", allPlaintextACKsAndVersionsPreserved=True, sameRoleCiphertextRestored=True, + keyMaterialLogged=False, sharedKmsTouched=False, backupRestorationUsed=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + if fleet.reserved: + try: fleet.stop(); fleet.snapshot("D08-failure-ciphertext-scene", quiescent=True) + except Exception as capture: result["failureEvidenceError"] = str(capture) + finally: + for path, token in active: + try: fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: fleet.record("stop-error", {"error": str(exc)}) + if context: + try: fleet.stop(); fleet.start() + except Exception as exc: iam_dirty = True; fleet.record("IAM-recovery-start-failed", {"error": str(exc)}) + if not iam_dirty: + for item in context: + for operation, needed in (("detach", item["attached"]), ("remove-user", True), ("remove-policy", item["policyCreated"])): + if not needed: continue + try: iam_exchange(probe, fleet, operation, item["user"], policy=item["policy"] if operation != "remove-user" else None) + except Exception as exc: iam_dirty = True; fleet.record("IAM-recovery-DIRTY", {"operation": operation, "error": str(exc)}) + if iam_dirty: + fleet.stop() + fleet.record("retained-key-authority-environment", {"state": "DIRTY", "namespacesRetained": True}) + result["cleanup"] = "PASS" if not iam_dirty and fleet.cleanup() else "FAIL" + fleet.record("branch-result", result) + return result + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f"Heal workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ("candidate", "rc-bin", "evidence", "tool-directory"): parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(set(nodes)) == len(nodes) + and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "four current Heal nodes required") + require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)) + and bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "verified workflow candidate required") + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "D08 executor omitted a canonical encryption branch") + results = [] + try: + for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): + for branch in BRANCHES: + results.append(run_branch(args, topology, selected, branch)) + if results[-1]["cleanup"] != "PASS": break + if results[-1]["cleanup"] != "PASS": break + finally: + complete = len(results) == 8 and all(item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, + "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", "complete": complete}) + require(complete, "D08 requires all four encryption branches on H1/H2 with exact ciphertext/plaintext/key-negative proofs") + + +if __name__ == "__main__": + try: main() + except Exception as exc: + print(f"Heal encrypted object validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/encryption_runtime.py b/scripts/validation_heal/encryption_runtime.py new file mode 100644 index 00000000..fa685976 --- /dev/null +++ b/scripts/validation_heal/encryption_runtime.py @@ -0,0 +1,68 @@ +"""D08 static KMS material only inside the stopped run-owned service namespace.""" + +import base64 +import hashlib +import json +import os +import re +import stat + + +def settings_for(state, key_id): + return {"RUSTFS_KMS_ENABLE": "true", "RUSTFS_KMS_BACKEND": "static", "RUSTFS_KMS_DEFAULT_KEY_ID": key_id, + "RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY": "true", "RUSTFS_KMS_STATIC_SECRET_KEY_FILE": str(state / "static-kms-secret")} + + +def require(value, message): + if not value: raise ValueError(message) + + +def configure(state, owner, request, fault): + key_id = request.get("keyId") + material = request.get("keyBase64") + require(key_id == "rv-heal-key-" + owner["runId"] and isinstance(material, str), "encryption key is outside its exact run") + decoded = base64.b64decode(material, validate=True) + require(len(decoded) == 32 and len(set(decoded)) > 1, "fresh AES-256 material required") + value = {"schemaVersion": 1, "runId": owner["runId"], "backend": "static", "keyId": key_id, + "secretPath": str(state / "static-kms-secret"), "keyMaterialLogged": False, "sharedKmsTouched": False} + with fault.directory(state) as fd: + original = fault.read_file(fd, "service.env", 65536) + require(not re.search(rb"(?m)^RUSTFS_KMS_", original), "run already has an encryption dependency configuration") + secret = (key_id + ":" + material + "\n").encode() + fault.new_file(fd, "static-kms-secret", secret) + value["secretSha256"] = hashlib.sha256(secret).hexdigest() + settings = settings_for(state, key_id) + updated = original + "".join(f"{name}={item}\n" for name, item in settings.items()).encode() + value.update(settings=settings, serviceEnvSha256=hashlib.sha256(updated).hexdigest()) + fault.new_file(fd, "encryption-intent.json", fault.encode(value)) + fault.new_file(fd, "service-encryption.env", updated) + os.replace("service-encryption.env", "service.env", src_dir_fd=fd, dst_dir_fd=fd) + os.fsync(fd) + fault.new_file(fd, "encryption-config.json", fault.encode(value)) + return {name: value[name] for name in ("schemaVersion", "runId", "backend", "keyId", "keyMaterialLogged", "sharedKmsTouched")} + + +def attest(state, owner, checked, digest, environment=None): + marker = checked(state / "encryption-config.json", state) + intent = checked(state / "encryption-intent.json", state) + if not marker.exists(): + require(not intent.exists(), "partial encryption configuration remains unresolved") + return None + require(marker.stat().st_size < 8192, "encryption attestation exceeds its bound") + value = json.loads(marker.read_text()) + secret = checked(state / "static-kms-secret", state) + require(stat.S_ISREG(secret.stat().st_mode) and stat.S_IMODE(secret.stat().st_mode) == 0o600 + and secret.stat().st_nlink == 1 and secret.stat().st_uid == marker.stat().st_uid, + "dedicated key material no longer has its private file ownership") + require(value.get("schemaVersion") == 1 and value.get("runId") == owner["runId"] + and value.get("backend") == "static" and value.get("keyId") == "rv-heal-key-" + owner["runId"] + and value.get("secretPath") == str(state / "static-kms-secret") + and value.get("settings") == settings_for(state, value["keyId"]) + and digest(checked(state / "static-kms-secret", state)) == value["secretSha256"] + and digest(checked(state / "service.env", state)) == value["serviceEnvSha256"], + "run-owned KMS configuration or key material changed") + if environment is not None: + require(all(environment.get(name) == item for name, item in value["settings"].items()), + "candidate process did not load the exact dedicated KMS configuration") + return {"backend": "static", "keyId": value["keyId"], "runOwnedSecretFileUnchanged": True, + "loadedByCandidate": environment is not None, "sharedKmsTouched": False, "keyMaterialLogged": False} diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 5a61b2f1..08e46853 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -407,6 +407,12 @@ def fault_module(): return heal_fault +def encryption_module(): + if "encryption_runtime" in globals(): return globals()["encryption_runtime"] + from . import encryption_runtime + return encryption_runtime + + def gate_module(): if "gate" in globals(): return globals()["gate"] from . import drive_gate @@ -696,6 +702,8 @@ def snapshot(request): require(all(environment.get(key) == value for key, value in expected_config.items()), "bound Heal runtime configuration differs") files, formats = [], [] + encryption = encryption_module().attest(state, owner, checked, digest, + None if request.get("quiescent") is True else environment) limits = resource_limits(owner.get("resourceProfile", "standard")) tool = state / "heal-meta-inspect" if owner.get("inspectorSha256"): @@ -733,7 +741,8 @@ def snapshot(request): "storageClassEnvironment": None if request.get("quiescent") is True else { key: environment.get(key) for key in ("RUSTFS_STORAGE_CLASS_INLINE_BLOCK", "RUSTFS_STORAGE_CLASS_STANDARD")}, "resourceLimits": limits, - "files": sorted(files, key=lambda value: (value["disk"], value["path"])), "formats": formats} + "files": sorted(files, key=lambda value: (value["disk"], value["path"])), "formats": formats, + "encryptionConfigAttestation": encryption} def logs(request): @@ -810,6 +819,12 @@ def format_schema_fault(request): return globals()["schema_faults"].install(fd, replacement, owner, request, fault) +def configure_encryption(request): + state, _, owner = owned(request["runId"]) + stopped(owner) + return encryption_module().configure(state, owner, request, fault_module()) + + def cleanup(request): state, roots, owner = owned(request["runId"]) command("systemctl", "stop", owner["unit"], check=False) @@ -918,6 +933,7 @@ def main(request): for action in ("quorum-gate", "quorum-gate-probe"): actions[action] = quorum_gate_operation actions["format-schema-fault"] = format_schema_fault + actions["encryption-configure"] = configure_encryption actions.update({"replace-drive": replace_drive, "replacement-state": replacement_state, "replacement-probe": replacement_probe}) actions.update({"format-export": format_export, "foreign-format": foreign_format, "restore-replacement": restore_replacement}) actions.update({"process": process, "process-metrics": process_metrics, "process-signal": process_signal, "process-graceful": process_graceful, "process-crash": process_crash}) diff --git a/scripts/validation_heal/heal_probe.py b/scripts/validation_heal/heal_probe.py index 2eda949e..159f7f70 100644 --- a/scripts/validation_heal/heal_probe.py +++ b/scripts/validation_heal/heal_probe.py @@ -83,7 +83,7 @@ def redact(value): if isinstance(value, list): return [redact(item) for item in value] if isinstance(value, str): - for name in ("HEAL_ACCESS_KEY", "HEAL_SECRET_KEY", "HEAL_SESSION_TOKEN", "HEAL_SSE_C_KEY"): + for name in ("HEAL_ACCESS_KEY", "HEAL_SECRET_KEY", "HEAL_SESSION_TOKEN", "HEAL_SSE_C_KEY", "HEAL_SSE_C_WRONG_KEY"): secret = os.environ.get(name) if secret: value = value.replace(secret, "[REDACTED]") @@ -202,8 +202,8 @@ def s3(self, operation: str, params: dict, body: bytes | None = None, *, allow_g validate_s3_request(self.binding, operation, params, allow_governance_bypass=allow_governance_bypass) outgoing = dict(params) for key in ("SSECustomerKey", "CopySourceSSECustomerKey"): - if outgoing.get(key) == "$HEAL_SSE_C_KEY": - outgoing[key] = os.environ["HEAL_SSE_C_KEY"] + if outgoing.get(key) in {"$HEAL_SSE_C_KEY", "$HEAL_SSE_C_WRONG_KEY"}: + outgoing[key] = os.environ[outgoing[key][1:]] if body is not None: require(operation in {"put_object", "upload_part"}, "body not allowed for this operation") outgoing["Body"] = body @@ -331,13 +331,18 @@ def _open_admin(self, outgoing): opener = request.build_opener(request.ProxyHandler({}), NoRedirect()) return opener.open(outgoing, timeout=30) - def inventory(self, bucket: str, *, unversioned_current_tags: bool = False, page_size: int | None = None) -> dict: + def inventory(self, bucket: str, *, unversioned_current_tags: bool = False, page_size: int | None = None, + object_read_options: dict | None = None) -> dict: """Enumerate every version/marker, then hash complete version-addressed GETs. Only call before injection or after attribution/physical proof: a GET can itself trigger read repair. Never use this to watch scanner progress. """ bucket_allowed(self.binding, bucket) + read_options = {} if object_read_options is None else object_read_options + require(isinstance(read_options, dict) and set(read_options) <= {"SSECustomerAlgorithm", "SSECustomerKey"} + and (not read_options or read_options == {"SSECustomerAlgorithm": "AES256", "SSECustomerKey": "$HEAL_SSE_C_KEY"}), + "inventory decryption options must use the reviewed in-memory customer-key reference") require(page_size is None or type(page_size) is int and 1 <= page_size <= 1000, "invalid explicit inventory page size") client, entries, count, byte_count = self.s3_client(), [], 0, 0 @@ -361,7 +366,7 @@ def inventory(self, bucket: str, *, unversioned_current_tags: bool = False, page count += 1 require(count <= self.binding["maxInventoryObjects"], "inventory object budget exceeded") params = {"Bucket": bucket, "Key": version["Key"], "VersionId": version["VersionId"]} - result = self.s3("get_object", params) + result = self.s3("get_object", {**params, **read_options}) require(result["acknowledged"], "version GET failed: " + canonical(result)) tag_params = dict(params) if unversioned_current_tags: @@ -379,7 +384,8 @@ def inventory(self, bucket: str, *, unversioned_current_tags: bool = False, page "properties": {key: response[key] for key in ( "ETag", "ContentLength", "ContentType", "ContentEncoding", "ContentDisposition", "CacheControl", "Metadata", "StorageClass", - "ServerSideEncryption", "ObjectLockMode", "ObjectLockRetainUntilDate", + "ServerSideEncryption", "SSEKMSKeyId", "SSECustomerAlgorithm", "SSECustomerKeyMD5", + "ObjectLockMode", "ObjectLockRetainUntilDate", "ObjectLockLegalHoldStatus") if key in response}, "tags": sorted(tags["response"].get("TagSet", []), key=lambda item: item["Key"])}) require(count <= self.binding["maxInventoryObjects"], "inventory object budget exceeded") diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 8020c017..5bd05e0b 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -184,7 +184,7 @@ def remote(self, node, action, **params): source += "\nimport types\nmrf = types.ModuleType('mrf_records')\nexec(" + repr( NODE_SOURCE.with_name("mrf_records.py").read_text()) + ", mrf.__dict__)\n" if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline", - "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault"}: + "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure"}: source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" if action.startswith("bucket-record-"): @@ -193,6 +193,9 @@ def remote(self, node, action, **params): if action == "format-schema-fault": source += "\nimport types\nschema_faults = types.ModuleType('schema_faults')\nexec(" + repr( NODE_SOURCE.with_name("schema_faults.py").read_text()) + ", schema_faults.__dict__)\n" + if action in {"encryption-configure", "snapshot"}: + source += "\nimport types\nencryption_runtime = types.ModuleType('encryption_runtime')\nexec(" + repr( + NODE_SOURCE.with_name("encryption_runtime.py").read_text()) + ", encryption_runtime.__dict__)\n" replacement_actions = {"start", "cleanup", "audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot", "format-schema-fault"} if action in {"start", "cleanup", "audit-finalize", "drive-gate", "replace-drive", "quorum-gate", "quorum-gate-probe"}: source += "\nimport types\nquorum_gate_backend = types.ModuleType('quorum_gate')\nexec(" + repr( diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 69f645ef..067687d4 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 3c4d051e..8110ec4a 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -34,6 +34,7 @@ case "${case_id}" in heal.disk-replacement-unformatted) driver=unformatted.py ;; heal.below-write-quorum) driver=write_quorum.py ;; heal.foreign-format-and-unknown-schema) driver=schema.py ;; + heal.encrypted-objects) driver=encrypted.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index 41498711..5b5b358c 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -98,6 +98,13 @@ def files(snapshot): def map_target(snapshot, bucket, key, role, part_number, version_id="null", *, preferred_nodes=None, latest_required=True, multi_set=False, set_index=0): + return _map_external_target(snapshot, bucket, key, role, part_number, version_id, + preferred_nodes=preferred_nodes, latest_required=latest_required, multi_set=multi_set, set_index=set_index) + + +def _map_external_target(snapshot, bucket, key, role, part_number, version_id="null", *, preferred_nodes=None, + latest_required=True, multi_set=False, set_index=0, part_policy=None): + """Share set/role authority checks while keeping transformed part policy explicit.""" require(type(multi_set) is bool and type(set_index) is int and 0 <= set_index < (2 if multi_set else 1), "invalid explicit object set mapping") validate_topology(snapshot, "H1" if len(snapshot) == 1 else "H2", expected_sets=2 if multi_set else 1) @@ -141,8 +148,12 @@ def map_target(snapshot, bucket, key, role, part_number, version_id="null", *, p require(logical is None or logical == logical_entry, "baseline versions or part geometry lack one full-set authority") logical = logical_entry parts = [part for part in version["parts"] if part["number"] == part_number] - require(len(parts) == 1 and parts[0]["size"] == parts[0]["actualSize"] > 0, - "unsupported compressed/encrypted or ambiguous fixture part") + require(len(parts) == 1, "ambiguous fixture part") + if part_policy is None: + require(parts[0]["size"] == parts[0]["actualSize"] > 0, + "unsupported compressed/encrypted or ambiguous fixture part") + else: + part_policy(version, parts[0]) relative = f"{bucket}/{key}/{data_dir}/part.{part_number}" entry = indexed.get((node, disk, relative)) require(entry and entry["size"] > 32, "external part is missing from the healthy baseline") diff --git a/test_validation_heal_encrypted.py b/test_validation_heal_encrypted.py new file mode 100644 index 00000000..aa9390b3 --- /dev/null +++ b/test_validation_heal_encrypted.py @@ -0,0 +1,152 @@ +import base64 +import copy +import hashlib +import json +import os +from pathlib import Path +import sys +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import encrypted, encryption_runtime, heal_fault +from scripts.validation_heal.heal_probe import Probe, redact +from scripts.validation_heal.shards import map_target +from scripts.validation_heal.matrix import AUTOMATIC_CASES, select +import test_validation_heal as shard_fixtures +from test_validation_heal_authorization import binding + + +class EncryptedObjectTests(unittest.TestCase): + def geometry(self): + before = shard_fixtures.HealShardOracleTest().snapshot() + key, bucket = "objects/0008.bin", "rv-heal-0123456789abcdef" + expected = {"key": key, "versionId": "null", "isLatest": True, "deleteMarker": False, + "payload": {"size": encrypted.SIZE, "sha256": "plaintext"}} + for observation in before.values(): + for entry in observation["files"]: + if entry["path"] == f"{bucket}/{key}/xl.meta": + for version in entry["metadata"]["versions"]: + version["size"] = encrypted.SIZE + 16000 + version["parts"][0].update(size=encrypted.SIZE + 16000, actualSize=encrypted.SIZE) + return before, bucket, expected + + def test_ciphertext_geometry_keeps_plain_mapper_strict_and_does_not_rewrite_evidence(self): + before, bucket, expected = self.geometry() + saved = copy.deepcopy(before) + target = encrypted.map_cipher_target(before, bucket, expected, "data") + self.assertTrue(target["encryptedPhysicalLayer"]) + self.assertEqual(target["plaintextACK"], expected["payload"]) + self.assertNotEqual(target["ciphertextPartSize"], expected["payload"]["size"]) + self.assertEqual(before, saved) + with self.assertRaisesRegex(ValueError, "encrypted"): + map_target(before, bucket, expected["key"], "data", 1) + + def test_ciphertext_policy_rejects_plaintext_downgrade_wrong_ack_and_unsupported_transforms(self): + for kind in ("plain", "actual", "size", "multipart", "inline", "unbounded", "bool"): + before, bucket, expected = self.geometry() + meta = next(entry for entry in before["node1"]["files"] if entry["path"] == f"{bucket}/{expected['key']}/xl.meta") + version = meta["metadata"]["versions"][0] + if kind == "plain": version["parts"][0]["size"] = encrypted.SIZE + if kind == "actual": version["parts"][0]["actualSize"] -= 1 + if kind == "size": version["size"] -= 1 + if kind == "multipart": version["parts"].append(copy.deepcopy(version["parts"][0])) + if kind == "inline": version["inline"] = {"length": 1} + if kind == "unbounded": version["parts"][0]["size"] = encrypted.SIZE + 2 * 1024**2 + if kind == "bool": version["parts"][0]["actualSize"] = True + with self.subTest(kind=kind), self.assertRaises(ValueError): encrypted.map_cipher_target(before, bucket, expected, "data") + + def test_customer_key_is_resolved_only_for_sdk_and_both_key_forms_are_redacted(self): + key, wrong = "0123456789abcdef0123456789abcdef", "fedcba9876543210fedcba9876543210" + probe = Probe(binding()) + sdk = unittest.mock.Mock() + sdk.head_object.return_value = {"ResponseMetadata": {"HTTPStatusCode": 200}} + with patch.dict(os.environ, {"HEAL_SSE_C_KEY": key, "HEAL_SSE_C_WRONG_KEY": wrong}), patch.object(probe, "s3_client", return_value=sdk): + for reference, actual in (("$HEAL_SSE_C_KEY", key), ("$HEAL_SSE_C_WRONG_KEY", wrong)): + observed = probe.s3("head_object", {"Bucket": probe.binding["bucketPrefix"], "Key": "encrypted", + "SSECustomerAlgorithm": "AES256", "SSECustomerKey": reference}) + self.assertTrue(observed["acknowledged"]) + self.assertEqual(sdk.head_object.call_args.kwargs["SSECustomerKey"], actual) + self.assertEqual(observed["params"]["SSECustomerKey"], "[REDACTED]") + self.assertNotIn(actual, json.dumps(redact({"error": "rejected " + actual}))) + + def test_encryption_headers_reject_silent_plaintext_or_wrong_key_identity(self): + with patch.dict(os.environ, {"HEAL_SSE_C_KEY": "0123456789abcdef0123456789abcdef"}): + good = {"ServerSideEncryption": "aws:kms", "SSEKMSKeyId": "owned-key"} + encrypted.encryption_headers(good, "encrypted-versioned", "owned-key") + for changed in ({}, {**good, "SSEKMSKeyId": "other"}, {**good, "ServerSideEncryption": "AES256"}): + with self.assertRaises(ValueError): encrypted.encryption_headers(changed, "encrypted-versioned", "owned-key") + raw_key = os.environ["HEAL_SSE_C_KEY"].encode() + customer = {"SSECustomerAlgorithm": "AES256", "SSECustomerKeyMD5": base64.b64encode(hashlib.md5(raw_key, usedforsecurity=False).digest()).decode()} + encrypted.encryption_headers(customer, "sse-c", "unused") + with self.assertRaises(ValueError): encrypted.encryption_headers({**customer, "ServerSideEncryption": "AES256"}, "sse-c", "unused") + + def test_key_refusal_requires_reviewed_explicit_error_and_never_accepts_missing_object(self): + for customer, status, code in ((True, 400, "InvalidRequest"), (False, 403, "AccessDenied")): + good = {"acknowledged": False, "resultUnknown": False, + "response": {"ResponseMetadata": {"HTTPStatusCode": status}, "Error": {"Code": code}}} + self.assertTrue(encrypted.denied_key(good, customer=customer)["missingOrWrongKeyIsNotObjectAbsence"]) + for kind in ("transport", "payload", "absence", "success"): + bad = copy.deepcopy(good) + if kind == "transport": bad["resultUnknown"] = True + if kind == "payload": bad["payload"] = {"sha256": "decrypted"} + if kind == "absence": bad["response"] = {"ResponseMetadata": {"HTTPStatusCode": 404}, "Error": {"Code": "NoSuchKey"}} + if kind == "success": bad["acknowledged"] = True + with self.subTest(customer=customer, kind=kind), self.assertRaises(ValueError): encrypted.denied_key(bad, customer=customer) + + def test_versioned_fixture_preserves_real_distinct_acks_and_every_generation(self): + sequence = [0] + def invoke(operation, params, body=None): + if operation == "get_bucket_versioning": return {"Status": "Enabled"} + if operation == "put_object": + sequence[0] += 1 + return {"VersionId": f"real-{sequence[0]}", "ServerSideEncryption": "aws:kms", "SSEKMSKeyId": "owned-key"} + return {} + recorder = types.SimpleNamespace(call=invoke) + with patch.object(encrypted, "COUNT", 2): + expected = encrypted.seed_encrypted(recorder, "bucket", "encrypted-versioned", "owned-key") + self.assertEqual(len(expected), 6) + self.assertEqual(sum(item["isLatest"] for item in expected), 2) + self.assertEqual(len({item["versionId"] for item in expected}), 6) + self.assertEqual(len({item["payload"]["sha256"] for item in expected}), 6) + with patch.object(encrypted, "COUNT", 1), self.assertRaises(ValueError): + encrypted.seed_encrypted(types.SimpleNamespace(call=lambda *args: {"Status": "Enabled", "VersionId": "null", + "ServerSideEncryption": "aws:kms", "SSEKMSKeyId": "owned-key"}), "bucket", "encrypted-versioned", "owned-key") + + def test_inventory_decryption_options_cannot_override_identity_or_expose_raw_key(self): + probe = Probe.__new__(Probe) + probe.binding = {"bucketPrefix": "rv-heal-test-bucket"} + for options in ({"Bucket": "unowned"}, {"Key": "other"}, {"VersionId": "other"}, + {"SSECustomerAlgorithm": "AES256", "SSECustomerKey": "raw-secret"}, {"SSECustomerKey": "$HEAL_SSE_C_WRONG_KEY"}): + with self.subTest(options=options), self.assertRaises(ValueError): + probe.inventory("rv-heal-test-bucket", object_read_options=options) + + @unittest.skipUnless(sys.platform == "linux", "actual Linux FD/mount guard executes in CI") + def test_static_dependency_keeps_private_key_file_and_configured_process_identity(self): + with tempfile.TemporaryDirectory() as folder: + state = Path(folder) + original = b"RUSTFS_ACCESS_KEY=temporary\n" + (state / "service.env").write_bytes(original) + owner = {"runId": "gh-12345-1-h1"} + key_id = "rv-heal-key-" + owner["runId"] + key = base64.b64encode(bytes(range(32))).decode() + proof = encryption_runtime.configure(state, owner, {"keyId": key_id, "keyBase64": key}, heal_fault) + self.assertNotIn(key, json.dumps(proof)) + self.assertFalse(proof["sharedKmsTouched"]) + self.assertNotIn(key.encode(), (state / "service.env").read_bytes()) + settings = encryption_runtime.settings_for(state, key_id) + digest = lambda path: hashlib.sha256(path.read_bytes()).hexdigest() + checked = lambda path, boundary: path + self.assertTrue(encryption_runtime.attest(state, owner, checked, digest, settings)["loadedByCandidate"]) + with self.assertRaises(ValueError): encryption_runtime.attest(state, owner, checked, digest, {**settings, "RUSTFS_KMS_BACKEND": "local"}) + (state / "static-kms-secret").chmod(0o644) + with self.assertRaises(ValueError): encryption_runtime.attest(state, owner, checked, digest, settings) + with self.assertRaises(ValueError): encryption_runtime.configure(state, owner, {"keyId": key_id, "keyBase64": key}, heal_fault) + + def test_all_source_encryption_branches_have_an_automatic_entry(self): + self.assertEqual(select("D08")[0]["variants"], list(encrypted.BRANCHES)) + self.assertIn(encrypted.CASE_ID, AUTOMATIC_CASES) + + +if __name__ == "__main__": unittest.main() From e5ba96b32b5bddb29f00c37ad8b2e4411f6fd935 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 07:21:24 +0800 Subject: [PATCH 092/343] Migrate all three H2 coordinator crash mechanisms with durable responsibility boundaries --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 61 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/coordinator_crash.py | 434 ++++++++++++++++++ scripts/validation_heal/cursor.py | 15 +- scripts/validation_heal/fleet_node.py | 28 +- scripts/validation_heal/healthy.py | 5 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/mrf_records.py | 5 +- scripts/validation_heal/mrf_replay.py | 55 ++- scripts/validation_heal/replacement.py | 32 +- .../validation_heal/replacement_journal.py | 65 +++ scripts/validation_heal/root_restart.py | 33 +- scripts/validation_heal/run_case.sh | 1 + test_validation_heal_coordinator_crash.py | 264 +++++++++++ 15 files changed, 958 insertions(+), 48 deletions(-) create mode 100644 scripts/validation_heal/coordinator_crash.py create mode 100644 scripts/validation_heal/replacement_journal.py create mode 100644 test_validation_heal_coordinator_crash.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index ce238281..84bdd963 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -25,6 +25,7 @@ on: - test_validation_heal_scopes.py - test_validation_heal_bucket_metadata.py - test_validation_heal_write_quorum.py + - test_validation_heal_coordinator_crash.py - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py @@ -66,6 +67,7 @@ on: - test_validation_heal_scopes.py - test_validation_heal_bucket_metadata.py - test_validation_heal_write_quorum.py + - test_validation_heal_coordinator_crash.py - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py @@ -100,7 +102,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 5fd6eff2..7d060aca 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 125 currently dispatchable IDs include all 74 non-Heal cases and fifty-one Heal cases. Forty-five have live PASS evidence; the other 113 have no live PASS claim. +case evidence. The 126 currently dispatchable IDs include all 74 non-Heal cases and fifty-two Heal cases. Forty-six have live PASS evidence; the other 112 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 33 Heal cases remain +The other 32 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -1036,7 +1036,7 @@ four regression verdicts have their own archive and never increment a fleet or canonical-case PASS count. Existing tests assert captured lock events internally; the collector does not fabricate emitted barrier logs. Missing critical-point fixtures, Admin nolock semantics and required fleet overlap remain explicit gaps. -The current implementation has 274 focused Heal/catalog checks, including the +The current implementation has 285 focused Heal/catalog checks, including the Linux-only D11, C09 and D08 ownership/mount/recovery checks. Native compilation and execution still require the separate hosted job; local parser checks are not native PASS. @@ -1175,3 +1175,58 @@ SSE-S3 records that its service-managed key has no caller decryption-key argument. Failed IAM cleanup retains a DIRTY stopped namespace. Nine targeted geometry, authority, version, redaction and Linux file-ownership tests pass; D08 still requires fresh complete live evidence and has no live PASS claim. + + +A02 `38094094085` passed every applicable H1/H2 root, bucket, object, prefix, +recursive and actual two-set selector branch. H1 keeps the source's selector +inapplicability explicit; H2 tests the actual independent sets. Its complete +physical/streaming and all-peer ACK manifests, cleanup PASS and independent +canonical gate increase the complete live total to 46/158. F06 `38093199055` +retained FAIL/cleanup PASS on the already queued old harness; its durable-stop +query assertion is fixed at `62d0ca9` and remains queued for a fresh run. +B08 `38093907057` retained FAIL/cleanup PASS: H1 lacks the exact incarnation +receipt, while H2's actual default layout cannot preserve R with the selected +write/lock-loss plan. No layout or missing identity is assumed sufficient. + +F04 `heal.coordinator-process-crash` now dispatches its three separate source +mechanisms on actual H2. It records a candidate-bound persistence support table +and only sends SIGKILL through Linux pidfd after freezing the unique original +coordinator at nonzero and unfinished physical progress. Target and other peer +process birth identities must remain unchanged throughout recovery. No host or +platform restart is issued, and an unobserved window cannot pass. + +The root branch ACKs the full DX fixture, places missing original data shards +outside the coordinator, freezes the exact durable admission owner and requires +the original token to replay with complete responsibility. SIGKILL must leave +the original intent bytes unchanged; it makes no graceful elapsed-time or cursor +checkpoint claim. Repeated terminal restart, original record disk, full physical +hashes/streaming and all-peer S3 inventories remain mandatory. + +The MRF branch ACKs sixty-four independent never-versioned DB writes while one +original node is offline. Its isolated profile enables only MRF, with one worker, +one task per set and a replay batch of one. Actual process settings and the +configuration source digest are retained. Fresh accepted MRF task/start logs, +exact version/null representation and a partial target census bind the paused +coordinator to a still-committed original ACK responsibility. The complete +CRC/commit/lifecycle checkpoint remains byte-identical across the crash. +The lost volatile token must return its explicit unknown-token error; a separate +new automatic task must prove the interrupted ACK, while every ACK's original +responsibility retires with a verified receipt. Automatic rescan success is never +reported as original-token recovery. The final census, all surviving original +shards, streaming checks and S3 inventories gate the branch. + +The replacement branch independently ACKs 1025 external objects and preserves +an inaccessible original drive archive. Its coordinator is distinct from the +live target. An exact-name read-only journal captures the schema-seven sealed +intent for the original generation, endpoint, physical device and filesystem +identity before and after SIGKILL. The original generation must finish on every +peer after the same coordinator unit restarts, with the original replacement +UUID/set slot, full same-role shard hashes, streaming and ACK inventories. +No manual replacement compensation or backup rescue earns repair credit. + +Eleven focused F04 tests cover process continuity/exit, exact root intent, +replacement seals/name/identity and filesystem guards, partial MRF authority, +source attribution, bounded configuration and mandatory three-mechanism scoring. +F04 has no live PASS claim yet. The source-bound MRF log reader also accepts the +reviewed Rust Option debug spelling without merging different version identities; +existing E05 receipts and every original branch remain independently required. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 419bb7ea..bc2ae905 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/coordinator_crash.py b/scripts/validation_heal/coordinator_crash.py new file mode 100644 index 00000000..2335124c --- /dev/null +++ b/scripts/validation_heal/coordinator_crash.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 +"""F04: distinct H2 root, MRF and replacement coordinator SIGKILL boundaries.""" + +from __future__ import annotations + +import argparse +import copy +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +from urllib.parse import quote + +try: + from .heal_probe import Probe, canonical, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .cursor import run_topology as dx_topology + from . import root_restart, mrf_replay, replacement, replacement_journal + from .mrf_records import checkpoint, REFERENCE as MRF_REFERENCE + from .shards import files + from .scanner_repair import event_time + from .stop import validate_unknown_token + from .unformatted import binding as lab_binding + from .fleet_node import PROCESS_FIELDS +except ImportError: + from heal_probe import Probe, canonical, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from cursor import run_topology as dx_topology + import root_restart, mrf_replay, replacement, replacement_journal + from mrf_records import checkpoint, REFERENCE as MRF_REFERENCE + from shards import files + from scanner_repair import event_time + from stop import validate_unknown_token + from unformatted import binding as lab_binding + from fleet_node import PROCESS_FIELDS + + +CASE_ID = "heal.coordinator-process-crash" +BRANCHES = ("root-coordinator-kill", "mrf-coordinator-kill", "replacement-coordinator-kill") +MRF_COUNT = 64 +REPLACEMENT_COUNT = 1025 +WINDOW_SECONDS = 90 +REFERENCE = {"candidateCommit": root_restart.REFERENCE["candidateCommit"], + "root": root_restart.REFERENCE, "mrf": MRF_REFERENCE, "replacement": replacement_journal.REFERENCE, + "concurrencyConfigSource": {"crates/config/src/constants/heal.rs": + "82b8614dfa7cc111e694138035e4a84ee23035c719f0286fc0565d7181ca014f"}, + "supportTable": { + "root": "durable original admission and token; SIGKILL cannot checkpoint elapsed timeout or cursor", + "mrf": "durable exact ACK responsibility; task token is volatile, automatic rescan uses a new token", + "replacement": "durable original generation and target identities; target process remains alive"}, + "fault": "one exact run-owned paused coordinator process, Linux pidfd SIGKILL; same boot/roots/candidate"} + + +def processes(fleet, label): + return fleet.record(label, {node: fleet.remote(node, "process") for node in fleet.nodes}) + + +def continuity(before, after, owner, *, private_mounts=False): + require(len(before) == 4 and set(before) == set(after) and owner in before, + "coordinator crash omitted an actual H2 peer") + for node, old in before.items(): + new = after[node] + ignored = {"pid", "startTicks"} | ({"mountNamespace"} if private_mounts else set()) if node == owner else set() + require(all(new.get(name) == old.get(name) for name in PROCESS_FIELDS if name not in ignored), + "coordinator restart changed a target/peer process, boot, candidate, unit or environment") + require(type(new.get("pid")) is int and new["pid"] > 1 and type(new.get("startTicks")) is int, + "coordinator recovery omitted an actual running process") + if node == owner: + require((new["pid"], new["startTicks"]) != (old["pid"], old["startTicks"]) + and isinstance(new.get("mountNamespace"), str) and new["mountNamespace"], + "coordinator did not actually restart in a witnessed namespace") + return {"onlyCoordinatorRestarted": True, "everyTargetAndPeerKeptOriginalProcess": True, + "sameBootUnitCandidateSettingsAndRoots": True, "privateMountNamespaceMayRestart": private_mounts} + + +def freeze(fleet, owner, before): + identity = before[owner] + fleet.frozen_processes[owner] = identity + fleet.record("coordinator-SIGSTOP", fleet.remote(owner, "process-signal", signal="SIGSTOP", expected=identity)) + current = fleet.record("coordinator-frozen-process", fleet.remote(owner, "process")) + require(current.get("state") == "T" and all(current.get(name) == identity.get(name) for name in PROCESS_FIELDS), + "coordinator freeze missed its original process birth") + return identity + + +def kill(fleet, owner, identity, peers): + proof = fleet.record("coordinator-actual-SIGKILL", fleet.remote(owner, "process-crash", expected=identity)) + require(proof.get("actualSignal") == 9 and proof.get("signal") == "SIGKILL" and proof.get("graceful") is False + and proof.get("pidfdBound") is True and proof.get("platformEventIssued") is False, + "coordinator interruption lacks its exact non-graceful process-bound exit") + del fleet.frozen_processes[owner] + for node in fleet.nodes: + if node == owner: continue + actual = fleet.record("peer-kept-running-during-crash", fleet.remote(node, "process")) + require(all(actual.get(name) == peers[node].get(name) for name in PROCESS_FIELDS), + "target or non-coordinator peer changed process during SIGKILL") + fleet.record("exited-coordinator-unit-released", fleet.remote(owner, "stop")) + return proof + + +def replacement_checkpoint(observed, generation): + token, endpoint = generation["taskId"], generation["targetSlots"][0] + require(observed.get("taskId") == token and observed.get("readOnly") is True + and observed.get("stable") is True and len(observed.get("records", [])) == 12, + "replacement checkpoint lost its exact generation or whole physical-root census") + seen, intents, seals = set(), {}, set() + for row in observed["records"]: + pair = row.get("disk"), row.get("kind") + require(type(pair[0]) is int and 0 <= pair[0] < 4 and pair[1] in replacement_journal.FILES + and pair not in seen and type(row.get("present")) is bool, + "replacement checkpoint duplicated or invented a disk/name identity") + seen.add(pair) + suffix = replacement_journal.FILES[pair[1]][0] + require(row.get("relativePath") == f".rustfs.sys/buckets/ahm-replacement/{token}_{suffix}", + "replacement checkpoint escaped the exact reviewed generation name") + if not row["present"]: continue + require(type(row.get("size")) is int and row["size"] > 0 + and bool(re.fullmatch(r"[0-9a-f]{64}", row.get("sha256", ""))), "replacement checkpoint lacks whole-file identity") + if pair[1] == "seal": + require(row.get("sealed") is True and row["size"] == 6 + and row["sha256"] == hashlib.sha256(b"sealed").hexdigest(), "replacement intent is not committed") + seals.add(pair[0]) + elif pair[1] == "proof": + raise ValueError("NOT_OBSERVED: replacement already has completion proof before its required crash") + else: + value = row.get("value", {}) + require(type(value.get("schema_version")) is int and value["schema_version"] == 7 + and value.get("task_id") == token and value.get("replacement_generation") == token + and value.get("set_disk_id") == generation["setDiskId"] + and value.get("replacement_targets") == [endpoint] and value.get("completed") is False, + "replacement intent lost its original generation/set/target or is already complete") + identities = value.get("replacement_target_identities") + require(isinstance(identities, list) and len(identities) == 1 and identities[0].get("endpoint") == endpoint + and isinstance(identities[0].get("canonical_path"), str) and identities[0]["canonical_path"].startswith("/") + and isinstance(identities[0].get("physical_device_ids"), list) and identities[0]["physical_device_ids"] + and all(isinstance(item, str) and item for item in identities[0]["physical_device_ids"]) + and isinstance(identities[0].get("filesystem_identity"), str) and identities[0]["filesystem_identity"], + "replacement checkpoint omitted the actual target path/device/filesystem identity") + intents[pair[0]] = row + require(intents and set(intents) <= seals, "NOT_OBSERVED: coordinator has no sealed durable replacement intent") + return {"generation": token, "sealedRecordDisks": sorted(intents), "originalTargetBound": True, + "physicalPartialWindowRequiredSeparately": True} + + +def interrupt_replacement(fleet, owner, proofs, original_format, generation): + target = proofs[0]["node"] + require(owner != target and len(fleet.nodes) == 4, "F04 requires a coordinator distinct from the live replacement target") + before = processes(fleet, "replacement-coordinator-and-target-processes") + identity = freeze(fleet, owner, before) + scene = fleet.snapshot("replacement-coordinator-frozen-scene") + progress = replacement.target_progress(scene, proofs) + require(progress["partial"], "NOT_OBSERVED: replacement was not physically partial at coordinator freeze") + require(all(scene[node]["formats"] == original_format[node]["formats"] for node in fleet.nodes), + "replacement changed its target UUID/set before the coordinator crash") + original = fleet.record("replacement-coordinator-durable-checkpoint", fleet.remote(owner, + "replacement-records", taskId=generation["taskId"])) + checkpoint_proof = replacement_checkpoint(original, generation) + exit_proof = kill(fleet, owner, identity, before) + after = fleet.record("replacement-coordinator-records-after-SIGKILL", fleet.remote(owner, + "replacement-records", taskId=generation["taskId"], quiescent=True)) + replacement_checkpoint(after, generation) + require(canonical(original) == canonical(after), "SIGKILL lost or rewrote the coordinator's committed replacement intent") + stopped_scene = fleet.record("replacement-target-still-partial-after-coordinator-exit", { + node: fleet.remote(node, "snapshot", quiescent=node == owner) for node in fleet.nodes}) + after_progress = replacement.target_progress(stopped_scene, proofs) + require(after_progress["partial"] and set(progress["completeDataVersions"]) <= set(after_progress["completeDataVersions"]), + "coordinator exit lost restored parts or missed the unfinished replacement window") + require(all(stopped_scene[node]["formats"] == original_format[node]["formats"] for node in fleet.nodes), + "coordinator crash changed a replacement disk identity") + fleet.record("same-replacement-coordinator-restarted", fleet.remote(owner, "start")) + fleet.wait_ready() + restarted = processes(fleet, "replacement-processes-after-coordinator-restart") + continuity_proof = continuity(before, restarted, owner, private_mounts=True) + archive = fleet.record("original-target-archive-still-refused", fleet.remote(target, "replacement-probe")) + require(archive.get("serviceReadDenied") is True and archive.get("sameOriginalServiceProcess") is True, + "live target gained backup access during the coordinator restart") + return {"branch": "replacement-coordinator-kill", "coordinator": owner, "targetKeptRunning": target, + "originalGeneration": generation, "before": progress, "afterActualExit": after_progress, + "checkpoint": checkpoint_proof, "actualExit": exit_proof, "processContinuity": continuity_proof, + "sameCommittedGenerationRequiredForCompletion": True, "adminCompensationTasks": 0} + + +def mrf_progress(snapshot, authorities, bucket, obligations, target): + """Observe partial physical ACK replay; final cohort streaming is a separate gate.""" + validate_topology(snapshot, "H2") + current, original, complete = files(snapshot), files(authorities), [] + for item in obligations: + wanted, metadata_path = item["versionId"], f"{bucket}/{item['key']}/xl.meta" + candidates = [] + for (node, disk, path), entry in original.items(): + if node == target or path != metadata_path: continue + versions = [value for value in entry.get("metadata", {}).get("versions", []) + if (value.get("versionId") or "null") == wanted] + require(len(versions) == 1, "MRF physical authority omitted its exact ACK version") + value = versions[0] + logical = {name: value.get(name) for name in + ("versionId", "isLatest", "deleted", "dataDir", "size", "parts", "usesLegacyChecksum")} + logical["erasure"] = {name: value.get("erasure", {}).get(name) for name in + ("data_blocks", "parity_blocks", "block_size", "distribution", "algorithm")} + candidates.append(logical) + require(len(candidates) == 12 and all(value == candidates[0] for value in candidates) + and candidates[0]["deleted"] is False and candidates[0]["isLatest"] is True, + "MRF progress has no single original twelve-member ACK authority") + authority = candidates[0] + restored = 0 + for layout in snapshot[target]["formats"]: + disk = layout["disk"] + metadata = current.get((target, disk, metadata_path), {}).get("metadata", {}) + matches = [value for value in metadata.get("versions", []) if (value.get("versionId") or "null") == wanted] + if not matches: continue + require(len(matches) == 1 and metadata.get("bucket") == bucket and metadata.get("key") == item["key"], + "MRF replay confused physical key/version identities") + version, erasure = matches[0], matches[0].get("erasure", {}) + logical = {name: version.get(name) for name in authority if name != "erasure"} + logical["erasure"] = {name: erasure.get(name) for name in authority["erasure"]} + geometry = layout["value"]["xl"] + slot = geometry["sets"][0].index(geometry["this"]) + require(logical == authority and type(erasure.get("index")) is int + and erasure["index"] == erasure["distribution"][slot] + and version.get("role") == ("data" if erasure["index"] <= erasure["data_blocks"] else "parity"), + "MRF replay changed logical authority or its actual target set role") + parts = version.get("parts", []) + require(version.get("inline") is None and len(parts) == 1 and parts[0].get("number") == 1 + and parts[0].get("size") == parts[0].get("actualSize") > 1024**2, + "MRF progress requires its reviewed external plaintext part") + path = f"{bucket}/{item['key']}/{version['dataDir']}/part.1" + entry = current.get((target, disk, path)) + if entry: + require(type(entry.get("size")) is int and entry["size"] > 32 + and bool(re.fullmatch(r"[0-9a-f]{64}", entry.get("sha256", ""))), + "MRF progress lacks complete part-file identity") + restored += 1 + if restored == 4: complete.append(list(mrf_replay.identity(item))) + return {"completeACKs": complete, "pendingACKs": len(obligations) - len(complete), + "partial": 0 < len(complete) < len(obligations), "finalStreamingProofClaimed": False} + + +def active_mrf_task(events, obligations, bindings, after, target): + candidates = [] + for item in obligations: + for owner in bindings[mrf_replay.identity(item)]: + node = owner["node"] + if node == target: continue + rows = events.get(node, []) + for start in rows: + token, stamp = start.get("task_id"), event_time(start) + if not (start.get("target") == "rustfs::heal::task" and start.get("stage") == "start" + and start.get("message") == "Heal object started" and start.get("bucket") == item["bucket"] + and start.get("object") == item["key"] and "version_id" in start + and mrf_replay.source_version(start["version_id"], item["versionId"]) + and isinstance(token, str) and token and stamp is not None and stamp >= after): continue + accepted = any(row.get("target") == "rustfs::heal::manager" and row.get("source") == "mrf" + and row.get("message") == "Heal queue request accepted" and row.get("result") == "accepted" + and row.get("request_id") == token and event_time(row) is not None + and after <= event_time(row) <= stamp for row in rows) + terminal = any(row.get("target") == "rustfs::heal::manager" and row.get("task_id") == token + and row.get("state") == "task_completed" and event_time(row) is not None + and event_time(row) >= stamp for row in rows) + if accepted and not terminal: candidates.append((node, token, item)) + require(len(candidates) == len({(node, token) for node, token, _ in candidates}), + "MRF source duplicated an active task identity") + return candidates + + +def interrupt_mrf(probe, fleet, window, binding, bucket, expected, obligations, bindings, rejoin_time, + before, authorities, target): + fleet.record("MRF-crash-support-table", REFERENCE) + fleet.record("MRF-pre-registered-crash-window", {"objects": MRF_COUNT, "windowSeconds": WINDOW_SECONDS, + "maxConcurrentHeals": 1, "maxConcurrentPerSet": 1, "replayBatch": 1, + "actualProcessConfiguration": {node: value["backgroundConfig"] for node, value in before.items()}, + "partialPhysicalProgressRequired": True, "fixedSleepUsedAsWindow": False}) + deadline = time.monotonic() + WINDOW_SECONDS + chosen = None + while chosen is None: + events = window.capture() + scene = fleet.snapshot("MRF-coordinator-running-window") + progress = mrf_progress(scene, authorities, bucket, obligations, target) + for owner, token, obligation in active_mrf_task(events, obligations, bindings, rejoin_time, target): + if not progress["partial"] or list(mrf_replay.identity(obligation)) in progress["completeACKs"]: continue + endpoint = "node" + str(fleet.nodes.index(owner) + 1) + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + quote(obligation["key"], safe="/") + observed = fleet.record("MRF-original-volatile-task-running", Probe(binding, endpoint).admin(path, [("clientToken", token)], b"")) + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), + "MRF active source task lost its exact status before freeze") + if observed["response"].get("summary") == "running": + chosen = owner, token, obligation, endpoint, path + break + require(chosen is not None or time.monotonic() < deadline, "NOT_OBSERVED: no active MRF coordinator with partial physical ACK replay") + if chosen is None: time.sleep(.1) + owner, token, obligation, endpoint, path = chosen + peers = processes(fleet, "MRF-processes-before-coordinator-freeze") + process_identity = freeze(fleet, owner, peers) + frozen = fleet.snapshot("MRF-frozen-coordinator-partial-scene") + progress = mrf_progress(frozen, authorities, bucket, obligations, target) + require(progress["partial"] and list(mrf_replay.identity(obligation)) not in progress["completeACKs"], + "NOT_OBSERVED: original MRF task finished before its physical crash boundary") + require(any(node == owner and active == token for node, active, _ in + active_mrf_task(window.capture(), obligations, bindings, rejoin_time, target)), + "NOT_OBSERVED: MRF task became terminal before the coordinator freeze") + raw = fleet.record("MRF-frozen-committed-records", fleet.remote(owner, "mrf-records")) + committed = checkpoint(raw) + # The original binding already checked the actual enqueue time. Require + # that exact intent digest below, rather than re-dating it at rejoin. + bound = mrf_replay.pending_bindings({owner: committed}, [obligation], 0) + original_owner = next(item for item in bindings[mrf_replay.identity(obligation)] if item["node"] == owner) + recovered_owner = bound[mrf_replay.identity(obligation)][0] + require(all(recovered_owner[name] == original_owner[name] for name in + ("node", "checkpointOwner", "responsibilityId", "bucketIncarnationId")) + and recovered_owner["sequence"] >= original_owner["sequence"], "MRF pending crash checkpoint changed its original ACK responsibility") + exit_proof = kill(fleet, owner, process_identity, peers) + after = fleet.record("MRF-committed-records-after-SIGKILL", fleet.remote(owner, "mrf-records", quiescent=True)) + require(canonical(raw) == canonical(after), "SIGKILL changed the frozen committed MRF record bytes") + fleet.record("same-MRF-coordinator-restarted", fleet.remote(owner, "start")) + fleet.wait_ready() + restarted = processes(fleet, "MRF-processes-after-coordinator-restart") + continuity_proof = continuity(peers, restarted, owner) + unknown = fleet.record("lost-volatile-MRF-token-refused", Probe(binding, endpoint).admin(path, [("clientToken", token)], b"")) + validate_unknown_token(unknown, token) + return {"branch": "mrf-coordinator-kill", "coordinator": owner, "targetKeptRunning": target, + "volatileOriginalTaskId": token, "ackIdentity": list(mrf_replay.identity(obligation)), + "originalResponsibilityId": original_owner["responsibilityId"], "partialPhysicalWindow": progress, + "originalIntentDigest": original_owner["intentDigest"], "frozenIntentDigest": recovered_owner["intentDigest"], + "actualExit": exit_proof, "processContinuity": continuity_proof, + "oldTokenCompletedClaim": False, "newAutomaticTaskRequired": True, "manualCompensationTasks": 0} + + +def root_observer(*args, **kwargs): + return root_restart.observe_task(*args, **kwargs, interruption="sigkill") + + +def run_replacement(args, nodes): + directory = args.evidence / "replacement-coordinator-kill" + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-h2" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="replacement", resource_profile="full-restart") + fleet.node_runtime_profiles = {node: "replacement-target" for node in nodes[1:]} + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == REFERENCE["candidateCommit"], "unreviewed coordinator durability boundary") + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + fleet.record("registered-coordinator-support-table", REFERENCE) + fleet.record("registered-replacement-crash-budget", {"objects": REPLACEMENT_COUNT, + "objectBytes": mrf_replay.SIZE, "rebuildSeconds": replacement.REBUILD_SECONDS, + "requiresNonzeroAndUnfinishedPhysicalProgress": True}) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = lab_binding(args, candidate, nodes, run_id, bucket) + binding.update(maxInventoryObjects=REPLACEMENT_COUNT, maxInventoryBytes=2 * 1024**3, + leaseExpiresAt=(datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat()) + fleet.record("binding", binding) + probe, active = Probe(binding), [] + result = {"topology": "H2", "branch": "replacement-coordinator-kill", "verdict": "RUNNING", "cleanup": "PENDING"} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("actual-topology", validate_topology(fleet.snapshot("replacement-H2-topology"), "H2")) + recorder = Recorder(probe, directory / "fixture") + expected = seed(recorder, bucket, "DB", REPLACEMENT_COUNT, [mrf_replay.SIZE]) + baseline = fleet.record("replacement-full-S3-baseline", probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("replacement-independent-ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + fleet.stop() + proof = replacement.run_branch(recorder, fleet, bucket, "replacement-coordinator-kill", expected, baseline, + active, [], coordinator_interruptor=interrupt_replacement) + require(proof["responsibility"]["coordinatorInterruption"]["targetKeptRunning"] == nodes[-1], + "replacement coordinator branch did not keep its actual target running") + result.update(verdict="PASS", proof=proof) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + if fleet.reserved: + try: fleet.stop(); fleet.snapshot("replacement-coordinator-failure", quiescent=True) + except Exception as capture: result["failureEvidenceError"] = str(capture) + finally: + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("branch-result", result) + return result + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f"Heal workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ("candidate", "rc-bin", "evidence", "tool-directory"): parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(set(nodes)) == len(nodes) and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), + "F04 requires the actual four current Heal nodes") + require(bool(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha)) + and bool(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", ""))), "verified workflow candidate required") + require(json.loads(args.candidate.read_text())["source_sha"] == REFERENCE["candidateCommit"], "unreviewed coordinator persistence support") + recipe = select(CASE_ID)[0] + require(recipe["variants"] == list(BRANCHES) and recipe["contract"]["probes"]["topology"] == "H2", + "F04 omitted its source branches or topology") + results = [] + try: + root_args = copy.copy(args) + root_args.evidence = args.evidence / "root-coordinator-kill" + root_args.evidence.mkdir() + root = dx_topology(root_args, "H2", nodes[:4], observer=root_observer, branches=(BRANCHES[0],), + source_branches=BRANCHES, runtime_profile="root-restart", policy_reference=REFERENCE, target_nodes=nodes[1:4]) + results.append({**root, "branch": BRANCHES[0]}) + if root["cleanup"] == "PASS": + mrf_args = copy.copy(args) + mrf_args.evidence = args.evidence / "mrf-coordinator-kill" + mrf_args.evidence.mkdir() + mrf = mrf_replay.run_branch(mrf_args, "offline-put", nodes[:4], after_rejoin=interrupt_mrf, + put_count=MRF_COUNT, fixture="DB") + results.append({**mrf, "branch": BRANCHES[1], "mechanismScenario": mrf["branch"]}) + if results[-1]["cleanup"] == "PASS": results.append(run_replacement(args, nodes[:4])) + finally: + complete = len(results) == 3 and all(item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, + "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", "complete": complete}) + require(complete, "F04 requires all three H2 coordinator mechanisms, real partial SIGKILL windows and complete cleanup") + + +if __name__ == "__main__": + try: main() + except Exception as exc: + print(f"Heal coordinator crash validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/cursor.py b/scripts/validation_heal/cursor.py index 658b4a41..79162bce 100644 --- a/scripts/validation_heal/cursor.py +++ b/scripts/validation_heal/cursor.py @@ -298,7 +298,8 @@ def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, or def run_topology(args, topology, nodes, *, observer=observe_task, branches=BRANCHES, count=OBJECTS, fixture_builder=None, missing_indices=None, - source_branches=None, runtime_profile="manual", policy_reference=LIMIT_REFERENCE): + source_branches=None, runtime_profile="manual", policy_reference=LIMIT_REFERENCE, + target_nodes=None): directory = args.evidence / topology directory.mkdir() run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" @@ -343,6 +344,18 @@ def run_topology(args, topology, nodes, *, observer=observe_task, branches=BRANC cohorts = map_cohorts(original, bucket, expected, count=count) fleet.record("all-external-baseline", verify_all_parts(fleet, cohorts, "baseline-part-batch", count=count)) targets = [cohorts[index] for index in missing_indices] + if target_nodes is not None: + require(target_nodes and len(set(target_nodes)) == len(target_nodes) and set(target_nodes) <= set(nodes), + "DX fault placement escaped its actual bound fleet") + selected = [] + for target in targets: + shard = next((entry for entry in target["cohort"] if entry["node"] in target_nodes + and entry["role"] == target["role"]), None) + require(shard is not None, "DX fault has no requested role outside the coordinator") + selected.append({**target, **shard}) + targets = selected + fleet.record("explicit-fault-placement", {"nodes": target_nodes, + "targets": [{name: target[name] for name in ("node", "disk", "key", "role", "index")} for target in targets]}) receipts = [] for index, target in enumerate(targets): params = {"disk": target["disk"], "relativePath": target["relativePath"], "kind": "missing", diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 08e46853..a093bb79 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -33,11 +33,13 @@ def runtime_config(profile): - require(profile in {"manual", "offline-replay", "durable-mrf", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(profile in {"manual", "offline-replay", "durable-mrf", "mrf-crash", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") if profile == "scope-isolation": return {**BACKGROUND_CONFIG, "RUSTFS_ERASURE_SET_DRIVE_COUNT": "8"} - if profile == "durable-mrf": - return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} + if profile in {"durable-mrf", "mrf-crash"}: + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug", + **({"RUSTFS_HEAL_MAX_CONCURRENT_HEALS": "1", "RUSTFS_HEAL_MAX_CONCURRENT_PER_SET": "1", + "RUSTFS_HEAL_MRF_REPLAY_BATCH": "1"} if profile == "mrf-crash" else {})} if profile in {"replacement", "replacement-target", "schema-refusal"}: return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_AUTO_HEAL_ENABLE": "true" if profile == "replacement" else "false", "RUSTFS_HEAL_INTERVAL_SECS": "10", "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} @@ -231,7 +233,7 @@ def start(request): "--property=KillMode=control-group", "--property=TimeoutStopSec=30", "--property=RuntimeMaxSec=" + str(resource_limits(owner.get("resourceProfile", "standard"))["runtimeMaxSeconds"]), "--property=MemoryMax=4G", - *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") in {"root-restart", "replacement", "replacement-target", "schema-refusal"} else []), + *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") in {"root-restart", "durable-mrf", "mrf-crash", "replacement", "replacement-target", "schema-refusal"} else []), "--property=WorkingDirectory=" + str(state), "--property=EnvironmentFile=" + str(state / "service.env"), *gate_module().start_properties(gate_module().checked_gate(state, roots)), @@ -379,15 +381,15 @@ def process_graceful(request): def process_crash(request): - require(owned(request["runId"])[2].get("runtimeProfile") in {"replacement", "replacement-target"} - and process(request).get("state") == "T", "crash observation requires the paused owned replacement service") + require(owned(request["runId"])[2].get("runtimeProfile") in {"replacement", "replacement-target", "root-restart", "durable-mrf", "mrf-crash"} + and process(request).get("state") == "T", "crash observation requires a paused owned recovery service") before = pidfd_signal(request, signal.SIGKILL) deadline = time.monotonic() + 30 while True: after = process(request) if after["pid"] == 0: break require(all(after.get(name) == before[name] for name in PROCESS_FIELDS), "another process replaced the crash target") - require(time.monotonic() < deadline, "owned replacement process did not exit after SIGKILL") + require(time.monotonic() < deadline, "owned recovery process did not exit after SIGKILL") time.sleep(0.1) service = after["service"] require(after["bootId"] == before["bootId"] and after["binarySha256"] == before["binarySha256"] @@ -918,6 +920,17 @@ def mrf_records(request): return module.read_records(state, roots, owner, checked) +def replacement_records(request): + require(set(request) <= {"runId", "action", "taskId", "quiescent"}, + "replacement journal accepts no paths or mutation options") + state, roots, owner = owned(request["runId"]) + require(owner.get("runtimeProfile") in {"replacement", "replacement-target"} + and digest(state / "rustfs") == owner["binarySha256"], "replacement checkpoint candidate/runtime changed") + if request.get("quiescent") is True: stopped(owner) + else: require(process(request)["pid"] > 1, "replacement checkpoint has no live owned service") + return globals()["replacement_journal"].read_records(roots, request.get("taskId"), checked) + + def main(request): require(os.geteuid() == 0, "node adapter requires passwordless sudo") actions = {"preflight": preflight, "reserve": reserve, "start": start, "stop": stop, "snapshot": snapshot, @@ -934,6 +947,7 @@ def main(request): actions[action] = quorum_gate_operation actions["format-schema-fault"] = format_schema_fault actions["encryption-configure"] = configure_encryption + actions["replacement-records"] = replacement_records actions.update({"replace-drive": replace_drive, "replacement-state": replacement_state, "replacement-probe": replacement_probe}) actions.update({"format-export": format_export, "foreign-format": foreign_format, "restore-replacement": restore_replacement}) actions.update({"process": process, "process-metrics": process_metrics, "process-signal": process_signal, "process-graceful": process_graceful, "process-crash": process_crash}) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 5bd05e0b..84658079 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"manual", "offline-replay", "durable-mrf", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "offline-replay", "durable-mrf", "mrf-crash", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -183,6 +183,9 @@ def remote(self, node, action, **params): if action == "mrf-records": source += "\nimport types\nmrf = types.ModuleType('mrf_records')\nexec(" + repr( NODE_SOURCE.with_name("mrf_records.py").read_text()) + ", mrf.__dict__)\n" + if action == "replacement-records": + source += "\nimport types\nreplacement_journal = types.ModuleType('replacement_journal')\nexec(" + repr( + NODE_SOURCE.with_name("replacement_journal.py").read_text()) + ", replacement_journal.__dict__)\n" if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline", "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure"}: source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 067687d4..b9ccca09 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/mrf_records.py b/scripts/validation_heal/mrf_records.py index 30e39e64..e89a9308 100644 --- a/scripts/validation_heal/mrf_records.py +++ b/scripts/validation_heal/mrf_records.py @@ -20,7 +20,8 @@ "crates/heal/src/heal/mrf_queue/snapshot.rs": "ecfcb5ee63edef5c5d3d6713edc127e16f489522afc5fc02dc1044ccca241d14", "crates/heal/src/heal/mrf_queue/partial_write.rs": "20415e0e985daea8fc62c5e152ed438e75008c33a2a592275e1dd9d1d4fee771", "crates/common/src/mrf_channel.rs": "075bb84bcbbe7d76674f730c64c8b69fe37d0790ec75da46365ba2e0cab34e0f", - "crates/heal/src/heal/manager/scheduler.rs": "79ce1f9c278c664231b6a1eca7c521f8e98742fe6d17d8f852368da4e77cc33b"}, + "crates/heal/src/heal/manager/scheduler.rs": "79ce1f9c278c664231b6a1eca7c521f8e98742fe6d17d8f852368da4e77cc33b", + "crates/heal/src/heal/task/heal_object.rs": "17bb72783b6fd58afdc0b81c24558c1f5aff867fa3fb6ee27bd7f264076d8f6e"}, "journal": "complete CRC32/ISO-HDLC records; format 1, layouts 1/2/3", "commit": "RFMRFC01 version 1; owner, sequence, payload length and two SHA-256 digests", "lifecycle": "RFMFLC01 version 1; checksummed JSON, same committed owner and sequence", @@ -48,7 +49,7 @@ def verify_reference(candidate): def read_records(state, roots, owner, checked): """Exact allowlist, no directory enumeration, no write descriptor.""" - require(owner.get("runtimeProfile") == "durable-mrf", "MRF records require the dedicated run-owned runtime") + require(owner.get("runtimeProfile") in {"durable-mrf", "mrf-crash"}, "MRF records require the dedicated run-owned runtime") require(hashlib.sha256((state / "rustfs").read_bytes()).hexdigest() == owner["binarySha256"], "MRF candidate changed") def collect(): diff --git a/scripts/validation_heal/mrf_replay.py b/scripts/validation_heal/mrf_replay.py index c85f3b6a..e0d1f86d 100644 --- a/scripts/validation_heal/mrf_replay.py +++ b/scripts/validation_heal/mrf_replay.py @@ -29,7 +29,7 @@ from .node_outage import quorum_plan, put_ack, delete_ack, replica_census, validate_deleted_read from .scanner_repair import JournalWindow, event_time from .shards import files, map_target, verify_cohort - from .versions import verify_quorum_contract + from .versions import public_version_id, verify_quorum_contract except ImportError: from heal_probe import Probe, compare_inventories, payload, require, save from heal_workload import Recorder, seed, verify_seed @@ -40,7 +40,7 @@ from node_outage import quorum_plan, put_ack, delete_ack, replica_census, validate_deleted_read from scanner_repair import JournalWindow, event_time from shards import files, map_target, verify_cohort - from versions import verify_quorum_contract + from versions import public_version_id, verify_quorum_contract CASE_ID = "heal.mrf-offline-write-replay" @@ -54,16 +54,20 @@ def identity(item): return item["bucket"], item["key"], item["versionId"], item["poolIndex"], item["setIndex"], item["kind"] -def acknowledge(recorder, bucket, branch, expected): +def acknowledge(recorder, bucket, branch, expected, *, put_count=3, fixture="DV"): + require(type(put_count) is int and 3 <= put_count <= 64 + and (branch == "offline-put" or put_count == 3), "unreviewed MRF workload size") + require(fixture == "DV" or fixture == "DB" and branch == "offline-put", "unsupported MRF ACK fixture") manifest, obligations, deleted = copy.deepcopy(expected), [], [] - for index in range(3): + count = put_count if branch == "offline-put" else 3 + for index in range(count): if branch == "offline-put": key = f"offline/new-{index:02d}.bin" body = payload(f"{bucket}/{key}/offline", SIZE + index) response = recorder.call("put_object", {"Bucket": bucket, "Key": key, "Metadata": {"heal-fixture": "durable-MRF", "branch": branch}, "Tagging": "suite=heal&fixture=DV"}, body) - manifest = put_ack(manifest, key, body, response, "DV") - version = response["VersionId"] + manifest = put_ack(manifest, key, body, response, fixture) + version = public_version_id(response.get("VersionId")) absent = False elif branch == "offline-version-delete": entry = next(entry for entry in expected if entry["key"] == f"objects/{index + 1:04d}.bin" @@ -82,7 +86,7 @@ def acknowledge(recorder, bucket, branch, expected): require(response.get("ResponseMetadata", {}).get("HTTPStatusCode") in {200, 204}, "MRF workload lacks an explicit commit ACK") obligations.append({"bucket": bucket, "key": key, "versionId": version, "poolIndex": 0, "setIndex": 0, "kind": 3, "authoritativelyAbsentRequired": absent}) - require(len({identity(item) for item in obligations}) == 3, "MRF workload reused an ACK identity") + require(len({identity(item) for item in obligations}) == count, "MRF workload reused an ACK identity") return manifest, obligations, deleted @@ -129,6 +133,13 @@ def retirement(checkpoints, obligations, bindings): "operatorAcceptanceUsed": False, "responsibilityCount": len(obligations)} +def source_version(value, wanted): + # heal_object.rs logs Option<&str> with Rust debug syntax. Preserve the + # exact version; only the reviewed null representations denote no version. + if wanted == "null": return value in {None, "", "None", "null", "00000000-0000-0000-0000-000000000000"} + return isinstance(value, str) and value in {wanted, f'Some("{wanted}")'} + + def source_tasks(events, obligation, owners, after): tasks = [] for node in {owner["node"] for owner in owners}: @@ -137,7 +148,8 @@ def source_tasks(events, obligation, owners, after): stamp, token = event_time(start), start.get("task_id") if not (start.get("target") == "rustfs::heal::task" and start.get("message") == "Heal object started" and start.get("stage") == "start" and start.get("bucket") == obligation["bucket"] - and start.get("object") == obligation["key"] and start.get("version_id") == obligation["versionId"] + and start.get("object") == obligation["key"] and "version_id" in start + and source_version(start["version_id"], obligation["versionId"]) and isinstance(token, str) and token and stamp is not None and stamp >= after): continue admission = next((event for event in rows if event.get("target") == "rustfs::heal::manager" @@ -163,7 +175,7 @@ def verified_receipt(status, obligation, owners): item = receipt.get("identity", {}) incarnation = {owner["bucketIncarnationId"] for owner in owners} require(item.get("kind") == "object" and item.get("bucket") == obligation["bucket"] - and item.get("object") == obligation["key"] and item.get("versionId") == obligation["versionId"] + and item.get("object") == obligation["key"] and public_version_id(item.get("versionId")) == obligation["versionId"] and type(item.get("poolIndex")) is int and item["poolIndex"] == obligation["poolIndex"] and type(item.get("setIndex")) is int and item["setIndex"] == obligation["setIndex"] and item.get("bucketIncarnationId") in incarnation, "MRF proof belongs to another ACK, set or bucket generation") @@ -230,11 +242,12 @@ def wait_replay(fleet, window, binding, nodes, bucket, expected, obligations, bi time.sleep(2) -def run_branch(args, branch, nodes): +def run_branch(args, branch, nodes, *, after_rejoin=None, put_count=3, fixture="DV"): directory = args.evidence / branch directory.mkdir() run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-h2" - fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="durable-mrf") + fleet = Fleet(nodes, directory, run_id, args.tool_directory, + runtime_profile="durable-mrf" if after_rejoin is None else "mrf-crash") candidate = json.loads(args.candidate.read_text()) tool = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) contract = verify_quorum_contract(tool, candidate) @@ -255,8 +268,8 @@ def run_branch(args, branch, nodes): try: fleet.deploy(args.binary_sha, bucket, access, secret) recorder = Recorder(probe, directory / "fixture") - expected = seed(recorder, bucket, "DV", 4, [SIZE]) - baseline = fleet.record("s3-baseline", probe.inventory(bucket)) + expected = seed(recorder, bucket, fixture, 4, [SIZE]) + baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) before = fleet.snapshot("physical-baseline") fleet.record("topology-proof", validate_topology(before, "H2")) @@ -276,9 +289,9 @@ def run_branch(args, branch, nodes): fleet.record("offline-refusal", {"node": offline_node, "connectionRefused": True}) frozen = fleet.record("original-offline-scene", fleet.remote(offline_node, "snapshot", quiescent=True)) earliest_ms = int(time.time() * 1000) - 5000 - expected, obligations, deleted = acknowledge(recorder, bucket, branch, expected) + expected, obligations, deleted = acknowledge(recorder, bucket, branch, expected, put_count=put_count, fixture=fixture) fleet.record("offline-ACK-responsibility-manifest", {"expected": expected, "obligations": obligations, "deleted": deleted}) - acknowledged = fleet.record("offline-S3-ACK-proof", probe.inventory(bucket)) + acknowledged = fleet.record("offline-S3-ACK-proof", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) fleet.record("offline-S3-full-manifest", verify_seed(expected, acknowledged)) deadline = time.monotonic() + PERSIST_SECONDS while True: @@ -302,8 +315,17 @@ def run_branch(args, branch, nodes): fleet.record("registered-automatic-replay-budget", {"persistenceSeconds": PERSIST_SECONDS, "replaySeconds": REPLAY_SECONDS, "adminMutations": 0, "targetReadsBeforePhysicalProof": 0, "source": "MRF", "originalNodeAndDiskIdentitiesRequired": True}) + interruption = None if after_rejoin is None else after_rejoin(probe, fleet, window, binding, bucket, + expected, obligations, bindings, rejoin_time, before, authorities, offline_node) proofs, retired = wait_replay(fleet, window, binding, nodes, bucket, expected, obligations, bindings, rejoin_time, before, authorities, deleted_targets) + if interruption is not None: + interrupted_identity = tuple(interruption["ackIdentity"]) + require(interrupted_identity in proofs and + proofs[interrupted_identity]["source"]["taskId"] != interruption["volatileOriginalTaskId"], + "MRF automatic rescan was mistaken for completion of the lost volatile task") + interruption["newAutomaticTaskId"] = proofs[interrupted_identity]["source"]["taskId"] + result["coordinatorInterruption"] = interruption fleet.record("MRF-verified-responsibility-retirement", {"retirement": retired, "proofs": [{"identity": list(key), **value} for key, value in proofs.items()]}) retirement(read_checkpoints(fleet, nodes, "MRF-repeat-retirement"), obligations, bindings) @@ -316,7 +338,8 @@ def run_branch(args, branch, nodes): fleet.start() for endpoint in binding["endpoints"]: peer = Probe(binding, endpoint) - inventory = fleet.record("MRF-all-peer-S3", {"endpoint": endpoint, **peer.inventory(bucket)}) + inventory = fleet.record("MRF-all-peer-S3", {"endpoint": endpoint, + **peer.inventory(bucket, unversioned_current_tags=fixture == "DB")}) fleet.record("MRF-all-peer-manifest", {**verify_seed(expected, inventory), **compare_inventories(acknowledged, inventory)}) for entry in deleted: validate_deleted_read(fleet.record("MRF-deleted-version-refusal", peer.s3("get_object", diff --git a/scripts/validation_heal/replacement.py b/scripts/validation_heal/replacement.py index fa6ece07..ff001762 100644 --- a/scripts/validation_heal/replacement.py +++ b/scripts/validation_heal/replacement.py @@ -24,6 +24,7 @@ AUTOMATIC_BRANCHES = ("replacement-unversioned", "replacement-versioned-marker") GRACEFUL_BRANCHES = ("target-partial-graceful", "versioned-target-partial") CRASH_BRANCHES = ("target-partial-sigkill", "target-partial-sigkill-versioned") +COORDINATOR_BRANCHES = ("replacement-coordinator-kill",) FORMAT_SECONDS = 90 REBUILD_SECONDS = 300 STATUS_PATH = "/rustfs/admin/v4/heal/replacement-recovery" @@ -170,7 +171,8 @@ def coordinator(events, token, target): return owners[0] -def automatic_completion(probe, fleet, endpoint, earliest, *, interruption=None, proofs=(), original_format=None, journal=None): +def automatic_completion(probe, fleet, endpoint, earliest, *, interruption=None, proofs=(), original_format=None, + journal=None, coordinator_interruptor=None): deadline, original, timeline = time.monotonic() + REBUILD_SECONDS, None, [] interrupted = None while True: @@ -192,8 +194,12 @@ def automatic_completion(probe, fleet, endpoint, earliest, *, interruption=None, owner = coordinator(journal.capture(), original["taskId"], proofs[0]["node"]) fleet.record("replacement-distinct-coordinator-proof", {"node": owner, "target": proofs[0]["node"], "originalGeneration": original, "source": "fresh accepted auto_heal and exact erasure task start"}) - interrupted = interrupt_target(fleet, proofs[0]["node"], proofs, interruption, original_format) - interrupted["coordinatorKeptRunning"] = owner + if interruption == "coordinator-sigkill": + require(coordinator_interruptor is not None, "replacement coordinator observer missing") + interrupted = coordinator_interruptor(fleet, owner, proofs, original_format, original) + else: + interrupted = interrupt_target(fleet, proofs[0]["node"], proofs, interruption, original_format) + interrupted["coordinatorKeptRunning"] = owner continue if ready and original is not None: require(len({canonical(records[0]) for records in observed.values()}) == 1, @@ -208,7 +214,8 @@ def automatic_completion(probe, fleet, endpoint, earliest, *, interruption=None, require(interruption is None or interrupted is not None, "NOT_OBSERVED: no actual partial target interruption occurred") return {"source": "automatic-replacement", "generation": original, "observations": len(timeline), "completionObservedOnEveryPeer": True, "budgetSeconds": REBUILD_SECONDS, - "adminHealMutations": 0, "targetS3ReadsBeforePhysicalProof": 0, "targetInterruption": interrupted} + "adminHealMutations": 0, "targetS3ReadsBeforePhysicalProof": 0, + "coordinatorInterruption" if interruption == "coordinator-sigkill" else "targetInterruption": interrupted} def wait_format(fleet, node, disk): @@ -287,13 +294,15 @@ def physical_rebuild(before, after, member): "backupRestorationUsed": False} -def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admissions): +def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admissions, *, coordinator_interruptor=None): try: from .versions import map_inline_target, validate_version_receipts, verify_version_s3 except ImportError: from versions import map_inline_target, validate_version_receipts, verify_version_s3 - automatic = branch in (*AUTOMATIC_BRANCHES, *GRACEFUL_BRANCHES, *CRASH_BRANCHES) - require(branch in (*BRANCHES, *AUTOMATIC_BRANCHES, *GRACEFUL_BRANCHES, *CRASH_BRANCHES), "unknown replacement branch") + automatic = branch in (*AUTOMATIC_BRANCHES, *GRACEFUL_BRANCHES, *CRASH_BRANCHES, *COORDINATOR_BRANCHES) + require(branch in (*BRANCHES, *AUTOMATIC_BRANCHES, *GRACEFUL_BRANCHES, *CRASH_BRANCHES, *COORDINATOR_BRANCHES), "unknown replacement branch") + require(branch not in COORDINATOR_BRANCHES or len(fleet.nodes) == 4 and coordinator_interruptor is not None, + "replacement coordinator crash requires actual H2 and its independent observer") require(not automatic or fleet.runtime_profile == "replacement", "automatic replacement lacks its isolated source configuration") before = fleet.snapshot("replacement-original-" + branch, quiescent=True) if automatic: @@ -318,7 +327,7 @@ def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admi proofs.append({**target, **shard}) earliest = int(time.time()) journal = JournalWindow(fleet, "replacement-interruption-source", event_targets=SOURCE_TARGETS) \ - if branch in (*GRACEFUL_BRANCHES, *CRASH_BRANCHES) else None + if branch in (*GRACEFUL_BRANCHES, *CRASH_BRANCHES, *COORDINATOR_BRANCHES) else None replaced = fleet.record("replacement-original-root-isolated", fleet.remote(node, "replace-drive", disk=disk, expectedFormatSha256=physical_format["sha256"])) require(replaced.get("oldRootMovedIntact") is True and replaced.get("newDataRootEmpty") is True @@ -338,8 +347,9 @@ def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admi if automatic: endpoint = replaced["dataPath"] if len(fleet.nodes) == 1 else f"http://rustfs-node{fleet.nodes.index(node) + 1}:9000{replaced['dataPath']}" receipts = automatic_completion(recorder.probe, fleet, endpoint, earliest, - interruption="graceful" if branch in GRACEFUL_BRANCHES else "sigkill" if branch in CRASH_BRANCHES else None, - proofs=proofs, original_format=initialized, journal=journal) + interruption="graceful" if branch in GRACEFUL_BRANCHES else "sigkill" if branch in CRASH_BRANCHES else + "coordinator-sigkill" if branch in COORDINATOR_BRANCHES else None, + proofs=proofs, original_format=initialized, journal=journal, coordinator_interruptor=coordinator_interruptor) else: status = run_task(recorder.probe, fleet, {"bucket": bucket, "key": ""}, 2, active, option_overrides={"recursive": True}, admissions=admissions) @@ -382,7 +392,7 @@ def run_branch(recorder, fleet, bucket, branch, expected, baseline, active, admi streaming.append({"key": target["key"], "versionId": target["versionId"], "proof": proof}) fleet.start() fleet.record("replacement-backup-service-refusal-after", fleet.remote(node, "replacement-probe")) - fixture = "DB" if branch in {"new-disk-data", "replacement-unversioned", "target-partial-graceful", "target-partial-sigkill"} else "DV" + fixture = "DB" if branch in {"new-disk-data", "replacement-unversioned", "target-partial-graceful", "target-partial-sigkill", *COORDINATOR_BRANCHES} else "DV" s3 = verify_version_s3(recorder.probe, fleet, bucket, expected, baseline, "replacement-s3-after", fixture=fixture) peers = [] for endpoint in recorder.probe.binding["endpoints"]: diff --git a/scripts/validation_heal/replacement_journal.py b/scripts/validation_heal/replacement_journal.py new file mode 100644 index 00000000..4e4611b1 --- /dev/null +++ b/scripts/validation_heal/replacement_journal.py @@ -0,0 +1,65 @@ +"""Read the exact candidate's run-owned replacement generation records.""" + +import hashlib +import json +import os +import stat +import uuid + + +REFERENCE = {"candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", "sourceFiles": { + "crates/heal/src/heal/resume.rs": "ed55baa03fafa79bf662393eafcf53b4f7752ca8b2f77824cc6deff085a0cf1d", + "crates/heal/src/heal/resume/replacement.rs": "d44d4b688a12fd09c4c9c48067e0f3dd024daf429ad4d4eeb63d4783bb6a143a"}, + "resumeSchema": 7, "commitSeal": "sealed", "role": "read-only generation witness; no replacement record synthesis"} +FILES = {"intent": ("ahm_replacement_intent.json", 1024**2), + "seal": ("ahm_replacement_intent_seal", 32), + "proof": ("ahm_replacement_completion_proof.json", 1024**2)} + + +def require(value, message): + if not value: raise ValueError(message) + + +def read_records(roots, token, checked): + require(isinstance(token, str) and str(uuid.UUID(token)) == token and uuid.UUID(token).int != 0, + "replacement checkpoint requires a canonical nonzero generation") + require(len(roots) == 4, "replacement checkpoint omitted an original physical root") + + def collect(): + rows = [] + for disk, root in enumerate(roots): + data = root / "data" + for kind, (suffix, limit) in FILES.items(): + relative = f".rustfs.sys/buckets/ahm-replacement/{token}_{suffix}" + path = checked(data / relative, data) + row = {"disk": disk, "kind": kind, "relativePath": relative, "present": False} + try: fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + except FileNotFoundError: + rows.append(row) + continue + with os.fdopen(fd, "rb") as stream: + before = os.fstat(stream.fileno()) + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_size <= limit, + "replacement record exceeds its exact regular-file bound") + raw = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + try: current = path.lstat() + except FileNotFoundError: return None + fields = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns", "st_nlink") + if len(raw) != before.st_size or any(getattr(before, field) != getattr(other, field) + for other in (after, current) for field in fields): return None + row.update(present=True, size=len(raw), sha256=hashlib.sha256(raw).hexdigest()) + if kind == "seal": + require(raw == b"sealed", "replacement generation has no reviewed commit seal") + row["sealed"] = True + else: + row["value"] = json.loads(raw) + require(isinstance(row["value"], dict), "replacement record is not a complete JSON object") + rows.append(row) + return rows + + for _ in range(3): + first, second = collect(), collect() + if first is not None and first == second: + return {"taskId": token, "readOnly": True, "stable": True, "records": second} + raise ValueError("replacement checkpoint did not stabilize inside its three-read budget") diff --git a/scripts/validation_heal/root_restart.py b/scripts/validation_heal/root_restart.py index 601c9d42..6a8c7049 100644 --- a/scripts/validation_heal/root_restart.py +++ b/scripts/validation_heal/root_restart.py @@ -176,8 +176,19 @@ def root_terminal(observed, token, owner, current, bucket, expected): "completedReportMatchesControlPlane": True, "retainedDiagnosticsAreFullManifest": False} -def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, original): - fleet.record("root-restart-contract", REFERENCE) +def crashed_owner(before, after): + require(canonical(before) == canonical(after), + "SIGKILL changed or moved the frozen original root intent without a graceful checkpoint") + return {"samePhysicalOwnerAndIntentBytes": True, "gracefulTimeoutCheckpointClaimed": False, + "recoveryBoundary": "replay the persisted admission; no cursor or elapsed-budget checkpoint claim"} + + +def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, original, interruption="graceful"): + require(interruption in {"graceful", "sigkill"}, "unsupported root interruption") + require(interruption != "sigkill" or len(fleet.nodes) == 4, "F04 root coordinator crash requires actual H2") + fleet.record("root-restart-contract", {**REFERENCE, "interruption": interruption, + "processBoundary": REFERENCE["processBoundary"] if interruption == "graceful" else + "coordinator-only pidfd SIGKILL; persisted admission replay, without shutdown checkpoint"}) started = fleet.record("root-admission", probe.admin(PATH, [], json.dumps(OPTIONS).encode())) require(started.get("httpStatus") == 200 and isinstance(started.get("response"), dict), "root admission failed") token = started["response"].get("clientToken") @@ -189,6 +200,8 @@ def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, or require(set(owners) == {"intent"}, "NOT_OBSERVED: original root owner finished before the interruption window") coordinator = owners["intent"]["node"] require(coordinator in fleet.nodes, "root coordinator is outside the actual bound fleet") + require(interruption != "sigkill" or all(target["node"] != coordinator for target in targets), + "NOT_OBSERVED: F04 root crash must keep every missing-shard target process alive") root_intent(admitted, token, coordinator) fleet.record("actual-root-coordinator", {"node": coordinator, "source": "unique physical admitted intent owner", "taskId": token}) processes = fleet.record("processes-before-interruption", {node: fleet.remote(node, "process") for node in fleet.nodes}) @@ -200,11 +213,16 @@ def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, or scene = fleet.snapshot("root-frozen-physical") pending, before_physical = partial_physical(original, scene, targets) before_record = root_intent(records(fleet, token, "root-intent-before-stop"), token, coordinator) - fleet.record("root-graceful-exit", fleet.remote(coordinator, "process-graceful", expected=identity)) + exit_proof = fleet.record("root-actual-exit", fleet.remote(coordinator, + "process-graceful" if interruption == "graceful" else "process-crash", expected=identity)) + if interruption == "sigkill": + require(exit_proof.get("actualSignal") == 9 and exit_proof.get("graceful") is False + and exit_proof.get("pidfdBound") is True and exit_proof.get("platformEventIssued") is False, + "root coordinator has no exact process-bound SIGKILL exit witness") del fleet.frozen_processes[coordinator] fleet.record("release-exited-unit", fleet.remote(coordinator, "stop")) after_record = root_intent(records(fleet, token, "root-intent-after-exit", stopped_node=coordinator), token, coordinator) - budget = same_owner(before_record, after_record) + budget = same_owner(before_record, after_record) if interruption == "graceful" else crashed_owner(before_record, after_record) exit_scenes = [] for index in range(3): exit_scenes.append(fleet.record("root-physical-after-exit", {node: fleet.remote(node, "snapshot", quiescent=node == coordinator) for node in fleet.nodes})) @@ -247,6 +265,13 @@ def observe_task(probe, fleet, binding, bucket, expected, targets, active, *, or root_terminal(journal_after, token, after_record, repeat, bucket, expected) require(canonical(journal_before) == canonical(journal_after), "repeat restart rewrote the completed root journal") require(physical_content(complete_scene) == physical_content(fleet.snapshot("root-repeat-live-scene")), "completed task revived and rewrote user files after repeat restart") + if interruption == "sigkill": + return {"root-coordinator-kill": {"verdict": "PASS", "originalTaskId": token, + "runningWindow": first["outcome"]["counters"], "beforeInterruptionPhysical": before_physical, + "afterActualExitPhysical": after_physical, "actualExit": exit_proof, "processContinuity": continuity, + "persistedAdmission": budget, "sameTaskRecovered": True, "newStartRequestIssued": False, + "completeResponsibility": terminal, "terminalJournal": journal, "repeatRestartIdle": repeat_idle, + "completedJournalBytesPreserved": True, "allPeerIdle": idle}} return {"root-partial-graceful": {"verdict": "PASS", "originalTaskId": token, "runningWindow": first["outcome"]["counters"], "beforeInterruptionPhysical": before_physical, "afterActualExitPhysical": after_physical, "sameTaskRecovered": True, "newStartRequestIssued": False, "processContinuity": continuity, diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 8110ec4a..829072ea 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -35,6 +35,7 @@ case "${case_id}" in heal.below-write-quorum) driver=write_quorum.py ;; heal.foreign-format-and-unknown-schema) driver=schema.py ;; heal.encrypted-objects) driver=encrypted.py ;; + heal.coordinator-process-crash) driver=coordinator_crash.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal_coordinator_crash.py b/test_validation_heal_coordinator_crash.py new file mode 100644 index 00000000..1a246469 --- /dev/null +++ b/test_validation_heal_coordinator_crash.py @@ -0,0 +1,264 @@ +import copy +from datetime import datetime +import hashlib +import json +import os +from pathlib import Path +import tempfile +import sys +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import coordinator_crash as executor, replacement_journal, root_restart, mrf_replay +from scripts.validation_heal import healthy, fleet_node +from scripts.validation_heal.matrix import AUTOMATIC_CASES, select +import test_validation_heal as fixtures + + +class CoordinatorCrashTests(unittest.TestCase): + token = "01234567-89ab-4def-8123-456789abcdef" + + def peers(self): + return {node: {**{name: name + "-" + node for name in fleet_node.PROCESS_FIELDS}, + "pid": 100 + index, "startTicks": 1000 + index} for index, node in enumerate(("node1", "node2", "node3", "node4"))} + + def test_coordinator_restart_keeps_every_target_process_and_boot(self): + before, after = self.peers(), self.peers() + after["node1"].update(pid=200, startTicks=2000) + self.assertTrue(executor.continuity(before, after, "node1")["everyTargetAndPeerKeptOriginalProcess"]) + for kind in ("target-pid", "target-mount", "owner-env", "owner-boot", "missing", "unchanged"): + bad = copy.deepcopy(after) + if kind == "target-pid": bad["node4"]["pid"] += 1 + elif kind == "target-mount": bad["node4"]["mountNamespace"] = "other" + elif kind == "owner-env": bad["node1"]["environmentFileSha256"] = "other" + elif kind == "owner-boot": bad["node1"]["bootId"] = "other" + elif kind == "missing": bad.pop("node2") + else: bad["node1"] = before["node1"] + with self.subTest(kind=kind), self.assertRaises(ValueError): executor.continuity(before, bad, "node1", private_mounts=True) + after["node1"]["mountNamespace"] = "mnt:[new]" + with self.assertRaises(ValueError): executor.continuity(before, after, "node1") + executor.continuity(before, after, "node1", private_mounts=True) + + def test_actual_kill_refuses_graceful_exit_platform_event_and_changed_peer(self): + peers = self.peers() + proof = {"actualSignal": 9, "signal": "SIGKILL", "graceful": False, "pidfdBound": True, "platformEventIssued": False} + for kind in ("good", "graceful", "signal", "platform", "pidfd", "peer"): + calls, changed = [], copy.deepcopy(proof) + if kind == "graceful": changed["graceful"] = True + elif kind == "signal": changed["actualSignal"] = 15 + elif kind == "platform": changed["platformEventIssued"] = True + elif kind == "pidfd": changed["pidfdBound"] = False + def remote(node, action, **params): + calls.append((node, action, params)) + if action == "process-crash": return changed + if action == "process": return {**peers[node], **({"pid": 999} if kind == "peer" else {})} + return {} + fleet = types.SimpleNamespace(nodes=list(peers), frozen_processes={"node1": peers["node1"]}, + remote=remote, record=lambda label, value: value) + with self.subTest(kind=kind): + if kind == "good": + executor.kill(fleet, "node1", peers["node1"], peers) + self.assertNotIn("node1", fleet.frozen_processes) + self.assertEqual(calls[0][2]["expected"], peers["node1"]) + self.assertEqual([node for node, action, _ in calls if action == "process"], list(peers)[1:]) + else: + with self.assertRaises(ValueError): executor.kill(fleet, "node1", peers["node1"], peers) + + def test_crash_preserves_original_root_intent_without_fabricated_timeout_checkpoint(self): + before = root_restart.root_intent(fixtures.HealRootRestartOracleTest().records(), self.token, "node1") + proof = root_restart.crashed_owner(before, copy.deepcopy(before)) + self.assertFalse(proof["gracefulTimeoutCheckpointClaimed"]) + for kind in ("disk", "sha", "timeout", "scope", "token"): + bad = copy.deepcopy(before) + if kind == "disk": bad["disk"] += 1 + elif kind == "sha": bad["sha256"] = "other" + elif kind == "timeout": bad["value"]["options"]["timeout"] = {"secs": 250, "nanos": 0} + elif kind == "scope": bad["value"]["heal_type"] = {"type": "bucket", "bucket": "other"} + else: bad["value"]["task_id"] = "another" + with self.subTest(kind=kind), self.assertRaises(ValueError): root_restart.crashed_owner(before, bad) + + def checkpoint(self): + endpoint = "http://rustfs-node4:9000/data/rustfs4/owned/data" + generation = {"taskId": self.token, "generation": self.token, "setDiskId": "pool_0_set_0", "targetSlots": [endpoint]} + intent = {"schema_version": 7, "task_id": self.token, "replacement_generation": self.token, + "set_disk_id": generation["setDiskId"], "replacement_targets": [endpoint], "completed": False, + "replacement_target_identities": [{"endpoint": endpoint, "canonical_path": "/data/rustfs4/owned/data", + "physical_device_ids": ["uuid:disk"], "filesystem_identity": "filesystem"}]} + rows = [] + for disk in range(4): + for kind, (suffix, _) in replacement_journal.FILES.items(): + row = {"disk": disk, "kind": kind, "present": disk == 2 and kind != "proof", + "relativePath": f".rustfs.sys/buckets/ahm-replacement/{self.token}_{suffix}"} + if row["present"]: + raw = b"sealed" if kind == "seal" else json.dumps(intent).encode() + row.update(size=len(raw), sha256=hashlib.sha256(raw).hexdigest()) + row.update({"sealed": True} if kind == "seal" else {"value": copy.deepcopy(intent)}) + rows.append(row) + return {"taskId": self.token, "readOnly": True, "stable": True, "records": rows}, generation + + def test_replacement_checkpoint_requires_sealed_exact_generation_and_target_identity(self): + observed, generation = self.checkpoint() + self.assertEqual(executor.replacement_checkpoint(observed, generation)["sealedRecordDisks"], [2]) + for kind in ("missing", "duplicate", "schema", "generation", "target", "device", "complete", "proof", "path", "unsealed", "hash"): + bad = copy.deepcopy(observed) + intent = next(row for row in bad["records"] if row["present"] and row["kind"] == "intent") + seal = next(row for row in bad["records"] if row["present"] and row["kind"] == "seal") + if kind == "missing": intent["present"] = False + elif kind == "duplicate": bad["records"][-1] = copy.deepcopy(bad["records"][0]) + elif kind == "schema": intent["value"]["schema_version"] = True + elif kind == "generation": intent["value"]["replacement_generation"] = "another" + elif kind == "target": intent["value"]["replacement_targets"] = ["another"] + elif kind == "device": intent["value"]["replacement_target_identities"][0]["physical_device_ids"] = [] + elif kind == "complete": intent["value"]["completed"] = True + elif kind == "proof": bad["records"][-1].update(present=True, size=10, sha256="a" * 64) + elif kind == "path": intent["relativePath"] = "foreign" + elif kind == "unsealed": seal["present"] = False + else: seal["sha256"] = "a" * 64 + with self.subTest(kind=kind), self.assertRaises(ValueError): executor.replacement_checkpoint(bad, generation) + + def test_replacement_journal_reads_only_exact_names_and_rejects_symlink_hardlink_and_oversize(self): + with tempfile.TemporaryDirectory() as folder: + roots = [Path(folder).resolve() / str(disk) for disk in range(4)] + for root in roots: (root / "data").mkdir(parents=True) + parent = roots[2] / "data/.rustfs.sys/buckets/ahm-replacement" + parent.mkdir(parents=True) + path = parent / (self.token + "_ahm_replacement_intent.json") + path.write_text('{"task_id":"' + self.token + '"}') + (parent / "unrelated_ahm_replacement_intent.json").write_text("untrusted non-JSON control") + observed = replacement_journal.read_records(roots, self.token, fleet_node.checked) + self.assertEqual(sum(row["present"] for row in observed["records"]), 1) + self.assertTrue(observed["readOnly"]) + raw = path.read_bytes() + for kind in ("symlink", "hardlink", "oversize"): + path.unlink() + if kind == "symlink": path.symlink_to(parent / "unrelated_ahm_replacement_intent.json") + elif kind == "hardlink": os.link(parent / "unrelated_ahm_replacement_intent.json", path) + else: path.write_bytes(b"x" * (1024**2 + 1)) + with self.subTest(kind=kind), self.assertRaises((ValueError, OSError)): + replacement_journal.read_records(roots, self.token, fleet_node.checked) + path.unlink(); path.write_bytes(raw) + with self.assertRaises(ValueError): replacement_journal.read_records(roots, "../other", fleet_node.checked) + + def mrf_scene(self): + bucket, _, source = fixtures.HealNodeOutageTest().snapshot() + scene, obligations = {}, [] + for node, value in source.items(): + scene[node] = {"formats": copy.deepcopy(value["formats"]), "files": []} + for index in range(2): + key = f"offline/new-{index:02d}.bin" + for entry in value["files"]: + row = copy.deepcopy(entry) + row["path"] = row["path"].replace("new.bin", key) + row["sha256"] = hashlib.sha256(row["path"].encode() + node.encode() + str(row["disk"]).encode()).hexdigest() + if row.get("metadata"): + row["metadata"]["key"] = key + row["metadata"]["versions"] = [row["metadata"]["versions"][0]] + row["metadata"]["versions"][0]["isLatest"] = True + scene[node]["files"].append(row) + if node == "node1": obligations.append({"bucket": bucket, "key": key, "versionId": "new-version", + "poolIndex": 0, "setIndex": 0, "kind": 3}) + authorities = copy.deepcopy(scene) + authorities["node4"]["files"] = [] + scene["node4"]["files"] = [row for row in scene["node4"]["files"] if "new-00" in row["path"]] + return scene, authorities, bucket, obligations + + def test_mrf_partial_progress_is_not_a_final_hash_or_streaming_proof(self): + scene, authorities, bucket, obligations = self.mrf_scene() + proof = executor.mrf_progress(scene, authorities, bucket, obligations, "node4") + self.assertTrue(proof["partial"]) + self.assertEqual(proof["pendingACKs"], 1) + self.assertFalse(proof["finalStreamingProofClaimed"]) + empty = copy.deepcopy(scene); empty["node4"]["files"] = [] + self.assertFalse(executor.mrf_progress(empty, authorities, bucket, obligations, "node4")["partial"]) + for kind in ("role", "version", "authority", "part"): + bad, originals = copy.deepcopy(scene), copy.deepcopy(authorities) + version = next(row for row in bad["node4"]["files"] if row.get("metadata"))["metadata"]["versions"][0] + if kind == "role": version["erasure"]["index"] = 1 + elif kind == "version": version["dataDir"] = "other" + elif kind == "authority": next(row for row in originals["node1"]["files"] if row.get("metadata"))["metadata"]["versions"][0]["size"] += 1 + else: next(row for row in bad["node4"]["files"] if not row.get("metadata"))["sha256"] = "incomplete" + with self.subTest(kind=kind), self.assertRaises(ValueError): executor.mrf_progress(bad, originals, bucket, obligations, "node4") + + def events(self): + stamp = "2026-10-11T00:00:00+00:00" + obligation = {"bucket": "owned", "key": "key", "versionId": "null", "poolIndex": 0, "setIndex": 0, "kind": 3} + rows = [{"target": "rustfs::heal::manager", "message": "Heal queue request accepted", "source": "mrf", + "result": "accepted", "request_id": self.token, "timestamp": stamp}, + {"target": "rustfs::heal::task", "message": "Heal object started", "stage": "start", "task_id": self.token, + "bucket": "owned", "object": "key", "version_id": "None", "timestamp": stamp}] + bindings = {mrf_replay.identity(obligation): [{"node": "node1"}]} + return {"node1": rows}, [obligation], bindings, datetime.fromisoformat(stamp) + + def test_mrf_source_requires_exact_ack_and_admission_without_terminal_completion(self): + events, obligations, bindings, stamp = self.events() + self.assertEqual(executor.active_mrf_task(events, obligations, bindings, stamp, "node4")[0][:2], ("node1", self.token)) + for kind in ("source", "version", "key", "terminal", "missing-version", "target"): + bad = copy.deepcopy(events) + if kind == "source": bad["node1"][0]["source"] = "scanner" + elif kind == "version": bad["node1"][1]["version_id"] = "foreign" + elif kind == "key": bad["node1"][1]["object"] = "foreign" + elif kind == "terminal": bad["node1"].append({"target": "rustfs::heal::manager", "task_id": self.token, + "state": "task_completed", "timestamp": stamp.isoformat()}) + elif kind == "missing-version": del bad["node1"][1]["version_id"] + self.assertEqual(executor.active_mrf_task(bad, obligations, bindings, stamp, "node1" if kind == "target" else "node4"), []) + + def test_rust_debug_version_representation_never_merges_distinct_versions(self): + for raw in ("version", 'Some("version")'): self.assertTrue(mrf_replay.source_version(raw, "version")) + for raw in ("None", "null", 'Some("other")', "other", 'Some("version") trailing'): + self.assertFalse(mrf_replay.source_version(raw, "version")) + for raw in (None, "None", "null", "00000000-0000-0000-0000-000000000000"): + self.assertTrue(mrf_replay.source_version(raw, "null")) + self.assertFalse(mrf_replay.source_version('Some("other")', "null")) + + def test_mrf_crash_admission_is_bounded_with_other_sources_disabled(self): + config = fleet_node.runtime_config("mrf-crash") + self.assertEqual(config["RUSTFS_HEAL_MRF_REPLAY_BATCH"], "1") + self.assertEqual(config["RUSTFS_HEAL_MAX_CONCURRENT_HEALS"], "1") + self.assertEqual(config["RUSTFS_HEAL_MAX_CONCURRENT_PER_SET"], "1") + self.assertEqual(config["RUSTFS_HEAL_MRF_ENABLE"], "true") + self.assertEqual(config["RUSTFS_SCANNER_ENABLED"], "false") + self.assertEqual(config["RUSTFS_HEAL_AUTO_HEAL_ENABLE"], "false") + self.assertNotIn("RUSTFS_HEAL_MAX_CONCURRENT_HEALS", fleet_node.runtime_config("durable-mrf")) + + def test_remote_checkpoint_module_is_carried_in_the_actual_ssh_program(self): + sources = [] + def invoke(args, **options): + sources.append(options["input"]) + compile(sources[-1], "node-source", "exec") + return types.SimpleNamespace(returncode=0, stdout='{"readOnly":true}', stderr="") + with tempfile.TemporaryDirectory() as folder, patch.object(healthy.subprocess, "run", side_effect=invoke): + fleet = healthy.Fleet(["node1"], Path(folder), "gh-12345-1-h2") + self.assertTrue(fleet.remote("node1", "replacement-records", taskId=self.token)["readOnly"]) + self.assertIn("ahm-replacement/", sources[0]) + self.assertIn("def read_records(roots, token, checked):", sources[0]) + + def test_all_three_mechanisms_are_required_and_dirty_cleanup_stops_dispatch(self): + self.assertEqual(select("F04")[0]["variants"], list(executor.BRANCHES)) + self.assertIn(executor.CASE_ID, AUTOMATIC_CASES) + for dirty in (False, True): + with tempfile.TemporaryDirectory() as folder: + output = Path(folder) + candidate = output / "candidate.json" + candidate.write_text(json.dumps({"source_sha": executor.REFERENCE["candidateCommit"]})) + argv = ["program", "--candidate", str(candidate), "--rc-bin", "rc", "--evidence", str(output), + "--tool-directory", "tools", "--case-id", executor.CASE_ID, "--testing-commit", "a" * 40, + "--binary-sha", "b" * 64] + root = {"topology": "H2", "verdict": "FAIL", "cleanup": "FAIL" if dirty else "PASS"} + mrf = {"topology": "H2", "branch": "offline-put", "verdict": "PASS", "cleanup": "PASS"} + replacement_result = {"topology": "H2", "branch": executor.BRANCHES[2], "verdict": "PASS", "cleanup": "PASS"} + with patch.object(sys, "argv", argv), patch.dict(os.environ, {"GITHUB_RUN_ID": "12345"}), \ + patch.object(executor, "dx_topology", return_value=root) as root_run, \ + patch.object(executor.mrf_replay, "run_branch", return_value=mrf) as mrf_run, \ + patch.object(executor, "run_replacement", return_value=replacement_result) as replacement_run: + with self.assertRaises(ValueError): executor.main() + summary = json.loads((output / "healthy-summary.json").read_text()) + self.assertFalse(summary["complete"]) + self.assertEqual(summary["cleanup"], "FAIL" if dirty else "PASS") + self.assertEqual(len(summary["results"]), 1 if dirty else 3) + self.assertEqual(root_run.call_args.kwargs["target_nodes"], ["vm001", "vm002", "vm003"]) + self.assertEqual(mrf_run.call_count, 0 if dirty else 1) + self.assertEqual(replacement_run.call_count, 0 if dirty else 1) + + +if __name__ == "__main__": unittest.main() From c593556fe8f34b5d30aac91cb59378a8aedde5ac Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 07:24:23 +0800 Subject: [PATCH 093/343] Use actual writable roots and a balanced H2 fixture for quorum validation --- docs/validation-migration-map.md | 18 ++++++++++++++++- scripts/validation_heal/fleet_node.py | 5 ++++- scripts/validation_heal/healthy.py | 2 +- scripts/validation_heal/write_quorum.py | 15 ++++++++++++-- test_validation_heal_write_quorum.py | 26 +++++++++++++++++++++++-- 5 files changed, 59 insertions(+), 7 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 7d060aca..5e7555b3 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1036,7 +1036,7 @@ four regression verdicts have their own archive and never increment a fleet or canonical-case PASS count. Existing tests assert captured lock events internally; the collector does not fabricate emitted barrier logs. Missing critical-point fixtures, Admin nolock semantics and required fleet overlap remain explicit gaps. -The current implementation has 285 focused Heal/catalog checks, including the +The current implementation has 287 focused Heal/catalog checks, including the Linux-only D11, C09 and D08 ownership/mount/recovery checks. Native compilation and execution still require the separate hosted job; local parser checks are not native PASS. @@ -1230,3 +1230,19 @@ source attribution, bounded configuration and mandatory three-mechanism scoring. F04 has no live PASS claim yet. The source-bound MRF log reader also accepts the reviewed Rust Option debug spelling without merging different version identities; existing E05 receipts and every original branch remain independently required. + + +B08's read-only plan now measures writable drives independently of readable +sources, including the candidate's actual default H2 R=12/W=12 geometry. Five +owned read-only roots leave fifteen payload sources and all sixteen metadata +authorities readable while only eleven roots remain writable. Numerical R= read, "layout cannot preserve R while removing W") + require(total - 1 >= read, "layout cannot retain R payload sources after the original missing part") lock = {"mode": "local" if len(hosts) == 1 else "distributed", "clients": len(hosts), "sharedQuorum": (len(hosts) + 1) // 2, "exclusiveQuorum": len(hosts) // 2 + 1} if branch == "lock-quorum-absent" and len(hosts) == 1: @@ -165,7 +170,8 @@ def run_branch(args, topology, nodes, branch): directory = args.evidence / topology / branch directory.mkdir(parents=True) run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" - fleet = Fleet(nodes, directory, run_id, args.tool_directory) + lock_fixture = topology == "H2" and branch == "lock-quorum-absent" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="lock-quorum" if lock_fixture else "manual") candidate = json.loads(args.candidate.read_text()) require(candidate["source_sha"] == LOCK_REFERENCE["candidateCommit"], "B08 lock client policy is unreviewed") bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] @@ -179,6 +185,7 @@ def run_branch(args, topology, nodes, branch): "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), "maxInventoryObjects": 256, "maxInventoryBytes": 512 * 1024**2} fleet.record("lock-policy-source-contract", LOCK_REFERENCE) + fleet.record("storage-boundary-source-contract", STORAGE_REFERENCE) contract = verify_quorum_contract(fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)), candidate) fleet.record("binding", binding) probe, active = Probe(binding), [] @@ -186,6 +193,10 @@ def run_branch(args, topology, nodes, branch): put_observed, put_key, put_payload, original = None, None, None, None try: fleet.deploy(args.binary_sha, bucket, access, secret) + if lock_fixture: + processes = fleet.snapshot("lock-fixture-actual-storage-configuration") + require(all(value["storageClassEnvironment"]["RUSTFS_STORAGE_CLASS_STANDARD"] == "EC:8" + for value in processes.values()), "lock fixture did not load its actual sixteen-drive eight-parity configuration") expected = seed(Recorder(probe, directory / "fixture"), bucket, "DB", 128, SIZES) baseline = fleet.record("S3-baseline", probe.inventory(bucket, unversioned_current_tags=True)) fleet.record("ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) diff --git a/test_validation_heal_write_quorum.py b/test_validation_heal_write_quorum.py index cccbe400..8c99d088 100644 --- a/test_validation_heal_write_quorum.py +++ b/test_validation_heal_write_quorum.py @@ -52,16 +52,38 @@ def test_h1_distributed_loss_is_explicitly_inapplicable_and_retains_both_source_ def test_layout_and_source_uncertainty_cannot_be_assumed_sufficient(self): snapshot, target, _ = self.scene(16) - for kind in ("duplicate-member", "other-node", "unrecognized", "different-policy", "no-read-write-gap"): + for kind in ("duplicate-member", "other-node", "unrecognized", "different-policy", "insufficient-read"): broken, contract = copy.deepcopy(target), dict(self.contract) if kind == "duplicate-member": broken["cohort"][-1] = copy.deepcopy(broken["cohort"][0]) elif kind == "other-node": broken["cohort"][-1]["node"] = "foreign" elif kind == "unrecognized": contract["recognized"] = False elif kind == "different-policy": contract["writeQuorumRule"] = "assumed" - else: broken["logical"]["erasure"].update(data_blocks=10, parity_blocks=6) + else: broken["logical"]["erasure"].update(data_blocks=16, parity_blocks=0) with self.subTest(kind=kind), self.assertRaises(ValueError): executor.plan_for(snapshot, broken, executor.BRANCHES[0], contract) + def test_read_only_boundary_can_preserve_reads_when_numerical_r_equals_w(self): + source = oracle.HealReadQuorumBoundaryTest() + snapshot, expected = source.scene(total=16, data=12) + target = shards.map_target(snapshot, source.bucket, expected[0]["key"], "data", 1) + plan = executor.plan_for(snapshot, target, executor.BRANCHES[0], self.contract) + self.assertEqual(plan["readQuorum"], plan["writeQuorum"]) + self.assertEqual(plan["readableMetadataAuthorities"], 16) + self.assertEqual(plan["readablePayloadSources"], 15) + self.assertEqual(plan["writableDrives"], 11) + self.assertEqual(sum(map(len, plan["gates"].values())), 5) + target = {**target, **next(row for row in target["cohort"] if row["node"] == "node4")} + with self.assertRaisesRegex(ValueError, "preserve R"): + executor.plan_for(snapshot, target, executor.BRANCHES[1], self.contract) + + def test_fresh_h2_lock_fixture_has_a_source_reviewed_balanced_layout(self): + from scripts.validation_heal.fleet_node import runtime_config, BACKGROUND_CONFIG + config = runtime_config("lock-quorum") + self.assertEqual(config.pop("RUSTFS_STORAGE_CLASS_STANDARD"), "EC:8") + self.assertEqual(config, BACKGROUND_CONFIG) + self.assertNotIn("RUSTFS_STORAGE_CLASS_STANDARD", runtime_config("manual")) + self.assertEqual(executor.STORAGE_REFERENCE["candidateCommit"], executor.LOCK_REFERENCE["candidateCommit"]) + def test_measured_read_and_write_refusals_must_match_every_root(self): _, _, plan = self.scene(4) observed = {"node1": {"disks": [{"disk": disk, "readableMetadata": True, From acdd69218406b0596d79b6aab15481f8271e35c6 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 07:38:48 +0800 Subject: [PATCH 094/343] Migrate bounded disk and node flapping with durable ACK and rebuild interruption proofs --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 38 +- scripts/validation_heal/drive_gate.py | 12 + scripts/validation_heal/flapping.py | 554 ++++++++++++++++++ scripts/validation_heal/fleet_node.py | 34 +- scripts/validation_heal/healthy.py | 6 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/mrf_records.py | 5 +- scripts/validation_heal/mrf_replay.py | 6 +- scripts/validation_heal/run_case.sh | 1 + test_validation_heal_flapping.py | 235 ++++++++ test_validation_heal_mrf.py | 25 +- 12 files changed, 906 insertions(+), 16 deletions(-) create mode 100644 scripts/validation_heal/flapping.py create mode 100644 test_validation_heal_flapping.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 84bdd963..1f632159 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -26,6 +26,7 @@ on: - test_validation_heal_bucket_metadata.py - test_validation_heal_write_quorum.py - test_validation_heal_coordinator_crash.py + - test_validation_heal_flapping.py - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py @@ -68,6 +69,7 @@ on: - test_validation_heal_bucket_metadata.py - test_validation_heal_write_quorum.py - test_validation_heal_coordinator_crash.py + - test_validation_heal_flapping.py - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py @@ -102,7 +104,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 5e7555b3..fc029f90 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 126 currently dispatchable IDs include all 74 non-Heal cases and fifty-two Heal cases. Forty-six have live PASS evidence; the other 112 have no live PASS claim. +case evidence. The 127 currently dispatchable IDs include all 74 non-Heal cases and fifty-three Heal cases. Forty-six have live PASS evidence; the other 112 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1246,3 +1246,39 @@ configuration source hash is archived. Two regression tests cover the default R=W read-only plan and fresh balanced lock fixture. H1's missing incarnation receipt remains a product/evidence FAIL; the identity oracle is not relaxed. A fresh full B08 run is required and no new live PASS is claimed. + +B10 `heal.disk-and-node-flapping` now executes all three H2 branches on the +current auto-testing nodes. Each branch preregisters three cycles, 16/16/64 +unique degraded PUT ACKs, four foreground workers, exact fault/persistence/ +replay windows, a total deadline, queue/task/retry limits and CPU/FD/RSS +budgets. The foreground ACK ledger records concurrent unique PUT, HEAD, +control GET and LIST without retrying uncertain writes. Live drive rearming +requires the exact original process birth, an independent service mount +namespace and removal of DAC bypass from all capability sets. Every cycle +has service read/write refusal plus exact-member Admin offline/online +witnesses, or an owned-unit stop plus an exact connection refusal. Original +host, candidate, runtime, disk UUID and set-slot identities remain bound. + +The third `interrupt-rebuild-again` cycle additionally requires an exact +running MRF task and both completed and incomplete physical ACK targets; +it loses the original node again, reproves incomplete physical progress +and requires the original pending durable responsibility IDs/incarnations +and nondecreasing owner sequences to survive. Every cycle then needs actual +MRF admission/start/terminal proof, original responsibility retirement, +complete physical cohorts and streaming integrity before target GET. The +final gate rechecks every peer's full ACK inventory and unchanged physical +results. Manual compensation and backups cannot satisfy convergence. This +executor has no canonical live PASS yet; 31 Heal executors remain. + +E05 `38094551700` retains FAIL/cleanup PASS at `6354298`. Its PUT and marker +branches exhausted the in-memory journal budget with unrelated HTTP/HTTP2 +frame debug events before they could prove complete replay. Its historical +version DELETE branch had no exact committed durable MRF obligation within +the registered persistence window; the final checkpoint held only unrelated +system-bucket records, which cannot discharge the owned ACKs. The corrected +MRF profiles use the candidate's reviewed tracing filter grammar to enable +only Heal manager/task/MRF debug over an info baseline, attest both +`RUST_LOG` and `RUSTFS_OBS_LOGGER_LEVEL`, and retain complete raw journal +pages while selecting the required attribution targets in memory. The +60,000-event/page budgets and exact durable responsibility assertions stay +in force. All three E05 branches still require a fresh complete live run. diff --git a/scripts/validation_heal/drive_gate.py b/scripts/validation_heal/drive_gate.py index fc2bc94e..a7507207 100644 --- a/scripts/validation_heal/drive_gate.py +++ b/scripts/validation_heal/drive_gate.py @@ -121,6 +121,18 @@ def start_properties(value): return properties +def validate_live_boundary(pid): + require(type(pid) is int and pid > 1, "live drive gate requires a real owned process") + root = Path(f"/proc/{pid}") + require((root / "ns/mnt").readlink() != Path("/proc/self/ns/mnt").readlink(), + "live drive gate requires the preconfigured private mount namespace") + fields = dict(line.split(":", 1) for line in (root / "status").read_text().splitlines() if ":" in line) + require(all(name in fields and int(fields[name].strip(), 16) & DENY_DAC_CAPS == 0 + for name in ("CapEff", "CapPrm", "CapBnd")), + "live drive gate requires DAC bypass removed from effective, permitted and bounding capabilities") + return {"privateMountNamespace": True, "dacBypassRemoved": True} + + def process_identity(pid): text = Path(f"/proc/{pid}/status").read_text() fields = dict(line.split(":", 1) for line in text.splitlines() if ":" in line) diff --git a/scripts/validation_heal/flapping.py b/scripts/validation_heal/flapping.py new file mode 100644 index 00000000..cd1d0106 --- /dev/null +++ b/scripts/validation_heal/flapping.py @@ -0,0 +1,554 @@ +#!/usr/bin/env python3 +"""B10: three bounded member-loss cycles and an observed rebuild interruption.""" + +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor +import copy +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +from urllib import request +from urllib.error import URLError +from urllib.parse import quote + +try: + from .heal_probe import Probe, canonical, compare_inventories, payload, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .fleet_node import PROCESS_FIELDS + from .disk_loss import quorum_plan as drive_quorum, member_state + from .node_outage import quorum_plan as node_quorum, put_ack, replica_census + from .mrf_records import verify_reference + from .mrf_replay import identity, pending_bindings, read_checkpoints, retirement, wait_replay, source_version + from .scanner_repair import JournalWindow, event_time + from .shards import files, map_target, verify_cohort + from .stale_rejoin import logical_metadata + from .versions import verify_quorum_contract +except ImportError: + from heal_probe import Probe, canonical, compare_inventories, payload, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from fleet_node import PROCESS_FIELDS + from disk_loss import quorum_plan as drive_quorum, member_state + from node_outage import quorum_plan as node_quorum, put_ack, replica_census + from mrf_records import verify_reference + from mrf_replay import identity, pending_bindings, read_checkpoints, retirement, wait_replay, source_version + from scanner_repair import JournalWindow, event_time + from shards import files, map_target, verify_cohort + from stale_rejoin import logical_metadata + from versions import verify_quorum_contract + + +CASE_ID = "heal.disk-and-node-flapping" +BRANCHES = ("drive-flap-3", "node-flap-3", "interrupt-rebuild-again") +SIZE = 1024**2 + 73 +PROTOCOL = {"cycles": 3, "uniqueACKsPerCycle": [16, 16, 64], "foregroundConcurrency": 4, + "faultWindowSeconds": 120, "persistenceSeconds": 30, "replaySeconds": 240, + "partialWindowSeconds": 90, "branchDeadlineSeconds": 1200, + "extraThirdCycleInterruption": 1, "maxActiveTasks": 4, "maxQueuedTasks": 512, + "maxAcceptedTasksPerACKPerOwner": 64, "maxSystemTasksPerOwner": 256, + "maxFDs": 4096, "maxRssBytes": 4 * 1024**3, + "maxSampledCpuCores": 8, "maxForegroundErrorRate": 0, + "adminCompensationTasks": 0, "backupRestorationUsed": False} + + +def same_process(before, after): + require(all(before.get(name) == after.get(name) for name in PROCESS_FIELDS), + "flapping changed an unrelated process or a live-gated service birth") + + +def node_return(before, after): + require(before.get("pid", 0) > 1 and after.get("pid", 0) > 1 + and (before["pid"], before["startTicks"]) != (after["pid"], after["startTicks"]) + and all(before.get(name) == after.get(name) for name in + ("bootId", "unit", "executable", "binarySha256", "commandSha256", "environmentFileSha256", "settings")), + "node flap did not restart the exact original unit, candidate, boot and runtime") + + +def untouched(before, after, members): + left, right = files(before), files(after) + require({node: value["formats"] for node, value in before.items()} == + {node: value["formats"] for node, value in after.items()}, "flap changed original disk UUIDs or set slots") + require(all(right.get(key) == value for key, value in left.items()), + "flap changed an original acknowledged shard or control") + require(all(key[:2] not in members for key in set(right) - set(left)), + "offline target received a new ACK shard while inaccessible") + return {"originalShardAndControlBytesUnchanged": True, "offlineMembersReceivedNoACKFiles": True} + + +def replay_progress(scene, authorities, bucket, obligations, members): + """Observe exact original member roles; this is not final streaming verification.""" + validate_topology(scene, "H2") + require(members and len(members) in {1, 4}, "flap progress requires the exact original lost members") + current, original, complete = files(scene), files(authorities), [] + layouts = {(node, row["disk"]): row["value"]["xl"] for node, value in scene.items() for row in value["formats"]} + require(members <= set(layouts), "flap target members escaped the original topology") + for item in obligations: + require(item["bucket"] == bucket and item["versionId"] == "null", "flap progress crossed an ACK identity") + path = f"{bucket}/{item['key']}/xl.meta" + sources = [entry["metadata"] for (node, disk, name), entry in original.items() + if name == path and (node, disk) not in members and entry.get("metadata")] + require(len(sources) == 16 - len(members) and + all(canonical(logical_metadata(value)) == canonical(logical_metadata(sources[0])) for value in sources), + "flap has no complete surviving ACK authority") + authority = logical_metadata(sources[0]) + versions = authority.get("versions", []) + require(len(versions) == 1 and versions[0].get("versionId") == "null" + and versions[0].get("isLatest") is True and versions[0].get("deleted") is False, + "flap authority is a different version or marker") + restored = 0 + for member in members: + metadata = current.get((*member, path), {}).get("metadata") + if metadata is None: continue + require(canonical(logical_metadata(metadata)) == canonical(authority), "flap replay changed ACK metadata authority") + version = metadata["versions"][0] + erasure, layout = version["erasure"], layouts[member] + slot = layout["sets"][0].index(layout["this"]) + require(type(erasure.get("index")) is int and erasure["index"] == erasure["distribution"][slot] + and version.get("role") == ("data" if erasure["index"] <= erasure["data_blocks"] else "parity"), + "flap replay put an ACK on the wrong original set role") + parts = version.get("parts", []) + require(version.get("inline") is None and version.get("dataDir") and len(parts) == 1 + and parts[0].get("number") == 1 and parts[0].get("size") == parts[0].get("actualSize") > 1024**2, + "flap progress requires one actual external plaintext ACK part") + part = current.get((*member, f"{bucket}/{item['key']}/{version['dataDir']}/part.1")) + if part is not None: + require(type(part.get("size")) is int and part["size"] > 32 + and re.fullmatch(r"[0-9a-f]{64}", part.get("sha256", "")), "flap replay lacks an actual complete part identity") + restored += 1 + if restored == len(members): complete.append(list(identity(item))) + return {"completeACKs": complete, "pendingACKs": len(obligations) - len(complete), + "partial": 0 < len(complete) < len(obligations), "finalStreamingProofClaimed": False} + + +def retained_pending(checkpoints, obligations, bindings, pending): + require(pending and pending <= {identity(item) for item in obligations}, "interruption has no exact incomplete ACKs") + for wanted in pending: + for owner in bindings[wanted]: + value = checkpoints[owner["node"]] + rows = [row for row in value["intents"] if identity(row) == wanted] + require(value["present"] and value["owner"] == owner["checkpointOwner"] + and value["sequence"] >= owner["sequence"] and len(rows) == 1 + and rows[0]["lifecycle"]["responsibility_id"] == owner["responsibilityId"] + and rows[0]["lifecycle"]["source_bucket_incarnation_id"] == owner["bucketIncarnationId"] + and rows[0]["lifecycle"]["state"] == {"state": "active"} + and rows[0]["lifecycle"].get("last_operator_acceptance") is None, + "incomplete interrupted ACK was retired, rebound, rolled back or operator-accepted") + return {"pendingOriginalResponsibilities": len(pending), "ownerIncarnationAndResponsibilityStable": True} + + +def bounded_admissions(events, obligations): + lookup = {(item["bucket"], item["key"], item["versionId"]): identity(item) for item in obligations} + tokens, counts, system_counts = {}, {}, {} + for node, rows in events.items(): + accepted = {row.get("request_id") for row in rows if row.get("target") == "rustfs::heal::manager" + and row.get("message") == "Heal queue request accepted" and row.get("source") == "mrf" and row.get("result") == "accepted"} + for row in rows: + if row.get("target") != "rustfs::heal::task" or row.get("message") != "Heal object started": continue + wanted = (row.get("bucket"), row.get("object"), "null") + require((wanted in lookup or wanted[0] == ".rustfs.sys" and isinstance(wanted[1], str) and wanted[1]) + and "version_id" in row and source_version(row["version_id"], "null"), + "flapping source task escaped owned ACKs or the internal system bucket") + bound = lookup[wanted] if wanted in lookup else (*wanted, "internal-system") + token = row.get("task_id") + require(isinstance(token, str) and token in accepted and event_time(row) is not None, + "flapping task has no actual accepted MRF admission") + previous = tokens.get(token) + require(previous is None or previous == (node, bound), "flap task token changed owner or ACK identity") + if previous is None: + tokens[token] = (node, bound) + key = node, bound + counts[key] = counts.get(key, 0) + 1 + require(counts[key] <= PROTOCOL["maxAcceptedTasksPerACKPerOwner"], "flapping exceeded its registered retry/task budget") + if wanted not in lookup: + system_counts[node] = system_counts.get(node, 0) + 1 + require(system_counts[node] <= PROTOCOL["maxSystemTasksPerOwner"], "flapping exceeded its registered internal-system task budget") + return {"distinctAcceptedTaskIds": len(tokens), "maxTasksForOneACKOwner": max(counts.values(), default=0), + "internalSystemTasks": sum(system_counts.values()), "internalSystemTasksCertifyOwnedACKs": False, + "duplicateTokensCountedAsIndependentRepairs": False} + + +class Budget: + def __init__(self, fleet, binding): + self.fleet, self.binding = fleet, binding + self.deadline = time.monotonic() + PROTOCOL["branchDeadlineSeconds"] + self.previous = {} + self.samples = 0 + + def check(self): + require(time.monotonic() < self.deadline, "flapping exceeded its preregistered total deadline") + + def sample(self, nodes): + self.check() + for node in nodes: + value = self.fleet.record("flap-process-resource-sample", {"node": node, **self.fleet.remote(node, "process-metrics")}) + require(value.get("readOnly") is True and type(value.get("fileDescriptors")) is int + and 0 < value["fileDescriptors"] <= PROTOCOL["maxFDs"] + and type(value.get("rssBytes")) is int and 0 < value["rssBytes"] <= PROTOCOL["maxRssBytes"], + "flapping exceeded its frozen FD/RSS resource budget") + previous = self.previous.get(node) + if previous is not None: + if all(previous["identity"].get(name) == value["identity"].get(name) for name in PROCESS_FIELDS): + elapsed = (value["monotonicNs"] - previous["monotonicNs"]) / 1e9 + require(elapsed > 0 and type(value["clockTicksPerSecond"]) is int and value["clockTicksPerSecond"] > 0 + and value["clockTicksPerSecond"] == previous["clockTicksPerSecond"] + and value["cpuTicks"] >= previous["cpuTicks"] and value["clockDomain"] == previous["clockDomain"], + "flapping resource clock or counters regressed") + cores = (value["cpuTicks"] - previous["cpuTicks"]) / value["clockTicksPerSecond"] / elapsed + self.fleet.record("flap-CPU-interval", {"node": node, "cpuCores": cores, "elapsedSeconds": elapsed}) + require(cores <= PROTOCOL["maxSampledCpuCores"], "flapping exceeded its frozen CPU budget") + else: + require(node == self.fleet.nodes[-1], "flapping changed an unrelated peer process") + node_return(previous["identity"], value["identity"]) + self.previous[node] = value + endpoint = "node" + str(self.fleet.nodes.index(node) + 1) + observed = self.fleet.record("flap-queue-resource-sample", Probe(self.binding, endpoint).admin( + "/rustfs/admin/v3/background-heal/status", [], b"")) + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), "flap queue status unavailable") + body = observed["response"] + require(type(body.get("healActiveTasks")) is int and 0 <= body["healActiveTasks"] <= PROTOCOL["maxActiveTasks"] + and type(body.get("healQueueLength")) is int and 0 <= body["healQueueLength"] <= PROTOCOL["maxQueuedTasks"], + "flapping exceeded its registered active/queued task budget") + self.samples += 1 + + +def foreground(recorder, bucket, cycle, expected, deadline): + count = PROTOCOL["uniqueACKsPerCycle"][cycle - 1] + def put(index): + require(time.monotonic() < deadline, "foreground admission exceeded the registered offline window") + key = f"flap/cycle-{cycle}/nonce-{index:03d}.bin" + require(not any(entry["key"] == key for entry in expected), "flap reused a previous ACK key") + body = payload(f"{bucket}/{key}/0", SIZE + index) + ack = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "Metadata": {"heal-fixture": "DB-DL", "cycle": str(cycle), "nonce": str(index)}, + "Tagging": "suite=heal&fixture=flap"}, body, admission_deadline=deadline) + require(ack.get("ResponseMetadata", {}).get("HTTPStatusCode") == 200, "flap PUT lacks its explicit commit ACK") + recorder.call("head_object", {"Bucket": bucket, "Key": key}, admission_deadline=deadline) + return key, body, ack + def control_read(): + require(time.monotonic() < deadline, "control admission exceeded the registered offline window") + entry = expected[0] + observed = recorder.probe.s3("get_object", {"Bucket": bucket, "Key": entry["key"]}) + save(recorder.output / f"cycle-{cycle}-degraded-control-get.json", observed) + require(observed.get("acknowledged") is True and observed.get("payload") == entry["payload"], "degraded flap control GET differs from ACK") + recorder.call("list_objects_v2", {"Bucket": bucket, "Prefix": f"flap/cycle-{cycle}/"}, admission_deadline=deadline) + with ThreadPoolExecutor(max_workers=PROTOCOL["foregroundConcurrency"]) as pool: + reads = pool.submit(control_read) + writes = [pool.submit(put, index) for index in range(count)] + rows = [future.result() for future in writes] + reads.result() + manifest, obligations = copy.deepcopy(expected), [] + for key, body, ack in rows: + manifest = put_ack(manifest, key, body, ack, "DB") + obligations.append({"bucket": bucket, "key": key, "versionId": "null", "poolIndex": 0, "setIndex": 0, + "kind": 3, "authoritativelyAbsentRequired": False}) + return manifest, obligations + + +def refuse_endpoint(binding): + try: + with request.build_opener(request.ProxyHandler({})).open(binding["endpoints"]["node4"] + "/health/ready", timeout=3): + raise ValueError("flapped node still accepts endpoint requests") + except URLError as exc: + require(isinstance(exc.reason, ConnectionRefusedError), "node flap has no exact connection refusal") + return {"connectionRefused": True} + + +class MemberFault: + def __init__(self, fleet, binding, target, drive, budget): + self.fleet, self.binding, self.target, self.drive, self.budget = fleet, binding, target, drive, budget + self.node = fleet.nodes[-1] + self.baseline = None + self.processes = None + + def inject(self): + self.processes = {node: self.fleet.remote(node, "process") for node in self.fleet.nodes} + self.fleet.record("flap-processes-before-loss", self.processes) + if self.drive: + self.fleet.drive_gates.add(self.node) + self.fleet.record("flap-live-gate-arm", self.fleet.remote(self.node, "drive-gate-arm-live", + disk=self.target["disk"], expected=self.processes[self.node])) + self.fleet.record("flap-live-gate-active", self.fleet.remote(self.node, "drive-gate-activate")) + else: + stopped = self.fleet.record("flap-owned-node-stop", self.fleet.remote(self.node, "stop")) + require(stopped.get("quiescent") is True and stopped.get("service", {}).get("MainPID") == "0", "flap target unit did not stop") + return self.witness() + + def witness(self): + self.budget.check() + if not self.drive: + current = self.fleet.record("flap-stopped-process", self.fleet.remote(self.node, "process")) + require(current["pid"] == 0 and current["bootId"] == self.processes[self.node]["bootId"], "flap target stopped witness changed") + return self.fleet.record("flap-endpoint-refusal", refuse_endpoint(self.binding)) + current = self.fleet.remote(self.node, "process") + same_process(self.processes[self.node], current) + denial = self.fleet.record("flap-service-drive-refusal", self.fleet.remote(self.node, "drive-gate-probe")) + require(denial.get("errno") == 13 and denial.get("readProbe", {}).get("errno") == 13 + and denial.get("dacBypassRemoved") is True and denial.get("disk") == self.target["disk"], + "drive flap lacks exact service read/write denial") + deadline = min(time.monotonic() + 90, self.budget.deadline) + while True: + observed = self.fleet.record("flap-admin-member-offline", Probe(self.binding).admin( + "/rustfs/admin/v3/info", [], b"", method="GET")) + try: + proof = member_state(observed, self.target, 16, self.baseline) + return self.fleet.record("flap-drive-double-witness", {"service": denial, "admin": proof}) + except ValueError as exc: + if not str(exc).startswith("NOT_OBSERVED:"): raise + require(time.monotonic() < deadline, str(exc)) + time.sleep(1) + + def release(self): + if self.drive: + observed = self.fleet.record("flap-live-drive-rejoin", self.fleet.remote(self.node, "drive-gate-release-live")) + require(observed.get("sameOriginalProcess") is True and observed.get("serviceRestarted") is False, + "drive flap rejoin restarted its service") + self.fleet.drive_gates.remove(self.node) + else: + self.fleet.record("flap-original-node-rejoin", self.fleet.remote(self.node, "start")) + self.fleet.wait_ready() + returned = {node: self.fleet.remote(node, "process") for node in self.fleet.nodes} + for node in self.fleet.nodes: + if node == self.node and not self.drive: node_return(self.processes[node], returned[node]) + else: same_process(self.processes[node], returned[node]) + self.fleet.record("flap-process-continuity", returned) + if self.drive: + deadline = min(time.monotonic() + 90, self.budget.deadline) + while True: + observed = self.fleet.record("flap-admin-member-online", Probe(self.binding).admin( + "/rustfs/admin/v3/info", [], b"", method="GET")) + try: + online = member_state(observed, self.target, 16) + require(online == self.baseline, "flap original member returned with a different identity") + break + except ValueError: + require(time.monotonic() < deadline, "NOT_OBSERVED: original flapped drive did not return online") + time.sleep(1) + + +def interrupt_rebuild(fleet, window, binding, bucket, obligations, bindings, authorities, members, fault, budget, after, all_obligations): + deadline = min(time.monotonic() + PROTOCOL["partialWindowSeconds"], budget.deadline) + while True: + budget.sample(fleet.nodes) + events = window.capture() + bounded_admissions(events, all_obligations) + scene = fleet.snapshot("flap-third-cycle-rebuild-window") + progress = fleet.record("flap-third-cycle-physical-progress", replay_progress(scene, authorities, bucket, obligations, members)) + pending = {identity(item) for item in obligations} - {tuple(item) for item in progress["completeACKs"]} + active = None + if progress["partial"]: + for item in obligations: + if identity(item) not in pending: continue + for owner in bindings[identity(item)]: + endpoint = "node" + str(fleet.nodes.index(owner["node"]) + 1) + starts = [row for row in events[owner["node"]] if row.get("target") == "rustfs::heal::task" + and row.get("message") == "Heal object started" and row.get("bucket") == bucket + and row.get("object") == item["key"] and "version_id" in row and source_version(row["version_id"], "null") + and event_time(row) is not None and event_time(row) >= after] + for row in reversed(starts): + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + quote(item["key"], safe="/") + observed = fleet.record("flap-interrupted-task-running", Probe(binding, endpoint).admin(path, + [("clientToken", row["task_id"])], b"")) + require(observed.get("httpStatus") == 200, "third-cycle original task lost its exact status") + if observed.get("response", {}).get("summary") == "running": + active = {"node": owner["node"], "taskId": row["task_id"], "ACK": list(identity(item))} + break + if active: break + if active: break + if active: + interrupted_at = time.monotonic() + fault.inject() + frozen = fleet.record("flap-third-cycle-offline-progress", replay_progress( + {node: fleet.remote(node, "snapshot", quiescent=node == fault.node) for node in fleet.nodes}, + authorities, bucket, obligations, members)) + require(frozen["partial"] and active["ACK"] not in frozen["completeACKs"], + "NOT_OBSERVED: actual rebuild completed before the third-cycle repeated loss") + pending = {identity(item) for item in obligations} - {tuple(item) for item in frozen["completeACKs"]} + retained = retained_pending(read_checkpoints(fleet, fleet.nodes[:-1], "flap-interrupted-durable-records"), + obligations, bindings, pending) + fault.witness() + fault.release() + require(time.monotonic() - interrupted_at <= PROTOCOL["faultWindowSeconds"], + "third-cycle repeated loss exceeded its registered window budget") + return {"runningOriginalTask": active, "physicalWindow": frozen, "durable": retained, + "extraInterruptionCount": 1, "fixedSleepUsedAsWindow": False} + require(time.monotonic() < deadline, "NOT_OBSERVED: no exact running MRF task with partial third-cycle physical reconstruction") + time.sleep(.1) + + +def run_branch(args, branch, nodes): + directory = args.evidence / branch + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-h2" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="flapping") + candidate = json.loads(args.candidate.read_text()) + tool = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + contract = verify_quorum_contract(tool, candidate) + fleet.record("candidate-durable-MRF-reference", verify_reference(candidate)) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, 5)}, "defaultEndpoint": "node1", + "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 128, "maxInventoryBytes": 256 * 1024**2} + fleet.record("binding", binding) + fleet.record("preregistered-three-cycle-protocol", {**PROTOCOL, "branch": branch, + "lossKind": "one-original-drive" if branch == "drive-flap-3" else "one-original-node", + "thirdCycleExtraInterruptionRequired": branch == "interrupt-rebuild-again"}) + probe, result = Probe(binding), {"topology": "H2", "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING", "cycles": []} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + recorder = Recorder(probe, directory / "foreground-ACK-ledger") + expected = seed(recorder, bucket, "DB", 4, [SIZE]) + fleet.record("original-S3-manifest", verify_seed(expected, probe.inventory(bucket, unversioned_current_tags=True))) + before = fleet.snapshot("original-physical-baseline") + fleet.record("topology-proof", validate_topology(before, "H2")) + require(all(not value["intents"] for value in read_checkpoints(fleet, nodes, "fresh-MRF-baseline").values()), + "fresh flapping service has preexisting durable responsibilities") + target = map_target(before, bucket, "objects/0001.bin", "data", 1) + target = {**target, **next(shard for shard in target["cohort"] if shard["node"] == nodes[-1])} + drive = branch == "drive-flap-3" + members = {(nodes[-1], target["disk"])} if drive else {(nodes[-1], disk) for disk in range(4)} + fleet.record("actual-loss-quorum-plan", drive_quorum(target, contract) if drive else node_quorum(target, nodes[-1], contract)) + budget = Budget(fleet, binding) + fault = MemberFault(fleet, binding, target, drive, budget) + if drive: + fault.baseline = member_state(fleet.record("flap-original-admin-member", probe.admin( + "/rustfs/admin/v3/info", [], b"", method="GET")), target, 16) + all_obligations, all_bindings = [], {} + window = JournalWindow(fleet, "flapping-source-and-retry-window", + event_targets={"rustfs::heal::manager", "rustfs::heal::task"}) + for cycle in range(1, PROTOCOL["cycles"] + 1): + budget.sample(nodes) + original = fleet.snapshot("cycle-original-physical") + started = time.monotonic() + offline = fault.inject() + online = nodes if drive else nodes[:-1] + budget.sample(online) + earliest_ms = int(time.time() * 1000) - 5000 + expected, obligations = foreground(recorder, bucket, cycle, expected, + min(started + PROTOCOL["faultWindowSeconds"], budget.deadline)) + all_obligations.extend(obligations) + deadline = min(time.monotonic() + PROTOCOL["persistenceSeconds"], budget.deadline) + while True: + try: + checkpoints = read_checkpoints(fleet, online, "flap-offline-committed-ACKs") + bindings = pending_bindings(checkpoints, obligations, earliest_ms) + break + except ValueError as exc: + if not str(exc).startswith("NOT_OBSERVED:"): raise + require(time.monotonic() < deadline, "flap durable persistence exceeded its budget: " + str(exc)) + time.sleep(.2) + all_bindings.update(bindings) + fleet.record("cycle-durable-ACK-bindings", [{"identity": list(key), "owners": value} for key, value in bindings.items()]) + fault.witness() + authorities = {node: fleet.remote(node, "snapshot", quiescent=not drive and node == nodes[-1]) for node in nodes} + fleet.record("cycle-offline-authorities", authorities) + fleet.record("cycle-original-control-proof", untouched(original, authorities, members)) + require(time.monotonic() - started <= PROTOCOL["faultWindowSeconds"], "flap offline window exceeded its registered budget") + returned = datetime.now(timezone.utc) + fault.release() + interruption = None + if branch == "interrupt-rebuild-again" and cycle == 3: + interruption = interrupt_rebuild(fleet, window, binding, bucket, obligations, bindings, authorities, + members, fault, budget, returned, all_obligations) + def observe(): + budget.sample(nodes) + bounded_admissions(window.capture(), all_obligations) + proofs, retired = wait_replay(fleet, window, binding, nodes, bucket, expected, obligations, bindings, + returned, original, authorities, [], observe=observe, deadline_limit=budget.deadline) + fleet.record("cycle-full-ACK-task-proofs", [{"identity": list(key), **value} for key, value in proofs.items()]) + reconstructed = fleet.snapshot("cycle-complete-physical-before-target-GET") + cohorts = replica_census(reconstructed, bucket, expected, authorities) + for cohort in cohorts: verify_cohort(fleet, cohort, "cycle-full-streaming-proof") + inventory = fleet.record("cycle-complete-S3-manifest", probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("cycle-ACK-content-proof", verify_seed(expected, inventory)) + result["cycles"].append({"cycle": cycle, "offlineWitness": offline, "uniqueACKs": len(obligations), + "sourceTasks": len(proofs), "retirement": retired, "interruption": interruption, + "sameOriginalDiskUUIDs": True, "physicalAndStreamingBeforeTargetGET": True}) + budget.sample(nodes) + fleet.stop() + reconstructed = fleet.snapshot("final-quiescent-physical", quiescent=True) + cohorts = replica_census(reconstructed, bucket, expected, before) + for cohort in cohorts: verify_cohort(fleet, cohort, "final-full-streaming-proof") + retired = retirement(read_checkpoints(fleet, nodes, "final-original-responsibility-retirement", quiescent=True), all_obligations, all_bindings) + admissions = bounded_admissions(window.capture(), all_obligations) + fleet.start() + for endpoint in binding["endpoints"]: + actual = fleet.record("final-full-peer-manifest", {"endpoint": endpoint, + **Probe(binding, endpoint).inventory(bucket, unversioned_current_tags=True)}) + fleet.record("final-peer-ACK-proof", {**verify_seed(expected, actual), **compare_inventories(inventory, actual)}) + fleet.stop() + require(physical_content(reconstructed) == physical_content(fleet.snapshot("final-after-S3-verification", quiescent=True)), + "target verification altered the frozen product reconstruction") + budget.check() + result.update(verdict="PASS", uniqueACKs=len(all_obligations), resourceSamples=budget.samples, + boundedAdmissions=admissions, durableRetirement=retired, fullPeerManifestEqual=True, + originalDiskUUIDsPreserved=True, adminCompensationTasks=0, backupRestorationUsed=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + if fleet.reserved: + try: + fleet.stop() + fleet.snapshot("flapping-failure-physical", quiescent=True) + read_checkpoints(fleet, nodes, "flapping-failure-durable-records", quiescent=True) + except Exception as evidence_error: + result["failureEvidenceError"] = str(evidence_error) + finally: + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("branch-result", result) + return result + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f"Heal workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ("candidate", "rc-bin", "evidence", "tool-directory"): + parser.add_argument("--" + name, type=Path, required=True) + for name in ("testing-commit", "binary-sha"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + args = parser.parse_args() + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(set(nodes)) == len(nodes) and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), + "four current Heal nodes required") + require(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha), "invalid candidate binary digest") + require(re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")), "workflow fleet identity required") + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "flapping executor omitted a source branch") + results = [] + try: + for branch in BRANCHES: + results.append(run_branch(args, branch, nodes[:4])) + if results[-1]["cleanup"] != "PASS": break + finally: + complete = len(results) == len(BRANCHES) and all(item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "complete": complete, + "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL"}) + require(complete, "B10 requires all three bounded H2 branches, actual third-cycle interruption and full cleanup") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"Heal flapping validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 602041b3..f0b9e89a 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -33,15 +33,19 @@ def runtime_config(profile): - require(profile in {"manual", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(profile in {"manual", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") if profile == "lock-quorum": return {**BACKGROUND_CONFIG, "RUSTFS_STORAGE_CLASS_STANDARD": "EC:8"} if profile == "scope-isolation": return {**BACKGROUND_CONFIG, "RUSTFS_ERASURE_SET_DRIVE_COUNT": "8"} - if profile in {"durable-mrf", "mrf-crash"}: - return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug", + if profile in {"durable-mrf", "mrf-crash", "flapping"}: + # Keep actual MRF admission/start/terminal evidence without enabling + # unrelated HTTP/HTTP2 frame debugging on the high-volume fleet. + logging_filter = "info,rustfs::heal::manager=debug,rustfs::heal::task=debug,rustfs::heal::mrf=debug" + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true", + "RUSTFS_OBS_LOGGER_LEVEL": logging_filter, "RUST_LOG": logging_filter, **({"RUSTFS_HEAL_MAX_CONCURRENT_HEALS": "1", "RUSTFS_HEAL_MAX_CONCURRENT_PER_SET": "1", - "RUSTFS_HEAL_MRF_REPLAY_BATCH": "1"} if profile == "mrf-crash" else {})} + "RUSTFS_HEAL_MRF_REPLAY_BATCH": "1"} if profile in {"mrf-crash", "flapping"} else {})} if profile in {"replacement", "replacement-target", "schema-refusal"}: return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_AUTO_HEAL_ENABLE": "true" if profile == "replacement" else "false", "RUSTFS_HEAL_INTERVAL_SECS": "10", "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} @@ -236,6 +240,8 @@ def start(request): "--property=KillMode=control-group", "--property=TimeoutStopSec=30", "--property=RuntimeMaxSec=" + str(resource_limits(owner.get("resourceProfile", "standard"))["runtimeMaxSeconds"]), "--property=MemoryMax=4G", + *(["--property=PrivateMounts=true", "--property=CapabilityBoundingSet=~CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH"] + if owner.get("runtimeProfile") == "flapping" else []), *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") in {"root-restart", "durable-mrf", "mrf-crash", "replacement", "replacement-target", "schema-refusal"} else []), "--property=WorkingDirectory=" + str(state), "--property=EnvironmentFile=" + str(state / "service.env"), @@ -550,6 +556,24 @@ def drive_gate_state(request): return gate_module().checked_gate(state, roots) or {"state": "CLEAR"} +def drive_gate_arm_live(request): + """Re-arm one owned root without changing the running service's birth.""" + require(set(request) == {"runId", "action", "disk", "expected"}, "live arm requires an exact owned process binding") + state, roots, owner = owned(request["runId"]) + require(owner.get("runtimeProfile") == "flapping", "live arm requires the preconfigured flapping service") + before = process(request) + require(before["pid"] > 1 and isinstance(request["expected"], dict) + and all(before.get(name) == request["expected"].get(name) for name in PROCESS_FIELDS), + "live drive gate crossed the expected process birth") + module = gate_module() + module.validate_live_boundary(before["pid"]) + require(quorum_gate_module().checked(state, roots) is None, "live drive gate cannot share a quorum gate") + receipt = module.apply_gate(state, roots, request["disk"], "permission-denied") + after = process(request) + require(all(before.get(name) == after.get(name) for name in PROCESS_FIELDS), "live arm crossed its owned process identity") + return {**receipt, "processBefore": before, "processAfter": after, "sameOriginalProcess": True} + + def drive_gate_release_live(request): state, roots, _ = owned(request["runId"]) gate = gate_module().checked_gate(state, roots) @@ -942,7 +966,7 @@ def main(request): "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part, "verify-parts": verify_parts, "verify-inline": verify_inline, "drive-gate": drive_gate, "drive-gate-probe": drive_gate_probe, "drive-gate-activate": drive_gate_activate, "gate-state": drive_gate_state, - "drive-gate-release-live": drive_gate_release_live} + "drive-gate-release-live": drive_gate_release_live, "drive-gate-arm-live": drive_gate_arm_live} actions["task-records"] = task_records for action in ("bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover"): actions[action] = bucket_record_operation diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index c1e7f551..4ceeb519 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"manual", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -200,10 +200,10 @@ def remote(self, node, action, **params): source += "\nimport types\nencryption_runtime = types.ModuleType('encryption_runtime')\nexec(" + repr( NODE_SOURCE.with_name("encryption_runtime.py").read_text()) + ", encryption_runtime.__dict__)\n" replacement_actions = {"start", "cleanup", "audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot", "format-schema-fault"} - if action in {"start", "cleanup", "audit-finalize", "drive-gate", "replace-drive", "quorum-gate", "quorum-gate-probe"}: + if action in {"start", "cleanup", "audit-finalize", "drive-gate", "drive-gate-arm-live", "replace-drive", "quorum-gate", "quorum-gate-probe"}: source += "\nimport types\nquorum_gate_backend = types.ModuleType('quorum_gate')\nexec(" + repr( NODE_SOURCE.with_name("quorum_gate.py").read_text()) + ", quorum_gate_backend.__dict__)\n" - if action in {"drive-gate", "drive-gate-probe", "drive-gate-activate", "gate-state", "drive-gate-release-live"} | replacement_actions: + if action in {"drive-gate", "drive-gate-arm-live", "drive-gate-probe", "drive-gate-activate", "gate-state", "drive-gate-release-live"} | replacement_actions: source += "\nimport types\ngate = types.ModuleType('drive_gate')\nexec(" + repr( NODE_SOURCE.with_name("drive_gate.py").read_text()) + ", gate.__dict__)\n" if action in replacement_actions: diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index b9ccca09..d7be34e5 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/mrf_records.py b/scripts/validation_heal/mrf_records.py index e89a9308..3b620e1c 100644 --- a/scripts/validation_heal/mrf_records.py +++ b/scripts/validation_heal/mrf_records.py @@ -21,7 +21,8 @@ "crates/heal/src/heal/mrf_queue/partial_write.rs": "20415e0e985daea8fc62c5e152ed438e75008c33a2a592275e1dd9d1d4fee771", "crates/common/src/mrf_channel.rs": "075bb84bcbbe7d76674f730c64c8b69fe37d0790ec75da46365ba2e0cab34e0f", "crates/heal/src/heal/manager/scheduler.rs": "79ce1f9c278c664231b6a1eca7c521f8e98742fe6d17d8f852368da4e77cc33b", - "crates/heal/src/heal/task/heal_object.rs": "17bb72783b6fd58afdc0b81c24558c1f5aff867fa3fb6ee27bd7f264076d8f6e"}, + "crates/heal/src/heal/task/heal_object.rs": "17bb72783b6fd58afdc0b81c24558c1f5aff867fa3fb6ee27bd7f264076d8f6e", + "crates/obs/src/telemetry/filter.rs": "f46372640433f231f89e3c3f092a7d9292f1e870b386685b3595238cfc9847d6"}, "journal": "complete CRC32/ISO-HDLC records; format 1, layouts 1/2/3", "commit": "RFMRFC01 version 1; owner, sequence, payload length and two SHA-256 digests", "lifecycle": "RFMFLC01 version 1; checksummed JSON, same committed owner and sequence", @@ -49,7 +50,7 @@ def verify_reference(candidate): def read_records(state, roots, owner, checked): """Exact allowlist, no directory enumeration, no write descriptor.""" - require(owner.get("runtimeProfile") in {"durable-mrf", "mrf-crash"}, "MRF records require the dedicated run-owned runtime") + require(owner.get("runtimeProfile") in {"durable-mrf", "mrf-crash", "flapping"}, "MRF records require the dedicated run-owned runtime") require(hashlib.sha256((state / "rustfs").read_bytes()).hexdigest() == owner["binarySha256"], "MRF candidate changed") def collect(): diff --git a/scripts/validation_heal/mrf_replay.py b/scripts/validation_heal/mrf_replay.py index e0d1f86d..75dcc34a 100644 --- a/scripts/validation_heal/mrf_replay.py +++ b/scripts/validation_heal/mrf_replay.py @@ -207,9 +207,11 @@ def deleted_parts_absent(snapshot, targets): def wait_replay(fleet, window, binding, nodes, bucket, expected, obligations, bindings, rejoin_time, - before, authorities, deleted_targets): + before, authorities, deleted_targets, *, observe=None, deadline_limit=None): deadline, proofs = time.monotonic() + REPLAY_SECONDS, {} + if deadline_limit is not None: deadline = min(deadline, deadline_limit) while True: + if observe is not None: observe() events = window.capture() for obligation in obligations: key = identity(obligation) @@ -308,7 +310,7 @@ def run_branch(args, branch, nodes, *, after_rejoin=None, put_count=3, fixture=" require(physical_content({offline_node: frozen}) == physical_content({offline_node: still_offline}), "MRF ACKs reached the stopped original node") authorities = fleet.record("offline-physical-authority", {node: fleet.remote(node, "snapshot", quiescent=node == offline_node) for node in nodes}) deleted_targets = [map_target(before, bucket, entry["key"], "data", 1, entry["versionId"], latest_required=False) for entry in deleted] - window = JournalWindow(fleet, "MRF-rejoin") + window = JournalWindow(fleet, "MRF-rejoin", event_targets={"rustfs::heal::manager", "rustfs::heal::task"}) rejoin_time = datetime.now(timezone.utc) fleet.record("rejoin-original-node", fleet.remote(offline_node, "start")) fleet.wait_ready() diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 829072ea..60f87379 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -36,6 +36,7 @@ case "${case_id}" in heal.foreign-format-and-unknown-schema) driver=schema.py ;; heal.encrypted-objects) driver=encrypted.py ;; heal.coordinator-process-crash) driver=coordinator_crash.py ;; + heal.disk-and-node-flapping) driver=flapping.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal_flapping.py b/test_validation_heal_flapping.py new file mode 100644 index 00000000..069655a1 --- /dev/null +++ b/test_validation_heal_flapping.py @@ -0,0 +1,235 @@ +import copy +from pathlib import Path +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import flapping, fleet_node, drive_gate +from scripts.validation_heal.matrix import AUTOMATIC_CASES, select +from scripts.validation_heal.mrf_replay import identity +import test_validation_heal_coordinator_crash as crash_fixtures +import test_validation_heal_mrf as mrf_fixtures + + +class FlappingTests(unittest.TestCase): + def scene(self): + scene, authorities, bucket, obligations = crash_fixtures.CoordinatorCrashTests().mrf_scene() + for value in (scene, authorities): + for node in value.values(): + for row in node["files"]: + for version in row.get("metadata", {}).get("versions", []): + version["versionId"] = "null" + for item in obligations: item["versionId"] = "null" + return scene, authorities, bucket, obligations + + def test_node_partial_replay_requires_real_original_members_and_full_ack_roles(self): + scene, authorities, bucket, obligations = self.scene() + members = {("node4", disk) for disk in range(4)} + proof = flapping.replay_progress(scene, authorities, bucket, obligations, members) + self.assertTrue(proof["partial"]) + self.assertFalse(proof["finalStreamingProofClaimed"]) + for kind in ("role", "version", "authority", "part", "uuid", "member"): + bad, original = copy.deepcopy(scene), copy.deepcopy(authorities) + metadata = next(row for row in bad["node4"]["files"] if row.get("metadata"))["metadata"] + version = metadata["versions"][0] + selected = members + if kind == "role": version["erasure"]["index"] = 1 + elif kind == "version": version["versionId"] = "other" + elif kind == "authority": next(row for row in original["node1"]["files"] if row.get("metadata"))["metadata"]["versions"][0]["size"] += 1 + elif kind == "part": next(row for row in bad["node4"]["files"] if not row.get("metadata"))["sha256"] = "missing" + elif kind == "uuid": bad["node4"]["formats"][0]["value"]["xl"]["this"] = "foreign" + else: selected = {("node5", disk) for disk in range(4)} + with self.subTest(kind=kind), self.assertRaises(ValueError): + flapping.replay_progress(bad, original, bucket, obligations, selected) + + def test_single_drive_progress_uses_fifteen_surviving_authorities(self): + scene, authorities, bucket, obligations = self.scene() + full = copy.deepcopy(scene["node4"]["files"]) + for row in copy.deepcopy(full): + row["path"] = row["path"].replace("new-00", "new-01") + if row.get("metadata"): row["metadata"]["key"] = row["metadata"]["key"].replace("new-00", "new-01") + full.append(row) + authorities["node4"]["files"] = [row for row in full if row["disk"] != 3] + scene["node4"]["files"] = [row for row in full if row["disk"] != 3 or "new-00" in row["path"]] + proof = flapping.replay_progress(scene, authorities, bucket, obligations, {("node4", 3)}) + self.assertTrue(proof["partial"]) + self.assertEqual(proof["pendingACKs"], 1) + authorities["node4"]["files"] = [row for row in authorities["node4"]["files"] + if not (row.get("metadata") and "new-01" in row["path"] and row["disk"] == 0)] + with self.assertRaises(ValueError): + flapping.replay_progress(scene, authorities, bucket, obligations, {("node4", 3)}) + + def test_zero_and_complete_reconstruction_cannot_certify_an_interruption_window(self): + scene, authorities, bucket, obligations = self.scene() + members = {("node4", disk) for disk in range(4)} + empty = copy.deepcopy(scene); empty["node4"]["files"] = [] + self.assertFalse(flapping.replay_progress(empty, authorities, bucket, obligations, members)["partial"]) + for row in copy.deepcopy(scene["node4"]["files"]): + row["path"] = row["path"].replace("new-00", "new-01") + if row.get("metadata"): row["metadata"]["key"] = row["metadata"]["key"].replace("new-00", "new-01") + scene["node4"]["files"].append(row) + self.assertFalse(flapping.replay_progress(scene, authorities, bucket, obligations, members)["partial"]) + + def test_incomplete_ack_cannot_retire_rebind_or_rollback_during_repeated_loss(self): + fixture = mrf_fixtures.DurableMrfTests() + record = fixture.observed() + from scripts.validation_heal.mrf_records import checkpoint + value, item = checkpoint(record), fixture.obligation() + lifecycle = value["intents"][0]["lifecycle"] + owner = {"node": "node1", "checkpointOwner": value["owner"], "sequence": value["sequence"], + "responsibilityId": lifecycle["responsibility_id"], "bucketIncarnationId": lifecycle["source_bucket_incarnation_id"]} + wanted = identity(item) + bindings = {wanted: [owner]} + self.assertEqual(flapping.retained_pending({"node1": value}, [item], bindings, {wanted})["pendingOriginalResponsibilities"], 1) + for kind in ("absent", "retired", "owner", "sequence", "responsibility", "incarnation", "accepted", "held"): + bad = copy.deepcopy(value) + if kind == "absent": bad["present"] = False + elif kind == "retired": bad["intents"] = [] + elif kind == "owner": bad["owner"] = "other" + elif kind == "sequence": bad["sequence"] -= 1 + elif kind == "responsibility": bad["intents"][0]["lifecycle"]["responsibility_id"] = "other" + elif kind == "incarnation": bad["intents"][0]["lifecycle"]["source_bucket_incarnation_id"] = "other" + elif kind == "accepted": bad["intents"][0]["lifecycle"]["last_operator_acceptance"] = {"accepted": True} + else: bad["intents"][0]["lifecycle"]["state"] = {"state": "held"} + with self.subTest(kind=kind), self.assertRaises(ValueError): + flapping.retained_pending({"node1": bad}, [item], bindings, {wanted}) + + def events(self): + events, obligations, _, _ = crash_fixtures.CoordinatorCrashTests().events() + return events, obligations + + def test_task_storm_and_cross_ack_token_reuse_fail_without_counting_duplicates(self): + events, obligations = self.events() + proof = flapping.bounded_admissions(events, obligations) + self.assertEqual(proof["distinctAcceptedTaskIds"], 1) + events["node1"].append(copy.deepcopy(events["node1"][-1])) + self.assertEqual(flapping.bounded_admissions(events, obligations)["distinctAcceptedTaskIds"], 1) + for kind in ("version", "source", "key", "owner", "storm"): + bad = copy.deepcopy(events) + if kind == "version": bad["node1"][-1]["version_id"] = 'Some("other")' + elif kind == "source": bad["node1"][0]["source"] = "scanner" + elif kind == "key": bad["node1"][-1]["object"] = "foreign" + elif kind == "owner": bad["node2"] = copy.deepcopy(bad["node1"]) + else: + pair = bad["node1"][:2] + for index in range(flapping.PROTOCOL["maxAcceptedTasksPerACKPerOwner"]): + admission, start = copy.deepcopy(pair) + admission["request_id"] = start["task_id"] = f"token-{index}" + bad["node1"].extend((admission, start)) + with self.subTest(kind=kind), self.assertRaises(ValueError): flapping.bounded_admissions(bad, obligations) + + def test_internal_system_mrf_tasks_remain_bounded_and_cannot_discharge_owned_acks(self): + events, obligations = self.events() + admission, start = copy.deepcopy(events["node1"][:2]) + admission["request_id"] = start["task_id"] = "system-token" + start["bucket"], start["object"] = ".rustfs.sys", "scanner/durable-dirty-producer-replay/owned.json" + events["node1"].extend((admission, start)) + proof = flapping.bounded_admissions(events, obligations) + self.assertEqual(proof["internalSystemTasks"], 1) + self.assertFalse(proof["internalSystemTasksCertifyOwnedACKs"]) + for index in range(flapping.PROTOCOL["maxSystemTasksPerOwner"]): + admission, start = copy.deepcopy((admission, start)) + admission["request_id"] = start["task_id"] = f"system-token-{index}" + start["object"] = f"scanner/durable-dirty-producer-replay/{index}.json" + events["node1"].extend((admission, start)) + with self.assertRaises(ValueError): flapping.bounded_admissions(events, obligations) + + def process(self, pid=10): + return {name: {"pid": pid, "startTicks": pid * 100}.get(name, name) for name in fleet_node.PROCESS_FIELDS} + + def test_drive_rearm_does_not_accept_recycled_process_or_dac_bypass(self): + before = self.process() + owner = {"runtimeProfile": "flapping"} + calls = [] + gate = types.SimpleNamespace(validate_live_boundary=lambda pid: calls.append(("boundary", pid)), + apply_gate=lambda state, roots, disk, mode: calls.append(("arm", disk, mode)) or {"state": "ARMED"}) + quorum = types.SimpleNamespace(checked=lambda state, roots: None) + request = {"runId": "owned", "action": "drive-gate-arm-live", "disk": 1, "expected": before} + with patch.object(fleet_node, "owned", return_value=(Path("/owned"), [Path("/disk")], owner)), \ + patch.object(fleet_node, "process", return_value=before), \ + patch.object(fleet_node, "gate_module", return_value=gate), \ + patch.object(fleet_node, "quorum_gate_module", return_value=quorum): + self.assertTrue(fleet_node.drive_gate_arm_live(request)["sameOriginalProcess"]) + self.assertEqual(calls, [("boundary", 10), ("arm", 1, "permission-denied")]) + calls.clear() + with self.assertRaises(ValueError): + fleet_node.drive_gate_arm_live({**request, "expected": self.process(11)}) + self.assertEqual(calls, []) + with patch.object(gate, "validate_live_boundary", side_effect=ValueError("DAC bypass")): + with self.assertRaises(ValueError): fleet_node.drive_gate_arm_live(request) + self.assertEqual(calls, []) + + def test_three_drive_gate_cycles_preserve_recovery_intent_and_original_inode(self): + with tempfile.TemporaryDirectory() as folder: + state, root = Path(folder) / "state", Path(folder) / "root" + state.mkdir(); root.mkdir(); data = root / "data"; data.mkdir(mode=0o700) + inode = data.stat().st_ino + original = Path.lstat + def rooted(path): + observed = original(path) + return types.SimpleNamespace(st_mode=observed.st_mode, st_uid=0, st_nlink=observed.st_nlink, + st_size=observed.st_size, st_dev=observed.st_dev, st_ino=observed.st_ino) + for cycle in range(3): + # Only ownership is simulated. The actual inode, recovery + # marker and three permission changes persist on the filesystem. + with patch.object(Path, "lstat", rooted): + drive_gate.apply_gate(state, [root], 0, "permission-denied") + drive_gate.activate_gate(state, [root]) + self.assertEqual(data.stat().st_mode & 0o777, 0) + self.assertTrue((state / "drive-gate.json").exists()) + drive_gate.release_gate(state, [root]) + self.assertEqual(data.stat().st_ino, inode) + self.assertEqual(data.stat().st_mode & 0o777, 0o700) + self.assertFalse((state / "drive-gate.json").exists()) + + def test_original_node_return_requires_new_birth_without_host_candidate_or_settings_change(self): + before, after = self.process(10), self.process(11) + flapping.node_return(before, after) + for name in ("pid", "bootId", "binarySha256", "environmentFileSha256", "settings"): + bad = copy.deepcopy(after) + if name == "pid": bad = copy.deepcopy(before) + else: bad[name] = "changed" + with self.subTest(name=name), self.assertRaises(ValueError): flapping.node_return(before, bad) + with self.assertRaises(ValueError): flapping.same_process(before, after) + + def test_bounded_resources_reject_fd_rss_cpu_queue_and_total_deadline(self): + resource = {"readOnly": True, "fileDescriptors": 10, "rssBytes": 100, "identity": self.process(), + "monotonicNs": 2 * 10**9, "clockTicksPerSecond": 100, "cpuTicks": 100, "clockDomain": "bootId"} + binding = {"endpoints": {"node1": "http://node1"}} + status = {"httpStatus": 200, "response": {"healActiveTasks": 0, "healQueueLength": 0}} + for kind in ("good", "fd", "rss", "cpu", "queue", "deadline", "peer"): + value, result = copy.deepcopy(resource), copy.deepcopy(status) + if kind == "fd": value["fileDescriptors"] = flapping.PROTOCOL["maxFDs"] + 1 + elif kind == "rss": value["rssBytes"] = flapping.PROTOCOL["maxRssBytes"] + 1 + elif kind == "cpu": value["cpuTicks"] = 100000 + elif kind == "queue": result["response"]["healQueueLength"] = flapping.PROTOCOL["maxQueuedTasks"] + 1 + elif kind == "peer": value["identity"] = self.process(11) + fleet = types.SimpleNamespace(nodes=["node1", "node4"], record=lambda label, value: value, + remote=lambda *args, **kwargs: value) + with patch.object(flapping, "Probe", return_value=types.SimpleNamespace(admin=lambda *args, **kwargs: result)): + budget = flapping.Budget(fleet, binding) + previous = copy.deepcopy(resource); previous["monotonicNs"] = 10**9; previous["cpuTicks"] = 90 + budget.previous["node1"] = previous + if kind == "deadline": budget.deadline = 0 + with self.subTest(kind=kind): + if kind == "good": budget.sample(["node1"]) + else: + with self.assertRaises(ValueError): budget.sample(["node1"]) + + def test_all_source_branches_are_dispatched_with_isolated_runtime_and_bounded_protocol(self): + self.assertEqual(select(flapping.CASE_ID)[0]["variants"], list(flapping.BRANCHES)) + self.assertIn(flapping.CASE_ID, AUTOMATIC_CASES) + config = fleet_node.runtime_config("flapping") + self.assertEqual(config["RUSTFS_HEAL_MRF_ENABLE"], "true") + self.assertEqual(config["RUSTFS_HEAL_MAX_CONCURRENT_HEALS"], "1") + self.assertEqual(config["RUSTFS_HEAL_MRF_REPLAY_BATCH"], "1") + self.assertEqual(config["RUSTFS_SCANNER_ENABLED"], "false") + self.assertEqual(config["RUSTFS_HEAL_AUTO_HEAL_ENABLE"], "false") + self.assertEqual(flapping.PROTOCOL["cycles"], 3) + self.assertEqual(len(flapping.PROTOCOL["uniqueACKsPerCycle"]), 3) + self.assertLess(sum(flapping.PROTOCOL["uniqueACKsPerCycle"]), 128) + + +if __name__ == "__main__": + unittest.main() diff --git a/test_validation_heal_mrf.py b/test_validation_heal_mrf.py index 75d5b507..1d43afd6 100644 --- a/test_validation_heal_mrf.py +++ b/test_validation_heal_mrf.py @@ -175,7 +175,9 @@ def test_reader_never_enumerates_or_opens_credential_records(self): secret = roots[0] / "data" / ".rustfs.sys" / "credentials.json"; secret.write_text("private") owner = {"runtimeProfile": "durable-mrf", "binarySha256": hashlib.sha256(b"candidate").hexdigest()} with patch("scripts.validation_heal.mrf_records.os.walk", side_effect=AssertionError("no directory traversal")): - result = records.read_records(state, roots, owner, lambda path, boundary: path) + for profile in ("durable-mrf", "mrf-crash", "flapping"): + result = records.read_records(state, roots, {**owner, "runtimeProfile": profile}, lambda path, boundary: path) + self.assertTrue(result["stable"]) self.assertTrue(result["stable"]) self.assertTrue(records.checkpoint(result)["allAllowlistedFilesAbsent"]) self.assertNotIn("private", json.dumps(result)) @@ -190,9 +192,30 @@ def test_candidate_contract_and_runtime_keep_other_repair_sources_disabled(self) self.assertEqual(config["RUSTFS_HEAL_MRF_ENABLE"], "true") self.assertEqual(config["RUSTFS_SCANNER_ENABLED"], "false") self.assertEqual(config["RUSTFS_HEAL_AUTO_HEAL_ENABLE"], "false") + self.assertEqual(config["RUST_LOG"], config["RUSTFS_OBS_LOGGER_LEVEL"]) + self.assertTrue(config["RUST_LOG"].startswith("info,")) + for target in ("manager", "task", "mrf"): + self.assertIn("rustfs::heal::" + target + "=debug", config["RUST_LOG"]) self.assertEqual(select("E05")[0]["variants"], list(replay.BRANCHES)) self.assertIn(replay.CASE_ID, AUTOMATIC_CASES) + def test_source_filter_keeps_complete_raw_pages_and_exact_heal_events(self): + noise = {"target": "h2::codec::framed_read", "message": "frame"} + start = {"target": "rustfs::heal::task", "message": "Heal object started", "task_id": self.responsibility} + pages = [] + def remote(node, action, **params): + if params.get("checkpoint"): + return {"complete": True, "afterCursor": "baseline"} + value = {"complete": True, "beforeCursor": params["afterCursor"], "afterCursor": "next", "hasMore": False, + "journal": "\n".join([json.dumps(noise)] * 20 + [json.dumps(start)])} + return value + fleet = types.SimpleNamespace(nodes=["node1"], remote=remote, + record=lambda label, value: pages.append(value) or value) + window = replay.JournalWindow(fleet, "MRF-rejoin", event_targets={"rustfs::heal::manager", "rustfs::heal::task"}) + self.assertEqual(window.capture()["node1"], [start]) + self.assertEqual(len(pages[-1]["journal"].splitlines()), 21) + self.assertIn("h2::codec::framed_read", pages[-1]["journal"]) + def test_replay_observer_uses_real_journal_interface_and_requires_exact_task_before_retirement(self): obligation = self.obligation() bindings = replay.pending_bindings({"node1": records.checkpoint(self.observed())}, [obligation], 1) From 6caa9f4e9e203d90b6bbcc39ded20a76334bd557 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 08:09:20 +0800 Subject: [PATCH 095/343] test: migrate bounded capacity faults and quota recovery --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 59 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/bounded_volume.py | 302 +++++++++++++++ scripts/validation_heal/capacity.py | 360 ++++++++++++++++++ scripts/validation_heal/fleet_node.py | 68 +++- scripts/validation_heal/heal_probe.py | 16 +- scripts/validation_heal/healthy.py | 11 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 1 + test_validation_heal_capacity.py | 200 ++++++++++ 11 files changed, 1016 insertions(+), 9 deletions(-) create mode 100644 scripts/validation_heal/bounded_volume.py create mode 100644 scripts/validation_heal/capacity.py create mode 100644 test_validation_heal_capacity.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 1f632159..afde2afd 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -27,6 +27,7 @@ on: - test_validation_heal_write_quorum.py - test_validation_heal_coordinator_crash.py - test_validation_heal_flapping.py + - test_validation_heal_capacity.py - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py @@ -70,6 +71,7 @@ on: - test_validation_heal_write_quorum.py - test_validation_heal_coordinator_crash.py - test_validation_heal_flapping.py + - test_validation_heal_capacity.py - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py @@ -104,7 +106,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index fc029f90..2fa8a646 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 127 currently dispatchable IDs include all 74 non-Heal cases and fifty-three Heal cases. Forty-six have live PASS evidence; the other 112 have no live PASS claim. +case evidence. The 128 currently dispatchable IDs include all 74 non-Heal cases and fifty-four Heal cases. Forty-six have live PASS evidence; the other 112 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1282,3 +1282,60 @@ only Heal manager/task/MRF debug over an info baseline, attest both pages while selecting the required attribution targets in memory. The 60,000-event/page budgets and exact durable responsibility assertions stay in force. All three E05 branches still require a fresh complete live run. + + +I02 `heal.enospc-inodes-and-quota` now executes all three source branches +separately on H1 and H2. The two capacity branches allocate exactly one +128 MiB ext4 image with at most 1,024 inodes on each original independent +backing drive. Images, filesystem UUIDs, mounts and undo paths are registered +before allocation; only these bounded regular files are formatted. Each +owned service binds the allocated data roots in a private mount namespace. +The bounded volumes do not add physical drives to the H1/H2 topology. + +Block or inode exhaustion fills only the dedicated padding directory and +must produce actual ENOSPC in both the filler and a probe with the original +candidate process birth, mount namespace and effective UID/GID. The original +shared drives must retain at least 1 GiB of free space. Missing DB and completed +MPU parts remain unresolved during the exhausted interval; all other business +parts and metadata remain frozen. Removing only this run's filler permits +independent manual repair, exact responsibility reconciliation, same-role +streaming bitrot verification and complete all-peer S3/MPU ACK checks. + +The hard-quota branch uses only the owned bucket and the candidate's reviewed +1 MiB and 256 MiB limits. It requires the exact InvalidRequest quota refusal +for a single new logical PUT, preserves existing object/MPU ACKs and verifies +existing-object shard repair under the quota. Quota is logical object +admission, so a successful repair of an existing shard is not a new logical +PUT. Both capacity modes also require a distinct, acknowledged PUT after +release. Every uncertain negative or released-capacity PUT is resolved across +all peers without retries; unresolved receipts or recovery drift retain a +DIRTY namespace. File-backup rescue never earns product repair credit. + +The prior B10 commit omitted its top-level dispatcher arm even though the +Heal sub-dispatcher was present. The arm is corrected together with I02. A +catalog-wide automatic-dispatch contract now checks that all 54 registered +Heal IDs reach candidate preflight through the top-level entry. The B10 live +queue has not started and must use the corrected immutable harness revision. +This change leaves 30 Heal executors to implement and makes no new canonical +live PASS claim. The focused suite has 310 tests with five platform skips; +the catalog still preserves all 158 exact source IDs and hashes, and the +84/301/532 source matrix is unchanged. + +F05 `38095216675` and J06 `38095795125` retain FAIL/cleanup PASS at the old +harness revision because Admin evidence exceeded its previous response +budget. J06 completed only the first no-heal phase, which cannot certify an +ABBA group. The queued fresh F05/J06 runs use the reviewed 8 MiB response +contract. D11 `38096389824` also retains FAIL/cleanup PASS for all eight +H1/H2 instances: versioning/lock configuration byte restoration did not match +the frozen original; policy/tag record decoding failed outside its witnessed +mutation; unknown-record startup did not become ready. No D11 PASS or +successful unknown-record refusal is claimed. + +Native tools-only run `38096433036` binds driver +`ae9b5cbabfeaff440d9b8ee0b2fe805f17aff5b3` and auto-testing +`acdd69218406b0596d79b6aab15481f8271e35c6`. Its candidate resolver and tool/native +jobs use hosted ubuntu-latest with independent tools-only concurrency and +empty fleet credentials. The metadata inspector build passed; the native +candidate test executable remains in compilation. This exact audited hosted +run is excluded from the local fleet-busy classifier. Unknown active runs +still block dispatch, and native results cannot count as canonical fleet PASS. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index bc2ae905..a4d91693 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/bounded_volume.py b/scripts/validation_heal/bounded_volume.py new file mode 100644 index 00000000..ded9e705 --- /dev/null +++ b/scripts/validation_heal/bounded_volume.py @@ -0,0 +1,302 @@ +"""I02: fixed-size filesystems bound only into the run-owned service's roots.""" + +import errno +import json +import os +from pathlib import Path +import re +import shutil +import stat +import subprocess +import time +import uuid + + +MARKER = "bounded-volumes.json" +IMAGE_BYTES = 128 * 1024**2 +MAX_INODES = 1024 +MIN_SHARED_FREE = 1024**3 +PROBE = '''import errno,json,os,sys +folder=sys.argv[1] +info=os.statvfs(folder) +result={"uid":os.geteuid(),"gid":os.getegid(),"device":os.stat(folder).st_dev, + "freeBlocks":info.f_bfree,"availableBlocks":info.f_bavail,"freeInodes":info.f_ffree} +name=folder+"/.i02-write-witness" +fd=None +try: + fd=os.open(name,os.O_CREAT|os.O_EXCL|os.O_WRONLY|os.O_NOFOLLOW,0o600) + os.write(fd,b"i02"*2048) + os.fsync(fd) +except OSError as exc: + result["writeErrno"]=exc.errno +else: + result["writeErrno"]=0 +finally: + if fd is not None: + os.close(fd) + os.unlink(name) +print(json.dumps(result)) +''' + + +def require(value, message): + if not value: + raise ValueError(message) + + +def command(*args): + result = subprocess.run(args, capture_output=True, text=True, timeout=90) + require(result.returncode == 0, f"bounded-volume {args[0]} failed: {result.stderr[-800:]}") + return result.stdout + + +def regular(path, *, size=None): + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_uid == 0 and info.st_nlink == 1 + and stat.S_IMODE(info.st_mode) == 0o600 and (size is None or info.st_size == size), + "bounded-volume file ownership/type/size differs") + return info + + +def read(state, roots, owner): + path = state / MARKER + if not path.exists(): + return None + require(regular(path).st_size < 16384, "bounded-volume marker exceeds budget") + value = json.loads(path.read_text()) + require(value.get("schemaVersion") == 1 and value.get("runId") == owner["runId"] + and value.get("branch") in {"bounded-volume-enospc", "bounded-inode-exhaustion"} + and value.get("imageBytes") == IMAGE_BYTES and value.get("maxInodes") == MAX_INODES + and len(value.get("volumes", [])) == len(roots) == 4, + "bounded-volume contract differs from its original run") + for index, row in enumerate(value["volumes"]): + require(type(row.get("disk")) is int and row["disk"] == index and row.get("image") == str(roots[index] / "bounded-volume.img") + and row.get("mount") == str(state / "bounded" / f"disk{index}") + and str(uuid.UUID(row["uuid"])) == row["uuid"], "bounded-volume paths or filesystem identities differ") + folder = Path(row["mount"]) + require(not any(path.is_symlink() for path in (folder, *folder.parents)), "symlink in bounded mount boundary") + return value + + +def loop_for(image): + rows = json.loads(command("losetup", "--json", "--list", "--output", "NAME,BACK-FILE,BACK-INO,BACK-MAJ:MIN"))["loopdevices"] + selected = [row for row in rows if row["back-file"] == str(image)] + if selected: + info = regular(image) + require(all(int(row["back-ino"]) == info.st_ino + and row["back-maj:min"] == f"{os.major(info.st_dev)}:{os.minor(info.st_dev)}" for row in selected), + "loop attachment belongs to a different backing inode/device") + found = [row["name"] for row in selected] + require(len(found) <= 1 and all(re.fullmatch(r"/dev/loop[0-9]+", item) for item in found), + "bounded image has ambiguous loop attachments") + return found[0] if found else None + + +def mounted(row): + observed = subprocess.run(["findmnt", "--json", "--mountpoint", row["mount"], "--output", + "SOURCE,FSTYPE,UUID,TARGET,MAJ:MIN,OPTIONS"], capture_output=True, text=True, timeout=15) + if observed.returncode == 1: + return None + require(observed.returncode == 0, "bounded mount identity unavailable") + entries = json.loads(observed.stdout).get("filesystems", []) + require(len(entries) == 1, "bounded mount is ambiguous") + entry = entries[0] + require(entry["target"] == row["mount"] and entry["fstype"] == "ext4" and entry["uuid"] == row["uuid"] + and entry["source"] == loop_for(Path(row["image"])) and "rw" in entry["options"].split(","), + "bounded mount no longer belongs to its exact image/UUID") + return entry + + +def create(state, roots, owner, branch): + require(owner.get("runtimeProfile") == "bounded-capacity" and read(state, roots, owner) is None + and branch in {"bounded-volume-enospc", "bounded-inode-exhaustion"}, "invalid bounded capacity allocation") + require(all(not any((root / "data").iterdir()) and not (root / "bounded-volume.img").exists() + and shutil.disk_usage(root).free >= MIN_SHARED_FREE + IMAGE_BYTES for root in roots), + "bounded allocation requires empty owned roots and shared-drive safety margin") + value = {"schemaVersion": 1, "runId": owner["runId"], "branch": branch, "imageBytes": IMAGE_BYTES, + "maxInodes": MAX_INODES, "volumes": [{"disk": index, "image": str(root / "bounded-volume.img"), + "mount": str(state / "bounded" / f"disk{index}"), "uuid": str(uuid.uuid4())} + for index, root in enumerate(roots)]} + # The complete undo identity exists before allocating an image or loop. + fd = os.open(state / MARKER, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(value, stream) + stream.flush() + os.fsync(stream.fileno()) + (state / "bounded").mkdir(mode=0o700) + for row in value["volumes"]: + image = Path(row["image"]) + descriptor = os.open(image, os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_NOFOLLOW, 0o600) + try: + os.posix_fallocate(descriptor, 0, IMAGE_BYTES) + os.fsync(descriptor) + finally: + os.close(descriptor) + regular(image, size=IMAGE_BYTES) + # mkfs receives an exclusively created bounded regular file, never a + # supplied device. The subsequently attached loop is derived from it. + command("mkfs.ext4", "-F", "-q", "-m", "0", "-N", str(MAX_INODES), "-U", row["uuid"], str(image)) + loop = command("losetup", "--find", "--show", "--nooverlap", str(image)).strip() + require(loop == loop_for(image), "new loop identity differs") + folder = Path(row["mount"]) + folder.mkdir(mode=0o700) + command("mount", "-t", "ext4", "-o", "nodev,nosuid,noexec", loop, str(folder)) + mounted(row) + folder.chmod(0o700) + for name in ("data", "quarantine", "padding"): + (folder / name).mkdir(mode=0o700) + info = os.statvfs(folder) + require(info.f_blocks * info.f_frsize <= IMAGE_BYTES and 0 < info.f_files <= MAX_INODES, + "actual bounded filesystem exceeds registered block/inode limits") + return {**value, "hostSharedMountRemounted": False, "allocatedBytesPerNode": len(roots) * IMAGE_BYTES, + "independentPhysicalDriveCountIncreased": False} + + +def views(state, roots, owner): + value = read(state, roots, owner) + if value is None: + require(owner.get("runtimeProfile") != "bounded-capacity", "bounded runtime has no durable allocation intent") + return roots + result = [] + for original, row in zip(roots, value["volumes"], strict=True): + require(not any((original / "data").iterdir()), "candidate wrote outside its private bounded binding") + regular(Path(row["image"]), size=IMAGE_BYTES) + require(mounted(row) is not None, "bounded filesystem is not mounted") + folder = Path(row["mount"]) + require(folder.stat().st_uid == 0 and stat.S_IMODE(folder.stat().st_mode) == 0o700, + "bounded filesystem root is not private/root-owned") + for name in ("data", "quarantine", "padding"): + path = folder / name + info = path.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and not path.is_symlink() + and info.st_dev == folder.stat().st_dev and stat.S_IMODE(info.st_mode) == 0o700, + "bounded data/quarantine/padding boundary differs") + result.append(folder) + return tuple(result) + + +def properties(state, roots, owner): + if read(state, roots, owner) is None: + require(owner.get("runtimeProfile") != "bounded-capacity", "bounded runtime is not configured") + return [] + actual = views(state, roots, owner) + return ["--property=PrivateMounts=true", "--property=BindPaths=" + " ".join( + f"{view / 'data'}:{root / 'data'}" for view, root in zip(actual, roots, strict=True))] + + +def fill(folder, branch): + padding = folder / "padding" + require(not any(padding.iterdir()), "bounded filler is already present") + deadline, written, count = time.monotonic() + 60, 0, 0 + if branch == "bounded-volume-enospc": + fd = os.open(padding / "blocks.bin", os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + try: + while True: + require(time.monotonic() < deadline and written < IMAGE_BYTES, "bounded block fill exceeded its budget") + try: + written += os.write(fd, b"\0" * min(1024**2, IMAGE_BYTES - written)) + os.fsync(fd) + except OSError as error: + require(error.errno == errno.ENOSPC, "bounded fill reached an unrelated IO error") + break + finally: + os.close(fd) + else: + require(branch == "bounded-inode-exhaustion", "unreviewed bounded fill mode") + while True: + require(count < MAX_INODES and time.monotonic() < deadline, "bounded inode fill exceeded its budget") + try: + fd = os.open(padding / f"inode-{count:04d}", os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + except OSError as error: + require(error.errno == errno.ENOSPC, "bounded inode fill reached an unrelated IO error") + break + os.fsync(fd) + os.close(fd) + count += 1 + info = os.statvfs(folder) + require(info.f_bfree == 0 if branch == "bounded-volume-enospc" else info.f_ffree == 0, + "bounded fill did not exhaust its registered resource") + return {"allocatedFillerBytes": written, "allocatedFillerInodes": count, "actualErrno": errno.ENOSPC, + "freeBlocks": info.f_bfree, "freeInodes": info.f_ffree} + + +def release_filler(folder): + padding = folder / "padding" + entries = list(padding.iterdir()) + require(len(entries) <= MAX_INODES and all(path.name == "blocks.bin" or re.fullmatch(r"inode-[0-9]{4}", path.name) + for path in entries), "unrecognized file in bounded filler; retain namespace") + for path in entries: + require(regular(path).st_size <= IMAGE_BYTES, "bounded filler grew outside its limit") + path.unlink() + descriptor = os.open(padding, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + info = os.statvfs(folder) + require(info.f_bfree > 0 and info.f_ffree > 0, "bounded resource was not released") + return {"removedOwnedFillerFiles": len(entries), "freeBlocks": info.f_bfree, "freeInodes": info.f_ffree} + + +def witness(state, roots, owner, pid, *, exhausted): + actual = views(state, roots, owner) + namespace = Path(f"/proc/{pid}/ns/mnt") + require(pid > 1 and namespace.readlink() != Path('/proc/self/ns/mnt').readlink(), + "bounded witness lacks a private candidate mount namespace") + fields = dict(line.split(":", 1) for line in Path(f"/proc/{pid}/status").read_text().splitlines() if ":" in line) + require(list(map(int, fields["Uid"].split()))[1] == list(map(int, fields["Gid"].split()))[1] == 0, + "bounded witness service UID/GID changed") + branch = read(state, roots, owner)["branch"] + rows = [] + for disk, (root, view) in enumerate(zip(roots, actual, strict=True)): + raw = json.loads(command("nsenter", "--mount=" + str(namespace), "python3", "-c", PROBE, str(root / "data"))) + validate_probe(raw, view.stat().st_dev, branch, exhausted) + rows.append({"disk": disk, **raw, "sameServiceUidAndMountNamespace": True}) + require(all(shutil.disk_usage(root).free >= MIN_SHARED_FREE for root in roots), + "shared backing drive crossed its retained safety margin") + return {"branch": branch, "disks": rows, "hostSharedMountRemounted": False, + "realMediaDamageClaimed": False, "sharedDriveSafetyMarginBytes": MIN_SHARED_FREE} + + +def validate_probe(raw, device, branch, exhausted): + require(branch in {"bounded-volume-enospc", "bounded-inode-exhaustion"} and type(exhausted) is bool + and all(type(raw.get(name)) is int and raw[name] >= 0 for name in + ("uid", "gid", "device", "writeErrno", "freeBlocks", "freeInodes", "availableBlocks")), + "bounded probe omitted a real integer resource/identity value") + require(raw["uid"] == raw["gid"] == 0 and raw["device"] == device + and raw["writeErrno"] == (errno.ENOSPC if exhausted else 0), + "candidate mount/UID did not witness the exact bounded write error") + if exhausted: + require(raw["freeBlocks"] == 0 if branch == "bounded-volume-enospc" else raw["freeInodes"] == 0, + "candidate resource view differs from the exhausted filesystem") + else: + require(raw["freeBlocks"] > 0 and raw["freeInodes"] > 0, "released candidate resource remains exhausted") + + +def dispose(state, roots, owner): + value = read(state, roots, owner) + if value is None: + return {"boundedVolumesAbsent": True} + for row in value["volumes"]: + image, folder = Path(row["image"]), Path(row["mount"]) + if image.exists(): + info = regular(image) + require(info.st_size in {0, IMAGE_BYTES}, "partially allocated image has unrecognized size") + entry = mounted(row) + if entry is not None: + command("umount", str(folder)) + require(mounted(row) is None, "bounded filesystem remains mounted") + loop = loop_for(image) + if loop: + command("losetup", "--detach", loop) + require(loop_for(image) is None, "bounded loop remains attached") + image.unlink() + else: + require(not folder.exists() or not os.path.ismount(folder), "missing image has a retained mount") + if folder.exists(): + require(not folder.is_symlink() and not any(folder.iterdir()), "bounded mountpoint is not empty after detach") + folder.rmdir() + if (state / "bounded").exists(): + (state / "bounded").rmdir() + (state / MARKER).unlink() + return {"boundedVolumesRemoved": True, "originalSharedMountsRetained": True} diff --git a/scripts/validation_heal/capacity.py b/scripts/validation_heal/capacity.py new file mode 100644 index 00000000..8528b9bd --- /dev/null +++ b/scripts/validation_heal/capacity.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 +"""I02: actual bounded ENOSPC/inode exhaustion and owned hard-quota recovery.""" + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys + +try: + from .heal_probe import Probe, compare_inventories, payload, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .shards import (map_target, run_task, validate_fault, validate_repaired_physical, + verify_cohort, pending_inventory, verify_ranges) + from .versions import validate_version_receipts, verify_quorum_contract + from .drive_gate import validate_unwritable + from .stop import idle_fleet + from .write_quorum import resolve_put + from .bounded_volume import IMAGE_BYTES, MAX_INODES, MIN_SHARED_FREE +except ImportError: + from heal_probe import Probe, compare_inventories, payload, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from shards import (map_target, run_task, validate_fault, validate_repaired_physical, + verify_cohort, pending_inventory, verify_ranges) + from versions import validate_version_receipts, verify_quorum_contract + from drive_gate import validate_unwritable + from stop import idle_fleet + from write_quorum import resolve_put + from bounded_volume import IMAGE_BYTES, MAX_INODES, MIN_SHARED_FREE + + +CASE_ID = "heal.enospc-inodes-and-quota" +BRANCHES = ("bounded-volume-enospc", "bounded-inode-exhaustion", "bucket-quota") +SIZE = 5 * 1024**2 + 97 +CONTRACT = {"candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", + "sourceFiles": {"rustfs/src/admin/handlers/quota.rs": + "6a2b062d3d1c4cd81d934c570d21cb44dcd2c1883f6a85d2265e01b466924f5c", + "rustfs/src/error.rs": "656d38972c06339015c174405571a94f19d8dd64a680e1a7ca631f80205f99a7"}, + "quota": {"blockedNewLogicalPUT": {"httpStatus": 400, "code": "InvalidRequest", + "message": "Bucket quota exceeded."}, "limitBytes": 1024**2, "releasedLimitBytes": 256 * 1024**2, + "existingObjectShardRepair": "does not reserve a new S3 object or increase logical bucket usage"}, + "volume": {"imageBytesPerOriginalDisk": IMAGE_BYTES, "maximumInodesPerImage": MAX_INODES, + "sharedDriveFreeMarginBytes": MIN_SHARED_FREE, "loopVolumesIncreasePhysicalTopologyCount": False}, + "load": {"DBObjects": 8, "completedMPUObjects": 1, "pendingMPUs": 2, + "uniqueNegativePUTs": 1, "uniqueReleasedCapacityPUTs": 1, "uncertainRequestRetries": 0, "fillDeadlineSecondsPerDisk": 60}} + + +def refusal(observed, *, quota): + require(observed.get("acknowledged") is False and observed.get("clientCallInvoked") is True, + "capacity-negative PUT succeeded or was never invoked") + if observed.get("resultUnknown") is True: + return {"state": "UNKNOWN_PENDING_RESOLUTION", "retryAllowed": False} + response = observed.get("response") or {} + error = response.get("Error") or {} + if quota: + require(response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 400 + and error.get("Code") == "InvalidRequest" and error.get("Message", "").startswith("Bucket quota exceeded."), + "quota-negative PUT did not return the reviewed actual hard-quota refusal") + else: + require(response.get("ResponseMetadata", {}).get("HTTPStatusCode") in {500, 503} + and error.get("Code") in {"ServiceUnavailable", "XMinioWriteQuorum", "InternalError"}, + "capacity-negative PUT lacks a reviewed structured storage refusal") + require(observed.get("resultUnknown") is False, "capacity PUT is neither refused nor explicitly unresolved") + return {"state": "REFUSED", "code": error["Code"], "retryAllowed": False} + + +def quota_config(probe, fleet, bucket, limit): + path = "/rustfs/admin/v3/quota/" + bucket + body = json.dumps({"quota": limit, "quota_type": "HARD"}).encode() + configured = fleet.record("owned-bucket-quota-PUT", probe.admin(path, [], body, method="PUT")) + require(configured.get("httpStatus") == 200 and configured.get("response", {}).get("bucket") == bucket + and configured["response"].get("quota") == limit and configured["response"].get("quota_type") == "HARD", + "hard quota was not acknowledged on the exact owned bucket") + for endpoint in probe.binding["endpoints"]: + observed = fleet.record("all-peer-owned-quota-GET", Probe(probe.binding, endpoint).admin(path, [], b"", method="GET")) + require(observed.get("httpStatus") == 200 and observed.get("response", {}).get("bucket") == bucket + and observed["response"].get("quota") == limit and observed["response"].get("quota_type") == "HARD", + "peer did not observe the exact acknowledged quota limit") + return {"bucket": bucket, "limitBytes": limit, "allPeersVerified": True, "sharedFilesystemQuotaChanged": False} + + +def verify_s3(fleet, binding, bucket, expected, baseline, pending): + result = {} + for endpoint in binding["endpoints"]: + probe = Probe(binding, endpoint) + inventory = fleet.record("all-peer-final-complete-S3", {"endpoint": endpoint, + **probe.inventory(bucket, unversioned_current_tags=True)}) + proof = {**verify_seed(expected, inventory), **compare_inventories(baseline, inventory)} + require(fleet.record("all-peer-pending-MPU", pending_inventory(probe, bucket)) == pending, + "capacity failure changed an acknowledged pending MPU part or UploadId") + proof["ranges"] = [verify_ranges(probe, fleet, bucket, entry["key"], expected_body=payload( + f"{bucket}/{entry['key']}/0", entry["payload"]["size"])) for entry in expected] + result[endpoint] = proof + return result + + +def run_branch(args, topology, nodes, branch): + directory = args.evidence / topology / branch + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + bounded = branch != "bucket-quota" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, + runtime_profile="bounded-capacity" if bounded else "manual") + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == CONTRACT["candidateCommit"], "unreviewed I02 error/quota contract") + manifest = fleet.record("candidate-bound-inspector", verify_inspector(args.tool_directory, candidate)) + verify_quorum_contract(manifest, candidate) + fleet.record("registered-capacity-contract", CONTRACT) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 128, "maxInventoryBytes": 512 * 1024**2} + fleet.record("binding", binding) + probe, active, allocated = Probe(binding), [], [] + result = {"topology": topology, "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} + attempted, negative_body, negative_key = None, None, "capacity-negative/unique.bin" + released_attempted, released_body, released_key = None, None, "capacity-released/different-nonce.bin" + quota_set = False + fill_initiated = False + + def allocate(): + for node in nodes: + allocated.append(node) + fleet.record("fixed-owned-volume-allocation", {"node": node, **fleet.remote( + node, "bounded-volumes-create", branch=branch)}) + + def release(): + nonlocal quota_set + if bounded: + for node in allocated: + identity = fleet.remote(node, "process") + if identity["pid"]: + fleet.record("release-only-owned-filler", {"node": node, **fleet.remote( + node, "bounded-volumes-release", expected=identity)}) + else: + fleet.record("release-only-owned-filler", {"node": node, **fleet.remote( + node, "bounded-volumes-release", quiescent=True)}) + elif quota_set: + quota_config(probe, fleet, bucket, CONTRACT["quota"]["releasedLimitBytes"]) + quota_set = False + + try: + fleet.deploy(args.binary_sha, bucket, access, secret, before_start=allocate if bounded else None) + recorder = Recorder(probe, directory / "fixture") + expected = seed(recorder, bucket, "DB", 8, [SIZE]) + expected += seed(recorder, bucket, "DM", 1, [SIZE], existing_bucket=True) + pending = fleet.record("independent-pending-MPU-ACKs", pending_inventory(probe, bucket)) + baseline = fleet.record("complete-S3-baseline", probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("all-fixture-ACKs", {"expected": expected, **verify_seed(expected, baseline)}) + fleet.stop() + original = fleet.snapshot("original-complete-physical", quiescent=True) + fleet.record("actual-independent-topology", validate_topology(original, topology)) + targets = [map_target(original, bucket, expected[index]["key"], "data", 1) for index in (0, 8)] + for target in targets: + fleet.record("original-target-streaming-proof", verify_cohort(fleet, target, "original-target")) + for index, target in enumerate(targets): + params = {"disk": target["disk"], "faultId": f"I02-target-{index}"} + fleet.faults.append((target["node"], params)) + receipt = fleet.record("exact-missing-part", fleet.remote(target["node"], "apply-fault", **params, + relativePath=target["relativePath"], kind="missing", expectedSha256=target["sha256"])) + scene = fleet.snapshot("exact-missing-part-witness", quiescent=True) + prior = original if index == 0 else faulted + fleet.record("missing-part-proof", validate_fault(prior, scene, target, receipt)) + faulted = scene + fleet.start() + require(physical_content(faulted) == physical_content(fleet.snapshot("manual-source-isolation-before-fault")), + "capacity target changed before the registered capacity fault") + if bounded: + fill_initiated = True + for node in nodes: + identity = fleet.remote(node, "process") + fleet.record("actual-resource-exhaustion", {"node": node, **fleet.remote( + node, "bounded-volumes-fill", expected=identity)}) + else: + quota_set = True + result["quota"] = quota_config(probe, fleet, bucket, CONTRACT["quota"]["limitBytes"]) + negative_body = payload(f"{bucket}/{negative_key}/0", SIZE) + # This unique call has durable request and ACK files even when the + # expected refusal causes Recorder to reject it. No retry is issued. + negative = Recorder(probe, directory / "negative-PUT") + attempted = {"acknowledged": False, "resultUnknown": True, "clientCallInvoked": True, + "reason": "durable request must be resolved if its ACK cannot be read"} + try: + negative.call("put_object", {"Bucket": bucket, "Key": negative_key}, negative_body) + except ValueError: + pass + ack = json.loads((negative.output / "000001-ack.json").read_text()) + attempted = ack["observation"] + result["negativePUT"] = refusal(attempted, quota=not bounded) + if not bounded: + require(result["negativePUT"]["state"] == "REFUSED", "quota dependency failed without its exact quota refusal") + result["negativeTasks"] = [] + for entry in (expected[0], expected[8]): + current = run_task(probe, fleet, {"bucket": bucket, "key": entry["key"]}, 2, active) + proof = validate_unwritable(current, bucket, entry) if bounded else validate_version_receipts( + current, bucket, [entry], 2, required_repairs={(entry["key"], "null")}) + result["negativeTasks"].append(proof) + if bounded: + for node in nodes: + identity = fleet.remote(node, "process") + fleet.record("resource-still-exhausted-after-Heal", {"node": node, **fleet.remote( + node, "bounded-volumes-witness", expected=identity)}) + fleet.stop() + negative_scene = fleet.snapshot("negative-complete-physical-before-release", quiescent=True) + if bounded: + require(physical_content(faulted) == physical_content(negative_scene), + "unwritable Heal or PUT changed a healthy source, metadata or the missing target") + else: + result["quotaPhysical"] = validate_repaired_physical(original, negative_scene, targets[0], repair_targets=targets) + for target in targets: + fleet.record("quota-existing-object-streaming-proof", verify_cohort(fleet, target, "quota-repaired")) + for node in nodes: + fleet.record("negative-owned-service-diagnostics", {"node": node, **fleet.remote(node, "logs")}) + if bounded: + release() + fleet.start() + else: + fleet.start() + release() + require(physical_content(negative_scene) == physical_content(fleet.snapshot("capacity-return-before-recovery-Heal")), + "capacity return altered files before the independent recovery task") + result["recoveryTasks"] = [] + for entry in (expected[0], expected[8]): + current = run_task(probe, fleet, {"bucket": bucket, "key": entry["key"]}, 2, active) + result["recoveryTasks"].append(validate_version_receipts(current, bucket, [entry], 2, + required_repairs={(entry["key"], "null")} if bounded else ())) + idle_fleet(fleet, binding) + fleet.stop() + repaired = fleet.snapshot("complete-product-repaired-before-S3", quiescent=True) + result["physical"] = validate_repaired_physical(original, repaired, targets[0], repair_targets=targets) + for target in targets: + fleet.record("product-reconstructed-streaming-proof", verify_cohort(fleet, target, "all-repaired-targets")) + fleet.faults.clear() + fleet.start() + reads = [fleet.record("single-negative-PUT-final-resolution", Probe(binding, endpoint).s3("get_object", { + "Bucket": bucket, "Key": negative_key})) for endpoint in binding["endpoints"]] + result["putResolution"] = resolve_put(attempted, reads, {"size": len(negative_body), + "sha256": hashlib.sha256(negative_body).hexdigest()}) + require(result["putResolution"]["state"] == "ABSENT", "capacity-refused unique PUT committed unexpectedly") + result["s3"] = verify_s3(fleet, binding, bucket, expected, baseline, pending) + fleet.stop() + require(physical_content(repaired) == physical_content(fleet.snapshot("physical-after-full-S3", quiescent=True)), + "S3 verification changed the frozen product-repaired scene") + fleet.start() + released_body = payload(f"{bucket}/{released_key}/0", SIZE) + released_payload = {"size": len(released_body), "sha256": hashlib.sha256(released_body).hexdigest()} + positive = Recorder(probe, directory / "capacity-released-PUT") + released_attempted = {"acknowledged": False, "resultUnknown": True, "clientCallInvoked": True} + try: + positive.call("put_object", {"Bucket": bucket, "Key": released_key}, released_body) + finally: + ack_path = positive.output / "000001-ack.json" + if ack_path.exists(): + released_attempted = json.loads(ack_path.read_text())["observation"] + released_reads = [] + for endpoint in binding["endpoints"]: + observed = fleet.record("released-capacity-success-ACK-GET", Probe(binding, endpoint).s3( + "get_object", {"Bucket": bucket, "Key": released_key})) + require(observed.get("acknowledged") is True and observed.get("resultUnknown") is False + and observed.get("payload") == released_payload, "released capacity PUT ACK cannot be redeemed") + released_reads.append(observed) + result["releasedPUTResolution"] = resolve_put(released_attempted, released_reads, released_payload) + result["releasedCapacityPUT"] = {"differentUniqueKey": released_key, "allPeersRedeemed": True, + "originalNegativePUTRetried": False} + result["verdict"] = "PASS" + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + try: + fleet.stop() + fleet.snapshot("physical-original-failure-before-any-rescue", quiescent=True) + except Exception as evidence_error: + result["failureEvidenceError"] = str(evidence_error) + finally: + for path, token in active: + try: fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: fleet.record("stop-unresolved-error", {"error": str(exc)}) + recovery_failed = False + try: + if fill_initiated or quota_set: + release() + if attempted and "putResolution" not in result: + if fleet.remote(nodes[0], "process")["pid"] == 0: + fleet.start() + reads = [fleet.record("failed-case-PUT-resolution", Probe(binding, endpoint).s3("get_object", { + "Bucket": bucket, "Key": negative_key})) for endpoint in binding["endpoints"]] + result["putResolution"] = resolve_put(attempted, reads, {"size": len(negative_body), + "sha256": hashlib.sha256(negative_body).hexdigest()}) + if released_attempted and "releasedPUTResolution" not in result: + if fleet.remote(nodes[0], "process")["pid"] == 0: + fleet.start() + reads = [fleet.record("failed-case-released-PUT-resolution", Probe(binding, endpoint).s3("get_object", { + "Bucket": bucket, "Key": released_key})) for endpoint in binding["endpoints"]] + result["releasedPUTResolution"] = resolve_put(released_attempted, reads, {"size": len(released_body), + "sha256": hashlib.sha256(released_body).hexdigest()}) + except Exception as exc: + recovery_failed = True + result["recoveryError"] = str(exc) + if recovery_failed: + result["cleanup"] = "FAIL" + fleet.record("retained-unresolved-capacity-environment", {"state": "DIRTY", "namespacesRetained": True, + "productRepairClaimed": False, "failureVerdictPreserved": True}) + else: + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("branch-result", result) + return result + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f"I02 workflow interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ("candidate", "rc-bin", "evidence", "tool-directory"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--case-id", choices=[CASE_ID], required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + args = parser.parse_args() + require(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha) and re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")), + "immutable workflow candidate identity required") + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "I02 source branch omitted") + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(nodes) == len(set(nodes)) and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", n) for n in nodes), + "four current Heal nodes required") + results = [] + try: + for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): + for branch in BRANCHES: + results.append(run_branch(args, topology, selected, branch)) + if results[-1]["cleanup"] != "PASS": break + if results[-1]["cleanup"] != "PASS": break + finally: + complete = len(results) == 6 and all(row["verdict"] == "PASS" and row["cleanup"] == "PASS" for row in results) + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "complete": complete, + "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL"}) + require(complete, "I02 requires all real H1/H2 capacity faults, complete ACK/physical proofs and clean finalization") + + +if __name__ == "__main__": + try: main() + except Exception as exc: + print(f"I02 capacity validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index f0b9e89a..5c0caab5 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -33,7 +33,9 @@ def runtime_config(profile): - require(profile in {"manual", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(profile in {"manual", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + if profile == "bounded-capacity": + return {**BACKGROUND_CONFIG, "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true"} if profile == "lock-quorum": return {**BACKGROUND_CONFIG, "RUSTFS_STORAGE_CLASS_STANDARD": "EC:8"} if profile == "scope-isolation": @@ -247,6 +249,7 @@ def start(request): "--property=EnvironmentFile=" + str(state / "service.env"), *gate_module().start_properties(gate_module().checked_gate(state, roots)), *quorum_gate_module().properties(state, roots), + *(volume_module().properties(state, roots, owner) if owner.get("runtimeProfile") == "bounded-capacity" else []), *replacement_module().start_properties(state, roots), str(state / "rustfs"), *owner["volumes"]) return {"unit": owner["unit"], "started": True} @@ -436,6 +439,53 @@ def quorum_gate_module(): return quorum_gate +def volume_module(): + if "bounded_volume_backend" in globals(): return globals()["bounded_volume_backend"] + from . import bounded_volume + return bounded_volume + + +def volume_views(state, roots, owner): + return volume_module().views(state, roots, owner) if owner.get("runtimeProfile") == "bounded-capacity" else roots + + +def volume_operation(request): + state, roots, owner = owned(request["runId"]) + require(owner.get("runtimeProfile") == "bounded-capacity", "bounded operation requires its exact dedicated runtime") + module = volume_module() + if request["action"] == "bounded-volumes-create": + require(set(request) == {"runId", "action", "branch"}, "bounded allocation accepts no supplied path or limits") + stopped(owner) + return module.create(state, roots, owner, request["branch"]) + if request["action"] == "bounded-volumes-release" and request.get("quiescent") is True: + require(set(request) == {"runId", "action", "quiescent"}, "bounded recovery accepts no supplied path") + stopped(owner) + changed = [module.release_filler(view) for view in module.views(state, roots, owner)] + return {"changes": changed, "ownedFillerReleased": True, "productRepairClaimed": False, + "quiescent": True, "privateMountRemovedByStoppedService": True} + require(set(request) == {"runId", "action", "expected"}, "bounded operation lacks exact process binding") + before = process(request) + require(before["pid"] > 1 and isinstance(request["expected"], dict) + and all(before.get(name) == request["expected"].get(name) for name in PROCESS_FIELDS), + "bounded operation crossed its expected original process birth") + views = module.views(state, roots, owner) + if request["action"] == "bounded-volumes-fill": + branch = module.read(state, roots, owner)["branch"] + changed = [module.fill(view, branch) for view in views] + observed = module.witness(state, roots, owner, before["pid"], exhausted=True) + elif request["action"] == "bounded-volumes-release": + changed = [module.release_filler(view) for view in views] + observed = module.witness(state, roots, owner, before["pid"], exhausted=False) + else: + require(request["action"] == "bounded-volumes-witness", "unsupported bounded operation") + changed = [] + observed = module.witness(state, roots, owner, before["pid"], exhausted=True) + after = process(request) + require(all(before.get(name) == after.get(name) for name in PROCESS_FIELDS), + "bounded operation crossed a candidate process identity") + return {**observed, "changes": changed, "processBefore": before, "processAfter": after, "sameOriginalProcess": True} + + def quorum_gate_operation(request): state, roots, owner = owned(request["runId"]) module = quorum_gate_module() @@ -590,8 +640,9 @@ def drive_gate_release_live(request): def fault_root(request): - _, roots, owner = owned(request["runId"]) + state, roots, owner = owned(request["runId"]) stopped(owner) + roots = volume_views(state, roots, owner) index = request.get("disk") require(type(index) is int and 0 <= index < len(roots), "invalid physical disk index") return roots[index], owner @@ -622,6 +673,7 @@ def recover_fault(request): def verify_part(request): state, roots, owner = owned(request["runId"]) stopped(owner) + roots = volume_views(state, roots, owner) require(owner.get("inspectorSha256"), "candidate-bound bitrot verifier is required") tool = checked(state / "heal-meta-inspect", state) require(digest(tool) == owner["inspectorSha256"], "bitrot verifier changed") @@ -652,6 +704,7 @@ def verify_part_owned(request, roots, owner, tool): def verify_parts(request): state, roots, owner = owned(request["runId"]) stopped(owner) + roots = volume_views(state, roots, owner) entries = request.get("parts") require(isinstance(entries, list) and 1 <= len(entries) <= 128 and all(isinstance(entry, dict) and set(entry) == {"disk", "relativePath", "versionId", "partNumber"} @@ -716,6 +769,7 @@ def snapshot_buckets(owner, request): def snapshot(request): state, roots, owner = owned(request["runId"]) + roots = volume_views(state, roots, owner) pid = int(command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"], check=False).stdout.strip() or "0") if request.get("quiescent") is True: stopped(owner) @@ -771,7 +825,10 @@ def snapshot(request): key: environment.get(key) for key in ("RUSTFS_STORAGE_CLASS_INLINE_BLOCK", "RUSTFS_STORAGE_CLASS_STANDARD")}, "resourceLimits": limits, "files": sorted(files, key=lambda value: (value["disk"], value["path"])), "formats": formats, - "encryptionConfigAttestation": encryption} + "encryptionConfigAttestation": encryption, + "boundedVolumeViews": None if owner.get("runtimeProfile") != "bounded-capacity" else { + "actualBackingRoots": [str(root) for root in roots], + "privateServiceBindingsRequired": True, "originalPhysicalDriveCountIncreased": False}} def logs(request): @@ -859,9 +916,12 @@ def cleanup(request): command("systemctl", "stop", owner["unit"], check=False) pid = command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"], check=False).stdout.strip() require(pid in {"", "0"}, "run-owned service remains active") + stopped(owner) gate_module().release_gate(state, roots) quorum_gate_module().release(state, roots) replacement_module().attest(state, roots) + if owner.get("runtimeProfile") == "bounded-capacity": + volume_module().dispose(state, roots, owner) command("systemctl", "reset-failed", owner["unit"], check=False) for root in (*roots, state): for directory, children, names in os.walk(root, followlinks=False): @@ -972,6 +1032,8 @@ def main(request): actions[action] = bucket_record_operation for action in ("quorum-gate", "quorum-gate-probe"): actions[action] = quorum_gate_operation + for action in ("bounded-volumes-create", "bounded-volumes-fill", "bounded-volumes-release", "bounded-volumes-witness"): + actions[action] = volume_operation actions["format-schema-fault"] = format_schema_fault actions["encryption-configure"] = configure_encryption actions["replacement-records"] = replacement_records diff --git a/scripts/validation_heal/heal_probe.py b/scripts/validation_heal/heal_probe.py index 159f7f70..9f0a99a3 100644 --- a/scripts/validation_heal/heal_probe.py +++ b/scripts/validation_heal/heal_probe.py @@ -258,12 +258,26 @@ def admin(self, path: str, query: list[tuple[str, str]], body: bytes, from heal_signing import sign_admin_request with observation_phase(observation, 'requestPreparation'): validate_binding(self.binding) - require(path.startswith("/rustfs/admin/v3/heal/") or path in { + quota_path = path.startswith("/rustfs/admin/v3/quota/") + require(path.startswith("/rustfs/admin/v3/heal/") or quota_path or path in { "/rustfs/admin/v3/background-heal/status", "/rustfs/admin/v4/heal/replacement-recovery", "/rustfs/admin/v3/info"}, "only bounded Heal evidence routes are allowed") if path == "/rustfs/admin/v3/info": require(method == "GET" and not query and body == b"" and auth == "signed", "server-info evidence requires an exact signed read-only GET") + if quota_path: + bucket = path.removeprefix("/rustfs/admin/v3/quota/") + bucket_allowed(self.binding, bucket) + require(method in {"GET", "PUT"} and not query and auth == "signed", + "owned quota evidence requires an exact signed GET/PUT") + if method == "GET": + require(body == b"", "quota GET cannot carry a mutation body") + else: + quota = json.loads(body) + require(isinstance(quota, dict) and set(quota) == {"quota", "quota_type"} + and quota["quota_type"] == "HARD" and type(quota["quota"]) is int + and quota["quota"] in {1024**2, 256 * 1024**2}, + "quota mutation is outside the two registered I02 limits") require("?" not in path and "#" not in path and len(body) <= 2 * 1024**2, "invalid Admin path or body budget") require(not any(part in {".", ".."} for part in unquote(path).split("/")), diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 4ceeb519..8852ad6e 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"manual", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -180,6 +180,9 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() + if self.runtime_profile == "bounded-capacity": + source += "\nimport types\nbounded_volume_backend = types.ModuleType('bounded_volume')\nexec(" + repr( + NODE_SOURCE.with_name("bounded_volume.py").read_text()) + ", bounded_volume_backend.__dict__)\n" if action == "mrf-records": source += "\nimport types\nmrf = types.ModuleType('mrf_records')\nexec(" + repr( NODE_SOURCE.with_name("mrf_records.py").read_text()) + ", mrf.__dict__)\n" @@ -211,6 +214,8 @@ def remote(self, node, action, **params): NODE_SOURCE.with_name("replace_drive.py").read_text()) + ", replacement.__dict__)\n" source += "\nmain(" + repr(value) + ")\n" timeout = 600 if self.resource_profile in {"cursor", "full-restart"} and action == "snapshot" else 180 + if action in {"bounded-volumes-create", "bounded-volumes-fill"}: + timeout = 600 result = subprocess.run([*self.ssh(node), "sudo -n python3 -"], input=source, capture_output=True, text=True, timeout=timeout) require(result.returncode == 0, f"Heal node {node} {action} failed: {result.stderr[-1200:]}") @@ -327,6 +332,10 @@ def cleanup(self): if (self.faults or self.bucket_record_faults) and not failures: try: self.stop() + if self.runtime_profile == "bounded-capacity": + for node in self.reserved: + self.record("bounded-filler-recovery", {"node": node, **self.remote( + node, "bounded-volumes-release", quiescent=True)}) for node, params in self.faults: self.record("fault-recovery", {"node": node, **self.remote(node, "recover-fault", **params)}) for node, params in self.bucket_record_faults: diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index d7be34e5..16fc0d2a 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 60f87379..498c727d 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -37,6 +37,7 @@ case "${case_id}" in heal.encrypted-objects) driver=encrypted.py ;; heal.coordinator-process-crash) driver=coordinator_crash.py ;; heal.disk-and-node-flapping) driver=flapping.py ;; + heal.enospc-inodes-and-quota) driver=capacity.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal_capacity.py b/test_validation_heal_capacity.py new file mode 100644 index 00000000..974589ec --- /dev/null +++ b/test_validation_heal_capacity.py @@ -0,0 +1,200 @@ +import copy +import errno +import json +import os +from pathlib import Path +from types import SimpleNamespace +from io import BytesIO +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from scripts.validation_heal import bounded_volume as volume, capacity, fleet_node +from scripts.validation_heal.matrix import AUTOMATIC_CASES, select +from scripts.validation_heal.heal_probe import Probe +from test_validation_heal_authorization import binding + + +ROOT = Path(__file__).parent + + +class CapacityContractTests(unittest.TestCase): + def test_all_catalogued_automatic_cases_reach_the_top_level_dispatcher(self): + for case in sorted(AUTOMATIC_CASES): + with self.subTest(case=case): + observed = subprocess.run(["bash", str(ROOT / "scripts/run_validation_case.sh"), + case, "/nonexistent/candidate.json", "a" * 40, "/nonexistent/evidence"], + capture_output=True, text=True, timeout=5) + self.assertEqual(observed.returncode, 2) + self.assertIn("candidate manifest missing", observed.stderr) + self.assertNotIn("executor is not migrated", observed.stderr) + self.assertIn(capacity.CASE_ID, AUTOMATIC_CASES) + self.assertEqual(select(capacity.CASE_ID)[0]["variants"], list(capacity.BRANCHES)) + + def test_inner_dispatch_uses_the_actual_capacity_executor(self): + observed = subprocess.run(["bash", str(ROOT / "scripts/validation_heal/run_case.sh"), + "--validation-case", capacity.CASE_ID, "--dry-run"], capture_output=True, text=True, timeout=5) + self.assertEqual(observed.returncode, 0, observed.stderr) + source = (ROOT / "scripts/validation_heal/run_case.sh").read_text() + self.assertIn("heal.enospc-inodes-and-quota) driver=capacity.py", source) + + def test_capacity_refusals_do_not_accept_unrelated_denials_or_success(self): + observed = {"acknowledged": False, "resultUnknown": False, "clientCallInvoked": True, + "response": {"ResponseMetadata": {"HTTPStatusCode": 400}, "Error": { + "Code": "InvalidRequest", "Message": "Bucket quota exceeded. Current usage: 1 bytes, limit: 2 bytes"}}} + self.assertEqual(capacity.refusal(observed, quota=True)["state"], "REFUSED") + for change in ("success", "not-invoked", "auth", "generic-invalid", "wrong-http"): + broken = copy.deepcopy(observed) + if change == "success": broken["acknowledged"] = True + elif change == "not-invoked": broken["clientCallInvoked"] = False + elif change == "auth": broken["response"]["Error"]["Code"] = "AccessDenied" + elif change == "generic-invalid": broken["response"]["Error"]["Message"] = "bad query" + else: broken["response"]["ResponseMetadata"]["HTTPStatusCode"] = 503 + with self.subTest(change=change), self.assertRaises(ValueError): + capacity.refusal(broken, quota=True) + unknown = {"acknowledged": False, "resultUnknown": True, "clientCallInvoked": True} + self.assertFalse(capacity.refusal(unknown, quota=False)["retryAllowed"]) + storage = {**observed, "response": {"ResponseMetadata": {"HTTPStatusCode": 503}, + "Error": {"Code": "ServiceUnavailable"}}} + self.assertEqual(capacity.refusal(storage, quota=False)["state"], "REFUSED") + with self.assertRaises(ValueError): capacity.refusal(observed, quota=False) + + def test_exact_candidate_probe_distinguishes_blocks_inodes_and_release(self): + observed = {"uid": 0, "gid": 0, "device": 8, "writeErrno": errno.ENOSPC, + "freeBlocks": 0, "availableBlocks": 0, "freeInodes": 91} + volume.validate_probe(observed, 8, capacity.BRANCHES[0], True) + with self.assertRaises(ValueError): volume.validate_probe(observed, 8, capacity.BRANCHES[1], True) + inode = {**observed, "freeInodes": 0, "freeBlocks": 11, "availableBlocks": 11} + volume.validate_probe(inode, 8, capacity.BRANCHES[1], True) + released = {**inode, "freeInodes": 30, "writeErrno": 0} + volume.validate_probe(released, 8, capacity.BRANCHES[1], False) + for name, value in (("uid", 1000), ("device", 9), ("writeErrno", errno.EACCES), + ("freeInodes", True), ("writeErrno", 0)): + with self.subTest(name=name, value=value), self.assertRaises(ValueError): + volume.validate_probe({**inode, name: value}, 8, capacity.BRANCHES[1], True) + with self.assertRaises(ValueError): volume.validate_probe(inode, 8, capacity.BRANCHES[1], False) + + def test_fill_is_bounded_and_requires_real_enospc(self): + stats = SimpleNamespace(f_bfree=0, f_ffree=900) + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + (folder / "padding").mkdir() + with patch.object(volume.os, "open", return_value=7), patch.object(volume.os, "close"), \ + patch.object(volume.os, "write", side_effect=[1024**2, OSError(errno.ENOSPC, "full")]), \ + patch.object(volume.os, "fsync"), patch.object(volume.os, "statvfs", return_value=stats): + result = volume.fill(folder, capacity.BRANCHES[0]) + self.assertEqual(result["allocatedFillerBytes"], 1024**2) + self.assertEqual(result["actualErrno"], errno.ENOSPC) + with patch.object(volume.os, "open", side_effect=OSError(errno.ENOSPC, "inodes")), \ + patch.object(volume.os, "statvfs", return_value=SimpleNamespace(f_bfree=500, f_ffree=0)): + self.assertEqual(volume.fill(folder, capacity.BRANCHES[1])["actualErrno"], errno.ENOSPC) + with patch.object(volume.os, "open", side_effect=OSError(errno.EIO, "unrelated")), self.assertRaises(ValueError): + volume.fill(folder, capacity.BRANCHES[1]) + + def test_recovery_rejects_unrecognized_filler_without_deleting_it(self): + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + (folder / "padding").mkdir() + unexpected = folder / "padding" / "acknowledged-object" + unexpected.write_bytes(b"preserve") + with self.assertRaises(ValueError): volume.release_filler(folder) + self.assertEqual(unexpected.read_bytes(), b"preserve") + + def test_loop_detach_binding_requires_the_exact_backing_inode_and_device(self): + row = {"name": "/dev/loop7", "back-file": "/owned/image", "back-ino": 31, + "back-maj:min": f"{os.major(2049)}:{os.minor(2049)}"} + with patch.object(volume, "command", return_value=json.dumps({"loopdevices": [row]})), \ + patch.object(volume, "regular", return_value=SimpleNamespace(st_ino=31, st_dev=2049)): + self.assertEqual(volume.loop_for(Path("/owned/image")), "/dev/loop7") + for change in ("inode", "device", "duplicate", "nonloop"): + rows = [dict(row)] + if change == "inode": rows[0]["back-ino"] = 99 + elif change == "device": rows[0]["back-maj:min"] = "999:999" + elif change == "duplicate": rows.append(dict(row)) + else: rows[0]["name"] = "/dev/sda" + with self.subTest(change=change), patch.object(volume, "command", return_value=json.dumps({"loopdevices": rows})), \ + self.assertRaises(ValueError): + volume.loop_for(Path("/owned/image")) + + def test_only_the_dedicated_runtime_can_allocate_bounded_volumes(self): + with tempfile.TemporaryDirectory() as directory: + state = Path(directory) + with self.assertRaises(ValueError): + volume.create(state, (), {"runtimeProfile": "manual", "runId": "gh-1-1-h1"}, capacity.BRANCHES[0]) + self.assertFalse((state / volume.MARKER).exists()) + config = fleet_node.runtime_config("bounded-capacity") + self.assertEqual(config["RUSTFS_UNSAFE_BYPASS_DISK_CHECK"], "true") + self.assertEqual(config["RUSTFS_HEAL_MRF_ENABLE"], "false") + self.assertEqual(config["RUSTFS_HEAL_AUTO_HEAL_ENABLE"], "false") + self.assertEqual(config["RUSTFS_SCANNER_ENABLED"], "false") + + def test_allocation_registers_exact_undo_before_formatting_only_bounded_files(self): + with tempfile.TemporaryDirectory() as directory: + state = Path(directory) / "state" + state.mkdir() + roots = tuple(Path(directory) / f"root{index}" for index in range(4)) + for root in roots: + (root / "data").mkdir(parents=True) + owner = {"runtimeProfile": "bounded-capacity", "runId": "gh-1-1-h1"} + calls = [] + def checked_command(*args): + calls.append(args) + self.assertTrue((state / volume.MARKER).exists()) + intent = json.loads((state / volume.MARKER).read_text()) + self.assertEqual(len(intent["volumes"]), 4) + if args[0] == "mkfs.ext4": + self.assertIn(Path(args[-1]).parent, roots) + self.assertEqual(Path(args[-1]).name, "bounded-volume.img") + self.assertNotIn("/dev/", args[-1]) + return "/dev/loop7\n" if args[0] == "losetup" else "" + with patch.object(volume, "command", side_effect=checked_command), \ + patch.object(volume, "regular", return_value=SimpleNamespace(st_size=100)), \ + patch.object(volume, "loop_for", return_value="/dev/loop7"), \ + patch.object(volume, "mounted", return_value={"verified": True}), \ + patch.object(volume.os, "posix_fallocate", create=True) as allocate, \ + patch.object(volume.os, "statvfs", return_value=SimpleNamespace(f_blocks=32768, f_frsize=4096, f_files=1024)), \ + patch.object(volume.shutil, "disk_usage", return_value=SimpleNamespace(free=10 * 1024**3)): + result = volume.create(state, roots, owner, capacity.BRANCHES[0]) + self.assertEqual(allocate.call_count, 4) + self.assertTrue(all(call.args[1:] == (0, volume.IMAGE_BYTES) for call in allocate.call_args_list)) + self.assertEqual(sum(args[0] == "mkfs.ext4" for args in calls), 4) + self.assertFalse(result["hostSharedMountRemounted"]) + self.assertFalse(result["independentPhysicalDriveCountIncreased"]) + + def test_registered_quota_probe_signs_and_preserves_the_exact_request(self): + class Response(BytesIO): + code = 200 + def __init__(self): + super().__init__(b"{}") + self.headers = {"Content-Length": "2"} + probe = Probe(binding()) + path = "/rustfs/admin/v3/quota/" + probe.binding["bucketPrefix"] + with patch.object(probe, "credentials", return_value=("access", "secret")), \ + patch.object(probe, "_open_admin", side_effect=lambda outgoing: Response()) as outgoing: + for limit in (1024**2, 256 * 1024**2): + body = json.dumps({"quota": limit, "quota_type": "HARD"}).encode() + observed = probe.admin(path, [], body, method="PUT") + self.assertEqual(observed["requestBody"].encode(), body) + self.assertEqual(outgoing.call_args.args[0].data, body) + self.assertIn("Authorization", outgoing.call_args.args[0].headers) + self.assertEqual(probe.admin(path, [], b"", method="GET")["httpStatus"], 200) + + def test_quota_probe_refuses_foreign_buckets_unregistered_limits_and_auth(self): + probe = Probe(binding()) + path = "/rustfs/admin/v3/quota/" + probe.binding["bucketPrefix"] + for route, method, body, auth in ( + ("/rustfs/admin/v3/quota/foreign-bucket", "PUT", b'{"quota":1048576,"quota_type":"HARD"}', "signed"), + (path, "DELETE", b"", "signed"), + (path, "PUT", b'{"quota":123,"quota_type":"HARD"}', "signed"), + (path, "PUT", b'{"quota":true,"quota_type":"HARD"}', "signed"), + (path, "GET", b"mutation", "signed"), + (path, "GET", b"", "anonymous")): + with self.subTest(route=route, method=method, body=body, auth=auth), \ + patch.object(probe, "_open_admin") as outgoing, self.assertRaises(ValueError): + probe.admin(route, [], body, method=method, auth=auth) + outgoing.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From 6072cab721a15390fd96b23f877ed03a8ba69c71 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 08:18:28 +0800 Subject: [PATCH 096/343] docs: record verified native regression results --- docs/validation-migration-map.md | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 2fa8a646..6c5dfee5 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1036,9 +1036,10 @@ four regression verdicts have their own archive and never increment a fleet or canonical-case PASS count. Existing tests assert captured lock events internally; the collector does not fabricate emitted barrier logs. Missing critical-point fixtures, Admin nolock semantics and required fleet overlap remain explicit gaps. -The current implementation has 287 focused Heal/catalog checks, including the -Linux-only D11, C09 and D08 ownership/mount/recovery checks. Native compilation and execution -still require the separate hosted job; local parser checks are not native PASS. +The current implementation has 310 focused Heal/catalog checks, including the +Linux-only ownership/mount/recovery checks. Hosted native compilation and all +four exact regressions completed in run `38096433036`; their independently +verified identities and limits are recorded below. J03 corrected-budget run `38089238767` passed every applicable within-window, @@ -1048,8 +1049,8 @@ physical/streaming integrity, all-peer S3 evidence, cleanup PASS and the independent canonical selected-case gate. This increases complete live PASS to 45/158; branch-only successes and native regressions remain outside that count. A06/F06/F01 and A02/E05/F05/J06/D11 queues continue serially on the -current auto-testing fleet. The optional native build waits for those queues -and uses the existing Functional Chain tools-only driver, because the reusable +current auto-testing fleet. The optional native build uses the existing +Functional Chain tools-only driver on hosted Linux, because the reusable workflow is still in a draft PR and cannot be dispatched directly from the default branch. @@ -1335,7 +1336,19 @@ Native tools-only run `38096433036` binds driver `ae9b5cbabfeaff440d9b8ee0b2fe805f17aff5b3` and auto-testing `acdd69218406b0596d79b6aab15481f8271e35c6`. Its candidate resolver and tool/native jobs use hosted ubuntu-latest with independent tools-only concurrency and -empty fleet credentials. The metadata inspector build passed; the native -candidate test executable remains in compilation. This exact audited hosted -run is excluded from the local fleet-busy classifier. Unknown active runs -still block dispatch, and native results cannot count as canonical fleet PASS. +empty fleet credentials. The workflow, metadata inspector build, optimized +native compilation and all four exact-name regression processes completed +successfully. Independent verification matched the downloaded executable +SHA-256 `0c9e005ae877f741a62100f0d70af6c2256d1078d546ac97e963d6979aa05bc1`, +candidate source tree `8071fb0a9e5353eb7d3fa58c8a58c8622473f2bd`, Cargo.lock +SHA-256 `d8f7bc416ba67d19873d34e9389e5aa823b22d7d7552706a5a810cee26f6eda2` +and collector SHA-256 +`b095d4e1cdb26be9d16cc82f589e29137ec84666e4675a9acc7dfbed761a0f28`. +Each original listing and result log proves exactly one non-ignored selected +test ran and passed. The final-rename and lock-owner tests contain actual +deterministic candidate assertions; overwrite and version-delete use +uncontrolled concurrency and do not prove deterministic commit fences. +Admin nolock, PUT cleanup-tail, other critical points and fleet overlap remain +unproven. These four native results do not increase the 46/158 complete +canonical live PASS count. This exact audited hosted run is excluded from +the local fleet-busy classifier; unknown active runs still block dispatch. From 601f249f91af2f6a68f46c0903371dd807899b4d Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 08:32:45 +0800 Subject: [PATCH 097/343] test: migrate network partition and lock quorum validation --- .../workflows/validation-catalog-tests.yml | 10 +- docs/validation-migration-map.md | 35 ++ scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 39 ++ scripts/validation_heal/healthy.py | 13 + scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/network.py | 466 ++++++++++++++++++ scripts/validation_heal/network_gate.py | 312 ++++++++++++ scripts/validation_heal/run_case.sh | 1 + test_validation_heal_network.py | 287 +++++++++++ 10 files changed, 1164 insertions(+), 3 deletions(-) create mode 100644 scripts/validation_heal/network.py create mode 100644 scripts/validation_heal/network_gate.py create mode 100644 test_validation_heal_network.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index afde2afd..8a2e69e1 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -28,6 +28,7 @@ on: - test_validation_heal_coordinator_crash.py - test_validation_heal_flapping.py - test_validation_heal_capacity.py + - test_validation_heal_network.py - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py @@ -72,6 +73,7 @@ on: - test_validation_heal_coordinator_crash.py - test_validation_heal_flapping.py - test_validation_heal_capacity.py + - test_validation_heal_network.py - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py @@ -106,7 +108,13 @@ jobs: - run: python3 -B -m unittest -v test_validation_pool.py - run: python3 -B -m unittest -v test_validation_replication.py - run: python3 -m pip install -r scripts/validation_ilm/requirements.txt - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_oidc.py test_validation_webhooks.py + - name: Install network namespace test dependency + run: | + if ! command -v nft >/dev/null; then + sudo apt-get update + sudo apt-get install --no-install-recommends -y nftables + fi + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 6c5dfee5..e54ce8e4 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1352,3 +1352,38 @@ Admin nolock, PUT cleanup-tail, other critical points and fleet overlap remain unproven. These four native results do not increase the 46/158 complete canonical live PASS count. This exact audited hosted run is excluded from the local fleet-busy classifier; unknown active runs still block dispatch. + +F01 `38096950043` passed both original-root graceful-interruption and same-record +disk-resume branches on H1/H2 at immutable harness +`21b4ff9656710867dbcbaced08e1804110c6bdf7`. Independent artifact inspection and +the canonical selected-case gate verified every branch, cleanup PASS, actual +partial physical repair, original task/record ownership, non-refunded timeout, +unchanged peer processes and complete physical/streaming/all-peer S3 evidence. +Complete canonical live PASS is now 47/158. The older F01 response-budget FAIL +remains in its original archive. + +I03 `heal.network-partition-and-lock-quorum` now has all five H2 source branches: +minority partition, read boundary, write boundary, lock boundary and rejoin. +Each uses a fresh run-owned four-node/sixteen-member EC:8 service and an exact +versioned ACK fixture. The actual R=8/W=9 geometry and shared/exclusive lock +quorums are frozen independently. A two/two split proves the exact readable +boundary and unavailable exclusive lock quorum; a three/one split retains +three lock clients while private read-only gates leave precisely eight writable +roots. Historical and damaged versions use their own available-shard counts. + +All four independent 20-minute expiry timers and root-owned recovery intents +precede the first atomic nftables transaction. Each table matches only the +registered peer IPv4 addresses and TCP source/destination port 9000; no shared +table is flushed. Every peer direction is probed with actual drop-counter +increments, and management TCP 22 remains reachable. Each group independently +executes exact-version GET, one unique PUT without retries, Heal and partial +background-coverage checks. Every original version/control must remain intact; +unknown writes require all-peer resolution after network recovery. Owned rules +must be removed before independent repair, exact version responsibility, +same-role streaming checks and complete S3 manifests. A distinct recovered PUT +must redeem its actual VersionId, bytes, metadata and tags on every peer. +Foreign rules, lost expiry guards, unknown ACKs or failed recovery retain DIRTY +evidence. No I03 live PASS is claimed. The implementation now has 129 automatic +executors (74 non-Heal, 55 Heal); 29 Heal executors remain. The focused suite +contains 332 tests, including the real Linux private-network nftables gate; +six platform-specific tests are skipped locally on macOS. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index a4d91693..d0ce920f 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 5c0caab5..8414b149 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -911,12 +911,49 @@ def configure_encryption(request): return encryption_module().configure(state, owner, request, fault_module()) +def network_operation(request): + state, _, owner = owned(request["runId"]) + module = globals()["network_gate_backend"] + module.identities(owner) + action = request["action"] + require(set(request) <= {"runId", "action", "addresses", "localIndex", "blockedIndices"}, + "network adapter accepts no external paths, ports, rules or commands") + if action == "network-release": + return module.release(state, owner) + identity = process(request) + require(identity["pid"] > 1 and identity["executable"] == str(state / "rustfs"), + "network fault requires an actual owned candidate process") + listeners = command("ss", "-H", "-lntp", "sport = :9000").stdout + pids = {int(value) for value in re.findall(r"pid=([0-9]+)", listeners)} + require(pids == {identity["pid"]} and listeners.strip(), + "TCP 9000 is not exclusively owned by this run's candidate") + if action == "network-preflight": + values = module.resolve() + local = [] + for index, address in enumerate(values): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as client: + try: client.bind((address, 0)) + except OSError: continue + local.append(index) + require(len(local) == 1, "fleet aliases do not bind exactly one actual local address") + return {"addresses": values, "localIndex": local[0], "process": identity, + "exclusiveServicePort": 9000, "nftVersion": command("nft", "--version").stdout.strip()} + if action == "network-prepare": + return module.prepare(state, owner, request, globals()["NETWORK_GATE_SOURCE"]) + if action == "network-activate": + return module.activate(state, owner) + require(action == "network-observe", "unsupported network fault operation") + return {**module.observe(state, owner), "process": identity} + + def cleanup(request): state, roots, owner = owned(request["runId"]) command("systemctl", "stop", owner["unit"], check=False) pid = command("systemctl", "show", "-p", "MainPID", "--value", owner["unit"], check=False).stdout.strip() require(pid in {"", "0"}, "run-owned service remains active") stopped(owner) + if (state / "network-gate.json").exists(): + globals()["network_gate_backend"].release(state, owner) gate_module().release_gate(state, roots) quorum_gate_module().release(state, roots) replacement_module().attest(state, roots) @@ -1036,6 +1073,8 @@ def main(request): actions[action] = volume_operation actions["format-schema-fault"] = format_schema_fault actions["encryption-configure"] = configure_encryption + for action in ("network-preflight", "network-prepare", "network-activate", "network-observe", "network-release"): + actions[action] = network_operation actions["replacement-records"] = replacement_records actions.update({"replace-drive": replace_drive, "replacement-state": replacement_state, "replacement-probe": replacement_probe}) actions.update({"format-export": format_export, "foreign-format": foreign_format, "restore-replacement": restore_replacement}) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 8852ad6e..26f1886c 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -150,6 +150,7 @@ def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_prof self.bucket_record_faults = [] self.drive_gates = set() self.quorum_gates = set() + self.network_gates = set() self.secret_values = set() self.frozen_processes = {} self.snapshot_buckets = None @@ -180,6 +181,10 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() + if action.startswith("network-") or action == "cleanup": + network_source = NODE_SOURCE.with_name("network_gate.py").read_text() + source += "\nimport types\nnetwork_gate_backend = types.ModuleType('network_gate')\nexec(" + repr( + network_source) + ", network_gate_backend.__dict__)\nNETWORK_GATE_SOURCE = " + repr(network_source) + "\n" if self.runtime_profile == "bounded-capacity": source += "\nimport types\nbounded_volume_backend = types.ModuleType('bounded_volume')\nexec(" + repr( NODE_SOURCE.with_name("bounded_volume.py").read_text()) + ", bounded_volume_backend.__dict__)\n" @@ -299,6 +304,14 @@ def wait_ready(self): def cleanup(self): failures = [] + if self.network_gates: + try: + self.stop() + for node in tuple(self.network_gates): + self.record("network-gate-recovery", {"node": node, **self.remote(node, "network-release")}) + self.network_gates.remove(node) + except Exception as exc: + failures.append({"error": str(exc), "state": "DIRTY", "namespacesRetained": True}) for node, identity in tuple(self.frozen_processes.items()): try: current = self.remote(node, "process") diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 16fc0d2a..244b7028 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/network.py b/scripts/validation_heal/network.py new file mode 100644 index 00000000..6c230940 --- /dev/null +++ b/scripts/validation_heal/network.py @@ -0,0 +1,466 @@ +#!/usr/bin/env python3 +"""I03: actual bidirectional partitions, independent quorum witnesses and ACK reconciliation.""" + +import argparse +import copy +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +from urllib.parse import quote + +try: + from .heal_probe import Probe, canonical, compare_inventories, payload, require, save + from .heal_workload import Recorder, RecordedCallError, verify_seed + from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .shards import files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort + from .versions import public_version_id, validate_version_receipts, verify_quorum_contract, validate_unrecoverable_get + from .write_quorum import expected_reads, resolve_put, negative_put, LOCK_REFERENCE, STORAGE_REFERENCE + from .stop import idle_fleet + from .network_gate import TTL_SECONDS +except ImportError: + from heal_probe import Probe, canonical, compare_inventories, payload, require, save + from heal_workload import Recorder, RecordedCallError, verify_seed + from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from shards import files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort + from versions import public_version_id, validate_version_receipts, verify_quorum_contract, validate_unrecoverable_get + from write_quorum import expected_reads, resolve_put, negative_put, LOCK_REFERENCE, STORAGE_REFERENCE + from stop import idle_fleet + from network_gate import TTL_SECONDS + + +CASE_ID = 'heal.network-partition-and-lock-quorum' +BRANCHES = ('minority-partition', 'read-boundary', 'write-boundary', 'lock-boundary', 'rejoin') +SIZE = 5 * 1024**2 + 97 +REFERENCE = {'candidateCommit': LOCK_REFERENCE['candidateCommit'], 'lock': LOCK_REFERENCE, + 'storage': STORAGE_REFERENCE, 'topology': 'four original nodes, sixteen original members, explicit EC:8', + 'network': {'servicePort': 9000, 'managementPort': 22, 'ttlSeconds': TTL_SECONDS, + 'rules': 'one owned nft table per host; exact peer IPv4 and TCP source/destination port', + 'mutation': 'all expiry timers armed before the first atomic table transaction'}, + 'load': {'fixtureObjects': 4, 'versionsPerObject': 2, 'partitionPUTs': 2, 'recoveredPUTs': 1, + 'automaticRetries': 0, 'negativeTaskDeadlineSeconds': 330}, + 'writePolicy': 'three-peer groups retain R but use independent private gates to measure W-1; two-peer groups lose W and exclusive lock quorum'} + + +def plan_for(snapshot, target, branch): + nodes = list(snapshot) + geometry = target['logical']['erasure'] + cohort = target['cohort'] + require(branch in BRANCHES and len(nodes) == 4 and geometry['data_blocks'] == geometry['parity_blocks'] == 8 + and len(cohort) == len({(row['node'], row['disk']) for row in cohort}) == 16 + and all({row['disk'] for row in cohort if row['node'] == node} == set(range(4)) for node in nodes), + 'I03 requires the observed balanced sixteen-member cross-node set') + groups = [[0, 1], [2, 3]] if branch in {'read-boundary', 'lock-boundary'} else [[0, 1, 2], [3]] + require(target['node'] == nodes[3], 'partition defect must be on the isolated fourth original member') + gates = {nodes[0]: [0, 1, 2, 3]} if len(groups[0]) == 3 else {} + sides = [] + for group in groups: + slots = [row for row in cohort if nodes.index(row['node']) in group] + sources = len(slots) - int(3 in group) + writable = len(slots) - sum(len(gates.get(nodes[index], [])) for index in group) + require(writable < 9 and len(group) < 3 or writable == 8 and len(group) == 3, + 'partition side still has safe write admission') + sides.append({'members': group, 'endpoint': f'node{group[0] + 1}', 'readableSources': sources, + 'metadataAuthorities': len(slots), 'writableDrives': writable, 'lockClients': len(group), + 'GETAllowed': sources >= 8, 'PUTAllowed': False, 'exclusiveLockQuorumPresent': len(group) >= 3}) + require(sides[0]['readableSources'] >= 8 and (len(groups[0]) != 2 or sides[0]['readableSources'] == 8), + 'read boundary does not preserve the exact actual R sources') + return {'groups': groups, 'sides': sides, 'readQuorum': 8, 'writeQuorum': 9, + 'sharedLockQuorum': 2, 'exclusiveLockQuorum': 3, 'gates': gates, + 'blocked': {nodes[index]: [other for other in range(4) if not any(index in group and other in group for group in groups)] + for index in range(4)}} + + +def validate_witness(observed, plan, nodes): + require(set(observed) == set(nodes), 'network witness omitted an original node') + for index, node in enumerate(nodes): + value = observed[node] + require(value.get('state') == 'ACTIVE' and value.get('localIndex') == index + and value.get('expiryArmed') is True and value.get('process', {}).get('pid', 0) > 1, + 'network witness has no actual original service or active expiry') + rows = value.get('probes', []) + require(len(rows) == 3 and {row['peerIndex'] for row in rows} == set(range(4)) - {index}, + 'network probes omitted or duplicated a peer direction') + require(all(row['service']['connected'] is (row['peerIndex'] not in plan['blocked'][node]) + and row['management']['connected'] is True for row in rows), + 'partition matrix or management-channel preservation differs') + require(sum(row['packets'] for row in value['countersAfter']) > sum(row['packets'] for row in value['countersBefore']), + 'network probe has no packet-counter evidence') + return {'allTwelvePeerDirectionsVerified': True, 'actualPacketCounters': True, + 'allManagementDirectionsReachable': True, 'servicesNotStoppedToSimulatePartition': True} + + +def validate_partial(value, peers): + coverage = value.get('coverage') or {} + require(value.get('clusterStatusComplete') is False and value.get('state') in {'degraded', 'active'} + and all(type(coverage.get(name)) is int for name in ('expected', 'responded', 'unknown')) + and coverage['expected'] == 4 and coverage['responded'] == peers and coverage['unknown'] == 4 - peers + and isinstance(coverage.get('reasons'), list) and coverage['reasons'], + 'partitioned endpoint falsely reports complete/global health or hides actual peer coverage') + return {'responded': peers, 'unknown': 4 - peers, 'globalCompleteClaimed': False} + + +def validate_negative_task(current, bucket, expected, target): + outcome = current.get('outcome') or {} + execution = outcome.get('execution', {}) + require(current.get('summary') in {'finished', 'stopped'} and execution.get('state') in { + 'completed', 'completed_with_errors', 'aborted'} and outcome.get('coverage') in {'partial', 'complete', 'unknown'} + and current.get('truncated', False) is False and outcome.get('objectsTruncated') is False, + 'partition task has no bounded explicit unresolved terminal result') + wanted = {(row['key'], row['versionId']) for row in expected} + target_pair = target['key'], target['versionId'] + counters = outcome.get('counters', {}) + for name in ('processed', 'healed', 'unchanged', 'skipped', 'failed', 'unknown', 'attemptFailures'): + require(type(counters.get(name)) is int and counters[name] >= 0, 'partition task omits typed error counters') + require(counters.get('overflowed') is False and counters['healed'] == 0 + and counters['processed'] == sum(counters[name] for name in ('healed', 'unchanged', 'skipped', 'failed')) + and counters['unknown'] <= counters['skipped'] and counters['processed'] <= len(wanted), + 'partition task hides unresolved work or claims an unsafe repair') + seen, unresolved, states = set(), set(), [] + for row in outcome.get('objects', []): + identity = row.get('identity', {}) + pair = identity.get('object'), public_version_id(identity.get('versionId')) + require(identity.get('bucket') == bucket and pair in wanted and pair not in seen, + 'partition task reports foreign or duplicate responsibility') + state = row.get('disposition', {}).get('state') + require(state in {'verified_healthy', 'failed', 'deferred', 'unknown', 'cancelled'}, + 'partition task reports repaired/absent or an unrelated disposition') + if state == 'verified_healthy': + require(pair != target_pair and identity.get('bucketIncarnationId') + and type(identity.get('poolIndex')) is int and identity['poolIndex'] == 0 + and type(identity.get('setIndex')) is int and identity['setIndex'] == 0, + 'missing target was reported healthy, or a healthy receipt lacks authority') + else: + unresolved.add(pair) + seen.add(pair) + states.append(state) + require(counters['processed'] == len(seen), 'bounded partition counters differ from exact version receipts') + require(counters['unchanged'] == states.count('verified_healthy') and counters['failed'] == states.count('failed') + and counters['skipped'] == sum(states.count(state) for state in ('deferred', 'unknown', 'cancelled')) + and counters['unknown'] == states.count('unknown') and counters['attemptFailures'] >= counters['failed'], + 'partition counters conceal the actual unresolved receipt dispositions or failed attempts') + require(target_pair in unresolved or execution == {'state': 'aborted', 'reason': 'untraversable'} + and outcome['coverage'] != 'complete' and counters['processed'] == 0, + 'partition task lost the damaged version obligation or silently completed it') + require(not counters['failed'] or execution['state'] == 'completed_with_errors', + 'partition task erases its failed execution status') + return {'missingVersionUnresolved': True, 'unsafeRepairClaimed': False, + 'execution': execution, 'coverage': outcome['coverage'], 'counters': counters} + + +def negative_heal(probe, fleet, bucket, expected, target, active): + path = '/rustfs/admin/v3/heal/' + quote(bucket, safe='') + '/' + quote(target['key'], safe='/') + admission = fleet.record('partition-Heal-admission', probe.admin(path, [], json.dumps(BASE_OPTIONS).encode())) + if admission.get('httpStatus') != 200: + # No retry: require an actual structured storage/lock refusal. An + # absent token or transport exception alone cannot prove safe refusal. + body = admission.get('response') + if isinstance(body, str): + import xml.etree.ElementTree as ET + body = {item.tag.rsplit('}', 1)[-1]: item.text for item in ET.fromstring(body).iter()} + require(admission.get('httpStatus') in {500, 503} and isinstance(body, dict) + and (body.get('Code') or body.get('code')) in {'InternalError', 'XMinioWriteQuorum', 'XMinioLockTimeout', 'ServiceUnavailable'} + and not body.get('clientToken'), 'partition Heal admission did not return a structured safe refusal') + return {'admissionRefused': True, 'repairClaimed': False, 'requestRetried': False} + token = admission.get('response', {}).get('clientToken') + require(isinstance(token, str) and token, 'partition Heal admitted without an actual token') + active.append((probe.endpoint_name, path, token)) + deadline = time.monotonic() + 330 + while True: + response = fleet.record('partition-Heal-status', probe.admin(path, [('clientToken', token)], b'')) + require(response.get('httpStatus') == 200 and isinstance(response.get('response'), dict), + 'partition task status disappeared or has an unknown response') + current = response['response'] + if current.get('summary') != 'running': + break + require(time.monotonic() < deadline, 'partition task exceeded its frozen unresolved-task deadline') + time.sleep(0.2) + proof = validate_negative_task(current, bucket, expected, target) + repeated = fleet.record('partition-terminal-repeat', probe.admin(path, [('clientToken', token)], b'')) + require(repeated.get('httpStatus') == 200 and canonical(repeated.get('response', {}).get('outcome')) == canonical(current['outcome']), + 'negative partition responsibility changed at its original token') + active.remove((probe.endpoint_name, path, token)) + return {**proof, 'taskId': token, 'requestRetried': False} + + +def project(snapshot, bucket, keys): + result = copy.deepcopy(snapshot) + prefixes = tuple(bucket + '/' + key + '/' for key in keys) + for value in result.values(): + value['files'] = [row for row in value['files'] if not row['path'].startswith(prefixes)] + return result + + +def unique_put(probe, directory, bucket, key, rows): + body = payload(f'{bucket}/{key}/0', SIZE) + entry = {'endpoint': probe.endpoint_name, 'key': key, 'payload': {'size': SIZE, 'sha256': hashlib.sha256(body).hexdigest()}} + rows.append(entry) # Register before invocation, including collector failures. + recorder = Recorder(probe, directory / key.replace('/', '-')) + try: + response = recorder.call('put_object', {'Bucket': bucket, 'Key': key, + 'ContentType': 'application/octet-stream', 'Metadata': {'heal-fixture': 'I03', 'nonce': key}, + 'Tagging': 'suite=heal&fixture=I03'}, body) + except RecordedCallError as exc: + entry['observed'] = exc.observation + else: + entry['observed'] = {'acknowledged': True, 'resultUnknown': False, 'clientCallInvoked': True, + 'response': response, 'payload': entry['payload']} + return entry + + +def resolve_writes(fleet, binding, bucket, rows): + for row in rows: + reads = [fleet.record('all-peer-unique-write-resolution', Probe(binding, endpoint).s3('get_object', { + 'Bucket': bucket, 'Key': row['key']})) for endpoint in binding['endpoints']] + observed = row.get('observed', {'acknowledged': False, 'resultUnknown': True}) + resolution = resolve_put(observed, reads, row['payload']) + if resolution['state'] == 'COMMITTED': + versions = {read.get('response', {}).get('VersionId') for read in reads} + require(len(versions) == 1 and next(iter(versions)) not in {None, 'null'}, + 'peers disagree on the unique committed version identity') + row['versionId'] = next(iter(versions)) + require(not observed.get('acknowledged') or observed['response'].get('VersionId') == row['versionId'], + 'unique PUT ACK was replaced by a different committed version') + row['resolution'] = resolution + return {'allOriginalRequestsResolved': True, 'automaticRetries': 0} + + +def release_network(fleet): + for node in tuple(fleet.network_gates): + value = fleet.record('release-owned-network-table', {'node': node, **fleet.remote(node, 'network-release')}) + require(value['state'] == 'CLEAR' and value.get('rulesRemaining') == 0, + 'network recovery did not remove every owned rule') + fleet.network_gates.remove(node) + require(value.get('expiryTriggered') is False, 'partition expired before its complete evidence window') + + +def run_branch(args, nodes, branch): + directory = args.evidence / 'H2' / branch + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-h2" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile='lock-quorum') + candidate = json.loads(args.candidate.read_text()) + require(candidate['source_sha'] == REFERENCE['candidateCommit'], 'network lock policy is not reviewed for this candidate') + bucket = 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {'schemaVersion': 1, 'runId': run_id, 'jumpHostOnly': True, 'exclusiveHealService': True, + 'leaseExpiresAt': (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), 'bucketPrefix': bucket, + 'endpoints': {f'node{index}': f'http://rustfs-node{index}:9000' for index in range(1, 5)}, 'defaultEndpoint': 'node1', + 'candidateCommit': candidate['source_sha'], 'harnessCommit': args.testing_commit, + 'candidatePackageSha256': candidate['package_sha256'], 'candidateBinarySha256': args.binary_sha, + 'rcBinarySha256': hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), 'maxInventoryObjects': 128, + 'maxInventoryBytes': 512 * 1024**2} + fleet.record('binding', binding) + fleet.record('network-source-contract', REFERENCE) + contract = verify_quorum_contract(fleet.record('inspector-build', verify_inspector(args.tool_directory, candidate)), candidate) + result = {'topology': 'H2', 'branch': branch, 'verdict': 'RUNNING', 'cleanup': 'PENDING'} + probe, active, recovery_active, rows = Probe(binding), [], [], [] + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + recorder = Recorder(probe, directory / 'fixture') + recorder.call('create_bucket', {'Bucket': bucket}) + recorder.call('put_bucket_versioning', {'Bucket': bucket, 'VersioningConfiguration': {'Status': 'Enabled'}}) + expected = [] + for index in range(4): + key = f'objects/{index:04d}.bin' + for version in range(2): + body = payload(f'{bucket}/{key}/{version}', SIZE) + ack = recorder.call('put_object', {'Bucket': bucket, 'Key': key, + 'ContentType': 'application/octet-stream', 'Metadata': {'heal-fixture': 'DV', 'nonce': f'{index}-{version}'}, + 'Tagging': 'suite=heal&fixture=DV'}, body) + require(ack.get('VersionId') not in {None, 'null'}, 'versioned network fixture has no exact ACK version') + expected.append({'key': key, 'versionId': ack['VersionId'], 'isLatest': version == 1, 'deleteMarker': False, + 'payload': {'size': SIZE, 'sha256': hashlib.sha256(body).hexdigest()}}) + baseline = fleet.record('all-version-S3-baseline', probe.inventory(bucket)) + fleet.record('exact-ACK-manifest', {'expected': expected, **verify_seed(expected, baseline)}) + fleet.stop() + original = fleet.snapshot('physical-baseline', quiescent=True) + validate_topology(original, 'H2') + selected = expected[1] + target = map_target(original, bucket, selected['key'], 'data', 1, selected['versionId']) + target = {**target, **next(row for row in target['cohort'] if row['node'] == nodes[3])} + cohorts = [map_target(original, bucket, entry['key'], 'data', 1, entry['versionId'], latest_required=False) for entry in expected] + plan = fleet.record('actual-partition-quorum-plan', plan_for(original, target, branch)) + params = {'disk': target['disk'], 'faultId': 'I03-missing-target'} + registration = target['node'], params + fleet.faults.append(registration) + receipt = fleet.record('exact-missing-target', fleet.remote(target['node'], 'apply-fault', **params, + relativePath=target['relativePath'], kind='missing', expectedSha256=target['sha256'])) + faulted = fleet.snapshot('physical-exact-fault', quiescent=True) + validate_fault(original, faulted, target, receipt) + for node, disks in plan['gates'].items(): + fleet.quorum_gates.add(node) + fleet.record('independent-W-minus-one-gate', {'node': node, **fleet.remote(node, 'quorum-gate', disks=disks)}) + fleet.start() + preflight = fleet.record('network-actual-address-process-attestation', {node: fleet.remote(node, 'network-preflight') for node in nodes}) + addresses = preflight[nodes[0]]['addresses'] + require(all(value['addresses'] == addresses and value['localIndex'] == index + and value['exclusiveServicePort'] == 9000 for index, value in enumerate(preflight.values())), + 'actual host address, service ownership or peer alias order differs') + # Every independent expiry is ready before the first rule is installed. + for index, node in enumerate(nodes): + fleet.network_gates.add(node) + fleet.record('network-expiry-and-rule-intent', {'node': node, **fleet.remote(node, 'network-prepare', + addresses=addresses, localIndex=index, blockedIndices=plan['blocked'][node])}) + for node in nodes: + fleet.record('atomic-owned-table-activation', {'node': node, **fleet.remote(node, 'network-activate')}) + def witness(label): + raw = fleet.record(label, {node: fleet.remote(node, 'network-observe') for node in nodes}) + return validate_witness(raw, plan, nodes) + result['networkBefore'] = witness('all-directed-peer-probes-before') + gates = fleet.record('actual-service-readable-writable-roots', {node: fleet.remote(node, 'quorum-gate-probe', + relativePath=f"{bucket}/{selected['key']}/xl.meta", expectedReads=expected_reads(original, node, f"{bucket}/{selected['key']}/xl.meta")) for node in nodes}) + for side in plan['sides']: + require(sum(row['writable'] for index in side['members'] for row in gates[nodes[index]]['disks']) == side['writableDrives'] + and all(row['readableMetadata'] and row['sameServiceUidAndMountNamespace'] for index in side['members'] + for row in gates[nodes[index]]['disks']), 'actual independent R/W witnesses differ from partition plan') + for side in plan['sides']: + endpoint = side['endpoint'] + local_probe = Probe(binding, endpoint) + for version in expected[:2]: + read = fleet.record('partition-explicit-version-GET', local_probe.s3('get_object', { + 'Bucket': bucket, 'Key': version['key'], 'VersionId': version['versionId']})) + available = side['metadataAuthorities'] - int(3 in side['members'] and version['versionId'] == selected['versionId']) + if available >= plan['readQuorum']: + require(read.get('acknowledged') is True and read.get('resultUnknown') is False + and read.get('payload') == version['payload'], 'R-capable partition cannot redeem its exact original ACK') + else: + validate_unrecoverable_get(read, contract) + row = unique_put(local_probe, directory, bucket, 'partition/' + endpoint + '.bin', rows) + row['negativeProof'] = negative_put(row['observed']) + result.setdefault('partitionTasks', {})[endpoint] = negative_heal(local_probe, fleet, bucket, expected[:2], target, active) + status = fleet.record('partition-raw-background', local_probe.admin('/rustfs/admin/v3/background-heal/status', [], b'')) + require(status.get('httpStatus') == 200 and isinstance(status.get('response'), dict), 'partition background status unavailable') + result.setdefault('coverage', {})[endpoint] = validate_partial(status['response'], len(side['members'])) + result['networkAfter'] = witness('all-directed-peer-probes-after') + fleet.record('partition-process-resource-observations', {node: fleet.remote(node, 'process-metrics') for node in nodes}) + fleet.stop() + scene = fleet.snapshot('negative-scene-before-release', quiescent=True) + require(physical_content(faulted) == physical_content(project(scene, bucket, [row['key'] for row in rows])), + 'partition writes/Heal changed an original version, control or damaged target') + for node in nodes: + fleet.record('partition-peer-logs', {'node': node, **fleet.remote(node, 'logs')}) + release_network(fleet) + for node in tuple(fleet.quorum_gates): + fleet.record('release-owned-W-gate', fleet.remote(node, 'quorum-gate', mode='clear')) + fleet.quorum_gates.remove(node) + fleet.start() + require(physical_content(scene) == physical_content(fleet.snapshot('all-peer-return-before-independent-repair')), + 'network recovery repaired a target before the independent product task') + repaired = run_task(probe, fleet, {'bucket': bucket, 'key': selected['key']}, 2, recovery_active, + status_endpoints=tuple(binding['endpoints'])[1:]) + result['recoveryResponsibility'] = validate_version_receipts(repaired, bucket, expected[:2], 2, + required_repairs={(selected['key'], selected['versionId'])}) + idle_fleet(fleet, binding) + fleet.stop() + after = fleet.snapshot('physical-rebuilt-before-any-recovered-GET', quiescent=True) + result['physical'] = validate_repaired_physical(original, project(after, bucket, [row['key'] for row in rows]), target) + result['streaming'] = [verify_cohort(fleet, cohort, 'every-original-version-streaming') for cohort in cohorts] + fleet.faults.remove(registration) + fleet.start() + result['writeResolution'] = resolve_writes(fleet, binding, bucket, rows) + require(all(row['resolution']['state'] == 'ABSENT' for row in rows), 'a W/lock-negative partition nevertheless committed a new version') + released = unique_put(probe, directory, bucket, 'recovered/unique.bin', rows) + require(released['observed']['acknowledged'] is True and released['observed']['response'].get('VersionId') not in {None, 'null'}, + 'fully recovered cluster cannot acknowledge a distinct safe PUT') + resolve_writes(fleet, binding, bucket, [released]) + augmented = expected + [{'key': released['key'], 'versionId': released['versionId'], 'isLatest': True, + 'deleteMarker': False, 'payload': released['payload']}] + inventories = [] + for endpoint in binding['endpoints']: + inventory = fleet.record('all-peer-complete-final-S3', Probe(binding, endpoint).inventory(bucket)) + verify_seed(augmented, inventory) + subset = {**inventory, 'entries': [row for row in inventory['entries'] if row['key'] != released['key']]} + compare_inventories(baseline, subset) + inventories.append(inventory) + for inventory in inventories[1:]: + compare_inventories(inventories[0], inventory) + require(all(row['properties'].get('Metadata') == {'heal-fixture': 'I03', 'nonce': released['key']} + and row['tags'] == [{'Key': 'fixture', 'Value': 'I03'}, {'Key': 'suite', 'Value': 'heal'}] + for inventory in inventories for row in inventory['entries'] if row['key'] == released['key']), + 'recovered PUT lost its public metadata or tags') + result.update(verdict='PASS', everyOriginalVersionPreserved=True, noForkOrStaleOverwrite=True) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict='FAIL', failure={'type': type(exc).__name__, 'message': str(exc)}) + finally: + dirty = False + active.extend(('node1', path, token) for path, token in recovery_active) + for endpoint, path, token in active: + try: + fleet.record('stop-original-partition-task', Probe(binding, endpoint).admin(path, [('clientToken', token), ('forceStop', 'true')], b'')) + except Exception as exc: + fleet.record('stop-partition-task-error', {'error': str(exc)}) + try: + if fleet.network_gates or fleet.quorum_gates: + fleet.stop() + release_network(fleet) + for node in tuple(fleet.quorum_gates): + fleet.record('failed-branch-W-release', fleet.remote(node, 'quorum-gate', mode='clear')) + fleet.quorum_gates.remove(node) + fleet.start() + pending = [row for row in rows if 'resolution' not in row] + if pending: + fleet.stop() + fleet.start() + resolve_writes(fleet, binding, bucket, pending) + except Exception as exc: + dirty = True + result['recoveryError'] = str(exc) + if dirty: + fleet.stop() + fleet.record('retain-unresolved-network-or-write', {'state': 'DIRTY', 'namespacesRetained': True, 'writes': rows}) + clean = False + else: + clean = fleet.cleanup() + result.update(cleanup='PASS' if clean else 'FAIL', writes=rows) + fleet.record('branch-result', result) + return result + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f'I03 interrupted by signal {signum}') + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ('candidate', 'rc-bin', 'evidence', 'tool-directory'): + parser.add_argument('--' + name, type=Path, required=True) + parser.add_argument('--case-id', choices=[CASE_ID], required=True) + parser.add_argument('--testing-commit', required=True) + parser.add_argument('--binary-sha', required=True) + args = parser.parse_args() + require(re.fullmatch(r'[0-9a-f]{64}', args.binary_sha) and re.fullmatch(r'[1-9][0-9]*', os.environ.get('GITHUB_RUN_ID', '')), + 'immutable workflow candidate identity required') + require(select(CASE_ID)[0]['variants'] == list(BRANCHES), 'I03 source branch omitted') + nodes = os.environ.get('RUSTFS_NODES', 'vm000 vm001 vm002 vm003').split() + require(len(nodes) >= 4 and len(nodes) == len(set(nodes)) and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in nodes), + 'four distinct current Heal nodes required') + results = [] + try: + for branch in BRANCHES: + results.append(run_branch(args, nodes[:4], branch)) + if results[-1]['cleanup'] != 'PASS': + break + finally: + complete = len(results) == len(BRANCHES) and all(row['verdict'] == row['cleanup'] == 'PASS' for row in results) + save(args.evidence / 'healthy-summary.json', {'caseId': CASE_ID, 'results': results, 'complete': complete, + 'cleanup': 'PASS' if results and all(row['cleanup'] == 'PASS' for row in results) else 'FAIL'}) + require(complete, 'I03 requires all five real partition/quorum/recovery branches and complete ACK resolution') + + +if __name__ == '__main__': + try: + main() + except Exception as exc: + print(f'I03 network validation failed: {exc}', file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/network_gate.py b/scripts/validation_heal/network_gate.py new file mode 100644 index 00000000..59a42b8e --- /dev/null +++ b/scripts/validation_heal/network_gate.py @@ -0,0 +1,312 @@ +"""I03: one owned nftables table, with a pre-armed independent expiry timer.""" + +from datetime import datetime, timedelta, timezone +import hashlib +import ipaddress +import json +import os +from pathlib import Path +import re +import socket +import stat +import subprocess +import sys + + +MARKER = 'network-gate.json' +SCRIPT = 'network-gate-expiry.py' +EXPIRY = 'network-gate-expired.json' +TTL_SECONDS = 1200 +PORT = 9000 + + +def require(value, message): + if not value: + raise ValueError(message) + + +def command(*args, input=None): + result = subprocess.run(args, input=input, capture_output=True, text=True, timeout=30) + require(result.returncode == 0, f'{args[0]} network operation failed: {result.stderr[-600:]}') + return result.stdout + + +def private_file(path, limit=128 * 1024): + info = path.lstat() + require(not path.is_symlink() and stat.S_ISREG(info.st_mode) and info.st_uid == 0 + and info.st_nlink == 1 and stat.S_IMODE(info.st_mode) == 0o600 and info.st_size <= limit, + 'network artifact is not a bounded private root-owned regular file') + with os.fdopen(os.open(path, os.O_RDONLY | os.O_NOFOLLOW), 'rb') as stream: + before = os.fstat(stream.fileno()) + raw = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + require(len(raw) == before.st_size <= limit and all(getattr(before, name) == getattr(after, name) + for name in ('st_ino', 'st_dev', 'st_size', 'st_mtime_ns', 'st_ctime_ns', 'st_nlink')), + 'network artifact changed while reading') + return raw + + +def write(path, value, *, replace=False): + target = path.with_suffix('.pending') if replace else path + fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, 'wb') as stream: + stream.write(value if isinstance(value, bytes) else json.dumps(value).encode()) + stream.flush() + os.fsync(stream.fileno()) + if replace: + os.replace(target, path) + descriptor = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def identities(owner): + require(re.fullmatch(r'gh-[1-9][0-9]*-[1-9][0-9]*-h2', owner.get('runId', '')) + and owner.get('runtimeProfile') == 'lock-quorum', 'network fault requires its exact H2 EC:8 service') + suffix = hashlib.sha256((owner['runId'] + '/' + owner['hostname']).encode()).hexdigest()[:24] + return 'rv_i03_' + suffix, 'rv-i03-expiry-' + suffix + + +def addresses(values): + require(isinstance(values, list) and len(values) == len(set(values)) == 4, + 'network topology requires four distinct IPv4 addresses') + for value in values: + address = ipaddress.ip_address(value) + require(address.version == 4 and str(address) == value and address.is_private + and not (address.is_loopback or address.is_unspecified or address.is_link_local or address.is_multicast), + 'network target must be a canonical private unicast fleet address') + return values + + +def resolve(): + values = [] + for index in range(1, 5): + found = {row[4][0] for row in socket.getaddrinfo(f'rustfs-node{index}', PORT, socket.AF_INET, socket.SOCK_STREAM)} + require(len(found) == 1, 'peer alias has ambiguous IPv4 resolution') + values.append(found.pop()) + return addresses(values) + + +def spec(table, blocked): + result = [{'add': {'table': {'family': 'inet', 'name': table}}}] + for chain in ('input', 'output'): + result.append({'add': {'chain': {'family': 'inet', 'table': table, 'name': chain, + 'type': 'filter', 'hook': chain, 'prio': -10, 'policy': 'accept'}}}) + for peer in blocked: + for field in ('sport', 'dport'): + result.append({'add': {'rule': {'family': 'inet', 'table': table, 'chain': chain, + 'comment': f'{table}:{chain}:{peer}:{field}', 'expr': [ + {'match': {'op': '==', 'left': {'meta': {'key': 'nfproto'}}, 'right': 'ipv4'}}, + {'match': {'op': '==', 'left': {'meta': {'key': 'l4proto'}}, 'right': 'tcp'}}, + {'match': {'op': '==', 'left': {'payload': {'protocol': 'ip', + 'field': 'saddr' if chain == 'input' else 'daddr'}}, 'right': peer}}, + {'match': {'op': '==', 'left': {'payload': {'protocol': 'tcp', 'field': field}}, 'right': PORT}}, + {'counter': None}, {'drop': None}]}}}) + return {'nftables': result} + + +def table_present(table): + values = json.loads(command('nft', '-j', 'list', 'tables'))['nftables'] + return any(row.get('table', {}).get('family') == 'inet' and row['table'].get('name') == table for row in values if 'table' in row) + + +def validate_table(value, table, blocked): + expected = spec(table, blocked)['nftables'] + chains = {row['add']['chain']['name']: row['add']['chain'] for row in expected if 'chain' in row['add']} + wanted = {row['add']['rule']['comment']: row['add']['rule'] for row in expected if 'rule' in row['add']} + seen_chains, seen_rules, tables, counters = set(), set(), 0, [] + for row in value.get('nftables', []): + if 'metainfo' in row: + continue + require(len(row) == 1 and next(iter(row)) in {'table', 'chain', 'rule'}, 'foreign network table object') + kind, item = next(iter(row.items())) + require(item.get('family') == 'inet' and (item.get('name') if kind == 'table' else item.get('table')) == table, + 'network table identity changed') + if kind == 'table': + require(set(item) <= {'family', 'name', 'handle'} and tables == 0, 'network table flags or ownership changed') + tables += 1 + elif kind == 'chain': + name = item.get('name') + require(name in chains and name not in seen_chains and {k: v for k, v in item.items() if k != 'handle'} == chains[name], + 'network chain has an unexpected hook, priority or policy') + seen_chains.add(name) + else: + name = item.get('comment') + require(name in wanted and name not in seen_rules, 'foreign or duplicate network rule') + normalized = dict(item) + normalized.pop('handle', None) + normalized.pop('index', None) + expressions = [] + for expression in normalized.get('expr', []): + if 'counter' in expression: + count = expression['counter'] + require(set(expression) == {'counter'} and isinstance(count, dict) and set(count) == {'packets', 'bytes'} + and all(type(count[k]) is int and count[k] >= 0 for k in count), 'invalid network packet counter') + counters.append({'rule': name, **count}) + expressions.append({'counter': None}) + else: + expressions.append(expression) + normalized['expr'] = expressions + require(normalized == wanted[name], 'network rule changed its addresses, TCP port or drop expression') + seen_rules.add(name) + require(tables == 1 and seen_chains == set(chains) and seen_rules == set(wanted) + and len(counters) == len(wanted), 'network table omits a registered rule or chain') + return counters + + +def read_marker(state, owner): + if not (state / MARKER).exists(): + return None + value = json.loads(private_file(state / MARKER)) + table, timer = identities(owner) + values = addresses(value.get('addresses')) + local, blocked = value.get('localIndex'), value.get('blockedIndices') + require(type(local) is int and 0 <= local < 4 and isinstance(blocked, list) and blocked == sorted(set(blocked)) + and 0 < len(blocked) <= 3 and all(type(i) is int and 0 <= i < 4 and i != local for i in blocked) + and value.get('runId') == owner['runId'] and value.get('hostname') == owner['hostname'] + and value.get('binarySha256') == owner['binarySha256'] and value.get('unit') == owner['unit'] + and value.get('schemaVersion') == 1 and value.get('table') == table and value.get('timer') == timer + and value.get('phase') in {'armed', 'active'} and value.get('ttlSeconds') == TTL_SECONDS + and value.get('spec') == spec(table, [values[i] for i in blocked]) + and value.get('scriptSha256') == hashlib.sha256(private_file(state / SCRIPT)).hexdigest(), + 'network ownership, exact rules or expiry script changed; retain namespace') + require(datetime.fromisoformat(value['expiresAt']).tzinfo is not None, 'network timer expiry has no timezone') + return value + + +def prepare(state, owner, request, source): + table, timer = identities(owner) + values = addresses(request.get('addresses')) + local, blocked = request.get('localIndex'), request.get('blockedIndices') + require(values == resolve() and type(local) is int and 0 <= local < 4 + and isinstance(blocked, list) and blocked == sorted(set(blocked)) and 0 < len(blocked) <= 3 + and all(type(i) is int and 0 <= i < 4 and i != local for i in blocked), 'network peer binding differs') + # Binding a socket to this address is an independent ownership check. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as test: + test.bind((values[local], 0)) + require(not (state / MARKER).exists() and not (state / SCRIPT).exists() and not (state / EXPIRY).exists() + and not table_present(table), 'network namespace or table is already occupied') + for suffix in ('.timer', '.service'): + require(command('systemctl', 'show', timer + suffix, '-p', 'LoadState', '--value').strip() == 'not-found', + 'network expiry unit is already occupied') + command('nft', '-j', '-c', '-f', '-', input=json.dumps(spec(table, [values[i] for i in blocked]))) + write(state / SCRIPT, source.encode()) + value = {'schemaVersion': 1, 'runId': owner['runId'], 'hostname': owner['hostname'], + 'unit': owner['unit'], 'binarySha256': owner['binarySha256'], 'table': table, 'timer': timer, + 'addresses': values, 'localIndex': local, 'blockedIndices': blocked, 'ttlSeconds': TTL_SECONDS, + 'expiresAt': (datetime.now(timezone.utc) + timedelta(seconds=TTL_SECONDS)).isoformat(), 'phase': 'armed', + 'scriptSha256': hashlib.sha256(source.encode()).hexdigest(), 'spec': spec(table, [values[i] for i in blocked])} + write(state / MARKER, value) + command('systemd-run', '--quiet', '--unit=' + timer, '--on-active=' + str(TTL_SECONDS) + 's', + '--timer-property=AccuracySec=1s', '--property=RemainAfterExit=yes', '/usr/bin/python3', str(state / SCRIPT), str(state)) + require(command('systemctl', 'show', timer + '.timer', '-p', 'ActiveState', '--value').strip() == 'active', + 'network expiry timer was not armed before mutation') + return {**value, 'state': 'ARMED', 'rulesApplied': False, 'expiryArmedBeforeMutation': True} + + +def activate(state, owner): + value = read_marker(state, owner) + require(value and value['phase'] == 'armed' and not table_present(value['table']) + and not (state / EXPIRY).exists() + and datetime.now(timezone.utc) < datetime.fromisoformat(value['expiresAt']) + and command('systemctl', 'show', value['timer'] + '.timer', '-p', 'ActiveState', '--value').strip() == 'active', + 'network activation requires its still-armed independent timer') + # nft's whole transaction creates only this table; it never flushes rules. + command('nft', '-j', '-f', '-', input=json.dumps(value['spec'])) + counters = validate_table(json.loads(command('nft', '-j', 'list', 'table', 'inet', value['table'])), + value['table'], [value['addresses'][i] for i in value['blockedIndices']]) + value['phase'] = 'active' + write(state / MARKER, value, replace=True) + return {'state': 'ACTIVE', 'table': value['table'], 'counters': counters, + 'onlyExactFleetTCP9000': True, 'unrelatedTablesChanged': False} + + +def observe(state, owner): + value = read_marker(state, owner) + require(value and value['phase'] == 'active' and not (state / EXPIRY).exists() + and datetime.now(timezone.utc) < datetime.fromisoformat(value['expiresAt']) + and command('systemctl', 'show', value['timer'] + '.timer', '-p', 'ActiveState', '--value').strip() == 'active', + 'network partition expired or lost its independent recovery timer') + counters = validate_table(json.loads(command('nft', '-j', 'list', 'table', 'inet', value['table'])), + value['table'], [value['addresses'][i] for i in value['blockedIndices']]) + probes = [] + for index, address in enumerate(value['addresses']): + if index == value['localIndex']: + continue + row = {'peerIndex': index, 'peerAddress': address} + for port, label in ((PORT, 'service'), (22, 'management')): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as client: + client.settimeout(1) + client.bind((value['addresses'][value['localIndex']], 0)) + try: + client.connect((address, port)) + except OSError as exc: + row[label] = {'connected': False, 'error': type(exc).__name__, 'errno': exc.errno} + else: + row[label] = {'connected': True} + require(row['service']['connected'] is (index not in value['blockedIndices']) + and row['management']['connected'] is True, + 'actual service isolation or preserved management connectivity differs') + probes.append(row) + after = validate_table(json.loads(command('nft', '-j', 'list', 'table', 'inet', value['table'])), + value['table'], [value['addresses'][i] for i in value['blockedIndices']]) + require(sum(row['packets'] for row in after) > sum(row['packets'] for row in counters), + 'network isolation did not hit a real owned packet rule') + return {'state': 'ACTIVE', 'localIndex': value['localIndex'], 'table': value['table'], + 'probes': probes, 'countersBefore': counters, 'countersAfter': after, 'managementPort': 22, + 'allServicesRemainRunning': True, 'expiryArmed': True} + + +def release(state, owner, *, expired=False): + value = read_marker(state, owner) + if value is None: + table, _ = identities(owner) + require(not table_present(table), 'network table has no ownership intent; preserve namespace') + return {'state': 'CLEAR', 'gateWasAbsent': True} + # Stop the expiry worker before inspecting/deleting, avoiding a concurrent + # deletion. A TTL worker does not stop its own unit. + if not expired: + command('systemctl', 'stop', value['timer'] + '.timer', value['timer'] + '.service') + counters = [] + if table_present(value['table']): + counters = validate_table(json.loads(command('nft', '-j', 'list', 'table', 'inet', value['table'])), + value['table'], [value['addresses'][i] for i in value['blockedIndices']]) + command('nft', '-j', '-f', '-', input=json.dumps({'nftables': [{'delete': {'table': { + 'family': 'inet', 'name': value['table']}}}]})) + require(not table_present(value['table']), 'owned network table was not removed') + receipt = {'state': 'CLEAR', 'table': value['table'], 'rulesRemaining': 0, + 'removedCounters': counters, 'expiryTriggered': expired or (state / EXPIRY).exists(), + 'unrelatedTablesChanged': False} + if expired: + write(state / EXPIRY, receipt) + else: + for suffix in ('.timer', '.service'): + require(command('systemctl', 'show', value['timer'] + suffix, '-p', 'ActiveState', '--value').strip() in {'inactive', 'failed'}, + 'network expiry unit remains active') + (state / MARKER).unlink() + (state / SCRIPT).unlink() + return receipt + + +def expiry_main(): + require(os.geteuid() == 0 and len(sys.argv) == 2, 'root-owned expiry invocation required') + state = Path(sys.argv[1]) + require(state.is_absolute() and state.parent == Path('/var/lib/rustfs-auto-testing/heal') + and re.fullmatch(r'gh-[1-9][0-9]*-[1-9][0-9]*-h2', state.name), 'expiry namespace is not a registered H2 run') + for entry in (state, *state.parents): + require(not entry.is_symlink() and stat.S_ISDIR(entry.stat().st_mode) and entry.stat().st_uid == 0, + 'expiry namespace ownership changed') + # owner.json is created by the existing node reserve path (mode 0644). + path = state / 'owner.json' + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_uid == 0 and info.st_nlink == 1 and info.st_size < 65536 + and info.st_mode & 0o022 == 0 and not path.is_symlink(), 'expiry owner changed') + owner = json.loads(path.read_text()) + require(owner['runId'] == state.name and owner['hostname'] == socket.gethostname(), 'expiry owner differs from actual host') + print(json.dumps(release(state, owner, expired=True))) + + +if __name__ == '__main__': + expiry_main() diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 498c727d..6fc46cc9 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -38,6 +38,7 @@ case "${case_id}" in heal.coordinator-process-crash) driver=coordinator_crash.py ;; heal.disk-and-node-flapping) driver=flapping.py ;; heal.enospc-inodes-and-quota) driver=capacity.py ;; + heal.network-partition-and-lock-quorum) driver=network.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal_network.py b/test_validation_heal_network.py new file mode 100644 index 00000000..8f929631 --- /dev/null +++ b/test_validation_heal_network.py @@ -0,0 +1,287 @@ +"""Network fault ownership, real kernel rules and negative quorum truth.""" + +import copy +import errno +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from scripts.validation_heal import network, network_gate as gate, fleet_node +from scripts.validation_heal.matrix import AUTOMATIC_CASES, select +from scripts.validation_heal import shards +import test_validation_heal as oracle + + +class NetworkTests(unittest.TestCase): + def scene(self, branch='read-boundary'): + base = oracle.HealReadQuorumBoundaryTest() + snapshot, expected = base.scene(total=16, data=8) + target = shards.map_target(snapshot, base.bucket, expected[0]['key'], 'data', 1) + target = {**target, **next(row for row in target['cohort'] if row['node'] == 'node4')} + return snapshot, target, network.plan_for(snapshot, target, branch) + + def test_both_actual_r_w_and_lock_boundaries_are_independent(self): + for branch in network.BRANCHES: + snapshot, target, plan = self.scene(branch) + self.assertEqual(plan['readQuorum'], 8) + self.assertEqual(plan['writeQuorum'], 9) + self.assertEqual(plan['exclusiveLockQuorum'], 3) + self.assertEqual(plan['sides'][0]['writableDrives'], 8) + self.assertGreaterEqual(plan['sides'][0]['readableSources'], 8) + self.assertTrue(all(side['writableDrives'] < plan['writeQuorum'] for side in plan['sides'])) + if branch in {'read-boundary', 'lock-boundary'}: + self.assertEqual(plan['sides'][0]['readableSources'], 8) + self.assertEqual(plan['sides'][1]['readableSources'], 7) + self.assertFalse(plan['gates']) + self.assertTrue(all(side['lockClients'] < 3 for side in plan['sides'])) + else: + self.assertEqual(plan['sides'][0]['lockClients'], 3) + self.assertTrue(plan['sides'][0]['exclusiveLockQuorumPresent']) + self.assertEqual(plan['gates'], {'node1': [0, 1, 2, 3]}) + self.assertEqual(plan['sides'][1]['readableSources'], 3) + for index, node in enumerate(snapshot): + for other in plan['blocked'][node]: + self.assertIn(index, plan['blocked'][list(snapshot)[other]]) + + def test_foreign_geometry_duplicate_slots_and_wrong_defect_member_are_rejected(self): + for kind in ('geometry', 'duplicate', 'foreign-node', 'target-node', 'branch'): + snapshot, target, _ = self.scene() + branch = 'read-boundary' + if kind == 'geometry': target['logical']['erasure']['data_blocks'] = 12 + elif kind == 'duplicate': target['cohort'][-1] = copy.deepcopy(target['cohort'][0]) + elif kind == 'foreign-node': target['cohort'][-1]['node'] = 'foreign' + elif kind == 'target-node': target['node'] = 'node1' + else: branch = 'invented' + with self.subTest(kind=kind), self.assertRaises(ValueError): + network.plan_for(snapshot, target, branch) + + def test_partial_coverage_cannot_be_converted_into_global_idle(self): + value = {'clusterStatusComplete': False, 'state': 'degraded', + 'coverage': {'expected': 4, 'responded': 2, 'unknown': 2, 'reasons': ['partition']}} + network.validate_partial(value, 2) + for kind in ('complete', 'idle', 'hidden-peer', 'no-reason', 'untyped'): + broken = copy.deepcopy(value) + if kind == 'complete': broken['clusterStatusComplete'] = True + elif kind == 'idle': broken['state'] = 'idle' + elif kind == 'hidden-peer': broken['coverage']['unknown'] = 0 + elif kind == 'no-reason': broken['coverage']['reasons'] = [] + else: broken['coverage']['responded'] = True + with self.subTest(kind=kind), self.assertRaises(ValueError): + network.validate_partial(broken, 2) + + def negative(self, state='deferred'): + expected = [{'key': 'objects/a', 'versionId': 'v0'}, {'key': 'objects/a', 'versionId': 'v1'}] + target = {'key': 'objects/a', 'versionId': 'v1'} + objects = [{'identity': {'bucket': 'owned', 'object': 'objects/a', 'versionId': version, + 'bucketIncarnationId': 'incarnation', 'poolIndex': 0, 'setIndex': 0}, + 'disposition': {'state': 'verified_healthy' if version == 'v0' else state}} for version in ('v0', 'v1')] + failed = state == 'failed' + current = {'summary': 'stopped' if failed else 'finished', 'truncated': False, + 'outcome': {'execution': {'state': 'completed_with_errors' if failed else 'completed'}, + 'coverage': 'complete', 'objectsTruncated': False, 'objects': objects, + 'counters': {'processed': 2, 'healed': 0, 'unchanged': 1, 'failed': int(failed), + 'skipped': int(not failed), 'unknown': int(state == 'unknown'), + 'attemptFailures': int(failed), 'overflowed': False}}} + return current, expected, target + + def test_failed_deferred_and_unknown_remain_unresolved_without_unsafe_repair(self): + for state in ('failed', 'deferred', 'unknown'): + current, expected, target = self.negative(state) + self.assertFalse(network.validate_negative_task(current, 'owned', expected, target)['unsafeRepairClaimed']) + current, expected, target = self.negative() + for kind in ('healed', 'false-healthy', 'hidden-failure', 'foreign', 'duplicate', 'lost-target', 'uncounted-unknown'): + broken = copy.deepcopy(current) + objects, counts = broken['outcome']['objects'], broken['outcome']['counters'] + if kind == 'healed': counts['healed'] = 1 + elif kind == 'false-healthy': objects[-1]['disposition']['state'] = 'verified_healthy' + elif kind == 'hidden-failure': objects[-1]['disposition']['state'] = 'failed' + elif kind == 'foreign': objects[-1]['identity']['bucket'] = 'foreign' + elif kind == 'duplicate': objects[-1] = copy.deepcopy(objects[0]) + elif kind == 'lost-target': objects.pop(); counts.update(processed=1, skipped=0) + else: objects[-1]['disposition']['state'] = 'unknown' + with self.subTest(kind=kind), self.assertRaises(ValueError): + network.validate_negative_task(broken, 'owned', expected, target) + + def test_structured_abort_preserves_untraversable_responsibility_without_success(self): + current, expected, target = self.negative() + current['outcome'].update(execution={'state': 'aborted', 'reason': 'untraversable'}, coverage='partial', objects=[]) + current['outcome']['counters'].update(processed=0, unchanged=0, skipped=0) + network.validate_negative_task(current, 'owned', expected, target) + current['outcome']['coverage'] = 'complete' + with self.assertRaises(ValueError): network.validate_negative_task(current, 'owned', expected, target) + + def test_write_identity_mismatch_stays_pending_and_no_retry_is_issued(self): + row = {'key': 'unique', 'payload': {'size': 1, 'sha256': 'a'}, + 'observed': {'acknowledged': True, 'resultUnknown': False, 'response': {'VersionId': 'ack-version'}}} + class Fleet: + def record(self, label, value): return value + observed = {'acknowledged': True, 'resultUnknown': False, 'payload': row['payload'], 'response': {'VersionId': 'foreign'}} + with patch.object(network, 'Probe') as probe, self.assertRaises(ValueError): + probe.return_value.s3.return_value = observed + network.resolve_writes(Fleet(), {'endpoints': {'node1': 'url', 'node2': 'url2'}}, 'bucket', [row]) + self.assertNotIn('resolution', row) + + def test_owned_address_allowlist_rejects_nonfleet_destinations(self): + gate.addresses(['10.0.0.1', '10.0.0.2', '10.0.0.3', '10.0.0.4']) + for address in ('127.0.0.1', '0.0.0.0', '169.254.1.1', '8.8.8.8', '224.0.0.1', '::1', '10.0.0.2'): + with self.subTest(address=address), self.assertRaises(ValueError): + gate.addresses([address, '10.0.0.2', '10.0.0.3', '10.0.0.4']) + + def table(self): + table, blocked = 'rv_i03_test', ['10.0.0.2', '10.0.0.3'] + rows = [] + for entry in gate.spec(table, blocked)['nftables']: + kind, value = next(iter(entry['add'].items())) + value = copy.deepcopy(value) + value['handle'] = len(rows) + 1 + if kind == 'rule': + value['expr'][-2]['counter'] = {'packets': 3, 'bytes': 180} + rows.append({kind: value}) + return {'nftables': [{'metainfo': {'json_schema_version': 1}}] + rows}, table, blocked + + def test_exact_rules_exclude_ssh_and_reject_foreign_expressions_before_deletion(self): + raw, table, blocked = self.table() + counters = gate.validate_table(raw, table, blocked) + self.assertEqual(len(counters), 8) + for kind in ('ssh', 'address', 'accept', 'priority', 'foreign-rule', 'missing', 'chain'): + broken = copy.deepcopy(raw) + rule = next(row['rule'] for row in broken['nftables'] if 'rule' in row) + chain = next(row['chain'] for row in broken['nftables'] if 'chain' in row) + if kind == 'ssh': rule['expr'][3]['match']['right'] = 22 + elif kind == 'address': rule['expr'][2]['match']['right'] = '10.0.0.9' + elif kind == 'accept': rule['expr'][-1] = {'accept': None} + elif kind == 'priority': chain['prio'] = -100 + elif kind == 'foreign-rule': rule['comment'] = 'foreign' + elif kind == 'missing': broken['nftables'].pop() + else: chain['hook'] = 'forward' + with self.subTest(kind=kind), self.assertRaises(ValueError): + gate.validate_table(broken, table, blocked) + + def test_all_peer_directions_and_packet_counter_hits_are_required(self): + snapshot, _, plan = self.scene() + nodes = list(snapshot) + observed = {node: {'state': 'ACTIVE', 'localIndex': index, 'expiryArmed': True, 'process': {'pid': 123}, + 'probes': [{'peerIndex': other, 'service': {'connected': other not in plan['blocked'][node]}, + 'management': {'connected': True}} for other in range(4) if other != index], + 'countersBefore': [{'packets': 0}], 'countersAfter': [{'packets': 1}]} for index, node in enumerate(nodes)} + network.validate_witness(observed, plan, nodes) + for kind in ('stopped-service', 'management-blocked', 'still-connected', 'missing-peer', 'no-hit', 'expired'): + broken = copy.deepcopy(observed) + first = broken[nodes[0]] + if kind == 'stopped-service': first['process']['pid'] = 0 + elif kind == 'management-blocked': first['probes'][0]['management']['connected'] = False + elif kind == 'still-connected': first['probes'][-1]['service']['connected'] = True + elif kind == 'missing-peer': first['probes'].pop() + elif kind == 'no-hit': first['countersAfter'][0]['packets'] = 0 + else: first['expiryArmed'] = False + with self.subTest(kind=kind), self.assertRaises(ValueError): network.validate_witness(broken, plan, nodes) + + def test_source_branches_and_existing_dispatch_contract_remain_complete(self): + self.assertEqual(select(network.CASE_ID)[0]['variants'], list(network.BRANCHES)) + self.assertIn(network.CASE_ID, AUTOMATIC_CASES) + source = Path('scripts/validation_heal/run_case.sh').read_text() + self.assertIn(network.CASE_ID + ') driver=network.py', source) + + +class NetworkExpiryTests(unittest.TestCase): + owner = {'runId': 'gh-123-1-h2', 'hostname': 'node1', 'unit': 'owned.service', + 'runtimeProfile': 'lock-quorum', 'binarySha256': 'a' * 64} + peers = ['10.17.0.1', '10.17.0.2', '10.17.0.3', '10.17.0.4'] + + def prepare(self, folder, *, timer_active=True): + calls = [] + def command(*args, input=None): + calls.append((args, input)) + if args[:4] == ('nft', '-j', 'list', 'tables'): return '{"nftables":[]}' + if 'LoadState' in args: return 'not-found' + if 'ActiveState' in args: return 'active' if timer_active else 'inactive' + return '' + source = Path(gate.__file__).read_text() + with patch.object(gate, 'command', side_effect=command), patch.object(gate, 'resolve', return_value=self.peers), patch.object(gate.socket, 'socket'): + value = gate.prepare(folder, self.owner, {'addresses': self.peers, 'localIndex': 0, 'blockedIndices': [2, 3]}, source) + return value, calls + + def test_independent_expiry_is_armed_and_intent_is_durable_before_any_rule_mutation(self): + with tempfile.TemporaryDirectory() as folder: + state = Path(folder) + value, calls = self.prepare(state) + self.assertEqual(value['state'], 'ARMED') + self.assertEqual(json.loads((state / gate.MARKER).read_text())['phase'], 'armed') + self.assertTrue((state / gate.SCRIPT).is_file()) + self.assertEqual((state / gate.MARKER).stat().st_mode & 0o777, 0o600) + self.assertTrue(any(args[0] == 'systemd-run' for args, _ in calls)) + self.assertFalse(any(args[:4] == ('nft', '-j', '-f', '-') for args, _ in calls)) + + def test_failed_timer_keeps_undo_intent_and_never_installs_rules(self): + with tempfile.TemporaryDirectory() as folder: + state = Path(folder) + with self.assertRaisesRegex(ValueError, 'timer was not armed'): + self.prepare(state, timer_active=False) + self.assertTrue((state / gate.MARKER).is_file()) + self.assertEqual(json.loads((state / gate.MARKER).read_text())['phase'], 'armed') + + def test_lost_timer_refuses_activation_before_touching_nft(self): + with tempfile.TemporaryDirectory() as folder: + value, _ = self.prepare(Path(folder)) + with patch.object(gate, 'read_marker', return_value=value), patch.object(gate, 'table_present', return_value=False), \ + patch.object(gate, 'command', return_value='inactive') as command, self.assertRaises(ValueError): + gate.activate(Path(folder), self.owner) + self.assertTrue(all(call.args[0] != 'nft' for call in command.call_args_list)) + + def test_foreign_rule_blocks_cleanup_instead_of_deleting_the_table(self): + with tempfile.TemporaryDirectory() as folder: + state = Path(folder) + value, _ = self.prepare(state) + broken = {'nftables': [{'table': {'family': 'inet', 'name': value['table']}}, + {'rule': {'family': 'inet', 'table': value['table'], 'chain': 'input', + 'comment': 'foreign', 'expr': [{'accept': None}]}}]} + def command(*args, input=None): + if args[:4] == ('nft', '-j', 'list', 'table'): return json.dumps(broken) + return '' + with patch.object(gate, 'read_marker', return_value=value), patch.object(gate, 'table_present', return_value=True), \ + patch.object(gate, 'command', side_effect=command) as observed, self.assertRaises(ValueError): + gate.release(state, self.owner) + self.assertTrue((state / gate.MARKER).exists()) + self.assertFalse(any(call.args[:4] == ('nft', '-j', '-f', '-') for call in observed.call_args_list)) + + +class NetworkKernelTests(unittest.TestCase): + @unittest.skipUnless(sys.platform == 'linux', 'requires an isolated Linux network namespace') + def test_real_nft_json_round_trip_and_foreign_rule_refusal(self): + self.assertIsNotNone(shutil.which('nft'), 'Linux CI must install nftables; this gate may not skip silently') + program = '''import copy,json +from scripts.validation_heal import network_gate as gate +table='rv_i03_kernel_test' +peers=['10.17.0.2','10.17.0.3'] +spec=gate.spec(table,peers) +gate.command('nft','-j','-c','-f','-',input=json.dumps(spec)) +gate.command('nft','-j','-f','-',input=json.dumps(spec)) +raw=json.loads(gate.command('nft','-j','list','table','inet',table)) +assert len(gate.validate_table(raw,table,peers))==8 +foreign={'nftables':[{'add':{'rule':{'family':'inet','table':table,'chain':'input','expr':[{'accept':None}]}}}]} +gate.command('nft','-j','-f','-',input=json.dumps(foreign)) +try: + gate.validate_table(json.loads(gate.command('nft','-j','list','table','inet',table)),table,peers) +except ValueError: + pass +else: + raise AssertionError('foreign rule was accepted') +gate.command('nft','-j','-f','-',input=json.dumps({'nftables':[{'delete':{'table':{'family':'inet','name':table}}}]})) +assert not gate.table_present(table) +print('PASS: actual private-network kernel rules and deletion guards') +''' + command = ['unshare', '--net', sys.executable, '-B', '-c', program] + if os.geteuid() != 0: command = ['sudo', '-n', *command] + result = subprocess.run(command, capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn('PASS: actual private-network kernel rules', result.stdout) + + +if __name__ == '__main__': + unittest.main() From 314bc36aa6db5902724a7dac054662774acf4fed Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 08:34:42 +0800 Subject: [PATCH 098/343] test: retain exact kernel rule mismatch diagnostics --- scripts/validation_heal/network_gate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/network_gate.py b/scripts/validation_heal/network_gate.py index 59a42b8e..32bc15c2 100644 --- a/scripts/validation_heal/network_gate.py +++ b/scripts/validation_heal/network_gate.py @@ -149,7 +149,8 @@ def validate_table(value, table, blocked): else: expressions.append(expression) normalized['expr'] = expressions - require(normalized == wanted[name], 'network rule changed its addresses, TCP port or drop expression') + require(normalized == wanted[name], 'network rule changed its addresses, TCP port or drop expression: ' + + json.dumps({'observed': normalized, 'expected': wanted[name]}, sort_keys=True)) seen_rules.add(name) require(tables == 1 and seen_chains == set(chains) and seen_rules == set(wanted) and len(counters) == len(wanted), 'network table omits a registered rule or chain') From f8b538a9fdc19da82d3556aa998481fe420c74c7 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 08:36:36 +0800 Subject: [PATCH 099/343] test: match the canonical nft payload rule representation --- scripts/validation_heal/network_gate.py | 4 ++-- test_validation_heal_network.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/validation_heal/network_gate.py b/scripts/validation_heal/network_gate.py index 32bc15c2..6737d753 100644 --- a/scripts/validation_heal/network_gate.py +++ b/scripts/validation_heal/network_gate.py @@ -98,8 +98,8 @@ def spec(table, blocked): for field in ('sport', 'dport'): result.append({'add': {'rule': {'family': 'inet', 'table': table, 'chain': chain, 'comment': f'{table}:{chain}:{peer}:{field}', 'expr': [ - {'match': {'op': '==', 'left': {'meta': {'key': 'nfproto'}}, 'right': 'ipv4'}}, - {'match': {'op': '==', 'left': {'meta': {'key': 'l4proto'}}, 'right': 'tcp'}}, + # IP/TCP payload selectors imply their protocol guards. + # nft removes redundant explicit meta guards on export. {'match': {'op': '==', 'left': {'payload': {'protocol': 'ip', 'field': 'saddr' if chain == 'input' else 'daddr'}}, 'right': peer}}, {'match': {'op': '==', 'left': {'payload': {'protocol': 'tcp', 'field': field}}, 'right': PORT}}, diff --git a/test_validation_heal_network.py b/test_validation_heal_network.py index 8f929631..2c4d2f3b 100644 --- a/test_validation_heal_network.py +++ b/test_validation_heal_network.py @@ -153,8 +153,8 @@ def test_exact_rules_exclude_ssh_and_reject_foreign_expressions_before_deletion( broken = copy.deepcopy(raw) rule = next(row['rule'] for row in broken['nftables'] if 'rule' in row) chain = next(row['chain'] for row in broken['nftables'] if 'chain' in row) - if kind == 'ssh': rule['expr'][3]['match']['right'] = 22 - elif kind == 'address': rule['expr'][2]['match']['right'] = '10.0.0.9' + if kind == 'ssh': rule['expr'][1]['match']['right'] = 22 + elif kind == 'address': rule['expr'][0]['match']['right'] = '10.0.0.9' elif kind == 'accept': rule['expr'][-1] = {'accept': None} elif kind == 'priority': chain['prio'] = -100 elif kind == 'foreign-rule': rule['comment'] = 'foreign' From 7698230c53184168d0174e21e4aa34c0ca158aad Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 08:37:18 +0800 Subject: [PATCH 100/343] test: refuse unguarded IPv6 peer aliases before partition --- scripts/validation_heal/network_gate.py | 5 ++++- test_validation_heal_network.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/network_gate.py b/scripts/validation_heal/network_gate.py index 6737d753..10177d40 100644 --- a/scripts/validation_heal/network_gate.py +++ b/scripts/validation_heal/network_gate.py @@ -83,7 +83,10 @@ def addresses(values): def resolve(): values = [] for index in range(1, 5): - found = {row[4][0] for row in socket.getaddrinfo(f'rustfs-node{index}', PORT, socket.AF_INET, socket.SOCK_STREAM)} + rows = socket.getaddrinfo(f'rustfs-node{index}', PORT, socket.AF_UNSPEC, socket.SOCK_STREAM) + require(rows and all(row[0] == socket.AF_INET for row in rows), + 'peer alias exposes an unguarded IPv6 path; no partition will be installed') + found = {row[4][0] for row in rows} require(len(found) == 1, 'peer alias has ambiguous IPv4 resolution') values.append(found.pop()) return addresses(values) diff --git a/test_validation_heal_network.py b/test_validation_heal_network.py index 2c4d2f3b..6556949e 100644 --- a/test_validation_heal_network.py +++ b/test_validation_heal_network.py @@ -133,6 +133,12 @@ def test_owned_address_allowlist_rejects_nonfleet_destinations(self): with self.subTest(address=address), self.assertRaises(ValueError): gate.addresses([address, '10.0.0.2', '10.0.0.3', '10.0.0.4']) + def test_ipv6_alias_cannot_bypass_the_registered_ipv4_partition(self): + rows = [(gate.socket.AF_INET, gate.socket.SOCK_STREAM, 6, '', ('10.0.0.1', 9000)), + (gate.socket.AF_INET6, gate.socket.SOCK_STREAM, 6, '', ('fd00::1', 9000, 0, 0))] + with patch.object(gate.socket, 'getaddrinfo', return_value=rows), self.assertRaisesRegex(ValueError, 'unguarded IPv6'): + gate.resolve() + def table(self): table, blocked = 'rv_i03_test', ['10.0.0.2', '10.0.0.3'] rows = [] From 9b93e7570db7b4af39608ba4876e0852952261d9 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 08:39:08 +0800 Subject: [PATCH 101/343] docs: record network kernel validation and live queue --- docs/validation-migration-map.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index e54ce8e4..73934603 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1385,5 +1385,17 @@ must redeem its actual VersionId, bytes, metadata and tags on every peer. Foreign rules, lost expiry guards, unknown ACKs or failed recovery retain DIRTY evidence. No I03 live PASS is claimed. The implementation now has 129 automatic executors (74 non-Heal, 55 Heal); 29 Heal executors remain. The focused suite -contains 332 tests, including the real Linux private-network nftables gate; +contains 333 tests, including the real Linux private-network nftables gate; six platform-specific tests are skipped locally on macOS. + +All six I03 checks passed at `7698230c53184168d0174e21e4aa34c0ca158aad`. +The Linux gate installed, inspected and removed real kernel rules inside a +private network namespace and rejected an injected foreign rule. The adapter +uses nft's canonical IP/TCP payload expressions, retains exact address/port/drop +checks, and refuses aliases with an unguarded IPv6 route before mutation. +The serial I03 live plan binds this immutable SHA and the existing candidate +package/tool runs; it has no live verdict yet. Current RustFS CI has runner +shutdown failures in Workspace Test and Lint, Connect Short Credential Boundary +and S3 Implemented Tests; their original job logs show shutdown/cancellation, +not a completed passing check. The two optional workflow files remain the only +RustFS changes. From c24d590d52edc457f9af180ecc050b1e90a703d5 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 08:55:19 +0800 Subject: [PATCH 102/343] test: migrate exact multi-pool Heal target isolation --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 48 ++ scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 16 +- scripts/validation_heal/healthy.py | 7 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/multi_pool.py | 502 ++++++++++++++++++ scripts/validation_heal/multi_pool_layout.py | 42 ++ scripts/validation_heal/run_case.sh | 1 + scripts/validation_heal/shards.py | 24 +- test_validation_heal_multi_pool.py | 274 ++++++++++ 11 files changed, 908 insertions(+), 14 deletions(-) create mode 100644 scripts/validation_heal/multi_pool.py create mode 100644 scripts/validation_heal/multi_pool_layout.py create mode 100644 test_validation_heal_multi_pool.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 8a2e69e1..fe17f79f 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -22,6 +22,7 @@ on: - test_validation_heal_mrf.py - test_validation_heal_unformatted.py - test_validation_heal_performance.py + - test_validation_heal_multi_pool.py - test_validation_heal_scopes.py - test_validation_heal_bucket_metadata.py - test_validation_heal_write_quorum.py @@ -67,6 +68,7 @@ on: - test_validation_heal_mrf.py - test_validation_heal_unformatted.py - test_validation_heal_performance.py + - test_validation_heal_multi_pool.py - test_validation_heal_scopes.py - test_validation_heal_bucket_metadata.py - test_validation_heal_write_quorum.py @@ -114,7 +116,7 @@ jobs: sudo apt-get update sudo apt-get install --no-install-recommends -y nftables fi - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 73934603..acb4d02b 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1399,3 +1399,51 @@ shutdown failures in Workspace Test and Lint, Connect Short Credential Boundary and S3 Implemented Tests; their original job logs show shutdown/cancellation, not a completed passing check. The two optional workflow files remain the only RustFS changes. + + +H06 `heal.multi-pool-target-isolation` now dispatches all three original H5 +branches: selected pool, selected set and unselected control fault. It reuses +the four current Heal nodes and sixteen original independent mounts. Two +separate endpoint-expansion arguments create two actual pools with two +four-peer sets each; the candidate layout/parser source binds the argument +order and exact public pool cmdlines. The runtime explicitly uses set width +four and EC:2. Full format/UUID/set mappings and decoded object geometry must +prove actual cross-node EC 2+2 before any fault. + +Bounded DB and two-version DV fixtures must populate all four observed +pool/set combinations. Every acknowledged version is mapped without an +assumed placement hash or internal PUT selector. A measured stop of one owned +peer must leave three physical slots per set and redeem every original +version from all three survivors; this is process loss, not platform power +loss. Every selected and control scope contains a distinct DB fault and a +historical DV fault. The control branch also gates one exact unselected root +read-only inside the candidate service namespace, witnessing EROFS before +and after selected Heal without remounting the host's shared drive. + +Exact selected version responsibility, counters, settings, pool/set/disk +identities, public pool capacity and logical quota policy are checked. +Unselected missing parts must remain missing before any target GET or rescue. +Only after that physical proof may independent control tasks repair them. +Complete physical, same-role streaming and all-peer version/metadata/tag +manifests are required. Uncertain seed requests are registered before calls, +retain durable request/ACK files, stop further mutation and require exact +all-peer resolution; unresolved bucket mutations or ACK/version drift retain +DIRTY namespaces and block later branches. No H06 live PASS is claimed. + +The current implementation has 130 automatic executors (74 non-Heal, 56 Heal); +28 Heal executors remain. The complete local validation suite ran 352 tests +successfully with six Linux-only skips on macOS. Catalog source IDs/hashes, +84/301/532 Heal matrix, Python syntax, Bash syntax, ShellCheck error checks, +scoped actionlint and whitespace checks passed. The pre-H06 documentation +HEAD `9b93e7570db7b4af39608ba4876e0852952261d9` has all six CI checks passing; +H06 still needs its own CI and serial live run. + +C09 run `38098069878` completed FAIL with cleanup PASS. All eight H1/H2 +branches retain failures: some injected foreign/future formats prevented +readiness or repair admission, while other negative tasks did not satisfy +failed-execution/incomplete-responsibility evidence. D08 run `38099278528` +completed FAIL with cleanup PASS: SSE-S3, SSE-KMS and encrypted-versioned +branches passed on both topologies, but SSE-C ACK/GET classification failed +on both. Neither run increases the 47/158 complete canonical live PASS count. +The RustFS aggregate Test and Lint check also failed because its three +runner-shutdown lanes failed; its final summary preserves those exact failures. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index d0ce920f..b038d922 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 8414b149..7d8a54e8 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -25,7 +25,7 @@ MOUNTS = tuple(Path(f"/data/rustfs{index}") for index in range(1, 5)) STATE_ROOT = Path("/var/lib/rustfs-auto-testing/heal") -RUN_PATTERN = re.compile(r"gh-[1-9][0-9]*-[1-9][0-9]*-(?:h1|h2)") +RUN_PATTERN = re.compile(r"gh-[1-9][0-9]*-[1-9][0-9]*-(?:h1|h2|h5)") BACKGROUND_CONFIG = {"RUSTFS_SCANNER_ENABLED": "false", "RUSTFS_HEAL_ENABLED": "true", "RUSTFS_HEAL_AUTO_HEAL_ENABLE": "false", "RUSTFS_HEAL_MRF_ENABLE": "false", "RUSTFS_SHARD_INTEGRITY_WRITE": "true", "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED": "true", @@ -33,7 +33,9 @@ def runtime_config(profile): - require(profile in {"manual", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(profile in {"manual", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + if profile == "multi-pool": + return {**BACKGROUND_CONFIG, "RUSTFS_ERASURE_SET_DRIVE_COUNT": "4", "RUSTFS_STORAGE_CLASS_STANDARD": "EC:2"} if profile == "bounded-capacity": return {**BACKGROUND_CONFIG, "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true"} if profile == "lock-quorum": @@ -177,16 +179,22 @@ def reserve(request): for name in ("access", "secret"): require(bool(re.fullmatch(r"[A-Za-z0-9]{16,80}", request[name])), "invalid runtime credential") volumes = request["volumes"] - require(isinstance(volumes, list) and len(volumes) in {4, 16}, "unsupported automatic Heal topology") + require(isinstance(volumes, list) and len(volumes) in {2, 4, 16}, "unsupported automatic Heal topology") local_paths = [str(root / "data") for root in paths(run_id)[1]] h1 = run_id.endswith("-h1") require(profile != "lock-quorum" or not h1, "eight-parity lock boundary requires actual sixteen-drive H2") expected = local_paths if h1 else [f"http://rustfs-node{node}:9000{path}" for node in range(1, 5) for path in local_paths] + if profile == "multi-pool": + require(run_id.endswith("-h5"), "multi-pool service requires its explicit H5 namespace") + expected = [f"http://rustfs-node{{1...4}}:9000/data/rustfs{{{first}...{first + 1}}}/heal-validation/{run_id}/data" + for first in (1, 3)] + else: + require(not run_id.endswith("-h5"), "H5 cannot use an H1/H2 runtime") if profile == "scope-isolation": require(not h1, "two-set runtime requires the four-node topology") expected = [f"http://rustfs-node{node}:9000{path}" for path in local_paths for node in range(1, 5)] - require(volumes == expected, "endpoint sequence differs from H1/H2 deployment contract") + require(volumes == expected, "endpoint sequence differs from the explicit Heal deployment contract") state, roots = paths(run_id) owner = {"runId": run_id, "unit": unit(run_id), "hostname": observed["hostname"], "mounts": observed["mounts"], "binarySha256": binary_sha, diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 26f1886c..325a8c9e 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"manual", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -254,6 +254,11 @@ def deploy(self, binary_sha, bucket, access, secret, *, before_start=None): paths = checked[self.nodes[0]]["dataRoots"] volumes = paths if len(self.nodes) == 1 else [f"http://rustfs-node{index}:9000{path}" for index in range(1, 5) for path in paths] + if self.runtime_profile == "multi-pool": + require(len(self.nodes) == 4, "H5 requires all four current Heal nodes") + try: from .multi_pool_layout import volumes as pool_volumes + except ImportError: from multi_pool_layout import volumes as pool_volumes + volumes = pool_volumes(self.run_id) if self.runtime_profile == "scope-isolation": require(len(self.nodes) == 4, "two-set isolation requires the four-node topology") volumes = [f"http://rustfs-node{index}:9000{path}" for path in paths for index in range(1, 5)] diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 244b7028..3e525419 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/multi_pool.py b/scripts/validation_heal/multi_pool.py new file mode 100644 index 00000000..cbd6d4ff --- /dev/null +++ b/scripts/validation_heal/multi_pool.py @@ -0,0 +1,502 @@ +#!/usr/bin/env python3 +"""H06: real two-pool/two-set selection with unselected faults preserved until proof.""" + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys + +try: + from .heal_probe import Probe, compare_inventories, payload, require, save + from .heal_workload import Recorder, RecordedCallError, verify_seed + from .healthy import BASE_OPTIONS, Fleet, physical_content + from .inspector import verify as verify_inspector + from .matrix import select + from .multi_pool_layout import validate as validate_layout, volumes + from .shards import files, map_target, run_task, validate_fault, validate_repaired_physical + from .scopes import physical_selection, verify_parts, REFERENCE as SELECTOR_REFERENCE + from .stop import idle_fleet + from .versions import public_version_id + from .write_quorum import endpoint_refused, expected_reads + from ..validation_pool.pool_status import validate_pool_inventory +except ImportError: + from heal_probe import Probe, compare_inventories, payload, require, save + from heal_workload import Recorder, RecordedCallError, verify_seed + from healthy import BASE_OPTIONS, Fleet, physical_content + from inspector import verify as verify_inspector + from matrix import select + from multi_pool_layout import validate as validate_layout, volumes + from shards import files, map_target, run_task, validate_fault, validate_repaired_physical + from scopes import physical_selection, verify_parts, REFERENCE as SELECTOR_REFERENCE + from stop import idle_fleet + from versions import public_version_id + from write_quorum import endpoint_refused, expected_reads + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'validation_pool')) + from pool_status import validate_pool_inventory + + +CASE_ID = 'heal.multi-pool-target-isolation' +BRANCHES = ('selected-pool', 'selected-set', 'unselected-control-fault') +SIZE = 1024**2 + 73 +REFERENCE = {'candidateCommit': SELECTOR_REFERENCE['candidateCommit'], 'selectors': SELECTOR_REFERENCE['sourceFiles'], + 'layoutSourceFiles': ['crates/ecstore/src/layout/disks_layout.rs', 'crates/utils/src/string.rs'], + 'topology': 'H5: two endpoint-expansion arguments, eight original independent mounts per pool, two four-peer sets per pool', + 'storageClass': 'explicit EC:2; every actual object must decode as 2+2', + 'boundedFixture': {'DBKeys': 48, 'DVKeys': 24, 'versionsPerDVKey': 2, 'bytesPerVersion': SIZE}, + 'placement': 'observe actual full-set XL2/version authority; no assumed pool hash or internal PUT selector', + 'scope': 'pool selector must match admitted pool; an unrestricted set may be null in the raw receipt, with independent exact physical binding', + 'control': 'unselected defects stay missing before any target GET or rescue; repair them in independent tasks only after proof'} + + +def fixture_call(recorder, ledger, operation, params, body=None): + row = {'operation': operation, 'params': params, 'resolved': False} + if body is not None: + row['payload'] = {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()} + ledger.append(row) # Register before the durable recorder or the network can fail. + try: + response = recorder.call(operation, params, body) + except RecordedCallError as exc: + row['observation'] = exc.observation + raise + row['observation'] = {'acknowledged': True, 'resultUnknown': False, 'response': response} + if operation == 'put_object': + version_id = response.get('VersionId', 'null') + require((params['Bucket'].endswith('-dv') and version_id not in {None, 'null'}) + or (not params['Bucket'].endswith('-dv') and version_id == 'null'), + 'H06 versioning mode differs from its exact PUT ACK') + row['versionId'] = version_id + row['resolved'] = True + return response + + +def resolve_fixture_inventory(rows, inventory): + """Resolve an interrupted seed against exact prior ACKs, never retry it.""" + entries = inventory['entries'] + require(all(not entry['deleteMarker'] for entry in entries), 'seed resolution contains unexpected markers') + consumed, resolutions = set(), [] + for row in rows: + require(row['operation'] == 'put_object', 'uncertain bucket mutation needs independent resolution; retain namespace') + params = row['params'] + matches = [entry for entry in entries if entry['key'] == params['Key'] and entry['payload'] == row['payload'] + and entry['properties'].get('ContentType') == params['ContentType'] + and entry['properties'].get('Metadata') == params['Metadata'] + and entry['tags'] == [{'Key': 'fixture', 'Value': 'H06'}, {'Key': 'suite', 'Value': 'heal'}]] + require(len(matches) <= 1, 'seed request has multiple or ambiguous committed versions') + observed = row.get('observation', {'acknowledged': False, 'resultUnknown': True}) + if matches: + entry = matches[0] + version = entry['versionId'] + require((params['Bucket'].endswith('-dv') and version not in {None, 'null'}) + or (not params['Bucket'].endswith('-dv') and version == 'null'), 'resolved seed versioning drift') + if observed.get('acknowledged'): + require(version == observed['response'].get('VersionId', 'null'), 'resolved seed replaced its PUT ACK version') + else: + require(observed.get('resultUnknown') is True, 'refused fixture PUT committed') + identity = entry['key'], version + require(identity not in consumed, 'seed resolution reused one version for different requests') + consumed.add(identity) + resolutions.append({'key': params['Key'], 'state': 'COMMITTED', 'versionId': version}) + else: + require(not observed.get('acknowledged') and not row['resolved'], 'a fixture PUT ACK cannot be redeemed') + resolutions.append({'key': params['Key'], 'state': 'ABSENT'}) + require(consumed == {(entry['key'], entry['versionId']) for entry in entries}, + 'seed resolution omitted foreign, altered or unexplained versions') + latest = {row['key']: row['versionId'] for row in resolutions if row['state'] == 'COMMITTED'} + require(all(entry['isLatest'] is (latest[entry['key']] == entry['versionId']) for entry in entries), + 'resolved fixture latest-version identity drifted') + return resolutions + + +def resolve_pending_fixture(fleet, binding, ledger): + pending = [row for row in ledger if not row['resolved']] + require(all(row['operation'] == 'put_object' for row in pending), + 'uncertain bucket mutation needs independent resolution; retain namespace') + for bucket in sorted({row['params']['Bucket'] for row in pending}): + rows = [row for row in ledger if row['operation'] == 'put_object' and row['params']['Bucket'] == bucket] + inventories = [fleet.record('failed-seed-all-peer-resolution', Probe(binding, endpoint).inventory(bucket, + unversioned_current_tags=not bucket.endswith('-dv'))) for endpoint in binding['endpoints']] + proofs = [resolve_fixture_inventory(rows, inventory) for inventory in inventories] + require(all(proof == proofs[0] for proof in proofs), 'seed resolution disagrees across original peers') + for inventory in inventories[1:]: + compare_inventories(inventories[0], inventory) + for row in pending: + if row['params']['Bucket'] == bucket: + row['resolved'] = True + row['resolution'] = proofs[0] + return {'allOriginalRequestsResolved': True, 'automaticRetries': 0} + + +def seed_fixtures(probe, directory, buckets, ledger): + recorder = Recorder(probe, directory / 'fixture') + expected = {} + for bucket, count, versions in ((buckets[0], 48, 1), (buckets[1], 24, 2)): + fixture_call(recorder, ledger, 'create_bucket', {'Bucket': bucket}) + if versions == 2: + fixture_call(recorder, ledger, 'put_bucket_versioning', {'Bucket': bucket, 'VersioningConfiguration': {'Status': 'Enabled'}}) + expected[bucket] = [] + for index in range(count): + key = f'objects/{index:04d}.bin' + for version in range(versions): + body = payload(f'{bucket}/{key}/{version}', SIZE) + response = fixture_call(recorder, ledger, 'put_object', {'Bucket': bucket, 'Key': key, + 'ContentType': 'application/octet-stream', 'Metadata': {'heal-fixture': 'H06', 'nonce': f'{index}-{version}'}, + 'Tagging': 'suite=heal&fixture=H06'}, body) + version_id = response.get('VersionId', 'null') + require((versions == 1 and version_id == 'null') or (versions == 2 and version_id not in {None, 'null'}), + 'H06 versioning mode differs from its exact PUT ACK') + expected[bucket].append({'key': key, 'versionId': version_id, 'deleteMarker': False, + 'isLatest': version == versions - 1, 'payload': {'size': SIZE, 'sha256': hashlib.sha256(body).hexdigest()}}) + return expected + + +def witness_peer_loss(fleet, binding, expected, mapped): + peer = fleet.nodes[-1] + before = fleet.record('H5-peer-process-before', fleet.remote(peer, 'process')) + require(before['pid'] > 1 and before['binarySha256'] == binding['candidateBinarySha256'], 'H5 peer lacks candidate process identity') + stopped = fleet.record('H5-one-owned-peer-stop', fleet.remote(peer, 'stop')) + require(stopped['quiescent'] is True and stopped['service']['MainPID'] == '0', 'H5 peer stop was not witnessed') + fleet.record('H5-stopped-peer-endpoint-refusal', endpoint_refused(binding, len(fleet.nodes) - 1)) + for identity, target in mapped.items(): + survivors = [member for member in target['cohort'] if member['node'] != peer] + require(len(survivors) == 3 and len({member['node'] for member in survivors}) == 3, + 'H5 peer loss did not leave exactly three independent slots') + bucket, key, version = identity + manifest = next(entry for entry in expected[bucket] if entry['key'] == key and entry['versionId'] == version) + for endpoint in list(binding['endpoints'])[:-1]: + read = fleet.record('H5-survivor-exact-version-read', Probe(binding, endpoint).s3('get_object', + {'Bucket': bucket, 'Key': key, 'VersionId': version})) + require(read.get('acknowledged') is True and read.get('resultUnknown') is False + and read.get('payload') == manifest['payload'] + and read['response'].get('VersionId', 'null') == version, 'H5 surviving peer lost acknowledged version/bytes') + fleet.record('H5-original-peer-return', fleet.remote(peer, 'start')) + fleet.wait_ready() + after = fleet.record('H5-peer-process-after', fleet.remote(peer, 'process')) + require(after['pid'] > 1 and after['bootId'] == before['bootId'] and after['unit'] == before['unit'] + and after['binarySha256'] == before['binarySha256'] + and (after['pid'], after['startTicks']) != (before['pid'], before['startTicks']), 'H5 peer return changed candidate/unit or did not restart') + return {'oneOwnedPeerStopped': True, 'exactVersions': len(mapped), 'survivingReadEndpoints': 3, + 'remainingSlotsPerActualSet': 3, 'fullNodePowerLossClaimed': False} + + +def map_versions(snapshot, expected): + layout = validate_layout(snapshot) + indexed, mapped = files(snapshot), {} + for bucket, entries in expected.items(): + for entry in entries: + scopes = set() + for node, value in snapshot.items(): + for fmt in value['formats']: + row = indexed.get((node, fmt['disk'], f"{bucket}/{entry['key']}/xl.meta"), {}) + versions = [version for version in row.get('metadata', {}).get('versions', []) + if public_version_id(version.get('versionId')) == entry['versionId']] + require(len(versions) <= 1, 'H06 metadata duplicates an acknowledged version') + if not versions: + continue + groups = fmt['value']['xl']['sets'] + set_ids = [index for index, group in enumerate(groups) if fmt['value']['xl']['this'] in group] + require(len(set_ids) == 1, 'H06 physical slot belongs to ambiguous sets') + scopes.add((fmt['disk'] // 2, set_ids[0])) + require(len(scopes) == 1, 'acknowledged version lacks a unique actual pool/set authority') + pool_index, set_index = next(iter(scopes)) + target = map_target(snapshot, bucket, entry['key'], 'data', 1, entry['versionId'], + latest_required=False, pool_index=pool_index, set_index=set_index) + require(target['logical']['erasure']['data_blocks'] == target['logical']['erasure']['parity_blocks'] == 2 + and len(target['cohort']) == len({row['node'] for row in target['cohort']}) == 4, + 'H06 object geometry is not actual cross-node EC 2+2') + mapped[(bucket, entry['key'], entry['versionId'])] = target + for bucket in expected: + require({(target['poolIndex'], target['setIndex']) for identity, target in mapped.items() if identity[0] == bucket} + == {(pool, group) for pool in range(2) for group in range(2)}, + 'NOT_OBSERVED: bounded DB/DV writes did not populate every actual pool/set') + by_key = {} + for (bucket, key, _), target in mapped.items(): + by_key.setdefault((bucket, key), set()).add((target['poolIndex'], target['setIndex'])) + require(all(len(scopes) == 1 for scopes in by_key.values()), 'fixture key versions span different pools/sets; require a new reviewed scope fixture') + return layout, mapped + + +def selection(mapped, branch): + require(branch in BRANCHES, 'unregistered H06 branch') + pool, group = (0, 0) if branch == 'unselected-control-fault' else (1, None if branch == 'selected-pool' else 1) + selected = {identity for identity, target in mapped.items() if target['poolIndex'] == pool + and (group is None or target['setIndex'] == group)} + require(selected and set(mapped) - selected, 'H06 selection has no target or unselected controls') + scopes = [(pool, index) for index in range(2)] if group is None else [(pool, group)] + scopes += [(1 - pool, 0)] + if group is not None: + scopes += [(pool, 1 - group)] + targets, keys = [], set() + for wanted_pool, wanted_group in scopes: + for versioned in (False, True): + candidates = sorted((identity, target) for identity, target in mapped.items() + if (target['poolIndex'], target['setIndex']) == (wanted_pool, wanted_group) + and identity[:2] not in keys and identity[0].endswith('-dv') is versioned + and (not versioned or target['logical']['isLatest'] is False)) + require(candidates, 'H06 scope lacks a distinct DB or historical DV target/control key') + identity, target = candidates[0] + targets.append(target) + keys.add(identity[:2]) + return {'recursive': True, 'pool': pool, 'set': group}, selected, targets + + +def responsibility(current, mapped, selected, repaired, options): + outcome = current.get('outcome') or {} + require(current.get('summary') == 'finished' and current.get('truncated', False) is False + and outcome.get('execution', {}).get('state') == 'completed' and outcome.get('coverage') == 'complete' + and outcome.get('objectsTruncated') is False and current.get('settings') == options, + 'H06 task lost exact selectors, complete execution or diagnostics') + objects = outcome.get('objects', []) + require(len(objects) == len(selected) and repaired and repaired <= selected, 'H06 lacks exact selected version responsibility') + seen, incarnations = set(), {} + for row in objects: + item = row.get('identity', {}) + identity = item.get('bucket'), item.get('object'), public_version_id(item.get('versionId')) + require(identity in selected and identity not in seen and item.get('kind') == 'object', + 'H06 outcome includes foreign, duplicate or unselected responsibility') + target = mapped[identity] + require(type(item.get('poolIndex')) is int and item['poolIndex'] == options['pool'] == target['poolIndex'], + 'H06 pool selector/receipt disagrees with its actual drive group') + reported_set = item.get('setIndex') + require(reported_set is None and options['set'] is None or type(reported_set) is int + and reported_set == target['setIndex'] and (options['set'] is None or reported_set == options['set']), + 'H06 set selector/receipt contradicts the actual version authority') + incarnation = item.get('bucketIncarnationId') + require(isinstance(incarnation, str) and incarnation and incarnations.setdefault(identity[0], incarnation) == incarnation, + 'H06 version receipts lack or mix bucket incarnations') + require(row.get('disposition', {}).get('state') == ('repaired' if identity in repaired else 'verified_healthy'), + 'H06 selected version is unknown, falsely healthy or falsely repaired') + seen.add(identity) + counts = {'processed': len(selected), 'healed': len(repaired), 'unchanged': len(selected) - len(repaired), + 'skipped': 0, 'failed': 0, 'unknown': 0, 'attemptFailures': 0, 'overflowed': False} + require(seen == selected and all(type(outcome.get('counters', {}).get(name)) is type(value) + and outcome['counters'][name] == value for name, value in counts.items()), + 'H06 canonical counters do not cover every exact selected ACK version') + return {'exactSelectedVersionResponsibility': True, 'repaired': len(repaired), 'versions': len(selected), + 'rawUnrestrictedSetNotRewritten': True, 'physicalScopeBindings': [ + {'bucket': identity[0], 'key': identity[1], 'versionId': identity[2], + 'poolIndex': mapped[identity]['poolIndex'], 'setIndex': mapped[identity]['setIndex'], + 'diskUUIDs': [row['diskUUID'] for row in mapped[identity]['cohort']]} for identity in sorted(selected)]} + + +def pool_status(probe, fleet, endpoint_expressions, label): + observed = fleet.record(label, probe.admin('/rustfs/admin/v3/pools/list', [], b'', method='GET')) + require(observed.get('httpStatus') == 200, 'H06 pool capacity/list observation failed') + values = validate_pool_inventory(observed.get('response'), 2) + require([row['id'] for row in values] == [0, 1] + and [row['cmdline'] for row in values] == endpoint_expressions and all(row['totalSize'] > 0 for row in values), + 'H06 public pool identities/capacity differ from its original endpoint arguments') + for row in values: + specific = fleet.record(label + '-selector', probe.admin('/rustfs/admin/v3/pools/status', + [('pool', str(row['id'])), ('by-id', 'true')], b'', method='GET')) + require(specific.get('httpStatus') == 200, 'H06 selected public pool status failed') + selected = validate_pool_inventory(specific.get('response'), 1)[0] + require(all(selected[name] == row[name] for name in ('id', 'cmdline', 'status', 'totalSize')), + 'H06 selected pool identity/state/total capacity disagrees with its pool list') + return [{name: row[name] for name in ('id', 'cmdline', 'status', 'totalSize')} for row in values] + + +def quota_status(probe, fleet, buckets, label): + result = {} + for bucket in buckets: + value = fleet.record(label, probe.admin('/rustfs/admin/v3/quota/' + bucket, [], b'', method='GET')) + response = value.get('response') or {} + require(value.get('httpStatus') == 200 and response.get('bucket') == bucket + and response.get('quota') is None and response.get('quota_type') == 'HARD', + 'H06 changed the owned bucket quota or queried a different logical bucket') + result[bucket] = {name: response[name] for name in ('bucket', 'quota', 'quota_type')} + return result + + +def run_branch(args, nodes, branch): + directory = args.evidence / 'H5' / branch + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-h5" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile='multi-pool') + candidate = json.loads(args.candidate.read_text()) + require(candidate['source_sha'] == REFERENCE['candidateCommit'], 'H06 selector/placement semantics are not reviewed for this candidate') + bucket = 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] + buckets = [bucket, bucket + '-dv'] + fleet.snapshot_buckets = buckets + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {'schemaVersion': 1, 'runId': run_id, 'jumpHostOnly': True, 'exclusiveHealService': True, + 'leaseExpiresAt': (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), 'bucketPrefix': bucket, + 'endpoints': {f'node{index}': f'http://rustfs-node{index}:9000' for index in range(1, 5)}, 'defaultEndpoint': 'node1', + 'candidateCommit': candidate['source_sha'], 'harnessCommit': args.testing_commit, + 'candidatePackageSha256': candidate['package_sha256'], 'candidateBinarySha256': args.binary_sha, + 'rcBinarySha256': hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), 'maxInventoryObjects': 256, + 'maxInventoryBytes': 512 * 1024**2} + fleet.record('binding', binding) + fleet.record('multi-pool-source-contract', REFERENCE) + fleet.record('inspector-build', verify_inspector(args.tool_directory, candidate)) + probe, active, ledger = Probe(binding), [], [] + result = {'topology': 'H5', 'branch': branch, 'verdict': 'RUNNING', 'cleanup': 'PENDING'} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + expected = seed_fixtures(probe, directory, buckets, ledger) + baseline = {name: fleet.record('full-baseline-S3', probe.inventory(name, + unversioned_current_tags=name == bucket)) for name in buckets} + for name in buckets: + fleet.record('fixture-ACK-oracle', {'bucket': name, **verify_seed(expected[name], baseline[name])}) + before_capacity = pool_status(probe, fleet, volumes(run_id), 'baseline-capacity-pool-binding') + before_quota = quota_status(probe, fleet, buckets, 'baseline-logical-quota-binding') + fleet.stop() + original = fleet.snapshot('all-pool-physical-baseline', quiescent=True) + layout, mapped = map_versions(original, expected) + fleet.record('all-pool-set-version-authorities', {'layout': layout, 'versions': [ + {'bucket': identity[0], 'key': identity[1], 'versionId': identity[2], + 'poolIndex': target['poolIndex'], 'setIndex': target['setIndex'], 'cohort': target['cohort']} + for identity, target in mapped.items()]}) + fleet.start() + result['nodeRedundancy'] = witness_peer_loss(fleet, binding, expected, mapped) + fleet.stop() + require(physical_content(original) == physical_content(fleet.snapshot('after-H5-peer-loss-before-fault', quiescent=True)), + 'healthy H5 peer loss modified the physical baseline') + deltas, selected, targets = selection(mapped, branch) + options = {**BASE_OPTIONS, **deltas} + result['selectors'] = options + fleet.record('selected-and-unselected-fault-plan', {'options': options, 'targets': targets, + 'selectedVersions': [list(identity) for identity in sorted(selected)]}) + current = original + registrations = {} + for index, target in enumerate(targets): + params = {'disk': target['disk'], 'faultId': f'H06-part-{index}'} + registration = target['node'], params + fleet.faults.append(registration) + registrations[(target['bucket'], target['key'], target['versionId'])] = registration + receipt = fleet.record('precise-pool-set-part-fault', fleet.remote(target['node'], 'apply-fault', **params, + relativePath=target['relativePath'], kind='missing', expectedSha256=target['sha256'])) + after = fleet.snapshot('physical-exact-fault', quiescent=True) + validate_fault(current, after, target, receipt) + current = after + readonly = None + if branch == 'unselected-control-fault': + readonly = next(target for target in targets if target['poolIndex'] != options['pool']) + fleet.quorum_gates.add(readonly['node']) + fleet.record('F4-unselected-private-readonly-intent', fleet.remote(readonly['node'], 'quorum-gate', disks=[readonly['disk']])) + repaired = {(target['bucket'], target['key'], target['versionId']) for target in targets + if (target['bucket'], target['key'], target['versionId']) in selected} + fleet.start() + def witness_readonly(label): + relative = f"{readonly['bucket']}/{readonly['key']}/xl.meta" + raw = fleet.record(label, fleet.remote(readonly['node'], 'quorum-gate-probe', relativePath=relative, + expectedReads=expected_reads(current, readonly['node'], relative))) + require({row['disk'] for row in raw['disks'] if not row['writable']} == {readonly['disk']}, + 'F4 unselected control did not witness the exact service-private read-only root') + return {'poolIndex': readonly['poolIndex'], 'setIndex': readonly['setIndex'], 'diskUUID': readonly['diskUUID'], **raw} + if readonly: + result['F4BeforeSelectedHeal'] = witness_readonly('F4-service-namespace-before-selected-Heal') + observed = run_task(probe, fleet, {'bucket': '', 'key': ''}, 2, active, + option_overrides=options, status_endpoints=tuple(binding['endpoints'])[1:]) + result['selectedResponsibility'] = responsibility(observed, mapped, selected, repaired, options) + if readonly: + result['F4AfterSelectedHeal'] = witness_readonly('F4-service-namespace-after-selected-Heal') + idle_fleet(fleet, binding) + fleet.stop() + isolated = fleet.snapshot('selected-repaired-unselected-still-missing-before-GET', quiescent=True) + selected_keys = {identity[:2] for identity in selected} + result['physicalIsolation'] = physical_selection(original, isolated, targets, selected_keys) + for identity in repaired: + fleet.faults.remove(registrations[identity]) + if readonly: + fleet.record('F4-release-after-unselected-physical-proof', fleet.remote(readonly['node'], 'quorum-gate', mode='clear')) + fleet.quorum_gates.remove(readonly['node']) + fleet.start() + require(pool_status(probe, fleet, volumes(run_id), 'selected-capacity-pool-binding') == before_capacity + and quota_status(probe, fleet, buckets, 'selected-logical-quota-binding') == before_quota, + 'selected Heal changed public pool capacity identity or logical quota policy') + for target in targets: + identity = target['bucket'], target['key'], target['versionId'] + if identity in repaired: + continue + local_scope = {key for key in mapped if key[:2] == identity[:2]} + control_options = {**BASE_OPTIONS, 'pool': target['poolIndex'], 'set': target['setIndex']} + control = run_task(probe, fleet, {'bucket': target['bucket'], 'key': target['key']}, 2, active, + option_overrides=control_options, status_endpoints=tuple(binding['endpoints'])[1:]) + fleet.record('independent-unselected-control-repair', responsibility(control, mapped, local_scope, {identity}, control_options)) + idle_fleet(fleet, binding) + fleet.stop() + final = fleet.snapshot('all-pool-rebuilt-before-business-GET', quiescent=True) + result['allPoolPhysical'] = validate_repaired_physical(original, final, targets[0], repair_targets=targets) + result['allVersionStreaming'] = verify_parts(fleet, mapped, 'complete-all-pool-version-streaming') + for identity, registration in registrations.items(): + if identity not in repaired: + fleet.faults.remove(registration) + fleet.start() + result['s3'] = {} + for endpoint in binding['endpoints']: + result['s3'][endpoint] = {} + for name in buckets: + observed = fleet.record('all-peer-all-pool-full-S3', Probe(binding, endpoint).inventory(name, + unversioned_current_tags=name == bucket)) + result['s3'][endpoint][name] = {**verify_seed(expected[name], observed), **compare_inventories(baseline[name], observed)} + result.update(verdict='PASS', unselectedControlsPreservedBeforeRecovery=True, crossPoolCopyCountedAsRepair=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict='FAIL', failure={'type': type(exc).__name__, 'message': str(exc)}) + finally: + for path, token in active: + try: fleet.record('stop-unresolved', probe.admin(path, [('clientToken', token), ('forceStop', 'true')], b'')) + except Exception as exc: fleet.record('stop-unresolved-error', {'error': str(exc)}) + pending = any(not row['resolved'] for row in ledger) + if pending: + try: + fleet.stop() + for node in tuple(fleet.quorum_gates): + fleet.record('failed-seed-gate-release', fleet.remote(node, 'quorum-gate', mode='clear')) + fleet.quorum_gates.remove(node) + fleet.start() + result['seedResolution'] = resolve_pending_fixture(fleet, binding, ledger) + except Exception as exc: + result['seedResolution'] = {'state': 'UNRESOLVED', 'error': str(exc)} + if any(not row['resolved'] for row in ledger): + try: fleet.stop() + except Exception as exc: fleet.record('retain-stop-error', {'error': str(exc)}) + fleet.record('retained-unresolved-seed', {'state': 'DIRTY', 'namespacesRetained': True, 'requests': ledger}) + clean = False + else: + clean = fleet.cleanup() + result['cleanup'] = 'PASS' if clean else 'FAIL' + fleet.record('branch-result', result) + return result + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f'H06 interrupted by signal {signum}') + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ('candidate', 'rc-bin', 'evidence', 'tool-directory'): + parser.add_argument('--' + name, type=Path, required=True) + parser.add_argument('--case-id', choices=[CASE_ID], required=True) + parser.add_argument('--testing-commit', required=True) + parser.add_argument('--binary-sha', required=True) + args = parser.parse_args() + require(re.fullmatch(r'[0-9a-f]{64}', args.binary_sha) and re.fullmatch(r'[1-9][0-9]*', os.environ.get('GITHUB_RUN_ID', '')), + 'immutable workflow candidate identity required') + require(select(CASE_ID)[0]['variants'] == list(BRANCHES), 'H06 source branch omitted') + nodes = os.environ.get('RUSTFS_NODES', 'vm000 vm001 vm002 vm003').split() + require(len(nodes) >= 4 and len(nodes) == len(set(nodes)) and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in nodes), + 'four distinct current Heal nodes required') + results = [] + try: + for branch in BRANCHES: + results.append(run_branch(args, nodes[:4], branch)) + if results[-1]['cleanup'] != 'PASS': break + finally: + complete = len(results) == 3 and all(row['verdict'] == row['cleanup'] == 'PASS' for row in results) + save(args.evidence / 'healthy-summary.json', {'caseId': CASE_ID, 'results': results, 'complete': complete, + 'cleanup': 'PASS' if results and all(row['cleanup'] == 'PASS' for row in results) else 'FAIL'}) + require(complete, 'H06 requires all actual H5 pool/set selectors, preserved control faults and full version/physical proof') + + +if __name__ == '__main__': + try: main() + except Exception as exc: + print(f'H06 multi-pool validation failed: {exc}', file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/multi_pool_layout.py b/scripts/validation_heal/multi_pool_layout.py new file mode 100644 index 00000000..cb9f2dcb --- /dev/null +++ b/scripts/validation_heal/multi_pool_layout.py @@ -0,0 +1,42 @@ +"""Actual H5 pools/sets over sixteen original independent endpoints.""" + +try: + from .heal_probe import require +except ImportError: + from heal_probe import require + + +def volumes(run_id): + require(run_id.endswith('-h5'), 'two-pool deployment requires an explicit H5 namespace') + return [f'http://rustfs-node{{1...4}}:9000/data/rustfs{{{first}...{first + 1}}}/heal-validation/{run_id}/data' + for first in (1, 3)] + + +def validate(snapshot): + require(len(snapshot) == 4, 'H5 requires the four actual original hosts') + rows = [(node, row) for node, value in snapshot.items() for row in value['formats']] + require(len(rows) == 16 and len({(node, row['disk']) for node, row in rows}) == 16 + and all({row['disk'] for owner, row in rows if owner == node} == set(range(4)) for node in snapshot), + 'H5 omitted or duplicated an original independent drive') + pools, all_ids = [], set() + for pool_index in range(2): + selected = [(node, row) for node, row in rows if row['disk'] // 2 == pool_index] + value = selected[0][1]['value'] + sets = value['xl']['sets'] + identities = [identity for group in sets for identity in group] + require(value['format'] == 'xl' and len(sets) == 2 and all(len(group) == 4 for group in sets) + and len(identities) == len(set(identities)) == 8 and not all_ids.intersection(identities), + 'H5 needs two distinct pools, each with two complete four-member sets') + require(all(row['value']['format'] == 'xl' and row['value']['id'] == value['id'] + and row['value']['xl']['sets'] == sets for _, row in selected) + and {row['value']['xl']['this'] for _, row in selected} == set(identities), + 'pool formats disagree or fail to cover their exact endpoint group') + require(all(sum(row['value']['xl']['this'] in group for owner, row in selected if owner == node) == 1 + for node in snapshot for group in sets), 'each H5 set must span all four actual hosts') + all_ids.update(identities) + pools.append({'poolIndex': pool_index, 'formatId': value['id'], 'actualSets': sets, + 'members': [{'node': node, 'disk': row['disk'], 'diskUUID': row['value']['xl']['this']} for node, row in selected]}) + require(set(pools[0]['actualSets'][0]).isdisjoint(set(pools[1]['actualSets'][0])), 'two pool selectors alias one physical set') + return {'topology': 'H5', 'nodes': 4, 'independentDrives': 16, 'pools': pools, + 'setCountPerPool': 2, 'minimumAvailableSlotsAfterOnePeerLoss': 3, + 'codingMustBeMeasuredPerObject': True} diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 6fc46cc9..4e6c3202 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -39,6 +39,7 @@ case "${case_id}" in heal.disk-and-node-flapping) driver=flapping.py ;; heal.enospc-inodes-and-quota) driver=capacity.py ;; heal.network-partition-and-lock-quorum) driver=network.py ;; + heal.multi-pool-target-isolation) driver=multi_pool.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index 5b5b358c..1ddb23dd 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -97,17 +97,24 @@ def files(snapshot): def map_target(snapshot, bucket, key, role, part_number, version_id="null", *, preferred_nodes=None, latest_required=True, - multi_set=False, set_index=0): + multi_set=False, set_index=0, pool_index=None): return _map_external_target(snapshot, bucket, key, role, part_number, version_id, - preferred_nodes=preferred_nodes, latest_required=latest_required, multi_set=multi_set, set_index=set_index) + preferred_nodes=preferred_nodes, latest_required=latest_required, multi_set=multi_set, set_index=set_index, pool_index=pool_index) def _map_external_target(snapshot, bucket, key, role, part_number, version_id="null", *, preferred_nodes=None, - latest_required=True, multi_set=False, set_index=0, part_policy=None): + latest_required=True, multi_set=False, set_index=0, part_policy=None, pool_index=None): """Share set/role authority checks while keeping transformed part policy explicit.""" - require(type(multi_set) is bool and type(set_index) is int and 0 <= set_index < (2 if multi_set else 1), + require(pool_index is None or type(pool_index) is int and 0 <= pool_index < 2 and multi_set is False, + "invalid explicit pool mapping") + require(type(multi_set) is bool and type(set_index) is int and 0 <= set_index < (2 if multi_set or pool_index is not None else 1), "invalid explicit object set mapping") - validate_topology(snapshot, "H1" if len(snapshot) == 1 else "H2", expected_sets=2 if multi_set else 1) + if pool_index is None: + validate_topology(snapshot, "H1" if len(snapshot) == 1 else "H2", expected_sets=2 if multi_set else 1) + else: + try: from .multi_pool_layout import validate as validate_pools + except ImportError: from multi_pool_layout import validate as validate_pools + validate_pools(snapshot) indexed = files(snapshot) cohort, logical = [], None for node, observation in snapshot.items(): @@ -115,6 +122,11 @@ def _map_external_target(snapshot, bucket, key, role, part_number, version_id="n disk = format_entry["disk"] identity = (node, disk, f"{bucket}/{key}/xl.meta") layout = format_entry["value"]["xl"] + if pool_index is not None and disk // 2 != pool_index: + other_versions = indexed.get(identity, {}).get("metadata", {}).get("versions", []) + require(not any((version.get("versionId") or "null") == version_id for version in other_versions), + "ACK version has an additional authority in an unselected pool") + continue if layout["this"] not in layout["sets"][set_index]: require(not any(owner == node and drive == disk and path.startswith(f"{bucket}/{key}/") for owner, drive, path in indexed), "object files span the selected and unselected erasure sets") @@ -169,7 +181,7 @@ def _map_external_target(snapshot, bucket, key, role, part_number, version_id="n target = next((entry for entry in cohort if entry["role"] == role and (preferred_nodes is None or entry["node"] in preferred_nodes)), None) require(target is not None, "requested shard role is not present") - return {**target, "bucket": bucket, "key": key, "poolIndex": 0, "setIndex": set_index, + return {**target, "bucket": bucket, "key": key, "poolIndex": 0 if pool_index is None else pool_index, "setIndex": set_index, "cohort": cohort, "logical": logical} diff --git a/test_validation_heal_multi_pool.py b/test_validation_heal_multi_pool.py new file mode 100644 index 00000000..2c01bbb4 --- /dev/null +++ b/test_validation_heal_multi_pool.py @@ -0,0 +1,274 @@ +import copy +from pathlib import Path +import tempfile +import types +import unittest +import uuid +from unittest.mock import Mock, patch + +from scripts.validation_heal import multi_pool as runner +from scripts.validation_heal.multi_pool_layout import validate, volumes +from scripts.validation_heal.fleet_node import runtime_config +from scripts.validation_heal.healthy import BASE_OPTIONS, validate_topology +from scripts.validation_heal.matrix import select, AUTOMATIC_CASES +from scripts.validation_heal.shards import files, map_target +from scripts.validation_heal.scopes import physical_selection + + +class MultiPoolTests(unittest.TestCase): + buckets = ('rv-heal-0123456789abcdef', 'rv-heal-0123456789abcdef-dv') + + def fixture(self): + sets = [[[f'p{pool}-s{group}-n{node}' for node in range(4)] for group in range(2)] for pool in range(2)] + snapshot = {f'node{node + 1}': {'formats': [], 'files': []} for node in range(4)} + expected = {bucket: [] for bucket in self.buckets} + for bucket in self.buckets: + for pool in range(2): + for group in range(2): + key = f'p{pool}/s{group}.bin' + versions = ('v1', 'v2') if bucket.endswith('-dv') else ('null',) + for offset, version in enumerate(versions): + expected[bucket].append({'key': key, 'versionId': version}) + for node, observed in enumerate(snapshot.values()): + disk = pool * 2 + group + metadata = {'bucket': bucket, 'key': key, 'versions': []} + for offset, version in enumerate(versions): + data_dir = str(uuid.UUID(int=pool * 100 + group * 10 + offset + 1)) + metadata['versions'].append({'versionId': None if version == 'null' else version, + 'isLatest': offset == len(versions) - 1, 'deleted': False, 'inline': None, + 'dataDir': data_dir, 'size': runner.SIZE, 'usesLegacyChecksum': False, + 'parts': [{'number': 1, 'size': runner.SIZE, 'actualSize': runner.SIZE}], + 'role': 'data' if node < 2 else 'parity', + 'erasure': {'index': node + 1, 'data_blocks': 2, 'parity_blocks': 2, + 'block_size': 1048576, 'distribution': [1, 2, 3, 4], 'algorithm': 'rs-vandermonde'}}) + observed['files'].append({'disk': disk, 'path': f'{bucket}/{key}/{data_dir}/part.1', + 'size': 400, 'sha256': f'{bucket}-{pool}-{group}-{offset}-{node}'}) + observed['files'].append({'disk': disk, 'path': f'{bucket}/{key}/xl.meta', 'size': 600, + 'sha256': f'meta-{bucket}-{pool}-{group}-{node}', 'metadata': metadata}) + for node, observed in enumerate(snapshot.values()): + for disk in range(4): + pool, group = disk // 2, disk % 2 + observed['formats'].append({'disk': disk, 'value': {'format': 'xl', 'id': 'deployment', + 'xl': {'this': sets[pool][group][node], 'sets': sets[pool]}}}) + return snapshot, expected + + def mapping(self): + snapshot, expected = self.fixture() + return runner.map_versions(snapshot, expected)[1] + + def receipt(self, branch): + mapped = self.mapping() + delta, selected, targets = runner.selection(mapped, branch) + options = {**BASE_OPTIONS, **delta} + repaired = {(target['bucket'], target['key'], target['versionId']) for target in targets} & selected + objects = [{'identity': {'kind': 'object', 'bucket': bucket, 'object': key, 'versionId': version, + 'poolIndex': options['pool'], 'setIndex': options['set'], 'bucketIncarnationId': 'incarnation-' + bucket}, + 'disposition': {'state': 'repaired' if (bucket, key, version) in repaired else 'verified_healthy'}} + for bucket, key, version in sorted(selected)] + current = {'summary': 'finished', 'truncated': False, 'settings': options, + 'outcome': {'execution': {'state': 'completed'}, 'coverage': 'complete', 'objectsTruncated': False, + 'objects': objects, 'counters': {'processed': len(selected), 'healed': len(repaired), + 'unchanged': len(selected) - len(repaired), 'failed': 0, 'skipped': 0, 'unknown': 0, + 'attemptFailures': 0, 'overflowed': False}}} + return current, mapped, selected, repaired, options + + def test_preserves_all_three_source_branches_and_real_two_pool_arguments(self): + self.assertEqual(select(runner.CASE_ID)[0]['variants'], list(runner.BRANCHES)) + self.assertIn(runner.CASE_ID, AUTOMATIC_CASES) + args = volumes('gh-123-1-h5') + self.assertEqual(len(args), 2) + self.assertIn('rustfs{1...2}', args[0]) + self.assertIn('rustfs{3...4}', args[1]) + self.assertEqual(runtime_config('multi-pool')['RUSTFS_ERASURE_SET_DRIVE_COUNT'], '4') + self.assertEqual(runtime_config('multi-pool')['RUSTFS_STORAGE_CLASS_STANDARD'], 'EC:2') + with self.assertRaises(ValueError): volumes('gh-123-1-h2') + + def test_original_h2_mapper_rejects_multi_pool_without_explicit_authority(self): + snapshot, expected = self.fixture() + with self.assertRaises(ValueError): validate_topology(snapshot, 'H2') + with self.assertRaises(ValueError): map_target(snapshot, self.buckets[0], 'p1/s1.bin', 'data', 1) + before = copy.deepcopy(snapshot) + layout, mapped = runner.map_versions(snapshot, expected) + self.assertEqual(snapshot, before) + self.assertEqual(layout['independentDrives'], 16) + self.assertEqual(len(mapped), 12) + self.assertEqual({(row['poolIndex'], row['setIndex']) for row in mapped.values()}, {(0, 0), (0, 1), (1, 0), (1, 1)}) + self.assertTrue(all(len(row['cohort']) == len({member['node'] for member in row['cohort']}) == 4 for row in mapped.values())) + + def test_rejects_missing_original_drives_wrong_pools_aliases_and_host_local_sets(self): + for kind in ('drive', 'pool', 'alias', 'host-local', 'duplicate'): + snapshot, _ = self.fixture() + if kind == 'drive': snapshot['node1']['formats'].pop() + elif kind == 'pool': snapshot['node1']['formats'][2]['value']['xl']['sets'] = copy.deepcopy(snapshot['node1']['formats'][0]['value']['xl']['sets']) + elif kind == 'alias': snapshot['node1']['formats'][2]['value']['xl']['this'] = snapshot['node1']['formats'][0]['value']['xl']['this'] + elif kind == 'duplicate': snapshot['node1']['formats'][0]['disk'] = 1 + else: + sets = [[f'p0-s{group}-n{node}' for group in range(2) for node in range(start, start + 2)] for start in (0, 2)] + for value in snapshot.values(): + for fmt in value['formats'][:2]: fmt['value']['xl']['sets'] = sets + with self.subTest(kind=kind), self.assertRaises(ValueError): validate(snapshot) + + def test_missing_ack_version_wrong_role_geometry_or_extra_pool_authority_fail(self): + for kind in ('missing', 'role', 'geometry', 'authority', 'unpopulated'): + snapshot, expected = self.fixture() + entry = next(row for row in snapshot['node1']['files'] if row['path'] == self.buckets[0] + '/p1/s1.bin/xl.meta') + if kind == 'missing': entry['metadata']['versions'].clear() + elif kind == 'role': entry['metadata']['versions'][0]['role'] = 'parity' + elif kind == 'geometry': entry['metadata']['versions'][0]['erasure']['parity_blocks'] = 3 + elif kind == 'authority': snapshot['node1']['files'].append({**copy.deepcopy(entry), 'disk': 0}) + else: expected[self.buckets[0]] = [row for row in expected[self.buckets[0]] if row['key'] != 'p0/s0.bin'] + with self.subTest(kind=kind), self.assertRaises(ValueError): runner.map_versions(snapshot, expected) + + def test_raw_pool_only_receipts_and_exact_version_scope_are_not_rewritten(self): + current, mapped, selected, repaired, options = self.receipt('selected-pool') + original = copy.deepcopy(current) + proof = runner.responsibility(current, mapped, selected, repaired, options) + self.assertEqual(current, original) + self.assertTrue(proof['rawUnrestrictedSetNotRewritten']) + self.assertEqual({row['setIndex'] for row in proof['physicalScopeBindings']}, {0, 1}) + self.assertEqual(proof['repaired'], 4) + self.assertEqual(proof['versions'], 6) + + def test_wrong_selectors_versions_responsibility_and_false_counts_fail(self): + for kind in ('pool', 'set', 'bool', 'version', 'duplicate', 'missing', 'foreign', 'unknown', 'counter', 'settings', 'truncated', 'incarnation'): + current, mapped, selected, repaired, options = self.receipt('selected-set') + item = current['outcome']['objects'][0] + if kind in ('pool', 'set', 'bool', 'version', 'incarnation'): + name, value = {'pool': ('poolIndex', None), 'set': ('setIndex', 0), 'bool': ('poolIndex', True), + 'version': ('versionId', 'foreign'), 'incarnation': ('bucketIncarnationId', '')}[kind] + item['identity'][name] = value + elif kind == 'duplicate': current['outcome']['objects'][-1] = copy.deepcopy(item) + elif kind == 'missing': current['outcome']['objects'].pop() + elif kind == 'foreign': item['identity']['bucket'] = 'foreign' + elif kind == 'unknown': item['disposition']['state'] = 'unknown' + elif kind == 'counter': current['outcome']['counters']['healed'] += 1 + elif kind == 'settings': current['settings'] = {**options, 'set': 0} + else: current['outcome']['objectsTruncated'] = True + with self.subTest(kind=kind), self.assertRaises(ValueError): runner.responsibility(current, mapped, selected, repaired, options) + + def test_control_missing_part_is_preserved_until_selected_physical_proof(self): + original, expected = self.fixture() + _, mapped = runner.map_versions(original, expected) + _, selected, targets = runner.selection(mapped, 'selected-set') + after = copy.deepcopy(original) + missing = {(target['node'], target['disk'], target['relativePath']) for target in targets + if (target['bucket'], target['key'], target['versionId']) not in selected} + for node, observed in after.items(): + observed['files'] = [row for row in observed['files'] if (node, row['disk'], row['path']) not in missing] + proof = physical_selection(original, after, targets, {identity[:2] for identity in selected}) + self.assertEqual(proof['selectedReconstructedParts'], 2) + self.assertEqual(proof['unselectedStillMissingParts'], 4) + identity = next(iter(missing)) + after[identity[0]]['files'].append(files(original)[identity]) + with self.assertRaises(ValueError): physical_selection(original, after, targets, {identity[:2] for identity in selected}) + + def seed_row(self, version='v1', resolved=False): + params = {'Bucket': self.buckets[1], 'Key': 'object', 'ContentType': 'application/octet-stream', 'Metadata': {'nonce': version}} + row = {'operation': 'put_object', 'params': params, 'payload': {'size': 1, 'sha256': version}, 'resolved': resolved, + 'observation': {'acknowledged': resolved, 'resultUnknown': not resolved, 'response': {'VersionId': version}}} + entry = {'key': 'object', 'versionId': version, 'deleteMarker': False, 'isLatest': True, 'payload': row['payload'], + 'properties': {'Metadata': params['Metadata'], 'ContentType': params['ContentType']}, + 'tags': [{'Key': 'fixture', 'Value': 'H06'}, {'Key': 'suite', 'Value': 'heal'}]} + return row, entry + + def test_unknown_seed_is_resolved_as_committed_or_absent_without_retry(self): + first, previous = self.seed_row('v1', True) + pending, newest = self.seed_row('v2') + rows = [first, pending] + absent = runner.resolve_fixture_inventory(rows, {'entries': [previous]}) + self.assertEqual(absent[-1]['state'], 'ABSENT') + previous['isLatest'] = False + committed = runner.resolve_fixture_inventory(rows, {'entries': [previous, newest]}) + self.assertEqual(committed[-1]['versionId'], 'v2') + + def test_unknown_seed_drift_ack_replacement_tags_extra_version_and_latest_fail(self): + for kind in ('payload', 'version', 'tags', 'extra', 'latest'): + row, entry = self.seed_row('v1', True) + if kind == 'payload': entry['payload'] = {'size': 2, 'sha256': 'drift'} + elif kind == 'version': entry['versionId'] = 'replacement' + elif kind == 'tags': entry['tags'] = [] + elif kind == 'latest': entry['isLatest'] = False + entries = [entry, {**entry, 'versionId': 'extra'}] if kind == 'extra' else [entry] + with self.subTest(kind=kind), self.assertRaises(ValueError): runner.resolve_fixture_inventory([row], {'entries': entries}) + + def test_pending_registration_precedes_recorder_or_ack_validation_failure(self): + ledger = [] + recorder = Mock() + def fail(*args): + self.assertEqual(len(ledger), 1) + raise OSError('collector failed before durable receipt') + recorder.call.side_effect = fail + with self.assertRaises(OSError): runner.fixture_call(recorder, ledger, 'create_bucket', {'Bucket': self.buckets[0]}) + self.assertFalse(ledger[0]['resolved']) + ledger = [] + recorder.call.side_effect = None + recorder.call.return_value = {'VersionId': 'null'} + with self.assertRaises(ValueError): runner.fixture_call(recorder, ledger, 'put_object', {'Bucket': self.buckets[1]}, b'x') + self.assertFalse(ledger[0]['resolved']) + + def test_uncertain_bucket_mutations_and_cross_peer_divergence_retain_dirty_boundary(self): + fleet = Mock() + with self.assertRaises(ValueError): runner.resolve_pending_fixture(fleet, {}, [{'operation': 'create_bucket', 'resolved': False}]) + fleet.record.assert_not_called() + row, entry = self.seed_row() + binding = {'endpoints': {'node1': 'a', 'node2': 'b'}} + fleet.record.side_effect = lambda label, value: value + with patch.object(runner, 'Probe') as probe: + probe.return_value.inventory.side_effect = [{'entries': [entry]}, {'entries': []}] + with self.assertRaises(ValueError): runner.resolve_pending_fixture(fleet, binding, [row]) + self.assertFalse(row['resolved']) + + def test_node_redundancy_requires_actual_owned_stop_and_exact_reads_from_every_survivor(self): + snapshot, expected = self.fixture() + _, mapped = runner.map_versions(snapshot, expected) + for entries in expected.values(): + for entry in entries: entry['payload'] = {'size': runner.SIZE, 'sha256': entry['key'] + entry['versionId']} + binding = {'candidateBinarySha256': 'candidate', 'endpoints': {f'node{i}': f'http://node{i}:9000' for i in range(1, 5)}} + for drift in (False, True): + fleet = Mock(nodes=list(snapshot)) + fleet.record.side_effect = lambda label, value: value + fleet.remote.side_effect = [ + {'pid': 5, 'startTicks': 1, 'unit': 'owned', 'bootId': 'boot', 'binarySha256': 'candidate'}, + {'quiescent': True, 'service': {'MainPID': '0'}}, {}, + {'pid': 6, 'startTicks': 2, 'unit': 'owned', 'bootId': 'boot', 'binarySha256': 'candidate'}] + with patch.object(runner, 'endpoint_refused', return_value={'connectionRefused': True}), patch.object(runner, 'Probe') as probe: + def read(operation, params): + entry = next(item for item in expected[params['Bucket']] if item['key'] == params['Key'] and item['versionId'] == params['VersionId']) + return {'acknowledged': True, 'resultUnknown': False, 'payload': {} if drift else entry['payload'], + 'response': {'VersionId': entry['versionId']}} + probe.return_value.s3.side_effect = read + if drift: + with self.assertRaises(ValueError): runner.witness_peer_loss(fleet, binding, expected, mapped) + else: + proof = runner.witness_peer_loss(fleet, binding, expected, mapped) + self.assertEqual(proof['exactVersions'], 12) + self.assertEqual(probe.return_value.s3.call_count, 36) + self.assertEqual({call.args[1] for call in probe.call_args_list}, {'node1', 'node2', 'node3'}) + + def test_unresolved_seed_prevents_namespace_cleanup_and_next_branch_admission(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + candidate = directory / 'candidate.json' + candidate.write_text(__import__('json').dumps({'source_sha': runner.REFERENCE['candidateCommit'], 'package_sha256': 'package'})) + rc = directory / 'rc' + rc.write_bytes(b'collector') + args = types.SimpleNamespace(evidence=directory / 'evidence', tool_directory=directory, + candidate=candidate, binary_sha='binary', testing_commit='harness', rc_bin=rc) + fleet = Mock(nodes=['node1', 'node2', 'node3', 'node4'], quorum_gates=set()) + fleet.record.side_effect = lambda label, value: value + def failed_seed(probe, path, buckets, ledger): + ledger.append({'operation': 'create_bucket', 'params': {'Bucket': buckets[0]}, 'resolved': False}) + raise OSError('unknown initial bucket mutation') + with patch.dict(runner.os.environ, {'GITHUB_RUN_ID': '123', 'GITHUB_RUN_ATTEMPT': '1'}), \ + patch.object(runner, 'Fleet', return_value=fleet), patch.object(runner, 'verify_inspector', return_value={}), \ + patch.object(runner, 'Probe'), patch.object(runner, 'seed_fixtures', side_effect=failed_seed): + result = runner.run_branch(args, fleet.nodes, 'selected-pool') + self.assertEqual(result['verdict'], 'FAIL') + self.assertEqual(result['cleanup'], 'FAIL') + self.assertEqual(result['seedResolution']['state'], 'UNRESOLVED') + fleet.cleanup.assert_not_called() + self.assertTrue(any(call.args[0] == 'retained-unresolved-seed' for call in fleet.record.call_args_list)) + + +if __name__ == '__main__': + unittest.main() From e3f603f59d8b04f2abd9e7781737a81b685258ad Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 08:55:59 +0800 Subject: [PATCH 103/343] test: retain three acknowledged versions in each H06 DV key --- docs/validation-migration-map.md | 2 +- scripts/validation_heal/multi_pool.py | 8 ++++---- test_validation_heal_multi_pool.py | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index acb4d02b..bab0a46a 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1410,7 +1410,7 @@ order and exact public pool cmdlines. The runtime explicitly uses set width four and EC:2. Full format/UUID/set mappings and decoded object geometry must prove actual cross-node EC 2+2 before any fault. -Bounded DB and two-version DV fixtures must populate all four observed +Bounded DB and three-version DV fixtures must populate all four observed pool/set combinations. Every acknowledged version is mapped without an assumed placement hash or internal PUT selector. A measured stop of one owned peer must leave three physical slots per set and redeem every original diff --git a/scripts/validation_heal/multi_pool.py b/scripts/validation_heal/multi_pool.py index cbd6d4ff..b2a35485 100644 --- a/scripts/validation_heal/multi_pool.py +++ b/scripts/validation_heal/multi_pool.py @@ -48,7 +48,7 @@ 'layoutSourceFiles': ['crates/ecstore/src/layout/disks_layout.rs', 'crates/utils/src/string.rs'], 'topology': 'H5: two endpoint-expansion arguments, eight original independent mounts per pool, two four-peer sets per pool', 'storageClass': 'explicit EC:2; every actual object must decode as 2+2', - 'boundedFixture': {'DBKeys': 48, 'DVKeys': 24, 'versionsPerDVKey': 2, 'bytesPerVersion': SIZE}, + 'boundedFixture': {'DBKeys': 48, 'DVKeys': 24, 'versionsPerDVKey': 3, 'bytesPerVersion': SIZE}, 'placement': 'observe actual full-set XL2/version authority; no assumed pool hash or internal PUT selector', 'scope': 'pool selector must match admitted pool; an unrestricted set may be null in the raw receipt, with independent exact physical binding', 'control': 'unselected defects stay missing before any target GET or rescue; repair them in independent tasks only after proof'} @@ -135,9 +135,9 @@ def resolve_pending_fixture(fleet, binding, ledger): def seed_fixtures(probe, directory, buckets, ledger): recorder = Recorder(probe, directory / 'fixture') expected = {} - for bucket, count, versions in ((buckets[0], 48, 1), (buckets[1], 24, 2)): + for bucket, count, versions in ((buckets[0], 48, 1), (buckets[1], 24, 3)): fixture_call(recorder, ledger, 'create_bucket', {'Bucket': bucket}) - if versions == 2: + if versions == 3: fixture_call(recorder, ledger, 'put_bucket_versioning', {'Bucket': bucket, 'VersioningConfiguration': {'Status': 'Enabled'}}) expected[bucket] = [] for index in range(count): @@ -148,7 +148,7 @@ def seed_fixtures(probe, directory, buckets, ledger): 'ContentType': 'application/octet-stream', 'Metadata': {'heal-fixture': 'H06', 'nonce': f'{index}-{version}'}, 'Tagging': 'suite=heal&fixture=H06'}, body) version_id = response.get('VersionId', 'null') - require((versions == 1 and version_id == 'null') or (versions == 2 and version_id not in {None, 'null'}), + require((versions == 1 and version_id == 'null') or (versions == 3 and version_id not in {None, 'null'}), 'H06 versioning mode differs from its exact PUT ACK') expected[bucket].append({'key': key, 'versionId': version_id, 'deleteMarker': False, 'isLatest': version == versions - 1, 'payload': {'size': SIZE, 'sha256': hashlib.sha256(body).hexdigest()}}) diff --git a/test_validation_heal_multi_pool.py b/test_validation_heal_multi_pool.py index 2c01bbb4..51d740cd 100644 --- a/test_validation_heal_multi_pool.py +++ b/test_validation_heal_multi_pool.py @@ -26,7 +26,7 @@ def fixture(self): for pool in range(2): for group in range(2): key = f'p{pool}/s{group}.bin' - versions = ('v1', 'v2') if bucket.endswith('-dv') else ('null',) + versions = ('v1', 'v2', 'v3') if bucket.endswith('-dv') else ('null',) for offset, version in enumerate(versions): expected[bucket].append({'key': key, 'versionId': version}) for node, observed in enumerate(snapshot.values()): @@ -91,7 +91,7 @@ def test_original_h2_mapper_rejects_multi_pool_without_explicit_authority(self): layout, mapped = runner.map_versions(snapshot, expected) self.assertEqual(snapshot, before) self.assertEqual(layout['independentDrives'], 16) - self.assertEqual(len(mapped), 12) + self.assertEqual(len(mapped), 16) self.assertEqual({(row['poolIndex'], row['setIndex']) for row in mapped.values()}, {(0, 0), (0, 1), (1, 0), (1, 1)}) self.assertTrue(all(len(row['cohort']) == len({member['node'] for member in row['cohort']}) == 4 for row in mapped.values())) @@ -127,7 +127,7 @@ def test_raw_pool_only_receipts_and_exact_version_scope_are_not_rewritten(self): self.assertTrue(proof['rawUnrestrictedSetNotRewritten']) self.assertEqual({row['setIndex'] for row in proof['physicalScopeBindings']}, {0, 1}) self.assertEqual(proof['repaired'], 4) - self.assertEqual(proof['versions'], 6) + self.assertEqual(proof['versions'], 8) def test_wrong_selectors_versions_responsibility_and_false_counts_fail(self): for kind in ('pool', 'set', 'bool', 'version', 'duplicate', 'missing', 'foreign', 'unknown', 'counter', 'settings', 'truncated', 'incarnation'): @@ -241,8 +241,8 @@ def read(operation, params): with self.assertRaises(ValueError): runner.witness_peer_loss(fleet, binding, expected, mapped) else: proof = runner.witness_peer_loss(fleet, binding, expected, mapped) - self.assertEqual(proof['exactVersions'], 12) - self.assertEqual(probe.return_value.s3.call_count, 36) + self.assertEqual(proof['exactVersions'], 16) + self.assertEqual(probe.return_value.s3.call_count, 48) self.assertEqual({call.args[1] for call in probe.call_args_list}, {'node1', 'node2', 'node3'}) def test_unresolved_seed_prevents_namespace_cleanup_and_next_branch_admission(self): From 6f4901573967c53b88239af361c1ab543f44fd4d Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 09:00:14 +0800 Subject: [PATCH 104/343] test: collect exact candidate conflicting authority regressions --- scripts/validation_heal/native_regressions.py | 17 +++++++++++++++++ test_validation_heal_native.py | 10 ++++++++++ 2 files changed, 27 insertions(+) diff --git a/scripts/validation_heal/native_regressions.py b/scripts/validation_heal/native_regressions.py index 686d93b7..600d0854 100644 --- a/scripts/validation_heal/native_regressions.py +++ b/scripts/validation_heal/native_regressions.py @@ -47,6 +47,23 @@ "fenceKind": "uncontrolled concurrency, not deterministic fence evidence", "relatedCases": ["heal.race-version-delete"], "limits": "does not prove a deterministic delete/repair commit fence"}, + "conflicting-authority-guard": { + "test": PREFIX + "dangling_delete_guard_preserves_conflicting_identities_without_writing_metadata", + "sourceAssertions": ["conflicting candidates share one explicit version ID but different data directories", + "both original and reversed candidate order return UnsafeToDelete", + "explicit-version regeneration refuses both conflicting authorities", + "missing drives retain FileNotFound; both original data-directory identities remain"], + "fenceKind": "deterministic authority decision assertions in the unmodified candidate test", + "relatedCases": ["heal.conflicting-subquorum-authority"], + "limits": "controlled guard decisions; does not emit or prove a concurrent runtime commit-pause fence"}, + "reconstructable-authority-preservation": { + "test": PREFIX + "heal_meta_quorum_failure_preserves_reconstructable_uncommitted_candidate", + "sourceAssertions": ["the actual read_all_fileinfo result is UnsafeToDelete", + "no_lock Heal returns typed FileNotFound without deleting the surviving metadata", + "surviving data-directory identity is unchanged; missing drives receive no manufactured metadata"], + "fenceKind": "deterministic preservation assertions in the unmodified candidate test", + "relatedCases": ["heal.conflicting-subquorum-authority"], + "limits": "separate native regression; does not certify a live split, overlap or runtime pause"}, } diff --git a/test_validation_heal_native.py b/test_validation_heal_native.py index b3d1b0c9..3702f24b 100644 --- a/test_validation_heal_native.py +++ b/test_validation_heal_native.py @@ -34,6 +34,16 @@ def test_uncontrolled_concurrency_is_explicitly_not_a_fence(self): self.assertIn('not deterministic fence', native.REGRESSIONS[key]['fenceKind']) self.assertIn('cannot invent emitted barrier logs', native.REGRESSIONS['read-repair-lock-owner']['limits']) + def test_authority_guards_bind_exact_candidate_tests_without_claiming_runtime_pause(self): + for key, suffix in ( + ('conflicting-authority-guard', 'dangling_delete_guard_preserves_conflicting_identities_without_writing_metadata'), + ('reconstructable-authority-preservation', 'heal_meta_quorum_failure_preserves_reconstructable_uncommitted_candidate')): + spec = native.REGRESSIONS[key] + self.assertEqual(spec['test'], native.PREFIX + suffix) + self.assertEqual(spec['relatedCases'], ['heal.conflicting-subquorum-authority']) + self.assertIn('deterministic', spec['fenceKind']) + self.assertIn('pause', spec['limits']) + def test_wrong_source_modified_contract_or_collector_fail_before_execution(self): base = {'schemaVersion': 1, 'candidateCommit': native.CANDIDATE, 'sourceFiles': {native.SOURCE: native.SOURCE_SHA}, 'sourceUnmodified': True, 'regressions': native.REGRESSIONS, 'optimized': True, From f63f3ece5f78670b6880a30390802dd658ebfb6d Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 09:06:51 +0800 Subject: [PATCH 105/343] test: allow exact read-only H5 pool identity observations --- scripts/validation_heal/heal_probe.py | 10 +++++++++- test_validation_heal_multi_pool.py | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/heal_probe.py b/scripts/validation_heal/heal_probe.py index 9f0a99a3..259d57ff 100644 --- a/scripts/validation_heal/heal_probe.py +++ b/scripts/validation_heal/heal_probe.py @@ -259,12 +259,20 @@ def admin(self, path: str, query: list[tuple[str, str]], body: bytes, with observation_phase(observation, 'requestPreparation'): validate_binding(self.binding) quota_path = path.startswith("/rustfs/admin/v3/quota/") - require(path.startswith("/rustfs/admin/v3/heal/") or quota_path or path in { + pool_path = path in {"/rustfs/admin/v3/pools/list", "/rustfs/admin/v3/pools/status"} + require(path.startswith("/rustfs/admin/v3/heal/") or quota_path or pool_path or path in { "/rustfs/admin/v3/background-heal/status", "/rustfs/admin/v4/heal/replacement-recovery", "/rustfs/admin/v3/info"}, "only bounded Heal evidence routes are allowed") if path == "/rustfs/admin/v3/info": require(method == "GET" and not query and body == b"" and auth == "signed", "server-info evidence requires an exact signed read-only GET") + if pool_path: + require(self.binding['runId'].endswith('-h5') and len(self.binding['endpoints']) == 4 + and method == 'GET' and body == b'' and auth == 'signed', + 'pool observations require the dedicated four-node H5 read-only binding') + allowed_query = query == [] if path.endswith('/list') else query in ( + [('pool', '0'), ('by-id', 'true')], [('pool', '1'), ('by-id', 'true')]) + require(allowed_query, 'pool observation selector escaped the two exact registered H5 pools') if quota_path: bucket = path.removeprefix("/rustfs/admin/v3/quota/") bucket_allowed(self.binding, bucket) diff --git a/test_validation_heal_multi_pool.py b/test_validation_heal_multi_pool.py index 51d740cd..a55918e0 100644 --- a/test_validation_heal_multi_pool.py +++ b/test_validation_heal_multi_pool.py @@ -1,4 +1,5 @@ import copy +from datetime import datetime, timedelta, timezone from pathlib import Path import tempfile import types @@ -83,6 +84,26 @@ def test_preserves_all_three_source_branches_and_real_two_pool_arguments(self): self.assertEqual(runtime_config('multi-pool')['RUSTFS_STORAGE_CLASS_STANDARD'], 'EC:2') with self.assertRaises(ValueError): volumes('gh-123-1-h2') + def test_h5_pool_observations_accept_only_exact_signed_get_queries(self): + binding = {'schemaVersion': 1, 'runId': 'gh-123-1-h5', 'jumpHostOnly': True, 'exclusiveHealService': True, + 'leaseExpiresAt': (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), + 'bucketPrefix': self.buckets[0], 'endpoints': {f'node{i}': f'http://rustfs-node{i}:9000' for i in range(1, 5)}, + 'defaultEndpoint': 'node1', 'candidateCommit': 'a' * 40, 'harnessCommit': 'b' * 40, + 'candidatePackageSha256': 'c' * 64, 'candidateBinarySha256': 'd' * 64, 'rcBinarySha256': 'e' * 64, + 'maxInventoryObjects': 256, 'maxInventoryBytes': 512 * 1024**2} + for path, query, method, auth, allowed in ( + ('list', [], 'GET', 'signed', True), ('status', [('pool', '1'), ('by-id', 'true')], 'GET', 'signed', True), + ('list', [], 'POST', 'signed', False), ('list', [], 'GET', 'anonymous', False), + ('list', [('pool', '0')], 'GET', 'signed', False), ('status', [('pool', '2'), ('by-id', 'true')], 'GET', 'signed', False), + ('status', [('pool', '0'), ('by-id', 'true'), ('pool', '1')], 'GET', 'signed', False)): + probe = runner.Probe(binding) + with self.subTest(path=path, query=query, method=method, auth=auth), patch.object(probe, 'credentials', side_effect=RuntimeError('allowed route reached signing')): + if allowed: + with self.assertRaisesRegex(RuntimeError, 'reached signing'): + probe.admin('/rustfs/admin/v3/pools/' + path, query, b'', method=method, auth=auth) + else: + with self.assertRaises(ValueError): probe.admin('/rustfs/admin/v3/pools/' + path, query, b'', method=method, auth=auth) + def test_original_h2_mapper_rejects_multi_pool_without_explicit_authority(self): snapshot, expected = self.fixture() with self.assertRaises(ValueError): validate_topology(snapshot, 'H2') From 5ab098530ca73adb170e2857869560e1567fcb93 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 09:11:19 +0800 Subject: [PATCH 106/343] docs: record complete stop-token live evidence and H5 CI --- docs/validation-migration-map.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index bab0a46a..1543a587 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1447,3 +1447,29 @@ branches passed on both topologies, but SSE-C ACK/GET classification failed on both. Neither run increases the 47/158 complete canonical live PASS count. The RustFS aggregate Test and Lint check also failed because its three runner-shutdown lanes failed; its final summary preserves those exact failures. + + +H06 read-only pool observations now have an explicit Probe allowlist: signed +GET only, four-node H5 binding, exact pool list or numeric pool 0/1 by-id +selectors, empty body and no duplicate/extra queries. All six CI checks passed +at `f63f3ece5f78670b6880a30390802dd658ebfb6d`. The pending H06 serial live plan +now binds that corrected immutable revision; the active plan was unchanged. + +A06 run `38099706345` passed all four stop/stale-token branches on H1/H2 using +the source's DX window fixture, at exact harness +`f248cb63d032cfc1658d32c508bce745f96b885f`. Independently inspected source branch +sets, topology coverage, nonzero interrupted progress, cancellation/physical +quietness, fresh task ownership and old-token behavior, complete recovered +physical/S3 manifests and cleanup all passed. The independent canonical gate +passed again. Complete canonical live PASS is now 48/158. Original A06 failures +remain archived and are not overwritten by this fresh successful run. + +The native collector additionally selects the candidate's two exact C07 +conflicting-authority/preservation regressions, with reviewed original source +assertions. These are controlled guard decisions, not emitted concurrent +runtime-pause fences or fleet PASS. The corrected hosted tools-only run +`38100407691` binds collector `6f4901573967c53b88239af361c1ab543f44fd4d` and is +still building native evidence. Earlier dispatch `38100369019` used an +incorrect complete collector revision and was cancelled; it is not a success +or native evidence source. C07 fleet migration is still in progress and does +not change the 130 automatic executors / 28 remaining Heal executor count. From 07627ade0667e67e5560fccbc08215458d506b04 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 09:27:58 +0800 Subject: [PATCH 107/343] test(validation): migrate conflicting subquorum authority case --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 39 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/authority.py | 220 +++++++++ scripts/validation_heal/authority_runner.py | 300 +++++++++++ scripts/validation_heal/fleet_node.py | 20 +- scripts/validation_heal/generation_fault.py | 177 +++++++ scripts/validation_heal/healthy.py | 18 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/run_case.sh | 1 + test_validation_heal_authority.py | 464 ++++++++++++++++++ 11 files changed, 1240 insertions(+), 7 deletions(-) create mode 100644 scripts/validation_heal/authority.py create mode 100644 scripts/validation_heal/authority_runner.py create mode 100644 scripts/validation_heal/generation_fault.py create mode 100644 test_validation_heal_authority.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index fe17f79f..337b9b4f 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -33,6 +33,7 @@ on: - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py + - test_validation_heal_authority.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -79,6 +80,7 @@ on: - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py + - test_validation_heal_authority.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -116,7 +118,7 @@ jobs: sudo apt-get update sudo apt-get install --no-install-recommends -y nftables fi - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_authority.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 1543a587..524aadd8 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 128 currently dispatchable IDs include all 74 non-Heal cases and fifty-four Heal cases. Forty-six have live PASS evidence; the other 112 have no live PASS claim. +case evidence. The 131 currently dispatchable IDs include all 74 non-Heal cases and fifty-seven Heal cases. Forty-eight have complete canonical live PASS evidence; the other 110 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1473,3 +1473,40 @@ still building native evidence. Earlier dispatch `38100369019` used an incorrect complete collector revision and was cancelled; it is not a success or native evidence source. C07 fleet migration is still in progress and does not change the 130 automatic executors / 28 remaining Heal executor count. + +C07 now has all three registered source branches on H1/H2. Its isolated EC:1 +profile establishes actual 3+1/R3 and 15+1/R15 sets; each exact half-generation +group is below its observed read quorum. Full A/B original-member trees are +archived while the owned units are stopped, then replayed through descriptor +relative operations with durable rename intents. The versioned branch creates +three distinct acknowledged generations before capturing A and explicitly +deletes prior versions so hidden historical metadata cannot create an authority +quorum. The unversioned branch tests distinct physical generations as in the +source gate; it does not claim a bucket-incarnation split. + +The complete census counts every live identity, binds original format UUID, +slot, role and geometry, and compares every chosen-generation byte. Every peer +must return the exact candidate-derived negative quorum GET error; a Heal task +must either receive the same typed admission refusal or prove explicit failed +or untraversable responsibility. Partial negative coverage remains partial, +and all peers have one 30-second budget to converge on the original path/token +and exact outcome. Original mixed/control bytes must remain unchanged before +fixture recovery. Only the complete last acknowledged B generation is rescued; +streaming bitrot and full all-peer S3 manifests then prove that ACK. Unknown +mutation receipts or partial recovery retain DIRTY namespaces. + +The driver downloads C07's native artifact from the same pinned successful +tools attempt, verifies original source/tree/Cargo.lock/collector/executable +identities and all six exact test listings/results before fleet deployment, +and maps only the two original authority guards. Their actual native fixture +has four members; repeating that independent proof in an H2 result does not +claim sixteen-member native execution or an emitted runtime-pause fence. + +The implementation now has 131 automatic executors (74 non-Heal, 57 Heal), +with 27 Heal executors remaining. The complete local suite ran 375 tests with +seven macOS skips, including 21 C07 census, native-evidence, pending-ACK, +recovery and filesystem tests. Linux CI additionally requires real mount +descriptors, root-owned generation recovery and nested bind-mount refusal. +The 158 IDs/hashes and 84/301/532 matrix, Bash syntax, scoped actionlint and +whitespace checks passed. C07 has no canonical live PASS claim; hosted native +run `38100407691` is still building. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index b038d922..66b589ef 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/authority.py b/scripts/validation_heal/authority.py new file mode 100644 index 00000000..49329e9e --- /dev/null +++ b/scripts/validation_heal/authority.py @@ -0,0 +1,220 @@ +"""C07 authority census and immutable native evidence for actual split fixtures.""" + +from collections import Counter +import argparse +import json +from pathlib import Path +import re + +try: + from .heal_probe import require + from .healthy import validate_topology + from .shards import files + from .versions import public_version_id + from . import native_regressions as native +except ImportError: + from heal_probe import require + from healthy import validate_topology + from shards import files + from versions import public_version_id + import native_regressions as native + + +CASE_ID = 'heal.conflicting-subquorum-authority' +BRANCHES = ('disjoint-version-quorums', 'disjoint-incarnations', 'native-authority-fence') +NATIVE_KEYS = ('conflicting-authority-guard', 'reconstructable-authority-preservation') +SOURCE_TREE = '8071fb0a9e5353eb7d3fa58c8a58c8622473f2bd' +CARGO_LOCK = 'd8f7bc416ba67d19873d34e9389e5aa823b22d7d7552706a5a810cee26f6eda2' + + +def object_files(snapshot, bucket, key): + prefix = bucket + '/' + key + '/' + indexed = files(snapshot) + return {(node, disk): [{'path': path.removeprefix(prefix), 'size': entry['size'], 'sha256': entry['sha256']} + for (owner, drive, path), entry in sorted(indexed.items()) + if owner == node and drive == disk and path.startswith(prefix)] + for node, value in snapshot.items() for disk in range(len(value['formats']))} + + +def census(snapshot, bucket, key, topology): + layout = validate_topology(snapshot, topology) + indexed = files(snapshot) + identities, slots, geometry = Counter(), [], None + for node, observed in snapshot.items(): + for fmt in observed['formats']: + disk = fmt['disk'] + row = indexed.get((node, disk, f'{bucket}/{key}/xl.meta')) + require(row and row['metadata'].get('bucket') == bucket and row['metadata'].get('key') == key, + 'C07 lost an original metadata member or its exact object identity') + versions = row['metadata']['versions'] + latest = [version for version in versions if version.get('isLatest') is True] + require(len(latest) == 1 and latest[0].get('deleted') is False and latest[0].get('inline') is None, + 'C07 requires one external current candidate on every original member') + version = latest[0] + erasure = version['erasure'] + coding = {name: erasure[name] for name in ('data_blocks', 'parity_blocks', 'block_size', 'distribution', 'algorithm')} + require(geometry is None or geometry == coding, 'C07 generations use different coding/distribution contracts') + geometry = coding + members = fmt['value']['xl']['sets'][0] + slot = members.index(fmt['value']['xl']['this']) + require(type(erasure['index']) is int and erasure['index'] == erasure['distribution'][slot] + and version.get('role') == ('data' if erasure['index'] <= erasure['data_blocks'] else 'parity'), + 'C07 object role does not bind its original format slot') + # Count every live version separately. An older version hidden in + # the other group's metadata must not silently reach read quorum. + live = [item for item in versions if item.get('deleted') is False] + for candidate in live: + identity = {name: candidate.get(name) for name in ('dataDir', 'size', 'parts', 'deleted')} + identity['versionId'] = public_version_id(candidate.get('versionId')) + require(isinstance(identity['dataDir'], str) and identity['dataDir'], 'C07 version has no physical data identity') + digest = json.dumps(identity, sort_keys=True, separators=(',', ':')) + identities[digest] += 1 + slots.append({'node': node, 'disk': disk, 'diskUUID': fmt['value']['xl']['this'], + 'metadataSha256': row['sha256'], 'versionId': public_version_id(version.get('versionId')), + 'dataDir': version['dataDir'], 'setSlot': slot}) + total = len(slots) + require(type(geometry['data_blocks']) is int and type(geometry['parity_blocks']) is int + and geometry['data_blocks'] + geometry['parity_blocks'] == total + and sorted(geometry['distribution']) == list(range(1, total + 1)) + and len({row['diskUUID'] for row in slots}) == total, + 'C07 census lacks one exact erasure set and all independent original members') + return {'topology': topology, 'layout': layout, 'coding': geometry, 'N': total, + 'R': geometry['data_blocks'], 'groups': [{'identity': json.loads(identity), 'members': count} + for identity, count in sorted(identities.items())], 'slots': slots} + + +def validate_split(snapshot, generation_a, generation_b, bucket, key, topology, branch): + require(branch in BRANCHES[:2], 'native evidence cannot stand in for a physical split') + before_a = census(generation_a, bucket, key, topology) + before_b = census(generation_b, bucket, key, topology) + mixed = census(snapshot, bucket, key, topology) + require(before_a['N'] == before_b['N'] == mixed['N'] and before_a['coding'] == before_b['coding'] == mixed['coding'] + and len(before_a['groups']) == len(before_b['groups']) == 1 + and before_a['groups'][0]['members'] == before_b['groups'][0]['members'] == mixed['N'], + 'C07 archived generations are not independently complete single authorities') + identities = [row['groups'][0]['identity'] for row in (before_a, before_b)] + require(identities[0] != identities[1] and identities[0]['dataDir'] != identities[1]['dataDir'], + 'C07 generations reuse one logical or physical authority') + if branch == 'disjoint-version-quorums': + require(identities[0]['versionId'] != identities[1]['versionId'] + and all(identity['versionId'] != 'null' for identity in identities), 'C07 version groups are not distinct real ACKs') + else: + require(all(identity['versionId'] == 'null' for identity in identities), + 'C07 incarnation branch needs distinct unversioned physical generations, not bucket-incarnation claims') + require(len(mixed['groups']) == 2 and all(row['identity'] in identities and row['members'] == mixed['N'] // 2 + and row['members'] < mixed['R'] for row in mixed['groups']), + 'C07 live identities do not form two independently sub-quorum groups') + a, b, current = (object_files(value, bucket, key) for value in (generation_a, generation_b, snapshot)) + members = sorted(current) + require(set(a) == set(b) == set(current) and mixed['N'] == len(members), 'C07 split omitted original members') + for index, member in enumerate(members): + require(current[member] == (a if index < len(members) // 2 else b)[member], + 'C07 split does not preserve every exact chosen generation byte') + return {**mixed, 'everyLiveIdentityBelowR': True, 'crossVersionUnionClaimedAsAuthority': False, + 'bucketIncarnationSplitClaimed': False} + + +def validate_negative_task(status, bucket, options): + outcome = status.get('outcome') or {} + execution = outcome.get('execution') or {} + require(status.get('summary') == 'finished' and status.get('truncated', False) is False + and status.get('settings') == options and outcome.get('objectsTruncated') is False + and execution.get('state') in {'completed_with_errors', 'aborted'} + and outcome.get('coverage') in {'complete', 'partial', 'unknown'}, + 'C07 Heal lost its explicit terminal refusal/selector/coverage evidence') + counters = outcome.get('counters') or {} + names = ('processed', 'healed', 'unchanged', 'skipped', 'failed', 'unknown', 'attemptFailures') + require(all(type(counters.get(name)) is int and counters[name] >= 0 for name in names) + and counters.get('overflowed') is False and counters['healed'] == counters['unchanged'] == 0 + and (counters['failed'] + counters['unknown'] > 0 or execution == {'state': 'aborted', 'reason': 'untraversable'}), + 'C07 task claimed repair/health or concealed its unresolved responsibility') + objects = outcome.get('objects') + require(isinstance(objects, list), 'C07 task omitted its responsibility diagnostics') + if execution.get('state') == 'aborted': + require(execution == {'state': 'aborted', 'reason': 'untraversable'} + and outcome['coverage'] in {'partial', 'unknown'}, + 'C07 abort concealed its untraversable responsibility or claimed full coverage') + else: + require(bool(objects), 'C07 failed task has no exact conflict responsibility') + identities = set() + for row in objects: + identity = row.get('identity') or {} + require(identity.get('bucket') == bucket and identity.get('object') == 'conflict.bin' + and identity.get('kind') == 'object' and bool(identity.get('bucketIncarnationId')) + and row.get('disposition', {}).get('state') in {'failed', 'unknown', 'deferred'}, + 'C07 task attributed a false repair or refusal to another object') + require(type(identity.get('poolIndex')) is int and identity['poolIndex'] == options['pool'] == 0 + and type(identity.get('setIndex')) is int and identity['setIndex'] == options['set'] == 0, + 'C07 refusal escaped the exact requested original pool/set') + key = (identity['bucketIncarnationId'], public_version_id(identity.get('versionId'))) + require(key not in identities, 'C07 refused-task diagnostics repeat one responsibility identity') + identities.add(key) + states = [row['disposition']['state'] for row in objects] + require(counters['processed'] == len(objects) and counters['failed'] == states.count('failed') + and counters['skipped'] == states.count('unknown') + states.count('deferred') + and counters['unknown'] == states.count('unknown') and counters['attemptFailures'] >= counters['failed'], + 'C07 refused-task counters conceal missing or mismatched object diagnostics') + return {'safeExplicitRefusal': True, 'healed': 0, 'verifiedHealthy': 0, + 'runtimeCommitPauseClaimed': False, 'coverage': outcome['coverage']} + + +def verify_native(directory, candidate): + directory = Path(directory) + tools, evidence = directory / 'native-tools', directory / 'native-evidence' + manifest = json.loads((tools / 'native-regressions.json').read_text()) + build_run = json.loads((directory / 'tool-run.json').read_text()) + require(type(build_run.get('id')) is int and build_run['id'] > 0 + and type(build_run.get('run_attempt')) is int and build_run['run_attempt'] > 0 + and build_run.get('status') == 'completed' and build_run.get('conclusion') == 'success' + and build_run.get('path') in {'.github/workflows/rustfs-functional-chain.yml', '.github/workflows/rustfs-validation-tools.yml'} + and manifest.get('runId') == str(build_run['id']) and manifest.get('runAttempt') == str(build_run['run_attempt']), + 'C07 native artifact is not bound to the downloaded successful tools build attempt') + require(candidate['source_sha'] == native.CANDIDATE and manifest.get('schemaVersion') == 1 + and manifest.get('candidateCommit') == native.CANDIDATE and manifest.get('candidateSourceTree') == SOURCE_TREE + and manifest.get('candidateCargoLockSha256') == CARGO_LOCK + and manifest.get('sourceFiles') == {native.SOURCE: native.SOURCE_SHA} + and manifest.get('sourceUnmodified') is True and manifest.get('optimized') is True + and manifest.get('collectorSha256') == native.sha(Path(native.__file__)) + and manifest.get('regressions') == native.REGRESSIONS and manifest.get('fleetPassClaimed') is False, + 'C07 native artifact changed its exact source/build/collector contract') + require(manifest.get('binaryName') == 'ecstore-native-tests' and re.fullmatch(r'[0-9a-f]{64}', manifest.get('binarySha256', '')), + 'C07 native executable path or digest is not registered') + binary = tools / 'ecstore-native-tests' + require(binary.is_file() and not binary.is_symlink() and native.sha(binary) == manifest['binarySha256'], + 'C07 native executable differs from its hosted source build') + summary = json.loads((evidence / 'native-summary.json').read_text()) + require(summary.get('complete') is True and summary.get('fleetPassClaimed') is False + and summary.get('canonicalCasePassClaimed') is False and summary.get('sourceUnmodified') is True, + 'C07 native regressions did not complete independently') + records = summary.get('results', []) + require(len(records) == len(native.REGRESSIONS) and {row['regression'] for row in records} == set(native.REGRESSIONS), + 'C07 native summary omitted, duplicated or added exact tests') + listing = (tools / 'test-inventory.log').read_text() + runtime_listing = (evidence / 'runtime-inventory.log').read_text() + proofs = [] + for key in native.REGRESSIONS: + spec = native.REGRESSIONS[key] + row = next(record for record in records if record['regression'] == key) + require(row == json.loads((evidence / (key + '.json')).read_text()) and row.get('verdict') == 'PASS' + and row.get('test') == spec['test'] and row.get('candidateCommit') == native.CANDIDATE + and row.get('binarySha256') == manifest['binarySha256'] and row.get('emittedBarrierLogsClaimed') is False + and row.get('fleetPassClaimed') is False + and row.get('command') == ['native-tools/ecstore-native-tests', spec['test'], '--exact', '--nocapture', '--test-threads=1'] + and row.get('sourceAssertions') == spec['sourceAssertions'] and row.get('fenceKind') == spec['fenceKind'], + 'C07 native result does not bind one exact original test execution') + native.exact_listing(listing, spec['test']) + native.exact_listing(runtime_listing, spec['test']) + proof = native.test_result((evidence / (key + '.log')).read_text(), spec['test'], row.get('exitCode')) + if key in NATIVE_KEYS: + proofs.append({'regression': key, 'test': spec['test'], **proof}) + return {'candidateCommit': native.CANDIDATE, 'binarySha256': manifest['binarySha256'], 'results': proofs, + 'toolsBuildRun': build_run['id'], 'toolsBuildAttempt': build_run['run_attempt'], + 'nativeRegressionPassed': True, 'fleetPassClaimed': False, 'runtimeCommitPauseClaimed': False} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--directory', type=Path, required=True) + parser.add_argument('--candidate', type=Path, required=True) + args = parser.parse_args() + print(json.dumps(verify_native(args.directory, json.loads(args.candidate.read_text())), indent=2)) diff --git a/scripts/validation_heal/authority_runner.py b/scripts/validation_heal/authority_runner.py new file mode 100644 index 00000000..570014cc --- /dev/null +++ b/scripts/validation_heal/authority_runner.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +"""Execute C07 live authority splits and independently bound native guard evidence.""" + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +from urllib.parse import quote +import xml.etree.ElementTree as ET + +try: + from .authority import CASE_ID, BRANCHES, census, object_files, validate_split, validate_negative_task, verify_native + from .heal_probe import Probe, canonical, compare_inventories, payload, require, save + from .heal_workload import Recorder, verify_seed + from .healthy import BASE_OPTIONS, Fleet, physical_content + from .inspector import verify as verify_inspector + from .matrix import select + from .native_regressions import CANDIDATE + from .scopes import verify_parts + from .shards import map_target + from .stop import idle_fleet + from .versions import validate_unrecoverable_get, verify_quorum_contract +except ImportError: + from authority import CASE_ID, BRANCHES, census, object_files, validate_split, validate_negative_task, verify_native + from heal_probe import Probe, canonical, compare_inventories, payload, require, save + from heal_workload import Recorder, verify_seed + from healthy import BASE_OPTIONS, Fleet, physical_content + from inspector import verify as verify_inspector + from matrix import select + from native_regressions import CANDIDATE + from scopes import verify_parts + from shards import map_target + from stop import idle_fleet + from versions import validate_unrecoverable_get, verify_quorum_contract + + +SIZE = 1024**2 + 73 + + +def negative_heal(probe, fleet, bucket, options, active, contract): + path = '/rustfs/admin/v3/heal/' + quote(bucket, safe='') + '/conflict.bin' + started = fleet.record('negative-authority-admission', probe.admin(path, [], json.dumps(options).encode())) + expected_error = contract['unrecoverableGet'] + if started.get('httpStatus') == expected_error['httpStatus']: + body = started.get('response') + if isinstance(body, dict): + code = body.get('Code') or body.get('code') + require(not body.get('clientToken'), 'refused authority admission returned a task token') + else: + require(isinstance(body, str) and '= 4 and len(nodes) == len(set(nodes)) and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in nodes), + 'C07 requires four distinct current Heal nodes') + results = [] + try: + for topology in ('H1', 'H2'): + for branch in BRANCHES: + results.append(run_branch(args, topology, nodes[:1] if topology == 'H1' else nodes[:4], branch)) + if results[-1]['cleanup'] != 'PASS': break + if results[-1]['cleanup'] != 'PASS': break + finally: + complete = len(results) == 6 and all(row['verdict'] == row['cleanup'] == 'PASS' for row in results) + save(args.evidence / 'healthy-summary.json', {'caseId': CASE_ID, 'results': results, 'complete': complete, + 'cleanup': 'PASS' if results and all(row['cleanup'] == 'PASS' for row in results) else 'FAIL'}) + require(complete, 'C07 needs both actual live topologies, both split branches, independent native guards and complete cleanup') + + +if __name__ == '__main__': + try: main() + except Exception as exc: + print(f'C07 authority validation failed: {exc}', file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 7d8a54e8..c4bd4194 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -33,7 +33,9 @@ def runtime_config(profile): - require(profile in {"manual", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(profile in {"manual", "authority-split", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + if profile == "authority-split": + return {**BACKGROUND_CONFIG, "RUSTFS_STORAGE_CLASS_STANDARD": "EC:1"} if profile == "multi-pool": return {**BACKGROUND_CONFIG, "RUSTFS_ERASURE_SET_DRIVE_COUNT": "4", "RUSTFS_STORAGE_CLASS_STANDARD": "EC:2"} if profile == "bounded-capacity": @@ -1039,6 +1041,20 @@ def task_records(request): return {"taskId": token, "binarySha256": owner["binarySha256"], "readOnly": True, "records": records} +def generation_operation(request): + state, roots, owner = owned(request['runId']) + stopped(owner) + require(digest(state / 'rustfs') == owner['binarySha256'], 'generation candidate binary changed') + disk = request['disk'] + require(type(disk) is int and 0 <= disk < 4, 'generation operation escaped an original independent root') + operation = request['action'] + require(operation in {'generation-capture', 'generation-install', 'generation-recover'}, 'unregistered generation operation') + backend = globals()['generation_backend'] + with fault_module().directory(roots[disk]) as root: + method = {'generation-capture': backend.capture, 'generation-install': backend.install, 'generation-recover': backend.recover}[operation] + return method(root, owner, request, fault_module()) + + def mrf_records(request): require(set(request) <= {"runId", "action", "quiescent"}, "MRF observer does not accept paths or mutation options") state, roots, owner = owned(request["runId"]) @@ -1073,6 +1089,8 @@ def main(request): "drive-gate-activate": drive_gate_activate, "gate-state": drive_gate_state, "drive-gate-release-live": drive_gate_release_live, "drive-gate-arm-live": drive_gate_arm_live} actions["task-records"] = task_records + for action in ('generation-capture', 'generation-install', 'generation-recover'): + actions[action] = generation_operation for action in ("bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover"): actions[action] = bucket_record_operation for action in ("quorum-gate", "quorum-gate-probe"): diff --git a/scripts/validation_heal/generation_fault.py b/scripts/validation_heal/generation_fault.py new file mode 100644 index 00000000..b60a2319 --- /dev/null +++ b/scripts/validation_heal/generation_fault.py @@ -0,0 +1,177 @@ +"""C07: preserve two exact stopped object generations and restore only the last ACK. + +All operations are relative to the original run-owned root's descriptors. +Archive copies and recovery are fixture operations, never product repair. +""" + +import json +import os +import re +import stat +import uuid + + +def require(value, message): + if not value: + raise ValueError(message) + + +def scope(owner, request): + bucket, key = request['bucket'], request['key'] + require(isinstance(bucket, str) and re.fullmatch(r'rv-heal-[0-9a-f]{16}(?:-dv)?', bucket) + and (bucket == owner['bucket'] or bucket == owner['bucket'] + '-dv') + and key == 'conflict.bin', 'generation fixture escaped its exact owned bucket/key') + require(owner.get('runtimeProfile') == 'authority-split', 'generation replay requires the registered C07 runtime') + return [bucket, key] + + +def tree(fd, fault): + """Read the complete small object tree; reject aliases or unexpected leaves.""" + values = {} + names = sorted(os.listdir(fd)) + require('xl.meta' in names and 1 < len(names) <= 17, 'generation lacks bounded external metadata/data directories') + for name in names: + if name == 'xl.meta': + values[name] = fault.read_file(fd, name) + continue + require(str(uuid.UUID(name)) == name and uuid.UUID(name).int != 0, 'generation contains an unreviewed object child') + with fault.parent(fd, [name]) as part_fd: + parts = sorted(os.listdir(part_fd)) + require(parts and len(parts) <= 16 and all(re.fullmatch(r'part\.[1-9][0-9]*', part) for part in parts), + 'generation contains unreviewed multipart leaves') + for part in parts: + values[name + '/' + part] = fault.read_file(part_fd, part) + require(sum(map(len, values.values())) <= 32 * 1024**2, 'generation exceeds its per-drive byte budget') + manifest = [{'path': name, 'size': len(raw), 'sha256': fault.digest(raw)} for name, raw in sorted(values.items())] + return values, manifest + + +def tree_sha(manifest, fault): + return fault.digest(json.dumps(manifest, sort_keys=True, separators=(',', ':')).encode()) + + +def copy_into(fd, values, fault): + require(not os.listdir(fd), 'exclusive generation copy target is not empty') + for name, raw in sorted(values.items()): + parts = name.split('/') + if len(parts) == 1: + fault.new_file(fd, name, raw) + else: + require(len(parts) == 2, 'generation copy depth changed') + if parts[0] not in os.listdir(fd): + os.mkdir(parts[0], 0o700, dir_fd=fd) + os.fsync(fd) + with fault.parent(fd, parts[:1]) as child: + fault.new_file(child, parts[1], raw) + os.fsync(fd) + + +def archive(root_fd, label, fault): + return fault.parent(root_fd, ['quarantine', 'C07-' + label]) + + +def capture(root_fd, owner, request, fault): + parts = scope(owner, request) + label = request['generation'] + require(label in {'a', 'b'}, 'unregistered generation archive') + with fault.parent(root_fd, ['data', *parts]) as original: + values, manifest = tree(original, fault) + require(manifest == request['expectedFiles'], 'capture differs from the frozen full-drive object tree') + record = {'schemaVersion': 1, 'runId': owner['runId'], 'bucket': parts[0], 'key': parts[1], + 'disk': request['disk'], 'generation': label, 'mountId': fault.mount_id(root_fd), + 'device': os.fstat(root_fd).st_dev, 'files': manifest, 'treeSha256': tree_sha(manifest, fault), + 'backupRestorationIsProductRepair': False} + with fault.parent(root_fd, ['quarantine']) as quarantine: + os.mkdir('C07-' + label, 0o700, dir_fd=quarantine) + os.fsync(quarantine) + with archive(root_fd, label, fault) as saved: + fault.new_file(saved, 'manifest.json', fault.encode(record)) + os.mkdir('object', 0o700, dir_fd=saved) + with fault.parent(saved, ['object']) as destination: + copy_into(destination, values, fault) + require(tree(destination, fault)[1] == manifest, 'archived generation is incomplete') + fault.new_file(saved, 'complete.json', fault.encode(record)) + return record + + +def captured(root_fd, owner, request, label, fault): + parts = scope(owner, request) + with archive(root_fd, label, fault) as saved: + record = json.loads(fault.read_file(saved, 'complete.json', 65536)) + require(record == json.loads(fault.read_file(saved, 'manifest.json', 65536)) + and record['schemaVersion'] == 1 and record['runId'] == owner['runId'] + and (record['bucket'], record['key'], record['disk'], record['generation']) + == (parts[0], parts[1], request['disk'], label) + and record['mountId'] == fault.mount_id(root_fd) and record['device'] == os.fstat(root_fd).st_dev, + 'archived generation ownership or original device drifted') + with fault.parent(saved, ['object']) as original: + values, manifest = tree(original, fault) + require(manifest == record['files'] and tree_sha(manifest, fault) == record['treeSha256'], + 'archived generation bytes differ from the captured authority') + return values, record + + +def install(root_fd, owner, request, fault): + parts = scope(owner, request) + old_values, old = captured(root_fd, owner, request, 'a', fault) + _, current = captured(root_fd, owner, request, 'b', fault) + require(old['treeSha256'] != current['treeSha256'] + and request['expectedSceneSha256'] == current['treeSha256'], 'generation fault has no distinct frozen last ACK') + with fault.parent(root_fd, ['data', *parts]) as live: + require(tree(live, fault)[1] == current['files'], 'last acknowledged object drifted before generation replay') + intent = {'schemaVersion': 1, 'runId': owner['runId'], 'disk': request['disk'], 'bucket': parts[0], 'key': parts[1], + 'mountId': fault.mount_id(root_fd), 'device': os.fstat(root_fd).st_dev, + 'originalTreeSha256': current['treeSha256'], 'injectedTreeSha256': old['treeSha256'], + 'restoreGeneration': 'b', 'backupRestorationIsProductRepair': False} + with fault.parent(root_fd, ['quarantine']) as quarantine: + require('C07-pre-split' not in os.listdir(quarantine), 'generation replay would replace a retained scene') + fault.new_file(quarantine, 'C07-replay-intent.json', fault.encode(intent)) + with fault.parent(root_fd, ['data', parts[0]]) as bucket: + os.rename(parts[1], 'C07-pre-split', src_dir_fd=bucket, dst_dir_fd=quarantine) + os.fsync(bucket) + os.fsync(quarantine) + os.mkdir(parts[1], 0o700, dir_fd=bucket) + with fault.parent(bucket, [parts[1]]) as destination: + copy_into(destination, old_values, fault) + require(tree(destination, fault)[1] == old['files'], 'generation replay did not install every exact archived byte') + os.fsync(bucket) + fault.new_file(quarantine, 'C07-replay-applied.json', fault.encode(intent)) + return {**intent, 'state': 'APPLIED', 'oldAndNewArchivesPreserved': True} + + +def recover(root_fd, owner, request, fault): + parts = scope(owner, request) + values, original = captured(root_fd, owner, request, 'b', fault) + with fault.parent(root_fd, ['quarantine']) as quarantine: + intent = json.loads(fault.read_file(quarantine, 'C07-replay-intent.json', 65536)) + require(intent['schemaVersion'] == 1 and intent['runId'] == owner['runId'] + and (intent['bucket'], intent['key'], intent['disk']) == (parts[0], parts[1], request['disk']) + and intent['restoreGeneration'] == 'b' and intent['originalTreeSha256'] == original['treeSha256'] + and intent['mountId'] == fault.mount_id(root_fd) and intent['device'] == os.fstat(root_fd).st_dev, + 'recovery cannot establish original run/device/last-ACK ownership') + with fault.parent(root_fd, ['data', parts[0]]) as bucket: + if 'C07-failed-current' in os.listdir(quarantine): + # A completed rename can precede a lost SSH receipt. Confirm + # exact restoration; partial recovery remains DIRTY. + with fault.parent(bucket, [parts[1]]) as live: + require(tree(live, fault)[1] == original['files'], 'partial prior recovery cannot be discarded') + else: + if parts[1] in os.listdir(bucket): + observed = os.stat(parts[1], dir_fd=bucket, follow_symlinks=False) + require(stat.S_ISDIR(observed.st_mode) and observed.st_uid == 0 + and observed.st_dev == os.fstat(root_fd).st_dev, 'fault scene no longer owns its original directory') + os.rename(parts[1], 'C07-failed-current', src_dir_fd=bucket, dst_dir_fd=quarantine) + else: + os.mkdir('C07-failed-current', 0o700, dir_fd=quarantine) + os.fsync(quarantine) + os.fsync(bucket) + os.mkdir(parts[1], 0o700, dir_fd=bucket) + with fault.parent(bucket, [parts[1]]) as live: + copy_into(live, values, fault) + require(tree(live, fault)[1] == original['files'], 'recovery did not restore every last-ACK byte') + os.fsync(bucket) + receipt = {**intent, 'state': 'CLEAR', 'originalLastACKGenerationRestored': True, + 'failedScenePreserved': True, 'productRepairClaimed': False} + if 'C07-replay-recovered.json' not in os.listdir(quarantine): + fault.new_file(quarantine, 'C07-replay-recovered.json', fault.encode(receipt)) + return receipt diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 325a8c9e..755ef367 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"manual", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(runtime_profile in {"manual", "authority-split", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -151,6 +151,7 @@ def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_prof self.drive_gates = set() self.quorum_gates = set() self.network_gates = set() + self.generation_faults = [] self.secret_values = set() self.frozen_processes = {} self.snapshot_buckets = None @@ -194,8 +195,11 @@ def remote(self, node, action, **params): if action == "replacement-records": source += "\nimport types\nreplacement_journal = types.ModuleType('replacement_journal')\nexec(" + repr( NODE_SOURCE.with_name("replacement_journal.py").read_text()) + ", replacement_journal.__dict__)\n" + if action.startswith("generation-"): + source += "\nimport types\ngeneration_backend = types.ModuleType('generation_fault')\nexec(" + repr( + NODE_SOURCE.with_name("generation_fault.py").read_text()) + ", generation_backend.__dict__)\n" if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline", - "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure"}: + "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure"} or action.startswith("generation-"): source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" if action.startswith("bucket-record-"): @@ -309,6 +313,16 @@ def wait_ready(self): def cleanup(self): failures = [] + if self.generation_faults: + try: + self.stop() + for node, params in tuple(self.generation_faults): + receipt = self.record("generation-last-ACK-recovery", {"node": node, **self.remote(node, "generation-recover", **params)}) + require(receipt.get("state") == "CLEAR" and receipt.get("originalLastACKGenerationRestored") is True, + "generation replay lacks complete last-ACK restoration") + self.generation_faults.remove((node, params)) + except Exception as exc: + failures.append({"error": str(exc), "state": "DIRTY", "namespacesRetained": True}) if self.network_gates: try: self.stop() diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 3e525419..bdb46788 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 4e6c3202..3f243473 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -40,6 +40,7 @@ case "${case_id}" in heal.enospc-inodes-and-quota) driver=capacity.py ;; heal.network-partition-and-lock-quorum) driver=network.py ;; heal.multi-pool-target-isolation) driver=multi_pool.py ;; + heal.conflicting-subquorum-authority) driver=authority_runner.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal_authority.py b/test_validation_heal_authority.py new file mode 100644 index 00000000..e10c128b --- /dev/null +++ b/test_validation_heal_authority.py @@ -0,0 +1,464 @@ +import copy +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import types +import unittest +import uuid +from unittest.mock import patch + +from scripts.validation_heal import generation_fault as generation, heal_fault as fault +from scripts.validation_heal import authority, authority_runner as runner, native_regressions as native +from scripts.validation_heal.healthy import BASE_OPTIONS, Fleet +from scripts.validation_heal.matrix import AUTOMATIC_CASES, select +import test_validation_heal as oracle + + +class AuthorityCensusTests(unittest.TestCase): + bucket = 'rv-heal-0123456789abcdef' + + def scene(self, total=4, branch='disjoint-version-quorums'): + first, _ = oracle.HealReadQuorumBoundaryTest().scene(total=total, data=total - 1) + for value in first.values(): + for entry in value['files']: + entry['path'] = entry['path'].replace('boundary/exact-r-read.bin', 'conflict.bin') + if 'metadata' in entry: + entry['metadata']['key'] = 'conflict.bin' + version = entry['metadata']['versions'][0] + version['versionId'] = str(uuid.UUID(int=11)) if branch == authority.BRANCHES[0] else None + second = copy.deepcopy(first) + for value in second.values(): + for entry in value['files']: + entry['sha256'] += '-generation-b' + if 'metadata' in entry: + version = entry['metadata']['versions'][0] + old_dir = version['dataDir'] + version['dataDir'] = str(uuid.UUID(int=2)) + version['versionId'] = str(uuid.UUID(int=12)) if branch == authority.BRANCHES[0] else None + else: + entry['path'] = entry['path'].replace(old_dir, str(uuid.UUID(int=2))) + mixed = copy.deepcopy(second) + members = sorted((node, row['disk']) for node, value in first.items() for row in value['formats']) + for node, disk in members[:total // 2]: + mixed[node]['files'] = [row for row in mixed[node]['files'] if row['disk'] != disk] + mixed[node]['files'].extend(copy.deepcopy([row for row in first[node]['files'] if row['disk'] == disk])) + return first, second, mixed + + def check(self, first, second, mixed, total=4, branch='disjoint-version-quorums'): + return authority.validate_split(mixed, first, second, self.bucket, 'conflict.bin', 'H1' if total == 4 else 'H2', branch) + + def test_both_original_topologies_and_real_identity_branches_have_no_authority_quorum(self): + for total in (4, 16): + for branch in authority.BRANCHES[:2]: + with self.subTest(total=total, branch=branch): + proof = self.check(*self.scene(total, branch), total, branch) + self.assertEqual(proof['R'], total - 1) + self.assertEqual([row['members'] for row in proof['groups']], [total // 2] * 2) + self.assertFalse(proof['crossVersionUnionClaimedAsAuthority']) + self.assertFalse(proof['bucketIncarnationSplitClaimed']) + + def test_hidden_live_history_in_other_group_cannot_be_ignored_by_latest_only_census(self): + first, second, mixed = self.scene() + for disk in (2, 3): + row = next(row for row in mixed['node1']['files'] if row['disk'] == disk and 'metadata' in row) + old = copy.deepcopy(next(row for row in first['node1']['files'] if row['disk'] == disk and 'metadata' in row)['metadata']['versions'][0]) + old['isLatest'] = False + row['metadata']['versions'].append(old) + with self.assertRaisesRegex(ValueError, 'sub-quorum'): + self.check(first, second, mixed) + + def test_changed_bytes_uuid_slot_role_geometry_and_missing_members_are_not_split_proofs(self): + for kind in ('bytes', 'format', 'slot', 'role', 'geometry', 'missing', 'duplicate', 'same-generation', 'wrong-half'): + first, second, mixed = self.scene() + value = mixed['node1'] + version = next(row['metadata']['versions'][0] for row in value['files'] if row['disk'] == 0 and 'metadata' in row) + if kind == 'bytes': value['files'][0]['sha256'] = 'changed' + elif kind == 'format': value['formats'][0]['value']['xl']['this'] = 'foreign' + elif kind == 'slot': version['erasure']['index'] = 2 + elif kind == 'role': version['role'] = 'parity' + elif kind == 'geometry': version['erasure']['data_blocks'] = 2 + elif kind == 'missing': value['files'] = [row for row in value['files'] if row['disk'] != 0] + elif kind == 'duplicate': value['files'].append(copy.deepcopy(value['files'][0])) + elif kind == 'same-generation': second = copy.deepcopy(first) + else: + value['files'] = [row for row in value['files'] if row['disk'] != 0] + value['files'].extend(copy.deepcopy([row for row in second['node1']['files'] if row['disk'] == 0])) + with self.subTest(kind=kind), self.assertRaises(ValueError): self.check(first, second, mixed) + + def test_generation_sources_must_be_independent_and_branch_identity_exact(self): + first, second, mixed = self.scene() + frozen = copy.deepcopy((first, second)) + self.check(first, second, mixed) + self.assertEqual((first, second), frozen) + for branch in ('native-authority-fence', 'disjoint-incarnations'): + with self.subTest(branch=branch), self.assertRaises(ValueError): self.check(first, second, mixed, branch=branch) + with self.assertRaises(ValueError): self.check(second, second, mixed) + + +def negative_status(aborted=False): + objects = [] if aborted else [{'identity': {'bucket': 'owned', 'object': 'conflict.bin', 'kind': 'object', + 'versionId': 'null', 'bucketIncarnationId': 'incarnation', 'poolIndex': 0, 'setIndex': 0}, 'disposition': {'state': 'failed'}}] + return {'summary': 'finished', 'truncated': False, 'settings': BASE_OPTIONS, + 'outcome': {'execution': {'state': 'aborted', 'reason': 'untraversable'} if aborted else {'state': 'completed_with_errors'}, + 'coverage': 'partial' if aborted else 'complete', 'objectsTruncated': False, 'objects': objects, + 'counters': {'processed': len(objects), 'healed': 0, 'unchanged': 0, 'skipped': 0, 'failed': len(objects), + 'unknown': 0, 'attemptFailures': len(objects), 'overflowed': False}}} + + +class AuthorityResponsibilityTests(unittest.TestCase): + def test_explicit_failed_object_and_typed_untraversable_abort_preserve_negative_responsibility(self): + for aborted in (False, True): + value = negative_status(aborted) + proof = authority.validate_negative_task(value, 'owned', BASE_OPTIONS) + self.assertTrue(proof['safeExplicitRefusal']) + self.assertFalse(proof['runtimeCommitPauseClaimed']) + + def test_false_health_missing_identity_duplicate_counters_and_false_complete_abort_fail(self): + for kind in ('healthy', 'healed', 'foreign', 'pool', 'incarnation', 'duplicate', 'counters', 'empty', 'options', 'overflow', 'abort-coverage'): + value = negative_status(kind == 'abort-coverage') + outcome = value['outcome'] + if kind == 'healthy': outcome['objects'][0]['disposition']['state'] = 'verified_healthy' + elif kind == 'healed': outcome['counters']['healed'] = 1 + elif kind == 'foreign': outcome['objects'][0]['identity']['object'] = 'other.bin' + elif kind == 'pool': outcome['objects'][0]['identity']['poolIndex'] = True + elif kind == 'incarnation': outcome['objects'][0]['identity']['bucketIncarnationId'] = '' + elif kind == 'duplicate': outcome['objects'] *= 2; outcome['counters'].update(processed=2, failed=2, attemptFailures=2) + elif kind == 'counters': outcome['counters']['failed'] = 0 + elif kind == 'empty': outcome['objects'] = []; outcome['counters'].update(processed=0, failed=0, attemptFailures=0) + elif kind == 'options': value['settings'] = {**BASE_OPTIONS, 'remove': True} + elif kind == 'overflow': outcome['counters']['overflowed'] = True + else: outcome['coverage'] = 'complete' + with self.subTest(kind=kind), self.assertRaises(ValueError): authority.validate_negative_task(value, 'owned', BASE_OPTIONS) + + def test_peer_propagation_can_converge_for_partial_abort_without_claiming_full_coverage(self): + value = negative_status(True) + fleet = types.SimpleNamespace(record=lambda label, value: value) + probe = types.SimpleNamespace(binding={'endpoints': {'node1': 'url'}}, admin=unittest.mock.Mock(side_effect=[ + {'httpStatus': 200, 'response': {'clientToken': 'token'}}, {'httpStatus': 200, 'response': value}])) + active = [] + with patch.object(runner, 'Probe') as remote, patch.object(runner.time, 'sleep'): + remote.return_value.admin.side_effect = [{'httpStatus': 200, 'response': {'summary': 'running'}}, + {'httpStatus': 200, 'response': copy.deepcopy(value)}] + proof = runner.negative_heal(probe, fleet, 'owned', BASE_OPTIONS, active, + {'unrecoverableGet': {'httpStatus': 503, 'code': 'SlowDownRead'}}) + self.assertEqual(proof['coverage'], 'partial') + self.assertFalse(active) + + def test_missing_peer_status_and_wrong_public_error_cannot_be_success(self): + fleet = types.SimpleNamespace(record=lambda label, value: value) + contract = {'unrecoverableGet': {'httpStatus': 503, 'code': 'SlowDownRead'}} + probe = types.SimpleNamespace(binding={'endpoints': {'node1': 'url'}}, admin=unittest.mock.Mock(side_effect=[ + {'httpStatus': 200, 'response': {'clientToken': 'token'}}, {'httpStatus': 200, 'response': negative_status()}])) + active = [] + with patch.object(runner, 'Probe') as remote, self.assertRaisesRegex(ValueError, '404/unknown'): + remote.return_value.admin.return_value = {'httpStatus': 404, 'response': {}} + runner.negative_heal(probe, fleet, 'owned', BASE_OPTIONS, active, contract) + self.assertEqual(len(active), 1) + probe.admin = lambda *args: {'httpStatus': 503, 'response': {'Code': 'SlowDown'}} + with self.assertRaises(ValueError): runner.negative_heal(probe, fleet, 'owned', BASE_OPTIONS, [], contract) + + +class AuthorityNativeEvidenceTests(unittest.TestCase): + def fixture(self, directory): + tools, evidence = directory / 'native-tools', directory / 'native-evidence' + tools.mkdir(); evidence.mkdir() + binary = tools / 'ecstore-native-tests' + binary.write_bytes(b'fixed-test-executable') + manifest = {'schemaVersion': 1, 'candidateCommit': native.CANDIDATE, 'candidateSourceTree': authority.SOURCE_TREE, + 'candidateCargoLockSha256': authority.CARGO_LOCK, 'sourceFiles': {native.SOURCE: native.SOURCE_SHA}, + 'sourceUnmodified': True, 'optimized': True, 'collectorSha256': native.sha(Path(native.__file__)), + 'regressions': native.REGRESSIONS, 'fleetPassClaimed': False, 'binaryName': binary.name, + 'binarySha256': native.sha(binary), 'runId': '123', 'runAttempt': '1'} + (tools / 'native-regressions.json').write_text(json.dumps(manifest)) + run = {'id': 123, 'run_attempt': 1, 'status': 'completed', 'conclusion': 'success', + 'path': '.github/workflows/rustfs-functional-chain.yml'} + (directory / 'tool-run.json').write_text(json.dumps(run)) + listing = ''.join(spec['test'] + ': test\n' for spec in native.REGRESSIONS.values()) + (tools / 'test-inventory.log').write_text(listing) + (evidence / 'runtime-inventory.log').write_text(listing) + results = [] + for key, spec in native.REGRESSIONS.items(): + row = {'regression': key, 'test': spec['test'], 'verdict': 'PASS', 'candidateCommit': native.CANDIDATE, + 'binarySha256': manifest['binarySha256'], 'fleetPassClaimed': False, 'emittedBarrierLogsClaimed': False, + 'command': ['native-tools/ecstore-native-tests', spec['test'], '--exact', '--nocapture', '--test-threads=1'], + 'sourceAssertions': spec['sourceAssertions'], 'fenceKind': spec['fenceKind'], 'exitCode': 0} + (evidence / (key + '.json')).write_text(json.dumps(row)) + (evidence / (key + '.log')).write_text(f"running 1 test\ntest {spec['test']} ... ok\ntest result: ok. 1 passed; 0 failed; 0 ignored;\n") + results.append(row) + summary = {'complete': True, 'fleetPassClaimed': False, 'canonicalCasePassClaimed': False, + 'sourceUnmodified': True, 'results': results} + (evidence / 'native-summary.json').write_text(json.dumps(summary)) + return manifest, summary, run + + def verify(self, directory): + return authority.verify_native(directory, {'source_sha': native.CANDIDATE}) + + def test_all_six_original_tests_are_checked_and_only_two_authority_guards_are_mapped(self): + with tempfile.TemporaryDirectory() as folder: + directory = Path(folder) + self.fixture(directory) + proof = self.verify(directory) + self.assertEqual({row['regression'] for row in proof['results']}, set(authority.NATIVE_KEYS)) + self.assertEqual(proof['toolsBuildRun'], 123) + self.assertFalse(proof['runtimeCommitPauseClaimed']) + self.assertFalse(proof['fleetPassClaimed']) + + def test_source_artifact_attempt_zero_test_forged_complete_and_non_c07_row_failures_reject(self): + for kind in ('source', 'tree', 'lock', 'collector', 'binary', 'run', 'attempt', 'summary', 'duplicate', 'row', 'filtered', 'listing', 'path'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as folder: + directory = Path(folder) + manifest, summary, run = self.fixture(directory) + evidence = directory / 'native-evidence' + key = next(iter(native.REGRESSIONS)) + if kind == 'source': manifest['sourceUnmodified'] = False + elif kind == 'tree': manifest['candidateSourceTree'] = '0' * 40 + elif kind == 'lock': manifest['candidateCargoLockSha256'] = '0' * 64 + elif kind == 'collector': manifest['collectorSha256'] = '0' * 64 + elif kind == 'binary': (directory / 'native-tools/ecstore-native-tests').write_bytes(b'changed') + elif kind == 'run': run['conclusion'] = 'failure' + elif kind == 'attempt': run['run_attempt'] = 2 + elif kind == 'summary': summary['results'][0]['verdict'] = 'FAIL' + elif kind == 'duplicate': summary['results'][-1] = summary['results'][0] + elif kind == 'row': (evidence / (key + '.json')).write_text('{}') + elif kind == 'filtered': (evidence / (key + '.log')).write_text('running 0 tests\ntest result: ok. 0 passed; 0 failed; 0 ignored;\n') + elif kind == 'listing': (evidence / 'runtime-inventory.log').write_text('0 tests\n') + else: + summary['results'][0]['command'][0] = '/tmp/download/ecstore-native-tests' + (evidence / (key + '.json')).write_text(json.dumps(summary['results'][0])) + (directory / 'native-tools/native-regressions.json').write_text(json.dumps(manifest)) + (directory / 'tool-run.json').write_text(json.dumps(run)) + (evidence / 'native-summary.json').write_text(json.dumps(summary)) + with self.assertRaises(ValueError): self.verify(directory) + + +class AuthorityRecoveryTests(unittest.TestCase): + def test_failed_generation_recovery_retains_namespace_and_does_not_run_cleanup(self): + with tempfile.TemporaryDirectory() as folder: + fleet = Fleet(['vm000'], Path(folder), 'gh-123-1-h1', runtime_profile='authority-split') + fleet.reserved = ['node1'] + fleet.generation_faults.append(('node1', {'disk': 0, 'bucket': 'owned', 'key': 'conflict.bin'})) + with patch.object(fleet, 'stop'), patch.object(fleet, 'remote', side_effect=ValueError('partial archive')) as remote: + self.assertFalse(fleet.cleanup()) + self.assertTrue(fleet.generation_faults) + self.assertFalse(any(call.args[1] == 'cleanup' for call in remote.call_args_list)) + + def test_wrong_put_ack_keeps_durable_request_pending_and_never_deletes_namespace(self): + with tempfile.TemporaryDirectory() as folder: + directory = Path(folder) + candidate = directory / 'candidate.json' + candidate.write_text(json.dumps({'source_sha': native.CANDIDATE, 'package_sha256': 'a' * 64})) + binary = directory / 'rc'; binary.write_bytes(b'rc') + args = types.SimpleNamespace(evidence=directory / 'evidence', candidate=candidate, rc_bin=binary, + tool_directory=directory, testing_commit='b' * 40, binary_sha='c' * 64) + with patch.dict(os.environ, {'GITHUB_RUN_ID': '123', 'GITHUB_RUN_ATTEMPT': '1'}), \ + patch.object(runner, 'verify_inspector', return_value={}), patch.object(runner, 'verify_quorum_contract', return_value={}), \ + patch.object(runner, 'Fleet') as fleet, patch.object(runner, 'Recorder') as recorder: + fleet.return_value.record.side_effect = lambda label, value: value + recorder.return_value.call.side_effect = [{}, {'VersionId': 'foreign-version'}] + result = runner.run_branch(args, 'H1', ['vm000'], 'disjoint-incarnations') + self.assertEqual(result['verdict'], 'FAIL') + self.assertEqual(result['cleanup'], 'FAIL') + fleet.return_value.cleanup.assert_not_called() + fleet.return_value.stop.assert_called_once() + self.assertEqual(recorder.return_value.call.call_count, 2) + + def test_native_proof_uses_no_fleet_and_cannot_invent_h2_native_members(self): + with tempfile.TemporaryDirectory() as folder: + directory = Path(folder) + AuthorityNativeEvidenceTests().fixture(directory) + candidate = directory / 'candidate.json'; candidate.write_text(json.dumps({'source_sha': native.CANDIDATE})) + args = types.SimpleNamespace(evidence=directory / 'evidence', candidate=candidate, native_directory=directory) + with patch.object(runner, 'Fleet') as fleet: + result = runner.run_branch(args, 'H2', ['vm000', 'vm001', 'vm002', 'vm003'], 'native-authority-fence') + fleet.assert_not_called() + self.assertEqual(result['actualNativeUnitMembers'], 4) + self.assertFalse(result['fleetDeploymentPerformed']) + self.assertEqual(result['verdict'], 'PASS') + + def test_registered_source_branches_and_both_dispatch_paths_remain_complete(self): + self.assertEqual(select(authority.CASE_ID)[0]['variants'], list(authority.BRANCHES)) + self.assertIn(authority.CASE_ID, AUTOMATIC_CASES) + self.assertIn(authority.CASE_ID + ') driver=authority_runner.py', Path('scripts/validation_heal/run_case.sh').read_text()) + self.assertIn(authority.CASE_ID, Path('scripts/run_validation_case.sh').read_text()) + + +class GenerationFaultTests(unittest.TestCase): + bucket = 'rv-heal-0123456789abcdef' + data_dir = str(uuid.UUID(int=1)) + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name).resolve() + self.object = self.root / 'data' / self.bucket / 'conflict.bin' + self.object.mkdir(parents=True) + (self.root / 'quarantine').mkdir() + self.owner = {'runId': 'gh-123-1-h1', 'bucket': self.bucket, 'runtimeProfile': 'authority-split'} + self.request = {'disk': 0, 'bucket': self.bucket, 'key': 'conflict.bin'} + self.mount = patch.object(fault, 'mount_id', return_value=7) + self.mount.start() + self.seed('a') + + def tearDown(self): + self.mount.stop() + self.temporary.cleanup() + + def seed(self, label): + (self.object / 'xl.meta').write_bytes(('XL2 ' + label).encode()) + (self.object / self.data_dir).mkdir(exist_ok=True) + (self.object / self.data_dir / 'part.1').write_bytes((label * 123).encode()) + + def capture(self, label): + with fault.directory(self.root) as root, fault.parent(root, ['data', self.bucket, 'conflict.bin']) as obj: + manifest = generation.tree(obj, fault)[1] + return generation.capture(root, self.owner, {**self.request, 'generation': label, 'expectedFiles': manifest}, fault) + + def pair(self): + a = self.capture('a') + self.seed('b') + b = self.capture('b') + return a, b + + def install(self, b): + with fault.directory(self.root) as root: + return generation.install(root, self.owner, {**self.request, 'expectedSceneSha256': b['treeSha256']}, fault) + + def recover(self): + actual_stat = os.stat + def root_owned(path, **kwargs): + value = actual_stat(path, **kwargs) + return types.SimpleNamespace(st_mode=value.st_mode, st_uid=0, st_dev=value.st_dev) + with fault.directory(self.root) as root, patch.object(generation.os, 'stat', side_effect=root_owned): + return generation.recover(root, self.owner, self.request, fault) + + def test_complete_old_and_new_trees_are_preserved_then_only_last_ack_is_restored(self): + a, b = self.pair() + receipt = self.install(b) + self.assertEqual(receipt['injectedTreeSha256'], a['treeSha256']) + self.assertEqual((self.object / 'xl.meta').read_bytes(), b'XL2 a') + self.assertEqual((self.root / 'quarantine' / 'C07-pre-split' / 'xl.meta').read_bytes(), b'XL2 b') + result = self.recover() + self.assertEqual(result['state'], 'CLEAR') + self.assertFalse(result['productRepairClaimed']) + self.assertEqual((self.object / 'xl.meta').read_bytes(), b'XL2 b') + self.assertEqual((self.root / 'quarantine' / 'C07-failed-current' / 'xl.meta').read_bytes(), b'XL2 a') + self.assertEqual(self.recover(), result) + + def test_changed_live_bytes_or_archive_authority_refuse_before_first_rename(self): + _, b = self.pair() + self.seed('drift') + with self.assertRaises(ValueError): self.install(b) + self.assertFalse((self.root / 'quarantine' / 'C07-replay-intent.json').exists()) + self.seed('b') + (self.root / 'quarantine' / 'C07-a' / 'object' / 'xl.meta').write_bytes(b'foreign') + with self.assertRaises(ValueError): self.install(b) + self.assertFalse((self.root / 'quarantine' / 'C07-replay-intent.json').exists()) + + def test_hardlinks_symlinks_foreign_leaves_and_unknown_directories_cannot_enter_archive(self): + for kind in ('hardlink', 'symlink', 'leaf', 'directory'): + with self.subTest(kind=kind): + target = self.object / self.data_dir / 'part.1' + original = target.read_bytes() + if kind == 'hardlink': + alias = self.root / 'outside' + os.link(target, alias) + elif kind == 'symlink': + alias = self.root / 'outside' + alias.write_bytes(original) + target.unlink() + target.symlink_to(alias) + elif kind == 'leaf': + alias = self.object / self.data_dir / 'credential.json' + alias.write_bytes(b'foreign') + else: + alias = self.object / 'foreign-directory' + alias.mkdir() + with self.assertRaises((ValueError, OSError)): self.capture('a') + if kind == 'symlink': + target.unlink() + target.write_bytes(original) + if alias.is_dir(): alias.rmdir() + else: alias.unlink() + + def test_capture_rejects_an_incomplete_caller_manifest_and_duplicate_archive(self): + with fault.directory(self.root) as root: + with self.assertRaises(ValueError): generation.capture(root, self.owner, + {**self.request, 'generation': 'a', 'expectedFiles': []}, fault) + self.assertFalse((self.root / 'quarantine' / 'C07-a').exists()) + self.capture('a') + with self.assertRaises(FileExistsError): self.capture('a') + + def test_bucket_key_profile_and_device_drift_cannot_restore_an_unrelated_generation(self): + _, b = self.pair() + self.install(b) + for field, value in (('bucket', 'rv-heal-fedcba9876543210'), ('key', '../conflict.bin'), ('disk', 1)): + with self.subTest(field=field), fault.directory(self.root) as root, self.assertRaises((ValueError, FileNotFoundError)): + generation.recover(root, self.owner, {**self.request, field: value}, fault) + with fault.directory(self.root) as root, self.assertRaises(ValueError): + generation.recover(root, {**self.owner, 'runtimeProfile': 'manual'}, self.request, fault) + with patch.object(fault, 'mount_id', return_value=8), fault.directory(self.root) as root, self.assertRaises(ValueError): + generation.recover(root, self.owner, self.request, fault) + + def test_partial_recovery_remains_dirty_and_does_not_remove_the_failed_scene(self): + _, b = self.pair() + self.install(b) + failed = self.root / 'quarantine' / 'C07-failed-current' + self.object.rename(failed) + self.object.mkdir() + (self.object / 'xl.meta').write_bytes(b'partial') + with self.assertRaises(ValueError): self.recover() + self.assertEqual((failed / 'xl.meta').read_bytes(), b'XL2 a') + self.assertFalse((self.root / 'quarantine' / 'C07-replay-recovered.json').exists()) + + +class GenerationLinuxTests(unittest.TestCase): + @unittest.skipUnless(sys.platform == 'linux', 'requires original Linux mount descriptors and root-owned recovery') + def test_real_descriptors_restore_last_ack_and_reject_nested_bind_mount(self): + program = '''import json,subprocess,tempfile,uuid +from pathlib import Path +from scripts.validation_heal import generation_fault as generation,heal_fault as fault +with tempfile.TemporaryDirectory() as folder: + root=Path(folder).resolve() + bucket='rv-heal-0123456789abcdef' + obj=root/'data'/bucket/'conflict.bin' + obj.mkdir(parents=True) + (root/'quarantine').mkdir() + data_dir=str(uuid.UUID(int=1)) + (obj/data_dir).mkdir() + owner={'runId':'gh-123-1-h1','bucket':bucket,'runtimeProfile':'authority-split'} + request={'disk':0,'bucket':bucket,'key':'conflict.bin'} + captured={} + for label in ('a','b'): + (obj/'xl.meta').write_bytes(('XL2 '+label).encode()) + (obj/data_dir/'part.1').write_bytes(label.encode()*512) + with fault.directory(root) as fd,fault.parent(fd,['data',bucket,'conflict.bin']) as child: + manifest=generation.tree(child,fault)[1] + captured[label]=generation.capture(fd,owner,{**request,'generation':label,'expectedFiles':manifest},fault) + with fault.directory(root) as fd: + generation.install(fd,owner,{**request,'expectedSceneSha256':captured['b']['treeSha256']},fault) + assert (obj/'xl.meta').read_bytes()==b'XL2 a' + restored=generation.recover(fd,owner,request,fault) + assert restored['state']=='CLEAR' and restored['originalLastACKGenerationRestored'] + assert (obj/'xl.meta').read_bytes()==b'XL2 b' + assert generation.recover(fd,owner,request,fault)==restored + outside=root/'outside' + outside.mkdir() + subprocess.run(['mount','--bind',str(outside),str(obj)],check=True) + try: + with fault.directory(root) as fd: + try: generation.capture(fd,owner,{**request,'generation':'a','expectedFiles':[]},fault) + except ValueError as error: assert 'mount' in str(error) + else: raise AssertionError('nested bind mount was accepted') + finally: subprocess.run(['umount',str(obj)],check=True) +print(json.dumps({'realMountDescriptors':True,'lastACKRestored':True,'nestedBindRefused':True})) +''' + result = subprocess.run(['sudo', '-n', 'unshare', '-m', sys.executable, '-B', '-c', program], + capture_output=True, text=True, timeout=60) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertTrue(json.loads(result.stdout)['nestedBindRefused']) + + +if __name__ == '__main__': + unittest.main() From e024ade0ee5ca37129a7103eae3efc6d5bf99174 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 09:32:38 +0800 Subject: [PATCH 108/343] test(validation): verify original hosted native execution paths --- docs/validation-migration-map.md | 19 +++++++++++++++++++ scripts/validation_heal/authority.py | 3 ++- test_validation_heal_authority.py | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 524aadd8..c5fcdd60 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1510,3 +1510,22 @@ descriptors, root-owned generation recovery and nested bind-mount refusal. The 158 IDs/hashes and 84/301/532 matrix, Bash syntax, scoped actionlint and whitespace checks passed. C07 has no canonical live PASS claim; hosted native run `38100407691` is still building. + +Hosted build `38100407691` has now completed successfully. Independent local +artifact verification confirmed all six exact tests against unchanged source +tree `8071fb0a9e5353eb7d3fa58c8a58c8622473f2bd`, candidate Cargo.lock +`d8f7bc416ba67d19873d34e9389e5aa823b22d7d7552706a5a810cee26f6eda2`, +collector `6f4901573967c53b88239af361c1ab543f44fd4d` / SHA-256 +`ca6d2087416761ba3b5398366391b9f5d8108b59272f5759ea4b67752d9caf45`, +and executable `0c9e005ae877f741a62100f0d70af6c2256d1078d546ac97e963d6979aa05bc1`. +The original collector resolves its executable path before recording commands; +C07 verifies that exact hosted path, without replacing it with a downloaded +local path. Both authority guards passed, independently of any fleet result. +No new canonical PASS is claimed. + +All six auto-testing checks passed at `07627ade0667e67e5560fccbc08215458d506b04`. +The Linux contract job ran all 375 validation tests without skips, including +real original mount-descriptor recovery and nested bind-mount refusal. The +C07 pending serial plan uses this successful native tools attempt; the active +queue resumed F06 in run `38102031029` after the hosted build completed. Shared +fleet locks, cleanup guards and the active plan are retained. diff --git a/scripts/validation_heal/authority.py b/scripts/validation_heal/authority.py index 49329e9e..4b46ab5a 100644 --- a/scripts/validation_heal/authority.py +++ b/scripts/validation_heal/authority.py @@ -25,6 +25,7 @@ NATIVE_KEYS = ('conflicting-authority-guard', 'reconstructable-authority-preservation') SOURCE_TREE = '8071fb0a9e5353eb7d3fa58c8a58c8622473f2bd' CARGO_LOCK = 'd8f7bc416ba67d19873d34e9389e5aa823b22d7d7552706a5a810cee26f6eda2' +HOSTED_BINARY = '/home/runner/work/rustfs/rustfs/native-tools/ecstore-native-tests' def object_files(snapshot, bucket, key): @@ -199,7 +200,7 @@ def verify_native(directory, candidate): and row.get('test') == spec['test'] and row.get('candidateCommit') == native.CANDIDATE and row.get('binarySha256') == manifest['binarySha256'] and row.get('emittedBarrierLogsClaimed') is False and row.get('fleetPassClaimed') is False - and row.get('command') == ['native-tools/ecstore-native-tests', spec['test'], '--exact', '--nocapture', '--test-threads=1'] + and row.get('command') == [HOSTED_BINARY, spec['test'], '--exact', '--nocapture', '--test-threads=1'] and row.get('sourceAssertions') == spec['sourceAssertions'] and row.get('fenceKind') == spec['fenceKind'], 'C07 native result does not bind one exact original test execution') native.exact_listing(listing, spec['test']) diff --git a/test_validation_heal_authority.py b/test_validation_heal_authority.py index e10c128b..fb2705df 100644 --- a/test_validation_heal_authority.py +++ b/test_validation_heal_authority.py @@ -183,7 +183,7 @@ def fixture(self, directory): for key, spec in native.REGRESSIONS.items(): row = {'regression': key, 'test': spec['test'], 'verdict': 'PASS', 'candidateCommit': native.CANDIDATE, 'binarySha256': manifest['binarySha256'], 'fleetPassClaimed': False, 'emittedBarrierLogsClaimed': False, - 'command': ['native-tools/ecstore-native-tests', spec['test'], '--exact', '--nocapture', '--test-threads=1'], + 'command': [authority.HOSTED_BINARY, spec['test'], '--exact', '--nocapture', '--test-threads=1'], 'sourceAssertions': spec['sourceAssertions'], 'fenceKind': spec['fenceKind'], 'exitCode': 0} (evidence / (key + '.json')).write_text(json.dumps(row)) (evidence / (key + '.log')).write_text(f"running 1 test\ntest {spec['test']} ... ok\ntest result: ok. 1 passed; 0 failed; 0 ignored;\n") From ad464b760df89aa6f942079f2a1185608d912695 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 09:34:32 +0800 Subject: [PATCH 109/343] test(validation): preserve original shard integrity commitments --- docs/validation-migration-map.md | 10 +++++++ scripts/validation_heal/generation_fault.py | 15 +++++++++-- test_validation_heal_authority.py | 30 +++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index c5fcdd60..af5de7b0 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1529,3 +1529,13 @@ real original mount-descriptor recovery and nested bind-mount refusal. The C07 pending serial plan uses this successful native tools attempt; the active queue resumed F06 in run `38102031029` after the hosted build completed. Shared fleet locks, cleanup guards and the active plan are retained. + +C07's complete generation archive includes the fixed candidate's actual +`part.N.integrity.` commitments, independently observed in the retained +A06 physical evidence. Only canonical non-nil commitment UUIDs, one commitment +per existing part and the bounded part/commitment budget are accepted. Every +commitment's exact bytes and name follow its chosen archived generation; +payload-less, repeated or ambiguous commitments are rejected. Two additional +regressions verify complete A/B commitment preservation and invalid-leaf +refusal. The updated local suite runs 377 tests with seven macOS skips; the +real Linux generation test also includes a commitment file. diff --git a/scripts/validation_heal/generation_fault.py b/scripts/validation_heal/generation_fault.py index b60a2319..46f90d2f 100644 --- a/scripts/validation_heal/generation_fault.py +++ b/scripts/validation_heal/generation_fault.py @@ -37,10 +37,21 @@ def tree(fd, fault): require(str(uuid.UUID(name)) == name and uuid.UUID(name).int != 0, 'generation contains an unreviewed object child') with fault.parent(fd, [name]) as part_fd: parts = sorted(os.listdir(part_fd)) - require(parts and len(parts) <= 16 and all(re.fullmatch(r'part\.[1-9][0-9]*', part) for part in parts), - 'generation contains unreviewed multipart leaves') + require(parts and len(parts) <= 32, 'generation exceeds its part/commitment file budget') + payloads, commitments = set(), set() for part in parts: + match = re.fullmatch(r'part\.([1-9][0-9]*)(?:\.integrity\.([0-9a-f-]{36}))?', part) + require(match is not None, 'generation contains unreviewed multipart leaves') + number, nonce = match.groups() + if nonce is None: + payloads.add(number) + else: + require(str(uuid.UUID(nonce)) == nonce and uuid.UUID(nonce).int != 0 and number not in commitments, + 'generation has ambiguous or noncanonical shard commitment identity') + commitments.add(number) values[name + '/' + part] = fault.read_file(part_fd, part) + require(0 < len(payloads) <= 16 and commitments <= payloads, + 'generation commitment has no exact corresponding original part') require(sum(map(len, values.values())) <= 32 * 1024**2, 'generation exceeds its per-drive byte budget') manifest = [{'path': name, 'size': len(raw), 'sha256': fault.digest(raw)} for name, raw in sorted(values.items())] return values, manifest diff --git a/test_validation_heal_authority.py b/test_validation_heal_authority.py index fb2705df..56f07d60 100644 --- a/test_validation_heal_authority.py +++ b/test_validation_heal_authority.py @@ -390,6 +390,35 @@ def test_capture_rejects_an_incomplete_caller_manifest_and_duplicate_archive(sel self.capture('a') with self.assertRaises(FileExistsError): self.capture('a') + def test_full_protected_generation_preserves_exact_candidate_integrity_sidecar(self): + name = 'part.1.integrity.' + str(uuid.UUID(int=77)) + sidecar = self.object / self.data_dir / name + sidecar.write_bytes(b'candidate-shard-integrity-a') + a = self.capture('a') + self.seed('b'); sidecar.unlink() + newest_sidecar = self.object / self.data_dir / ('part.1.integrity.' + str(uuid.UUID(int=78))) + newest_sidecar.write_bytes(b'candidate-shard-integrity-b') + b = self.capture('b') + self.assertTrue(any(row['path'].endswith(name) for row in a['files'])) + self.install(b) + self.assertEqual(sidecar.read_bytes(), b'candidate-shard-integrity-a') + self.assertFalse(newest_sidecar.exists()) + self.recover() + self.assertEqual(newest_sidecar.read_bytes(), b'candidate-shard-integrity-b') + self.assertFalse(sidecar.exists()) + + def test_commitment_without_payload_duplicate_nonce_and_nil_nonce_are_rejected(self): + for name in ('part.2.integrity.' + str(uuid.UUID(int=77)), 'part.1.integrity.' + str(uuid.UUID(int=0)), + 'part.1.integrity.' + str(uuid.UUID(int=78))): + with self.subTest(name=name): + target = self.object / self.data_dir / name + target.write_bytes(b'commitment') + first = self.object / self.data_dir / ('part.1.integrity.' + str(uuid.UUID(int=77))) + if name.endswith(str(uuid.UUID(int=78))): first.write_bytes(b'original-commitment') + with self.assertRaises(ValueError): self.capture('a') + target.unlink() + if first.exists(): first.unlink() + def test_bucket_key_profile_and_device_drift_cannot_restore_an_unrelated_generation(self): _, b = self.pair() self.install(b) @@ -433,6 +462,7 @@ def test_real_descriptors_restore_last_ack_and_reject_nested_bind_mount(self): for label in ('a','b'): (obj/'xl.meta').write_bytes(('XL2 '+label).encode()) (obj/data_dir/'part.1').write_bytes(label.encode()*512) + (obj/data_dir/('part.1.integrity.'+str(uuid.UUID(int=77)))).write_bytes(label.encode()*32) with fault.directory(root) as fd,fault.parent(fd,['data',bucket,'conflict.bin']) as child: manifest=generation.tree(child,fault)[1] captured[label]=generation.capture(fd,owner,{**request,'generation':label,'expectedFiles':manifest},fault) From 0766f029abd0e62858f64f04855d502d756dd4f1 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 09:44:08 +0800 Subject: [PATCH 110/343] test(validation): add candidate bound shared reference fixture codec --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 15 + .../prepare_heal_shared_fixture.py | 37 +++ scripts/validation_heal/shared_fixture.py | 75 +++++ .../tools/heal_shared_version_fixture.rs | 275 ++++++++++++++++++ test_validation_heal_orphan.py | 72 +++++ 6 files changed, 477 insertions(+), 1 deletion(-) create mode 100644 scripts/validation_heal/prepare_heal_shared_fixture.py create mode 100644 scripts/validation_heal/shared_fixture.py create mode 100644 scripts/validation_heal/tools/heal_shared_version_fixture.rs create mode 100644 test_validation_heal_orphan.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 337b9b4f..7b334a01 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -34,6 +34,7 @@ on: - test_validation_heal_schema.py - test_validation_heal_native.py - test_validation_heal_authority.py + - test_validation_heal_orphan.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -81,6 +82,7 @@ on: - test_validation_heal_schema.py - test_validation_heal_native.py - test_validation_heal_authority.py + - test_validation_heal_orphan.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -118,7 +120,7 @@ jobs: sudo apt-get update sudo apt-get install --no-install-recommends -y nftables fi - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_authority.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index af5de7b0..9b752b93 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1539,3 +1539,18 @@ payload-less, repeated or ambiguous commitments are rejected. Two additional regressions verify complete A/B commitment preservation and invalid-leaf refusal. The updated local suite runs 377 tests with seven macOS skips; the real Linux generation test also includes a commitment file. + +C10's shared-version branch requires the candidate's real metadata codec, +two independent same-payload ACK versions and physically identical original +parts. Its encoder is built as an optional immutable helper alongside the +read-only inspector, without modifying the candidate source. It reads bounded +original metadata from stdin and emits a separately encoded result on stdout; +it has no filesystem mutation API and makes no physical byte-equality claim. +Only the destination data-directory reference and its source commitment +descriptor may change; all version identities and unrelated metadata are +reloaded and compared before output. Three actual codec tests must pass against +the candidate before the artifact can be consumed. Python guards additionally +bind source/tree/Cargo.lock/helper/binary/role and the successful build attempt, +and reject incomplete, ignored or zero-test output. This is C10 prerequisite +tooling; the full five-branch executor is still in progress and the count +remains 131 automatic / 27 remaining Heal executors. diff --git a/scripts/validation_heal/prepare_heal_shared_fixture.py b/scripts/validation_heal/prepare_heal_shared_fixture.py new file mode 100644 index 00000000..c1f6d524 --- /dev/null +++ b/scripts/validation_heal/prepare_heal_shared_fixture.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Prepare the bounded C10 metadata encoder against one unmodified candidate.""" + +import argparse +import hashlib +import json +from pathlib import Path + +try: + from .prepare_heal_meta_inspector import prepare +except ImportError: + from prepare_heal_meta_inspector import prepare + + +ROLE = 'controlled C10 shared-reference metadata encoder; not service under test' + + +def prepare_shared(source, commit, output): + prepare(source, commit, output) + helper = Path(__file__).with_name('tools') / 'heal_shared_version_fixture.rs' + (output / 'src/main.rs').write_bytes(helper.read_bytes()) + manifest = output / 'Cargo.toml' + manifest.write_text(manifest.read_text().replace('name = "heal-meta-inspect"', 'name = "heal-shared-fixture"')) + path = output / 'source.json' + proof = json.loads(path.read_text()) + proof.update(role=ROLE, helperSourceSha256=hashlib.sha256(helper.read_bytes()).hexdigest(), + filesystemMutationPerformed=False, physicalByteEqualityClaimed=False) + path.write_text(json.dumps(proof, indent=2) + '\n') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source', type=Path, required=True) + parser.add_argument('--commit', required=True) + parser.add_argument('--out', type=Path, required=True) + args = parser.parse_args() + prepare_shared(args.source, args.commit, args.out) diff --git a/scripts/validation_heal/shared_fixture.py b/scripts/validation_heal/shared_fixture.py new file mode 100644 index 00000000..92cdc5b8 --- /dev/null +++ b/scripts/validation_heal/shared_fixture.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Verify a separately built C10 encoder; ordinary execution needs no checkout.""" + +import argparse +import hashlib +import json +from pathlib import Path +import re +import stat + +try: + from .authority import SOURCE_TREE, CARGO_LOCK + from .heal_probe import require + from .native_regressions import CANDIDATE + from .prepare_heal_shared_fixture import ROLE +except ImportError: + from authority import SOURCE_TREE, CARGO_LOCK + from heal_probe import require + from native_regressions import CANDIDATE + from prepare_heal_shared_fixture import ROLE + + +TESTS = ('tests::exact_existing_versions_share_one_directory_without_changing_other_metadata', + 'tests::malformed_absent_same_and_already_shared_versions_are_refused', + 'tests::incompatible_part_and_deleted_destination_are_refused') + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def verify(directory, candidate): + directory = Path(directory) + paths = {name: directory / name for name in ('heal-shared-fixture', 'heal-shared-fixture.json', + 'shared-Cargo.lock', 'shared-fixture-tests.log', 'tool-run.json')} + for path in paths.values(): + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and 0 < info.st_size <= 256 * 1024**2, + 'C10 fixture artifact must contain bounded original regular files') + manifest = json.loads(paths['heal-shared-fixture.json'].read_text()) + run = json.loads(paths['tool-run.json'].read_text()) + require(type(run.get('id')) is int and run['id'] > 0 and type(run.get('run_attempt')) is int and run['run_attempt'] > 0 + and run.get('status') == 'completed' and run.get('conclusion') == 'success' + and run.get('path') in {'.github/workflows/rustfs-functional-chain.yml', '.github/workflows/rustfs-validation-tools.yml'} + and manifest.get('runId') == str(run['id']) and manifest.get('runAttempt') == str(run['run_attempt']), + 'C10 encoder is not bound to one pinned successful source build attempt') + helper = Path(__file__).with_name('tools') / 'heal_shared_version_fixture.rs' + require(candidate['source_sha'] == CANDIDATE and manifest.get('schemaVersion') == 1 + and manifest.get('candidateCommit') == CANDIDATE and manifest.get('candidateSourceTree') == SOURCE_TREE + and manifest.get('candidateLockSha256') == CARGO_LOCK and manifest.get('built') is True + and manifest.get('sourceUnmodified') is True and manifest.get('target') == 'x86_64-unknown-linux-gnu' + and manifest.get('role') == ROLE and manifest.get('filesystemMutationPerformed') is False + and manifest.get('physicalByteEqualityClaimed') is False + and manifest.get('helperSourceSha256') == sha(helper) + and manifest.get('binarySha256') == sha(paths['heal-shared-fixture']) + and manifest.get('sharedLockSha256') == sha(paths['shared-Cargo.lock']), + 'C10 encoder changed its source/codec/build/artifact or claimed unobserved physical equality') + with paths['heal-shared-fixture'].open('rb') as stream: + header = stream.read(20) + require(header[:6] == b'\x7fELF\x02\x01' and header[18:20] == b'\x3e\x00', 'C10 encoder is not Linux x86_64 ELF') + log = paths['shared-fixture-tests.log'].read_text() + require(re.search(r'(?m)^running 3 tests\s*$', log) + and re.search(r'test result: ok\. 3 passed; 0 failed; 0 ignored;', log) + and all(len(re.findall(r'(?m)^test ' + re.escape(name) + r' \.\.\. ok\s*$', log)) == 1 for name in TESTS), + 'C10 codec build did not execute all three original transformation/refusal tests') + return {**manifest, 'toolsBuildRun': run['id'], 'toolsBuildAttempt': run['run_attempt'], + 'exactCodecTestsPassed': True, 'fleetPassClaimed': False} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--directory', type=Path, required=True) + parser.add_argument('--candidate', type=Path, required=True) + args = parser.parse_args() + print(json.dumps(verify(args.directory, json.loads(args.candidate.read_text())), indent=2)) diff --git a/scripts/validation_heal/tools/heal_shared_version_fixture.rs b/scripts/validation_heal/tools/heal_shared_version_fixture.rs new file mode 100644 index 00000000..6bc7ae59 --- /dev/null +++ b/scripts/validation_heal/tools/heal_shared_version_fixture.rs @@ -0,0 +1,275 @@ +// Encode a controlled C10 shared-reference fixture with the exact candidate +// codec. The node verifies both original physical generations before calling +// this stdin/stdout encoder; this program has no filesystem mutation API. +use rustfs_filemeta::FileMeta; +use std::io::{Read, Write}; + +fn main() -> Result<(), Box> { + let args: Vec = std::env::args().collect(); + if args.len() != 5 { + return Err( + "usage: shared-fixture BUCKET KEY SOURCE_VERSION DEST_VERSION < XL_META".into(), + ); + } + let mut original = Vec::new(); + std::io::stdin() + .take(64 * 1024 * 1024 + 1) + .read_to_end(&mut original)?; + let (encoded, receipt) = encode(&original, &args[1], &args[2], &args[3], &args[4])?; + eprintln!("{}", receipt); + std::io::stdout().write_all(&encoded)?; + Ok(()) +} + +fn encode( + original: &[u8], + bucket: &str, + key: &str, + source_version: &str, + destination_version: &str, +) -> Result<(Vec, serde_json::Value), Box> { + if original.is_empty() || original.len() > 64 * 1024 * 1024 { + return Err("requires bounded original metadata".into()); + } + let mut meta = FileMeta::load(original)?; + meta.data.validate()?; + let decoded = meta.into_file_info_versions(bucket, key, true)?; + let lookup = |id: &str| { + decoded + .versions + .iter() + .filter(|version| { + version + .version_id + .is_some_and(|value| value.to_string() == id) + }) + .collect::>() + }; + let sources = lookup(source_version); + let destinations = lookup(destination_version); + if sources.len() != 1 || destinations.len() != 1 { + return Err("requires exactly one existing version for each ACK identity".into()); + } + let src = sources[0]; + let dst = destinations[0]; + if src.version_id == dst.version_id + || src.deleted + || dst.deleted + || src.is_remote() + || dst.is_remote() + || src.inline_data() + || dst.inline_data() + || src.size <= 0 + || src.size != dst.size + || src.parts.is_empty() + || src.parts.len() != dst.parts.len() + || src.erasure.algorithm != dst.erasure.algorithm + || src.erasure.data_blocks != dst.erasure.data_blocks + || src.erasure.parity_blocks != dst.erasure.parity_blocks + || src.erasure.block_size != dst.erasure.block_size + || src.erasure.index != dst.erasure.index + || src.erasure.distribution != dst.erasure.distribution + || src.erasure.checksums != dst.erasure.checksums + || src.parts.iter().zip(&dst.parts).any(|(left, right)| { + left.etag != right.etag + || left.number != right.number + || left.size != right.size + || left.actual_size != right.actual_size + || left.index != right.index + || left.checksums != right.checksums + || left.error != right.error + }) + { + return Err("requires compatible distinct external plaintext versions".into()); + } + let src_dir = src.data_dir.ok_or("source data directory absent")?; + let dst_dir = dst.data_dir.ok_or("destination data directory absent")?; + if src_dir.is_nil() || dst_dir.is_nil() || src_dir == dst_dir { + return Err("requires two distinct original physical generations".into()); + } + let before = meta + .versions + .iter() + .map(|version| version.parse_version_meta()) + .collect::, _>>()?; + let mut changed = before + .iter() + .find(|version| version.get_version_id() == dst.version_id) + .ok_or("shallow destination absent")? + .clone(); + let source_object = before + .iter() + .find(|version| version.get_version_id() == src.version_id) + .and_then(|version| version.object.as_ref()) + .ok_or("source is not an object")?; + let changed_object = changed + .object + .as_mut() + .ok_or("destination is not an object")?; + changed_object.data_dir = Some(src_dir); + // A shared directory must also share the source commitment descriptor. + // Preserve all other destination metadata, including its ACK identity. + let is_integrity = |key: &str| { + key == "x-rustfs-internal-shard-integrity-v1" + || key == "x-minio-internal-shard-integrity-v1" + }; + changed_object.meta_sys.retain(|key, _| !is_integrity(key)); + for (key, value) in &source_object.meta_sys { + if is_integrity(key) { + changed_object.meta_sys.insert(key.clone(), value.clone()); + } + } + meta.add_version_filemata(changed.clone())?; + let encoded = meta.marshal_msg()?; + if encoded.len() > 64 * 1024 * 1024 { + return Err("encoded fixture exceeds metadata budget".into()); + } + let reloaded = FileMeta::load(&encoded)?; + if reloaded.versions.len() != before.len() { + return Err("fixture changed the original version set".into()); + } + for version in &reloaded.versions { + let value = version.parse_version_meta()?; + let old = before + .iter() + .find(|item| item.get_version_id() == value.get_version_id()) + .ok_or("fixture invented a version")?; + let expected = if value.get_version_id() == dst.version_id { + &changed + } else { + old + }; + if &value != expected { + return Err("fixture changed unrelated version metadata".into()); + } + } + let receipt = serde_json::json!({"sourceVersion":source_version,"destinationVersion":destination_version, + "sharedDataDir":src_dir.to_string(),"retiredDataDir":dst_dir.to_string(), + "versionCount":before.len(),"filesystemMutationPerformed":false,"physicalByteEqualityClaimed":false}); + Ok((encoded, receipt)) +} + +#[cfg(test)] +mod tests { + use super::*; + use rustfs_filemeta::{ChecksumInfo, ErasureInfo, FileInfo, ObjectPartInfo}; + use rustfs_utils::HashAlgorithm; + + const SOURCE: &str = "00000000-0000-0000-0000-000000000001"; + const DESTINATION: &str = "00000000-0000-0000-0000-000000000002"; + const SOURCE_DIR: &str = "00000000-0000-0000-0000-000000000003"; + const DESTINATION_DIR: &str = "00000000-0000-0000-0000-000000000004"; + + fn fixture() -> FileMeta { + let mut meta = FileMeta::new(); + for (version, directory) in [(SOURCE, SOURCE_DIR), (DESTINATION, DESTINATION_DIR)] { + let info = FileInfo { + volume: "bucket".to_owned(), + name: "shared.bin".to_owned(), + version_id: Some(version.parse().unwrap()), + data_dir: Some(directory.parse().unwrap()), + size: 1024, + parts: vec![ObjectPartInfo { + number: 1, + size: 1024, + actual_size: 1024, + etag: "identical-ack-etag".to_owned(), + ..Default::default() + }], + erasure: ErasureInfo { + algorithm: "ReedSolomon".to_owned(), + data_blocks: 2, + parity_blocks: 2, + block_size: 1024 * 1024, + index: 1, + distribution: vec![1, 2, 3, 4], + checksums: vec![ChecksumInfo { + part_number: 1, + algorithm: HashAlgorithm::HighwayHash256S, + hash: Default::default(), + }], + }, + ..Default::default() + }; + meta.add_version(info).unwrap(); + } + meta + } + + #[test] + fn exact_existing_versions_share_one_directory_without_changing_other_metadata() { + let original = fixture().marshal_msg().unwrap(); + let source_before = FileMeta::load(&original) + .unwrap() + .into_file_info_versions("bucket", "shared.bin", true) + .unwrap() + .versions + .into_iter() + .find(|version| version.version_id == Some(SOURCE.parse().unwrap())) + .unwrap(); + let (encoded, receipt) = + encode(&original, "bucket", "shared.bin", SOURCE, DESTINATION).unwrap(); + let decoded = FileMeta::load(&encoded) + .unwrap() + .into_file_info_versions("bucket", "shared.bin", true) + .unwrap(); + assert_eq!(decoded.versions.len(), 2); + assert!( + decoded + .versions + .iter() + .all(|version| version.data_dir == Some(SOURCE_DIR.parse().unwrap())) + ); + assert_eq!(receipt["retiredDataDir"], DESTINATION_DIR); + assert_eq!(receipt["filesystemMutationPerformed"], false); + assert_eq!(receipt["physicalByteEqualityClaimed"], false); + assert_eq!( + decoded + .versions + .into_iter() + .find(|version| version.version_id == Some(SOURCE.parse().unwrap())) + .unwrap(), + source_before + ); + } + + #[test] + fn malformed_absent_same_and_already_shared_versions_are_refused() { + assert!(encode(b"invalid", "bucket", "shared.bin", SOURCE, DESTINATION).is_err()); + let original = fixture().marshal_msg().unwrap(); + assert!(encode(&original, "bucket", "shared.bin", SOURCE, SOURCE).is_err()); + assert!(encode(&original, "bucket", "shared.bin", SOURCE, SOURCE_DIR).is_err()); + let (shared, _) = encode(&original, "bucket", "shared.bin", SOURCE, DESTINATION).unwrap(); + assert!(encode(&shared, "bucket", "shared.bin", SOURCE, DESTINATION).is_err()); + } + + #[test] + fn incompatible_part_and_deleted_destination_are_refused() { + for deleted in [false, true] { + let mut meta = fixture(); + let mut info = meta + .into_file_info_versions("bucket", "shared.bin", true) + .unwrap() + .versions + .into_iter() + .find(|version| version.version_id == Some(DESTINATION.parse().unwrap())) + .unwrap(); + if deleted { + info.deleted = true; + } else { + info.parts[0].etag = "different-payload-ack".to_owned(); + } + meta.add_version(info).unwrap(); + assert!( + encode( + &meta.marshal_msg().unwrap(), + "bucket", + "shared.bin", + SOURCE, + DESTINATION + ) + .is_err() + ); + } + } +} diff --git a/test_validation_heal_orphan.py b/test_validation_heal_orphan.py new file mode 100644 index 00000000..148a12f1 --- /dev/null +++ b/test_validation_heal_orphan.py @@ -0,0 +1,72 @@ +"""C10 controlled-codec build identity and preservation guards.""" + +import json +from pathlib import Path +import tempfile +import unittest + +from scripts.validation_heal import shared_fixture as shared + + +class SharedFixtureArtifactTests(unittest.TestCase): + def fixture(self, root): + binary = root / 'heal-shared-fixture' + binary.write_bytes(b'\x7fELF\x02\x01' + b'\0' * 12 + b'\x3e\x00' + b'fixed-codec') + lock = root / 'shared-Cargo.lock'; lock.write_text('frozen helper graph\n') + (root / 'shared-fixture-tests.log').write_text('running 3 tests\n' + ''.join( + 'test ' + name + ' ... ok\n' for name in shared.TESTS) + 'test result: ok. 3 passed; 0 failed; 0 ignored;\n') + run = {'id': 123, 'run_attempt': 1, 'status': 'completed', 'conclusion': 'success', + 'path': '.github/workflows/rustfs-validation-tools.yml'} + (root / 'tool-run.json').write_text(json.dumps(run)) + manifest = {'schemaVersion': 1, 'built': True, 'sourceUnmodified': True, 'candidateCommit': shared.CANDIDATE, + 'candidateSourceTree': shared.SOURCE_TREE, 'candidateLockSha256': shared.CARGO_LOCK, + 'target': 'x86_64-unknown-linux-gnu', 'role': shared.ROLE, 'filesystemMutationPerformed': False, + 'physicalByteEqualityClaimed': False, 'runId': '123', 'runAttempt': '1', + 'helperSourceSha256': shared.sha(Path(shared.__file__).with_name('tools') / 'heal_shared_version_fixture.rs'), + 'binarySha256': shared.sha(binary), 'sharedLockSha256': shared.sha(lock)} + (root / 'heal-shared-fixture.json').write_text(json.dumps(manifest)) + return manifest, run + + def test_verified_codec_is_distinct_from_read_only_inspector_and_live_proof(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder); self.fixture(root) + proof = shared.verify(root, {'source_sha': shared.CANDIDATE}) + self.assertTrue(proof['exactCodecTestsPassed']) + self.assertFalse(proof['filesystemMutationPerformed']) + self.assertFalse(proof['physicalByteEqualityClaimed']) + self.assertFalse(proof['fleetPassClaimed']) + + def test_foreign_source_artifact_role_commit_and_build_attempt_cannot_be_installed(self): + for field, value in (('role', 'read-only decoder; not the RustFS service under test'), ('candidateSourceTree', '0' * 40), + ('candidateLockSha256', '0' * 64), ('helperSourceSha256', '0' * 64), ('built', False), + ('physicalByteEqualityClaimed', True), ('runAttempt', '2')): + with self.subTest(field=field), tempfile.TemporaryDirectory() as folder: + root = Path(folder); manifest, _ = self.fixture(root) + manifest[field] = value + (root / 'heal-shared-fixture.json').write_text(json.dumps(manifest)) + with self.assertRaises(ValueError): shared.verify(root, {'source_sha': shared.CANDIDATE}) + + def test_zero_filtered_ignored_or_incomplete_codec_tests_do_not_certify_a_build(self): + for log in ('running 0 tests\ntest result: ok. 0 passed; 0 failed; 0 ignored;\n', + 'running 3 tests\ntest result: ok. 2 passed; 0 failed; 1 ignored;\n', + 'running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored;\n'): + with self.subTest(log=log), tempfile.TemporaryDirectory() as folder: + root = Path(folder); self.fixture(root) + (root / 'shared-fixture-tests.log').write_text(log) + with self.assertRaises(ValueError): shared.verify(root, {'source_sha': shared.CANDIDATE}) + + def test_changed_executable_or_failed_remote_build_is_refused(self): + for kind in ('binary', 'run', 'symlink'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as folder: + root = Path(folder); _, run = self.fixture(root) + binary = root / 'heal-shared-fixture' + if kind == 'binary': binary.write_bytes(b'foreign') + elif kind == 'run': + run['conclusion'] = 'failure'; (root / 'tool-run.json').write_text(json.dumps(run)) + else: + binary.rename(root / 'outside'); binary.symlink_to(root / 'outside') + with self.assertRaises(ValueError): shared.verify(root, {'source_sha': shared.CANDIDATE}) + + +if __name__ == '__main__': + unittest.main() From cce6971af37bca12e1c6d1d07e006de3139a6533 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 09:52:06 +0800 Subject: [PATCH 111/343] test(validation): provide valid shared-version codec timestamps --- scripts/validation_heal/prepare_heal_shared_fixture.py | 2 ++ .../validation_heal/tools/heal_shared_version_fixture.rs | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/prepare_heal_shared_fixture.py b/scripts/validation_heal/prepare_heal_shared_fixture.py index c1f6d524..e5a18726 100644 --- a/scripts/validation_heal/prepare_heal_shared_fixture.py +++ b/scripts/validation_heal/prepare_heal_shared_fixture.py @@ -21,6 +21,8 @@ def prepare_shared(source, commit, output): (output / 'src/main.rs').write_bytes(helper.read_bytes()) manifest = output / 'Cargo.toml' manifest.write_text(manifest.read_text().replace('name = "heal-meta-inspect"', 'name = "heal-shared-fixture"')) + with manifest.open('a') as stream: + stream.write('[dev-dependencies]\ntime = "0.3"\n') path = output / 'source.json' proof = json.loads(path.read_text()) proof.update(role=ROLE, helperSourceSha256=hashlib.sha256(helper.read_bytes()).hexdigest(), diff --git a/scripts/validation_heal/tools/heal_shared_version_fixture.rs b/scripts/validation_heal/tools/heal_shared_version_fixture.rs index 6bc7ae59..a26ca641 100644 --- a/scripts/validation_heal/tools/heal_shared_version_fixture.rs +++ b/scripts/validation_heal/tools/heal_shared_version_fixture.rs @@ -162,12 +162,19 @@ mod tests { fn fixture() -> FileMeta { let mut meta = FileMeta::new(); - for (version, directory) in [(SOURCE, SOURCE_DIR), (DESTINATION, DESTINATION_DIR)] { + for (index, (version, directory)) in [(SOURCE, SOURCE_DIR), (DESTINATION, DESTINATION_DIR)] + .into_iter() + .enumerate() + { let info = FileInfo { volume: "bucket".to_owned(), name: "shared.bin".to_owned(), version_id: Some(version.parse().unwrap()), data_dir: Some(directory.parse().unwrap()), + mod_time: Some( + time::OffsetDateTime::from_unix_timestamp(1_700_000_000 + index as i64) + .unwrap(), + ), size: 1024, parts: vec![ObjectPartInfo { number: 1, From 6cc8cb79a0f8024b0089a43d31b0708a46bbae4c Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 09:55:47 +0800 Subject: [PATCH 112/343] test(validation): preserve shared-codec rejection inputs --- .../tools/heal_shared_version_fixture.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/validation_heal/tools/heal_shared_version_fixture.rs b/scripts/validation_heal/tools/heal_shared_version_fixture.rs index a26ca641..7c24a17f 100644 --- a/scripts/validation_heal/tools/heal_shared_version_fixture.rs +++ b/scripts/validation_heal/tools/heal_shared_version_fixture.rs @@ -262,9 +262,18 @@ mod tests { .find(|version| version.version_id == Some(DESTINATION.parse().unwrap())) .unwrap(); if deleted { - info.deleted = true; + // A deleted payload with erasure geometry is purge-pending, + // not a canonical delete marker in the candidate codec. + info = FileInfo { + version_id: info.version_id, + mod_time: info.mod_time, + deleted: true, + ..Default::default() + }; } else { - info.parts[0].etag = "different-payload-ack".to_owned(); + // Single-part ETags are not stored in PartETags. Change a + // durable part field so the rejection input survives load. + info.parts[0].size -= 1; } meta.add_version(info).unwrap(); assert!( @@ -275,7 +284,8 @@ mod tests { SOURCE, DESTINATION ) - .is_err() + .is_err(), + "incompatible destination was accepted (delete_marker={deleted})" ); } } From b0fc7d15ba170bd592e6f4012c1ba15125186afb Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 10:09:23 +0800 Subject: [PATCH 113/343] feat(validation): migrate complete C10 orphan safety branches --- docs/validation-migration-map.md | 59 ++- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/fleet_node.py | 45 +- scripts/validation_heal/generation_fault.py | 5 +- scripts/validation_heal/healthy.py | 20 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/orphan.py | 501 ++++++++++++++++++++ scripts/validation_heal/orphan_fault.py | 237 +++++++++ scripts/validation_heal/run_case.sh | 1 + test_validation_heal_orphan.py | 326 +++++++++++++ 10 files changed, 1186 insertions(+), 12 deletions(-) create mode 100644 scripts/validation_heal/orphan.py create mode 100644 scripts/validation_heal/orphan_fault.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 9b752b93..c23594bc 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 131 currently dispatchable IDs include all 74 non-Heal cases and fifty-seven Heal cases. Forty-eight have complete canonical live PASS evidence; the other 110 have no live PASS claim. +case evidence. The 132 currently dispatchable IDs include all 74 non-Heal cases and fifty-eight Heal cases. Forty-nine have complete canonical live PASS evidence; the other 109 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1551,6 +1551,57 @@ descriptor may change; all version identities and unrelated metadata are reloaded and compared before output. Three actual codec tests must pass against the candidate before the artifact can be consumed. Python guards additionally bind source/tree/Cargo.lock/helper/binary/role and the successful build attempt, -and reject incomplete, ignored or zero-test output. This is C10 prerequisite -tooling; the full five-branch executor is still in progress and the count -remains 131 automatic / 27 remaining Heal executors. +and reject incomplete, ignored or zero-test output. This artifact is the prerequisite for the complete C10 executor described below; +tool success remains independent of the canonical live verdict. + + +C10 now dispatches all five source branches on both H1 and H2. Runtime profiles +freeze the source's 0/300/30-second dangling policy, and each trigger records +server LastModified, original-node clock agreement and the actual elapsed age. +All independent original roots archive their complete objects, including real +integrity sidecars and absent post-DELETE trees. Durable arm records precede +controlled replay or shared conversion. Rescue uses the last acknowledged +post-DELETE authority on every original root, preserves the failed scene and +cannot restore the deleted old generation. A shared seed is retired before +later ACKs; unknown writes, lost arm receipts and partial recovery retain DIRTY +namespaces and block cleanup. + +The shared branch proves two independently acknowledged original generations +have identical physical parts before the candidate codec changes only their +shared data-directory/commitment references. Both full peer S3 inventories and +all physical references must agree before the later third version is created. +The retained shared references and delete-marker controls survive orphan +removal. Pending MPU uses a bounded, descriptor-relative, read-only census of +the isolated run's complete multipart tree, exact all-peer UploadId/list_parts +checks, and an acknowledged completion followed by full payload verification. +Product physical and S3 assertions run before any fixture rescue. Every live +part receives candidate-bound streaming verification before final S3 checks. + +Hosted tools run `38103474555` attempt 1 succeeded at driver +`9b4cc40abd5ea3a0a5ee6c52fb7e113b57102d6e` and helper +`6cc8cb79a0f8024b0089a43d31b0708a46bbae4c`. Independent original artifact/API +verification confirms all three codec tests, source/tree/Cargo.lock/helper/role +and binary `eb2991cf6f84b6998a474c51cd142c2b1c84106fd7f49f569fce7a241dc4ecea`. +Runs `38102872003` and `38103273662` failed on invalid synthetic timestamp and +non-persisted/non-marker rejection inputs; these failures remain retained. +The corrected inputs keep all transformation/refusal assertions. The earlier +wrong-SHA dispatch `38102848357` was cancelled and is not tool evidence. +Ordinary C10 execution consumes this artifact without a RustFS checkout. + +The complete local validation suite now runs 397 tests with eight macOS skips. +C10 has 20 tests covering source completeness, typed deferred/refusal results, +all-member live reference/part/commitment/control safety, absent-authority +recovery, foreign device/candidate refusal, uncertain ACK retention, shared +seed retirement and pending-MPU alias refusal. Its real Linux descriptor, +root-owned recovery and nested bind-mount test is mandatory in CI. C10 still +requires canonical live evidence; there are 132 automatic executors and 26 +remaining Heal executors. + +F06 `heal.cancelled-task-restart` passed in run `38102031029` at harness +`f248cb63d032cfc1658d32c508bce745f96b885f`: both H1/H2 partial cancellations, +quiet physical windows, persisted cancellation after restart, no old-task +replay, new-token repair, all-peer S3 and cleanup passed. Its independently +rerun canonical gate passed, increasing complete live PASS to 49. +F07's coordinator process-crash run `38103118033` remains canonical FAIL with +cleanup PASS. The serial fleet queue continues under its existing lock and +unclean-environment guard. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 66b589ef..ce2d871b 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index c4bd4194..3514a11a 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -33,7 +33,10 @@ def runtime_config(profile): - require(profile in {"manual", "authority-split", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(profile in {"orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + if profile in {"orphan-preserve", "orphan-before", "orphan-after"}: + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS": + {"orphan-preserve": "0", "orphan-before": "300", "orphan-after": "30"}[profile]} if profile == "authority-split": return {**BACKGROUND_CONFIG, "RUSTFS_STORAGE_CLASS_STANDARD": "EC:1"} if profile == "multi-pool": @@ -1055,6 +1058,43 @@ def generation_operation(request): return method(root, owner, request, fault_module()) +def orphan_operation(request): + state, roots, owner = owned(request['runId']) + stopped(owner) + require(owner.get('runtimeProfile') in {'orphan-preserve', 'orphan-before', 'orphan-after'} + and digest(state / 'rustfs') == owner['binarySha256'], 'C10 runtime/candidate changed') + fault, generation, backend = fault_module(), globals()['generation_backend'], globals()['orphan_backend'] + action = request['action'] + tool = state / 'heal-shared-fixture' + if action == 'orphan-install-tool': + require(re.fullmatch(r'[0-9a-f]{64}', request['sha256']), 'C10 encoder digest absent') + with fault.directory(state) as fd: + raw = fault.read_file(fd, tool.name) + require(fault.digest(raw) == request['sha256'] and os.stat(tool, follow_symlinks=False).st_uid == 0, + 'C10 uploaded encoder changed or is not privately root owned') + fault.new_file(fd, 'shared-tool.json', fault.encode({'runId': owner['runId'], + 'ownerSha256': digest(state / 'owner.json'), 'binarySha256': owner['binarySha256'], 'sha256': request['sha256']})) + tool.chmod(0o500) + return {'installed': True, 'sha256': request['sha256']} + disk = request['disk'] + require(type(disk) is int and 0 <= disk < 4, 'C10 escaped an original independent root') + with fault.directory(roots[disk]) as root: + if action == 'orphan-multipart': return backend.multipart(root, fault) + if action == 'orphan-observe': + present, _, files = backend.scene(root, owner, request, fault, generation) + return {'present': present, 'files': files, 'nodeEpoch': time.time()} + if action == 'orphan-convert': + with fault.directory(state) as fd: + record = json.loads(fault.read_file(fd, 'shared-tool.json', 65536)) + require(record == {'runId': owner['runId'], 'ownerSha256': digest(state / 'owner.json'), + 'binarySha256': owner['binarySha256'], 'sha256': digest(tool)}, 'C10 encoder binding changed') + return backend.convert(root, owner, request, fault, generation, checked(tool, state)) + methods = {'orphan-capture': backend.capture, 'orphan-arm': backend.arm, 'orphan-install': backend.install, + 'orphan-recover': backend.recover, 'orphan-disarm': backend.disarm} + require(action in methods, 'unregistered C10 operation') + return methods[action](root, owner, request, fault, generation) + + def mrf_records(request): require(set(request) <= {"runId", "action", "quiescent"}, "MRF observer does not accept paths or mutation options") state, roots, owner = owned(request["runId"]) @@ -1091,6 +1131,9 @@ def main(request): actions["task-records"] = task_records for action in ('generation-capture', 'generation-install', 'generation-recover'): actions[action] = generation_operation + for action in ('orphan-install-tool', 'orphan-observe', 'orphan-multipart', 'orphan-convert', + 'orphan-capture', 'orphan-arm', 'orphan-install', 'orphan-recover', 'orphan-disarm'): + actions[action] = orphan_operation for action in ("bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover"): actions[action] = bucket_record_operation for action in ("quorum-gate", "quorum-gate-probe"): diff --git a/scripts/validation_heal/generation_fault.py b/scripts/validation_heal/generation_fault.py index 46f90d2f..4b0f93f2 100644 --- a/scripts/validation_heal/generation_fault.py +++ b/scripts/validation_heal/generation_fault.py @@ -25,11 +25,12 @@ def scope(owner, request): return [bucket, key] -def tree(fd, fault): +def tree(fd, fault, *, allow_sparse=False): """Read the complete small object tree; reject aliases or unexpected leaves.""" values = {} names = sorted(os.listdir(fd)) - require('xl.meta' in names and 1 < len(names) <= 17, 'generation lacks bounded external metadata/data directories') + require(len(names) <= 17 and (allow_sparse or 'xl.meta' in names and len(names) > 1), + 'generation lacks bounded external metadata/data directories') for name in names: if name == 'xl.meta': values[name] = fault.read_file(fd, name) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 755ef367..26a205ec 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"manual", "authority-split", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(runtime_profile in {"orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -152,6 +152,7 @@ def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_prof self.quorum_gates = set() self.network_gates = set() self.generation_faults = [] + self.orphan_faults = [] self.secret_values = set() self.frozen_processes = {} self.snapshot_buckets = None @@ -195,13 +196,16 @@ def remote(self, node, action, **params): if action == "replacement-records": source += "\nimport types\nreplacement_journal = types.ModuleType('replacement_journal')\nexec(" + repr( NODE_SOURCE.with_name("replacement_journal.py").read_text()) + ", replacement_journal.__dict__)\n" - if action.startswith("generation-"): + if action.startswith("generation-") or action.startswith("orphan-"): source += "\nimport types\ngeneration_backend = types.ModuleType('generation_fault')\nexec(" + repr( NODE_SOURCE.with_name("generation_fault.py").read_text()) + ", generation_backend.__dict__)\n" if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline", - "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure"} or action.startswith("generation-"): + "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure"} or action.startswith("generation-") or action.startswith("orphan-"): source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" + if action.startswith("orphan-"): + source += "\nimport types\norphan_backend = types.ModuleType('orphan_fault')\nexec(" + repr( + NODE_SOURCE.with_name("orphan_fault.py").read_text()) + ", orphan_backend.__dict__)\n" if action.startswith("bucket-record-"): source += "\nimport types\nbucket_records = types.ModuleType('bucket_records')\nexec(" + repr( NODE_SOURCE.with_name("bucket_records.py").read_text()) + ", bucket_records.__dict__)\n" @@ -313,6 +317,16 @@ def wait_ready(self): def cleanup(self): failures = [] + if self.orphan_faults: + try: + self.stop() + for node, params in tuple(self.orphan_faults): + receipt = self.record("orphan-last-ACK-recovery", {"node": node, **self.remote(node, "orphan-recover", **params)}) + require(receipt.get("state") == "CLEAR" and receipt.get("lastACKAuthorityRestored") is True, + "C10 lacks complete last-ACK authority restoration") + self.orphan_faults.remove((node, params)) + except Exception as exc: + failures.append({"error": str(exc), "state": "DIRTY", "namespacesRetained": True}) if self.generation_faults: try: self.stop() diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index bdb46788..2bafb064 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/orphan.py b/scripts/validation_heal/orphan.py new file mode 100644 index 00000000..c198d502 --- /dev/null +++ b/scripts/validation_heal/orphan.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python3 +"""C10: actual grace, exact shared references and pending-MPU preservation.""" + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import subprocess +import sys +import time +import uuid + +try: + from .authority import object_files + from .heal_probe import Probe, canonical, compare_inventories, payload, require, save + from .heal_workload import Recorder, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .native_regressions import CANDIDATE + from .shared_fixture import verify as verify_shared + from .shards import files, map_target, run_task + from .scopes import verify_parts + from .stop import idle_fleet +except ImportError: + from authority import object_files + from heal_probe import Probe, canonical, compare_inventories, payload, require, save + from heal_workload import Recorder, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from native_regressions import CANDIDATE + from shared_fixture import verify as verify_shared + from shards import files, map_target, run_task + from scopes import verify_parts + from stop import idle_fleet + + +CASE_ID = 'heal.dangling-and-orphan-safety' +BRANCHES = ('remove-false', 'remove-true-before-grace', 'remove-true-after-grace', 'shared-version-reference', 'pending-mpu') +SIZE = 2 * 1024**2 + 123 +KEY = 'orphan.bin' + + +def members(snapshot): + return {(node, row['disk']) for node, value in snapshot.items() for row in value['formats']} + + +def references(snapshot, bucket, key): + indexed = files(snapshot) + result = {} + for node, disk in members(snapshot): + metadata = indexed.get((node, disk, f'{bucket}/{key}/xl.meta'), {}).get('metadata', {}) + require(not metadata or metadata.get('bucket') == bucket and metadata.get('key') == key, + 'C10 physical metadata changed bucket/key identity') + values = metadata.get('versions', []) + require(len({value['versionId'] for value in values}) == len(values), 'C10 duplicate physical version identity') + result[(node, disk)] = {value['versionId']: value for value in values} + return result + + +def require_shared(snapshot, bucket, versions): + require(len(versions) == 2 and len(set(versions)) == 2, 'C10 shared fixture needs two independent ACK versions') + for values in references(snapshot, bucket, KEY).values(): + live = [values.get(version) for version in versions] + require(all(value and value.get('deleted') is False and value.get('inline') is None for value in live) + and live[0].get('dataDir') and live[0]['dataDir'] == live[1].get('dataDir'), + 'C10 lost a shared live version reference') + + +def physical_safety(before, after, authority, bucket, expected, branch, doomed, directory, target): + require(members(before) == members(after) == members(authority), 'C10 lost an original physical member') + require({node: value['formats'] for node, value in before.items()} == + {node: value['formats'] for node, value in after.items()}, 'C10 changed original format/UUID/set identity') + left, right = files(before), files(after) + prefix = bucket + '/' + KEY + '/' + controls = lambda values: {identity: value for identity, value in values.items() if not identity[2].startswith(prefix)} + require(controls(left) == controls(right), 'C10 changed control bytes or decoded metadata') + original_refs, final_refs = references(authority, bucket, KEY), references(after, bucket, KEY) + selected = [entry for entry in expected if entry['key'] == KEY] + wanted = {entry['versionId']: entry for entry in selected} + for member, values in final_refs.items(): + allowed = set(wanted) + if branch == 'remove-true-before-grace' and member == target: allowed.add(doomed) + require(set(values) == allowed, 'C10 lost, invented or resurrected a version/marker reference') + for version, entry in wanted.items(): + old, new = original_refs[member].get(version), values.get(version) + require(old and new and new.get('deleted') is entry['deleteMarker'] + and new.get('dataDir') == old.get('dataDir'), 'C10 live version/marker or dataDir changed') + if entry['deleteMarker']: continue + data_prefix = prefix + old['dataDir'] + '/' + original = {identity: value for identity, value in left.items() if identity[:2] == member and identity[2].startswith(data_prefix)} + actual = {identity: value for identity, value in right.items() if identity[:2] == member and identity[2].startswith(data_prefix)} + require(original and actual == original, 'C10 changed referenced payloads or integrity sidecars') + if branch == 'pending-mpu': + require(physical_content(before) == physical_content(after), 'C10 Heal changed an unfaulted pending-MPU control object') + elif branch == 'remove-false': + require(physical_content(before) == physical_content(after), 'C10 remove=false changed protected orphan evidence') + elif branch == 'remove-true-before-grace': + old = {identity: value for identity, value in left.items() if identity[:2] == target and identity[2].startswith(prefix + directory + '/')} + new = {identity: value for identity, value in right.items() if identity[:2] == target and identity[2].startswith(prefix + directory + '/')} + require(old and old == new and final_refs[target][doomed].get('dataDir') == directory, + 'C10 pre-grace deleted protected stale metadata or shards') + else: + require(not any(identity[2].startswith(prefix + directory + '/') for identity in right), + 'C10 expired or unreferenced replay was not actually removed') + if branch == 'shared-version-reference': + require_shared(after, bucket, [entry['versionId'] for entry in selected if not entry['deleteMarker']]) + return {'allOriginalMembersChecked': len(members(after)), 'liveReferencesAndPartsPreserved': True, + 'healthyControlsAndFormatsPreserved': True, 'backupRescueIsProductRepair': False} + + +def terminal_safety(status, branch, bucket, options): + require(status.get('summary') == 'finished' and status.get('truncated', False) is False + and status.get('settings') == options, 'C10 terminal receipt is absent/truncated or changed its request') + outcome = status.get('outcome') or {} + counters = outcome.get('counters') or {} + require(outcome.get('coverage') == 'complete' and outcome.get('objectsTruncated', False) is False + and all(type(counters.get(name)) is int and counters[name] >= 0 for name in ('processed', 'healed', 'failed', 'unknown')) + and counters['unknown'] == 0, 'C10 responsibility is missing, partial or unknown') + if branch == 'remove-false': + require(outcome.get('execution', {}).get('state') == 'completed_with_errors' + and counters['healed'] == 0 and counters['failed'] == 1 + and 'Object not found: ' + bucket + '/' + KEY in (status.get('detail') or ''), + 'C10 metadata-less protected orphan lacks its exact not-found refusal') + elif branch == 'remove-true-before-grace': + deferred = [value for value in outcome.get('objects', []) + if value.get('identity', {}).get('bucket') == bucket and value.get('identity', {}).get('object') == KEY + and value.get('disposition', {}).get('state') == 'deferred' + and value['disposition'].get('details', {}).get('reason') == 'dangling_delete_grace'] + require(deferred and counters['failed'] == 0 and outcome.get('execution', {}).get('state') == 'completed', + 'C10 pre-grace preservation lacks an exact typed deferred responsibility') + else: + require(counters['failed'] == 0 and outcome.get('execution', {}).get('state') == 'completed', + 'C10 physical safety has no successful terminal responsibility') + return {'canonicalCounters': counters, 'fullOriginalResponsibility': True} + + +def capture(fleet, snapshot, bucket, label): + result = {} + for (node, disk), expected in sorted(object_files(snapshot, bucket, KEY).items()): + params = {'disk': disk, 'bucket': bucket, 'key': KEY} + observed = fleet.record('C10-stopped-object-presence', {'node': node, **fleet.remote(node, 'orphan-observe', **params)}) + require(observed['files'] == expected and abs(observed['nodeEpoch'] - time.time()) <= 3, + 'C10 stopped archive or real host clock differs from the physical witness') + result[(node, disk)] = fleet.record('C10-archive-' + label, {'node': node, **fleet.remote(node, 'orphan-capture', + **params, generation=label, expectedFiles=expected, expectedPresent=observed['present'])}) + return result + + +def arm_all(fleet, snapshot, bucket, label): + capture(fleet, snapshot, bucket, label) + for node, disk in sorted(members(snapshot)): + params = {'disk': disk, 'bucket': bucket, 'key': KEY, 'generation': label} + # Register before SSH: a lost arm receipt leaves the whole namespace DIRTY. + fleet.orphan_faults.append((node, params)) + fleet.record('C10-arm-original-member', {'node': node, **fleet.remote(node, 'orphan-arm', **params)}) + + +def recover_all(fleet): + for node, params in tuple(fleet.orphan_faults): + receipt = fleet.record('C10-recover-original-member', {'node': node, **fleet.remote(node, 'orphan-recover', **params)}) + require(receipt.get('state') == 'CLEAR' and receipt.get('lastACKAuthorityRestored') is True, + 'C10 original member is not completely recovered') + fleet.orphan_faults.remove((node, params)) + + +def pending_snapshot(fleet): + return {(node, disk): fleet.record('C10-read-only-MPU-census', {'node': node, 'disk': disk, + **fleet.remote(node, 'orphan-multipart', disk=disk)})['files'] for node in fleet.nodes for disk in range(4)} + + +def verify_public(fleet, binding, bucket, expected, deleted, label): + for endpoint in binding['endpoints']: + peer = Probe(binding, endpoint) + inventory = fleet.record(label + '-full-inventory', peer.inventory(bucket)) + fleet.record(label + '-ACK-comparison', verify_seed(expected, inventory)) + for version in deleted: + observed = fleet.record(label + '-deleted-version', peer.s3('get_object', {'Bucket': bucket, 'Key': KEY, 'VersionId': version})) + response = observed.get('response') or {} + require(observed['acknowledged'] is False and observed['resultUnknown'] is False and 'payload' not in observed + and response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 404 + and response.get('Error', {}).get('Code') in {'NoSuchVersion', 'NoSuchKey'}, 'C10 deleted version resurrected') + current = next((entry for entry in expected if entry['key'] == KEY and entry['isLatest']), None) + if current is None or current['deleteMarker']: + observed = fleet.record(label + '-current-absence', peer.s3('get_object', {'Bucket': bucket, 'Key': KEY})) + response = observed.get('response') or {} + headers = response.get('ResponseMetadata', {}).get('HTTPHeaders', {}) + require(not observed['acknowledged'] and not observed['resultUnknown'] and 'payload' not in observed + and response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 404 + and response.get('Error', {}).get('Code') == 'NoSuchKey', 'C10 absent current/marker resurrected') + if current: + require(headers.get('x-amz-delete-marker') == 'true' + and headers.get('x-amz-version-id') in {None, current['versionId']}, 'C10 current marker header/identity changed') + + +def run_branch(args, topology, nodes, branch): + directory = args.evidence / topology / branch + directory.mkdir(parents=True) + candidate = json.loads(args.candidate.read_text()) + require(candidate['source_sha'] == CANDIDATE, 'C10 requires its reviewed candidate codec/grace policy') + result = {'topology': topology, 'branch': branch, 'verdict': 'RUNNING', 'cleanup': 'PENDING'} + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + runtime = 'orphan-before' if branch == BRANCHES[1] else 'orphan-after' if branch == BRANCHES[2] else 'orphan-preserve' + grace = 300 if runtime == 'orphan-before' else 30 if runtime == 'orphan-after' else 0 + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile=runtime) + bucket = 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {'schemaVersion': 1, 'runId': run_id, 'jumpHostOnly': True, 'exclusiveHealService': True, + 'leaseExpiresAt': (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), 'bucketPrefix': bucket, + 'endpoints': {f'node{index}': f'http://rustfs-node{index}:9000' for index in range(1, len(nodes) + 1)}, + 'defaultEndpoint': 'node1', 'candidateCommit': candidate['source_sha'], 'harnessCommit': args.testing_commit, + 'candidatePackageSha256': candidate['package_sha256'], 'candidateBinarySha256': args.binary_sha, + 'rcBinarySha256': hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), 'maxInventoryObjects': 128, + 'maxInventoryBytes': 512 * 1024**2} + fleet.record('binding', binding) + fleet.record('inspector-build', verify_inspector(args.tool_directory, candidate)) + shared = fleet.record('shared-encoder-build', verify_shared(args.shared_directory, candidate)) + probe, active, pending_mutation = Probe(binding), [], None + expected, deleted, recovery_snapshot = [], [], None + try: + def install_tool(): + if branch != 'shared-version-reference': return + raw = (args.shared_directory / 'heal-shared-fixture').read_bytes() + for node in nodes: + path = '/var/lib/rustfs-auto-testing/heal/' + run_id + '/heal-shared-fixture' + receipt = subprocess.run([*fleet.ssh(node), 'sudo -n dd of=' + path + ' conv=excl,fsync oflag=nofollow bs=1M status=none'], + input=raw, capture_output=True, timeout=120) + require(receipt.returncode == 0, 'C10 exclusive private encoder upload failed') + fleet.record('C10-install-owned-encoder', {'node': node, **fleet.remote(node, 'orphan-install-tool', sha256=shared['binarySha256'])}) + fleet.deploy(args.binary_sha, bucket, access, secret, before_start=install_tool) + recorder = Recorder(probe, directory / 'fixture') + def call(operation, params, body=None, check=None): + nonlocal pending_mutation + pending_mutation = {'operation': operation, 'params': params} + response = recorder.call(operation, params, body) + if check is not None: check(response) + pending_mutation = None + return response + def unique_version(response): + version = response.get('VersionId') + require(isinstance(version, str) and str(uuid.UUID(version)) == version and uuid.UUID(version).int != 0 + and version not in {entry['versionId'] for entry in expected} | set(deleted), + 'C10 ACK lost its distinct exact non-null version identity') + def put(key, label, raw=None): + body = payload('C10/' + label, SIZE) if raw is None else raw + response = call('put_object', {'Bucket': bucket, 'Key': key, 'ContentType': 'application/octet-stream', + 'Metadata': {'heal-fixture': 'C10', 'ack': label}}, body, unique_version) + for entry in expected: + if entry['key'] == key: entry['isLatest'] = False + entry = {'key': key, 'versionId': response['VersionId'], 'isLatest': True, 'deleteMarker': False, + 'payload': {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}} + expected.append(entry) + head = fleet.record('C10-server-generation-age', probe.s3('head_object', {'Bucket': bucket, 'Key': key, 'VersionId': entry['versionId']})) + require(head['acknowledged'], 'C10 generation age has no successful server HEAD') + info = head['response'] + require(info.get('ContentLength') == len(body) and info.get('ETag') == response.get('ETag') + and info.get('ContentType') == 'application/octet-stream' + and info.get('Metadata') == {'heal-fixture': 'C10', 'ack': label}, 'C10 HEAD differs from original PUT ACK/properties') + entry.update(properties={name: info[name] for name in ( + 'ETag', 'ContentLength', 'ContentType', 'ContentEncoding', 'ContentDisposition', 'CacheControl', 'Metadata', + 'StorageClass', 'ServerSideEncryption', 'SSEKMSKeyId', 'SSECustomerAlgorithm', 'SSECustomerKeyMD5', + 'ObjectLockMode', 'ObjectLockRetainUntilDate', 'ObjectLockLegalHoldStatus') if name in info}, tags=[]) + value = head['response']['LastModified'] + epoch = value.timestamp() if isinstance(value, datetime) else datetime.fromisoformat(value).timestamp() + require(abs(epoch - time.time()) <= 5, 'C10 fixture server clock/age precondition failed') + return entry, epoch + call('create_bucket', {'Bucket': bucket}) + call('put_bucket_versioning', {'Bucket': bucket, 'VersioningConfiguration': {'Status': 'Enabled'}}) + put('control.bin', 'control') + # A separate acknowledged marker control exercises DM preservation in every branch. + marker_data, _ = put('marker-control.bin', 'marker-control') + def marker_ack(response): + unique_version(response) + require(response.get('DeleteMarker') is True and response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 204, + 'C10 marker DELETE lacks its exact successful ACK') + marker = call('delete_object', {'Bucket': bucket, 'Key': 'marker-control.bin'}, check=marker_ack) + marker_data['isLatest'] = False + expected.append({'key': 'marker-control.bin', 'versionId': marker['VersionId'], 'deleteMarker': True, 'isLatest': True}) + raw = payload('C10/shared-first', SIZE) + first, first_time = put(KEY, 'first', raw) + if branch == 'shared-version-reference': + second, _ = put(KEY, 'second', raw) + baseline = fleet.record('C10-independent-original-ACK-S3', probe.inventory(bucket)) + verify_seed(expected, baseline) + fleet.stop() + seed = fleet.snapshot('C10-stopped-shared-seed', quiescent=True) + validate_topology(seed, topology) + source_map = map_target(seed, bucket, KEY, 'data', 1, first['versionId'], latest_required=False) + dest_map = map_target(seed, bucket, KEY, 'data', 1, second['versionId']) + recovery_snapshot = seed + arm_all(fleet, seed, bucket, 'seed') + for left, right in zip(source_map['cohort'], dest_map['cohort'], strict=True): + require((left['node'], left['disk']) == (right['node'], right['disk']), 'C10 shared member ordering changed') + fleet.record('C10-exact-codec-shared-conversion', {'node': left['node'], **fleet.remote(left['node'], 'orphan-convert', + disk=left['disk'], bucket=bucket, key=KEY, sourceVersion=first['versionId'], destinationVersion=second['versionId'], + sourceDataDir=left['dataDir'], destinationDataDir=right['dataDir'])}) + converted = fleet.snapshot('C10-shared-all-original-physical', quiescent=True) + require_shared(converted, bucket, [first['versionId'], second['versionId']]) + fleet.start() + for endpoint in binding['endpoints']: + compare_inventories(baseline, fleet.record('C10-shared-all-peer-S3', Probe(binding, endpoint).inventory(bucket))) + fleet.stop() + confirmed = fleet.snapshot('C10-shared-before-later-ACK', quiescent=True) + require(physical_content(converted) == physical_content(confirmed), 'C10 shared representation drifted after GET') + for node, params in tuple(fleet.orphan_faults): + receipt = fleet.record('C10-retire-seed-rescue-before-later-ACK', {'node': node, **fleet.remote(node, 'orphan-disarm', + **params, expectedFiles=object_files(confirmed, bucket, KEY)[(node, params['disk'])])}) + require(receipt.get('state') == 'RETIRED' and receipt.get('laterACKMayProceed') is True, 'C10 seed rescue was not retired') + fleet.orphan_faults.remove((node, params)) + recovery_snapshot = None + fleet.start() + doomed, doomed_time = put(KEY, 'doomed-third') + else: + doomed, doomed_time = first, first_time + if branch.startswith('remove-true-'): put(KEY, 'second-retained') + fleet.record('C10-full-baseline-S3', verify_seed(expected, probe.inventory(bucket))) + fleet.stop() + original = fleet.snapshot('C10-all-original-generations', quiescent=True) + validate_topology(original, topology) + doomed_map = map_target(original, bucket, KEY, 'data', 1, doomed['versionId'], latest_required=not branch.startswith('remove-true-')) + target = doomed_map['node'], doomed_map['disk'] + data_dir = doomed_map['dataDir'] + if branch == 'pending-mpu': + fleet.start() + upload = call('create_multipart_upload', {'Bucket': bucket, 'Key': KEY}, check=lambda response: + require(isinstance(response.get('UploadId'), str) and response['UploadId'], 'C10 MPU create ACK omitted UploadId'))['UploadId'] + pending_body = payload('C10/pending-part', SIZE) + part = call('upload_part', {'Bucket': bucket, 'Key': KEY, 'UploadId': upload, 'PartNumber': 1}, pending_body, + lambda response: require(isinstance(response.get('ETag'), str) and response['ETag'], 'C10 part ACK omitted ETag')) + fleet.stop() + before = authority = fleet.snapshot('C10-pending-MPU-object-before', quiescent=True) + pending_before = pending_snapshot(fleet) + require(len(pending_before) == (4 if topology == 'H1' else 16) and all(pending_before.values()), + 'C10 did not witness actual pending-MPU files on every original root') + fleet.start() + else: + capture(fleet, original, bucket, 'old') + fleet.start() + def delete_ack(response): + require(response.get('VersionId') == doomed['versionId'] and response.get('DeleteMarker') in {None, False} + and response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 204, 'C10 exact-version DELETE ACK mismatch') + call('delete_object', {'Bucket': bucket, 'Key': KEY, 'VersionId': doomed['versionId']}, check=delete_ack) + deleted.append(doomed['versionId']); expected.remove(doomed) + if branch == 'shared-version-reference': + marker = call('delete_object', {'Bucket': bucket, 'Key': KEY}, check=marker_ack) + for entry in expected: + if entry['key'] == KEY: entry['isLatest'] = False + expected.append({'key': KEY, 'versionId': marker['VersionId'], 'deleteMarker': True, 'isLatest': True}) + fleet.record('C10-post-delete-all-S3-authority', verify_seed(expected, probe.inventory(bucket))) + fleet.stop() + authority = fleet.snapshot('C10-post-delete-all-physical-authority', quiescent=True) + require(all(doomed['versionId'] not in values for values in references(authority, bucket, KEY).values()) + and not any(identity[2].startswith(bucket + '/' + KEY + '/' + data_dir + '/') for identity in files(authority)), + 'C10 DELETE did not remove the exact stale reference/payload before controlled replay') + recovery_snapshot = authority + arm_all(fleet, authority, bucket, 'authority') + fleet.record('C10-controlled-single-root-replay', {'node': target[0], **fleet.remote(target[0], 'orphan-install', + disk=target[1], bucket=bucket, key=KEY, dataDir=data_dir, + mode='object' if branch.startswith('remove-true-') else 'data')}) + before = fleet.snapshot('C10-fault-before-GET', quiescent=True) + present = {identity[:2] for identity in files(before) if identity[2].startswith(bucket + '/' + KEY + '/' + data_dir + '/')} + require(present == {target}, 'C10 replay is not confined to one exact original member') + fleet.start() + if branch == 'remove-true-after-grace': + while time.time() < doomed_time + grace + 4: time.sleep(min(1, max(0, doomed_time + grace + 4 - time.time()))) + age = time.time() - doomed_time + require(branch != 'remove-true-before-grace' or 0 <= age < grace - 5, 'C10 missed the real pre-grace window') + require(branch != 'remove-true-after-grace' or age >= grace + 3, 'C10 real grace period has not elapsed') + fleet.record('C10-real-age-trigger', {'ageSeconds': age, 'serverLastModifiedEpoch': doomed_time, 'graceSeconds': grace, + 'globalClockChanged': False, 'remove': branch != 'remove-false'}) + from_options = {'recursive': branch != 'remove-false', 'remove': branch != 'remove-false'} + try: from .healthy import BASE_OPTIONS + except ImportError: from healthy import BASE_OPTIONS + options = {**BASE_OPTIONS, 'scanMode': 2, **from_options} + status = run_task(probe, fleet, {'bucket': bucket, 'key': KEY if branch == 'remove-false' else ''}, 2, active, + status_endpoints=tuple(binding['endpoints'])[1:], option_overrides=from_options) + idle_fleet(fleet, binding) + fleet.stop() + after = fleet.snapshot('C10-product-physical-before-target-GET', quiescent=True) + result['physical'] = physical_safety(before, after, authority, bucket, expected, branch, + doomed['versionId'], data_dir, target) + result['terminal'] = terminal_safety(status, branch, bucket, options) + # Product S3 behavior must be observed before fixture rescue can hide + # lost references, resurrected versions or changed current semantics. + fleet.start() + verify_public(fleet, binding, bucket, expected, deleted, 'C10-product-before-rescue') + fleet.stop() + if branch == 'pending-mpu': + require(pending_before == pending_snapshot(fleet), 'C10 Heal mutated pending-MPU bytes') + fleet.start() + for endpoint in binding['endpoints']: + observed = fleet.record('C10-all-peer-list-parts', Probe(binding, endpoint).s3('list_parts', + {'Bucket': bucket, 'Key': KEY, 'UploadId': upload})) + response = observed.get('response') or {} + require(observed['acknowledged'] and response.get('IsTruncated', False) is False + and [{'number': row['PartNumber'], 'size': row['Size'], 'etag': row['ETag']} for row in response.get('Parts', [])] + == [{'number': 1, 'size': SIZE, 'etag': part['ETag']}], 'C10 pending UploadId/part ACK changed') + completed = call('complete_multipart_upload', {'Bucket': bucket, 'Key': KEY, 'UploadId': upload, + 'MultipartUpload': {'Parts': [{'PartNumber': 1, 'ETag': part['ETag']}]}}, check=unique_version) + for entry in expected: + if entry['key'] == KEY: entry['isLatest'] = False + expected.append({'key': KEY, 'versionId': completed['VersionId'], 'isLatest': True, 'deleteMarker': False, + 'payload': {'size': SIZE, 'sha256': hashlib.sha256(pending_body).hexdigest()}}) + result['pendingMPUPhysicalAndListPartsPreserved'] = True + else: + # Preserve the product result above, then undo only the controlled + # replay using all original post-DELETE authorities before GET. + recover_all(fleet) + recovered = fleet.snapshot('C10-restored-last-DELETE-before-GET', quiescent=True) + require(physical_content(recovered) == physical_content(authority), 'C10 restoration changed last DELETE authority') + recovery_snapshot = None + fleet.start() + fleet.stop() + restored = fleet.snapshot('C10-final-current-ACK-physical', quiescent=True) + mapping = {(bucket, entry['key'], entry['versionId']): map_target(restored, bucket, entry['key'], 'data', 1, + entry['versionId'], latest_required=entry['isLatest']) for entry in expected if not entry['deleteMarker']} + result['streaming'] = verify_parts(fleet, mapping, 'C10-all-live-ACK-streaming') + fleet.start() + verify_public(fleet, binding, bucket, expected, deleted, 'C10-all-peer-final-S3') + result.update(verdict='PASS', postDeleteAuthorityRestored=True, backupRestorationIsProductRepair=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict='FAIL', failure={'type': type(exc).__name__, 'message': str(exc)}) + finally: + for path, token in active: + try: fleet.record('C10-stop-unresolved-task', probe.admin(path, [('clientToken', token), ('forceStop', 'true')], b'')) + except Exception as exc: fleet.record('C10-stop-error', {'error': str(exc)}) + if pending_mutation is not None: + try: fleet.stop() + except Exception as exc: fleet.record('C10-retain-stop-error', {'error': str(exc)}) + fleet.record('C10-uncertain-ACK-retained', {'state': 'DIRTY', 'request': pending_mutation, + 'namespacesRetained': True, 'oldAuthorityRestorationProhibited': True}) + clean = False + else: + recovery_ok = True + if fleet.orphan_faults: + try: + fleet.stop() + for node in fleet.nodes: + fleet.record('C10-before-rescue-original-logs', {'node': node, **fleet.remote(node, 'logs')}) + recover_all(fleet) + restored = fleet.snapshot('C10-failure-restored-authority', quiescent=True) + require(recovery_snapshot is not None and physical_content(restored) == physical_content(recovery_snapshot), + 'C10 failed-case restoration differs from the last known ACK authority') + fleet.start() + verify_public(fleet, binding, bucket, expected, deleted, 'C10-failure-rescued-S3') + except Exception as exc: + fleet.record('C10-recovery-DIRTY', {'state': 'DIRTY', 'namespacesRetained': True, 'error': str(exc)}) + try: fleet.stop() + except Exception as stop_error: fleet.record('C10-recovery-stop-error', {'error': str(stop_error)}) + recovery_ok = False + clean = fleet.cleanup() if recovery_ok else False + result['cleanup'] = 'PASS' if clean else 'FAIL' + fleet.record('branch-result', result) + return result + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f'C10 interrupted by signal {signum}') + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ('candidate', 'rc-bin', 'evidence', 'tool-directory'): + parser.add_argument('--' + name, type=Path, required=True) + parser.add_argument('--shared-directory', type=Path, default=os.environ.get('VALIDATION_HEAL_SHARED_DIRECTORY')) + parser.add_argument('--case-id', choices=[CASE_ID], required=True) + parser.add_argument('--testing-commit', required=True) + parser.add_argument('--binary-sha', required=True) + args = parser.parse_args() + require(re.fullmatch(r'[0-9a-f]{64}', args.binary_sha) and re.fullmatch(r'[1-9][0-9]*', os.environ.get('GITHUB_RUN_ID', '')), + 'C10 requires immutable workflow/binary identities') + require(select(CASE_ID)[0]['variants'] == list(BRANCHES), 'C10 omitted a source branch') + require(args.shared_directory is not None, 'C10 requires its successful exact-candidate encoder before deployment') + save(args.evidence / 'shared-preflight.json', verify_shared(args.shared_directory, json.loads(args.candidate.read_text()))) + nodes = os.environ.get('RUSTFS_NODES', 'vm000 vm001 vm002 vm003').split() + require(len(nodes) >= 4 and len(nodes) == len(set(nodes)) and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in nodes), + 'C10 requires four distinct current Heal nodes') + results = [] + try: + for topology in ('H1', 'H2'): + for branch in BRANCHES: + results.append(run_branch(args, topology, nodes[:1] if topology == 'H1' else nodes[:4], branch)) + if results[-1]['cleanup'] != 'PASS': break + if results[-1]['cleanup'] != 'PASS': break + finally: + complete = len(results) == 10 and all(row['verdict'] == row['cleanup'] == 'PASS' for row in results) + save(args.evidence / 'healthy-summary.json', {'caseId': CASE_ID, 'results': results, 'complete': complete, + 'cleanup': 'PASS' if results and all(row['cleanup'] == 'PASS' for row in results) else 'FAIL'}) + require(complete, 'C10 requires all five actual H1/H2 branches and complete original-member cleanup') + + +if __name__ == '__main__': + try: main() + except Exception as exc: + print(f'C10 orphan validation failed: {exc}', file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/orphan_fault.py b/scripts/validation_heal/orphan_fault.py new file mode 100644 index 00000000..bf203099 --- /dev/null +++ b/scripts/validation_heal/orphan_fault.py @@ -0,0 +1,237 @@ +"""C10 stopped descriptor archives, shared conversion and post-delete recovery.""" + +import json +import os +import re +import stat +import subprocess +import uuid + + +def require(value, message): + if not value: + raise ValueError(message) + + +def scope(owner, request): + require(owner.get('runtimeProfile') in {'orphan-preserve', 'orphan-before', 'orphan-after'} + and request.get('bucket') == owner['bucket'] and request.get('key') == 'orphan.bin', + 'C10 escaped its registered runtime or exact owned object') + require(type(request.get('disk')) is int and 0 <= request['disk'] < 4, 'C10 original disk absent') + return [owner['bucket'], 'orphan.bin'] + + +def scene(root, owner, request, fault, generation): + parts = scope(owner, request) + with fault.parent(root, ['data', parts[0]]) as bucket: + if parts[1] not in os.listdir(bucket): + return False, {}, [] + with fault.parent(bucket, [parts[1]]) as obj: + values, files = generation.tree(obj, fault, allow_sparse=True) + return True, values, files + + +def identity(root, owner, request, fault): + return {'schemaVersion': 1, 'runId': owner['runId'], 'disk': request['disk'], + 'bucket': owner['bucket'], 'key': 'orphan.bin', 'mountId': fault.mount_id(root), + 'device': os.fstat(root).st_dev, 'binarySha256': owner['binarySha256']} + + +def capture(root, owner, request, fault, generation): + label = request['generation'] + require(label in {'seed', 'old', 'authority'}, 'unregistered C10 archive') + present, values, files = scene(root, owner, request, fault, generation) + require(files == request['expectedFiles'] and present == request['expectedPresent'], + 'C10 archive differs from its complete stopped physical witness') + record = {**identity(root, owner, request, fault), 'generation': label, 'present': present, + 'files': files, 'treeSha256': generation.tree_sha(files, fault), 'productRepairClaimed': False} + with fault.parent(root, ['quarantine']) as quarantine: + os.mkdir('C10-' + label, 0o700, dir_fd=quarantine) + os.fsync(quarantine) + with fault.parent(quarantine, ['C10-' + label]) as saved: + fault.new_file(saved, 'manifest.json', fault.encode(record)) + os.mkdir('object', 0o700, dir_fd=saved) + with fault.parent(saved, ['object']) as obj: + generation.copy_into(obj, values, fault) + require(generation.tree(obj, fault, allow_sparse=True)[1] == files, 'C10 archive incomplete') + fault.new_file(saved, 'complete.json', fault.encode(record)) + return record + + +def captured(root, owner, request, label, fault, generation): + scope(owner, request) + require(label in {'seed', 'old', 'authority'}, 'unregistered C10 saved generation') + with fault.parent(root, ['quarantine', 'C10-' + label]) as saved: + record = json.loads(fault.read_file(saved, 'complete.json', 65536)) + require(record == json.loads(fault.read_file(saved, 'manifest.json', 65536)) + and all(record.get(key) == value for key, value in identity(root, owner, request, fault).items()) + and record['generation'] == label and type(record['present']) is bool, + 'C10 archive candidate, ownership, device or mount changed') + with fault.parent(saved, ['object']) as obj: + values, files = generation.tree(obj, fault, allow_sparse=True) + require(files == record['files'] and generation.tree_sha(files, fault) == record['treeSha256'] + and (record['present'] or not files), 'C10 archived bytes differ from authority') + return values, record + + +def arm(root, owner, request, fault, generation): + label = request['generation'] + require(label in {'seed', 'authority'}, 'C10 recovery cannot restore a deleted old generation') + _, record = captured(root, owner, request, label, fault, generation) + present, _, files = scene(root, owner, request, fault, generation) + require(present == record['present'] and files == record['files'], 'C10 authority drifted before arming') + intent = {**identity(root, owner, request, fault), 'restoreGeneration': label, + 'originalTreeSha256': record['treeSha256'], 'originalPresent': record['present'], 'productRepairClaimed': False} + with fault.parent(root, ['quarantine']) as quarantine: + fault.new_file(quarantine, 'C10-' + label + '-intent.json', fault.encode(intent)) + return {**intent, 'state': 'ARMED'} + + +def intent(root, owner, request, label, fault, generation): + values, record = captured(root, owner, request, label, fault, generation) + with fault.parent(root, ['quarantine']) as quarantine: + value = json.loads(fault.read_file(quarantine, 'C10-' + label + '-intent.json', 65536)) + require(all(value.get(key) == item for key, item in identity(root, owner, request, fault).items()) + and value['restoreGeneration'] == label and value['originalTreeSha256'] == record['treeSha256'] + and value['originalPresent'] == record['present'] + and 'C10-' + label + '-disarmed.json' not in os.listdir(quarantine), + 'C10 recovery authority is absent, foreign or already retired by later ACKs') + return values, record, value + + +def install(root, owner, request, fault, generation): + parts = scope(owner, request) + _, authority, _ = intent(root, owner, request, 'authority', fault, generation) + old, _ = captured(root, owner, request, 'old', fault, generation) + present, current, files = scene(root, owner, request, fault, generation) + require(present == authority['present'] and files == authority['files'], 'C10 last DELETE authority drifted') + directory = request['dataDir'] + require(str(uuid.UUID(directory)) == directory and uuid.UUID(directory).int != 0, + 'C10 stale payload has no exact canonical data directory') + stale = {name: raw for name, raw in old.items() if name.startswith(directory + '/')} + require(stale and not any(name.startswith(directory + '/') for name in current), + 'C10 replay would overwrite a live physical reference') + mode = request['mode'] + require(mode in {'object', 'data'}, 'unregistered C10 replay mode') + values = old if mode == 'object' else {**current, **stale} + with fault.parent(root, ['quarantine']) as quarantine, fault.parent(root, ['data', parts[0]]) as bucket: + fault.new_file(quarantine, 'C10-replay-intent.json', fault.encode({'mode': mode, 'dataDir': directory, + 'restoreGeneration': 'authority', 'postDeleteTreeSha256': authority['treeSha256']})) + require('C10-pre-replay' not in os.listdir(quarantine), 'C10 retained scene already exists') + if present: + os.rename(parts[1], 'C10-pre-replay', src_dir_fd=bucket, dst_dir_fd=quarantine) + else: + os.mkdir('C10-pre-replay', 0o700, dir_fd=quarantine) + os.fsync(quarantine); os.fsync(bucket) + os.mkdir(parts[1], 0o700, dir_fd=bucket) + with fault.parent(bucket, [parts[1]]) as obj: + generation.copy_into(obj, values, fault) + actual, manifest = generation.tree(obj, fault, allow_sparse=True) + require(actual == values, 'C10 replay did not preserve all original sidecars and parts') + fault.new_file(quarantine, 'C10-replay-applied.json', fault.encode({'mode': mode, 'dataDir': directory, + 'files': manifest, 'state': 'APPLIED', 'postDeleteAuthorityRetained': True})) + return {'state': 'APPLIED', 'mode': mode, 'dataDir': directory, 'files': manifest} + + +def recover(root, owner, request, fault, generation): + parts = scope(owner, request) + label = request['generation'] + values, authority, original_intent = intent(root, owner, request, label, fault, generation) + name = 'C10-' + label + '-failed-current' + with fault.parent(root, ['quarantine']) as quarantine, fault.parent(root, ['data', parts[0]]) as bucket: + if name in os.listdir(quarantine): + present, _, files = scene(root, owner, request, fault, generation) + require(present == authority['present'] and files == authority['files'], + 'C10 interrupted recovery remains DIRTY; never discard a partial restore') + else: + if parts[1] in os.listdir(bucket): + with fault.parent(bucket, [parts[1]]) as obj: + require(os.fstat(obj).st_uid == 0, 'C10 live scene no longer root owned') + os.rename(parts[1], name, src_dir_fd=bucket, dst_dir_fd=quarantine) + else: + os.mkdir(name, 0o700, dir_fd=quarantine) + os.fsync(bucket); os.fsync(quarantine) + if authority['present']: + os.mkdir(parts[1], 0o700, dir_fd=bucket) + with fault.parent(bucket, [parts[1]]) as obj: + generation.copy_into(obj, values, fault) + os.fsync(bucket) + present, _, files = scene(root, owner, request, fault, generation) + require(present == authority['present'] and files == authority['files'], 'C10 authority restoration incomplete') + receipt = {**original_intent, 'state': 'CLEAR', 'lastACKAuthorityRestored': True, 'failedScenePreserved': True} + receipt_name = 'C10-' + label + '-recovered.json' + if receipt_name not in os.listdir(quarantine): fault.new_file(quarantine, receipt_name, fault.encode(receipt)) + return receipt + + +def disarm(root, owner, request, fault, generation): + label = request['generation'] + require(label == 'seed', 'only a verified shared seed representation may retire recovery before a later ACK') + _, _, original = intent(root, owner, request, label, fault, generation) + present, _, files = scene(root, owner, request, fault, generation) + require(present and files == request['expectedFiles'], 'C10 shared fixture drifted before retiring seed rescue') + with fault.parent(root, ['quarantine']) as quarantine: + fault.new_file(quarantine, 'C10-seed-disarmed.json', fault.encode({**original, + 'state': 'RETIRED', 'verifiedSharedFiles': files, 'laterACKMayProceed': True})) + return {'state': 'RETIRED', 'laterACKMayProceed': True} + + +def convert(root, owner, request, fault, generation, tool): + scope(owner, request) + _, saved, _ = intent(root, owner, request, 'seed', fault, generation) + present, values, files = scene(root, owner, request, fault, generation) + require(present and files == saved['files'], 'C10 shared seed changed before codec conversion') + source, destination = request['sourceDataDir'], request['destinationDataDir'] + for directory in (source, destination): + require(str(uuid.UUID(directory)) == directory and uuid.UUID(directory).int != 0, + 'C10 shared directory is not canonical') + left = {name.removeprefix(source + '/'): raw for name, raw in values.items() + if name.startswith(source + '/') and re.fullmatch(r'part\.[1-9][0-9]*', name.split('/')[-1])} + right = {name.removeprefix(destination + '/'): raw for name, raw in values.items() + if name.startswith(destination + '/') and re.fullmatch(r'part\.[1-9][0-9]*', name.split('/')[-1])} + require(source != destination and left and left == right, + 'C10 independently ACKed generations do not contain identical physical payload parts') + result = subprocess.run([str(tool), owner['bucket'], 'orphan.bin', request['sourceVersion'], request['destinationVersion']], + input=values['xl.meta'], capture_output=True, timeout=60) + require(result.returncode == 0 and 0 < len(result.stdout) <= fault.LIMIT, + 'candidate codec refused the exact shared fixture') + receipt = json.loads(result.stderr) + require(receipt.get('sourceVersion') == request['sourceVersion'] and receipt.get('destinationVersion') == request['destinationVersion'] + and receipt.get('sharedDataDir') == source and receipt.get('retiredDataDir') == destination + and receipt.get('filesystemMutationPerformed') is False and receipt.get('physicalByteEqualityClaimed') is False, + 'C10 codec receipt lost its original ACK identity') + with fault.parent(root, ['quarantine']) as quarantine, fault.parent(root, ['data', owner['bucket'], 'orphan.bin']) as obj: + fault.new_file(quarantine, 'C10-shared-intent.json', fault.encode(receipt)) + fault.new_file(obj, 'C10-converted-meta', result.stdout) + os.rename(destination, 'C10-retired-shared-data', src_dir_fd=obj, dst_dir_fd=quarantine) + os.rename('xl.meta', 'C10-original-shared-meta', src_dir_fd=obj, dst_dir_fd=quarantine) + os.rename('C10-converted-meta', 'xl.meta', src_dir_fd=obj, dst_dir_fd=obj) + os.fsync(obj); os.fsync(quarantine) + expected = {name: raw for name, raw in values.items() if not name.startswith(destination + '/')} + expected['xl.meta'] = result.stdout + require(generation.tree(obj, fault, allow_sparse=True)[0] == expected, + 'C10 shared conversion changed unrelated bytes or lost source commitments') + fault.new_file(quarantine, 'C10-shared-applied.json', fault.encode(receipt)) + return {**receipt, 'state': 'APPLIED', 'physicalByteEqualityVerified': True, 'retiredBytesPreserved': True} + + +def multipart(root, fault): + """Read the entire isolated run's MPU tree without following aliases.""" + files, total = [], 0 + def visit(fd, prefix): + nonlocal total + require(len(prefix) <= 8, 'C10 MPU nesting exceeds its fixture budget') + for name in sorted(os.listdir(fd)): + require(re.fullmatch(r'[A-Za-z0-9_.-]{1,256}', name) and name not in {'.', '..'}, 'C10 MPU leaf is unreviewed') + info = os.stat(name, dir_fd=fd, follow_symlinks=False) + if stat.S_ISDIR(info.st_mode): + with fault.parent(fd, [name]) as child: visit(child, prefix + [name]) + else: + raw = fault.read_file(fd, name) + total += len(raw) + require(len(files) < 512 and total <= 32 * 1024**2, 'C10 MPU census exceeds complete evidence budget') + files.append({'path': '/'.join(prefix + [name]), 'size': len(raw), 'sha256': fault.digest(raw)}) + with fault.parent(root, ['data', '.rustfs.sys']) as system: + if 'multipart' in os.listdir(system): + with fault.parent(system, ['multipart']) as pending: visit(pending, []) + return {'readOnly': True, 'files': files, 'totalBytes': total} diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 3f243473..89ac82bf 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -41,6 +41,7 @@ case "${case_id}" in heal.network-partition-and-lock-quorum) driver=network.py ;; heal.multi-pool-target-isolation) driver=multi_pool.py ;; heal.conflicting-subquorum-authority) driver=authority_runner.py ;; + heal.dangling-and-orphan-safety) driver=orphan.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal_orphan.py b/test_validation_heal_orphan.py index 148a12f1..61b3d5cb 100644 --- a/test_validation_heal_orphan.py +++ b/test_validation_heal_orphan.py @@ -1,11 +1,22 @@ """C10 controlled-codec build identity and preservation guards.""" +import copy import json +import os +import shutil +import subprocess +import sys +import types +import uuid +from unittest.mock import patch from pathlib import Path import tempfile import unittest from scripts.validation_heal import shared_fixture as shared +from scripts.validation_heal import orphan, orphan_fault as backend, generation_fault as generation, heal_fault as fault +from scripts.validation_heal.matrix import select, AUTOMATIC_CASES +from scripts.validation_heal.healthy import BASE_OPTIONS, Fleet class SharedFixtureArtifactTests(unittest.TestCase): @@ -46,6 +57,7 @@ def test_foreign_source_artifact_role_commit_and_build_attempt_cannot_be_install (root / 'heal-shared-fixture.json').write_text(json.dumps(manifest)) with self.assertRaises(ValueError): shared.verify(root, {'source_sha': shared.CANDIDATE}) + def test_zero_filtered_ignored_or_incomplete_codec_tests_do_not_certify_a_build(self): for log in ('running 0 tests\ntest result: ok. 0 passed; 0 failed; 0 ignored;\n', 'running 3 tests\ntest result: ok. 2 passed; 0 failed; 1 ignored;\n', @@ -68,5 +80,319 @@ def test_changed_executable_or_failed_remote_build_is_refused(self): with self.assertRaises(ValueError): shared.verify(root, {'source_sha': shared.CANDIDATE}) + + +class OrphanDescriptorTests(unittest.TestCase): + bucket = 'rv-heal-0123456789abcdef' + old_dir, live_dir = str(uuid.UUID(int=11)), str(uuid.UUID(int=12)) + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name).resolve() + self.object = self.root / 'data' / self.bucket / orphan.KEY + self.object.mkdir(parents=True) + (self.root / 'quarantine').mkdir() + self.owner = {'runId': 'gh-123-1-h1', 'bucket': self.bucket, 'runtimeProfile': 'orphan-preserve', 'binarySha256': 'a' * 64} + self.request = {'disk': 0, 'bucket': self.bucket, 'key': orphan.KEY} + self.mount = patch.object(fault, 'mount_id', return_value=7); self.mount.start() + self.seed(self.old_dir, b'old') + + def tearDown(self): + self.mount.stop(); self.temporary.cleanup() + + def seed(self, directory, raw): + (self.object / 'xl.meta').write_bytes(b'XL2 ' + raw) + (self.object / directory).mkdir(exist_ok=True) + (self.object / directory / 'part.1').write_bytes(raw * 256) + (self.object / directory / ('part.1.integrity.' + str(uuid.UUID(int=77)))).write_bytes(raw + b'-commitment') + + def invoke(self, name, **kwargs): + with fault.directory(self.root) as root: + return getattr(backend, name)(root, self.owner, {**self.request, **kwargs}, fault, generation) + + def capture(self, label): + with fault.directory(self.root) as root: + present, _, files = backend.scene(root, self.owner, self.request, fault, generation) + return self.invoke('capture', generation=label, expectedFiles=files, expectedPresent=present) + + def authority(self, absent=False): + self.capture('old') + shutil.rmtree(self.object) + if not absent: + self.object.mkdir(); self.seed(self.live_dir, b'live') + saved = self.capture('authority') + self.invoke('arm', generation='authority') + return saved + + def recover(self, label='authority'): + actual = os.fstat + def root_owned(fd): + info = actual(fd) + fields = {name: getattr(info, name) for name in dir(info) if name.startswith('st_')} + return types.SimpleNamespace(**{**fields, 'st_uid': 0}) + with patch.object(backend.os, 'fstat', side_effect=root_owned): + return self.invoke('recover', generation=label) + + def test_post_delete_recovery_never_reinstates_old_version_and_preserves_sidecars(self): + saved = self.authority() + self.invoke('install', mode='object', dataDir=self.old_dir) + self.assertEqual((self.object / 'xl.meta').read_bytes(), b'XL2 old') + receipt = self.recover() + self.assertTrue(receipt['lastACKAuthorityRestored']) + self.assertEqual(receipt['originalTreeSha256'], saved['treeSha256']) + self.assertEqual((self.object / 'xl.meta').read_bytes(), b'XL2 live') + self.assertFalse((self.object / self.old_dir).exists()) + self.assertTrue((self.root / 'quarantine/C10-authority-failed-current' / self.old_dir).is_dir()) + self.assertEqual(receipt, self.recover()) + + def test_absent_post_delete_authority_remains_absent_after_metadata_less_replay(self): + self.authority(absent=True) + self.invoke('install', mode='data', dataDir=self.old_dir) + self.assertFalse((self.object / 'xl.meta').exists()) + self.assertTrue((self.object / self.old_dir).exists()) + self.recover() + self.assertFalse(self.object.exists()) + self.assertTrue((self.root / 'quarantine/C10-authority-failed-current' / self.old_dir).is_dir()) + + def test_live_reference_and_commitment_remain_exact_during_data_only_replay(self): + self.authority() + original = (self.object / self.live_dir / 'part.1').read_bytes() + self.invoke('install', mode='data', dataDir=self.old_dir) + self.assertEqual((self.object / 'xl.meta').read_bytes(), b'XL2 live') + self.assertEqual((self.object / self.live_dir / 'part.1').read_bytes(), original) + self.assertEqual((self.object / self.old_dir / ('part.1.integrity.' + str(uuid.UUID(int=77)))).read_bytes(), b'old-commitment') + + def test_old_generation_is_never_a_recovery_authority_and_foreign_identity_is_refused(self): + self.authority() + with self.assertRaises(ValueError): self.invoke('arm', generation='old') + for field, value in (('key', '../escape'), ('bucket', 'foreign'), ('disk', 4)): + with self.subTest(field=field), self.assertRaises(ValueError): self.invoke('capture', **{field: value}, + generation='authority', expectedFiles=[], expectedPresent=False) + with patch.object(fault, 'mount_id', return_value=8), self.assertRaises(ValueError): self.recover() + self.owner['binarySha256'] = 'b' * 64 + with self.assertRaises(ValueError): self.recover() + + def test_partial_restore_is_retained_and_never_certified_clear(self): + self.authority() + self.invoke('install', mode='object', dataDir=self.old_dir) + self.object.rename(self.root / 'quarantine/C10-authority-failed-current') + self.object.mkdir(); (self.object / 'xl.meta').write_bytes(b'partial') + with self.assertRaises(ValueError): self.recover() + self.assertFalse((self.root / 'quarantine/C10-authority-recovered.json').exists()) + self.assertTrue((self.root / 'quarantine/C10-authority-failed-current' / self.old_dir).exists()) + + def test_sparse_mode_does_not_relax_the_original_c07_generation_contract(self): + shutil.rmtree(self.object / self.old_dir) + with fault.directory(self.object) as fd: + with self.assertRaises(ValueError): generation.tree(fd, fault) + self.assertEqual(set(generation.tree(fd, fault, allow_sparse=True)[0]), {'xl.meta'}) + (self.object / 'xl.meta').unlink() + with fault.directory(self.object) as fd: + with self.assertRaises(ValueError): generation.tree(fd, fault) + self.assertEqual(generation.tree(fd, fault, allow_sparse=True), ({}, [])) + + def test_symlinks_hardlinks_and_payload_less_commitments_are_not_archived(self): + part = self.object / self.old_dir / 'part.1' + for kind in ('symlink', 'hardlink', 'missing-part'): + raw = part.read_bytes() + if kind == 'symlink': + part.unlink(); part.symlink_to(self.root / 'outside') + elif kind == 'hardlink': os.link(part, self.root / 'outside') + else: part.unlink() + with self.subTest(kind=kind), self.assertRaises((ValueError, OSError)): self.capture('seed') + if kind == 'hardlink': (self.root / 'outside').unlink() + else: + if part.is_symlink(): part.unlink() + part.write_bytes(raw) + + def test_encoder_refuses_nonidentical_physical_parts_before_it_can_execute(self): + self.seed(self.live_dir, b'different') + self.capture('seed'); self.invoke('arm', generation='seed') + with fault.directory(self.root) as root, patch.object(backend.subprocess, 'run') as execute, self.assertRaisesRegex(ValueError, 'identical physical'): + backend.convert(root, self.owner, {**self.request, 'sourceDataDir': self.old_dir, 'destinationDataDir': self.live_dir, + 'sourceVersion': str(uuid.UUID(int=1)), 'destinationVersion': str(uuid.UUID(int=2))}, fault, generation, self.root / 'unavailable') + execute.assert_not_called() + + def test_retired_shared_seed_cannot_be_recovered_after_later_ack_is_allowed(self): + self.capture('seed'); self.invoke('arm', generation='seed') + with fault.directory(self.root) as root: + manifest = backend.scene(root, self.owner, self.request, fault, generation)[2] + self.invoke('disarm', generation='seed', expectedFiles=manifest) + with self.assertRaisesRegex(ValueError, 'retired'): self.recover('seed') + + def test_mpu_census_is_complete_read_only_and_refuses_aliases(self): + folder = self.root / 'data/.rustfs.sys/multipart/hash/upload'; folder.mkdir(parents=True) + (folder / 'part.1').write_bytes(b'pending-part') + (folder / 'xl.meta').write_bytes(b'pending-metadata') + with fault.directory(self.root) as root: + result = backend.multipart(root, fault) + self.assertTrue(result['readOnly']); self.assertEqual(len(result['files']), 2) + self.assertEqual(result, backend.multipart(root, fault)) + (folder / 'alias').symlink_to(folder / 'part.1') + with self.assertRaises(OSError): backend.multipart(root, fault) + + +class OrphanOracleTests(unittest.TestCase): + bucket = 'rv-heal-0123456789abcdef' + def status(self, branch): + status = {'summary': 'finished', 'truncated': False, 'settings': {**BASE_OPTIONS, 'scanMode': 2, + 'recursive': branch != 'remove-false', 'remove': branch != 'remove-false'}, 'outcome': { + 'execution': {'state': 'completed'}, 'coverage': 'complete', 'objectsTruncated': False, + 'counters': {'processed': 1, 'healed': 0, 'failed': 0, 'unknown': 0}, 'objects': []}} + if branch == 'remove-false': + status['detail'] = 'Object not found: ' + self.bucket + '/' + orphan.KEY + status['outcome']['counters']['failed'] = 1 + status['outcome']['execution']['state'] = 'completed_with_errors' + if branch == 'remove-true-before-grace': + status['outcome']['objects'] = [{'identity': {'bucket': self.bucket, 'object': orphan.KEY}, + 'disposition': {'state': 'deferred', 'details': {'reason': 'dangling_delete_grace'}}}] + return status + + def test_all_exact_terminal_branches_and_typed_grace_are_required(self): + for branch in orphan.BRANCHES: + status = self.status(branch) + orphan.terminal_safety(status, branch, self.bucket, status['settings']) + for mutation in ('reason', 'key', 'partial', 'unknown', 'truncated', 'settings'): + status = self.status('remove-true-before-grace'); options = copy.deepcopy(status['settings']) + if mutation == 'reason': status['outcome']['objects'][0]['disposition']['details']['reason'] = 'retry' + elif mutation == 'key': status['outcome']['objects'][0]['identity']['object'] = 'other' + elif mutation == 'partial': status['outcome']['coverage'] = 'partial' + elif mutation == 'unknown': status['outcome']['counters']['unknown'] = 1 + elif mutation == 'truncated': status['outcome']['objectsTruncated'] = True + else: status['settings']['remove'] = False + with self.subTest(mutation=mutation), self.assertRaises(ValueError): orphan.terminal_safety(status, 'remove-true-before-grace', self.bucket, options) + + def test_source_case_has_all_five_branches_on_both_required_topologies(self): + source = select(orphan.CASE_ID)[0] + self.assertEqual(source['variants'], list(orphan.BRANCHES)) + self.assertEqual(source['contract']['probes']['topology'], 'H1,H2') + self.assertIn(orphan.CASE_ID, AUTOMATIC_CASES) + + def scene(self, branch): + live, stale = str(uuid.UUID(int=12)), str(uuid.UUID(int=11)) + version, doomed = str(uuid.UUID(int=1)), str(uuid.UUID(int=2)) + expected = [] if branch == 'remove-false' else [{'key': orphan.KEY, 'versionId': version, 'deleteMarker': False}] + versions = [] if not expected else [{'versionId': version, 'deleted': False, 'dataDir': live, 'inline': None}] + if branch == 'shared-version-reference': + shared_version = str(uuid.UUID(int=3)) + versions.append({'versionId': shared_version, 'deleted': False, 'dataDir': live, 'inline': None}) + expected.append({'key': orphan.KEY, 'versionId': shared_version, 'deleteMarker': False}) + authority = {'node1': {'formats': [{'disk': disk, 'sha256': str(disk)} for disk in range(4)], 'files': []}} + for disk in range(4): + authority['node1']['files'].append({'disk': disk, 'path': self.bucket + '/control.bin/xl.meta', 'sha256': 'control', 'size': 7}) + if versions: + authority['node1']['files'] += [ + {'disk': disk, 'path': self.bucket + '/' + orphan.KEY + '/xl.meta', 'sha256': 'authority', 'size': 7, + 'metadata': {'bucket': self.bucket, 'key': orphan.KEY, 'versions': copy.deepcopy(versions)}}, + {'disk': disk, 'path': self.bucket + '/' + orphan.KEY + '/' + live + '/part.1', 'sha256': 'live-part', 'size': 256}, + {'disk': disk, 'path': self.bucket + '/' + orphan.KEY + '/' + live + '/part.1.integrity.' + str(uuid.UUID(int=77)), 'sha256': 'live-commitment', 'size': 100}] + before = copy.deepcopy(authority) + if branch != 'pending-mpu': + before['node1']['files'].append({'disk': 0, 'path': self.bucket + '/' + orphan.KEY + '/' + stale + '/part.1', + 'sha256': 'stale-part', 'size': 256}) + if branch.startswith('remove-true-'): + meta = next(row for row in before['node1']['files'] if row['disk'] == 0 and row['path'].endswith(orphan.KEY + '/xl.meta')) + meta['metadata']['versions'].append({'versionId': doomed, 'deleted': False, 'dataDir': stale, 'inline': None}) + meta['sha256'] = 'old-authority' + after = copy.deepcopy(before if branch in {'remove-false', 'remove-true-before-grace', 'pending-mpu'} else authority) + return before, after, authority, expected, doomed, stale + + def test_actual_cleanup_preservation_and_all_original_controls_are_required(self): + for branch in orphan.BRANCHES: + before, after, authority, expected, doomed, stale = self.scene(branch) + orphan.physical_safety(before, after, authority, self.bucket, expected, branch, doomed, stale, ('node1', 0)) + for kind in ('control', 'format', 'member', 'sidecar', 'orphan'): + changed = copy.deepcopy(after) + if kind == 'control': changed['node1']['files'][0]['sha256'] = 'modified' + elif kind == 'format': changed['node1']['formats'][0]['sha256'] = 'modified' + elif kind == 'member': changed['node1']['formats'].pop() + elif kind == 'sidecar': + if branch == 'remove-false': continue + next(row for row in changed['node1']['files'] if '.integrity.' in row['path'])['sha256'] = 'modified' + else: + if branch == 'pending-mpu': continue + if branch in {'remove-true-after-grace', 'shared-version-reference'}: + changed['node1']['files'].append(copy.deepcopy(before['node1']['files'][-1])) + else: changed['node1']['files'] = [row for row in changed['node1']['files'] if '/' + stale + '/' not in row['path']] + with self.subTest(branch=branch, kind=kind), self.assertRaises(ValueError): + orphan.physical_safety(before, changed, authority, self.bucket, expected, branch, doomed, stale, ('node1', 0)) + + def test_partial_original_member_recovery_blocks_cleanup_and_retains_the_namespace(self): + with tempfile.TemporaryDirectory() as folder: + fleet = Fleet(['node1'], Path(folder), 'gh-123-1-h1', runtime_profile='orphan-preserve') + fleet.reserved = ['node1'] + fleet.orphan_faults = [('node1', {'disk': 0, 'bucket': self.bucket, 'key': orphan.KEY, 'generation': 'authority'})] + calls = [] + def remote(node, action, **params): + calls.append(action) + if action == 'orphan-recover': return {'state': 'DIRTY', 'lastACKAuthorityRestored': False} + return {} + fleet.remote = remote + self.assertFalse(fleet.cleanup()) + self.assertNotIn('cleanup', calls) + self.assertEqual(len(fleet.orphan_faults), 1) + + def test_unknown_request_or_invalid_version_ack_never_restores_old_authority_or_cleans(self): + for failure in ('transport', 'wrong-version'): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as folder: + root = Path(folder) + candidate = root / 'candidate.json'; candidate.write_text(json.dumps({'source_sha': orphan.CANDIDATE, 'package_sha256': 'a' * 64})) + rc = root / 'rc'; rc.write_bytes(b'rc') + args = types.SimpleNamespace(evidence=root / 'evidence', candidate=candidate, rc_bin=rc, + tool_directory=root, shared_directory=root, testing_commit='b' * 40, binary_sha='c' * 64) + with patch.dict(os.environ, {'GITHUB_RUN_ID': '123', 'GITHUB_RUN_ATTEMPT': '1'}), \ + patch.object(orphan, 'verify_inspector', return_value={}), patch.object(orphan, 'verify_shared', return_value={}), \ + patch.object(orphan, 'Fleet') as fleet, patch.object(orphan, 'Recorder') as recorder: + fleet.return_value.record.side_effect = lambda label, value: value + recorder.return_value.call.side_effect = [TimeoutError('unknown original request')] if failure == 'transport' else [{}, {}, {'VersionId': 'foreign'}] + result = orphan.run_branch(args, 'H1', ['node1'], 'remove-false') + self.assertEqual(result['verdict'], 'FAIL'); self.assertEqual(result['cleanup'], 'FAIL') + fleet.return_value.cleanup.assert_not_called() + fleet.return_value.stop.assert_called_once() + self.assertFalse(any(call.args[1] == 'orphan-recover' for call in fleet.return_value.remote.call_args_list)) + + +class OrphanLinuxTests(unittest.TestCase): + @unittest.skipUnless(sys.platform == 'linux', 'requires real Linux descriptors, root ownership and private bind mounts') + def test_actual_absent_authority_recovery_and_nested_mount_refusal(self): + program = '''import json,shutil,subprocess,tempfile,uuid +from pathlib import Path +from scripts.validation_heal import orphan_fault as backend,generation_fault as generation,heal_fault as fault +with tempfile.TemporaryDirectory() as folder: + root=Path(folder).resolve();bucket='rv-heal-0123456789abcdef';obj=root/'data'/bucket/'orphan.bin' + obj.mkdir(parents=True);(root/'quarantine').mkdir();directory=str(uuid.UUID(int=11));(obj/directory).mkdir() + (obj/'xl.meta').write_bytes(b'old-meta');(obj/directory/'part.1').write_bytes(b'old-part') + (obj/directory/('part.1.integrity.'+str(uuid.UUID(int=77)))).write_bytes(b'old-commitment') + owner={'runId':'gh-123-1-h1','bucket':bucket,'runtimeProfile':'orphan-preserve','binarySha256':'a'*64} + request={'disk':0,'bucket':bucket,'key':'orphan.bin'} + with fault.directory(root) as fd: + present,_,files=backend.scene(fd,owner,request,fault,generation) + backend.capture(fd,owner,{**request,'generation':'old','expectedPresent':present,'expectedFiles':files},fault,generation) + shutil.rmtree(obj) + backend.capture(fd,owner,{**request,'generation':'authority','expectedPresent':False,'expectedFiles':[]},fault,generation) + backend.arm(fd,owner,{**request,'generation':'authority'},fault,generation) + backend.install(fd,owner,{**request,'mode':'data','dataDir':directory},fault,generation) + assert not (obj/'xl.meta').exists() and (obj/directory/'part.1').read_bytes()==b'old-part' + receipt=backend.recover(fd,owner,{**request,'generation':'authority'},fault,generation) + assert receipt['lastACKAuthorityRestored'] and not obj.exists() + assert backend.recover(fd,owner,{**request,'generation':'authority'},fault,generation)==receipt + assert (root/'quarantine/C10-authority-failed-current'/directory/('part.1.integrity.'+str(uuid.UUID(int=77)))).read_bytes()==b'old-commitment' + obj.mkdir();outside=root/'outside';outside.mkdir() + subprocess.run(['mount','--bind',str(outside),str(obj)],check=True) + try: + with fault.directory(root) as fd: + try: backend.scene(fd,owner,request,fault,generation) + except ValueError as error: assert 'mount' in str(error) + else: raise AssertionError('nested mount accepted') + finally: subprocess.run(['umount',str(obj)],check=True) +print(json.dumps({'realPostDeleteAuthorityRestored':True,'nestedBindRefused':True})) +''' + result = subprocess.run(['sudo', '-n', 'unshare', '-m', sys.executable, '-B', '-c', program], capture_output=True, text=True, timeout=60) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertTrue(json.loads(result.stdout)['realPostDeleteAuthorityRestored']) + + + + if __name__ == '__main__': unittest.main() From 6f778ebfa27164746a0242a52dd31bebfc7110a3 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 10:11:17 +0800 Subject: [PATCH 114/343] docs(validation): record C10 Linux gates and exact F04 failures --- docs/validation-migration-map.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index c23594bc..2268bfe4 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1602,6 +1602,15 @@ F06 `heal.cancelled-task-restart` passed in run `38102031029` at harness quiet physical windows, persisted cancellation after restart, no old-task replay, new-token repair, all-peer S3 and cleanup passed. Its independently rerun canonical gate passed, increasing complete live PASS to 49. -F07's coordinator process-crash run `38103118033` remains canonical FAIL with -cleanup PASS. The serial fleet queue continues under its existing lock and +F04's coordinator process-crash run `38103118033` remains canonical FAIL with +cleanup PASS: root restart readiness, the MRF collector event budget and +replacement disk-UUID observation did not satisfy their gates. The serial fleet queue continues under its existing lock and unclean-environment guard. + + +All six auto-testing checks passed at C10 executor commit +`b0fc7d15ba170bd592e6f4012c1ba15125186afb`. Original Linux contract logs from +run `38104232517` show 397 Python tests in total with no skips, including real +C10 absent post-DELETE recovery and nested bind-mount refusal. The serial C10 +live plan binds this harness and successful shared tools run `38103474555`; +it waits for the existing queue and does not change an active plan or guard. From 23953a96b7c565da3bee9f7c9442d803ba9ab7cc Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 10:14:09 +0800 Subject: [PATCH 115/343] fix(validation): verify each shared ACK through its own metadata --- docs/validation-migration-map.md | 10 ++++++++++ scripts/validation_heal/orphan.py | 18 ++++++++++++++++- test_validation_heal_orphan.py | 33 +++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 2268bfe4..d9564a2f 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1614,3 +1614,13 @@ run `38104232517` show 397 Python tests in total with no skips, including real C10 absent post-DELETE recovery and nested bind-mount refusal. The serial C10 live plan binds this harness and successful shared tools run `38103474555`; it waits for the existing queue and does not change an active plan or guard. + + +C10 verifies each ACK version separately when two versions share one physical +part path. The general scope verifier keeps its independent-path requirement; +C10 reports version-addressed streaming checks and distinct physical parts +separately, without inventing independent disks. New regressions verify both +VersionId calls and complete shared-conversion/seed recovery with retained +original metadata, parts and commitments. The updated complete local suite +passes 399 tests with eight macOS skips; C10 now has 22 tests. Its queued live +plan is updated to this corrected revision before fleet dispatch. diff --git a/scripts/validation_heal/orphan.py b/scripts/validation_heal/orphan.py index c198d502..db096d4b 100644 --- a/scripts/validation_heal/orphan.py +++ b/scripts/validation_heal/orphan.py @@ -175,6 +175,22 @@ def pending_snapshot(fleet): **fleet.remote(node, 'orphan-multipart', disk=disk)})['files'] for node in fleet.nodes for disk in range(4)} +def verify_live_parts(fleet, mapping, label): + require(mapping, 'C10 streaming manifest is empty') + total, identities = 0, set() + for identity, target in mapping.items(): + require(identity == (target['bucket'], target['key'], target['versionId']), + 'C10 streaming manifest lost its exact version-addressed authority') + # Two independently ACKed shared versions deliberately reference the + # same path. Each version must validate its own metadata commitment; + # the general selector verifier still rejects aliases within a cohort. + proof = verify_parts(fleet, {identity: target}, label) + total += proof['exactStreamingParts'] + identities.update((member['node'], member['disk'], member['relativePath']) for member in target['cohort']) + return {'versionAddressedStreamingParts': total, 'distinctPhysicalParts': len(identities), + 'exactACKVersions': len(mapping), 'sharedPathsCountedAsIndependentDisks': False} + + def verify_public(fleet, binding, bucket, expected, deleted, label): for endpoint in binding['endpoints']: peer = Probe(binding, endpoint) @@ -421,7 +437,7 @@ def delete_ack(response): restored = fleet.snapshot('C10-final-current-ACK-physical', quiescent=True) mapping = {(bucket, entry['key'], entry['versionId']): map_target(restored, bucket, entry['key'], 'data', 1, entry['versionId'], latest_required=entry['isLatest']) for entry in expected if not entry['deleteMarker']} - result['streaming'] = verify_parts(fleet, mapping, 'C10-all-live-ACK-streaming') + result['streaming'] = verify_live_parts(fleet, mapping, 'C10-all-live-ACK-streaming') fleet.start() verify_public(fleet, binding, bucket, expected, deleted, 'C10-all-peer-final-S3') result.update(verdict='PASS', postDeleteAuthorityRestored=True, backupRestorationIsProductRepair=False) diff --git a/test_validation_heal_orphan.py b/test_validation_heal_orphan.py index 61b3d5cb..1c7602c4 100644 --- a/test_validation_heal_orphan.py +++ b/test_validation_heal_orphan.py @@ -220,6 +220,24 @@ def test_retired_shared_seed_cannot_be_recovered_after_later_ack_is_allowed(self self.invoke('disarm', generation='seed', expectedFiles=manifest) with self.assertRaisesRegex(ValueError, 'retired'): self.recover('seed') + def test_codec_conversion_preserves_retired_bytes_and_seed_rescue_can_restore_both_ack_generations(self): + self.seed(self.live_dir, b'old') + self.capture('seed'); self.invoke('arm', generation='seed') + source, destination = str(uuid.UUID(int=1)), str(uuid.UUID(int=2)) + receipt = {'sourceVersion': source, 'destinationVersion': destination, 'sharedDataDir': self.old_dir, + 'retiredDataDir': self.live_dir, 'filesystemMutationPerformed': False, 'physicalByteEqualityClaimed': False} + result = subprocess.CompletedProcess([], 0, b'controlled-encoded-meta', json.dumps(receipt).encode()) + with fault.directory(self.root) as root, patch.object(backend.subprocess, 'run', return_value=result): + proof = backend.convert(root, self.owner, {**self.request, 'sourceDataDir': self.old_dir, 'destinationDataDir': self.live_dir, + 'sourceVersion': source, 'destinationVersion': destination}, fault, generation, self.root / 'verified-codec') + self.assertTrue(proof['physicalByteEqualityVerified']) + self.assertEqual((self.object / 'xl.meta').read_bytes(), b'controlled-encoded-meta') + self.assertFalse((self.object / self.live_dir).exists()) + self.assertEqual((self.root / 'quarantine/C10-retired-shared-data/part.1').read_bytes(), b'old' * 256) + self.recover('seed') + self.assertEqual((self.object / 'xl.meta').read_bytes(), b'XL2 old') + self.assertEqual((self.object / self.live_dir / 'part.1').read_bytes(), b'old' * 256) + def test_mpu_census_is_complete_read_only_and_refuses_aliases(self): folder = self.root / 'data/.rustfs.sys/multipart/hash/upload'; folder.mkdir(parents=True) (folder / 'part.1').write_bytes(b'pending-part') @@ -268,6 +286,21 @@ def test_source_case_has_all_five_branches_on_both_required_topologies(self): self.assertEqual(source['contract']['probes']['topology'], 'H1,H2') self.assertIn(orphan.CASE_ID, AUTOMATIC_CASES) + def test_shared_paths_are_streamed_through_both_exact_ack_version_ids(self): + mapping = {} + for version in (str(uuid.UUID(int=1)), str(uuid.UUID(int=2))): + mapping[(self.bucket, orphan.KEY, version)] = {'bucket': self.bucket, 'key': orphan.KEY, 'versionId': version, + 'cohort': [{'node': 'node1', 'disk': disk, 'relativePath': 'shared/part.1'} for disk in range(4)]} + with patch.object(orphan, 'verify_parts', return_value={'exactStreamingParts': 4}) as verify: + proof = orphan.verify_live_parts(object(), mapping, 'exact-shared-ACK') + self.assertEqual(verify.call_count, 2) + self.assertEqual({next(iter(call.args[1]))[2] for call in verify.call_args_list}, {identity[2] for identity in mapping}) + self.assertEqual(proof['versionAddressedStreamingParts'], 8) + self.assertEqual(proof['distinctPhysicalParts'], 4) + self.assertFalse(proof['sharedPathsCountedAsIndependentDisks']) + next(iter(mapping.values()))['versionId'] = 'foreign' + with self.assertRaises(ValueError): orphan.verify_live_parts(object(), mapping, 'foreign-ACK') + def scene(self, branch): live, stale = str(uuid.UUID(int=12)), str(uuid.UUID(int=11)) version, doomed = str(uuid.UUID(int=1)), str(uuid.UUID(int=2)) From 7128b50818de3206f6fb368d4ea696e56da7206b Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 10:29:03 +0800 Subject: [PATCH 116/343] feat(validation): preserve all paired A10 option contracts --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 49 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/options.py | 575 ++++++++++++++++++ scripts/validation_heal/run_case.sh | 1 + scripts/validation_heal/write_quorum.py | 10 +- test_validation_heal_options.py | 239 ++++++++ test_validation_heal_write_quorum.py | 19 + 9 files changed, 895 insertions(+), 6 deletions(-) create mode 100644 scripts/validation_heal/options.py create mode 100644 test_validation_heal_options.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 7b334a01..7d7da719 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -35,6 +35,7 @@ on: - test_validation_heal_native.py - test_validation_heal_authority.py - test_validation_heal_orphan.py + - test_validation_heal_options.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -83,6 +84,7 @@ on: - test_validation_heal_native.py - test_validation_heal_authority.py - test_validation_heal_orphan.py + - test_validation_heal_options.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -120,7 +122,7 @@ jobs: sudo apt-get update sudo apt-get install --no-install-recommends -y nftables fi - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index d9564a2f..6bcebd4e 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 132 currently dispatchable IDs include all 74 non-Heal cases and fifty-eight Heal cases. Forty-nine have complete canonical live PASS evidence; the other 109 have no live PASS claim. +case evidence. The 133 currently dispatchable IDs include all 74 non-Heal cases and fifty-nine Heal cases. Forty-nine have complete canonical live PASS evidence; the other 109 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,7 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 32 Heal cases remain +The other 25 Heal cases remain operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -1624,3 +1624,48 @@ VersionId calls and complete shared-conversion/seed recovery with retained original metadata, parts and commitments. The updated complete local suite passes 399 tests with eight macOS skips; C10 now has 22 tests. Its queued live plan is updated to this corrected revision before fleet dispatch. + + +A10 now dispatches all nine source option branches on H1/H2, with fresh DB and +DV namespaces for both baseline and selected requests: 72 distinct instances. +Each pair changes only the reviewed option, or the inseparable pool/set +selector pair. The exact candidate Admin/selector source hashes and supported +fields are recorded in the contract; this is API coverage, not an rc claim. +H2 scope probes use two actual eight-drive sets across the original four +nodes. Recursive probes include a child prefix, an adjacent prefix and outside +controls; selectors bind each ACK version to its measured physical set. +Every historical version and marker must have complete, exact responsibility, +truthful counters and retained settings. `readRepair=true` and invalid selectors +must return HTTP 400 `InvalidArgument`, leave no active responsibility and +preserve both missing and bitflipped control parts. + +The remove pair reuses C10 descriptor capture, complete original-member census, +post-DELETE authority and safe recovery. Both DB and DV delete every original +target generation through separately recorded S3 ACKs, prove no live metadata +or payload reference, then replay only one owned data directory. Both phases +use the same data-only fault; only `remove` changes. Product physical and all +peer S3 checks precede recovery. An accepted parameter which fails its registered +physical behavior remains FAIL; it is never counted as default repair or as +option rejection. Unknown, wrong or duplicate mutation ACKs and incomplete +post-DELETE restoration retain DIRTY namespaces. This executor has 18 local +contract tests and still requires canonical live evidence. + +B08 live run `38104014806` remains FAIL with cleanup PASS: both write-loss +branches produced incomplete target identities; H2 lock loss also failed a +snapshot that incorrectly required a live PID on stopped peers. The collector +now attests online processes normally and reads every stopped original member +only with the quiescent check. Two regressions retain all physical members and +reject foreign stopped-node bindings. No quorum or identity assertion changed. +A new immutable live run is required; the old failure stays visible. + +The current flapping/MRF queue retains `38104306522` and `38104630791` as +canonical FAIL with cleanup PASS. F03 drive-offline observation, MRF admission +and source ownership checks failed. E03 offline PUT and marker branches passed, +but its exact offline version DELETE had no committed durable MRF responsibility +within the registered window. These partial results do not add canonical PASS. +All six checks passed at `23953a96b7c565da3bee9f7c9442d803ba9ab7cc`; Linux run +`38104493427` executed all 399 then-current tests without skips. The current +A10/B08 local suite passes 419 tests with eight macOS skips. There are now 133 +automatic executors (74 non-Heal, 59 Heal), 25 remaining Heal executors and +49/158 complete canonical live PASS results. All original 158 IDs/hashes and +84/301/532 Heal recipes/branches/topology templates remain unchanged. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index ce2d871b..4800a326 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 2bafb064..4aba5315 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.admin-option-contract", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/options.py b/scripts/validation_heal/options.py new file mode 100644 index 00000000..0192358d --- /dev/null +++ b/scripts/validation_heal/options.py @@ -0,0 +1,575 @@ +#!/usr/bin/env python3 +"""A10: paired Admin option probes on fresh DB/DV and owned C10 orphans.""" + +import argparse +import copy +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +from urllib.parse import quote +import uuid + +try: + from .heal_probe import Probe, compare_inventories, payload, require, save + from .heal_workload import Recorder, verify_seed + from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_rejection, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .heal_fault import bounded_fault_id + from .shards import files, map_target, validate_fault, validate_repaired_physical, peer_status_equal + from .versions import public_version_id + from .scopes import verify_parts + from .stop import idle_fleet + from . import orphan +except ImportError: + from heal_probe import Probe, compare_inventories, payload, require, save + from heal_workload import Recorder, verify_seed + from healthy import BASE_OPTIONS, Fleet, physical_content, validate_rejection, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from heal_fault import bounded_fault_id + from shards import files, map_target, validate_fault, validate_repaired_physical, peer_status_equal + from versions import public_version_id + from scopes import verify_parts + from stop import idle_fleet + import orphan + + +CASE_ID = 'heal.admin-option-contract' +BRANCHES = ('recursive-false', 'recursive-true', 'recreate', 'remove-false', 'remove-true', + 'update-parity', 'read-repair', 'selector-valid', 'selector-invalid') +FIXTURES = ('DB', 'DV') +SIZE = 1024**2 + 123 +KEYS = ('scope/target.bin', 'scope/child/target.bin', 'scope-extra/control.bin', 'other/control.bin', + *tuple(f'set-control/{number:02d}.bin' for number in range(12))) +REFERENCE = {'candidateCommit': orphan.CANDIDATE, + 'sourceFiles': {'rustfs/src/admin/handlers/heal.rs': '7e4007c29a8329ced8e48b88111400e79cffd845bd6ddc6243b79f3cd48e8199', + 'crates/heal/src/heal/channel.rs': '7f669ca43b9559db9b97fba6b419a6c76a94d76acd8526db132d74b7c534d6b0'}, + 'support': {'recursive': 'supported; false is exact object, true traverses child keys', + 'recreate': 'supported; request and retained settings must agree', + 'remove': 'supported; restricted to this run C10-proven unreferenced orphan', + 'updateParity': 'supported; deep parity corruption must be physically repaired', + 'readRepair': 'unsupported for Admin; HTTP 400 InvalidArgument before execution', + 'pool/set': 'supported paired selector; exact independently measured physical set'}, + 'dangerousEffects': {'remove': 'may delete unreferenced persisted bytes; never applied to a live fixture', + 'recreate': 'may reconstruct missing data; original ACK identities and complete controls stay fixed'}, + 'cliCoverageClaimed': False} + + +def paired_options(branch): + require(branch in BRANCHES, 'unknown A10 option branch') + baseline = {**BASE_OPTIONS, 'pool': None, 'set': None} + selected = dict(baseline) + if branch == 'recursive-false': baseline['recursive'] = True + elif branch == 'recursive-true': selected['recursive'] = True + elif branch == 'recreate': selected['recreate'] = True + elif branch == 'remove-true': selected['remove'] = True + elif branch == 'update-parity': selected['updateParity'] = True + elif branch == 'read-repair': selected['readRepair'] = True + elif branch.startswith('selector-'): + baseline['recursive'] = selected['recursive'] = True + if branch == 'selector-valid': selected.update(pool=0, set=0) + else: + baseline.update(pool=0, set=0) + selected.update(pool=99, set=99) + changed = {name for name in baseline if baseline[name] != selected[name]} + expected = {'recursive-false': {'recursive'}, 'recursive-true': {'recursive'}, 'recreate': {'recreate'}, + 'remove-false': set(), 'remove-true': {'remove'}, 'update-parity': {'updateParity'}, + 'read-repair': {'readRepair'}, 'selector-valid': {'pool', 'set'}, 'selector-invalid': {'pool', 'set'}}[branch] + require(changed == expected, 'A10 paired request changed unrelated defaults') + return baseline, selected + + +class FixtureRecorder(Recorder): + """Retain the namespace unless all mutation receipts and identities resolve.""" + def __init__(self, *args, fixture='DB', **kwargs): + super().__init__(*args, **kwargs) + require(fixture in FIXTURES, 'unknown A10 ACK versioning mode') + self.fixture = fixture + self.versions = set() + self.pending = None + + def call(self, operation, params, body=None, **kwargs): + self.pending = {'operation': operation, 'params': copy.deepcopy(params)} + value = super().call(operation, params, body, **kwargs) + # A successful HTTP status alone is insufficient for a durable identity. + if operation in {'put_object', 'delete_object'}: + version = value.get('VersionId') + require(version in {None, 'null'} or isinstance(version, str) and str(uuid.UUID(version)) == version + and uuid.UUID(version).int != 0, + 'A10 mutation receipt has an invalid version identity') + if operation == 'put_object': + require(isinstance(value.get('ETag'), str) and value['ETag'], 'A10 PUT ACK omitted ETag') + require((self.fixture == 'DB' and version in {None, 'null'}) + or (self.fixture == 'DV' and version not in {None, 'null'}), 'A10 PUT ACK changed its DB/DV mode') + identity = params['Key'], public_version_id(version) + require(identity not in self.versions, 'A10 PUT ACK reused an exact version identity') + self.versions.add(identity) + elif operation == 'delete_object': + if 'VersionId' in params: + require(version == params['VersionId'] and value.get('DeleteMarker') in {None, False}, + 'A10 exact data-version DELETE ACK changed identity or kind') + elif self.fixture == 'DV': + identity = params['Key'], public_version_id(version) + require(value.get('DeleteMarker') is True and version not in {None, 'null'} + and identity not in self.versions, 'A10 DELETE ACK omitted its independent marker identity') + self.versions.add(identity) + else: + require(version in {None, 'null'} and value.get('DeleteMarker') in {None, False}, + 'A10 DB DELETE ACK created a version or marker') + self.pending = None + return value + + +def seed_fixture(recorder, bucket, fixture, keys=KEYS): + require(fixture in FIXTURES and len(keys) == len(set(keys)), 'invalid A10 fixture') + recorder.call('create_bucket', {'Bucket': bucket}) + if fixture == 'DV': + recorder.call('put_bucket_versioning', {'Bucket': bucket, 'VersioningConfiguration': {'Status': 'Enabled'}}) + expected = [] + for key in keys: + for number in range(3 if fixture == 'DV' else 1): + body = payload(f'A10/{fixture}/{key}/{number}', SIZE) + response = recorder.call('put_object', {'Bucket': bucket, 'Key': key, + 'Metadata': {'heal-fixture': 'A10', 'generation': str(number)}, 'Tagging': 'suite=heal&fixture=' + fixture}, body) + recorder.pending = {'operation': 'put_object', 'bucket': bucket, 'key': key, 'reason': 'fixture identity unresolved'} + version = public_version_id(response.get('VersionId')) + require((fixture == 'DB' and version == 'null') or (fixture == 'DV' and version != 'null'), + 'A10 candidate ACK disagrees with DB/DV versioning') + require((key, version) not in {(entry['key'], entry['versionId']) for entry in expected}, + 'A10 PUT ACK reused a version') + expected.append({'key': key, 'versionId': version, 'isLatest': number == (2 if fixture == 'DV' else 0), + 'deleteMarker': False, 'payload': {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}}) + recorder.pending = None + if fixture == 'DV': + key = keys[-1] + response = recorder.call('delete_object', {'Bucket': bucket, 'Key': key}) + recorder.pending = {'operation': 'delete_object', 'bucket': bucket, 'key': key, 'reason': 'marker identity unresolved'} + version = public_version_id(response.get('VersionId')) + require(response.get('DeleteMarker') is True and version != 'null' + and version not in {entry['versionId'] for entry in expected}, 'A10 marker ACK is incomplete or duplicate') + next(entry for entry in expected if entry['key'] == key and entry['isLatest'])['isLatest'] = False + expected.append({'key': key, 'versionId': version, 'isLatest': True, 'deleteMarker': True}) + recorder.pending = None + return expected + + +def map_fixture(snapshot, bucket, expected, topology, *, multi_set=True): + validate_topology(snapshot, topology, expected_sets=2 if topology == 'H2' and multi_set else 1) + indexed, mapped = files(snapshot), {} + for entry in expected: + scopes = set() + for node, value in snapshot.items(): + for row in value['formats']: + if (node, row['disk'], f"{bucket}/{entry['key']}/xl.meta") in indexed: + layout = row['value']['xl'] + scopes.update(number for number, members in enumerate(layout['sets']) if layout['this'] in members) + require(len(scopes) == 1, 'A10 ACK has an ambiguous physical set') + set_index = next(iter(scopes)) + identity = (bucket, entry['key'], entry['versionId']) + if entry['deleteMarker']: + mapped[identity] = {'bucket': bucket, 'key': entry['key'], 'versionId': entry['versionId'], + 'poolIndex': 0, 'setIndex': set_index, 'deleteMarker': True} + else: + mapped[identity] = {**map_target(snapshot, bucket, entry['key'], 'data', 1, entry['versionId'], + latest_required=entry['isLatest'], multi_set=topology == 'H2' and multi_set, set_index=set_index), 'deleteMarker': False} + if topology == 'H2' and multi_set: + require({target['setIndex'] for target in mapped.values()} == {0, 1}, 'A10 fixture did not populate both actual sets') + require(len(mapped) == len(expected), 'A10 physical mapping omitted an ACK') + return mapped + + +def selection(branch, options, mapped): + if branch.startswith('recursive-'): + return {identity for identity in mapped if identity[1].startswith('scope/')} if options['recursive'] else set() + if branch.startswith('selector-'): + return {identity for identity, target in mapped.items() if options['pool'] is None + or (target['poolIndex'], target['setIndex']) == (options['pool'], options['set'])} + return {identity for identity in mapped if identity[1] == KEYS[0]} + + +def request_path(branch, bucket): + if branch.startswith('selector-'): return '/rustfs/admin/v3/heal/' + key = 'scope/' if branch.startswith('recursive-') else KEYS[0] + return '/rustfs/admin/v3/heal/' + quote(bucket, safe='') + '/' + quote(key, safe='/') + + +def run_request(probe, fleet, path, options, active, *, rejected=False): + raw = fleet.record('A10-admission', probe.admin(path, [], json.dumps(options).encode())) + if raw.get('httpStatus') == 200 and isinstance(raw.get('response'), dict) and raw['response'].get('clientToken'): + active.append((path, raw['response']['clientToken'])) + if rejected: + validate_rejection(raw, 'InvalidArgument') + return raw + require(raw.get('httpStatus') == 200 and isinstance(raw.get('response'), dict) + and active and active[-1][0] == path, 'A10 admission omitted its exact token') + token = active[-1][1] + deadline = time.monotonic() + 180 + while True: + raw = fleet.record('A10-status', probe.admin(path, [('clientToken', token)], b'')) + require(raw.get('httpStatus') == 200 and isinstance(raw.get('response'), dict), 'A10 original status is unavailable') + status = raw['response'] + if status.get('summary') != 'running': break + require(time.monotonic() < deadline, 'A10 task exceeded its frozen budget') + time.sleep(.2) + require(status.get('settings') == options, 'A10 candidate silently changed or ignored request fields') + for endpoint in probe.binding['endpoints']: + peer = Probe(probe.binding, endpoint) + convergence = time.monotonic() + 30 + while True: + repeat = fleet.record('A10-retained-status', peer.admin(path, [('clientToken', token)], b'')) + require(repeat.get('httpStatus') == 200 and isinstance(repeat.get('response'), dict), 'A10 retained task status is missing') + if (repeat['response'].get('settings') == options and peer_status_equal(status, repeat['response'], + task=(path, token), peer_task=(path, token))): break + require(time.monotonic() < convergence, 'A10 original responsibility did not converge on every peer') + time.sleep(.2) + active.remove((path, token)) + return status + + +def validate_outcome(status, mapped, selected, repaired, options): + outcome = status.get('outcome') or {} + require(status.get('summary') == 'finished' and status.get('settings') == options + and status.get('truncated', False) is False and outcome.get('coverage') == 'complete' + and outcome.get('objectsTruncated') is False, 'A10 lacks complete original task coverage/settings') + counters = outcome.get('counters') or {} + if not selected: + require(not repaired and outcome.get('execution', {}).get('state') == 'completed_with_errors' + and counters.get('healed') == 0 and counters.get('failed') == 1 and counters.get('unknown') == 0 + and 'Object not found: ' in (status.get('detail') or '') and not outcome.get('objects'), + 'A10 exact absent prefix was repaired, unknown or incorrectly claimed successful') + return {'exactPrefixNotFound': True, 'selectedACKs': 0, 'repairedACKs': 0} + require(repaired <= selected <= set(mapped) and outcome.get('execution', {}).get('state') == 'completed', + 'A10 responsibility escaped the physical scope or failed') + objects, seen, incarnations = outcome.get('objects'), set(), {} + require(isinstance(objects, list) and len(objects) == len(selected), 'A10 diagnostics omitted a version/marker or added a control') + for receipt in objects: + item = receipt.get('identity') or {} + identity = (item.get('bucket'), item.get('object'), public_version_id(item.get('versionId'))) + require(identity in selected and identity not in seen and item.get('kind') == 'object', 'A10 wrong or duplicate ACK identity') + target, scope = mapped[identity], (item.get('poolIndex'), item.get('setIndex')) + if (options['pool'], options['set']) != (None, None): + require(all(type(value) is int for value in scope) and scope == (options['pool'], options['set']) + == (target['poolIndex'], target['setIndex']), 'A10 explicit selector identity was lost or changed') + elif scope != (None, None): + require(all(type(value) is int for value in scope) and scope == (target['poolIndex'], target['setIndex']), + 'A10 unrestricted receipt contradicts its physical set') + incarnation = item.get('bucketIncarnationId') + require(isinstance(incarnation, str) and incarnation and incarnations.setdefault(identity[0], incarnation) == incarnation, + 'A10 lost or mixed a bucket incarnation') + state = 'repaired' if identity in repaired else 'metadata_healthy' if target['deleteMarker'] else 'verified_healthy' + require(receipt.get('disposition', {}).get('state') == state, 'A10 false repair, unknown or incomplete integrity') + seen.add(identity) + expected = {'processed': len(selected), 'healed': len(repaired), 'unchanged': len(selected) - len(repaired), + 'failed': 0, 'unknown': 0, 'skipped': 0, 'attemptFailures': 0, 'overflowed': False} + require(all(type(counters.get(name)) is type(value) and counters[name] == value for name, value in expected.items()), + 'A10 counters disagree with the complete independent ACK/physical manifest') + return {'selectedACKs': len(selected), 'repairedACKs': len(repaired), 'completeVersionsAndMarkers': True} + + +def physical_selection(original, after, targets, selected): + controls = [target for target in targets if (target['bucket'], target['key'], target['versionId']) not in selected] + repaired = [target for target in targets if target not in controls] + expected = copy.deepcopy(original) + actual = files(after) + for target in controls: + identity = target['node'], target['disk'], target['relativePath'] + # Bitflipped control bytes are witnessed separately below. The product + # physical oracle takes the exact fault scene as its control authority. + if target['faultKind'] == 'missing': + require(identity not in actual, 'A10 unselected missing control was silently repaired') + else: + require(identity in actual and {name: actual[identity][name] for name in ('sha256', 'size')} == target['faultBytes'], + 'A10 unselected corrupted control was silently repaired or changed') + for entry in expected[target['node']]['files']: + if (target['disk'], target['relativePath']) == (entry['disk'], entry['path']): + if target['faultKind'] == 'missing': expected[target['node']]['files'].remove(entry) + else: entry.update(sha256=target['faultBytes']['sha256'], size=target['faultBytes']['size']) + break + if repaired: + proof = validate_repaired_physical(expected, after, repaired[0], repair_targets=repaired) + else: + require(physical_content(expected) == physical_content(after), 'A10 no-repair/rejected request changed persisted bytes') + proof = {'allPersistedBytesUnchanged': True} + return {**proof, 'unselectedFaultsPreserved': len(controls), 'selectedPhysicalRepairs': len(repaired), + 'beforeAnyTargetGETOrBackupRescue': True} + + +def verify_public(fleet, binding, bucket, expected, baseline, fixture, label): + for endpoint in binding['endpoints']: + observed = fleet.record(label, Probe(binding, endpoint).inventory(bucket, unversioned_current_tags=fixture == 'DB')) + verify_seed(expected, observed) + compare_inventories(baseline, observed) + + +def scope_probe(fleet, probe, binding, recorder, bucket, topology, fixture, branch, options, active): + expected = seed_fixture(recorder, bucket, fixture) + baseline = fleet.record('A10-full-ACK-baseline', probe.inventory(bucket, unversioned_current_tags=fixture == 'DB')) + verify_seed(expected, baseline) + fleet.stop() + original = fleet.snapshot('A10-original-physical', quiescent=True) + mapped = map_fixture(original, bucket, expected, topology) + data = {identity: target for identity, target in mapped.items() if not target['deleteMarker']} + verify_parts(fleet, data, 'A10-all-original-streaming') + selected = selection(branch, options, mapped) + targets = [] + for index, key in enumerate(KEYS[:4]): + identity = next(identity for identity in data if identity[1] == key) + target = data[identity] + role = 'parity' if index % 2 or branch == 'update-parity' and index == 0 else 'data' + member = next(row for row in target['cohort'] if row['role'] == role) + targets.append({**target, **member, 'faultKind': 'bitflip' if role == 'parity' else 'missing'}) + if topology == 'H2': + for number in (0, 1): + if any(target['setIndex'] == number for target in targets): continue + target = next(target for identity, target in data.items() if target['setIndex'] == number) + targets.append({**target, 'faultKind': 'missing'}) + prior, registrations = original, [] + for index, target in enumerate(targets): + params = {'disk': target['disk'], 'faultId': bounded_fault_id('A10', branch, index)} + fleet.faults.append((target['node'], params)) + registrations.append((target['node'], params, target)) + raw = fleet.record('A10-fault', fleet.remote(target['node'], 'apply-fault', **params, + relativePath=target['relativePath'], expectedSha256=target['sha256'], kind=target['faultKind'], + **({'offset': target['size'] // 2} if target['faultKind'] == 'bitflip' else {}))) + scene = fleet.snapshot('A10-witness-fault', quiescent=True) + validate_fault(prior, scene, target, raw) + target['faultBytes'] = raw.get('fault') + prior = scene + fleet.start() + require(physical_content(prior) == physical_content(fleet.snapshot('A10-fault-immediately-before-START')), + 'NOT_OBSERVED: A10 faults changed before original task admission') + rejected = options['readRepair'] or options['pool'] == 99 + status = run_request(probe, fleet, request_path(branch, bucket), options, active, rejected=rejected) + idle_fleet(fleet, binding) + fleet.stop() + after = fleet.snapshot('A10-product-before-GET-or-rescue', quiescent=True) + if rejected: + require(physical_content(prior) == physical_content(after), 'A10 structured rejection mutated physical controls') + selected = set() + terminal = {'httpStatus': status['httpStatus'], 'structuredInvalidArgument': True, 'noActiveResponsibility': True} + else: + repaired = {(target['bucket'], target['key'], target['versionId']) for target in targets} & selected + terminal = validate_outcome(status, mapped, selected, repaired, options) + if not selected: + require('Object not found: ' + bucket + '/scope/' in status.get('detail', ''), 'A10 exact prefix failure names another key') + physical = physical_selection(original, after, targets, selected) + verify_parts(fleet, {identity: target for identity, target in data.items() if identity in selected}, + 'A10-selected-physical-streaming') if selected else None + # Restore only excluded controls after the product result has been frozen. + for node, params, target in registrations: + if (target['bucket'], target['key'], target['versionId']) not in selected: + receipt = fleet.record('A10-excluded-control-rescue', fleet.remote(node, 'recover-fault', **params)) + require(receipt.get('physicalRepairPassAllowed') is False, 'A10 backup rescue claimed product repair') + fleet.faults.remove((node, params)) + restored = fleet.snapshot('A10-all-physical-after-control-rescue', quiescent=True) + validate_repaired_physical(original, restored, targets[0], repair_targets=targets) + verify_parts(fleet, data, 'A10-all-final-physical-streaming') + fleet.start() + verify_public(fleet, binding, bucket, expected, baseline, fixture, 'A10-all-peer-final-S3') + fleet.stop() + require(physical_content(restored) == physical_content(fleet.snapshot('A10-physical-after-S3', quiescent=True)), + 'A10 GET changed the already-proven physical result') + return {'terminal': terminal, 'physical': physical, 'fixture': fixture, 'originalACKs': len(expected)} + + +def orphan_probe(fleet, probe, binding, recorder, bucket, topology, fixture, options, active): + # A10 reuses C10's complete descriptor archives and post-DELETE authority, + # while keeping the same data-only replay in both remove option phases. + keys = (orphan.KEY, 'control.bin', 'marker-control.bin') + expected = seed_fixture(recorder, bucket, fixture, keys) + doomed = next(entry for entry in expected if entry['key'] == orphan.KEY and entry['isLatest']) + if fixture == 'DV': + # Remove every orphan-key generation, with separate exact durable ACKs. + doomed_entries = [entry for entry in expected if entry['key'] == orphan.KEY] + else: doomed_entries = [doomed] + fleet.record('A10-orphan-original-S3', verify_seed(expected, probe.inventory(bucket, unversioned_current_tags=fixture == 'DB'))) + fleet.stop() + original = fleet.snapshot('A10-orphan-original-all-members', quiescent=True) + mapped = map_fixture(original, bucket, expected, topology, multi_set=False) + target = mapped[(bucket, orphan.KEY, doomed['versionId'])] + orphan.capture(fleet, original, bucket, 'old') + fleet.start() + for entry in doomed_entries: + params = {'Bucket': bucket, 'Key': orphan.KEY} + if fixture == 'DV': params['VersionId'] = entry['versionId'] + response = recorder.call('delete_object', params) + recorder.pending = {'operation': 'delete_object', 'params': params, 'reason': 'ACK identity not yet resolved'} + require(response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 204 + and response.get('DeleteMarker') in {None, False} + and public_version_id(response.get('VersionId')) == entry['versionId'], 'A10 orphan DELETE ACK changed exact identity') + recorder.pending = None + expected.remove(entry) + baseline = fleet.record('A10-last-DELETE-S3-authority', probe.inventory(bucket, unversioned_current_tags=fixture == 'DB')) + verify_seed(expected, baseline) + fleet.stop() + authority = fleet.snapshot('A10-last-DELETE-all-physical-authority', quiescent=True) + require(all(not values for values in orphan.references(authority, bucket, orphan.KEY).values()) + and not any(identity[2].startswith(bucket + '/' + orphan.KEY + '/') for identity in files(authority)), + 'A10 remove target has a surviving version/marker/data reference') + fleet.a10_orphan_authority = authority + fleet.a10_orphan_restored = False + orphan.arm_all(fleet, authority, bucket, 'authority') + try: + fleet.record('A10-owned-unreferenced-data-only-replay', fleet.remote(target['node'], 'orphan-install', + disk=target['disk'], bucket=bucket, key=orphan.KEY, dataDir=target['dataDir'], mode='data')) + before = fleet.snapshot('A10-orphan-fault-before-GET', quiescent=True) + require({identity[:2] for identity in files(before) if identity[2].startswith(bucket + '/' + orphan.KEY + '/')} + == {(target['node'], target['disk'])}, 'A10 replay escaped its one original member') + require(all(not values for values in orphan.references(before, bucket, orphan.KEY).values()), + 'A10 controlled orphan still has a live metadata reference') + fleet.start() + require(physical_content(before) == physical_content(fleet.snapshot('A10-orphan-immediately-before-START')), + 'NOT_OBSERVED: A10 orphan changed before admission') + status = run_request(probe, fleet, '/rustfs/admin/v3/heal/' + bucket + '/' + orphan.KEY, options, active) + idle_fleet(fleet, binding) + fleet.stop() + after = fleet.snapshot('A10-orphan-product-before-GET-or-rescue', quiescent=True) + branch = 'a10-remove-true' if options['remove'] else 'remove-false' + physical = orphan.physical_safety(before, after, authority, bucket, expected, branch, + doomed['versionId'], target['dataDir'], (target['node'], target['disk'])) + terminal = orphan.terminal_safety(status, branch, bucket, options) + fleet.start() + verify_public(fleet, binding, bucket, expected, baseline, fixture, 'A10-orphan-product-before-rescue-S3') + for endpoint in binding['endpoints']: + peer = Probe(binding, endpoint) + for entry in doomed_entries: + params = {'Bucket': bucket, 'Key': orphan.KEY} + if fixture == 'DV': params['VersionId'] = entry['versionId'] + observed = fleet.record('A10-deleted-orphan-not-resurrected', peer.s3('get_object', params)) + response = observed.get('response') or {} + require(not observed['acknowledged'] and not observed['resultUnknown'] and 'payload' not in observed + and response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 404 + and response.get('Error', {}).get('Code') in {'NoSuchKey', 'NoSuchVersion'}, 'A10 deleted ACK resurrected') + finally: + # These archives are the last DELETE authority, never the old PUT tree. + fleet.stop() + orphan.recover_all(fleet) + restored = fleet.snapshot('A10-orphan-last-DELETE-restored', quiescent=True) + require(physical_content(authority) == physical_content(restored), 'A10 orphan rescue changed last DELETE authority') + fleet.a10_orphan_restored = True + mapping = map_fixture(restored, bucket, expected, topology, multi_set=False) + verify_parts(fleet, {identity: target for identity, target in mapping.items() if not target['deleteMarker']}, + 'A10-orphan-all-live-controls-streaming') + fleet.start() + verify_public(fleet, binding, bucket, expected, baseline, fixture, 'A10-orphan-all-peer-final-S3') + return {'fixture': fixture, 'terminal': terminal, 'physical': physical, 'allOriginalPostDeleteAuthoritiesRestored': True, + 'C10UnreferencedOrphanProof': True, 'deletedACKs': len(doomed_entries), 'rescueIsProductRepair': False} + + +def run_instance(args, topology, nodes, branch, fixture, phase): + directory = args.evidence / topology / branch / fixture / phase + directory.mkdir(parents=True) + candidate = json.loads(args.candidate.read_text()) + require(candidate['source_sha'] == REFERENCE['candidateCommit'], 'A10 option contract is not reviewed for this candidate') + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + is_orphan = branch.startswith('remove-') + runtime = 'orphan-preserve' if is_orphan else 'scope-isolation' if topology == 'H2' else 'manual' + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile=runtime) + bucket = 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {'schemaVersion': 1, 'runId': run_id, 'jumpHostOnly': True, 'exclusiveHealService': True, + 'leaseExpiresAt': (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), 'bucketPrefix': bucket, + 'endpoints': {f'node{index}': f'http://rustfs-node{index}:9000' for index in range(1, len(nodes) + 1)}, + 'defaultEndpoint': 'node1', 'candidateCommit': candidate['source_sha'], 'harnessCommit': args.testing_commit, + 'candidatePackageSha256': candidate['package_sha256'], 'candidateBinarySha256': args.binary_sha, + 'rcBinarySha256': hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), 'maxInventoryObjects': 128, + 'maxInventoryBytes': 512 * 1024**2} + baseline_options, selected_options = paired_options(branch) + options = baseline_options if phase == 'baseline' else selected_options + fleet.record('A10-source-bound-contract', {**REFERENCE, 'branch': branch, 'fixture': fixture, 'phase': phase, + 'baselineOptions': baseline_options, 'selectedOptions': selected_options, + 'singleLogicalFieldDelta': 'pool/set selector pair' if branch.startswith('selector-') else branch}) + fleet.record('binding', binding) + fleet.record('inspector-build', verify_inspector(args.tool_directory, candidate)) + probe, active, recorder = Probe(binding), [], None + result = {'topology': topology, 'branch': branch, 'fixture': fixture, 'phase': phase, 'options': options, + 'verdict': 'RUNNING', 'cleanup': 'PENDING'} + seed_complete = False + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + recorder = FixtureRecorder(probe, directory / 'fixture-ACKs', fixture=fixture) + # Each phase owns a fresh namespace; failed/incomplete seeding cannot + # be hidden by a later default repair or by final namespace removal. + if is_orphan: + proof = orphan_probe(fleet, probe, binding, recorder, bucket, topology, fixture, options, active) + else: + proof = scope_probe(fleet, probe, binding, recorder, bucket, topology, fixture, branch, options, active) + seed_complete = True + result.update(verdict='PASS', proof=proof) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict='FAIL', failure={'type': type(exc).__name__, 'message': str(exc)}) + finally: + for path, token in active: + try: fleet.record('A10-stop-unresolved-original-token', probe.admin(path, [('clientToken', token), ('forceStop', 'true')], b'')) + except Exception as exc: fleet.record('A10-stop-error', {'error': str(exc)}) + unresolved = recorder is not None and recorder.pending is not None + incomplete_restore = hasattr(fleet, 'a10_orphan_authority') and not fleet.a10_orphan_restored + if unresolved or incomplete_restore: + try: fleet.stop() + except Exception as exc: fleet.record('A10-retain-stop-error', {'error': str(exc)}) + fleet.record('A10-unresolved-authority-retained', {'state': 'DIRTY', + 'request': recorder.pending if recorder is not None else None, 'namespacesRetained': True, + 'originalPostDeleteRestorationIncomplete': incomplete_restore}) + clean = False + else: + clean = fleet.cleanup() + result.update(cleanup='PASS' if clean else 'FAIL', fullProbeCompleted=seed_complete) + fleet.record('instance-result', result) + return result + + +def complete_results(results): + wanted = {(topology, branch, fixture, phase) for topology in ('H1', 'H2') for branch in BRANCHES + for fixture in FIXTURES for phase in ('baseline', 'selected')} + actual = {(row['topology'], row['branch'], row['fixture'], row['phase']) for row in results} + require(len(actual) == len(results) and actual <= wanted, 'A10 duplicate or foreign parameter instance') + return actual == wanted and all(row['verdict'] == row['cleanup'] == 'PASS' and row.get('fullProbeCompleted') is True + for row in results) + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f'A10 interrupted by signal {signum}') + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ('candidate', 'rc-bin', 'evidence', 'tool-directory'): + parser.add_argument('--' + name, type=Path, required=True) + parser.add_argument('--case-id', choices=[CASE_ID], required=True) + parser.add_argument('--testing-commit', required=True) + parser.add_argument('--binary-sha', required=True) + args = parser.parse_args() + require(re.fullmatch(r'[0-9a-f]{64}', args.binary_sha) and re.fullmatch(r'[1-9][0-9]*', os.environ.get('GITHUB_RUN_ID', '')), + 'A10 requires immutable workflow/binary identities') + require(select(CASE_ID)[0]['variants'] == list(BRANCHES), 'A10 omitted a source option') + nodes = os.environ.get('RUSTFS_NODES', 'vm000 vm001 vm002 vm003').split() + require(len(nodes) >= 4 and len(nodes) == len(set(nodes)) and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in nodes), + 'A10 requires four distinct current Heal nodes') + results = [] + try: + for topology in ('H1', 'H2'): + for branch in BRANCHES: + for fixture in FIXTURES: + for phase in ('baseline', 'selected'): + row = run_instance(args, topology, nodes[:1] if topology == 'H1' else nodes[:4], branch, fixture, phase) + results.append(row) + if row['cleanup'] != 'PASS': return + finally: + complete = complete_results(results) + save(args.evidence / 'healthy-summary.json', {'caseId': CASE_ID, 'results': results, 'complete': complete, + 'cleanup': 'PASS' if results and all(row['cleanup'] == 'PASS' for row in results) else 'FAIL'}) + require(complete, 'A10 requires every H1/H2 DB/DV baseline/selected option instance and complete cleanup') + + +if __name__ == '__main__': + try: main() + except Exception as exc: + print(f'A10 option validation failed: {exc}', file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 89ac82bf..b3c63b10 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -30,6 +30,7 @@ case "${case_id}" in heal.mrf-offline-write-replay) driver=mrf_replay.py ;; heal.performance-abba) driver=performance.py ;; heal.admin-scope-isolation) driver=scopes.py ;; + heal.admin-option-contract) driver=options.py ;; heal.bucket-metadata-integrity) driver=bucket_metadata.py ;; heal.disk-replacement-unformatted) driver=unformatted.py ;; heal.below-write-quorum) driver=write_quorum.py ;; diff --git a/scripts/validation_heal/write_quorum.py b/scripts/validation_heal/write_quorum.py index 6625c73d..c6fdbb48 100644 --- a/scripts/validation_heal/write_quorum.py +++ b/scripts/validation_heal/write_quorum.py @@ -108,6 +108,14 @@ def expected_reads(snapshot, node, relative): return result +def partition_snapshot(fleet, plan, label): + """Keep every physical member while attesting each actual process state.""" + offline = set(plan['offlineNodes']) + require(offline <= set(fleet.nodes), 'quorum snapshot has a foreign stopped node') + return fleet.record(label, {node: fleet.remote(node, 'snapshot', quiescent=node in offline) + for node in fleet.nodes}) + + def validate_gate_witness(observed, plan, online): require(set(observed) == set(online), "quorum probe omitted an available candidate") rows = [(node, row) for node, sample in observed.items() for row in sample["disks"]] @@ -240,7 +248,7 @@ def witness(label): expectedReads=expected_reads(original, node, relative)) for node in online}) return validate_gate_witness(raw, plan, online) fleet.record("quorums-before-operations", witness("service-quorum-before")) - require(physical_content(faulted) == physical_content(fleet.snapshot("physical-before-operations")), + require(physical_content(faulted) == physical_content(partition_snapshot(fleet, plan, "physical-before-operations")), "quorum loss changed the fault before its negative operations") read = fleet.record("read-without-write-admission", probe.s3("get_object", {"Bucket": bucket, "Key": selected["key"]})) require(read.get("acknowledged") is True and read.get("resultUnknown") is False diff --git a/test_validation_heal_options.py b/test_validation_heal_options.py new file mode 100644 index 00000000..cac85566 --- /dev/null +++ b/test_validation_heal_options.py @@ -0,0 +1,239 @@ +import copy +from pathlib import Path +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import options +from scripts.validation_heal.matrix import select, AUTOMATIC_CASES +from scripts.validation_heal.healthy import Fleet +import test_validation_heal_scopes as scope_fixtures + + +class OptionsTests(unittest.TestCase): + bucket = 'rv-heal-0123456789abcdef' + + def mapped(self): + rows = [(options.KEYS[0], 'null', 0, False), (options.KEYS[0], 'historic', 0, False), + (options.KEYS[1], 'child', 1, False), (options.KEYS[2], 'control', 1, False), + (options.KEYS[-1], 'marker', 0, True)] + return {(self.bucket, key, version): {'poolIndex': 0, 'setIndex': number, 'deleteMarker': marker} + for key, version, number, marker in rows} + + def status(self, branch='selector-valid'): + _, request = options.paired_options(branch) + mapped = self.mapped() + selected = options.selection(branch, request, mapped) + repaired = {identity for identity in selected if identity[2] == 'historic'} + receipts = [] + for identity in sorted(selected): + state = 'repaired' if identity in repaired else 'metadata_healthy' if mapped[identity]['deleteMarker'] else 'verified_healthy' + receipts.append({'identity': {'kind': 'object', 'bucket': identity[0], 'object': identity[1], + 'versionId': identity[2], 'poolIndex': request['pool'], 'setIndex': request['set'], + 'bucketIncarnationId': 'current-incarnation'}, 'disposition': {'state': state}}) + status = {'summary': 'finished', 'settings': request, 'outcome': {'execution': {'state': 'completed'}, + 'coverage': 'complete', 'objectsTruncated': False, 'objects': receipts, + 'counters': {'processed': len(selected), 'healed': len(repaired), 'unchanged': len(selected) - len(repaired), + 'failed': 0, 'unknown': 0, 'skipped': 0, 'attemptFailures': 0, 'overflowed': False}}} + return status, mapped, selected, repaired, request + + def test_all_source_options_topologies_and_two_fixture_types_are_preserved(self): + self.assertEqual(select(options.CASE_ID)[0]['variants'], list(options.BRANCHES)) + self.assertIn(options.CASE_ID, AUTOMATIC_CASES) + self.assertEqual(options.FIXTURES, ('DB', 'DV')) + self.assertEqual(options.REFERENCE['support']['readRepair'], + 'unsupported for Admin; HTTP 400 InvalidArgument before execution') + + def test_request_pairs_do_not_change_dangerous_or_unrelated_defaults(self): + for branch in options.BRANCHES: + baseline, selected = options.paired_options(branch) + self.assertFalse(baseline['remove']) + self.assertEqual(selected['remove'], branch == 'remove-true') + self.assertFalse(baseline['nolock']) + self.assertFalse(selected['nolock']) + self.assertEqual(baseline['scanMode'], selected['scanMode']) + changed = {name for name in baseline if baseline[name] != selected[name]} + self.assertEqual(len(changed), 2 if branch.startswith('selector-') else 0 if branch == 'remove-false' else 1) + + def test_prefix_exact_and_recursive_scopes_do_not_include_adjacent_control(self): + mapped = self.mapped() + baseline, selected = options.paired_options('recursive-true') + self.assertEqual(options.selection('recursive-true', baseline, mapped), set()) + identities = options.selection('recursive-true', selected, mapped) + self.assertEqual({identity[1] for identity in identities}, {options.KEYS[0], options.KEYS[1]}) + self.assertEqual(len(identities), 3) + self.assertNotIn((self.bucket, options.KEYS[2], 'control'), identities) + + def test_selector_uses_measured_sets_and_retains_historical_and_marker_identities(self): + status, mapped, selected, repaired, request = self.status() + proof = options.validate_outcome(status, mapped, selected, repaired, request) + self.assertEqual(proof['selectedACKs'], 3) + self.assertEqual(proof['repairedACKs'], 1) + self.assertIn((self.bucket, options.KEYS[-1], 'marker'), selected) + + def test_complete_oracle_rejects_lost_versions_wrong_scopes_markers_and_false_counters(self): + for kind in ('version', 'missing', 'duplicate', 'unknown', 'scope', 'null-scope', 'bool-scope', + 'incarnation', 'marker', 'counter', 'overflow', 'coverage', 'settings', 'truncated'): + with self.subTest(kind=kind): + status, mapped, selected, repaired, request = self.status() + receipt = status['outcome']['objects'][0] + if kind == 'version': receipt['identity']['versionId'] = 'unacknowledged' + elif kind == 'missing': status['outcome']['objects'].pop() + elif kind == 'duplicate': status['outcome']['objects'][-1] = copy.deepcopy(receipt) + elif kind == 'unknown': receipt['disposition']['state'] = 'unknown' + elif kind == 'scope': receipt['identity']['setIndex'] = 1 + elif kind == 'null-scope': receipt['identity']['poolIndex'] = None + elif kind == 'bool-scope': receipt['identity']['setIndex'] = False + elif kind == 'incarnation': receipt['identity']['bucketIncarnationId'] = '' + elif kind == 'marker': + marker = next(row for row in status['outcome']['objects'] if row['identity']['versionId'] == 'marker') + marker['disposition']['state'] = 'verified_healthy' + elif kind == 'counter': status['outcome']['counters']['healed'] += 1 + elif kind == 'overflow': status['outcome']['counters']['overflowed'] = True + elif kind == 'coverage': status['outcome']['coverage'] = 'partial' + elif kind == 'settings': status['settings'] = {**request, 'updateParity': True} + else: status['truncated'] = True + with self.assertRaises(ValueError): options.validate_outcome(status, mapped, selected, repaired, request) + + def test_nullable_unrestricted_scope_keeps_the_independent_physical_binding(self): + status, mapped, selected, repaired, request = self.status('recursive-true') + before = copy.deepcopy(status) + options.validate_outcome(status, mapped, selected, repaired, request) + self.assertEqual(before, status) + receipt = status['outcome']['objects'][0]['identity'] + identity = receipt['bucket'], receipt['object'], receipt['versionId'] + receipt.update(poolIndex=0, setIndex=1 - mapped[identity]['setIndex']) + with self.assertRaises(ValueError): options.validate_outcome(status, mapped, selected, repaired, request) + + def test_exact_missing_prefix_requires_truthful_not_found_and_no_object_receipts(self): + baseline, request = options.paired_options('recursive-false') + status = {'summary': 'finished', 'settings': request, 'detail': 'Object not found: ' + self.bucket + '/scope/', + 'outcome': {'coverage': 'complete', 'objectsTruncated': False, 'execution': {'state': 'completed_with_errors'}, + 'objects': [], 'counters': {'healed': 0, 'failed': 1, 'unknown': 0}}} + options.validate_outcome(status, self.mapped(), set(), set(), request) + for kind in ('success', 'repair', 'unknown', 'receipt', 'detail'): + changed = copy.deepcopy(status) + if kind == 'success': changed['outcome']['execution']['state'] = 'completed' + elif kind == 'repair': changed['outcome']['counters']['healed'] = 1 + elif kind == 'unknown': changed['outcome']['counters']['unknown'] = 1 + elif kind == 'receipt': changed['outcome']['objects'] = [{'disposition': {'state': 'unknown'}}] + else: changed['detail'] = '' + with self.assertRaises(ValueError): options.validate_outcome(changed, self.mapped(), set(), set(), request) + + def physical_scene(self): + fixture = scope_fixtures.ScopeTests() + original, mapped = fixture.snapshot(), fixture.mapping() + targets = [] + after = copy.deepcopy(original) + for number, target in enumerate(list(mapped.values())[:2]): + target = {**target, 'faultKind': 'missing' if number == 0 else 'bitflip', + 'faultBytes': None if number == 0 else {'sha256': 'flipped', 'size': target['size']}} + targets.append(target) + entries = after[target['node']]['files'] + entry = next(row for row in entries if (row['disk'], row['path']) == (target['disk'], target['relativePath'])) + if number == 0: entries.remove(entry) + else: entry['sha256'] = 'flipped' + return original, after, targets + + def test_rejected_request_keeps_missing_and_corrupted_controls_exact(self): + original, after, targets = self.physical_scene() + proof = options.physical_selection(original, after, targets, set()) + self.assertEqual(proof['selectedPhysicalRepairs'], 0) + self.assertEqual(proof['unselectedFaultsPreserved'], 2) + self.assertTrue(proof['beforeAnyTargetGETOrBackupRescue']) + + def test_unselected_control_repair_is_not_hidden_by_selected_repair_or_backup_rescue(self): + for control in (0, 1): + original, after, targets = self.physical_scene() + target = targets[control] + entries = after[target['node']]['files'] + entries[:] = [row for row in entries if (row['disk'], row['path']) != (target['disk'], target['relativePath'])] + entry = next(row for row in original[target['node']]['files'] + if (row['disk'], row['path']) == (target['disk'], target['relativePath'])) + entries.append(copy.deepcopy(entry)) + with self.assertRaisesRegex(ValueError, 'unselected'): options.physical_selection(original, after, targets, set()) + + def test_selected_physical_repair_leaves_corrupted_unselected_control_unchanged(self): + original, after, targets = self.physical_scene() + target = targets[0] + entry = next(row for row in original[target['node']]['files'] + if (row['disk'], row['path']) == (target['disk'], target['relativePath'])) + after[target['node']]['files'].append(copy.deepcopy(entry)) + selected = {(target['bucket'], target['key'], target['versionId'])} + proof = options.physical_selection(original, after, targets, selected) + self.assertEqual(proof['selectedPhysicalRepairs'], 1) + self.assertEqual(proof['unselectedFaultsPreserved'], 1) + + def test_wrong_or_absent_put_identity_remains_pending(self): + for response in ({'VersionId': 'not-a-version', 'ETag': 'etag'}, {'VersionId': 'null'}, + {'VersionId': '00000000-0000-0000-0000-000000000000', 'ETag': 'etag'}): + with tempfile.TemporaryDirectory() as name, patch.object(options.Recorder, 'call', return_value=response): + recorder = options.FixtureRecorder(types.SimpleNamespace(), Path(name) / 'ACKs') + with self.assertRaises(ValueError): recorder.call('put_object', {'Bucket': self.bucket, 'Key': 'owned.bin'}, b'body') + self.assertIsNotNone(recorder.pending) + + def test_transport_or_receipt_failure_retains_original_request(self): + with tempfile.TemporaryDirectory() as name, patch.object(options.Recorder, 'call', side_effect=RuntimeError('lost ACK')): + recorder = options.FixtureRecorder(types.SimpleNamespace(), Path(name) / 'ACKs') + with self.assertRaises(RuntimeError): recorder.call('delete_object', {'Bucket': self.bucket, 'Key': 'owned.bin'}) + self.assertEqual(recorder.pending['operation'], 'delete_object') + + def test_wrong_exact_delete_or_marker_ack_never_clears_mutation_guard(self): + version = '11111111-1111-4111-8111-111111111111' + for params, response in (({'VersionId': version}, {'VersionId': '22222222-2222-4222-8222-222222222222'}), + ({}, {'VersionId': version, 'DeleteMarker': False}), + ({}, {'VersionId': 'null', 'DeleteMarker': True})): + with tempfile.TemporaryDirectory() as name, patch.object(options.Recorder, 'call', return_value=response): + recorder = options.FixtureRecorder(types.SimpleNamespace(), Path(name) / 'ACKs', fixture='DV') + with self.assertRaises(ValueError): recorder.call('delete_object', {'Bucket': self.bucket, 'Key': 'owned.bin', **params}) + self.assertIsNotNone(recorder.pending) + + def test_duplicate_put_ack_is_retained_even_with_successful_http(self): + response = {'VersionId': '11111111-1111-4111-8111-111111111111', 'ETag': 'etag'} + with tempfile.TemporaryDirectory() as name, patch.object(options.Recorder, 'call', return_value=response): + recorder = options.FixtureRecorder(types.SimpleNamespace(), Path(name) / 'ACKs', fixture='DV') + params = {'Bucket': self.bucket, 'Key': 'owned.bin'} + recorder.call('put_object', params, b'one') + self.assertIsNone(recorder.pending) + with self.assertRaises(ValueError): recorder.call('put_object', params, b'two') + self.assertIsNotNone(recorder.pending) + + def test_positive_http_with_wrong_dv_mode_is_not_a_resolved_fixture_ack(self): + fake = types.SimpleNamespace(pending=None) + fake.call = lambda operation, params, body=None: {'VersionId': 'null', 'ETag': 'etag'} + with self.assertRaisesRegex(ValueError, 'DB/DV'): options.seed_fixture(fake, self.bucket, 'DV') + self.assertIsNotNone(fake.pending) + + def test_no_single_branch_checkpoint_or_duplicate_can_certify_canonical_completion(self): + rows = [{'topology': topology, 'branch': branch, 'fixture': fixture, 'phase': phase, + 'verdict': 'PASS', 'cleanup': 'PASS', 'fullProbeCompleted': True} + for topology in ('H1', 'H2') for branch in options.BRANCHES for fixture in options.FIXTURES + for phase in ('baseline', 'selected')] + self.assertEqual(len(rows), 72) + self.assertTrue(options.complete_results(rows)) + self.assertFalse(options.complete_results(rows[:-1])) + for name, value in (('verdict', 'FAIL'), ('cleanup', 'FAIL'), ('fullProbeCompleted', False)): + changed = copy.deepcopy(rows); changed[-1][name] = value + self.assertFalse(options.complete_results(changed)) + with self.assertRaises(ValueError): options.complete_results([*rows, rows[0]]) + + def test_rejected_successful_admission_is_retained_for_original_token_cancellation(self): + probe = types.SimpleNamespace(admin=lambda *args: {'httpStatus': 200, 'response': {'clientToken': 'wrong-admission'}}) + fleet = types.SimpleNamespace(record=lambda label, value: value) + active = [] + with self.assertRaises(ValueError): options.run_request(probe, fleet, '/rustfs/admin/v3/heal/', + options.paired_options('read-repair')[1], active, rejected=True) + self.assertEqual(active, [('/rustfs/admin/v3/heal/', 'wrong-admission')]) + + def test_a10_orphan_and_scope_node_programs_compile_with_the_actual_adapter(self): + def invoke(argv, **params): + compile(params['input'], 'remote-option-node', 'exec') + return types.SimpleNamespace(returncode=0, stdout='{"checked":true}', stderr='') + for profile in ('scope-isolation', 'orphan-preserve'): + with tempfile.TemporaryDirectory() as name, patch('scripts.validation_heal.healthy.subprocess.run', side_effect=invoke): + fleet = Fleet(['n0', 'n1', 'n2', 'n3'], Path(name), 'gh-1234-1-h2', runtime_profile=profile) + self.assertTrue(fleet.remote('n0', 'process-metrics')['checked']) + + +if __name__ == '__main__': unittest.main() diff --git a/test_validation_heal_write_quorum.py b/test_validation_heal_write_quorum.py index 8c99d088..78d84452 100644 --- a/test_validation_heal_write_quorum.py +++ b/test_validation_heal_write_quorum.py @@ -15,6 +15,25 @@ class WriteQuorumTests(unittest.TestCase): contract = {"recognized": True, "writeQuorumRule": "data-plus-one-if-equal-parity"} + def test_partition_snapshot_keeps_stopped_members_and_attests_online_processes(self): + import types + calls = [] + def remote(node, action, **params): + calls.append((node, action, params)) + return {'files': [node], 'formats': [node]} + fleet = types.SimpleNamespace(nodes=['n0', 'n1', 'n2', 'n3'], remote=remote, + record=lambda label, value: value) + result = executor.partition_snapshot(fleet, {'offlineNodes': ['n2', 'n3']}, 'partition') + self.assertEqual(set(result), set(fleet.nodes)) + self.assertEqual(calls, [(node, 'snapshot', {'quiescent': node in {'n2', 'n3'}}) for node in fleet.nodes]) + self.assertEqual(result['n2']['formats'], ['n2']) + + def test_partition_snapshot_rejects_foreign_process_bindings_before_reading(self): + import types + def forbidden(*args, **params): self.fail('foreign partition performed a node operation') + fleet = types.SimpleNamespace(nodes=['n0'], remote=forbidden, record=forbidden) + with self.assertRaises(ValueError): executor.partition_snapshot(fleet, {'offlineNodes': ['foreign']}, 'partition') + def scene(self, total=4, branch=executor.BRANCHES[0]): source = oracle.HealReadQuorumBoundaryTest() snapshot, expected = source.scene(total=total, data=total // 2) From b75f4a7602dca463c5aa532133ddc022f132051f Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 10:42:25 +0800 Subject: [PATCH 117/343] feat(validation): inspect current Heal resources without mutation --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 25 +++++ scripts/validation_heal/environment.py | 104 ++++++++++++++++++ scripts/validation_heal/fleet_node.py | 41 +++++++ test_validation_heal_environment.py | 74 +++++++++++++ 5 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 scripts/validation_heal/environment.py create mode 100644 test_validation_heal_environment.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 7d7da719..a53ff03b 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -36,6 +36,7 @@ on: - test_validation_heal_authority.py - test_validation_heal_orphan.py - test_validation_heal_options.py + - test_validation_heal_environment.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -85,6 +86,7 @@ on: - test_validation_heal_authority.py - test_validation_heal_orphan.py - test_validation_heal_options.py + - test_validation_heal_environment.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -122,7 +124,7 @@ jobs: sudo apt-get update sudo apt-get install --no-install-recommends -y nftables fi - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 6bcebd4e..86bb5441 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1669,3 +1669,28 @@ A10/B08 local suite passes 419 tests with eight macOS skips. There are now 133 automatic executors (74 non-Heal, 59 Heal), 25 remaining Heal executors and 49/158 complete canonical live PASS results. All original 158 IDs/hashes and 84/301/532 Heal recipes/branches/topology templates remain unchanged. + + +All six A10/B08 checks passed at `7128b50818de3206f6fb368d4ea696e56da7206b`. +Original Linux run `38105302558` executed all 419 tests with no skips. The +A10/B08 live plan retains that exact revision and successful inspector attempt +`38103474555`, under the original fleet lock and cleanup guard. + +`environment.inspect` is a separate read-only workflow operation, outside the +158 canonical IDs. It records the current four Heal hosts, all eight possible +mount endpoints on each, filesystem and physical-device identities, available +space and kernel/command interfaces. A shared partition, stacked/virtual +device, ordinary directory, duplicate filesystem identity or insufficient +space cannot certify an independent endpoint. H3/H4 resource shapes remain +separate from actual EC8+4 layout and fault proof. An installed cloud command +cannot certify platform restart permission or a compatible release pair. +No namespace is reserved, no service is stopped or installed, and no case is +scored. Five regressions guard these boundaries; the updated complete local +suite passes 424 tests with eight macOS skips. + +The six pending live plans now write their downloaded original evidence to +`/Volumes/Developer/KAI/validation-migration-evidence`; their candidate, tool +and harness identities are unchanged. This moves only future outputs because +the system disk has about one GiB free. The currently active flapping/MRF plan +and every existing artifact remain at their original locations. The local +`pending-output-locations.json` records each prior/new destination. diff --git a/scripts/validation_heal/environment.py b/scripts/validation_heal/environment.py new file mode 100644 index 00000000..93969f29 --- /dev/null +++ b/scripts/validation_heal/environment.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Read the current Heal fleet's capabilities; never execute or score a case.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +try: + from .heal_probe import require, save + from .healthy import Fleet +except ImportError: + from heal_probe import require, save + from healthy import Fleet + + +def eligible_mounts(value): + rows = value.get('mounts') + require(value.get('inspectionOnly') is True and value.get('mutated') is False + and value.get('canonicalPassClaimed') is False and isinstance(rows, list) and len(rows) == 8, + 'incomplete or mutating environment inspection') + require({row.get('target') for row in rows} == {f'/data/rustfs{number}' for number in range(1, 9)}, + 'environment inspection duplicated or omitted a mount endpoint') + eligible = {} + for row in rows: + if not row['mounted']: continue + mount = row.get('mount') or {} + if (row.get('independentBlockDeviceVerified') is True and row.get('writableMount') is True + and mount.get('uuid') and mount.get('target') == row['target'] + and mount.get('fstype') in {'xfs', 'ext4'} and row.get('availableBytes', 0) >= 2 * 1024**3): + eligible[row['target']] = row + # Two partitions from the same disk are one failure domain. Aliases or + # duplicated filesystem IDs cannot be counted as independent endpoints. + physical = [row['physicalDevice'] for row in eligible.values()] + identifiers = [row['mount']['uuid'] for row in eligible.values()] + require(len(physical) == len(set(physical)) and len(identifiers) == len(set(identifiers)), + 'environment inspection found aliased physical failure domains') + return eligible + + +def summarize(observed): + require(len(observed) == 4 and len({value.get('hostname') for value in observed.values()}) == 4 + and all(value.get('hostname') for value in observed.values()), 'inspection requires four actual distinct current Heal hosts') + mounts = {node: eligible_mounts(value) for node, value in observed.items()} + identifiers = [row['mount']['uuid'] for rows in mounts.values() for row in rows.values()] + require(len(identifiers) == len(set(identifiers)), 'environment inspection found shared or cloned filesystem identities') + nodes = list(observed) + available = lambda selected, count: all({f'/data/rustfs{number}' for number in range(1, count + 1)} <= set(mounts[node]) + for node in selected) + return {'independentEligibleEndpoints': sum(len(rows) for rows in mounts.values()), + 'eligibleEndpointsPerNode': {node: sorted(rows) for node, rows in mounts.items()}, + 'resourceShapes': {'H1': {'mountsAvailable': available(nodes[:1], 4), 'requiredEndpoints': 4}, + 'H2': {'mountsAvailable': available(nodes, 4), 'requiredEndpoints': 16}, + 'H3': {'mountsAvailable': available(nodes[:3], 4), 'requiredEndpoints': 12}, + 'H4': {'mountsAvailable': available(nodes[:3], 8), 'requiredEndpoints': 24}, + 'H6': {'platformRestartAuthorityVerified': False}}, + 'actualErasureLayoutVerified': False, 'faultBackendVerified': False, + 'compatibleReleasePairVerified': False, 'canonicalPassClaimed': False} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--candidate', type=Path, required=True) + parser.add_argument('--testing-commit', required=True) + parser.add_argument('--evidence', type=Path, required=True) + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + require(subprocess.check_output(['git', '-C', str(root), 'rev-parse', 'HEAD'], text=True).strip() == args.testing_commit + and re.fullmatch(r'[0-9a-f]{40}', args.testing_commit), 'environment inspection requires an exact harness checkout') + candidate = json.loads(args.candidate.read_text()) + require(re.fullmatch(r'[0-9a-f]{40}', candidate['source_sha']) and re.fullmatch(r'[0-9a-f]{64}', candidate['package_sha256']), + 'environment inspection has an invalid candidate identity') + workflow = os.environ.get('GITHUB_RUN_ID', '') + attempt = os.environ.get('GITHUB_RUN_ATTEMPT', '1') + require(re.fullmatch(r'[1-9][0-9]*', workflow) and re.fullmatch(r'[1-9][0-9]*', attempt), 'environment inspection requires workflow provenance') + nodes = (os.environ.get('RUSTFS_NODES') or 'vm000 vm001 vm002 vm003').split() + require(len(nodes) >= 4 and len(set(nodes)) == len(nodes) and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in nodes), + 'invalid current Heal node binding') + user = os.environ.get('RUSTFS_SSH_USER') or 'azureuser' + require(re.fullmatch(r'[A-Za-z_][A-Za-z0-9_-]*', user), 'invalid current Heal SSH user') + os.environ['RUSTFS_SSH_USER'] = user + args.evidence.mkdir(parents=True, exist_ok=False) + fleet = Fleet(nodes[:4], args.evidence, f'gh-{workflow}-{attempt}-h2') + result = {'schemaVersion': 1, 'runId': workflow, 'runAttempt': attempt, 'testingCommit': args.testing_commit, + 'candidate': candidate, 'inspectionSourceSha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + 'inspectionOnly': True, 'mutated': False, 'canonicalPassClaimed': False, 'complete': False} + try: + observed = {node: fleet.record('node-capabilities', {'node': node, **fleet.remote(node, 'environment-capabilities')}) + for node in fleet.nodes} + result.update(nodes=observed, resources=summarize(observed), complete=True) + finally: + save(args.evidence / 'environment-summary.json', result) + print(json.dumps(result['resources'], sort_keys=True)) + + +if __name__ == '__main__': + try: main() + except Exception as exc: + print(f'environment inspection failed: {exc}', file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 3514a11a..55bc4a26 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -143,6 +143,46 @@ def mounts(): return observed +def environment_capabilities(request): + """Read existing mounts and kernel interfaces without reserving a run.""" + paths(request['runId']) + observed = [] + for number in range(1, 9): + mount = Path(f'/data/rustfs{number}') + row = {'target': str(mount), 'mounted': False, 'independentBlockDeviceVerified': False} + raw = command('findmnt', '--json', '--mountpoint', str(mount), '--output', + 'TARGET,SOURCE,FSTYPE,UUID,MAJ:MIN,OPTIONS', check=False) + require(raw.returncode in {0, 1}, 'environment mount inventory failed') + if raw.returncode == 0: + values = json.loads(raw.stdout).get('filesystems', []) + require(len(values) == 1 and values[0].get('target') == str(mount), 'environment inventory has an ambiguous mount') + value = values[0] + row.update(mounted=True, mount=value, availableBytes=shutil.disk_usage(mount).free) + device = value.get('maj:min', '') + require(re.fullmatch(r'[0-9]+:[0-9]+', device), 'environment inventory omitted the actual device identity') + link = Path('/sys/dev/block') / device + if link.exists(): + block = link.resolve(strict=True) + if (block / 'partition').exists(): block = block.parent + # Stacked/virtual devices need a separately reviewed fault + # backend; directory or partition counts cannot prove H4. + stacked = (block / 'slaves').is_dir() and any((block / 'slaves').iterdir()) + physical = (block / 'dev').read_text().strip() + require(re.fullmatch(r'[0-9]+:[0-9]+', physical), 'environment physical device is ambiguous') + row.update(physicalDevice=physical, blockName=block.name, + independentBlockDeviceVerified=not stacked and not block.name.startswith(('loop', 'dm-', 'ram'))) + row['writableMount'] = 'rw' in value.get('options', '').split(',') + observed.append(row) + binary = Path('/usr/bin/rustfs') + return {'hostname': socket.gethostname(), 'mounts': observed, 'inspectionOnly': True, 'mutated': False, + 'kernel': {'pidfd': hasattr(os, 'pidfd_open'), 'cgroupV2': Path('/sys/fs/cgroup/cgroup.controllers').is_file(), + 'fuseDevice': Path('/dev/fuse').exists() and stat.S_ISCHR(Path('/dev/fuse').stat().st_mode)}, + 'commands': {name: shutil.which(name) is not None for name in ('docker', 'fusermount3', 'nft', 'az')}, + 'installedBinarySha256': digest(binary) if binary.is_file() and not binary.is_symlink() else None, + 'platformRestartAuthorityVerified': False, 'compatibleReleasePairVerified': False, + 'canonicalPassClaimed': False} + + def resource_limits(profile): require(profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") # DX has 1,025 external objects on four local roots. The candidate keeps @@ -1129,6 +1169,7 @@ def main(request): "drive-gate-activate": drive_gate_activate, "gate-state": drive_gate_state, "drive-gate-release-live": drive_gate_release_live, "drive-gate-arm-live": drive_gate_arm_live} actions["task-records"] = task_records + actions["environment-capabilities"] = environment_capabilities for action in ('generation-capture', 'generation-install', 'generation-recover'): actions[action] = generation_operation for action in ('orphan-install-tool', 'orphan-observe', 'orphan-multipart', 'orphan-convert', diff --git a/test_validation_heal_environment.py b/test_validation_heal_environment.py new file mode 100644 index 00000000..e83f88a2 --- /dev/null +++ b/test_validation_heal_environment.py @@ -0,0 +1,74 @@ +import copy +from pathlib import Path +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import environment +from scripts.validation_heal.healthy import Fleet + + +class EnvironmentTests(unittest.TestCase): + def fixture(self, count=4): + return {f'n{node}': {'hostname': f'actual-host-{node}', 'inspectionOnly': True, 'mutated': False, + 'canonicalPassClaimed': False, 'mounts': [{'target': f'/data/rustfs{number}', 'mounted': number <= count, + 'independentBlockDeviceVerified': number <= count, 'writableMount': True, + 'physicalDevice': f'8:{number}', 'availableBytes': 4 * 1024**3, + 'mount': {'uuid': f'uuid-{node}-{number}', 'target': f'/data/rustfs{number}', 'fstype': 'xfs'}} + for number in range(1, 9)]} for node in range(4)} + + def test_four_current_mounts_prove_resources_for_h3_and_do_not_invent_h4(self): + result = environment.summarize(self.fixture()) + self.assertEqual(result['independentEligibleEndpoints'], 16) + self.assertTrue(result['resourceShapes']['H3']['mountsAvailable']) + self.assertFalse(result['resourceShapes']['H4']['mountsAvailable']) + self.assertFalse(result['actualErasureLayoutVerified']) + self.assertFalse(result['canonicalPassClaimed']) + self.assertFalse(result['resourceShapes']['H6']['platformRestartAuthorityVerified']) + + def test_eight_independent_mounts_are_only_a_resource_shape_not_live_ec84_proof(self): + result = environment.summarize(self.fixture(8)) + self.assertEqual(result['independentEligibleEndpoints'], 32) + self.assertTrue(result['resourceShapes']['H4']['mountsAvailable']) + self.assertEqual(result['resourceShapes']['H4']['requiredEndpoints'], 24) + self.assertFalse(result['faultBackendVerified']) + self.assertFalse(result['compatibleReleasePairVerified']) + + def test_partition_aliases_missing_endpoints_and_shared_filesystem_identities_fail(self): + for kind in ('partition', 'endpoint', 'shared', 'host', 'mutated', 'pass'): + observed = self.fixture() + if kind == 'partition': observed['n0']['mounts'][1]['physicalDevice'] = '8:1' + elif kind == 'endpoint': observed['n0']['mounts'][-1] = copy.deepcopy(observed['n0']['mounts'][0]) + elif kind == 'shared': observed['n1']['mounts'][0]['mount']['uuid'] = observed['n0']['mounts'][0]['mount']['uuid'] + elif kind == 'host': observed['n1']['hostname'] = observed['n0']['hostname'] + elif kind == 'mutated': observed['n0']['mutated'] = True + else: observed['n0']['canonicalPassClaimed'] = True + with self.subTest(kind=kind), self.assertRaises(ValueError): environment.summarize(observed) + + def test_directories_stacked_devices_readonly_mounts_or_low_space_do_not_count(self): + for kind in ('directory', 'stacked', 'readonly', 'space', 'fs'): + observed = self.fixture() + row = observed['n0']['mounts'][0] + if kind == 'directory': row['mounted'] = False + elif kind == 'stacked': row['independentBlockDeviceVerified'] = False + elif kind == 'readonly': row['writableMount'] = False + elif kind == 'space': row['availableBytes'] = 1024 + else: row['mount']['fstype'] = 'tmpfs' + result = environment.summarize(observed) + self.assertEqual(result['independentEligibleEndpoints'], 15) + self.assertFalse(result['resourceShapes']['H1']['mountsAvailable']) + + def test_current_remote_adapter_includes_a_compilable_readonly_inventory_action(self): + def invoke(argv, **params): + source = params['input'] + compile(source, 'remote-inventory', 'exec') + self.assertIn('environment_capabilities', source) + self.assertIn('environment-capabilities', source) + return types.SimpleNamespace(returncode=0, stdout='{"inspectionOnly":true,"mutated":false}', stderr='') + with tempfile.TemporaryDirectory() as name, patch('scripts.validation_heal.healthy.subprocess.run', side_effect=invoke): + fleet = Fleet(['n0'], Path(name), 'gh-1234-1-h2') + self.assertFalse(fleet.remote('n0', 'environment-capabilities')['mutated']) + + +if __name__ == '__main__': unittest.main() From e40fd9d74a902b9b9733c7bddfdc0b9d1b3f5c03 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 11:07:22 +0800 Subject: [PATCH 118/343] feat(validation): preserve I04 pressure and audited quota recovery --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-heal-operations.md | 2 +- docs/validation-migration-map.md | 41 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/capacity.py | 24 +- scripts/validation_heal/fleet_node.py | 38 +- scripts/validation_heal/healthy.py | 8 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/mrf_records.py | 2 +- scripts/validation_heal/queue_pressure.py | 456 ++++++++++++++++++ .../validation_heal/recover_environment.py | 91 +++- scripts/validation_heal/run_case.sh | 1 + test_validation_heal_queue_pressure.py | 168 +++++++ test_validation_heal_recovery.py | 128 ++++- 14 files changed, 947 insertions(+), 20 deletions(-) create mode 100644 scripts/validation_heal/queue_pressure.py create mode 100644 test_validation_heal_queue_pressure.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index a53ff03b..e1f3d124 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -37,6 +37,7 @@ on: - test_validation_heal_orphan.py - test_validation_heal_options.py - test_validation_heal_environment.py + - test_validation_heal_queue_pressure.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -87,6 +88,7 @@ on: - test_validation_heal_orphan.py - test_validation_heal_options.py - test_validation_heal_environment.py + - test_validation_heal_queue_pressure.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -124,7 +126,7 @@ jobs: sudo apt-get update sudo apt-get install --no-install-recommends -y nftables fi - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 9ca0b042..4cdb5c31 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -2,7 +2,7 @@ 此手册和 [84 项逐例操作卡](validation-heal-cases.md) 一起构成可执行的**人工操作方案 + 脚本辅助**。不是 84 个无人值守自动测试，也不是通过报告。操作卡列出 301 个分支模板；真实盘角色、阈值、损坏数量和拓扑仍需在每批 `parameterBindings` 展开，不能把模板数量当实测次数。 -公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 已接入 47 个 Heal 用例，完整清单通过 `matrix.py list` 查看，以 `automaticExecutor` 为准。入口使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 +公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 已接入 60 个 Heal 用例，完整清单通过 `matrix.py list` 查看，以 `automaticExecutor` 为准。入口使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 ## 1. 使用入口与交付物 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 86bb5441..014d56f4 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 133 currently dispatchable IDs include all 74 non-Heal cases and fifty-nine Heal cases. Forty-nine have complete canonical live PASS evidence; the other 109 have no live PASS claim. +case evidence. The 134 currently dispatchable IDs include all 74 non-Heal cases and sixty Heal cases. Forty-nine have complete canonical live PASS evidence; the other 109 have no live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1694,3 +1694,42 @@ and harness identities are unchanged. This moves only future outputs because the system disk has about one GiB free. The currently active flapping/MRF plan and every existing artifact remain at their original locations. The local `pending-output-locations.json` records each prior/new destination. + + +I04 `heal.queue-pressure-and-admission` now has all four source branches on +H1/H2. Its existing private Heal services freeze a two-slot queue and one +worker. The 769/129/129/129-object independent buckets have full ACK and +physical manifests; Merged returns the same token, Dropped carries its exact +structured overlap reason, and Full requires HTTP 503 SlowDown with +`admission=full, reason=none`. A live private drive refusal creates three +exact degraded-write ACKs. Actual Normal-priority MRF Full events must occur +while all three original committed responsibilities remain active. Recovery +keeps the original process, and retirement requires each successful exact +source task followed by all-member physical checks and every peer's complete +S3 manifest. Queue/source/priority counts, request latency, RSS, FD and thread +curves remain bounded by frozen budgets. Twelve negative oracle tests reject +false admission, wrong identities, premature retirement and resource drift. +No I04 canonical live PASS is claimed. There are now 134 automatic executors +(74 non-Heal, 60 Heal) and 24 remaining Heal executors. + +Read-only current-fleet inspection `38106333882` at helper +`b75f4a7602dca463c5aa532133ddc022f132051f` found 16 independent eligible +endpoints: four on each of the existing four Heal nodes. H1/H2/H3 resource +shapes are available; H4's required 24 endpoints are unavailable. This does +not prove actual EC layout or an H07 PASS. H6 restart authority and a +compatible immutable release pair remain unverified. Original Linux contract +run `38106020097` executed all 424 tests without skips. + +F04 `38105011828` remains FAIL with cleanup PASS: root canonical counters, +MRF partial replay admission and replacement disk identity failed their +independent gates. I02 `38106408335` remains FAIL with cleanup FAIL: bounded +block exhaustion was not witnessed, inode repair admission failed, and the +quota GET returned usage-unavailable before workload/Heal admission. Failure +recovery incorrectly called the stopped service. Quota rescue now restarts +only its owned service and independently checks config without claiming +usage or product repair. The narrowly supported archived H1 quota failure +can also restore both original missing parts against an unchanged complete +scene and remove its private bucket/quota namespace without starting a +service. Invoked/uncertain PUTs, changed candidate/run/versions, missing fault +receipts or another dirty instance reject finalization. Original failure +results remain immutable; any subsequent recovery has separate evidence. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 4800a326..c36f9dce 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/capacity.py b/scripts/validation_heal/capacity.py index 8528b9bd..78cc50c3 100644 --- a/scripts/validation_heal/capacity.py +++ b/scripts/validation_heal/capacity.py @@ -75,13 +75,18 @@ def refusal(observed, *, quota): return {"state": "REFUSED", "code": error["Code"], "retryAllowed": False} -def quota_config(probe, fleet, bucket, limit): +def quota_set_ack(probe, fleet, bucket, limit): path = "/rustfs/admin/v3/quota/" + bucket body = json.dumps({"quota": limit, "quota_type": "HARD"}).encode() configured = fleet.record("owned-bucket-quota-PUT", probe.admin(path, [], body, method="PUT")) require(configured.get("httpStatus") == 200 and configured.get("response", {}).get("bucket") == bucket and configured["response"].get("quota") == limit and configured["response"].get("quota_type") == "HARD", "hard quota was not acknowledged on the exact owned bucket") + return path + + +def quota_config(probe, fleet, bucket, limit): + path = quota_set_ack(probe, fleet, bucket, limit) for endpoint in probe.binding["endpoints"]: observed = fleet.record("all-peer-owned-quota-GET", Probe(probe.binding, endpoint).admin(path, [], b"", method="GET")) require(observed.get("httpStatus") == 200 and observed.get("response", {}).get("bucket") == bucket @@ -90,6 +95,21 @@ def quota_config(probe, fleet, bucket, limit): return {"bucket": bucket, "limitBytes": limit, "allPeersVerified": True, "sharedFilesystemQuotaChanged": False} +def release_quota(probe, fleet, bucket, limit): + # The failure scene is already archived before rescue. Config-only GET + # proves the restored limit without claiming an authoritative usage value. + if any(fleet.remote(node, "process")["pid"] == 0 for node in fleet.nodes): + fleet.start() + quota_set_ack(probe, fleet, bucket, limit) + for endpoint in probe.binding["endpoints"]: + observed = fleet.record("recovery-config-only-quota", Probe(probe.binding, endpoint).admin( + "/rustfs/admin/v3/get-bucket-quota", [("bucket", bucket)], b"", method="GET")) + value = observed.get("response") or {} + require(observed.get("httpStatus") == 200 and value.get("quota") == value.get("size") == limit + and value.get("quotatype") == "hard", "quota recovery did not restore the exact owned config") + return {"ownedQuotaConfigRestored": True, "authoritativeUsageClaimed": False, "productRepairClaimed": False} + + def verify_s3(fleet, binding, bucket, expected, baseline, pending): result = {} for endpoint in binding["endpoints"]: @@ -153,7 +173,7 @@ def release(): fleet.record("release-only-owned-filler", {"node": node, **fleet.remote( node, "bounded-volumes-release", quiescent=True)}) elif quota_set: - quota_config(probe, fleet, bucket, CONTRACT["quota"]["releasedLimitBytes"]) + fleet.record("quota-recovery-config-proof", release_quota(probe, fleet, bucket, CONTRACT["quota"]["releasedLimitBytes"])) quota_set = False try: diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 55bc4a26..1c221afd 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -33,7 +33,15 @@ def runtime_config(profile): - require(profile in {"orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(profile in {"queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + if profile == "queue-pressure": + logging_filter = "info,rustfs::heal::manager=debug,rustfs::heal::task=debug,rustfs::heal::mrf=debug" + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true", + "RUSTFS_OBS_LOGGER_LEVEL": logging_filter, "RUST_LOG": logging_filter, + "RUSTFS_HEAL_QUEUE_SIZE": "2", "RUSTFS_HEAL_MAX_CONCURRENT_HEALS": "1", + "RUSTFS_HEAL_MAX_CONCURRENT_PER_SET": "1", "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false", + "RUSTFS_HEAL_OVERLAP_POLICY": "merge", "RUSTFS_HEAL_LOW_PRIORITY_MERGE_ENABLE": "true", + "RUSTFS_HEAL_LOW_PRIORITY_DROP_WHEN_FULL": "true", "RUSTFS_HEAL_MRF_REPLAY_BATCH": "1"} if profile in {"orphan-preserve", "orphan-before", "orphan-after"}: return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS": {"orphan-preserve": "0", "orphan-before": "300", "orphan-after": "30"}[profile]} @@ -296,7 +304,7 @@ def start(request): "--property=RuntimeMaxSec=" + str(resource_limits(owner.get("resourceProfile", "standard"))["runtimeMaxSeconds"]), "--property=MemoryMax=4G", *(["--property=PrivateMounts=true", "--property=CapabilityBoundingSet=~CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH"] - if owner.get("runtimeProfile") == "flapping" else []), + if owner.get("runtimeProfile") in {"flapping", "queue-pressure"} else []), *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") in {"root-restart", "durable-mrf", "mrf-crash", "replacement", "replacement-target", "schema-refusal"} else []), "--property=WorkingDirectory=" + str(state), "--property=EnvironmentFile=" + str(state / "service.env"), @@ -663,7 +671,7 @@ def drive_gate_arm_live(request): """Re-arm one owned root without changing the running service's birth.""" require(set(request) == {"runId", "action", "disk", "expected"}, "live arm requires an exact owned process binding") state, roots, owner = owned(request["runId"]) - require(owner.get("runtimeProfile") == "flapping", "live arm requires the preconfigured flapping service") + require(owner.get("runtimeProfile") in {"flapping", "queue-pressure"}, "live arm requires a preconfigured private service") before = process(request) require(before["pid"] > 1 and isinstance(request["expected"], dict) and all(before.get(name) == request["expected"].get(name) for name in PROCESS_FIELDS), @@ -1044,6 +1052,29 @@ def audit_finalize(request): "backupRestorationUsed": False, "productRepairClaimed": False} +def quota_audit_finalize(request): + """Remove the private quota fixture only after exact pre-fault rescue.""" + require(set(request) == {"runId", "action", "bucket", "binarySha256", "expectedSceneSha256"}, + "quota finalization accepts only the exact archived namespace and scene") + state, roots, owner = owned(request["runId"]) + stopped(owner) + require(request["runId"].endswith("-h1") and owner.get("runtimeProfile") == "manual" + and owner.get("bucket") == request["bucket"] and request["binarySha256"] == owner["binarySha256"] + and digest(state / "rustfs") == owner["binarySha256"], "quota rescue candidate, bucket or private runtime differs") + require(owner.get("volumes") == [str(root / "data") for root in roots] + and all(not (state / name).exists() for name in ("network-gate.json", "drive-gate.json", "quorum-gate.json", "bounded-volumes.json")), + "quota rescue cannot remove a shared/foreign volume or another active fault backend") + observed = snapshot({"runId": request["runId"], "action": "snapshot", "quiescent": True}) + content = {key: observed[key] for key in ("files", "formats")} + scene_sha = hashlib.sha256(json.dumps(content, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + require(re.fullmatch(r"[0-9a-f]{64}", request.get("expectedSceneSha256", "")) is not None + and scene_sha == request["expectedSceneSha256"], "quota rescue did not restore the complete original fixture") + result = cleanup(request) + return {**result, "privateBucketQuotaNamespaceRemoved": True, "archivedPreFaultSceneEqual": True, + "sceneSha256": scene_sha, "serviceStartedDuringRecovery": False, + "backupRestorationUsed": True, "productRepairClaimed": False} + + def task_records(request): """Read three exact Admin journal names; never mutate system metadata.""" state, roots, owner = owned(request["runId"]) @@ -1170,6 +1201,7 @@ def main(request): "drive-gate-release-live": drive_gate_release_live, "drive-gate-arm-live": drive_gate_arm_live} actions["task-records"] = task_records actions["environment-capabilities"] = environment_capabilities + actions["quota-audit-finalize"] = quota_audit_finalize for action in ('generation-capture', 'generation-install', 'generation-recover'): actions[action] = generation_operation for action in ('orphan-install-tool', 'orphan-observe', 'orphan-multipart', 'orphan-convert', diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 26a205ec..5bba7c2e 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(runtime_profile in {"queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -183,7 +183,7 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() - if action.startswith("network-") or action == "cleanup": + if action.startswith("network-") or action in {"cleanup", "quota-audit-finalize"}: network_source = NODE_SOURCE.with_name("network_gate.py").read_text() source += "\nimport types\nnetwork_gate_backend = types.ModuleType('network_gate')\nexec(" + repr( network_source) + ", network_gate_backend.__dict__)\nNETWORK_GATE_SOURCE = " + repr(network_source) + "\n" @@ -215,8 +215,8 @@ def remote(self, node, action, **params): if action in {"encryption-configure", "snapshot"}: source += "\nimport types\nencryption_runtime = types.ModuleType('encryption_runtime')\nexec(" + repr( NODE_SOURCE.with_name("encryption_runtime.py").read_text()) + ", encryption_runtime.__dict__)\n" - replacement_actions = {"start", "cleanup", "audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot", "format-schema-fault"} - if action in {"start", "cleanup", "audit-finalize", "drive-gate", "drive-gate-arm-live", "replace-drive", "quorum-gate", "quorum-gate-probe"}: + replacement_actions = {"start", "cleanup", "audit-finalize", "quota-audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot", "format-schema-fault"} + if action in {"start", "cleanup", "audit-finalize", "quota-audit-finalize", "drive-gate", "drive-gate-arm-live", "replace-drive", "quorum-gate", "quorum-gate-probe"}: source += "\nimport types\nquorum_gate_backend = types.ModuleType('quorum_gate')\nexec(" + repr( NODE_SOURCE.with_name("quorum_gate.py").read_text()) + ", quorum_gate_backend.__dict__)\n" if action in {"drive-gate", "drive-gate-arm-live", "drive-gate-probe", "drive-gate-activate", "gate-state", "drive-gate-release-live"} | replacement_actions: diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 4aba5315..6fbe1b3c 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.admin-option-contract", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/mrf_records.py b/scripts/validation_heal/mrf_records.py index 3b620e1c..694e50aa 100644 --- a/scripts/validation_heal/mrf_records.py +++ b/scripts/validation_heal/mrf_records.py @@ -50,7 +50,7 @@ def verify_reference(candidate): def read_records(state, roots, owner, checked): """Exact allowlist, no directory enumeration, no write descriptor.""" - require(owner.get("runtimeProfile") in {"durable-mrf", "mrf-crash", "flapping"}, "MRF records require the dedicated run-owned runtime") + require(owner.get("runtimeProfile") in {"durable-mrf", "mrf-crash", "flapping", "queue-pressure"}, "MRF records require the dedicated run-owned runtime") require(hashlib.sha256((state / "rustfs").read_bytes()).hexdigest() == owner["binarySha256"], "MRF candidate changed") def collect(): diff --git a/scripts/validation_heal/queue_pressure.py b/scripts/validation_heal/queue_pressure.py new file mode 100644 index 00000000..f89dcb52 --- /dev/null +++ b/scripts/validation_heal/queue_pressure.py @@ -0,0 +1,456 @@ +#!/usr/bin/env python3 +"""I04: bounded multi-bucket pressure and durable responsibility through Full.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +import secrets +import sys +import time +from urllib.parse import quote +import xml.etree.ElementTree as ET + +try: + from .heal_probe import Probe, compare_inventories, require + from .heal_workload import Recorder, seed, verify_seed + from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from .fleet_node import PROCESS_FIELDS, runtime_config + from .inspector import verify as verify_inspector + from .matrix import select + from .cursor import SIZE, map_cohorts, validate_missing_batch, verify_part_manifest, main as generic_main + from .shards import map_target, validate_repaired_physical, verify_cohort + from .heal_fault import bounded_fault_id + from .stop import status, idle_fleet + from .scanner_repair import JournalWindow, event_time + from .mrf_records import verify_reference, REFERENCE as MRF_REFERENCE + from .mrf_replay import (acknowledge, identity, pending_bindings, read_checkpoints, retirement, + source_tasks, verified_receipt) +except ImportError: + from heal_probe import Probe, compare_inventories, require + from heal_workload import Recorder, seed, verify_seed + from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from fleet_node import PROCESS_FIELDS, runtime_config + from inspector import verify as verify_inspector + from matrix import select + from cursor import SIZE, map_cohorts, validate_missing_batch, verify_part_manifest, main as generic_main + from shards import map_target, validate_repaired_physical, verify_cohort + from heal_fault import bounded_fault_id + from stop import status, idle_fleet + from scanner_repair import JournalWindow, event_time + from mrf_records import verify_reference, REFERENCE as MRF_REFERENCE + from mrf_replay import (acknowledge, identity, pending_bindings, read_checkpoints, retirement, + source_tasks, verified_receipt) + + +CASE_ID = "heal.queue-pressure-and-admission" +BRANCHES = ("full-queue", "dropped-admission", "merged-duplicates", "cross-bucket-fairness") +COUNTS = (769, 129, 129, 129) +CAPACITY = 2 +MAX_ADMIN_REQUESTS = 24 +PRESSURE_SECONDS = 45 +DRAIN_SECONDS = 600 +RESOURCE_BOUNDS = {"rssBytes": 3 * 1024**3, "fileDescriptors": 8192, "threads": 2048} +REFERENCE = {"candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", + "sources": {"crates/heal/src/heal/manager.rs": + "ba9289942fbbe8ce2632686ae86b9d1be0ae9923420e3b091d67148b4ee02094", + "crates/heal/src/heal/mrf_queue.rs": MRF_REFERENCE["sourceFiles"]["crates/heal/src/heal/mrf_queue.rs"]}, + "adminPriority": "High", "partialWritePriority": "Normal", "queueCapacity": CAPACITY, + "maxConcurrentHeals": 1, "maxConcurrentPerSet": 1, + "droppedReason": "overlapping_paths", "mergedRule": "same scope returns the existing token", + "fullRule": "HTTP 503 SlowDown; admission=full, reason=none", + "fairness": "every admitted owned bucket must finish within the frozen drain budget", + "persistentProof": "all exact degraded-write ACKs remain committed during MRF Full; each retires only with its verified source task", + "nativeCriticalWindowClaimed": False} + + +def rejection(observed, *, admission, reason, code, http): + require(observed.get("httpStatus") == http and not observed.get("transportError"), + "queue rejection has a wrong status or an unknown transport result") + value = observed.get("response") + if isinstance(value, str): + value = {element.tag.rsplit("}", 1)[-1]: element.text for element in ET.fromstring(value).iter()} + require(isinstance(value, dict) and not value.get("clientToken") + and (value.get("Code") or value.get("code")) == code + and (value.get("Message") or value.get("message")) == + f"heal request not admitted: admission={admission}, reason={reason}", + "queue rejection erased its actual admission/reason or invented a task") + return {"httpStatus": http, "code": code, "admission": admission, "reason": reason, + "independentRepairCount": 0} + + +def queue_snapshot(value, peers): + require(value.get("clusterStatusComplete") is True + and value.get("coverage", {}).get("expected") == peers + and value["coverage"].get("responded") == peers and value["coverage"].get("unknown") == 0, + "queue pressure has incomplete actual peer coverage") + operations = value.get("healOperations") or {} + for name in ("queueLength", "activeTasks", "retryingTasks"): + require(type(operations.get(name)) is int and operations[name] >= 0, "queue pressure omitted typed backlog") + require(operations["queueLength"] <= peers * CAPACITY and operations["activeTasks"] <= peers, + "queue or active work exceeds its frozen capacity") + require(value.get("healQueueLength") == operations["queueLength"] + and value.get("healActiveTasks") == operations["activeTasks"], "queue counters disagree") + for name in ("queuedByPriority", "activeByPriority", "queuedBySource", "activeBySource"): + counts = operations.get(name) + require(isinstance(counts, dict) and counts and all(type(item) is int and item >= 0 for item in counts.values()), + "queue sample omitted actual source/priority counters") + require(sum(counts.values()) == operations["queueLength" if name.startswith("queued") else "activeTasks"], + "queue source/priority counters hide work") + telemetry = operations.get("admission") or {} + require(all(type(telemetry.get(name)) is int and telemetry[name] >= 0 + for name in ("accepted", "full", "dropped", "merged")), "queue sample omitted actual admission telemetry") + return operations + + +def resources(samples, baseline): + require(set(samples) == set(baseline), "resource sample dropped a current owned node") + for node, current in samples.items(): + require(current.get("readOnly") is True and current.get("clockDomain") == baseline[node].get("clockDomain") + and type(current.get("monotonicNs")) is int and current["monotonicNs"] > baseline[node]["monotonicNs"] + and all(current.get("identity", {}).get(name) == baseline[node]["identity"].get(name) for name in PROCESS_FIELDS), + "resource curve crossed its original process birth or monotonic clock") + require(all(type(current.get(name)) is int and 0 <= current[name] <= limit + for name, limit in RESOURCE_BOUNDS.items()), "queue pressure exceeded a pre-registered resource ceiling") + return {"withinFrozenLimits": True, "nodes": list(samples), "limits": RESOURCE_BOUNDS} + + +def active_window(current, count): + outcome = current.get("outcome") or {} + processed = outcome.get("counters", {}).get("processed") + return (current.get("summary") == "running" and outcome.get("execution", {}).get("state") == "running" + and type(processed) is int and 0 < processed < count) + + +def terminal(current, bucket, expected, missing): + wanted = {(entry["key"], entry["versionId"]): entry for entry in expected} + require(len(wanted) == len(expected) and missing <= set(wanted), "queue scope escaped its frozen ACK set") + outcome = current.get("outcome") or {} + require(current.get("summary") == "finished" and outcome.get("execution") == {"state": "completed"} + and outcome.get("coverage") == "complete", "queue task did not complete its actual responsibility") + counts = {"processed": len(expected), "healed": len(missing), "unchanged": len(expected) - len(missing), + "failed": 0, "skipped": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False} + counters = outcome.get("counters") or {} + require(all(type(counters.get(name)) is type(value) and counters[name] == value for name, value in counts.items()), + "queue task counters hide unresolved, false or failed repairs") + rows = outcome.get("objects") + require(isinstance(rows, list) and 0 < len(rows) <= 128 and outcome.get("objectsTruncated") is (len(expected) > 128), + "queue task confused bounded diagnostics with the independent complete manifest") + seen, incarnation = set(), None + for row in rows: + item = row.get("identity") or {} + pair = item.get("object"), item.get("versionId") or "null" + require(item.get("bucket") == bucket and pair in wanted and pair not in seen + and type(item.get("poolIndex")) is int and item["poolIndex"] == 0 + and type(item.get("setIndex")) is int and item["setIndex"] == 0 + and isinstance(item.get("bucketIncarnationId"), str) and item["bucketIncarnationId"], + "queue receipt crossed a bucket/version/incarnation/set or duplicated a result") + incarnation = incarnation or item["bucketIncarnationId"] + require(item["bucketIncarnationId"] == incarnation + and row.get("disposition", {}).get("state") == ("repaired" if pair in missing else "verified_healthy"), + "queue receipt claimed an incorrect disposition or bucket generation") + seen.add(pair) + return {"canonicalCounters": counters, "bucketIncarnationId": incarnation, + "diagnosticObjects": len(rows), "completePhysicalAndS3ManifestRequiredSeparately": True} + + +def retained(checkpoints, obligations, bindings): + """A newer sequence may publish the same responsibility; a new owner may not.""" + observed = {} + for obligation in obligations: + key = identity(obligation) + require(key in bindings, "retention proof lacks its original exact ACK binding") + for owner in bindings[key]: + current = checkpoints.get(owner["node"], {}) + require(current.get("present") is True and current.get("owner") == owner["checkpointOwner"] + and type(current.get("sequence")) is int and current["sequence"] >= owner["sequence"], + "queue failure lost or replaced the original committed MRF owner") + matches = [item for item in current.get("intents", []) if identity(item) == key] + require(len(matches) == 1 and matches[0].get("intentDigest") == owner["intentDigest"] + and matches[0].get("lifecycle", {}).get("responsibility_id") == owner["responsibilityId"] + and matches[0]["lifecycle"].get("source_bucket_incarnation_id") == owner["bucketIncarnationId"] + and matches[0]["lifecycle"].get("state") == {"state": "active"} + and matches[0]["lifecycle"].get("last_operator_acceptance") is None, + "queue failure retired, held, operator-accepted or altered the exact durable responsibility") + observed[str(key)] = {"owners": len(bindings[key]), "originalResponsibilityRetained": True} + return observed + + +def mrf_full_events(events, nodes, after): + return [{"node": node, "event": row} for node in nodes for row in events.get(node, []) + if row.get("target") == "rustfs::heal::manager" and row.get("event") == "heal_queue_admission" + and row.get("source") == "mrf" and row.get("priority") == "Normal" + and row.get("result") == "rejected_full" and row.get("queue_capacity") == CAPACITY + and row.get("queue_len") == CAPACITY and isinstance(row.get("request_id"), str) and row["request_id"] + and event_time(row) is not None and event_time(row) >= after] + + +class Pressure: + def __init__(self, probe, fleet, binding, active): + self.probe, self.fleet, self.binding, self.active = probe, fleet, binding, active + self.requests, self.samples = 0, [] + self.baseline = {node: fleet.record("resource-baseline", {"node": node, **fleet.remote(node, "process-metrics")}) + for node in fleet.nodes} + + def call(self, path): + self.requests += 1 + require(self.requests <= MAX_ADMIN_REQUESTS, "pressure exceeded its pre-registered request ceiling") + begin = time.monotonic_ns() + observed = self.fleet.record("bounded-pressure-request", self.probe.admin(path, [], + json.dumps({**BASE_OPTIONS, "recursive": True}).encode())) + self.fleet.record("bounded-admission-latency", {"path": path, "requestNumber": self.requests, + "beginMonotonicNs": begin, "endMonotonicNs": time.monotonic_ns(), + "httpStatus": observed.get("httpStatus"), "errorsIncludedInRequestCount": True}) + return observed + + def admit(self, path): + observed = self.call(path) + token = (observed.get("response") or {}).get("clientToken") if isinstance(observed.get("response"), dict) else None + require(observed.get("httpStatus") == 200 and isinstance(token, str) and token, "pressure request lacks an accepted token") + self.active.append((path, token)) + return token + + def sample(self): + observed = self.fleet.record("actual-queue-sample", self.probe.admin("/rustfs/admin/v3/background-heal/status", [], b"")) + require(observed.get("httpStatus") == 200, "pressure status failed") + operations = queue_snapshot(observed.get("response") or {}, len(self.fleet.nodes)) + samples = {node: self.fleet.record("resource-curve", {"node": node, **self.fleet.remote(node, "process-metrics")}) + for node in self.fleet.nodes} + self.fleet.record("bounded-resource-proof", resources(samples, self.baseline)) + self.samples.append({"monotonicNs": time.monotonic_ns(), "operations": operations}) + return operations + + +def run_topology(args, topology, nodes, **unused): + directory = args.evidence / topology + directory.mkdir() + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="queue-pressure", resource_profile="cursor") + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == REFERENCE["candidateCommit"], "unsupported candidate queue contract") + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + fleet.record("MRF-contract", verify_reference(candidate)) + prefix = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat(), "bucketPrefix": prefix, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": sum(COUNTS) + 16, "maxInventoryBytes": (sum(COUNTS) + 16) * SIZE} + fleet.record("binding", binding) + fleet.record("frozen-pressure-budget", {"source": REFERENCE, "runtime": runtime_config("queue-pressure"), + "bucketObjectCounts": list(COUNTS), "objectBytes": SIZE, "pressureSeconds": PRESSURE_SECONDS, + "drainSeconds": DRAIN_SECONDS, "maxAdminRequests": MAX_ADMIN_REQUESTS, "resourceBounds": RESOURCE_BOUNDS, + "durableDegradedWrites": 3, "maxConcurrentClients": 1, "fourSourceBranches": list(BRANCHES)}) + require(set(select(CASE_ID)[0]["variants"]) == set(BRANCHES), "source queue branches changed") + probe, active, scopes = Probe(binding), [], [] + result = {"topology": topology, "verdict": "RUNNING", "cleanup": "PENDING", "branches": {}} + try: + fleet.deploy(args.binary_sha, prefix, access, secret) + for index, count in enumerate(COUNTS): + bucket = f"{prefix}-queue-{index}" + recorder = Recorder(probe, directory / f"fixture-{index}") + expected = seed(recorder, bucket, "DX", count, [SIZE]) + baseline = fleet.record("full-baseline", probe.inventory(bucket, unversioned_current_tags=True)) + fleet.record("ACK-full-manifest", {"bucket": bucket, "expected": expected, **verify_seed(expected, baseline)}) + scopes.append({"bucket": bucket, "expected": expected, "baseline": baseline, + "path": "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/objects/"}) + durable_bucket = prefix + "-durable" + durable_recorder = Recorder(probe, directory / "durable-fixture") + durable_expected = seed(durable_recorder, durable_bucket, "DB", 4, [SIZE]) + durable_baseline = fleet.record("durable-baseline", probe.inventory(durable_bucket, unversioned_current_tags=True)) + verify_seed(durable_expected, durable_baseline) + fleet.stop() + original = fleet.snapshot("physical-baseline", quiescent=True) + fleet.record("topology-proof", validate_topology(original, topology)) + targets = [] + for index, scope in enumerate(scopes): + scope["cohorts"] = map_cohorts(original_subset(original, scope["bucket"]), scope["bucket"], scope["expected"], count=COUNTS[index]) + verify_part_manifest(fleet, scope["cohorts"], "baseline-part-manifest", count=COUNTS[index]) + target = scope["cohorts"][-1] + scope["missing"] = {(target["key"], target["versionId"])} + targets.append(target) + params = {"disk": target["disk"], "relativePath": target["relativePath"], "kind": "missing", + "faultId": bounded_fault_id("I04", "queue", index), "expectedSha256": target["sha256"]} + fleet.faults.append((target["node"], params)) + scope["fault"] = fleet.record("exact-missing-part", {"node": target["node"], **fleet.remote(target["node"], "apply-fault", **params)}) + faulted = fleet.snapshot("physical-fault", quiescent=True) + fleet.record("fault-proof", validate_missing_batch(original, faulted, targets, [scope["fault"] for scope in scopes])) + fleet.start() + require(physical_content(faulted) == physical_content(fleet.snapshot("physical-before-pressure")), + "ATTRIBUTION_NOT_OBSERVED: an owned fault changed before pressure admission") + initial_checkpoints = read_checkpoints(fleet, nodes, "MRF-empty-baseline") + require(all(not item["intents"] for item in initial_checkpoints.values()), "pressure fixture has unrelated durable responsibility") + window = JournalWindow(fleet, "pressure-admission", event_targets={"rustfs::heal::manager", "rustfs::heal::task"}) + pressure = Pressure(probe, fleet, binding, active) + first = scopes[0] + first["token"] = pressure.admit(first["path"]) + deadline = time.monotonic() + PRESSURE_SECONDS + while True: + running = status(probe, fleet, first["path"], first["token"], "pressure-running-window") + if active_window(running, COUNTS[0]): break + require(running.get("summary") == "running" and time.monotonic() < deadline, + "NOT_OBSERVED: no partial running task for queue pressure") + time.sleep(0.05) + merged = pressure.call(first["path"]) + require(merged.get("httpStatus") == 200 and merged.get("response", {}).get("clientToken") == first["token"], + "duplicate admission created another task instead of merging its original token") + dropped = rejection(pressure.call(first["path"] + "0001.bin"), admission="dropped", + reason="overlapping_paths", code="OperationAborted", http=409) + require(active_window(status(probe, fleet, first["path"], first["token"], "after-merge-and-drop"), COUNTS[0]), + "NOT_OBSERVED: Merged/Dropped were not bracketed by the same active task") + for scope in scopes[1:3]: + scope["admittedMonotonicNs"] = time.monotonic_ns() + scope["token"] = pressure.admit(scope["path"]) + require(len({scope["token"] for scope in scopes[:3]}) == 3, "independent buckets reused a task identity") + rejected = rejection(pressure.call(scopes[3]["path"]), admission="full", reason="none", code="SlowDown", http=503) + require(pressure.sample()["queueLength"] == CAPACITY, "NOT_OBSERVED: Full was not witnessed with the actual two-slot queue") + # Restrict the outage to one run-owned drive in the existing service's + # private mount namespace. No scheduler/process is stopped to fake Full. + gate_node, gate_disk = nodes[0], 0 + owner = fleet.record("original-gated-process", fleet.remote(gate_node, "process")) + fleet.drive_gates.add(gate_node) + fleet.record("permission-gate-armed", fleet.remote(gate_node, "drive-gate-arm-live", disk=gate_disk, expected=owner)) + fleet.record("permission-gate-activated", fleet.remote(gate_node, "drive-gate-activate")) + gate_proof = fleet.record("actual-service-refusal", fleet.remote(gate_node, "drive-gate-probe")) + require(gate_proof.get("sameServiceIdentity") is True and gate_proof.get("privateMountNamespace") is True + and gate_proof.get("name") == "EACCES" and gate_proof.get("readProbe", {}).get("errno") == 13, + "pressure degraded write has no same-namespace refusal witness") + earliest_ms, ack_start = int(time.time() * 1000) - 5000, datetime.now(timezone.utc) + durable_expected, obligations, _ = acknowledge(durable_recorder, durable_bucket, "offline-put", durable_expected, fixture="DB") + fleet.record("durable-ACK-ledger", {"expected": durable_expected, "obligations": obligations}) + deadline = time.monotonic() + PRESSURE_SECONDS + while True: + records = read_checkpoints(fleet, nodes, "committed-ACKs-during-Full") + try: + bindings = pending_bindings(records, obligations, earliest_ms) + except ValueError as exc: + require(str(exc).startswith("NOT_OBSERVED:") and time.monotonic() < deadline, + "durable pressure persistence failed: " + str(exc)) + time.sleep(0.5) + continue + all_intents = {identity(item) for value in records.values() for item in value["intents"]} + require(all_intents == {identity(item) for item in obligations}, "MRF Full isolation contains foreign or unexplained durable ACKs") + full = mrf_full_events(window.capture(), nodes, ack_start) + if full: break + require(time.monotonic() < deadline, "NOT_OBSERVED: the degraded-write MRF source never encountered real Full") + time.sleep(0.2) + # Capture again AFTER the rejection event, before rejoining. Admission + # alone cannot discharge any of the three exact durable responsibilities. + fleet.record("Full-durable-retention-proof", retained(read_checkpoints(fleet, nodes, "after-MRF-Full"), obligations, bindings)) + pressure.sample() + rejoin_time = datetime.now(timezone.utc) + released = fleet.record("same-process-drive-rejoin", fleet.remote(gate_node, "drive-gate-release-live")) + require(released.get("sameOriginalProcess") is True and released.get("serviceRestarted") is False, + "pressure release restarted the scheduler or replaced its original process") + fleet.drive_gates.remove(gate_node) + deadline, proofs, completed = time.monotonic() + DRAIN_SECONDS, {}, {} + while True: + operations = pressure.sample() + if "token" not in scopes[3] and operations["queueLength"] < CAPACITY: + retry = pressure.call(scopes[3]["path"]) + if retry.get("httpStatus") == 200: + token = retry.get("response", {}).get("clientToken") + require(isinstance(token, str) and token and token not in {scope["token"] for scope in scopes[:3]}, + "previously Full bucket lost its independent admitted token") + scopes[3].update(token=token, admittedMonotonicNs=time.monotonic_ns()) + active.append((scopes[3]["path"], token)) + else: + rejection(retry, admission="full", reason="none", code="SlowDown", http=503) + for scope in scopes: + if "token" not in scope or scope["bucket"] in completed: continue + current = status(probe, fleet, scope["path"], scope["token"], "per-bucket-drain") + if current.get("summary") != "running": + completed[scope["bucket"]] = terminal(current, scope["bucket"], scope["expected"], scope["missing"]) + completed[scope["bucket"]]["completionMonotonicNs"] = time.monotonic_ns() + active.remove((scope["path"], scope["token"])) + events = window.capture() + for obligation in obligations: + key = identity(obligation) + if key in proofs: continue + for task in source_tasks(events, obligation, bindings[key], rejoin_time): + endpoint = "node" + str(nodes.index(task["node"]) + 1) + path = "/rustfs/admin/v3/heal/" + quote(durable_bucket, safe="") + "/" + quote(obligation["key"], safe="/") + current = status(Probe(binding, endpoint), fleet, path, task["taskId"], "MRF-exact-ACK-completion") + proofs[key] = {"task": task, "receipt": verified_receipt(current, obligation, bindings[key])} + break + if len(completed) == len(scopes) and len(proofs) == len(obligations): + try: + retired = retirement(read_checkpoints(fleet, nodes, "exact-retirement-after-proof"), obligations, bindings) + break + except ValueError as exc: + require(str(exc).startswith("NOT_OBSERVED:") and time.monotonic() < deadline, + "durable queue retirement failed: " + str(exc)) + require(time.monotonic() < deadline, "queue drain exceeded its pre-registered cross-bucket completion budget") + time.sleep(2) + idle_fleet(fleet, binding) + fleet.stop() + repaired = fleet.snapshot("physical-product-result-before-GET", quiescent=True) + validate_repaired_physical(original_subset(original, None, excluded=durable_bucket), + original_subset(repaired, None, excluded=durable_bucket), targets[0], repair_targets=targets) + for index, scope in enumerate(scopes): + fleet.record("complete-external-product-proof", verify_part_manifest(fleet, scope["cohorts"], "repaired-part-manifest", count=COUNTS[index])) + for entry in durable_expected: + target = map_target(original_subset(repaired, durable_bucket), durable_bucket, entry["key"], "data", 1, entry["versionId"]) + fleet.record("all-degraded-ACK-parts", verify_cohort(fleet, target, "durable-all-member-streaming")) + retirement(read_checkpoints(fleet, nodes, "quiescent-retirement", quiescent=True), obligations, bindings) + fleet.start() + for endpoint in binding["endpoints"]: + peer = Probe(binding, endpoint) + for scope in scopes: + inventory = fleet.record("all-peer-S3", {"endpoint": endpoint, "bucket": scope["bucket"], + **peer.inventory(scope["bucket"], unversioned_current_tags=True)}) + fleet.record("all-peer-full-manifest", {**verify_seed(scope["expected"], inventory), + **compare_inventories(scope["baseline"], inventory)}) + inventory = fleet.record("all-peer-durable-S3", {"endpoint": endpoint, + **peer.inventory(durable_bucket, unversioned_current_tags=True)}) + fleet.record("all-peer-durable-manifest", verify_seed(durable_expected, inventory)) + fleet.stop() + require(physical_content(repaired) == physical_content(fleet.snapshot("physical-after-S3", quiescent=True)), + "verification GET supplied repairs absent from the prior physical result") + result.update(verdict="PASS", branches={ + "full-queue": {"verdict": "PASS", "adminRejection": rejected, "MRFRejections": full, + "durableRetirement": retired, "verifiedSourceACKs": len(proofs)}, + "dropped-admission": {"verdict": "PASS", "typedRejection": dropped, "independentRepairCount": 0}, + "merged-duplicates": {"verdict": "PASS", "originalTaskId": first["token"], "independentRepairCount": 0}, + "cross-bucket-fairness": {"verdict": "PASS", "completedBuckets": completed, "drainBudgetSeconds": DRAIN_SECONDS, + "requestCount": pressure.requests, "boundedQueueAndResourceSamples": pressure.samples}}, + completePhysicalAndS3ManifestsVerified=True, backupRestorationUsedAsRepair=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + try: + if fleet.reserved: + fleet.stop() + fleet.snapshot("physical-failure", quiescent=True) + read_checkpoints(fleet, nodes, "retained-failure-responsibility", quiescent=True) + except Exception as evidence_error: + result["failureEvidenceError"] = str(evidence_error) + finally: + for path, token in active: + try: + fleet.record("stop-unresolved-owned-task", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: + fleet.record("stop-error", {"error": str(exc)}) + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("topology-result", result) + return result + + +def original_subset(snapshot, bucket, *, excluded=None): + return {node: {**value, "files": [entry for entry in value["files"] + if (bucket is None or entry["path"].startswith(bucket + "/")) + and (excluded is None or not entry["path"].startswith(excluded + "/"))]} + for node, value in snapshot.items()} + + +if __name__ == "__main__": + try: + generic_main(case_id=CASE_ID, branches=BRANCHES, runner=run_topology) + except Exception as exc: + print(f"Heal queue-pressure validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index 4e3390d5..e2d05f3a 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Audit a failed run's rejected overlength fault against its archived physical scene.""" +"""Audit and finalize only explicitly supported, exactly archived failures.""" import argparse import hashlib @@ -16,8 +16,82 @@ from heal_probe import require, save +def quota_recovery_plan(artifact, failed_run, candidate): + """A private H1 quota failure before workload/Heal admits no unknown writes.""" + case = "heal.enospc-inodes-and-quota" + results = list(artifact.glob(f"*/run/{case}/cases/{case}.json")) + require(len(results) == 1, "quota recovery requires one exact I02 result") + result = json.loads(results[0].read_text()) + provenance = result.get("provenance") or {} + require(result.get("case_id") == case and result.get("verdict") == "FAIL" and result.get("cleanup_status") == "FAIL" + and provenance.get("source_commit") == candidate["source_sha"] + and provenance.get("candidate_sha256") == candidate["package_sha256"], "quota failed-result candidate or cleanup differs") + evidence = results[0].parents[1] / f"evidence/{case}" + summary = json.loads((evidence / "healthy-summary.json").read_text()) + retained = [row for row in summary["results"] if row.get("cleanup") != "PASS"] + require(summary.get("cleanup") == "FAIL" and len(retained) == 1, "quota recovery cannot omit another retained instance") + row = retained[0] + require(row.get("topology") == "H1" and row.get("branch") == "bucket-quota" and row.get("verdict") == "FAIL" + and row.get("failure", {}).get("message") == "peer did not observe the exact acknowledged quota limit" + and set(row) <= {"topology", "branch", "verdict", "cleanup", "failure", "recoveryError"}, + "quota recovery only supports failure before any negative PUT or Heal task") + folder = evidence / "H1/bucket-quota" + def one(suffix): + matches = list(folder.glob("*-" + suffix + ".json")) + require(len(matches) == 1, "quota recovery lacks one exact " + suffix) + return json.loads(matches[0].read_text()) + binding, before, failure = one("binding"), one("original-complete-physical"), one("physical-original-failure-before-any-rescue") + require(re.fullmatch(rf"gh-{failed_run}-[1-9][0-9]*-h1", binding["runId"]) is not None + and binding["candidateBinarySha256"] == provenance["binary_sha256"] + and binding["harnessCommit"] == provenance["testing_commit"] and len(before) == 1 + and set(before) == set(failure), "quota archived run/candidate/host differs") + node = next(iter(before)) + require(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) is not None, "quota archived node is invalid") + live = one("manual-source-isolation-before-fault") + require(all(value.get("quiescent") is True for scene in (before, failure) for value in scene.values()) + and set(live) == {node} and (live[node].get("backgroundConfig") or {}).get("RUSTFS_SCANNER_ENABLED") == "false" + and live[node].get("binarySha256") == provenance["binary_sha256"], + "quota archive has no stopped fixture and actual isolated candidate runtime") + quota = one("owned-bucket-quota-PUT") + read = one("all-peer-owned-quota-GET") + require(quota.get("path") == read.get("path") == "/rustfs/admin/v3/quota/" + binding["bucketPrefix"] + and quota.get("method") == "PUT" and quota.get("httpStatus") == 200 + and quota.get("response", {}).get("quota") == 1024**2 and quota["response"].get("quota_type") == "HARD" + and read.get("method") == "GET" and read.get("httpStatus") == 503 + and "authoritative bucket usage is not available yet" in str(read.get("response")), + "quota recovery lacks its exact acknowledged limit and usage-unavailable failure") + require(not (folder / "negative-PUT").exists() and not (folder / "capacity-released-PUT").exists(), + "quota recovery cannot finalize an invoked or uncertain workload") + receipts = [json.loads(path.read_text()) for path in folder.glob("*-exact-missing-part.json")] + require(len(receipts) == 2 and len({item.get("faultId") for item in receipts}) == 2, + "quota rescue requires both original fault receipts") + left = {(item["disk"], item["path"]): item for item in before[node]["files"]} + right = {(item["disk"], item["path"]): item for item in failure[node]["files"]} + require(before[node]["formats"] == failure[node]["formats"] and len(left) == len(before[node]["files"]) + and len(right) == len(failure[node]["files"]), "quota archived disk identity or file census differs") + faults, removed = [], set() + for receipt in receipts: + matches = [key for key, entry in left.items() if entry["path"] == receipt.get("relativePath") + and key not in right and receipt.get("before") == {"sha256": entry["sha256"], "size": entry["size"]}] + require(receipt.get("runId") == binding["runId"] and receipt.get("kind") == "missing" + and re.fullmatch(r"I02-[a-z0-9-]{1,48}", receipt.get("faultId", "")) is not None and len(matches) == 1 + and matches[0] not in removed, "quota fault receipt has a wrong/duplicate run, shard or backup identity") + removed.add(matches[0]) + faults.append({"disk": matches[0][0], "faultId": receipt["faultId"]}) + require({key: value for key, value in left.items() if key not in removed} == right, + "quota archived failure changed other content or versions; preserve namespace") + return [{"kind": "owned-quota", "runId": binding["runId"], "nodes": [node], "node": node, + "bucket": binding["bucketPrefix"], "binarySha256": provenance["binary_sha256"], + "before": before, "failure": failure, "faults": faults, + "expectedSceneSha256": hashlib.sha256(json.dumps(physical_content(before)[node], sort_keys=True, + separators=(",", ":")).encode()).hexdigest()}] + + def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") + if list(artifact.glob("*/run/heal.enospc-inodes-and-quota/cases/heal.enospc-inodes-and-quota.json")): + require(not list(artifact.glob("*/run/heal.object-size-boundaries/cases/*.json")), "mixed failed case recovery artifact") + return quota_recovery_plan(artifact, failed_run, candidate) results = list(artifact.glob("*/run/heal.object-size-boundaries/cases/heal.object-size-boundaries.json")) require(len(results) == 1, "failed artifact must have one exact D01 canonical result") result = json.loads(results[0].read_text()) @@ -77,10 +151,19 @@ def main(): fleet = Fleet(plan["nodes"], args.evidence, plan["runId"]) fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) current = fleet.snapshot("audited-physical-scene", quiescent=True) - require(physical_content(current) == physical_content(plan["before"]), "retained fleet drifted from archived pre-fault scene; no cleanup performed") + require(physical_content(current) == physical_content(plan.get("failure", plan["before"])), + "retained fleet drifted from its exact archived scene; no recovery performed") fleet.record("retained-service-logs", fleet.remote(plan["node"], "logs")) - proof = fleet.record("audited-finalization", fleet.remote(plan["node"], "audit-finalize", **{ - key: plan[key] for key in ("disk", "rejectedFaultId", "binarySha256", "expectedSceneSha256")})) + if plan.get("kind") == "owned-quota": + for fault in plan["faults"]: + fleet.record("quota-original-shard-rescue", fleet.remote(plan["node"], "recover-fault", **fault)) + require(physical_content(fleet.snapshot("complete-pre-fault-rescue", quiescent=True)) == physical_content(plan["before"]), + "quota rescue did not restore every original file; preserve namespace") + proof = fleet.record("audited-finalization", fleet.remote(plan["node"], "quota-audit-finalize", **{ + key: plan[key] for key in ("bucket", "binarySha256", "expectedSceneSha256")})) + else: + proof = fleet.record("audited-finalization", fleet.remote(plan["node"], "audit-finalize", **{ + key: plan[key] for key in ("disk", "rejectedFaultId", "binarySha256", "expectedSceneSha256")})) for node in plan["nodes"]: if node != plan["node"]: fleet.record("unchanged-peer-finalization", fleet.remote(node, "cleanup")) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index b3c63b10..5a5ebf66 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -31,6 +31,7 @@ case "${case_id}" in heal.performance-abba) driver=performance.py ;; heal.admin-scope-isolation) driver=scopes.py ;; heal.admin-option-contract) driver=options.py ;; + heal.queue-pressure-and-admission) driver=queue_pressure.py ;; heal.bucket-metadata-integrity) driver=bucket_metadata.py ;; heal.disk-replacement-unformatted) driver=unformatted.py ;; heal.below-write-quorum) driver=write_quorum.py ;; diff --git a/test_validation_heal_queue_pressure.py b/test_validation_heal_queue_pressure.py new file mode 100644 index 00000000..71fd6d85 --- /dev/null +++ b/test_validation_heal_queue_pressure.py @@ -0,0 +1,168 @@ +"""Reject false queue admission, responsibility retirement and resource proofs.""" + +import copy +from datetime import datetime, timezone +import json +import types +import unittest + +from scripts.validation_heal import queue_pressure as queue +from scripts.validation_heal.fleet_node import runtime_config +from scripts.validation_heal.matrix import AUTOMATIC_CASES, select + + +class QueuePressureTests(unittest.TestCase): + def status(self): + return {"clusterStatusComplete": True, "coverage": {"expected": 1, "responded": 1, "unknown": 0}, + "healQueueLength": 2, "healActiveTasks": 1, "healOperations": {"queueLength": 2, + "activeTasks": 1, "retryingTasks": 0, "queuedByPriority": {"high": 2, "normal": 0}, + "activeByPriority": {"high": 1}, "queuedBySource": {"admin": 2}, "activeBySource": {"admin": 1}, + "admission": {"accepted": 3, "full": 1, "dropped": 1, "merged": 1}}} + + def terminal(self): + expected = [{"key": "objects/0", "versionId": "null"}, {"key": "objects/1", "versionId": "null"}] + rows = [{"identity": {"bucket": "owned", "object": entry["key"], "versionId": "null", + "poolIndex": 0, "setIndex": 0, "bucketIncarnationId": "original-incarnation"}, + "disposition": {"state": "repaired" if index else "verified_healthy"}} + for index, entry in enumerate(expected)] + current = {"summary": "finished", "outcome": {"execution": {"state": "completed"}, "coverage": "complete", + "objectsTruncated": False, "objects": rows, "counters": {"processed": 2, "healed": 1, "unchanged": 1, + "failed": 0, "skipped": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False}}} + return current, expected, {("objects/1", "null")} + + def durable(self): + obligation = {"bucket": "owned", "key": "offline/new", "versionId": "null", "poolIndex": 0, "setIndex": 0, "kind": 3} + intent = {**obligation, "intentDigest": "original-digest", "lifecycle": {"responsibility_id": "original-id", + "source_bucket_incarnation_id": "original-incarnation", "state": {"state": "active"}, "last_operator_acceptance": None}} + records = {"node": {"present": True, "owner": "checkpoint-owner", "sequence": 9, "intents": [intent]}} + bindings = {queue.identity(obligation): [{"node": "node", "checkpointOwner": "checkpoint-owner", "sequence": 8, + "responsibilityId": "original-id", "bucketIncarnationId": "original-incarnation", "intentDigest": "original-digest"}]} + return records, [obligation], bindings + + def test_all_source_branches_and_automatic_registration(self): + self.assertEqual(set(select(queue.CASE_ID)[0]["variants"]), set(queue.BRANCHES)) + self.assertIn(queue.CASE_ID, AUTOMATIC_CASES) + + def test_profile_has_finite_queue_and_actual_two_priorities(self): + value = runtime_config("queue-pressure") + self.assertEqual(value["RUSTFS_HEAL_QUEUE_SIZE"], "2") + self.assertEqual(value["RUSTFS_HEAL_MAX_CONCURRENT_HEALS"], "1") + self.assertEqual(value["RUSTFS_HEAL_MAX_CONCURRENT_PER_SET"], "1") + self.assertEqual(value["RUSTFS_HEAL_MRF_ENABLE"], "true") + self.assertEqual(value["RUSTFS_SCANNER_ENABLED"], "false") + self.assertEqual(value["RUSTFS_HEAL_AUTO_HEAL_ENABLE"], "false") + self.assertNotEqual(queue.REFERENCE["adminPriority"], queue.REFERENCE["partialWritePriority"]) + + def test_full_has_exact_structured_rejection_and_no_admitted_task(self): + observed = {"httpStatus": 503, "response": {"Code": "SlowDown", "Message": "heal request not admitted: admission=full, reason=none"}} + expected = dict(admission="full", reason="none", code="SlowDown", http=503) + self.assertEqual(queue.rejection(observed, **expected)["independentRepairCount"], 0) + for changes in [{"httpStatus": 429}, {"transportError": "timeout"}, {"response": {**observed["response"], "clientToken": "invented"}}, + {"response": {**observed["response"], "Message": "queue was busy"}}, {"response": {"Code": "OperationAborted"}}]: + with self.subTest(changes=changes), self.assertRaises(ValueError): + queue.rejection({**observed, **changes}, **expected) + + def test_dropped_xml_preserves_its_distinct_reason(self): + value = {"httpStatus": 409, "response": "OperationAbortedheal request not admitted: admission=dropped, reason=overlapping_paths"} + self.assertEqual(queue.rejection(value, admission="dropped", reason="overlapping_paths", code="OperationAborted", http=409)["reason"], "overlapping_paths") + + def test_queue_requires_all_peers_and_consistent_source_priority_counts(self): + self.assertEqual(queue.queue_snapshot(self.status(), 1)["queueLength"], 2) + for name in ["coverage", "global-count", "source-count", "priority-count", "capacity", "bool-count", "admission"]: + value = self.status() + if name == "coverage": value["coverage"]["unknown"] = 1 + elif name == "global-count": value["healQueueLength"] = 1 + elif name == "source-count": value["healOperations"]["queuedBySource"]["admin"] = 1 + elif name == "priority-count": value["healOperations"]["activeByPriority"]["high"] = 0 + elif name == "capacity": value["healOperations"]["queueLength"] = 3 + elif name == "bool-count": value["healOperations"]["activeTasks"] = True + else: del value["healOperations"]["admission"]["full"] + with self.subTest(name=name), self.assertRaises(ValueError): queue.queue_snapshot(value, 1) + + def test_complete_scope_counters_and_dispositions_are_independent_of_idle(self): + value, expected, missing = self.terminal() + self.assertEqual(queue.terminal(value, "owned", expected, missing)["canonicalCounters"]["healed"], 1) + for name in ["coverage", "execution", "unknown", "attemptFailures", "wrong-bucket", "wrong-version", "wrong-set", "incarnation", "duplicate", "false-repair", "truncated"]: + value, expected, missing = self.terminal() + outcome = value["outcome"] + if name == "coverage": outcome["coverage"] = "partial" + elif name == "execution": outcome["execution"]["state"] = "completed_with_errors" + elif name in {"unknown", "attemptFailures"}: outcome["counters"][name] = 1 + elif name == "wrong-bucket": outcome["objects"][0]["identity"]["bucket"] = "foreign" + elif name == "wrong-version": outcome["objects"][0]["identity"]["versionId"] = "another-version" + elif name == "wrong-set": outcome["objects"][0]["identity"]["setIndex"] = 1 + elif name == "incarnation": outcome["objects"][0]["identity"]["bucketIncarnationId"] = "different" + elif name == "duplicate": outcome["objects"][1] = copy.deepcopy(outcome["objects"][0]) + elif name == "false-repair": outcome["objects"][0]["disposition"]["state"] = "repaired" + else: outcome["objectsTruncated"] = True + with self.subTest(name=name), self.assertRaises(ValueError): queue.terminal(value, "owned", expected, missing) + + def test_running_window_requires_partial_actual_processing(self): + value = {"summary": "running", "outcome": {"execution": {"state": "running"}, "counters": {"processed": 5}}} + self.assertTrue(queue.active_window(value, 10)) + for count in [0, 10, True, None]: + value["outcome"]["counters"]["processed"] = count + self.assertFalse(queue.active_window(value, 10)) + + def test_queue_failure_keeps_exact_durable_owner_and_generation(self): + records, obligations, bindings = self.durable() + self.assertTrue(queue.retained(records, obligations, bindings)) + for name in ["absent", "owner", "sequence", "missing", "duplicate", "digest", "responsibility", "incarnation", "held", "operator", "version"]: + records, obligations, bindings = self.durable() + current = records["node"]; item = current["intents"][0] + if name == "absent": current["present"] = False + elif name == "owner": current["owner"] = "different" + elif name == "sequence": current["sequence"] = 7 + elif name == "missing": current["intents"] = [] + elif name == "duplicate": current["intents"].append(copy.deepcopy(item)) + elif name == "digest": item["intentDigest"] = "changed" + elif name == "responsibility": item["lifecycle"]["responsibility_id"] = "new" + elif name == "incarnation": item["lifecycle"]["source_bucket_incarnation_id"] = "new" + elif name == "held": item["lifecycle"]["state"] = {"state": "held"} + elif name == "operator": item["lifecycle"]["last_operator_acceptance"] = {"actor": "operator"} + else: item["versionId"] = "other-version" + with self.subTest(name=name), self.assertRaises(ValueError): queue.retained(records, obligations, bindings) + + def test_mrf_full_is_real_source_priority_capacity_and_time(self): + after = datetime(2026, 10, 11, tzinfo=timezone.utc) + row = {"target": "rustfs::heal::manager", "event": "heal_queue_admission", "source": "mrf", "priority": "Normal", + "result": "rejected_full", "queue_capacity": 2, "queue_len": 2, "request_id": "actual-request", "timestamp": "2026-10-11T01:00:00Z"} + self.assertEqual(len(queue.mrf_full_events({"node": [row]}, ["node"], after)), 1) + for changes in [{"source": "admin"}, {"priority": "High"}, {"result": "accepted"}, {"queue_len": 1}, + {"queue_capacity": 99}, {"request_id": ""}, {"timestamp": "2026-10-10T23:59:59Z"}]: + self.assertEqual(queue.mrf_full_events({"node": [{**row, **changes}]}, ["node"], after), []) + self.assertEqual(queue.mrf_full_events({"foreign": [row]}, ["node"], after), []) + + def test_resource_ceiling_and_process_birth_are_preregistered(self): + baseline = {"node": {"clockDomain": "boot", "monotonicNs": 1, "identity": {"pid": 12, "startTicks": 100}}} + sample = {"node": {**baseline["node"], "monotonicNs": 2, "readOnly": True, "rssBytes": 123, "fileDescriptors": 30, "threads": 20}} + self.assertTrue(queue.resources(sample, baseline)["withinFrozenLimits"]) + for name in ["pid", "clock", "time", "rssBytes", "fileDescriptors", "threads", "missing-node"]: + current = copy.deepcopy(sample) + if name == "pid": current["node"]["identity"]["pid"] = 13 + elif name == "clock": current["node"]["clockDomain"] = "another-boot" + elif name == "time": current["node"]["monotonicNs"] = 1 + elif name == "missing-node": current = {} + else: current["node"][name] = queue.RESOURCE_BOUNDS[name] + 1 + with self.subTest(name=name), self.assertRaises(ValueError): queue.resources(current, baseline) + + def test_request_budget_counts_failed_and_merged_requests(self): + calls = [] + probe = types.SimpleNamespace(admin=lambda *args: calls.append(args) or {"httpStatus": 503}) + fleet = types.SimpleNamespace(nodes=[], record=lambda label, value: value) + pressure = queue.Pressure(probe, fleet, {}, []) + for _ in range(queue.MAX_ADMIN_REQUESTS): pressure.call("/owned") + with self.assertRaises(ValueError): pressure.call("/owned") + self.assertEqual(len(calls), queue.MAX_ADMIN_REQUESTS) + + def test_multi_bucket_physical_subset_keeps_every_format_and_control(self): + snapshot = {"node": {"formats": ["actual-format"], "files": [{"path": "a/objects/0/xl.meta"}, {"path": "b/objects/0/part.1"}]}} + a = queue.original_subset(snapshot, "a") + self.assertEqual(a["node"]["formats"], snapshot["node"]["formats"]) + self.assertEqual(a["node"]["files"], [snapshot["node"]["files"][0]]) + self.assertEqual(queue.original_subset(snapshot, None, excluded="b"), a) + self.assertEqual(len(snapshot["node"]["files"]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/test_validation_heal_recovery.py b/test_validation_heal_recovery.py index 2b84bd95..795457a9 100644 --- a/test_validation_heal_recovery.py +++ b/test_validation_heal_recovery.py @@ -8,7 +8,10 @@ from unittest.mock import patch from scripts.validation_heal.heal_fault import bounded_fault_id -from scripts.validation_heal.fleet_node import audit_finalize +from scripts.validation_heal.fleet_node import audit_finalize, quota_audit_finalize +from scripts.validation_heal.recover_environment import recovery_plan +from scripts.validation_heal.capacity import release_quota +from scripts.validation_heal.healthy import Fleet class RejectedFaultRecoveryTest(unittest.TestCase): @@ -50,5 +53,128 @@ def test_finalization_needs_no_intent_and_exact_archived_scene(self): cleanup.assert_not_called() +class QuotaRecoveryTests(unittest.TestCase): + def test_standalone_remote_recovery_loads_all_cleanup_backends(self): + import types + with tempfile.TemporaryDirectory() as temp, patch("scripts.validation_heal.healthy.subprocess.run", + return_value=types.SimpleNamespace(returncode=0, stdout='{}', stderr='')) as command: + fleet = Fleet(["node"], Path(temp), "gh-12345-1-h1") + fleet.remote("node", "quota-audit-finalize", bucket="rv-heal-0123456789abcdef", + binarySha256="a" * 64, expectedSceneSha256="b" * 64) + source = command.call_args.kwargs["input"].rsplit("\nmain(", 1)[0] + scope = {"__name__": "validation_remote_recovery"} + exec(compile(source, "standalone-recovery", "exec"), scope) + self.assertTrue(callable(scope["quota_audit_finalize"])) + self.assertTrue(callable(scope["quorum_gate_backend"].release)) + self.assertTrue(callable(scope["replacement"].attest)) + self.assertTrue(callable(scope["network_gate_backend"].release)) + + def archive(self, root): + case = "heal.enospc-inodes-and-quota" + base = root / "rustfs-validation-12345-1" / "run" / case + directory = base / "evidence" / case + branch = directory / "H1/bucket-quota" + branch.mkdir(parents=True) + def write(path, value): path.write_text(json.dumps(value)) + candidate = {"source_sha": "a" * 40, "package_sha256": "b" * 64} + provenance = {"source_commit": "a" * 40, "candidate_sha256": "b" * 64, + "binary_sha256": "c" * 64, "testing_commit": "d" * 40} + (base / "cases").mkdir() + write(base / "cases" / (case + ".json"), {"case_id": case, "verdict": "FAIL", "cleanup_status": "FAIL", "provenance": provenance}) + write(directory / "healthy-summary.json", {"cleanup": "FAIL", "results": [{"topology": "H1", "branch": "bucket-quota", + "verdict": "FAIL", "cleanup": "FAIL", "failure": {"message": "peer did not observe the exact acknowledged quota limit"}, "recoveryError": "connection refused"}]}) + bucket = "rv-heal-0123456789abcdef" + binding = {"runId": "gh-12345-1-h1", "bucketPrefix": bucket, "candidateBinarySha256": "c" * 64, "harnessCommit": "d" * 40} + write(branch / "00001-binding.json", binding) + entries = [{"disk": 0, "path": f"{bucket}/objects/{index}/part.1", "size": 100 + index, "sha256": str(index) * 64} for index in range(2)] + control = {"disk": 1, "path": bucket + "/control/xl.meta", "size": 5, "sha256": "e" * 64} + scene = {"node": {"quiescent": True, "formats": [{"disk": "actual-format"}], "files": [*entries, control], "backgroundConfig": None}} + write(branch / "00002-original-complete-physical.json", scene) + write(branch / "00003-physical-original-failure-before-any-rescue.json", {"node": {**scene["node"], "files": [control]}}) + write(branch / "00004-manual-source-isolation-before-fault.json", {"node": {"backgroundConfig": {"RUSTFS_SCANNER_ENABLED": "false"}, "binarySha256": "c" * 64}}) + write(branch / "00005-owned-bucket-quota-PUT.json", {"path": "/rustfs/admin/v3/quota/" + bucket, "method": "PUT", "httpStatus": 200, + "response": {"quota": 1024**2, "quota_type": "HARD"}}) + write(branch / "00006-all-peer-owned-quota-GET.json", {"path": "/rustfs/admin/v3/quota/" + bucket, "method": "GET", "httpStatus": 503, + "response": "authoritative bucket usage is not available yet"}) + for index, entry in enumerate(entries): + write(branch / f"0001{index}-exact-missing-part.json", {"runId": binding["runId"], "faultId": f"I02-target-{index}", + "kind": "missing", "relativePath": entry["path"], "before": {"size": entry["size"], "sha256": entry["sha256"]}}) + return candidate, branch, directory + + def test_recovery_is_bound_to_failed_workflow_candidate_and_both_backups(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate, _, _ = self.archive(root) + plan = recovery_plan(root, 12345, candidate)[0] + self.assertEqual(plan["kind"], "owned-quota") + self.assertEqual(len(plan["faults"]), 2) + self.assertEqual(plan["nodes"], ["node"]) + for run, value in [(12346, candidate), (12345, {**candidate, "package_sha256": "f" * 64})]: + with self.assertRaises(ValueError): recovery_plan(root, run, value) + + def test_uncertain_workload_and_collateral_drift_always_keep_namespace(self): + for changed in ["invoked-PUT", "unknown-ACK", "collateral", "wrong-run", "duplicate-fault", "other-retained"]: + with self.subTest(changed=changed), tempfile.TemporaryDirectory() as temp: + root = Path(temp); candidate, branch, directory = self.archive(root) + if changed == "invoked-PUT": (branch / "negative-PUT").mkdir() + elif changed == "unknown-ACK": + path = directory / "healthy-summary.json"; data = json.loads(path.read_text()) + data["results"][0]["negativePUT"] = {"state": "UNKNOWN"}; path.write_text(json.dumps(data)) + elif changed == "collateral": + path = branch / "00003-physical-original-failure-before-any-rescue.json"; data = json.loads(path.read_text()) + data["node"]["files"][0]["sha256"] = "new"; path.write_text(json.dumps(data)) + elif changed in {"wrong-run", "duplicate-fault"}: + path = branch / "00011-exact-missing-part.json"; data = json.loads(path.read_text()) + data["runId" if changed == "wrong-run" else "faultId"] = "gh-12346-1-h1" if changed == "wrong-run" else "I02-target-0" + path.write_text(json.dumps(data)) + else: + path = directory / "healthy-summary.json"; data = json.loads(path.read_text()) + data["results"].append({"cleanup": "FAIL", "branch": "another"}); path.write_text(json.dumps(data)) + with self.assertRaises(ValueError): recovery_plan(root, 12345, candidate) + + def test_node_finalization_requires_same_private_candidate_and_full_rescue(self): + scene = {"files": [{"path": "owned/key/part.1", "sha256": "actual"}], "formats": [{"disk": "original"}]} + scene_sha = hashlib.sha256(json.dumps(scene, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + with tempfile.TemporaryDirectory() as temp: + state = Path(temp); roots = [state / str(index) for index in range(4)] + bucket = "rv-heal-0123456789abcdef" + owner = {"runtimeProfile": "manual", "bucket": bucket, "binarySha256": "a" * 64, + "volumes": [str(root / "data") for root in roots]} + request = {"runId": "gh-12345-1-h1", "action": "quota-audit-finalize", "bucket": bucket, + "binarySha256": "a" * 64, "expectedSceneSha256": scene_sha} + with patch("scripts.validation_heal.fleet_node.owned", return_value=(state, roots, owner)), \ + patch("scripts.validation_heal.fleet_node.stopped"), patch("scripts.validation_heal.fleet_node.digest", return_value="a" * 64), \ + patch("scripts.validation_heal.fleet_node.snapshot", return_value=scene), \ + patch("scripts.validation_heal.fleet_node.cleanup", return_value={"cleanup": "PASS"}) as cleanup: + value = quota_audit_finalize(request) + self.assertTrue(value["privateBucketQuotaNamespaceRemoved"]) + self.assertFalse(value["serviceStartedDuringRecovery"]) + self.assertFalse(value["productRepairClaimed"]) + cleanup.reset_mock() + for changes in [{"bucket": "foreign"}, {"binarySha256": "b" * 64}, {"runId": "gh-12345-1-h2"}, {"expectedSceneSha256": "b" * 64}]: + with self.assertRaises(ValueError): quota_audit_finalize({**request, **changes}) + cleanup.assert_not_called() + (state / "drive-gate.json").write_text("active") + with self.assertRaises(ValueError): quota_audit_finalize(request) + cleanup.assert_not_called() + + def test_release_restarts_stopped_service_and_reads_config_without_usage_claim(self): + import types + events = [] + binding = {"endpoints": {"node1": "http://owned"}} + def admin(path, query, body, method): + events.append((method, path, query)) + if method == "PUT": return {"httpStatus": 200, "response": {"bucket": "owned", "quota": 100, "quota_type": "HARD"}} + return {"httpStatus": 200, "response": {"quota": 100, "size": 100, "quotatype": "hard"}} + probe = types.SimpleNamespace(binding=binding, admin=admin) + fleet = types.SimpleNamespace(nodes=["node"], remote=lambda *args: {"pid": 0}, + start=lambda: events.append("start"), record=lambda label, value: value) + with patch("scripts.validation_heal.capacity.Probe", return_value=probe): + result = release_quota(probe, fleet, "owned", 100) + self.assertEqual(events[0], "start") + self.assertEqual(events[-1], ("GET", "/rustfs/admin/v3/get-bucket-quota", [("bucket", "owned")])) + self.assertFalse(result["authoritativeUsageClaimed"]) + + if __name__ == "__main__": unittest.main() From 6ef733d6d62e6a9fe6f9e0d6e8f4f01cdb947165 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 11:12:34 +0800 Subject: [PATCH 119/343] fix(validation): resume exact interrupted quota recovery --- docs/validation-migration-map.md | 14 ++++++++ scripts/validation_heal/healthy.py | 2 +- .../validation_heal/recover_environment.py | 26 +++++++++++++- test_validation_heal_recovery.py | 35 ++++++++++++++++++- 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 014d56f4..eaea30ec 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1733,3 +1733,17 @@ scene and remove its private bucket/quota namespace without starting a service. Invoked/uncertain PUTs, changed candidate/run/versions, missing fault receipts or another dirty instance reject finalization. Original failure results remain immutable; any subsequent recovery has separate evidence. + +Recovery run `38107394081` rescued both exact original shards but failed +before namespace removal because its remote finalization snapshot lacked the +encryption attestation module. Both audited finalization actions now carry +that dependency. An interrupted quota rescue may resume only when every +remaining file and format matches the archived failed scene and each added +file is one of the exact original missing shards. The original fault intents +and backups are revalidated even for already restored bytes. Unexpected +content, missing controls, duplicate files or another host still retain the +namespace. Two regressions exercise remote dependency loading and every +accepted/rejected retry scene. The fleet remains guarded until a separate +successful recovery proves namespace removal; neither original FAIL changes. +Linux contract run `38107374734` independently confirms all 441 tests passed +with no skips at `e40fd9d74a902b9b9733c7bddfdc0b9d1b3f5c03`. diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 5bba7c2e..0d9a0f4c 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -212,7 +212,7 @@ def remote(self, node, action, **params): if action == "format-schema-fault": source += "\nimport types\nschema_faults = types.ModuleType('schema_faults')\nexec(" + repr( NODE_SOURCE.with_name("schema_faults.py").read_text()) + ", schema_faults.__dict__)\n" - if action in {"encryption-configure", "snapshot"}: + if action in {"encryption-configure", "snapshot", "audit-finalize", "quota-audit-finalize"}: source += "\nimport types\nencryption_runtime = types.ModuleType('encryption_runtime')\nexec(" + repr( NODE_SOURCE.with_name("encryption_runtime.py").read_text()) + ", encryption_runtime.__dict__)\n" replacement_actions = {"start", "cleanup", "audit-finalize", "quota-audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot", "format-schema-fault"} diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index e2d05f3a..dc6fba2a 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -134,6 +134,30 @@ def recovery_plan(artifact, failed_run, candidate): return plans +def recovery_scene_matches(current, plan): + """A retry may contain only the exact original shards already rescued.""" + observed = physical_content(current) + expected = physical_content(plan.get("failure", plan["before"])) + if plan.get("kind") != "owned-quota": + return observed == expected + original = physical_content(plan["before"]) + if set(observed) != set(expected): + return False + for node, scene in observed.items(): + if scene["formats"] != expected[node]["formats"]: + return False + files = {(entry["disk"], entry["path"]): entry for entry in scene["files"]} + failed = {(entry["disk"], entry["path"]): entry for entry in expected[node]["files"]} + before = {(entry["disk"], entry["path"]): entry for entry in original[node]["files"]} + if len(files) != len(scene["files"]) or not set(failed) <= set(files) <= set(before): + return False + if any(files[key] != entry for key, entry in failed.items()): + return False + if any(files[key] != before[key] for key in set(files) - set(failed)): + return False + return True + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--failed-run-id", type=int, required=True) @@ -151,7 +175,7 @@ def main(): fleet = Fleet(plan["nodes"], args.evidence, plan["runId"]) fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) current = fleet.snapshot("audited-physical-scene", quiescent=True) - require(physical_content(current) == physical_content(plan.get("failure", plan["before"])), + require(recovery_scene_matches(current, plan), "retained fleet drifted from its exact archived scene; no recovery performed") fleet.record("retained-service-logs", fleet.remote(plan["node"], "logs")) if plan.get("kind") == "owned-quota": diff --git a/test_validation_heal_recovery.py b/test_validation_heal_recovery.py index 795457a9..97a8e6cc 100644 --- a/test_validation_heal_recovery.py +++ b/test_validation_heal_recovery.py @@ -9,7 +9,7 @@ from scripts.validation_heal.heal_fault import bounded_fault_id from scripts.validation_heal.fleet_node import audit_finalize, quota_audit_finalize -from scripts.validation_heal.recover_environment import recovery_plan +from scripts.validation_heal.recover_environment import recovery_plan, recovery_scene_matches from scripts.validation_heal.capacity import release_quota from scripts.validation_heal.healthy import Fleet @@ -68,6 +68,17 @@ def test_standalone_remote_recovery_loads_all_cleanup_backends(self): self.assertTrue(callable(scope["quorum_gate_backend"].release)) self.assertTrue(callable(scope["replacement"].attest)) self.assertTrue(callable(scope["network_gate_backend"].release)) + self.assertTrue(callable(scope["encryption_module"]().attest)) + + def test_standalone_rejected_id_finalization_loads_snapshot_backend(self): + import types + with tempfile.TemporaryDirectory() as temp, patch("scripts.validation_heal.healthy.subprocess.run", + return_value=types.SimpleNamespace(returncode=0, stdout='{}', stderr='')) as command: + Fleet(["node"], Path(temp), "gh-12345-1-h1").remote("node", "audit-finalize") + source = command.call_args.kwargs["input"].rsplit("\nmain(", 1)[0] + scope = {"__name__": "validation_remote_recovery"} + exec(compile(source, "standalone-recovery", "exec"), scope) + self.assertTrue(callable(scope["encryption_module"]().attest)) def archive(self, root): case = "heal.enospc-inodes-and-quota" @@ -112,6 +123,28 @@ def test_recovery_is_bound_to_failed_workflow_candidate_and_both_backups(self): for run, value in [(12346, candidate), (12345, {**candidate, "package_sha256": "f" * 64})]: with self.assertRaises(ValueError): recovery_plan(root, run, value) + def test_retry_accepts_only_original_rescued_shards_and_no_other_drift(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate, _, _ = self.archive(root) + plan = recovery_plan(root, 12345, candidate)[0] + self.assertTrue(recovery_scene_matches(plan["failure"], plan)) + self.assertTrue(recovery_scene_matches(plan["before"], plan)) + partial = json.loads(json.dumps(plan["failure"])) + partial["node"]["files"].append(plan["before"]["node"]["files"][0]) + self.assertTrue(recovery_scene_matches(partial, plan)) + for changed in ["format", "control", "rescued", "missing-control", "extra", "duplicate", "node"]: + current = json.loads(json.dumps(partial)) + if changed == "format": current["node"]["formats"] = [] + elif changed == "control": current["node"]["files"][0]["sha256"] = "wrong" + elif changed == "rescued": current["node"]["files"][1]["sha256"] = "wrong" + elif changed == "missing-control": current["node"]["files"].pop(0) + elif changed == "extra": current["node"]["files"].append({"disk": 3, "path": "foreign"}) + elif changed == "duplicate": current["node"]["files"].append(current["node"]["files"][0]) + else: current["foreign"] = current.pop("node") + with self.subTest(changed=changed): + self.assertFalse(recovery_scene_matches(current, plan)) + def test_uncertain_workload_and_collateral_drift_always_keep_namespace(self): for changed in ["invoked-PUT", "unknown-ACK", "collateral", "wrong-run", "duplicate-fault", "other-retained"]: with self.subTest(changed=changed), tempfile.TemporaryDirectory() as temp: From aba6f5c6a36121adcc5a67663b767d0de5667ba9 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 11:20:44 +0800 Subject: [PATCH 120/343] test(validation): collect exact manager persistence regressions --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 27 ++ scripts/validation_heal/manager_native.py | 237 ++++++++++++++++++ test_validation_heal_manager_native.py | 126 ++++++++++ 4 files changed, 393 insertions(+), 1 deletion(-) create mode 100644 scripts/validation_heal/manager_native.py create mode 100644 test_validation_heal_manager_native.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index e1f3d124..645faae2 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -33,6 +33,7 @@ on: - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py + - test_validation_heal_manager_native.py - test_validation_heal_authority.py - test_validation_heal_orphan.py - test_validation_heal_options.py @@ -84,6 +85,7 @@ on: - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py + - test_validation_heal_manager_native.py - test_validation_heal_authority.py - test_validation_heal_orphan.py - test_validation_heal_options.py @@ -126,7 +128,7 @@ jobs: sudo apt-get update sudo apt-get install --no-install-recommends -y nftables fi - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index eaea30ec..a2b3ea26 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1747,3 +1747,30 @@ accepted/rejected retry scene. The fleet remains guarded until a separate successful recovery proves namespace removal; neither original FAIL changes. Linux contract run `38107374734` independently confirms all 441 tests passed with no skips at `e40fd9d74a902b9b9733c7bddfdc0b9d1b3f5c03`. + +Independent quota recovery `38107681061` at +`6ef733d6d62e6a9fe6f9e0d6e8f4f01cdb947165` completed with the owned service +stopped, both original shards restored and the complete archived scene hash +`979e39a78712ed39bba8af43acbd1187d57c9e5edd730f474cad3c2f2936d2d4`. +Its original finalization evidence proves private roots and bucket/quota +namespace removal. Original I02 canonical FAIL and cleanup FAIL are retained. +The serial queue resumed after independent successful-attempt API identity, +artifact hashes and exact failure/guard binding were checked. Rejected +short-SHA dispatch `38107655997` never entered recovery. Linux contract +`38107650661` executed all 443 tests without skips; all six checks passed. + +`manager_native.py` adds a separately collected exact-source manager suite +for F07/C11/J02 prerequisites. Fifteen original candidate tests cover corrupt, +future and stale checkpoints, real MRF payload/manifest write failures, +four deterministic migration publication interruptions, uncertain root +terminal publication, incarnation replay/checkpoint guards and outcome truth. +Hosted `build_manager_native` builds the unmodified `rustfs-heal` library with +the immutable candidate tree, lock and eight reviewed source hashes. Every +test runs by its complete name in a fresh process. Verification retains both +inventories, exact commands, executable and collector digests, original logs +and individual results. Seven negative evidence tests reject zero/ignored +execution, scope inflation, wrong build attempts, filtered names and modified +source or executable identities. This suite does not inject individual fsync +hooks, demonstrate physical power loss or execute canonical fleet branches; +it adds no automatic executor or live PASS count by itself. Existing C07 +native collector and pinned artifacts retain their exact contract. diff --git a/scripts/validation_heal/manager_native.py b/scripts/validation_heal/manager_native.py new file mode 100644 index 00000000..649044f9 --- /dev/null +++ b/scripts/validation_heal/manager_native.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""Exact candidate manager regressions for persistence and incarnation guards.""" + +import argparse +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import time + +try: + from . import native_regressions as native +except ImportError: + import native_regressions as native + + +SOURCE_TREE = "8071fb0a9e5353eb7d3fa58c8a58c8622473f2bd" +LOCK_SHA = "d8f7bc416ba67d19873d34e9389e5aa823b22d7d7552706a5a810cee26f6eda2" +SOURCE_FILES = { + "crates/heal/src/heal/resume/tests.rs": "cea1558f8659f01968a5592af55cd0626f1d309c9fecfa0bee26c36775229900", + "crates/heal/src/heal/resume/checkpoint.rs": "fe067b39ee406e16517bd665f7f49457daffa05d5d7bc298bd965e927b1381c1", + "crates/heal/src/heal/mrf_queue/snapshot.rs": "ecfcb5ee63edef5c5d3d6713edc127e16f489522afc5fc02dc1044ccca241d14", + "crates/heal/src/heal/mrf_queue/snapshot/migration.rs": "9163c87bcdee1439667f79aea2a36f6ffce14c6d7c64be0786306dd3a75478b9", + "crates/heal/src/heal/manager/tests/root_recovery.rs": "7c4a0cb7834390f053737a8950f24e070a01e1c74583d9db206ea1b620385594", + "crates/heal/src/heal/manager/root_recovery.rs": "ba6beba9d9d14985e4c5448d85727036388d3d72f28879ac5c67fc5f760048ba", + "crates/heal/src/heal/erasure_healer/resume_loop_tests/admin_outcome.rs": "f2251a9ab5f2df10a2299cd23bb3231615d946ce95a104002088f06235dde621", + "crates/heal/src/heal/storage.rs": "52ce6edff5b64b69a0d0c2a425b980984cf927fab79139e33d5ff9c7ad98efe9", +} +RESUME = "heal::resume::tests::" +SNAPSHOT = "heal::mrf_queue::snapshot::tests::" +ROOT = "heal::manager::tests::root_recovery::" +OUTCOME = "heal::erasure_healer::resume_loop_tests::admin_outcome::" + + +def spec(test, cases, assertions, kind="deterministic candidate guard assertions"): + return {"test": test, "relatedCases": cases, "sourceAssertions": assertions, + "fenceKind": kind, "fleetPassClaimed": False, "physicalPowerLossClaimed": False, + "emittedBarrierLogsClaimed": False} + + +F07 = ["heal.checkpoint-and-mrf-corruption"] +C11 = ["heal.bucket-incarnation-recreation"] +J02 = ["heal.outcome-counters-and-error-truth"] +REGRESSIONS = { + "checkpoint-truncated": spec(RESUME + "checkpoint_save_does_not_replace_a_non_empty_truncated_snapshot", F07, + ["save fails closed", "original truncated bytes remain", "task is blocked"]), + "checkpoint-future": spec(RESUME + "checkpoint_save_does_not_replace_a_future_schema_snapshot", F07, + ["future schema save fails closed", "unknown bytes remain", "task is blocked"]), + "checkpoint-digest": spec(RESUME + "checkpoint_digest_rejects_same_length_progress_tampering", F07, + ["same-length progress tampering fails load", "task is blocked"]), + "checkpoint-stale-manager": spec(RESUME + "stale_checkpoint_manager_cannot_overwrite_newer_progress", F07, + ["stale manager save is rejected", "reopened newer bucket/object cursor remains 4/20"]), + "checkpoint-page-order": spec(RESUME + "checkpoint_page_commit_keeps_ledger_until_cursor_is_durable", F07, + ["reload preserves completed page cursor, counters and ledger", "ledger prunes only after durable cursor"], + "real checkpoint publication and reload ordering; no injected fsync interruption"), + "mrf-payload-integrity": spec(SNAPSHOT + "whole_payload_integrity_is_required_even_with_a_valid_manifest", F07, + ["valid outer manifest cannot hide damaged full-record integrity"]), + "mrf-manifest-write-failure": spec(SNAPSHOT + "committed_snapshot_writer_manifest_failure_preserves_previous_anchor", F07, + ["actual successor manifest path blocks publication", "error remains visible", "reopened previous payload and manifest are unchanged"]), + "mrf-payload-write-failure": spec(SNAPSHOT + "committed_snapshot_writer_payload_failure_does_not_publish_manifest", F07, + ["actual successor payload path blocks staging", "error remains visible", "no successor manifest", "previous anchor survives reload"]), + "mrf-migration-boundaries": spec("heal::mrf_queue::snapshot::migration::tests::migration_commit_boundaries_and_lost_response_recover_idempotently", F07, + ["actual AfterPayload/BeforeManifest/AfterManifest/AfterReadback interruptions are injected", + "source bytes survive every interruption", "uncommitted recovery is refused", "same owner retry recovers identical records"], + "four actual migration publication interruption hooks; no physical power-loss or individual fsync-hook claim"), + "root-uncertain-publication": spec(ROOT + "root_recovery_uncertain_terminal_publication_preserves_commit_and_pending_fence", F07, + ["failure is injected after terminal write", "pending owner remains", "restart fences stale runnable work and restores committed outcome"], + "actual candidate terminal publication failure hook"), + "incarnation-root-replay": spec(ROOT + "bucket_incarnation_recovery_retires_old_owner_and_admits_successor", C11, + ["same incarnation replays original token", "new incarnation rejects old owner", "new token admits successor", + "reintroduced original pending bytes cannot defeat durable stale terminal"]), + "incarnation-deferred-metadata": spec(ROOT + "bucket_incarnation_metadata_failure_defers_replay_without_retiring_owner", C11, + ["metadata lookup failure preserves original pending owner instead of retiring or rebinding"]), + "incarnation-checkpoint": spec(OUTCOME + "admin_erasure_recreated_bucket_cannot_rebind_checkpoint", C11, + ["recreated incarnation returns StaleBucketIncarnation", "storage calls do not advance", "acknowledged healed counter is retained"]), + "outcome-truth": spec(OUTCOME + "admin_erasure_dispositions_do_not_invent_success_and_retries_count_once", J02, + ["verified/absent/unproven/transient cases remain distinct", "exact 5/0/2/0/3/1 counters", "eight attempt failures remain visible", + "exhausted missing-version retries remain deferred"]), + "outcome-wrong-identity": spec(OUTCOME + "admin_erasure_rejects_every_receipt_identity_mismatch", J02, + ["bucket/object/version/incarnation/pool/set mismatches each yield unknown and zero healed"]), +} +BUILD_ENV = {"CARGO_BUILD_JOBS": "2", "CARGO_INCREMENTAL": "0", + "CARGO_PROFILE_RELEASE_DEBUG": "0", "CARGO_PROFILE_RELEASE_STRIP": "debuginfo"} + + +def collectors(): + return {path.name: native.sha(path) for path in (Path(__file__), Path(native.__file__))} + + +def verify_manifest(directory): + manifest = json.loads((directory / "manager-native.json").read_text()) + native.require(manifest.get("schemaVersion") == 1 and manifest.get("candidateCommit") == native.CANDIDATE + and manifest.get("candidateSourceTree") == SOURCE_TREE and manifest.get("candidateCargoLockSha256") == LOCK_SHA + and manifest.get("sourceFiles") == SOURCE_FILES and manifest.get("collectorSources") == collectors() + and manifest.get("regressions") == REGRESSIONS and manifest.get("sourceUnmodified") is True + and manifest.get("optimized") is True and manifest.get("buildEnvironment") == BUILD_ENV + and manifest.get("binaryName") == "heal-manager-native-tests" and manifest.get("fleetPassClaimed") is False, + "manager native artifact changed its reviewed source, test or collector contract") + binary = directory / manifest["binaryName"] + native.require(binary.is_file() and not binary.is_symlink() and native.sha(binary) == manifest.get("binarySha256"), + "manager native executable identity differs") + return manifest, binary + + +def build(source, commit, out): + native.require(commit == native.CANDIDATE and subprocess.check_output( + ["git", "-C", str(source), "rev-parse", "HEAD"], text=True).strip() == commit, + "manager native tests require the exact reviewed candidate") + native.require(subprocess.check_output(["git", "-C", str(source), "rev-parse", "HEAD^{tree}"], text=True).strip() == SOURCE_TREE + and native.sha(source / "Cargo.lock") == LOCK_SHA and all(native.sha(source / path) == digest for path, digest in SOURCE_FILES.items()), + "manager candidate source/tree/lock differs") + for args in (["diff", "--quiet"], ["diff", "--cached", "--quiet"]): + native.require(subprocess.run(["git", "-C", str(source), *args]).returncode == 0, "candidate source is modified") + native.require(not subprocess.check_output(["git", "-C", str(source), "ls-files", "--others", "--exclude-standard"], text=True).strip(), + "candidate contains untracked source/configuration") + native.require(not out.exists() and shutil.disk_usage(source).free >= 12 * 1024**3, "fresh output and 12 GiB build space required") + out.mkdir(parents=True) + request = {"schemaVersion": 1, "candidateCommit": commit, "candidateSourceTree": SOURCE_TREE, + "candidateCargoLockSha256": LOCK_SHA, "sourceFiles": SOURCE_FILES, "collectorSources": collectors(), + "regressions": REGRESSIONS, "testingCommit": os.environ.get("AUTO_TESTING_SHA"), + "runId": os.environ.get("GITHUB_RUN_ID"), "runAttempt": os.environ.get("GITHUB_RUN_ATTEMPT"), + "fleetPassClaimed": False, "sourceUnmodified": True, "optimized": True, "buildEnvironment": BUILD_ENV} + native.save(out / "manager-build-request.json", request) + target = out / "target" + command = ["cargo", "test", "--locked", "--release", "-p", "rustfs-heal", "--lib", "--no-run", "--message-format=json"] + with (out / "cargo.jsonl").open("x") as stdout, (out / "cargo.stderr").open("x") as stderr: + result = subprocess.run(command, cwd=source, env={**os.environ, **BUILD_ENV, "CARGO_TARGET_DIR": str(target)}, + stdout=stdout, stderr=stderr, timeout=150 * 60) + native.require(result.returncode == 0, "manager native compilation failed; original diagnostics retained") + executables = [] + for line in (out / "cargo.jsonl").read_text().splitlines(): + try: row = json.loads(line) + except json.JSONDecodeError: continue + if row.get("reason") == "compiler-artifact" and row.get("profile", {}).get("test") is True \ + and row.get("target", {}).get("name") == "rustfs_heal" and row.get("executable"): + executables.append(Path(row["executable"])) + native.require(len(executables) == 1 and executables[0].resolve().is_relative_to(target.resolve()), "one owned manager test executable required") + binary = out / "heal-manager-native-tests" + shutil.copyfile(executables[0], binary) + binary.chmod(0o500) + listing = subprocess.run([str(binary), "--list"], capture_output=True, text=True, timeout=60) + (out / "test-inventory.log").write_text(listing.stdout + listing.stderr) + native.require(listing.returncode == 0, "manager native inventory failed") + for value in REGRESSIONS.values(): native.exact_listing(listing.stdout, value["test"]) + for args in (["diff", "--quiet"], ["diff", "--cached", "--quiet"]): + native.require(subprocess.run(["git", "-C", str(source), *args]).returncode == 0, "build changed tracked candidate source") + native.save(out / "manager-native.json", {**request, "binaryName": binary.name, "binarySha256": native.sha(binary), + "binaryKind": "native-regression-not-released-service", "buildCommand": command, + "toolchain": subprocess.check_output(["rustc", "-vV"], cwd=source, text=True)}) + shutil.rmtree(target) + + +def run(directory, out): + manifest, binary = verify_manifest(directory) + native.require(not out.exists(), "manager native evidence must be fresh") + out.mkdir(parents=True) + listing = subprocess.run([str(binary), "--list"], capture_output=True, text=True, timeout=60) + (out / "runtime-inventory.log").write_text(listing.stdout + listing.stderr) + native.require(listing.returncode == 0, "manager runtime inventory failed") + results = [] + for key, value in REGRESSIONS.items(): + native.exact_listing(listing.stdout, value["test"]) + verify_manifest(directory) + command = [str(binary), value["test"], "--exact", "--nocapture", "--test-threads=1"] + record = {"regression": key, **value, "candidateCommit": native.CANDIDATE, + "binarySha256": manifest["binarySha256"], "command": command} + started = time.monotonic_ns() + try: + child = subprocess.run(command, capture_output=True, text=True, timeout=120) + (out / (key + ".log")).write_text(child.stdout + child.stderr) + record["exitCode"] = child.returncode + native.test_result(child.stdout + child.stderr, value["test"], child.returncode) + record["verdict"] = "PASS" + except subprocess.TimeoutExpired as error: + parts = [part.decode(errors="replace") if isinstance(part, bytes) else part or "" for part in (error.stdout, error.stderr)] + (out / (key + ".log")).write_text("".join(parts)) + record.update(verdict="FAIL", reason="fixed 120-second native deadline exceeded") + except (OSError, ValueError) as error: + record.update(verdict="FAIL", reason=str(error)) + record["elapsedNanoseconds"] = time.monotonic_ns() - started + native.save(out / (key + ".json"), record) + results.append(record) + complete = all(row["verdict"] == "PASS" for row in results) + native.save(out / "manager-native-summary.json", {"results": results, "complete": complete, + "canonicalCasePassClaimed": False, "fleetPassClaimed": False, "physicalPowerLossClaimed": False}) + native.require(complete, "manager native failures remain in original evidence") + + +def verify_evidence(directory, evidence, candidate, run_id, run_attempt, helper_sha): + manifest, _ = verify_manifest(directory) + native.require(candidate["source_sha"] == native.CANDIDATE and manifest.get("runId") == str(run_id) + and manifest.get("runAttempt") == str(run_attempt) and manifest.get("testingCommit") == helper_sha + and re.fullmatch(r"[0-9a-f]{40}", helper_sha), "manager native evidence belongs to another build attempt") + summary = json.loads((evidence / "manager-native-summary.json").read_text()) + rows = summary.get("results", []) + native.require(summary.get("complete") is True and summary.get("canonicalCasePassClaimed") is False + and summary.get("fleetPassClaimed") is False and summary.get("physicalPowerLossClaimed") is False + and len(rows) == len(REGRESSIONS) and {row.get("regression") for row in rows} == set(REGRESSIONS), + "manager native summary is incomplete, duplicated or claims fleet success") + listings = [(path).read_text() for path in (directory / "test-inventory.log", evidence / "runtime-inventory.log")] + for row in rows: + key = row["regression"] + native.require(row == json.loads((evidence / (key + ".json")).read_text()) and row.get("verdict") == "PASS" + and row.get("binarySha256") == manifest["binarySha256"] and row.get("candidateCommit") == native.CANDIDATE + and all(row.get(name) == value for name, value in REGRESSIONS[key].items()), "manager result changed exact scope or identity") + command = row.get("command", []) + native.require(len(command) == 5 and Path(command[0]).name == manifest["binaryName"] + and command[1:] == [row["test"], "--exact", "--nocapture", "--test-threads=1"], "manager result lacks exact one-test invocation") + for listing in listings: native.exact_listing(listing, row["test"]) + native.test_result((evidence / (key + ".log")).read_text(), row["test"], row.get("exitCode")) + return {"complete": True, "candidateCommit": native.CANDIDATE, "binarySha256": manifest["binarySha256"], + "exactTests": len(rows), "canonicalCasePassClaimed": False, "fleetPassClaimed": False} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + compiling = commands.add_parser("build") + compiling.add_argument("--source", type=Path, required=True) + compiling.add_argument("--commit", required=True) + compiling.add_argument("--out", type=Path, required=True) + running = commands.add_parser("run") + running.add_argument("--directory", type=Path, required=True) + running.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + if args.command == "build": build(args.source.resolve(strict=True), args.commit, args.out.resolve()) + else: run(args.directory.resolve(strict=True), args.out.resolve()) + + +if __name__ == "__main__": + try: main() + except Exception as error: + print(f"manager native validation failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/test_validation_heal_manager_native.py b/test_validation_heal_manager_native.py new file mode 100644 index 00000000..5a27d8c3 --- /dev/null +++ b/test_validation_heal_manager_native.py @@ -0,0 +1,126 @@ +"""Manager native evidence must bind exact source, executable and real tests.""" + +import copy +import hashlib +import json +from pathlib import Path +import tempfile +import unittest + +from scripts.validation_heal import manager_native as suite + + +class ManagerNativeEvidenceTests(unittest.TestCase): + def fixture(self, root): + tools, evidence = root / "tools", root / "evidence" + tools.mkdir(); evidence.mkdir() + helper = "a" * 40 + binary = tools / "heal-manager-native-tests" + binary.write_bytes(b"original candidate unit executable") + digest = hashlib.sha256(binary.read_bytes()).hexdigest() + manifest = {"schemaVersion": 1, "candidateCommit": suite.native.CANDIDATE, + "candidateSourceTree": suite.SOURCE_TREE, "candidateCargoLockSha256": suite.LOCK_SHA, + "sourceFiles": suite.SOURCE_FILES, "collectorSources": suite.collectors(), + "regressions": suite.REGRESSIONS, "sourceUnmodified": True, "optimized": True, + "buildEnvironment": suite.BUILD_ENV, "binaryName": binary.name, "binarySha256": digest, + "fleetPassClaimed": False, "runId": "12345", "runAttempt": "1", "testingCommit": helper} + (tools / "manager-native.json").write_text(json.dumps(manifest)) + listing = "\n".join(value["test"] + ": test" for value in suite.REGRESSIONS.values()) + (tools / "test-inventory.log").write_text(listing) + (evidence / "runtime-inventory.log").write_text(listing) + rows = [] + for key, value in suite.REGRESSIONS.items(): + row = {"regression": key, **value, "candidateCommit": suite.native.CANDIDATE, + "binarySha256": digest, "verdict": "PASS", "exitCode": 0, "elapsedNanoseconds": 100, + "command": ["/hosted/tools/" + binary.name, value["test"], "--exact", "--nocapture", "--test-threads=1"]} + (evidence / (key + ".json")).write_text(json.dumps(row)) + (evidence / (key + ".log")).write_text("running 1 test\ntest " + value["test"] + " ... ok\n" + "test result: ok. 1 passed; 0 failed; 0 ignored; 99 filtered out; finished in 0.01s\n") + rows.append(row) + summary = {"results": rows, "complete": True, "canonicalCasePassClaimed": False, + "fleetPassClaimed": False, "physicalPowerLossClaimed": False} + (evidence / "manager-native-summary.json").write_text(json.dumps(summary)) + return tools, evidence, {"source_sha": suite.native.CANDIDATE}, helper + + def verify(self, fixture, **changes): + tools, evidence, candidate, helper = fixture + return suite.verify_evidence(tools, evidence, changes.get("candidate", candidate), + changes.get("run", 12345), changes.get("attempt", 1), changes.get("helper", helper)) + + def test_complete_exact_native_evidence_preserves_separate_scope(self): + with tempfile.TemporaryDirectory() as temp: + result = self.verify(self.fixture(Path(temp))) + self.assertEqual(result["exactTests"], 15) + self.assertFalse(result["canonicalCasePassClaimed"]) + self.assertFalse(result["fleetPassClaimed"]) + + def test_wrong_candidate_build_attempt_or_helper_is_rejected(self): + with tempfile.TemporaryDirectory() as temp: + fixture = self.fixture(Path(temp)) + for changes in [{"candidate": {"source_sha": "b" * 40}}, {"run": 12346}, {"attempt": 2}, {"helper": "b" * 40}]: + with self.subTest(changes=changes), self.assertRaises(ValueError): self.verify(fixture, **changes) + + def test_source_lock_collector_and_scope_drift_are_rejected(self): + with tempfile.TemporaryDirectory() as temp: + fixture = self.fixture(Path(temp)); path = fixture[0] / "manager-native.json" + original = json.loads(path.read_text()) + for field, value in [("candidateSourceTree", "b" * 40), ("candidateCargoLockSha256", "b" * 64), + ("sourceFiles", {}), ("collectorSources", {}), ("regressions", {}), + ("sourceUnmodified", False), ("optimized", False), ("fleetPassClaimed", True), + ("buildEnvironment", {}), ("binaryName", "../foreign")]: + path.write_text(json.dumps({**original, field: value})) + with self.subTest(field=field), self.assertRaises(ValueError): self.verify(fixture) + + def test_changed_or_symlinked_binary_is_rejected(self): + for changed in ["bytes", "symlink"]: + with tempfile.TemporaryDirectory() as temp: + fixture = self.fixture(Path(temp)); binary = fixture[0] / "heal-manager-native-tests" + if changed == "bytes": binary.write_bytes(b"modified") + else: + original = fixture[0] / "original"; binary.rename(original); binary.symlink_to(original) + with self.subTest(changed=changed), self.assertRaises(ValueError): self.verify(fixture) + + def test_zero_ignored_failed_or_missing_original_test_never_passes(self): + with tempfile.TemporaryDirectory() as temp: + fixture = self.fixture(Path(temp)); key = next(iter(suite.REGRESSIONS)); path = fixture[1] / (key + ".log") + original = path.read_text() + for output in ["running 0 tests\ntest result: ok. 0 passed; 0 failed; 0 ignored;\n", + original.replace(" ... ok", " ... ignored").replace("1 passed; 0 failed; 0 ignored", "0 passed; 0 failed; 1 ignored"), + original.replace(" ... ok", " ... FAILED"), original.replace("running 1 test", "running 2 tests")]: + path.write_text(output) + with self.assertRaises(ValueError): self.verify(fixture) + + def test_missing_duplicate_or_scope_changed_results_are_rejected(self): + with tempfile.TemporaryDirectory() as temp: + fixture = self.fixture(Path(temp)); path = fixture[1] / "manager-native-summary.json" + original = json.loads(path.read_text()) + for changed in ["missing", "duplicate", "scope", "power", "fleet"]: + value = copy.deepcopy(original) + if changed == "missing": value["results"].pop() + elif changed == "duplicate": value["results"][-1] = value["results"][0] + elif changed == "scope": value["results"][0]["emittedBarrierLogsClaimed"] = True + elif changed == "power": value["physicalPowerLossClaimed"] = True + else: value["fleetPassClaimed"] = True + path.write_text(json.dumps(value)) + with self.subTest(changed=changed), self.assertRaises(ValueError): self.verify(fixture) + + def test_listing_and_invocation_require_the_complete_exact_test_name(self): + for changed in ["missing", "duplicate", "filter", "parallel", "wrong-name"]: + with tempfile.TemporaryDirectory() as temp: + fixture = self.fixture(Path(temp)); key = next(iter(suite.REGRESSIONS)) + if changed in {"missing", "duplicate"}: + path = fixture[1] / "runtime-inventory.log"; listing = path.read_text() + path.write_text(listing.replace(suite.REGRESSIONS[key]["test"] + ": test", "") if changed == "missing" + else listing + "\n" + suite.REGRESSIONS[key]["test"] + ": test") + else: + path = fixture[1] / (key + ".json"); row = json.loads(path.read_text()) + if changed == "filter": row["command"].remove("--exact") + elif changed == "parallel": row["command"][-1] = "--test-threads=8" + else: row["command"][1] = "checkpoint" + path.write_text(json.dumps(row)) + summary_path = fixture[1] / "manager-native-summary.json"; summary = json.loads(summary_path.read_text()) + summary["results"][0] = row; summary_path.write_text(json.dumps(summary)) + with self.subTest(changed=changed), self.assertRaises(ValueError): self.verify(fixture) + + +if __name__ == "__main__": unittest.main() From 0fa6f5410407c759f8078b3e7b03ddfefe78f47f Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 11:29:30 +0800 Subject: [PATCH 121/343] test(validation): add bounded durable-record fault backend --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 16 ++ scripts/validation_heal/fleet_node.py | 30 +++ scripts/validation_heal/healthy.py | 7 +- scripts/validation_heal/journal_fault.py | 184 +++++++++++++++++ test_validation_heal_journal_fault.py | 186 ++++++++++++++++++ 6 files changed, 424 insertions(+), 3 deletions(-) create mode 100644 scripts/validation_heal/journal_fault.py create mode 100644 test_validation_heal_journal_fault.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 645faae2..e9308f54 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -34,6 +34,7 @@ on: - test_validation_heal_schema.py - test_validation_heal_native.py - test_validation_heal_manager_native.py + - test_validation_heal_journal_fault.py - test_validation_heal_authority.py - test_validation_heal_orphan.py - test_validation_heal_options.py @@ -86,6 +87,7 @@ on: - test_validation_heal_schema.py - test_validation_heal_native.py - test_validation_heal_manager_native.py + - test_validation_heal_journal_fault.py - test_validation_heal_authority.py - test_validation_heal_orphan.py - test_validation_heal_options.py @@ -128,7 +130,7 @@ jobs: sudo apt-get update sudo apt-get install --no-install-recommends -y nftables fi - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_journal_fault.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index a2b3ea26..f9949758 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1774,3 +1774,19 @@ source or executable identities. This suite does not inject individual fsync hooks, demonstrate physical power loss or execute canonical fleet branches; it adds no automatic executor or live PASS count by itself. Existing C07 native collector and pinned artifacts retain their exact contract. + +The F07 durable-record backend now reads only exact root/checkpoint task names +or the six allowlisted MRF slot files, without enumerating system metadata. +Injection requires the stopped owned recovery service, unchanged candidate, +actual run/disk/mount identity and archived complete file digest. An Admin +checkpoint must retain its original task, owned bucket incarnation and valid +candidate integrity digest. MRF injection additionally proves an actual +committed active responsibility on the selected slot. Controlled truncation, +future schema and CRC corruption retain original backups and intents before +mutation. Terminal reports, resume states, blocked fences, commit manifests +and unrelated records are read-only. Rescue revalidates backups and exact +injected/original bytes; advanced or removed product responsibility is never +resurrected. Distinct injection/rescue temporaries preserve interrupted +evidence. Ten file/remote-boundary regressions cover these guarantees. This +backend remains an F07 prerequisite; the six canonical branches and their +full live execution are still outstanding, with no count or PASS increase. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 1c221afd..b3067532 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -972,6 +972,34 @@ def configure_encryption(request): return encryption_module().configure(state, owner, request, fault_module()) +def journal_operation(request): + state, roots, owner = owned(request["runId"]) + stopped(owner) + require(digest(state / "rustfs") == owner["binarySha256"], "durable-record candidate changed") + require(owner.get("runtimeProfile") in {"root-restart", "mrf-crash"}, "durable records require the owned recovery runtime") + require(set(request) <= {"runId", "action", "family", "taskId", "disk", "relativePath", "faultId", "kind", "expectedBefore"}, + "durable-record adapter accepts no arbitrary system paths or commands") + module, fault = globals()["journal_fault_backend"], fault_module() + if request["action"] == "journal-census": + return module.census(roots, request, fault) + require(request["action"] in {"journal-apply", "journal-recover"} + and type(request.get("disk")) is int and 0 <= request["disk"] < 4, "invalid durable-record operation or slot") + root = roots[request["disk"]] + with fault.directory(root) as fd: + if request["action"] == "journal-apply": + if request["family"] == "mrf": + observed = globals()["mrf"].read_records(state, roots, owner, checked) + committed = globals()["mrf"].checkpoint(observed) + require(committed.get("present") is True and committed["intents"] + and all(item["bucket"] == owner["bucket"] and item.get("lifecycle", {}).get("state") == {"state": "active"} + for item in committed["intents"]) + and any(row["disk"] == request["disk"] and request["relativePath"] == + f".rustfs.sys/.heal-mrf-snapshot.{row['slot']}.bin" for row in committed["replicas"]), + "MRF fault lacks an actual committed active owned responsibility on this exact slot") + return module.install(fd, request, owner, fault, globals()["mrf"]) + return module.recover(fd, request, owner, fault) + + def network_operation(request): state, _, owner = owned(request["runId"]) module = globals()["network_gate_backend"] @@ -1202,6 +1230,8 @@ def main(request): actions["task-records"] = task_records actions["environment-capabilities"] = environment_capabilities actions["quota-audit-finalize"] = quota_audit_finalize + for action in ("journal-census", "journal-apply", "journal-recover"): + actions[action] = journal_operation for action in ('generation-capture', 'generation-install', 'generation-recover'): actions[action] = generation_operation for action in ('orphan-install-tool', 'orphan-observe', 'orphan-multipart', 'orphan-convert', diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 0d9a0f4c..d3fb0172 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -190,7 +190,7 @@ def remote(self, node, action, **params): if self.runtime_profile == "bounded-capacity": source += "\nimport types\nbounded_volume_backend = types.ModuleType('bounded_volume')\nexec(" + repr( NODE_SOURCE.with_name("bounded_volume.py").read_text()) + ", bounded_volume_backend.__dict__)\n" - if action == "mrf-records": + if action == "mrf-records" or action.startswith("journal-"): source += "\nimport types\nmrf = types.ModuleType('mrf_records')\nexec(" + repr( NODE_SOURCE.with_name("mrf_records.py").read_text()) + ", mrf.__dict__)\n" if action == "replacement-records": @@ -200,9 +200,12 @@ def remote(self, node, action, **params): source += "\nimport types\ngeneration_backend = types.ModuleType('generation_fault')\nexec(" + repr( NODE_SOURCE.with_name("generation_fault.py").read_text()) + ", generation_backend.__dict__)\n" if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline", - "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure"} or action.startswith("generation-") or action.startswith("orphan-"): + "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure"} or action.startswith("generation-") or action.startswith("orphan-") or action.startswith("journal-"): source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" + if action.startswith("journal-"): + source += "\nimport types\njournal_fault_backend = types.ModuleType('journal_fault')\nexec(" + repr( + NODE_SOURCE.with_name("journal_fault.py").read_text()) + ", journal_fault_backend.__dict__)\n" if action.startswith("orphan-"): source += "\nimport types\norphan_backend = types.ModuleType('orphan_fault')\nexec(" + repr( NODE_SOURCE.with_name("orphan_fault.py").read_text()) + ", orphan_backend.__dict__)\n" diff --git a/scripts/validation_heal/journal_fault.py b/scripts/validation_heal/journal_fault.py new file mode 100644 index 00000000..4381925e --- /dev/null +++ b/scripts/validation_heal/journal_fault.py @@ -0,0 +1,184 @@ +"""Bounded F07 faults on exact owned durable records; never search metadata.""" + +import base64 +import hashlib +import json +import os +import re +import uuid + + +LIMIT = 4 * 1024**2 + + +def require(value, message): + if not value: raise ValueError(message) + + +def task_id(value): + require(isinstance(value, str) and str(uuid.UUID(value)) == value and uuid.UUID(value).int != 0, + "journal task must be a canonical nonzero UUID") + return value + + +def names(family, token=None): + if family == "mrf": + require(token is None, "MRF census cannot invent a task-file binding") + return [f".rustfs.sys/.heal-mrf-{kind}.{slot}.bin" for slot in (0, 1) for kind in ("snapshot", "commit", "lifecycle")] + task_id(token) + if family == "root": + return [f".rustfs.sys/{prefix}{token}.json" for prefix in ("root-heal-", "terminal-root-heal-", "heal-terminal-report-")] + require(family == "checkpoint", "unknown durable-record family") + return [f".rustfs.sys/buckets/{token}_{suffix}" for suffix in + ("ahm_checkpoint.json", "ahm_resume_state.json", "ahm_checkpoint.blocked", "ahm_checkpoint.sha256")] + + +def relative(request): + path = request.get("relativePath") + require(path in names(request.get("family"), request.get("taskId")), "journal target is outside the exact record allowlist") + return path.split("/") + + +def entry(raw): + return {"size": len(raw), "sha256": hashlib.sha256(raw).hexdigest()} + + +def read(root_fd, path, fault): + with fault.parent(root_fd, ["data", *path[:-1]]) as directory: + return fault.read_file(directory, path[-1], LIMIT * (4 if "lifecycle" in path[-1] else 1)) + + +def census(roots, request, fault): + paths = names(request.get("family"), request.get("taskId")) + rows = [] + for disk, root in enumerate(roots): + with fault.directory(root) as fd: + for path in paths: + row = {"disk": disk, "relativePath": path, "present": False} + try: raw = read(fd, path.split("/"), fault) + except FileNotFoundError: rows.append(row); continue + row.update(present=True, **entry(raw), rawBase64=base64.b64encode(raw).decode()) + rows.append(row) + return {"readOnly": True, "quiescent": True, "family": request["family"], + "taskId": request.get("taskId"), "records": rows, "directoryEnumerationUsed": False} + + +def checkpoint_value(raw, token, bucket): + value = json.loads(raw) + require(value.get("task_id") == task_id(token) and value.get("schema_version") == 8, + "checkpoint does not belong to the exact original Admin task") + admin = value.get("admin") or {} + buckets = admin.get("buckets") + require(admin.get("completed") is False and isinstance(buckets, dict) and set(buckets) == {bucket} + and isinstance(buckets[bucket], str) and str(uuid.UUID(buckets[bucket])) == buckets[bucket] + and uuid.UUID(buckets[bucket]).int != 0, "checkpoint has no original owned bucket incarnation") + unsigned = json.loads(raw) + unsigned["integrity_digest"] = None + for field in ("processed_objects", "failed_objects", "skipped_objects"): + require(isinstance(unsigned.get(field), list) and all(isinstance(item, str) for item in unsigned[field]), + "checkpoint ledger has an unknown canonical representation") + unsigned[field].sort() + # Candidate serde_json::Value uses sorted object keys and UTF-8 strings. + canonical = json.dumps(unsigned, sort_keys=True, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode() + expected = base64.b64encode(hashlib.sha256(canonical).digest()).decode() + require(value.get("integrity_digest") == expected, "original checkpoint has no valid candidate integrity digest") + return value + + +def transform(raw, request, owner, mrf): + family, kind = request["family"], request.get("kind") + require(kind in {"truncate", "future-schema", "checksum"}, "unknown durable-record byte fault") + path = relative(request) + if family == "checkpoint": + require(path[-1].endswith("_ahm_checkpoint.json"), "only the exact validated Admin checkpoint may be corrupted") + value = checkpoint_value(raw, request["taskId"], owner["bucket"]) + require(kind in {"truncate", "future-schema"}, "checkpoint checksum is independently collected by native guards") + elif family == "root": + require(path[-1] == "root-heal-" + task_id(request["taskId"]) + ".json" and kind == "truncate", + "only the exact runnable root admission may be truncated") + value = json.loads(raw) + require(value.get("task_id") == request["taskId"] and value.get("schema") == 3 + and value.get("heal_type") == {"type": "bucket", "bucket": owner["bucket"]} + and isinstance(value.get("bucket_incarnation_id"), str), "root admission does not belong to this owned bucket") + task_id(value["bucket_incarnation_id"]) + else: + require(family == "mrf" and kind == "checksum" and re.fullmatch(r"\.heal-mrf-snapshot\.[01]\.bin", path[-1]), + "MRF mutation is restricted to the exact actual payload checksum") + intents = mrf.decode_journal(raw) + require(intents and all(item["bucket"] == owner["bucket"] and item["kind"] in {3, 4} for item in intents), + "MRF payload includes a foreign or unregistered responsibility") + # Corrupt a real record CRC without inventing, dropping or retagging ACK identities. + return raw[:-1] + bytes([raw[-1] ^ 1]) + if kind == "truncate": + require(len(raw) > 8, "cannot truncate an empty/unknown record") + return raw[:len(raw) // 2] + value["schema_version"] = 9 + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode() + + +def binding(root_fd, request, owner, fault): + require(owner.get("runtimeProfile") in {"root-restart", "mrf-crash"}, "journal mutation requires an owned recovery runtime") + require(owner["runId"] == request["runId"] and type(request.get("disk")) is int and 0 <= request["disk"] < 4, + "journal run or disk identity differs") + identifier = request.get("faultId") + require(isinstance(identifier, str) and re.fullmatch(r"F07-[a-z0-9-]{1,48}", identifier), "journal fault identity is invalid") + path = relative(request) + require(request["family"] == "checkpoint" and path[-1] == task_id(request["taskId"]) + "_ahm_checkpoint.json" + or request["family"] == "root" and path[-1] == "root-heal-" + task_id(request["taskId"]) + ".json" + or request["family"] == "mrf" and re.fullmatch(r"\.heal-mrf-snapshot\.[01]\.bin", path[-1]), + "journal mutation cannot replace a terminal, resume state, digest, blocked fence or commit record") + return {"schemaVersion": 1, "runId": owner["runId"], "disk": request["disk"], "family": request["family"], + "taskId": request.get("taskId"), "relativePath": "/".join(path), "faultId": identifier, + "binarySha256": owner["binarySha256"], "device": os.fstat(root_fd).st_dev, + "mountId": fault.mount_id(root_fd), "productRepairClaimed": False} + + +def replace(root_fd, path, raw, identifier, fault, *, rescue=False): + with fault.parent(root_fd, ["data", *path[:-1]]) as directory: + temporary = ".F07-" + identifier + (".rescue" if rescue else ".inject") + try: fault.new_file(directory, temporary, raw) + except FileExistsError: + require(fault.read_file(directory, temporary, LIMIT * 4) == raw, + "interrupted durable-record temporary differs; retain scene") + os.rename(temporary, path[-1], src_dir_fd=directory, dst_dir_fd=directory) + os.fsync(directory) + require(read(root_fd, path, fault) == raw, "durable-record replacement readback failed") + + +def install(root_fd, request, owner, fault, mrf): + identity = binding(root_fd, request, owner, fault) + path = relative(request) + before = read(root_fd, path, fault) + require(request.get("expectedBefore") == entry(before), "journal bytes drifted from their archived pre-fault identity") + injected = transform(before, request, owner, mrf) + intent = {**identity, "kind": request["kind"], "before": entry(before), "injected": entry(injected)} + with fault.parent(root_fd, ["quarantine"]) as backup: + fault.new_file(backup, identity["faultId"] + ".original", before) + fault.new_file(backup, identity["faultId"] + ".intent.json", fault.encode(intent)) + # The candidate is stopped by the node adapter; validate once more immediately before replacement. + require(read(root_fd, path, fault) == before, "durable record changed after backup; preserve the intent") + replace(root_fd, path, injected, identity["faultId"], fault) + fault.new_file(backup, identity["faultId"] + ".applied.json", fault.encode(intent)) + return {**intent, "applied": True, "originalBackupRetained": True} + + +def recover(root_fd, request, owner, fault): + identity = binding(root_fd, request, owner, fault) + with fault.parent(root_fd, ["quarantine"]) as backup: + intent = json.loads(fault.read_file(backup, identity["faultId"] + ".intent.json", 16384)) + require(all(intent.get(key) == value for key, value in identity.items()), "journal recovery lost original run/candidate/device/record identity") + before = fault.read_file(backup, identity["faultId"] + ".original", LIMIT * 4) + require(entry(before) == intent["before"], "journal original backup is corrupt") + path = relative(request) + current = read(root_fd, path, fault) + require(entry(current) in (intent["before"], intent["injected"]), + "product changed or advanced the durable responsibility; refusing old-record resurrection") + restored = current != before + if restored: + failed = identity["faultId"] + ".failed-" + entry(current)["sha256"] + try: fault.new_file(backup, failed, current) + except FileExistsError: require(fault.read_file(backup, failed, LIMIT * 4) == current, "preserved journal differs") + replace(root_fd, path, before, identity["faultId"], fault, rescue=True) + require(read(root_fd, path, fault) == before, "journal recovery did not preserve exact original bytes") + return {**identity, "restoredFromBackup": restored, "backupRetained": True, + "originalResponsibilityRecreated": False, "productRepairClaimed": False} diff --git a/test_validation_heal_journal_fault.py b/test_validation_heal_journal_fault.py new file mode 100644 index 00000000..adb3bf03 --- /dev/null +++ b/test_validation_heal_journal_fault.py @@ -0,0 +1,186 @@ +"""Exact durable-record corruption preserves ownership and refuses stale rescue.""" + +import base64 +import hashlib +import json +import os +from pathlib import Path +import struct +import tempfile +import types +import unittest +import uuid +from unittest.mock import patch +import zlib + +from scripts.validation_heal import journal_fault as journal, heal_fault as fault, mrf_records as mrf +from scripts.validation_heal.healthy import Fleet + + +class DurableRecordFaultTests(unittest.TestCase): + token = "12345678-1234-4234-8234-123456789abc" + incarnation = "22345678-1234-4234-8234-123456789abc" + bucket = "rv-heal-0123456789abcdef" + + def checkpoint(self): + value = {"task_id": self.token, "schema_version": 8, + "admin": {"completed": False, "buckets": {self.bucket: self.incarnation}}, + "processed_objects": ["objects/second", "objects/first"], "failed_objects": [], "skipped_objects": [], + "integrity_digest": None, "current_object_index": 2, "successful_objects": 2} + unsigned = {**value, "processed_objects": sorted(value["processed_objects"])} + value["integrity_digest"] = base64.b64encode(hashlib.sha256(json.dumps(unsigned, + sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode()).digest()).decode() + return json.dumps(value).encode() + + def fixture(self, root, kind="truncate", family="checkpoint"): + (root / "data/.rustfs.sys/buckets").mkdir(parents=True) + (root / "quarantine").mkdir() + owner = {"runtimeProfile": "root-restart", "runId": "gh-12345-1-h1", "bucket": self.bucket, "binarySha256": "a" * 64} + request = {"runId": owner["runId"], "family": family, "taskId": self.token, "disk": 0, + "faultId": "F07-checkpoint-0", "kind": kind, "relativePath": journal.names(family, self.token)[0]} + raw = self.checkpoint() if family == "checkpoint" else json.dumps({"schema": 3, "task_id": self.token, + "heal_type": {"type": "bucket", "bucket": self.bucket}, "bucket_incarnation_id": self.incarnation}).encode() + path = root / "data" / request["relativePath"] + path.write_bytes(raw) + request["expectedBefore"] = journal.entry(raw) + (root / "data/.rustfs.sys/unrelated.json").write_bytes(b"untouched control") + return owner, request, path, raw + + def test_real_file_fault_and_idempotent_rescue_retain_original_and_failed_bytes(self): + for kind in ["truncate", "future-schema"]: + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, raw = self.fixture(root, kind) + with fault.directory(root) as fd: + result = journal.install(fd, request, owner, fault, mrf) + self.assertEqual(result["before"], journal.entry(raw)) + self.assertNotEqual(path.read_bytes(), raw) + self.assertFalse(result["productRepairClaimed"]) + if kind == "future-schema": self.assertEqual(json.loads(path.read_bytes())["schema_version"], 9) + rescued = journal.recover(fd, request, owner, fault) + self.assertTrue(rescued["restoredFromBackup"]) + self.assertEqual(path.read_bytes(), raw) + self.assertFalse(journal.recover(fd, request, owner, fault)["restoredFromBackup"]) + self.assertEqual((root / "data/.rustfs.sys/unrelated.json").read_bytes(), b"untouched control") + self.assertEqual((root / "quarantine/F07-checkpoint-0.original").read_bytes(), raw) + self.assertEqual(len(list((root / "quarantine").glob("*.failed-*"))), 1) + + def test_root_fault_requires_exact_bucket_admission_and_preserves_terminal_names(self): + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, raw = self.fixture(root, family="root") + with fault.directory(root) as fd: + journal.install(fd, request, owner, fault, mrf) + journal.recover(fd, request, owner, fault) + self.assertEqual(path.read_bytes(), raw) + for field, value in [("task_id", str(uuid.uuid4())), ("bucket_incarnation_id", str(uuid.UUID(int=0))), + ("heal_type", {"type": "bucket", "bucket": "foreign"})]: + bad = json.loads(raw); bad[field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + journal.transform(json.dumps(bad).encode(), request, owner, mrf) + + def test_path_token_slot_profile_and_expected_bytes_cannot_escape_scope(self): + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, raw = self.fixture(root) + changes = [{"relativePath": ".rustfs.sys/config/config.json"}, {"relativePath": "../foreign"}, + {"relativePath": journal.names("checkpoint", self.token)[1]}, {"taskId": str(uuid.UUID(int=0))}, + {"taskId": "../escape"}, {"disk": True}, {"disk": 4}, {"runId": "gh-other-1-h1"}, + {"faultId": "C11-foreign"}, {"kind": "stale-generation"}, {"expectedBefore": {"sha256": "b" * 64, "size": len(raw)}}] + with fault.directory(root) as fd: + for change in changes: + with self.subTest(change=change), self.assertRaises(ValueError): journal.install(fd, {**request, **change}, owner, fault, mrf) + self.assertEqual(path.read_bytes(), raw) + with self.assertRaises(ValueError): journal.install(fd, request, {**owner, "runtimeProfile": "manual"}, fault, mrf) + + def test_corrupt_or_foreign_original_checkpoint_is_never_mutated(self): + owner = {"bucket": self.bucket} + request = {"family": "checkpoint", "taskId": self.token, "kind": "truncate", + "relativePath": journal.names("checkpoint", self.token)[0]} + raw = self.checkpoint() + for field, value in [("task_id", str(uuid.uuid4())), ("schema_version", 9), ("integrity_digest", "wrong"), + ("admin", {"completed": True, "buckets": {self.bucket: self.incarnation}}), + ("admin", {"completed": False, "buckets": {"foreign": self.incarnation}})]: + bad = json.loads(raw); bad[field] = value + with self.subTest(field=field), self.assertRaises(ValueError): journal.transform(json.dumps(bad).encode(), request, owner, mrf) + + def test_successor_deleted_or_foreign_bytes_are_not_overwritten_by_rescue(self): + for changed in ["advanced", "missing", "backup", "candidate", "device"]: + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, _ = self.fixture(root) + with fault.directory(root) as fd: + journal.install(fd, request, owner, fault, mrf) + if changed == "advanced": path.write_bytes(b"new product responsibility") + elif changed == "missing": path.unlink() + elif changed == "backup": (root / "quarantine/F07-checkpoint-0.original").write_bytes(b"bad backup") + elif changed == "candidate": owner["binarySha256"] = "b" * 64 + else: + p = root / "quarantine/F07-checkpoint-0.intent.json"; intent = json.loads(p.read_text()); intent["device"] = -1 + p.write_text(json.dumps(intent)) + current = path.read_bytes() if path.exists() else None + with self.subTest(changed=changed), self.assertRaises((ValueError, FileNotFoundError)): + journal.recover(fd, request, owner, fault) + self.assertEqual(path.read_bytes() if path.exists() else None, current) + + def test_record_symlink_hardlink_and_retained_backup_prevent_mutation(self): + for changed in ["symlink", "hardlink", "backup"]: + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, raw = self.fixture(root) + if changed == "symlink": path.unlink(); original = root / "original"; original.write_bytes(raw); path.symlink_to(original) + elif changed == "hardlink": os.link(path, root / "alias") + else: (root / "quarantine/F07-checkpoint-0.original").write_bytes(b"retained") + with fault.directory(root) as fd, self.subTest(changed=changed), self.assertRaises((OSError, ValueError)): + journal.install(fd, request, owner, fault, mrf) + self.assertEqual(path.read_bytes(), raw) + + def test_interrupted_rescue_temp_is_revalidated_without_erasing_injection_evidence(self): + for changed in [False, True]: + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, raw = self.fixture(root) + with fault.directory(root) as fd: + journal.install(fd, request, owner, fault, mrf) + injection = path.parent / ".F07-F07-checkpoint-0.inject" + injection.write_bytes(b"interrupted injection evidence") + rescue = path.parent / ".F07-F07-checkpoint-0.rescue" + rescue.write_bytes(b"wrong temporary" if changed else raw) + if changed: + current = path.read_bytes() + with self.assertRaises(ValueError): journal.recover(fd, request, owner, fault) + self.assertEqual(path.read_bytes(), current) + else: + journal.recover(fd, request, owner, fault) + self.assertEqual(path.read_bytes(), raw) + self.assertEqual(injection.read_bytes(), b"interrupted injection evidence") + + def test_mrf_crc_fault_retains_every_original_identity_and_refuses_foreign_payload(self): + bucket, key = self.bucket.encode(), b"offline/new.bin" + raw = struct.pack(" Date: Sun, 11 Oct 2026 11:47:33 +0800 Subject: [PATCH 122/343] test(validation): bind original Admin checkpoint observations --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 26 +++ scripts/validation_heal/checkpoint.py | 151 ++++++++++++++++++ scripts/validation_heal/fleet_node.py | 21 ++- scripts/validation_heal/journal_fault.py | 32 +++- test_validation_heal_checkpoint.py | 113 +++++++++++++ test_validation_heal_journal_fault.py | 47 ++++++ 7 files changed, 385 insertions(+), 9 deletions(-) create mode 100644 scripts/validation_heal/checkpoint.py create mode 100644 test_validation_heal_checkpoint.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index e9308f54..fb1ff60e 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -35,6 +35,7 @@ on: - test_validation_heal_native.py - test_validation_heal_manager_native.py - test_validation_heal_journal_fault.py + - test_validation_heal_checkpoint.py - test_validation_heal_authority.py - test_validation_heal_orphan.py - test_validation_heal_options.py @@ -88,6 +89,7 @@ on: - test_validation_heal_native.py - test_validation_heal_manager_native.py - test_validation_heal_journal_fault.py + - test_validation_heal_checkpoint.py - test_validation_heal_authority.py - test_validation_heal_orphan.py - test_validation_heal_options.py @@ -130,7 +132,7 @@ jobs: sudo apt-get update sudo apt-get install --no-install-recommends -y nftables fi - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_journal_fault.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index f9949758..0f6b97e8 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1790,3 +1790,29 @@ resurrected. Distinct injection/rescue temporaries preserve interrupted evidence. Ten file/remote-boundary regressions cover these guarantees. This backend remains an F07 prerequisite; the six canonical branches and their full live execution are still outstanding, with no count or PASS increase. + +Hosted manager run `38108127382` attempt 1 succeeded with driver +`b8126e50f78e40dfd108c7bf32162c751d4a87dd` and collector +`aba6f5c6a36121adcc5a67663b767d0de5667ba9`. Independent artifact verification +confirmed the immutable source tree/lock/eight source hashes, both collector +digests, executable SHA-256 +`3ec2d5cfb22db74ae44ddacb6548e0b30fbf0fc35359e0fa04bc4c88e83c431b`, +the complete build/runtime listings and fifteen original one-test commands, +logs and PASS results. The original four publication interruption hooks are +distinct from individual fsync hooks and physical power loss. This adds no +canonical fleet PASS. + +`checkpoint.py` locates the actual Admin erasure-set checkpoint selected by +an empty root request with explicit pool/set selectors. Ordinary bucket and +cluster scans cannot substitute for that record. Exact task/disk paths must +contain one schema-8 checkpoint and one schema-7 Admin resume state on the +same physical owner, with unchanged scope/options, valid candidate digest, +owned bucket incarnation and truthful durable progress. Read-only live +observation brackets two identical bounded record reads with the original +service birth, candidate, command and environment. The freeze/crash boundary +requires pidfd-bound SIGSTOP/SIGKILL and unchanged stopped record bytes before +any fault; no platform power event is claimed. Duplicate JSON fields, +non-finite values and duplicate dedup identities reject injection. Ten new +negative tests cover live replacement/process drift, missing/duplicate owners, +foreign scope and fabricated progress. These are F07/C11 prerequisites; +the remaining 24 complete executors are still outstanding. diff --git a/scripts/validation_heal/checkpoint.py b/scripts/validation_heal/checkpoint.py new file mode 100644 index 00000000..2946037f --- /dev/null +++ b/scripts/validation_heal/checkpoint.py @@ -0,0 +1,151 @@ +"""Locate original Admin erasure-set responsibility before F07/C11 faults.""" + +import base64 +import json +import time + +try: + from . import journal_fault as journal + from .healthy import BASE_OPTIONS + from .heal_probe import canonical, require + from .fleet_node import PROCESS_FIELDS +except ImportError: + import journal_fault as journal + from healthy import BASE_OPTIONS + from heal_probe import canonical, require + from fleet_node import PROCESS_FIELDS + + +PATH = "/rustfs/admin/v3/heal/" +OPTIONS = {**BASE_OPTIONS, "recursive": True} +REFERENCE = {"candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", "sourceFiles": { + "rustfs/src/admin/handlers/heal.rs": "7e4007c29a8329ced8e48b88111400e79cffd845bd6ddc6243b79f3cd48e8199", + "crates/heal/src/heal/channel.rs": "7f669ca43b9559db9b97fba6b419a6c76a94d76acd8526db132d74b7c534d6b0", + "crates/heal/src/heal/resume.rs": "ed55baa03fafa79bf662393eafcf53b4f7752ca8b2f77824cc6deff085a0cf1d", + "crates/heal/src/heal/resume/checkpoint.rs": "fe067b39ee406e16517bd665f7f49457daffa05d5d7bc298bd965e927b1381c1", + "crates/heal/src/heal/erasure_healer/admin.rs": "776c03e4dfbd27f51da506d8f637de3a3973d86d22dc83a7cda24f2e0d642ee5"}, + "route": "empty bucket/object plus explicit pool=0,set=0 selects Admin ErasureSet", + "checkpointSchema": 8, "resumeSchema": 7, "durableOwnerRequired": True, + "bucketOrClusterScanIsCheckpointProof": False, "canonicalPassClaimed": False} + + +def bytes_of(row): + require(row.get("present") is True and isinstance(row.get("rawBase64"), str), "record has no original bytes") + raw = base64.b64decode(row["rawBase64"], validate=True) + require(journal.entry(raw) == {name: row.get(name) for name in ("size", "sha256")}, + "record bytes differ from their exact archived size/digest") + return raw + + +def owned_checkpoint(observed, token, bucket, *, require_progress=True): + journal.task_id(token) + names = journal.names("checkpoint", token) + require(isinstance(observed, dict) and len(observed) in {1, 4}, "checkpoint census omitted bound nodes") + present = {} + for node, census in observed.items(): + rows = census.get("records") + require(census.get("readOnly") is True and census.get("family") == "checkpoint" + and census.get("taskId") == token and census.get("directoryEnumerationUsed") is False + and (census.get("quiescent") is True or census.get("stable") is True + and census.get("processBound") is True and census.get("independentReads") == 2) + and isinstance(rows, list) and len(rows) == 16, "checkpoint has no stopped or stable process-bound census") + seen = set() + for row in rows: + pair = row.get("disk"), row.get("relativePath") + require(type(pair[0]) is int and 0 <= pair[0] < 4 and pair[1] in names and pair not in seen + and type(row.get("present")) is bool, "checkpoint census duplicated or escaped its exact task/disk paths") + seen.add(pair) + if row["present"]: + require(pair[1] not in present, "Admin task has multiple physical checkpoint owners") + bytes_of(row) + present[pair[1]] = {"node": node, **row} + require(set(present) == set(names[:2]), + "NOT_OBSERVED: original runnable Admin checkpoint/resume pair is absent, blocked or has an unexpected digest record") + checkpoint, resume = present[names[0]], present[names[1]] + require((checkpoint["node"], checkpoint["disk"]) == (resume["node"], resume["disk"]), + "Admin checkpoint and resume state have different physical owners") + value = journal.checkpoint_value(bytes_of(checkpoint), token, bucket) + admin = value["admin"] + require(admin.get("set_disk_id") == "pool_0_set_0" and canonical(admin.get("options")) == canonical(OPTIONS), + "checkpoint changed the exact admitted set or Admin options") + outcome = admin.get("outcome") or {} + counters = outcome.get("counters") or {} + require(outcome.get("execution", {}).get("state") == "running" + and outcome.get("coverage") == "partial" + and all(type(counters.get(name)) is int and 0 <= counters[name] < 2**64 + for name in ("processed", "healed", "unchanged", "skipped", "failed", "unknown", "attemptFailures")) + and counters.get("overflowed") is False, "Admin checkpoint omitted truthful running outcome counters") + require(counters["processed"] == sum(counters[name] for name in ("healed", "unchanged", "skipped", "failed")) + and counters["unknown"] <= counters["skipped"] and counters["attemptFailures"] >= counters["failed"], + "Admin checkpoint counters do not account for the actual durable dispositions") + require(not require_progress or counters["processed"] > 0, + "NOT_OBSERVED: original Admin checkpoint has no durable object progress") + state = journal.decode_json(bytes_of(resume)) + require(type(state.get("schema_version")) is int and state["schema_version"] == 7 + and state.get("task_id") == token and state.get("task_type") == "admin_erasure_set" + and state.get("set_disk_id") == "pool_0_set_0" and state.get("completed") is False + and not state.get("replacement_generation") and not state.get("replacement_targets"), + "checkpoint companion is not the original runnable Admin erasure-set resume state") + return {"taskId": token, "bucket": bucket, "bucketIncarnationId": admin["buckets"][bucket], + "node": checkpoint["node"], "disk": checkpoint["disk"], "checkpoint": checkpoint, + "resume": resume, "value": value, "resumeValue": state, "originalDurableResponsibility": True} + + +def records(fleet, token, label, *, quiescent=False): + result = {} + for node in fleet.nodes: + params = {"family": "checkpoint", "taskId": token} + if not quiescent: + params["expected"] = fleet.record(label + "-process", {"node": node, **fleet.remote(node, "process")}) + result[node] = fleet.remote(node, "journal-census" if quiescent else "journal-observe", **params) + return fleet.record(label, result) + + +def admit(probe, fleet, active): + fleet.record("Admin-checkpoint-route-contract", REFERENCE) + result = fleet.record("Admin-erasure-set-admission", probe.admin(PATH, [], json.dumps(OPTIONS).encode())) + require(result.get("httpStatus") == 200 and isinstance(result.get("response"), dict), + "Admin erasure-set admission failed") + token = journal.task_id(result["response"].get("clientToken")) + active.append((PATH, token)) + return token + + +def freeze(fleet, token, bucket, *, timeout=60): + require(type(timeout) is int and 0 < timeout <= 120, "unreviewed checkpoint observation budget") + deadline = time.monotonic() + timeout + while True: + try: + owner = owned_checkpoint(records(fleet, token, "Admin-checkpoint-live"), token, bucket) + break + except ValueError as exc: + if not str(exc).startswith("NOT_OBSERVED:"): raise + require(time.monotonic() < deadline, "NOT_OBSERVED: actual durable Admin checkpoint window exceeded its frozen budget") + time.sleep(.1) + node = owner["node"] + identity = fleet.record("checkpoint-owner-process", fleet.remote(node, "process")) + require(node not in fleet.frozen_processes, "checkpoint owner is already tracked as frozen") + fleet.frozen_processes[node] = identity + fleet.record("checkpoint-owner-freeze", fleet.remote(node, "process-signal", signal="SIGSTOP", expected=identity)) + frozen = fleet.record("checkpoint-owner-frozen-process", fleet.remote(node, "process")) + require(frozen.get("state") == "T" and all(frozen.get(name) == identity.get(name) for name in PROCESS_FIELDS), + "checkpoint freeze missed the original process") + current = owned_checkpoint(records(fleet, token, "Admin-checkpoint-frozen"), token, bucket) + require((current["node"], current["disk"], current["bucketIncarnationId"]) == + (owner["node"], owner["disk"], owner["bucketIncarnationId"]), "checkpoint owner changed during freeze") + return current, identity + + +def stop_for_fault(fleet, token, bucket, frozen, identity): + node = frozen["node"] + require(fleet.frozen_processes.get(node) == identity, "checkpoint crash lost its tracked original process") + exit_proof = fleet.record("checkpoint-owner-exact-crash", fleet.remote(node, "process-crash", expected=identity)) + require(exit_proof.get("actualSignal") == 9 and exit_proof.get("pidfdBound") is True + and exit_proof.get("graceful") is False and exit_proof.get("platformEventIssued") is False, + "checkpoint interruption has no exact process-bound SIGKILL witness") + del fleet.frozen_processes[node] + fleet.stop() + current = owned_checkpoint(records(fleet, token, "Admin-checkpoint-stopped", quiescent=True), token, bucket) + require(current["checkpoint"] == frozen["checkpoint"] and current["resume"] == frozen["resume"], + "Admin durable responsibility changed between freeze and stopped fault boundary") + return current diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index b3067532..97e713f4 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -974,12 +974,27 @@ def configure_encryption(request): def journal_operation(request): state, roots, owner = owned(request["runId"]) - stopped(owner) require(digest(state / "rustfs") == owner["binarySha256"], "durable-record candidate changed") require(owner.get("runtimeProfile") in {"root-restart", "mrf-crash"}, "durable records require the owned recovery runtime") - require(set(request) <= {"runId", "action", "family", "taskId", "disk", "relativePath", "faultId", "kind", "expectedBefore"}, + require(set(request) <= {"runId", "action", "family", "taskId", "disk", "relativePath", "faultId", "kind", "expectedBefore", "expected"}, "durable-record adapter accepts no arbitrary system paths or commands") module, fault = globals()["journal_fault_backend"], fault_module() + if request["action"] == "journal-observe": + require(set(request) == {"runId", "action", "family", "taskId", "expected"}, + "live durable-record observation accepts only a process-bound exact family/task") + before = process(request) + expected = request.get("expected") + require(isinstance(expected, dict) and before.get("pid", 0) > 1 + and all(name in expected and before.get(name) == expected[name] for name in PROCESS_FIELDS), + "live durable-record observation lost its exact service birth or candidate") + observed = module.stable_census(roots, request, fault) + after = process(request) + require(all(after.get(name) == before.get(name) for name in PROCESS_FIELDS), + "service restarted while observing exact durable records") + return {**observed, "processIdentity": {name: before[name] for name in PROCESS_FIELDS}, + "processBound": True, "serviceFrozen": before.get("state") == after.get("state") == "T"} + require("expected" not in request, "stopped durable-record operations accept no live-process override") + stopped(owner) if request["action"] == "journal-census": return module.census(roots, request, fault) require(request["action"] in {"journal-apply", "journal-recover"} @@ -1230,7 +1245,7 @@ def main(request): actions["task-records"] = task_records actions["environment-capabilities"] = environment_capabilities actions["quota-audit-finalize"] = quota_audit_finalize - for action in ("journal-census", "journal-apply", "journal-recover"): + for action in ("journal-census", "journal-observe", "journal-apply", "journal-recover"): actions[action] = journal_operation for action in ('generation-capture', 'generation-install', 'generation-recover'): actions[action] = generation_operation diff --git a/scripts/validation_heal/journal_fault.py b/scripts/validation_heal/journal_fault.py index 4381925e..933a1080 100644 --- a/scripts/validation_heal/journal_fault.py +++ b/scripts/validation_heal/journal_fault.py @@ -48,7 +48,7 @@ def read(root_fd, path, fault): return fault.read_file(directory, path[-1], LIMIT * (4 if "lifecycle" in path[-1] else 1)) -def census(roots, request, fault): +def census(roots, request, fault, *, quiescent=True): paths = names(request.get("family"), request.get("taskId")) rows = [] for disk, root in enumerate(roots): @@ -59,12 +59,33 @@ def census(roots, request, fault): except FileNotFoundError: rows.append(row); continue row.update(present=True, **entry(raw), rawBase64=base64.b64encode(raw).decode()) rows.append(row) - return {"readOnly": True, "quiescent": True, "family": request["family"], + return {"readOnly": True, "quiescent": quiescent, "family": request["family"], "taskId": request.get("taskId"), "records": rows, "directoryEnumerationUsed": False} +def stable_census(roots, request, fault): + first = census(roots, request, fault, quiescent=False) + second = census(roots, request, fault, quiescent=False) + require(first == second, "NOT_OBSERVED: exact durable records changed between the two bounded reads") + return {**second, "stable": True, "independentReads": 2} + + +def decode_json(raw): + def unique_pairs(pairs): + result = {} + for key, value in pairs: + require(key not in result, "durable record contains duplicate JSON fields") + result[key] = value + return result + def invalid_constant(value): + raise ValueError("durable record contains a non-finite JSON value") + value = json.loads(raw, object_pairs_hook=unique_pairs, parse_constant=invalid_constant) + require(isinstance(value, dict), "durable record is not a JSON object") + return value + + def checkpoint_value(raw, token, bucket): - value = json.loads(raw) + value = decode_json(raw) require(value.get("task_id") == task_id(token) and value.get("schema_version") == 8, "checkpoint does not belong to the exact original Admin task") admin = value.get("admin") or {} @@ -72,10 +93,11 @@ def checkpoint_value(raw, token, bucket): require(admin.get("completed") is False and isinstance(buckets, dict) and set(buckets) == {bucket} and isinstance(buckets[bucket], str) and str(uuid.UUID(buckets[bucket])) == buckets[bucket] and uuid.UUID(buckets[bucket]).int != 0, "checkpoint has no original owned bucket incarnation") - unsigned = json.loads(raw) + unsigned = decode_json(raw) unsigned["integrity_digest"] = None for field in ("processed_objects", "failed_objects", "skipped_objects"): - require(isinstance(unsigned.get(field), list) and all(isinstance(item, str) for item in unsigned[field]), + require(isinstance(unsigned.get(field), list) and all(isinstance(item, str) for item in unsigned[field]) + and len(set(unsigned[field])) == len(unsigned[field]), "checkpoint ledger has an unknown canonical representation") unsigned[field].sort() # Candidate serde_json::Value uses sorted object keys and UTF-8 strings. diff --git a/test_validation_heal_checkpoint.py b/test_validation_heal_checkpoint.py new file mode 100644 index 00000000..d5bf790d --- /dev/null +++ b/test_validation_heal_checkpoint.py @@ -0,0 +1,113 @@ +"""An actual set checkpoint is distinct from a bucket/root scan and counters.""" + +import base64 +import copy +import hashlib +import json +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from scripts.validation_heal import checkpoint as reader, journal_fault as journal + + +TOKEN = "12345678-1234-4234-8234-123456789abc" +INCARNATION = "22345678-1234-4234-8234-123456789abc" +BUCKET = "rv-heal-0123456789abcdef" + + +def encoded(value, *, signed=False): + value = copy.deepcopy(value) + if signed: + value["integrity_digest"] = None + for field in ("processed_objects", "failed_objects", "skipped_objects"): value[field].sort() + unsigned = json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode() + value["integrity_digest"] = base64.b64encode(hashlib.sha256(unsigned).digest()).decode() + raw = json.dumps(value).encode() + return {"present": True, **journal.entry(raw), "rawBase64": base64.b64encode(raw).decode()} + + +class AdminCheckpointTests(unittest.TestCase): + def fixture(self): + value = {"schema_version": 8, "task_id": TOKEN, "checkpoint_time": 100, + "current_bucket_index": 0, "current_object_index": 2, + "processed_objects": ["objects/first", "objects/second"], "failed_objects": [], "skipped_objects": [], + "successful_objects": 2, "failed_object_count": 0, "skipped_object_count": 0, + "skipped_new_versions": 0, "skipped_ilm_expired": 0, "processed_bytes": 2, + "total_objects": 3, "total_bytes": 3, "baseline_generation": None, "baseline_known": False, + "counter_unknown": False, "integrity_digest": None, + "admin": {"set_disk_id": "pool_0_set_0", "options": reader.OPTIONS, "buckets": {BUCKET: INCARNATION}, + "completed": False, "outcome": {"execution": {"state": "running"}, "coverage": "partial", + "objects": [], "objectsTruncated": False, "counters": {"processed": 2, "healed": 1, "unchanged": 1, + "skipped": 0, "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False}}}} + resume = {"schema_version": 7, "task_id": TOKEN, "task_type": "admin_erasure_set", + "set_disk_id": "pool_0_set_0", "completed": False, "replacement_targets": [], "replacement_generation": None} + rows = [{"disk": disk, "relativePath": name, "present": False} + for disk in range(4) for name in journal.names("checkpoint", TOKEN)] + rows[4].update(encoded(value, signed=True)); rows[5].update(encoded(resume)) + return {"node": {"readOnly": True, "quiescent": True, "family": "checkpoint", "taskId": TOKEN, + "directoryEnumerationUsed": False, "records": rows}}, value, resume + + def test_exact_original_checkpoint_resume_pair_has_one_physical_owner(self): + observed, _, _ = self.fixture() + owner = reader.owned_checkpoint(observed, TOKEN, BUCKET) + self.assertEqual((owner["node"], owner["disk"], owner["bucketIncarnationId"]), ("node", 1, INCARNATION)) + self.assertTrue(owner["originalDurableResponsibility"]) + + def test_root_or_bucket_scan_without_checkpoint_cannot_be_used_as_durable_progress(self): + observed, _, _ = self.fixture() + for row in observed["node"]["records"]: row["present"] = False + with self.assertRaisesRegex(ValueError, "NOT_OBSERVED"): reader.owned_checkpoint(observed, TOKEN, BUCKET) + + def test_original_bytes_hash_exact_rows_and_unique_owner_are_required(self): + for change in ("bytes", "hash", "duplicate-row", "second-owner", "wrong-disk", "resume-owner", "omitted-disk", "blocked"): + observed, _, _ = self.fixture(); rows = observed["node"]["records"] + if change == "bytes": rows[4]["rawBase64"] = base64.b64encode(b"changed").decode() + elif change == "hash": rows[4]["sha256"] = "b" * 64 + elif change == "duplicate-row": rows[0] = copy.deepcopy(rows[4]) + elif change == "second-owner": observed["other"] = copy.deepcopy(observed["node"]) + elif change == "wrong-disk": rows[4]["disk"] = True + elif change == "resume-owner": rows[1].update({key: rows[5][key] for key in ("present", "size", "sha256", "rawBase64")}); rows[5]["present"] = False + elif change == "omitted-disk": rows.pop() + else: rows[6].update(encoded({"blocked": True})) + with self.subTest(change=change), self.assertRaises(ValueError): reader.owned_checkpoint(observed, TOKEN, BUCKET) + + def test_wrong_scope_incarnation_options_and_fabricated_progress_are_rejected(self): + changes = [("set_disk_id", "pool_0_set_1"), ("options", {**reader.OPTIONS, "pool": True}), + ("options", {**reader.OPTIONS, "nolock": True}), ("buckets", {"foreign": INCARNATION}), + ("buckets", {BUCKET: None}), ("completed", True)] + for key, new in changes: + observed, value, _ = self.fixture(); value["admin"][key] = new + observed["node"]["records"][4].update(encoded(value, signed=True)) + with self.subTest(key=key, new=new), self.assertRaises(ValueError): reader.owned_checkpoint(observed, TOKEN, BUCKET) + observed, value, _ = self.fixture() + for name in ("processed", "healed", "unchanged"): value["admin"]["outcome"]["counters"][name] = 0 + observed["node"]["records"][4].update(encoded(value, signed=True)) + with self.assertRaisesRegex(ValueError, "NOT_OBSERVED"): reader.owned_checkpoint(observed, TOKEN, BUCKET) + + def test_replacement_or_completed_resume_cannot_be_rebound_to_admin_task(self): + for key, new in [("task_id", INCARNATION), ("task_type", "erasure_set"), ("schema_version", 6), + ("set_disk_id", "pool_1_set_0"), ("completed", True), ("replacement_generation", TOKEN)]: + observed, _, resume = self.fixture(); resume[key] = new + observed["node"]["records"][5].update(encoded(resume)) + with self.subTest(key=key), self.assertRaises(ValueError): reader.owned_checkpoint(observed, TOKEN, BUCKET) + + def test_live_census_requires_two_stable_reads_bound_to_original_process(self): + observed, _, _ = self.fixture(); value = observed["node"] + value.update(quiescent=False, stable=True, processBound=True, independentReads=2) + reader.owned_checkpoint(observed, TOKEN, BUCKET) + for key, new in [("stable", False), ("processBound", False), ("independentReads", 1)]: + changed = copy.deepcopy(observed); changed["node"][key] = new + with self.subTest(key=key), self.assertRaises(ValueError): reader.owned_checkpoint(changed, TOKEN, BUCKET) + + def test_set_admission_keeps_explicit_selectors_and_actual_canonical_token(self): + active = []; calls = [] + fleet = SimpleNamespace(record=lambda label, value: value) + probe = SimpleNamespace(admin=lambda path, query, body: calls.append((path, query, json.loads(body))) + or {"httpStatus": 200, "response": {"clientToken": TOKEN}}) + self.assertEqual(reader.admit(probe, fleet, active), TOKEN) + self.assertEqual(calls, [(reader.PATH, [], reader.OPTIONS)]) + self.assertEqual(active, [(reader.PATH, TOKEN)]) + + +if __name__ == "__main__": unittest.main() diff --git a/test_validation_heal_journal_fault.py b/test_validation_heal_journal_fault.py index adb3bf03..d95aac35 100644 --- a/test_validation_heal_journal_fault.py +++ b/test_validation_heal_journal_fault.py @@ -15,6 +15,7 @@ from scripts.validation_heal import journal_fault as journal, heal_fault as fault, mrf_records as mrf from scripts.validation_heal.healthy import Fleet +from scripts.validation_heal import fleet_node class DurableRecordFaultTests(unittest.TestCase): @@ -171,6 +172,52 @@ def test_census_is_exact_and_never_enumerates_unrelated_system_metadata(self): self.assertEqual(base64.b64decode(present[0]["rawBase64"]), raw) self.assertFalse(result["directoryEnumerationUsed"]) + def test_stable_observation_rejects_atomic_replacement_between_reads(self): + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); _, request, path, raw = self.fixture(root) + stable = journal.stable_census([root], request, fault) + self.assertFalse(stable["quiescent"]) + self.assertTrue(stable["stable"]) + original = journal.census + calls = [] + def changed(*args, **kwargs): + value = original(*args, **kwargs) + if not calls: path.write_bytes(raw + b" ") + calls.append(value) + return value + with patch.object(journal, "census", side_effect=changed), self.assertRaisesRegex(ValueError, "NOT_OBSERVED"): + journal.stable_census([root], request, fault) + + def test_duplicate_and_nonfinite_json_never_become_candidate_checkpoint_bytes(self): + raw = self.checkpoint() + for bad in [raw[:-1] + b',"task_id":"foreign"}', raw.replace(b'"current_object_index": 2', b'"current_object_index": NaN')]: + with self.subTest(raw=bad), self.assertRaises(ValueError): journal.checkpoint_value(bad, self.token, self.bucket) + + def test_live_adapter_binds_both_reads_to_original_process_and_cannot_mutate(self): + identity = {name: name for name in fleet_node.PROCESS_FIELDS} + identity.update(pid=50, state="T") + owner = {"runtimeProfile": "root-restart", "binarySha256": "a" * 64} + observed = {"readOnly": True, "quiescent": False, "stable": True, "independentReads": 2} + request = {"runId": "gh-12345-1-h1", "action": "journal-observe", "family": "checkpoint", + "taskId": self.token, "expected": identity} + backend = types.SimpleNamespace(stable_census=lambda *args: observed) + with patch.object(fleet_node, "owned", return_value=(Path('/owned'), [], owner)), \ + patch.object(fleet_node, "digest", return_value="a" * 64), \ + patch.object(fleet_node, "fault_module", return_value=fault), \ + patch.dict(fleet_node.__dict__, journal_fault_backend=backend), \ + patch.object(fleet_node, "stopped", side_effect=AssertionError("read-only live observation must not stop service")): + with patch.object(fleet_node, "process", side_effect=[identity, identity]): + result = fleet_node.journal_operation(request) + self.assertTrue(result["processBound"]) + self.assertTrue(result["serviceFrozen"]) + self.assertFalse(result["quiescent"]) + for changed in [{**identity, "pid": 60}, {**identity, "binarySha256": "b" * 64}, + {**identity, "startTicks": 999}]: + with self.subTest(changed=changed), patch.object(fleet_node, "process", side_effect=[identity, changed]), self.assertRaises(ValueError): + fleet_node.journal_operation(request) + with patch.object(fleet_node, "process", return_value=identity), self.assertRaises(ValueError): + fleet_node.journal_operation({**request, "kind": "truncate"}) + def test_remote_action_loads_exact_record_and_fault_dependencies(self): with tempfile.TemporaryDirectory() as temp, patch("scripts.validation_heal.healthy.subprocess.run", return_value=types.SimpleNamespace(returncode=0, stdout='{}', stderr='')) as command: From 1ab525f9755d1a4174b0b552fa530b7821fdaf9b Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 11:55:35 +0800 Subject: [PATCH 123/343] test(validation): verify scoped manager evidence before fleet use --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 11 ++++ scripts/validation_heal/manager_fixture.py | 59 +++++++++++++++++++ test_validation_heal_manager_fixture.py | 51 ++++++++++++++++ 4 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 scripts/validation_heal/manager_fixture.py create mode 100644 test_validation_heal_manager_fixture.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index fb1ff60e..81fbc028 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -34,6 +34,7 @@ on: - test_validation_heal_schema.py - test_validation_heal_native.py - test_validation_heal_manager_native.py + - test_validation_heal_manager_fixture.py - test_validation_heal_journal_fault.py - test_validation_heal_checkpoint.py - test_validation_heal_authority.py @@ -88,6 +89,7 @@ on: - test_validation_heal_schema.py - test_validation_heal_native.py - test_validation_heal_manager_native.py + - test_validation_heal_manager_fixture.py - test_validation_heal_journal_fault.py - test_validation_heal_checkpoint.py - test_validation_heal_authority.py @@ -132,7 +134,7 @@ jobs: sudo apt-get update sudo apt-get install --no-install-recommends -y nftables fi - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 0f6b97e8..a6d8bd71 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1816,3 +1816,14 @@ non-finite values and duplicate dedup identities reject injection. Ten new negative tests cover live replacement/process drift, missing/duplicate owners, foreign scope and fabricated progress. These are F07/C11 prerequisites; the remaining 24 complete executors are still outstanding. + +The Functional Chain consumer downloads the manager artifact from the exact +successful tools run attempt before fleet mutation. `manager_fixture.py` +independently rechecks all fifteen original results, then exposes only the +ten F07, three C11 or two J02 prerequisite tests assigned to the selected case. +A failed/partial attempt, foreign workflow, wrong candidate, changed result +or unrelated case rejects the artifact. The same consumer independently +verified the retained `38108127382` artifact for all three scopes. It exports +no canonical fleet execution, individual fsync-hook, emitted barrier-log or +physical power-loss claim. These required evidence inputs do not make the +three incomplete canonical executors dispatchable. diff --git a/scripts/validation_heal/manager_fixture.py b/scripts/validation_heal/manager_fixture.py new file mode 100644 index 00000000..2cfcc8ea --- /dev/null +++ b/scripts/validation_heal/manager_fixture.py @@ -0,0 +1,59 @@ +"""Consume hosted manager evidence as scoped canonical-case prerequisites.""" + +import argparse +import json +from pathlib import Path +import re + +try: + from . import manager_native as suite +except ImportError: + import manager_native as suite + + +SCOPES = { + "heal.checkpoint-and-mrf-corruption": ( + "checkpoint-truncated", "checkpoint-future", "checkpoint-digest", "checkpoint-stale-manager", + "checkpoint-page-order", "mrf-payload-integrity", "mrf-manifest-write-failure", "mrf-payload-write-failure", + "mrf-migration-boundaries", "root-uncertain-publication"), + "heal.bucket-incarnation-recreation": ( + "incarnation-root-replay", "incarnation-deferred-metadata", "incarnation-checkpoint"), + "heal.outcome-counters-and-error-truth": ("outcome-truth", "outcome-wrong-identity"), +} + + +def verify(directory, candidate, case_id): + suite.native.require(case_id in SCOPES, "manager native suite has no reviewed scope for this canonical case") + directory = Path(directory) + api = json.loads((directory / "tool-run.json").read_text()) + suite.native.require(type(api.get("id")) is int and api["id"] > 0 + and type(api.get("run_attempt")) is int and api["run_attempt"] > 0 + and api.get("status") == "completed" and api.get("conclusion") == "success" + and isinstance(api.get("head_sha"), str) and re.fullmatch(r"[0-9a-f]{40}", api["head_sha"]) + and api.get("path") in {".github/workflows/rustfs-functional-chain.yml", ".github/workflows/rustfs-validation-tools.yml"}, + "manager artifact has no exact successful hosted tools build attempt") + tools, evidence = directory / "manager-tools", directory / "manager-evidence" + manifest = json.loads((tools / "manager-native.json").read_text()) + proof = suite.verify_evidence(tools, evidence, candidate, api["id"], api["run_attempt"], manifest.get("testingCommit")) + scoped = [] + for key in SCOPES[case_id]: + value = suite.REGRESSIONS[key] + suite.native.require(value["relatedCases"] == [case_id], "manager native mapping changed canonical scope") + scoped.append({"regression": key, **value}) + return {**proof, "caseId": case_id, "runId": api["id"], "runAttempt": api["run_attempt"], + "driverCommit": api["head_sha"], "collectorCommit": manifest["testingCommit"], + "casePrerequisiteTests": scoped, "originalEvidenceDirectory": str(evidence.resolve()), + "individualFsyncHooksClaimed": False, "physicalPowerLossClaimed": False, + "emittedBarrierLogsClaimed": False, "canonicalFleetBranchesExecuted": False} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=tuple(SCOPES), required=True) + args = parser.parse_args() + print(json.dumps(verify(args.directory, json.loads(args.candidate.read_text()), args.case_id), indent=2)) + + +if __name__ == "__main__": main() diff --git a/test_validation_heal_manager_fixture.py b/test_validation_heal_manager_fixture.py new file mode 100644 index 00000000..d22e529f --- /dev/null +++ b/test_validation_heal_manager_fixture.py @@ -0,0 +1,51 @@ +"""Hosted manager evidence cannot be rebound to another run or case scope.""" + +import json +from pathlib import Path +import tempfile +import unittest + +import test_validation_heal_manager_native as fixtures +from scripts.validation_heal import manager_fixture as consumer + + +class ManagerFixtureTests(unittest.TestCase): + def fixture(self, root): + tools, evidence, candidate, _ = fixtures.ManagerNativeEvidenceTests().fixture(root) + tools.rename(root / "manager-tools"); evidence.rename(root / "manager-evidence") + api = {"id": 12345, "run_attempt": 1, "status": "completed", "conclusion": "success", + "path": ".github/workflows/rustfs-functional-chain.yml", "head_sha": "b" * 40} + (root / "tool-run.json").write_text(json.dumps(api)) + return candidate, api + + def test_each_case_consumes_only_its_exact_original_prerequisite_tests(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp); candidate, _ = self.fixture(root) + for case, keys in consumer.SCOPES.items(): + value = consumer.verify(root, candidate, case) + self.assertEqual([row["regression"] for row in value["casePrerequisiteTests"]], list(keys)) + self.assertEqual(value["exactTests"], 15) + self.assertFalse(value["canonicalCasePassClaimed"]) + self.assertFalse(value["canonicalFleetBranchesExecuted"]) + self.assertFalse(value["individualFsyncHooksClaimed"]) + self.assertFalse(value["physicalPowerLossClaimed"]) + + def test_failed_partial_wrong_attempt_or_foreign_api_cannot_certify_artifact(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp); candidate, api = self.fixture(root) + changes = [("id", 12346), ("id", True), ("run_attempt", 2), ("status", "in_progress"), + ("conclusion", "failure"), ("path", ".github/workflows/ci.yml"), ("head_sha", "main")] + for field, new in changes: + (root / "tool-run.json").write_text(json.dumps({**api, field: new})) + with self.subTest(field=field), self.assertRaises(ValueError): + consumer.verify(root, candidate, next(iter(consumer.SCOPES))) + + def test_unrelated_case_and_changed_original_result_cannot_be_promoted(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp); candidate, _ = self.fixture(root) + with self.assertRaises(ValueError): consumer.verify(root, candidate, 'heal.admin-nolock-safety') + (root / 'manager-evidence/outcome-truth.log').write_text('running 0 tests\ntest result: ok. 0 passed; 0 failed; 0 ignored;\n') + with self.assertRaises(ValueError): consumer.verify(root, candidate, 'heal.outcome-counters-and-error-truth') + + +if __name__ == '__main__': unittest.main() From 9761f6483424fc03427a8051d8bfd7cd85adec17 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 12:10:44 +0800 Subject: [PATCH 124/343] test(validation): preserve captured checkpoint generations and recovery --- docs/validation-migration-map.md | 26 +++ scripts/validation_heal/checkpoint.py | 42 +++- scripts/validation_heal/fleet_node.py | 22 ++- scripts/validation_heal/healthy.py | 12 ++ scripts/validation_heal/journal_fault.py | 93 ++++++++- test_validation_heal_checkpoint.py | 42 ++++ test_validation_heal_journal_fault.py | 232 ++++++++++++++++++++++- 7 files changed, 449 insertions(+), 20 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index a6d8bd71..dc188477 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1827,3 +1827,29 @@ verified the retained `38108127382` artifact for all three scopes. It exports no canonical fleet execution, individual fsync-hook, emitted barrier-log or physical power-loss claim. These required evidence inputs do not make the three incomplete canonical executors dispatchable. + +F07 stale-generation injection now consumes an immutable earlier checkpoint +captured from the actual candidate process, rather than synthesizing a record. +The capture brackets two stable exact-path reads with the original service +birth and retains the original bytes and run/candidate/disk/device/mount/task +binding in separate quarantine files. A later generation must prove strictly +greater durable progress in the same incarnation, set and JSON-typed options, +with truthful monotonic disposition counters and an original runnable cursor. +The bounded orchestration waits for this actual advance before freezing the +owner. Capture does not mutate product metadata and does not claim a live CAS +barrier; the independently collected native stale-manager test retains that +separate responsibility. + +Fault backups preserve the newer pre-fault generation. Rescue reinstalls that +newer record, retains the injected older bytes and original capture, and +refuses to overwrite any advanced or deleted product responsibility. Fleet +cleanup now restores registered journal faults before namespace deletion; +failed restoration marks cleanup FAIL and retains every private namespace. +Eleven additional regressions exercise real-file capture/advance/inject/rescue, +immutable evidence, strict JSON identities, process drift, missing or foreign +capture proof, counter/cursor drift and retained failed cleanup. The complete +local contract suite executes 484 tests with eight macOS platform skips. These +are execution prerequisites: F07's six branches on both H1 and H2, including +bounded write failure and full retained-responsibility diagnostics/rescan, +remain required. Automatic coverage remains 134/158 with 24 Heal executors +outstanding; no canonical fleet PASS is added by these changes. diff --git a/scripts/validation_heal/checkpoint.py b/scripts/validation_heal/checkpoint.py index 2946037f..7e0eebb8 100644 --- a/scripts/validation_heal/checkpoint.py +++ b/scripts/validation_heal/checkpoint.py @@ -111,12 +111,51 @@ def admit(probe, fleet, active): return token -def freeze(fleet, token, bucket, *, timeout=60): +def capture_earlier(fleet, token, bucket, identifier, *, timeout=60): + require(type(timeout) is int and 0 < timeout <= 120, "unreviewed checkpoint capture budget") + deadline = time.monotonic() + timeout + while True: + try: + owner = owned_checkpoint(records(fleet, token, "Admin-checkpoint-before-capture"), token, bucket) + identity = fleet.record("checkpoint-capture-process", fleet.remote(owner["node"], "process")) + receipt = fleet.record("Admin-checkpoint-original-capture", fleet.remote(owner["node"], "journal-capture", + family="checkpoint", taskId=token, disk=owner["disk"], relativePath=owner["checkpoint"]["relativePath"], + faultId=identifier, expectedBefore=journal.entry(bytes_of(owner["checkpoint"])), expected=identity)) + require(receipt.get("processBound") is True and receipt.get("originalProductBytes") is True + and receipt.get("productMetadataMutated") is False + and all(journal.same_json(receipt.get(name), value) for name, value in { + "runId": fleet.run_id, "family": "checkpoint", "taskId": token, "disk": owner["disk"], + "relativePath": owner["checkpoint"]["relativePath"], "faultId": identifier, + "binarySha256": identity.get("binarySha256"), + "captured": journal.entry(bytes_of(owner["checkpoint"]))}.items()), + "checkpoint capture lost its original process, bytes or exact physical owner") + return {**owner, "captureId": identifier, "captureReceipt": receipt} + except ValueError as exc: + if not str(exc).startswith("NOT_OBSERVED:"): raise + require(time.monotonic() < deadline, "NOT_OBSERVED: actual earlier checkpoint capture exceeded its frozen budget") + time.sleep(.1) + + +def after_generation(current, earlier): + require(all(journal.same_json(current.get(name), earlier.get(name)) + for name in ("taskId", "bucket", "bucketIncarnationId", "node", "disk")), + "checkpoint advanced on a different original task, incarnation or physical owner") + require(current["value"]["admin"]["outcome"]["counters"]["processed"] > + earlier["value"]["admin"]["outcome"]["counters"]["processed"], + "NOT_OBSERVED: original checkpoint has not durably advanced past its captured generation") + # The same admission, counters, cursor and digest checks precede actual mutation. + journal.transform(bytes_of(current["checkpoint"]), {"family": "checkpoint", "taskId": current["taskId"], + "relativePath": current["checkpoint"]["relativePath"], "kind": "stale-generation"}, + {"bucket": current["bucket"]}, None, earlier=bytes_of(earlier["checkpoint"])) + + +def freeze(fleet, token, bucket, *, timeout=60, earlier=None): require(type(timeout) is int and 0 < timeout <= 120, "unreviewed checkpoint observation budget") deadline = time.monotonic() + timeout while True: try: owner = owned_checkpoint(records(fleet, token, "Admin-checkpoint-live"), token, bucket) + if earlier is not None: after_generation(owner, earlier) break except ValueError as exc: if not str(exc).startswith("NOT_OBSERVED:"): raise @@ -133,6 +172,7 @@ def freeze(fleet, token, bucket, *, timeout=60): current = owned_checkpoint(records(fleet, token, "Admin-checkpoint-frozen"), token, bucket) require((current["node"], current["disk"], current["bucketIncarnationId"]) == (owner["node"], owner["disk"], owner["bucketIncarnationId"]), "checkpoint owner changed during freeze") + if earlier is not None: after_generation(current, earlier) return current, identity diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 97e713f4..3f0ddc55 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -976,18 +976,30 @@ def journal_operation(request): state, roots, owner = owned(request["runId"]) require(digest(state / "rustfs") == owner["binarySha256"], "durable-record candidate changed") require(owner.get("runtimeProfile") in {"root-restart", "mrf-crash"}, "durable records require the owned recovery runtime") - require(set(request) <= {"runId", "action", "family", "taskId", "disk", "relativePath", "faultId", "kind", "expectedBefore", "expected"}, + require(set(request) <= {"runId", "action", "family", "taskId", "disk", "relativePath", "faultId", "kind", "expectedBefore", "expected", "captureId"}, "durable-record adapter accepts no arbitrary system paths or commands") module, fault = globals()["journal_fault_backend"], fault_module() - if request["action"] == "journal-observe": - require(set(request) == {"runId", "action", "family", "taskId", "expected"}, - "live durable-record observation accepts only a process-bound exact family/task") + if request["action"] in {"journal-observe", "journal-capture"}: + fields = {"runId", "action", "family", "taskId", "expected"} + if request["action"] == "journal-capture": + fields |= {"disk", "relativePath", "faultId", "expectedBefore"} + require(set(request) == fields, "live durable-record observation/capture has unknown mutation or path inputs") before = process(request) expected = request.get("expected") require(isinstance(expected, dict) and before.get("pid", 0) > 1 and all(name in expected and before.get(name) == expected[name] for name in PROCESS_FIELDS), "live durable-record observation lost its exact service birth or candidate") observed = module.stable_census(roots, request, fault) + if request["action"] == "journal-capture": + require(request["family"] == "checkpoint" and type(request["disk"]) is int and 0 <= request["disk"] < 4, + "earlier checkpoint capture has no exact owned Admin record/disk") + selected = [row for row in observed["records"] + if (row["disk"], row["relativePath"]) == (request["disk"], request["relativePath"])] + require(len(selected) == 1 and selected[0].get("present") is True + and module.same_json({name: selected[0].get(name) for name in ("size", "sha256")}, request["expectedBefore"]), + "NOT_OBSERVED: earlier checkpoint differs from its two stable process-bound reads") + with fault.directory(roots[request["disk"]]) as fd: + observed = {**module.capture(fd, request, owner, fault), "observation": observed} after = process(request) require(all(after.get(name) == before.get(name) for name in PROCESS_FIELDS), "service restarted while observing exact durable records") @@ -1245,7 +1257,7 @@ def main(request): actions["task-records"] = task_records actions["environment-capabilities"] = environment_capabilities actions["quota-audit-finalize"] = quota_audit_finalize - for action in ("journal-census", "journal-observe", "journal-apply", "journal-recover"): + for action in ("journal-census", "journal-observe", "journal-capture", "journal-apply", "journal-recover"): actions[action] = journal_operation for action in ('generation-capture', 'generation-install', 'generation-recover'): actions[action] = generation_operation diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index d3fb0172..2dd365ed 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -153,6 +153,7 @@ def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_prof self.network_gates = set() self.generation_faults = [] self.orphan_faults = [] + self.journal_faults = [] self.secret_values = set() self.frozen_processes = {} self.snapshot_buckets = None @@ -320,6 +321,17 @@ def wait_ready(self): def cleanup(self): failures = [] + if self.journal_faults: + try: + self.stop() + for node, params in tuple(self.journal_faults): + receipt = self.record("durable-record-recovery", {"node": node, **self.remote(node, "journal-recover", **params)}) + require(receipt.get("backupRetained") is True and receipt.get("originalResponsibilityRecreated") is False + and receipt.get("productRepairClaimed") is False, + "durable-record rescue lost the original evidence or claims product repair") + self.journal_faults.remove((node, params)) + except Exception as exc: + failures.append({"error": str(exc), "state": "DIRTY", "namespacesRetained": True}) if self.orphan_faults: try: self.stop() diff --git a/scripts/validation_heal/journal_fault.py b/scripts/validation_heal/journal_fault.py index 933a1080..e64a4589 100644 --- a/scripts/validation_heal/journal_fault.py +++ b/scripts/validation_heal/journal_fault.py @@ -43,6 +43,13 @@ def entry(raw): return {"size": len(raw), "sha256": hashlib.sha256(raw).hexdigest()} +def same_json(left, right): + # JSON scope identities distinguish booleans, integers and floating-point values. + def canonical(value): + return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + return canonical(left) == canonical(right) + + def read(root_fd, path, fault): with fault.parent(root_fd, ["data", *path[:-1]]) as directory: return fault.read_file(directory, path[-1], LIMIT * (4 if "lifecycle" in path[-1] else 1)) @@ -86,7 +93,7 @@ def invalid_constant(value): def checkpoint_value(raw, token, bucket): value = decode_json(raw) - require(value.get("task_id") == task_id(token) and value.get("schema_version") == 8, + require(value.get("task_id") == task_id(token) and type(value.get("schema_version")) is int and value["schema_version"] == 8, "checkpoint does not belong to the exact original Admin task") admin = value.get("admin") or {} buckets = admin.get("buckets") @@ -107,14 +114,45 @@ def checkpoint_value(raw, token, bucket): return value -def transform(raw, request, owner, mrf): +def transform(raw, request, owner, mrf, *, earlier=None): family, kind = request["family"], request.get("kind") - require(kind in {"truncate", "future-schema", "checksum"}, "unknown durable-record byte fault") + require(kind in {"truncate", "future-schema", "checksum", "stale-generation"}, "unknown durable-record byte fault") path = relative(request) if family == "checkpoint": require(path[-1].endswith("_ahm_checkpoint.json"), "only the exact validated Admin checkpoint may be corrupted") value = checkpoint_value(raw, request["taskId"], owner["bucket"]) - require(kind in {"truncate", "future-schema"}, "checkpoint checksum is independently collected by native guards") + require(kind in {"truncate", "future-schema", "stale-generation"}, "checkpoint checksum is independently collected by native guards") + if kind == "stale-generation": + require(isinstance(earlier, bytes), "stale checkpoint requires an original captured earlier record") + old = checkpoint_value(earlier, request["taskId"], owner["bucket"]) + require(isinstance(value["admin"].get("options"), dict) and value["admin"].get("set_disk_id") == "pool_0_set_0" + and all(same_json(old["admin"].get(name), value["admin"].get(name)) + for name in ("buckets", "options", "set_disk_id")), + "earlier checkpoint belongs to a different incarnation, set or admitted options") + before = old["admin"].get("outcome", {}).get("counters", {}) + after = value["admin"].get("outcome", {}).get("counters", {}) + fields = ("processed", "healed", "unchanged", "skipped", "failed", "unknown", "attemptFailures") + require(all(type(before.get(name)) is int and type(after.get(name)) is int + and 0 <= before[name] <= after[name] < 2**64 for name in fields) + and 0 < before["processed"] < after["processed"] + and before.get("overflowed") is False and after.get("overflowed") is False, + "captured checkpoint is not an earlier proven durable progress generation") + for checkpoint in (old, value): + require(type(checkpoint.get("checkpoint_time")) is int and 0 <= checkpoint["checkpoint_time"] < 2**64 + and all(type(checkpoint.get(name)) is int and 0 <= checkpoint[name] < 2**64 + for name in ("current_bucket_index", "current_object_index")) + and checkpoint["admin"].get("outcome", {}).get("execution") == {"state": "running"} + and checkpoint["admin"]["outcome"].get("coverage") == "partial", + "stale-generation fault has no original runnable cursor/time") + counters = checkpoint["admin"]["outcome"]["counters"] + require(counters["processed"] == sum(counters[name] for name in ("healed", "unchanged", "skipped", "failed")) + and counters["unknown"] <= counters["skipped"] and counters["attemptFailures"] >= counters["failed"], + "stale checkpoint counters do not account for original durable dispositions") + require(old["checkpoint_time"] <= value["checkpoint_time"] + and (old["current_bucket_index"], old["current_object_index"]) <= + (value["current_bucket_index"], value["current_object_index"]), + "earlier checkpoint has a future or incomparable scan position") + return earlier elif family == "root": require(path[-1] == "root-heal-" + task_id(request["taskId"]) + ".json" and kind == "truncate", "only the exact runnable root admission may be truncated") @@ -155,6 +193,37 @@ def binding(root_fd, request, owner, fault): "mountId": fault.mount_id(root_fd), "productRepairClaimed": False} +def capture(root_fd, request, owner, fault): + require(request.get("family") == "checkpoint", "only an actual Admin checkpoint has an earlier progress capture") + identity = binding(root_fd, request, owner, fault) + raw = read(root_fd, relative(request), fault) + checkpoint_value(raw, request["taskId"], owner["bucket"]) + require(same_json(request.get("expectedBefore"), entry(raw)), "NOT_OBSERVED: checkpoint changed before its earlier capture") + proof = {**identity, "captured": entry(raw), "originalProductBytes": True, "productMetadataMutated": False} + with fault.parent(root_fd, ["quarantine"]) as backup: + # Capture identity and bytes are immutable and distinct from fault backups. + fault.new_file(backup, identity["faultId"] + ".early.intent.json", fault.encode(proof)) + fault.new_file(backup, identity["faultId"] + ".early.checkpoint", raw) + return proof + + +def captured(root_fd, request, owner, fault): + identity = binding(root_fd, request, owner, fault) + identifier = request.get("captureId") + require(isinstance(identifier, str) and re.fullmatch(r"F07-[a-z0-9-]{1,48}", identifier) + and identifier != identity["faultId"] and request["family"] == "checkpoint", + "stale-generation capture has no separate original snapshot identity") + with fault.parent(root_fd, ["quarantine"]) as backup: + proof = decode_json(fault.read_file(backup, identifier + ".early.intent.json", 16384)) + require(all(same_json(proof.get(name), value) for name, value in {**identity, "faultId": identifier}.items()) + and proof.get("originalProductBytes") is True and proof.get("productMetadataMutated") is False, + "earlier checkpoint capture changed task, run, candidate, physical owner or record identity") + raw = fault.read_file(backup, identifier + ".early.checkpoint", LIMIT) + require(same_json(entry(raw), proof.get("captured")), "captured earlier product checkpoint bytes changed") + checkpoint_value(raw, request["taskId"], owner["bucket"]) + return raw + + def replace(root_fd, path, raw, identifier, fault, *, rescue=False): with fault.parent(root_fd, ["data", *path[:-1]]) as directory: temporary = ".F07-" + identifier + (".rescue" if rescue else ".inject") @@ -171,9 +240,13 @@ def install(root_fd, request, owner, fault, mrf): identity = binding(root_fd, request, owner, fault) path = relative(request) before = read(root_fd, path, fault) - require(request.get("expectedBefore") == entry(before), "journal bytes drifted from their archived pre-fault identity") - injected = transform(before, request, owner, mrf) + require(same_json(request.get("expectedBefore"), entry(before)), "journal bytes drifted from their archived pre-fault identity") + earlier = captured(root_fd, request, owner, fault) if request.get("kind") == "stale-generation" else None + require(earlier is not None or "captureId" not in request, "only a stale-generation fault consumes an earlier checkpoint") + injected = transform(before, request, owner, mrf, earlier=earlier) intent = {**identity, "kind": request["kind"], "before": entry(before), "injected": entry(injected)} + if earlier is not None: + intent.update(captureId=request["captureId"], capturedGeneration=entry(earlier)) with fault.parent(root_fd, ["quarantine"]) as backup: fault.new_file(backup, identity["faultId"] + ".original", before) fault.new_file(backup, identity["faultId"] + ".intent.json", fault.encode(intent)) @@ -187,10 +260,14 @@ def install(root_fd, request, owner, fault, mrf): def recover(root_fd, request, owner, fault): identity = binding(root_fd, request, owner, fault) with fault.parent(root_fd, ["quarantine"]) as backup: - intent = json.loads(fault.read_file(backup, identity["faultId"] + ".intent.json", 16384)) - require(all(intent.get(key) == value for key, value in identity.items()), "journal recovery lost original run/candidate/device/record identity") + intent = decode_json(fault.read_file(backup, identity["faultId"] + ".intent.json", 16384)) + require(all(same_json(intent.get(key), value) for key, value in identity.items()), "journal recovery lost original run/candidate/device/record identity") before = fault.read_file(backup, identity["faultId"] + ".original", LIMIT * 4) require(entry(before) == intent["before"], "journal original backup is corrupt") + if intent["kind"] == "stale-generation": + require(intent.get("captureId") == request.get("captureId"), "stale recovery changed its original earlier snapshot") + require(entry(captured(root_fd, request, owner, fault)) == intent.get("capturedGeneration"), + "stale recovery lost original earlier generation evidence") path = relative(request) current = read(root_fd, path, fault) require(entry(current) in (intent["before"], intent["injected"]), diff --git a/test_validation_heal_checkpoint.py b/test_validation_heal_checkpoint.py index d5bf790d..697ed214 100644 --- a/test_validation_heal_checkpoint.py +++ b/test_validation_heal_checkpoint.py @@ -109,5 +109,47 @@ def test_set_admission_keeps_explicit_selectors_and_actual_canonical_token(self) self.assertEqual(calls, [(reader.PATH, [], reader.OPTIONS)]) self.assertEqual(active, [(reader.PATH, TOKEN)]) + def test_earlier_capture_requires_original_bytes_bound_to_process_and_physical_owner(self): + observed, _, _ = self.fixture() + owner = reader.owned_checkpoint(observed, TOKEN, BUCKET) + identity = {"binarySha256": "a" * 64} + receipt = {"processBound": True, "originalProductBytes": True, "productMetadataMutated": False, + "runId": "gh-12345-1-h1", "family": "checkpoint", "taskId": TOKEN, "disk": 1, + "relativePath": owner["checkpoint"]["relativePath"], "faultId": "F07-early", "binarySha256": "a" * 64, + "captured": journal.entry(reader.bytes_of(owner["checkpoint"]))} + requests = [] + fleet = SimpleNamespace(run_id=receipt["runId"], record=lambda label, value: value, + remote=lambda node, action, **params: requests.append((node, action, params)) or + (identity if action == "process" else receipt)) + with patch.object(reader, "records", return_value=observed): + result = reader.capture_earlier(fleet, TOKEN, BUCKET, "F07-early") + self.assertEqual(result["captureId"], "F07-early") + self.assertEqual(requests[-1], ("node", "journal-capture", { + "family": "checkpoint", "taskId": TOKEN, "disk": 1, "relativePath": receipt["relativePath"], + "faultId": "F07-early", "expectedBefore": receipt["captured"], "expected": identity})) + for key, value in [("processBound", False), ("productMetadataMutated", True), ("disk", True), + ("binarySha256", "b" * 64), ("captured", {"size": 3, "sha256": "a" * 64})]: + original = receipt[key]; receipt[key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + reader.capture_earlier(fleet, TOKEN, BUCKET, "F07-early") + receipt[key] = original + + def test_stale_boundary_requires_actual_newer_progress_in_the_same_original_responsibility(self): + observed, value, _ = self.fixture() + earlier = reader.owned_checkpoint(observed, TOKEN, BUCKET) + with self.assertRaisesRegex(ValueError, "NOT_OBSERVED"): reader.after_generation(earlier, earlier) + value["checkpoint_time"] += 1; value["current_object_index"] += 1 + value["admin"]["outcome"]["counters"]["processed"] += 1 + value["admin"]["outcome"]["counters"]["unchanged"] += 1 + observed["node"]["records"][4].update(encoded(value, signed=True)) + current = reader.owned_checkpoint(observed, TOKEN, BUCKET) + reader.after_generation(current, earlier) + for key, new in [("disk", 2), ("node", "peer"), ("taskId", INCARNATION), ("bucketIncarnationId", TOKEN)]: + with self.subTest(key=key), self.assertRaises(ValueError): reader.after_generation({**current, key: new}, earlier) + changed = copy.deepcopy(current) + changed["value"]["admin"]["options"]["pool"] = False + changed["checkpoint"].update(encoded(changed["value"], signed=True)) + with self.assertRaises(ValueError): reader.after_generation(changed, earlier) + if __name__ == "__main__": unittest.main() diff --git a/test_validation_heal_journal_fault.py b/test_validation_heal_journal_fault.py index d95aac35..bf05b0ea 100644 --- a/test_validation_heal_journal_fault.py +++ b/test_validation_heal_journal_fault.py @@ -1,6 +1,7 @@ """Exact durable-record corruption preserves ownership and refuses stale rescue.""" import base64 +import copy import hashlib import json import os @@ -16,6 +17,7 @@ from scripts.validation_heal import journal_fault as journal, heal_fault as fault, mrf_records as mrf from scripts.validation_heal.healthy import Fleet from scripts.validation_heal import fleet_node +from scripts.validation_heal.checkpoint import OPTIONS class DurableRecordFaultTests(unittest.TestCase): @@ -23,16 +25,39 @@ class DurableRecordFaultTests(unittest.TestCase): incarnation = "22345678-1234-4234-8234-123456789abc" bucket = "rv-heal-0123456789abcdef" - def checkpoint(self): - value = {"task_id": self.token, "schema_version": 8, - "admin": {"completed": False, "buckets": {self.bucket: self.incarnation}}, - "processed_objects": ["objects/second", "objects/first"], "failed_objects": [], "skipped_objects": [], - "integrity_digest": None, "current_object_index": 2, "successful_objects": 2} - unsigned = {**value, "processed_objects": sorted(value["processed_objects"])} + def signed(self, value): + value = copy.deepcopy(value) + value["integrity_digest"] = None + unsigned = copy.deepcopy(value) + for name in ("processed_objects", "failed_objects", "skipped_objects"): unsigned[name].sort() value["integrity_digest"] = base64.b64encode(hashlib.sha256(json.dumps(unsigned, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode()).digest()).decode() return json.dumps(value).encode() + def checkpoint(self, processed=2, time=100): + return self.signed({"task_id": self.token, "schema_version": 8, "checkpoint_time": time, + "current_bucket_index": 0, "current_object_index": processed, + "admin": {"completed": False, "buckets": {self.bucket: self.incarnation}, "options": OPTIONS, + "set_disk_id": "pool_0_set_0", "outcome": {"execution": {"state": "running"}, "coverage": "partial", + "objects": [], "objectsTruncated": False, "counters": {"processed": processed, "healed": 1, + "unchanged": processed - 1, "skipped": 0, "failed": 0, "unknown": 0, + "attemptFailures": 0, "overflowed": False}}}, + "processed_objects": [f"objects/{i}" for i in reversed(range(processed))], + "failed_objects": [], "skipped_objects": [], "successful_objects": processed, + "failed_object_count": 0, "skipped_object_count": 0, "skipped_new_versions": 0, "skipped_ilm_expired": 0, + "processed_bytes": processed, "total_objects": 8, "total_bytes": 8, + "baseline_generation": None, "baseline_known": False, "counter_unknown": False}) + + def stale_fixture(self, root): + owner, request, path, early = self.fixture(root, kind="stale-generation") + capture_request = {key: value for key, value in request.items() if key != "kind"} + capture_request["faultId"] = "F07-early" + with fault.directory(root) as fd: journal.capture(fd, capture_request, owner, fault) + newer = self.checkpoint(processed=3, time=101) + path.write_bytes(newer) + request.update(expectedBefore=journal.entry(newer), captureId=capture_request["faultId"]) + return owner, request, path, early, newer + def fixture(self, root, kind="truncate", family="checkpoint"): (root / "data/.rustfs.sys/buckets").mkdir(parents=True) (root / "quarantine").mkdir() @@ -65,6 +90,120 @@ def test_real_file_fault_and_idempotent_rescue_retain_original_and_failed_bytes( self.assertEqual((root / "quarantine/F07-checkpoint-0.original").read_bytes(), raw) self.assertEqual(len(list((root / "quarantine").glob("*.failed-*"))), 1) + def test_stale_injection_uses_original_capture_and_rescue_restores_newer_pre_fault_bytes(self): + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, early, newer = self.stale_fixture(root) + with fault.directory(root) as fd: + result = journal.install(fd, request, owner, fault, mrf) + self.assertEqual(path.read_bytes(), early) + self.assertEqual(result["capturedGeneration"], journal.entry(early)) + self.assertEqual(result["before"], journal.entry(newer)) + self.assertEqual((root / "quarantine/F07-checkpoint-0.original").read_bytes(), newer) + self.assertTrue(journal.recover(fd, request, owner, fault)["restoredFromBackup"]) + self.assertEqual(path.read_bytes(), newer) + self.assertFalse(journal.recover(fd, request, owner, fault)["restoredFromBackup"]) + self.assertEqual((root / "quarantine/F07-early.early.checkpoint").read_bytes(), early) + self.assertEqual(next((root / "quarantine").glob("*.failed-*")).read_bytes(), early) + self.assertEqual((root / "data/.rustfs.sys/unrelated.json").read_bytes(), b"untouched control") + + def test_stale_generation_rejects_valid_digests_with_different_scopes_or_unproven_progress(self): + owner = {"bucket": self.bucket} + request = {"family": "checkpoint", "taskId": self.token, "kind": "stale-generation", + "relativePath": journal.names("checkpoint", self.token)[0]} + current = self.checkpoint(processed=3, time=101) + for change in ("task", "incarnation", "options", "option-type", "set", "equal", "newer", "zero", + "counter-type", "regressed-counter", "dishonest-counter", "overflow", "time", "time-type", + "cursor", "cursor-type", "completed", "coverage", "schema-type"): + old = json.loads(self.checkpoint()) + counters = old["admin"]["outcome"]["counters"] + if change == "task": old["task_id"] = str(uuid.uuid4()) + elif change == "incarnation": old["admin"]["buckets"][self.bucket] = str(uuid.uuid4()) + elif change == "options": old["admin"]["options"]["nolock"] = True + elif change == "option-type": old["admin"]["options"]["pool"] = False + elif change == "set": old["admin"]["set_disk_id"] = "pool_0_set_1" + elif change == "equal": old = json.loads(current) + elif change == "newer": old = json.loads(self.checkpoint(processed=4, time=102)) + elif change == "zero": counters.update(processed=0, healed=0, unchanged=0) + elif change == "counter-type": counters["healed"] = True + elif change == "regressed-counter": counters.update(healed=2, unchanged=0) + elif change == "dishonest-counter": counters["unchanged"] = 0 + elif change == "overflow": counters["overflowed"] = True + elif change == "time": old["checkpoint_time"] = 102 + elif change == "time-type": old["checkpoint_time"] = True + elif change == "cursor": old["current_object_index"] = 4 + elif change == "cursor-type": old["current_object_index"] = 2.0 + elif change == "completed": old["admin"]["completed"] = True + elif change == "coverage": old["admin"]["outcome"]["coverage"] = "complete" + else: old["schema_version"] = 8.0 + with self.subTest(change=change), self.assertRaises(ValueError): + journal.transform(current, request, owner, mrf, earlier=self.signed(old)) + self.assertEqual(journal.transform(current, request, owner, mrf, earlier=self.checkpoint()), self.checkpoint()) + + def test_missing_corrupt_or_foreign_capture_never_modifies_current_checkpoint(self): + for change in ("missing-proof", "missing-bytes", "proof", "bytes", "candidate", "device", "mount", + "disk", "disk-type", "run", "task", "path", "capture-id", "same-id", "entry-type", "duplicate-key"): + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, _, newer = self.stale_fixture(root) + proof_path = root / "quarantine/F07-early.early.intent.json" + bytes_path = root / "quarantine/F07-early.early.checkpoint" + proof = json.loads(proof_path.read_text()) + if change == "missing-proof": proof_path.unlink() + elif change == "missing-bytes": bytes_path.unlink() + elif change == "proof": proof_path.write_bytes(b'{"invalid":true}') + elif change == "bytes": bytes_path.write_bytes(b"changed product bytes") + elif change == "capture-id": request["captureId"] = "F07-foreign" + elif change == "same-id": request["captureId"] = request["faultId"] + elif change == "duplicate-key": proof_path.write_bytes(proof_path.read_bytes()[:-1] + b',"disk":0}') + else: + if change == "candidate": proof["binarySha256"] = "b" * 64 + elif change == "device": proof["device"] = -1 + elif change == "mount": proof["mountId"] = 2 + elif change == "disk": proof["disk"] = 1 + elif change == "disk-type": proof["disk"] = False + elif change == "run": proof["runId"] = "gh-other-1-h1" + elif change == "task": proof["taskId"] = str(uuid.uuid4()) + elif change == "path": proof["relativePath"] = ".rustfs.sys/foreign" + else: proof["captured"]["size"] = float(proof["captured"]["size"]) + proof_path.write_text(json.dumps(proof)) + with fault.directory(root) as fd, self.subTest(change=change), self.assertRaises((ValueError, FileNotFoundError)): + journal.install(fd, request, owner, fault, mrf) + self.assertEqual(path.read_bytes(), newer) + self.assertFalse((root / "quarantine/F07-checkpoint-0.original").exists()) + + def test_capture_is_immutable_and_rejects_drift_without_touching_product_bytes(self): + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, raw = self.fixture(root) + request.pop("kind") + with fault.directory(root) as fd: + with self.assertRaisesRegex(ValueError, "NOT_OBSERVED"): + journal.capture(fd, {**request, "expectedBefore": journal.entry(raw + b" ")}, owner, fault) + self.assertEqual(list((root / "quarantine").iterdir()), []) + captured = journal.capture(fd, request, owner, fault) + self.assertTrue(captured["originalProductBytes"]) + self.assertFalse(captured["productMetadataMutated"]) + with self.assertRaises(FileExistsError): journal.capture(fd, request, owner, fault) + self.assertEqual(path.read_bytes(), raw) + + def test_stale_rescue_rejects_advanced_deleted_checkpoint_or_lost_capture_identity(self): + for change in ("advance", "delete", "capture-id", "capture-bytes", "capture-proof", "device-type"): + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, _, _ = self.stale_fixture(root) + with fault.directory(root) as fd: + journal.install(fd, request, owner, fault, mrf) + if change == "advance": path.write_bytes(self.checkpoint(processed=4, time=102)) + elif change == "delete": path.unlink() + elif change == "capture-id": request["captureId"] = "F07-other" + elif change == "capture-bytes": (root / "quarantine/F07-early.early.checkpoint").write_bytes(b"changed") + elif change == "capture-proof": (root / "quarantine/F07-early.early.intent.json").unlink() + else: + p = root / "quarantine/F07-checkpoint-0.intent.json" + intent = json.loads(p.read_text()); intent["device"] = float(intent["device"]) + p.write_text(json.dumps(intent)) + current = path.read_bytes() if path.exists() else None + with self.subTest(change=change), self.assertRaises((ValueError, FileNotFoundError)): + journal.recover(fd, request, owner, fault) + self.assertEqual(path.read_bytes() if path.exists() else None, current) + def test_root_fault_requires_exact_bucket_admission_and_preserves_terminal_names(self): with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): root = Path(temp).resolve(); owner, request, path, raw = self.fixture(root, family="root") @@ -218,6 +357,48 @@ def test_live_adapter_binds_both_reads_to_original_process_and_cannot_mutate(sel with patch.object(fleet_node, "process", return_value=identity), self.assertRaises(ValueError): fleet_node.journal_operation({**request, "kind": "truncate"}) + def test_live_capture_requires_the_same_stable_record_and_service_birth(self): + for changed in (None, "record", "process"): + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, raw = self.fixture(root) + identity = {name: name for name in fleet_node.PROCESS_FIELDS} + identity.update(pid=50, state="S") + request.pop("kind"); request.update(action="journal-capture", expected=identity) + if changed == "record": request["expectedBefore"] = journal.entry(raw + b" ") + successor = {**identity, "startTicks": 999} if changed == "process" else identity + with patch.object(fleet_node, "owned", return_value=(root, [root], owner)), \ + patch.object(fleet_node, "digest", return_value=owner["binarySha256"]), \ + patch.object(fleet_node, "fault_module", return_value=fault), \ + patch.dict(fleet_node.__dict__, journal_fault_backend=journal), \ + patch.object(fleet_node, "stopped", side_effect=AssertionError("capture must not stop service")), \ + patch.object(fleet_node, "process", side_effect=[identity, successor]): + if changed: + with self.subTest(changed=changed), self.assertRaises(ValueError): fleet_node.journal_operation(request) + else: + result = fleet_node.journal_operation(request) + self.assertTrue(result["processBound"]) + self.assertFalse(result["serviceFrozen"]) + self.assertEqual(result["observation"]["independentReads"], 2) + self.assertEqual(result["captured"], journal.entry(raw)) + self.assertEqual(path.read_bytes(), raw) + captured = root / "quarantine/F07-checkpoint-0.early.checkpoint" + self.assertEqual(captured.exists(), changed != "record") + + def test_non_stale_fault_cannot_consume_a_capture_and_capture_cannot_mutate_records(self): + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, raw = self.fixture(root) + with fault.directory(root) as fd, self.assertRaises(ValueError): + journal.install(fd, {**request, "captureId": "F07-early"}, owner, fault, mrf) + identity = {name: name for name in fleet_node.PROCESS_FIELDS}; identity.update(pid=50, state="S") + capture = {**request, "action": "journal-capture", "expected": identity} + with patch.object(fleet_node, "owned", return_value=(root, [root], owner)), \ + patch.object(fleet_node, "digest", return_value=owner["binarySha256"]), \ + patch.object(fleet_node, "fault_module", return_value=fault), \ + patch.dict(fleet_node.__dict__, journal_fault_backend=journal), self.assertRaises(ValueError): + fleet_node.journal_operation(capture) + self.assertEqual(path.read_bytes(), raw) + self.assertEqual(list((root / "quarantine").iterdir()), []) + def test_remote_action_loads_exact_record_and_fault_dependencies(self): with tempfile.TemporaryDirectory() as temp, patch("scripts.validation_heal.healthy.subprocess.run", return_value=types.SimpleNamespace(returncode=0, stdout='{}', stderr='')) as command: @@ -229,5 +410,44 @@ def test_remote_action_loads_exact_record_and_fault_dependencies(self): self.assertTrue(callable(scope["fault_module"]().read_file)) self.assertTrue(callable(scope["mrf"].checkpoint)) + def test_fleet_cleanup_restores_newer_original_before_deleting_private_namespace(self): + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, _, newer = self.stale_fixture(root) + fleet = Fleet(["node"], root, owner["runId"]); fleet.reserved = ["node"] + fleet.journal_faults.append(("node", request)) + with fault.directory(root) as fd: journal.install(fd, request, owner, fault, mrf) + actions = [] + def remote(node, action, **params): + actions.append(action) + if action == "journal-recover": + with fault.directory(root) as fd: return journal.recover(fd, params, owner, fault) + if action == "cleanup": self.assertEqual(path.read_bytes(), newer) + return {} + with patch.object(fleet, "remote", side_effect=remote): self.assertTrue(fleet.cleanup()) + self.assertLess(actions.index("stop"), actions.index("journal-recover")) + self.assertLess(actions.index("journal-recover"), actions.index("cleanup")) + self.assertEqual(fleet.journal_faults, []) + + def test_fleet_cleanup_keeps_every_namespace_and_pending_fault_when_product_advanced(self): + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, path, _, _ = self.stale_fixture(root) + fleet = Fleet(["node", "peer"], root, owner["runId"]); fleet.reserved = ["node", "peer"] + fleet.journal_faults.append(("node", request)) + with fault.directory(root) as fd: journal.install(fd, request, owner, fault, mrf) + advanced = self.checkpoint(processed=4, time=102); path.write_bytes(advanced) + actions = [] + def remote(node, action, **params): + actions.append(action) + if action == "journal-recover": + with fault.directory(root) as fd: return journal.recover(fd, params, owner, fault) + return {} + with patch.object(fleet, "remote", side_effect=remote): self.assertFalse(fleet.cleanup()) + self.assertNotIn("cleanup", actions) + self.assertEqual(path.read_bytes(), advanced) + self.assertEqual(fleet.journal_faults, [("node", request)]) + recovery = json.loads(next(root.glob("*-recovery.json")).read_text()) + self.assertEqual(recovery["cleanup"], "FAIL") + self.assertTrue(recovery["errors"][0]["namespacesRetained"]) + if __name__ == "__main__": unittest.main() From 96700210c3e645d7320b57920e7072e5b9a47911 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 12:19:52 +0800 Subject: [PATCH 125/343] test(validation): bind journal failures to private bounded volumes --- docs/validation-migration-map.md | 15 +++++++ scripts/validation_heal/bounded_volume.py | 10 +++-- scripts/validation_heal/fleet_node.py | 29 +++++++------ scripts/validation_heal/healthy.py | 10 +++-- scripts/validation_heal/journal_fault.py | 4 +- test_validation_heal_capacity.py | 49 +++++++++++++++++++++ test_validation_heal_journal_fault.py | 52 +++++++++++++++++++++++ 7 files changed, 150 insertions(+), 19 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index dc188477..21a9753b 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1853,3 +1853,18 @@ are execution prerequisites: F07's six branches on both H1 and H2, including bounded write failure and full retained-responsibility diagnostics/rescan, remain required. Automatic coverage remains 134/158 with 24 Heal executors outstanding; no canonical fleet PASS is added by these changes. + +F07 also has a dedicated `journal-bounded` runtime using the existing +fixed 128-MiB-per-drive filesystem backend. Only fixed block exhaustion is +accepted for this profile; caller-supplied limits, arbitrary paths, inode +faults, root admissions and MRF payload mutations are rejected. The candidate +receives verified private service bindings on the same four original physical +drives, with scanner/MRF/automatic repair disabled. Exact checkpoint reads use +those actual bounded views, and capacity witnesses retain the original process +birth, namespace, UID, device and precise ENOSPC result. Filler is released +before registered journal rescue; failed release retains all namespaces. +Six new boundary regressions pass. The complete local suite now executes +490 tests with eight macOS skips. Linux `38110756752` independently confirms +the preceding captured-generation implementation executed all 484 tests +without skips and all six checks passed. No bounded-journal live execution or +complete F07 executor is claimed yet. diff --git a/scripts/validation_heal/bounded_volume.py b/scripts/validation_heal/bounded_volume.py index ded9e705..1a23b353 100644 --- a/scripts/validation_heal/bounded_volume.py +++ b/scripts/validation_heal/bounded_volume.py @@ -69,6 +69,8 @@ def read(state, roots, owner): and value.get("imageBytes") == IMAGE_BYTES and value.get("maxInodes") == MAX_INODES and len(value.get("volumes", [])) == len(roots) == 4, "bounded-volume contract differs from its original run") + require(owner.get("runtimeProfile") != "journal-bounded" or value["branch"] == "bounded-volume-enospc", + "bounded journal runtime has no reviewed inode-exhaustion branch") for index, row in enumerate(value["volumes"]): require(type(row.get("disk")) is int and row["disk"] == index and row.get("image") == str(roots[index] / "bounded-volume.img") and row.get("mount") == str(state / "bounded" / f"disk{index}") @@ -108,8 +110,10 @@ def mounted(row): def create(state, roots, owner, branch): - require(owner.get("runtimeProfile") == "bounded-capacity" and read(state, roots, owner) is None + require(owner.get("runtimeProfile") in {"bounded-capacity", "journal-bounded"} and read(state, roots, owner) is None and branch in {"bounded-volume-enospc", "bounded-inode-exhaustion"}, "invalid bounded capacity allocation") + require(owner.get("runtimeProfile") != "journal-bounded" or branch == "bounded-volume-enospc", + "bounded journal allocation accepts only fixed block exhaustion") require(all(not any((root / "data").iterdir()) and not (root / "bounded-volume.img").exists() and shutil.disk_usage(root).free >= MIN_SHARED_FREE + IMAGE_BYTES for root in roots), "bounded allocation requires empty owned roots and shared-drive safety margin") @@ -155,7 +159,7 @@ def create(state, roots, owner, branch): def views(state, roots, owner): value = read(state, roots, owner) if value is None: - require(owner.get("runtimeProfile") != "bounded-capacity", "bounded runtime has no durable allocation intent") + require(owner.get("runtimeProfile") not in {"bounded-capacity", "journal-bounded"}, "bounded runtime has no durable allocation intent") return roots result = [] for original, row in zip(roots, value["volumes"], strict=True): @@ -177,7 +181,7 @@ def views(state, roots, owner): def properties(state, roots, owner): if read(state, roots, owner) is None: - require(owner.get("runtimeProfile") != "bounded-capacity", "bounded runtime is not configured") + require(owner.get("runtimeProfile") not in {"bounded-capacity", "journal-bounded"}, "bounded runtime is not configured") return [] actual = views(state, roots, owner) return ["--property=PrivateMounts=true", "--property=BindPaths=" + " ".join( diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 3f0ddc55..4975e677 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -33,7 +33,7 @@ def runtime_config(profile): - require(profile in {"queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(profile in {"queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") if profile == "queue-pressure": logging_filter = "info,rustfs::heal::manager=debug,rustfs::heal::task=debug,rustfs::heal::mrf=debug" return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true", @@ -49,8 +49,10 @@ def runtime_config(profile): return {**BACKGROUND_CONFIG, "RUSTFS_STORAGE_CLASS_STANDARD": "EC:1"} if profile == "multi-pool": return {**BACKGROUND_CONFIG, "RUSTFS_ERASURE_SET_DRIVE_COUNT": "4", "RUSTFS_STORAGE_CLASS_STANDARD": "EC:2"} - if profile == "bounded-capacity": - return {**BACKGROUND_CONFIG, "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true"} + if profile in {"bounded-capacity", "journal-bounded"}: + return {**BACKGROUND_CONFIG, "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true", + **({"RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} + if profile == "journal-bounded" else {})} if profile == "lock-quorum": return {**BACKGROUND_CONFIG, "RUSTFS_STORAGE_CLASS_STANDARD": "EC:8"} if profile == "scope-isolation": @@ -305,12 +307,12 @@ def start(request): "--property=MemoryMax=4G", *(["--property=PrivateMounts=true", "--property=CapabilityBoundingSet=~CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH"] if owner.get("runtimeProfile") in {"flapping", "queue-pressure"} else []), - *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") in {"root-restart", "durable-mrf", "mrf-crash", "replacement", "replacement-target", "schema-refusal"} else []), + *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") in {"root-restart", "journal-bounded", "durable-mrf", "mrf-crash", "replacement", "replacement-target", "schema-refusal"} else []), "--property=WorkingDirectory=" + str(state), "--property=EnvironmentFile=" + str(state / "service.env"), *gate_module().start_properties(gate_module().checked_gate(state, roots)), *quorum_gate_module().properties(state, roots), - *(volume_module().properties(state, roots, owner) if owner.get("runtimeProfile") == "bounded-capacity" else []), + *(volume_module().properties(state, roots, owner) if owner.get("runtimeProfile") in {"bounded-capacity", "journal-bounded"} else []), *replacement_module().start_properties(state, roots), str(state / "rustfs"), *owner["volumes"]) return {"unit": owner["unit"], "started": True} @@ -430,7 +432,7 @@ def process_signal(request): def process_graceful(request): - require(owned(request["runId"])[2].get("runtimeProfile") in {"root-restart", "replacement", "replacement-target"} + require(owned(request["runId"])[2].get("runtimeProfile") in {"root-restart", "journal-bounded", "replacement", "replacement-target"} and process(request).get("state") == "T", "graceful observation requires the paused run-owned root service") before = pidfd_signal(request, signal.SIGTERM) # A paused process must receive CONT to handle TERM. Use the same pidfd @@ -454,7 +456,7 @@ def process_graceful(request): def process_crash(request): - require(owned(request["runId"])[2].get("runtimeProfile") in {"replacement", "replacement-target", "root-restart", "durable-mrf", "mrf-crash"} + require(owned(request["runId"])[2].get("runtimeProfile") in {"replacement", "replacement-target", "root-restart", "journal-bounded", "durable-mrf", "mrf-crash"} and process(request).get("state") == "T", "crash observation requires a paused owned recovery service") before = pidfd_signal(request, signal.SIGKILL) deadline = time.monotonic() + 30 @@ -507,12 +509,12 @@ def volume_module(): def volume_views(state, roots, owner): - return volume_module().views(state, roots, owner) if owner.get("runtimeProfile") == "bounded-capacity" else roots + return volume_module().views(state, roots, owner) if owner.get("runtimeProfile") in {"bounded-capacity", "journal-bounded"} else roots def volume_operation(request): state, roots, owner = owned(request["runId"]) - require(owner.get("runtimeProfile") == "bounded-capacity", "bounded operation requires its exact dedicated runtime") + require(owner.get("runtimeProfile") in {"bounded-capacity", "journal-bounded"}, "bounded operation requires its exact dedicated runtime") module = volume_module() if request["action"] == "bounded-volumes-create": require(set(request) == {"runId", "action", "branch"}, "bounded allocation accepts no supplied path or limits") @@ -887,7 +889,7 @@ def snapshot(request): "resourceLimits": limits, "files": sorted(files, key=lambda value: (value["disk"], value["path"])), "formats": formats, "encryptionConfigAttestation": encryption, - "boundedVolumeViews": None if owner.get("runtimeProfile") != "bounded-capacity" else { + "boundedVolumeViews": None if owner.get("runtimeProfile") not in {"bounded-capacity", "journal-bounded"} else { "actualBackingRoots": [str(root) for root in roots], "privateServiceBindingsRequired": True, "originalPhysicalDriveCountIncreased": False}} @@ -975,9 +977,12 @@ def configure_encryption(request): def journal_operation(request): state, roots, owner = owned(request["runId"]) require(digest(state / "rustfs") == owner["binarySha256"], "durable-record candidate changed") - require(owner.get("runtimeProfile") in {"root-restart", "mrf-crash"}, "durable records require the owned recovery runtime") + require(owner.get("runtimeProfile") in {"root-restart", "mrf-crash", "journal-bounded"}, "durable records require the owned recovery runtime") require(set(request) <= {"runId", "action", "family", "taskId", "disk", "relativePath", "faultId", "kind", "expectedBefore", "expected", "captureId"}, "durable-record adapter accepts no arbitrary system paths or commands") + if owner.get("runtimeProfile") == "journal-bounded": + require(request.get("family") == "checkpoint", "bounded journal runtime observes only the owned Admin checkpoint family") + roots = volume_views(state, roots, owner) module, fault = globals()["journal_fault_backend"], fault_module() if request["action"] in {"journal-observe", "journal-capture"}: fields = {"runId", "action", "family", "taskId", "expected"} @@ -1073,7 +1078,7 @@ def cleanup(request): gate_module().release_gate(state, roots) quorum_gate_module().release(state, roots) replacement_module().attest(state, roots) - if owner.get("runtimeProfile") == "bounded-capacity": + if owner.get("runtimeProfile") in {"bounded-capacity", "journal-bounded"}: volume_module().dispose(state, roots, owner) command("systemctl", "reset-failed", owner["unit"], check=False) for root in (*roots, state): diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 2dd365ed..524c240d 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(runtime_profile in {"queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -188,7 +188,7 @@ def remote(self, node, action, **params): network_source = NODE_SOURCE.with_name("network_gate.py").read_text() source += "\nimport types\nnetwork_gate_backend = types.ModuleType('network_gate')\nexec(" + repr( network_source) + ", network_gate_backend.__dict__)\nNETWORK_GATE_SOURCE = " + repr(network_source) + "\n" - if self.runtime_profile == "bounded-capacity": + if self.runtime_profile in {"bounded-capacity", "journal-bounded"}: source += "\nimport types\nbounded_volume_backend = types.ModuleType('bounded_volume')\nexec(" + repr( NODE_SOURCE.with_name("bounded_volume.py").read_text()) + ", bounded_volume_backend.__dict__)\n" if action == "mrf-records" or action.startswith("journal-"): @@ -324,6 +324,10 @@ def cleanup(self): if self.journal_faults: try: self.stop() + if self.runtime_profile == "journal-bounded": + for node in self.reserved: + self.record("bounded-journal-filler-recovery", {"node": node, **self.remote( + node, "bounded-volumes-release", quiescent=True)}) for node, params in tuple(self.journal_faults): receipt = self.record("durable-record-recovery", {"node": node, **self.remote(node, "journal-recover", **params)}) require(receipt.get("backupRetained") is True and receipt.get("originalResponsibilityRecreated") is False @@ -393,7 +397,7 @@ def cleanup(self): if (self.faults or self.bucket_record_faults) and not failures: try: self.stop() - if self.runtime_profile == "bounded-capacity": + if self.runtime_profile in {"bounded-capacity", "journal-bounded"}: for node in self.reserved: self.record("bounded-filler-recovery", {"node": node, **self.remote( node, "bounded-volumes-release", quiescent=True)}) diff --git a/scripts/validation_heal/journal_fault.py b/scripts/validation_heal/journal_fault.py index e64a4589..822ebdfb 100644 --- a/scripts/validation_heal/journal_fault.py +++ b/scripts/validation_heal/journal_fault.py @@ -177,12 +177,14 @@ def transform(raw, request, owner, mrf, *, earlier=None): def binding(root_fd, request, owner, fault): - require(owner.get("runtimeProfile") in {"root-restart", "mrf-crash"}, "journal mutation requires an owned recovery runtime") + require(owner.get("runtimeProfile") in {"root-restart", "mrf-crash", "journal-bounded"}, "journal mutation requires an owned recovery runtime") require(owner["runId"] == request["runId"] and type(request.get("disk")) is int and 0 <= request["disk"] < 4, "journal run or disk identity differs") identifier = request.get("faultId") require(isinstance(identifier, str) and re.fullmatch(r"F07-[a-z0-9-]{1,48}", identifier), "journal fault identity is invalid") path = relative(request) + require(owner.get("runtimeProfile") != "journal-bounded" or request["family"] == "checkpoint", + "bounded journal mutation cannot target root admissions or MRF payloads") require(request["family"] == "checkpoint" and path[-1] == task_id(request["taskId"]) + "_ahm_checkpoint.json" or request["family"] == "root" and path[-1] == "root-heal-" + task_id(request["taskId"]) + ".json" or request["family"] == "mrf" and re.fullmatch(r"\.heal-mrf-snapshot\.[01]\.bin", path[-1]), diff --git a/test_validation_heal_capacity.py b/test_validation_heal_capacity.py index 974589ec..87881e59 100644 --- a/test_validation_heal_capacity.py +++ b/test_validation_heal_capacity.py @@ -129,6 +129,55 @@ def test_only_the_dedicated_runtime_can_allocate_bounded_volumes(self): self.assertEqual(config["RUSTFS_HEAL_AUTO_HEAL_ENABLE"], "false") self.assertEqual(config["RUSTFS_SCANNER_ENABLED"], "false") + def test_bounded_journal_profile_requires_fixed_block_exhaustion_and_disables_background_sources(self): + with tempfile.TemporaryDirectory() as directory: + state = Path(directory) + owner = {"runtimeProfile": "journal-bounded", "runId": "gh-1-1-h1"} + with self.assertRaisesRegex(ValueError, "only fixed block"): + volume.create(state, (), owner, "bounded-inode-exhaustion") + self.assertFalse((state / volume.MARKER).exists()) + with self.assertRaises(ValueError): volume.views(state, (), owner) + with self.assertRaises(ValueError): volume.properties(state, (), owner) + config = fleet_node.runtime_config("journal-bounded") + self.assertEqual(config["RUSTFS_UNSAFE_BYPASS_DISK_CHECK"], "true") + self.assertEqual(config["RUSTFS_HEAL_PAGE_PARALLEL_ENABLE"], "false") + for field in ("RUSTFS_HEAL_MRF_ENABLE", "RUSTFS_HEAL_AUTO_HEAL_ENABLE", "RUSTFS_SCANNER_ENABLED"): + self.assertEqual(config[field], "false") + + def test_bounded_journal_start_uses_only_verified_private_service_bindings(self): + state, roots = Path('/owned/state'), [Path('/owned/disk' + str(i)) for i in range(4)] + owner = {"runtimeProfile": "journal-bounded", "unit": "owned-test.service", "binarySha256": "a" * 64, + "volumes": [str(root / 'data') for root in roots]} + bindings = ['--property=PrivateMounts=true', '--property=BindPaths=/verified/private/data:/owned/disk0/data'] + with patch.object(fleet_node, "owned", return_value=(state, roots, owner)), patch.object(fleet_node, "stopped"), \ + patch.object(fleet_node, "digest", return_value=owner["binarySha256"]), \ + patch.object(fleet_node, "command") as command, \ + patch.object(fleet_node, "gate_module", return_value=SimpleNamespace(checked_gate=lambda *args: None, start_properties=lambda *args: [])), \ + patch.object(fleet_node, "quorum_gate_module", return_value=SimpleNamespace(properties=lambda *args: [])), \ + patch.object(fleet_node, "replacement_module", return_value=SimpleNamespace(start_properties=lambda *args: [])), \ + patch.object(fleet_node, "volume_module", return_value=SimpleNamespace(properties=lambda *args: bindings)): + fleet_node.start({"runId": "gh-1-1-h1"}) + issued = command.call_args.args + self.assertEqual(issued[0], 'systemd-run') + for property in bindings: self.assertIn(property, issued) + self.assertIn('--property=RemainAfterExit=yes', issued) + self.assertEqual(issued[-4:], tuple(owner["volumes"])) + + def test_bounded_journal_capacity_refuses_arbitrary_limits_paths_or_process_birth(self): + owner = {"runtimeProfile": "journal-bounded"} + identity = {field: field for field in fleet_node.PROCESS_FIELDS}; identity['pid'] = 50 + request = {"runId": "gh-1-1-h1", "action": "bounded-volumes-fill", "expected": identity} + backend = SimpleNamespace(views=lambda *args: (), read=lambda *args: {'branch': 'bounded-volume-enospc'}, + fill=lambda *args: {}, witness=lambda *args, **kwargs: {}) + with patch.object(fleet_node, 'owned', return_value=(None, (), owner)), \ + patch.object(fleet_node, 'volume_module', return_value=backend): + with patch.object(fleet_node, 'process', side_effect=[identity, identity]): + self.assertTrue(fleet_node.volume_operation(request)['sameOriginalProcess']) + for extra in ({'path': '/foreign'}, {'imageBytes': 10**12}, {'branch': 'bounded-inode-exhaustion'}): + with self.subTest(extra=extra), self.assertRaises(ValueError): fleet_node.volume_operation({**request, **extra}) + with patch.object(fleet_node, 'process', return_value={**identity, 'startTicks': 999}), self.assertRaises(ValueError): + fleet_node.volume_operation(request) + def test_allocation_registers_exact_undo_before_formatting_only_bounded_files(self): with tempfile.TemporaryDirectory() as directory: state = Path(directory) / "state" diff --git a/test_validation_heal_journal_fault.py b/test_validation_heal_journal_fault.py index bf05b0ea..794377ac 100644 --- a/test_validation_heal_journal_fault.py +++ b/test_validation_heal_journal_fault.py @@ -449,5 +449,57 @@ def remote(node, action, **params): self.assertEqual(recovery["cleanup"], "FAIL") self.assertTrue(recovery["errors"][0]["namespacesRetained"]) + def test_bounded_journal_census_reads_verified_candidate_views_and_rejects_other_families(self): + with tempfile.TemporaryDirectory() as temp, patch.object(fault, "mount_id", return_value=1): + root = Path(temp).resolve(); owner, request, _, raw = self.fixture(root) + owner["runtimeProfile"] = "journal-bounded" + request = {key: request[key] for key in ("runId", "family", "taskId")}; request["action"] = "journal-census" + with patch.object(fleet_node, "owned", return_value=(root, ['/original/unbound-root'], owner)), \ + patch.object(fleet_node, "volume_views", return_value=[root]) as views, \ + patch.object(fleet_node, "digest", return_value=owner['binarySha256']), \ + patch.object(fleet_node, "fault_module", return_value=fault), patch.object(fleet_node, "stopped"), \ + patch.dict(fleet_node.__dict__, journal_fault_backend=journal): + result = fleet_node.journal_operation(request) + self.assertEqual(base64.b64decode(next(row for row in result['records'] if row['present'])['rawBase64']), raw) + views.assert_called_once() + for family in ('root', 'mrf'): + with self.subTest(family=family), self.assertRaises(ValueError): + fleet_node.journal_operation({**request, 'family': family}) + + def test_bounded_journal_cleanup_releases_filler_before_rescue_and_preserves_failed_release(self): + for failed in (False, True): + with tempfile.TemporaryDirectory() as temp: + fleet = Fleet(['node'], Path(temp), 'gh-12345-1-h1', runtime_profile='journal-bounded') + fleet.reserved = ['node']; fleet.journal_faults.append(('node', {'faultId': 'F07-checkpoint-0'})) + actions = [] + def remote(node, action, **params): + actions.append(action) + if action == 'bounded-volumes-release': + self.assertTrue(params['quiescent']) + if failed: raise ValueError('owned filesystem identity changed') + if action == 'journal-recover': + return {'backupRetained': True, 'originalResponsibilityRecreated': False, 'productRepairClaimed': False} + return {} + with patch.object(fleet, 'remote', side_effect=remote): self.assertEqual(fleet.cleanup(), not failed) + if failed: + self.assertNotIn('journal-recover', actions); self.assertNotIn('cleanup', actions) + self.assertTrue(fleet.journal_faults) + else: + self.assertLess(actions.index('bounded-volumes-release'), actions.index('journal-recover')) + self.assertLess(actions.index('journal-recover'), actions.index('cleanup')) + + def test_bounded_remote_journal_and_cleanup_embed_the_exact_capacity_backend(self): + with tempfile.TemporaryDirectory() as temp, patch('scripts.validation_heal.healthy.subprocess.run', + return_value=types.SimpleNamespace(returncode=0, stdout='{}', stderr='')) as command: + fleet = Fleet(['node'], Path(temp), 'gh-12345-1-h1', runtime_profile='journal-bounded') + for action in ('journal-census', 'cleanup'): + fleet.remote('node', action, family='checkpoint', taskId=self.token) + source = command.call_args.kwargs['input'].rsplit('\nmain(', 1)[0] + scope = {'__name__': 'bounded_journal_remote'} + exec(compile(source, 'bounded-journal-remote', 'exec'), scope) + self.assertTrue(callable(scope['volume_module']().views)) + self.assertTrue(callable(scope['volume_module']().dispose)) + if action == 'journal-census': self.assertTrue(callable(scope['journal_fault_backend'].census)) + if __name__ == "__main__": unittest.main() From ac3a5eaf8844a553dd298401367407e2369bc307 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 12:32:45 +0800 Subject: [PATCH 126/343] test(validation): bind root admission across fault boundary --- scripts/validation_heal/checkpoint.py | 71 +++++++++++++++++++++-- scripts/validation_heal/fleet_node.py | 5 +- test_validation_heal_checkpoint.py | 82 +++++++++++++++++++++++++++ test_validation_heal_journal_fault.py | 22 ++++++- 4 files changed, 172 insertions(+), 8 deletions(-) diff --git a/scripts/validation_heal/checkpoint.py b/scripts/validation_heal/checkpoint.py index 7e0eebb8..162e5781 100644 --- a/scripts/validation_heal/checkpoint.py +++ b/scripts/validation_heal/checkpoint.py @@ -23,6 +23,7 @@ "crates/heal/src/heal/channel.rs": "7f669ca43b9559db9b97fba6b419a6c76a94d76acd8526db132d74b7c534d6b0", "crates/heal/src/heal/resume.rs": "ed55baa03fafa79bf662393eafcf53b4f7752ca8b2f77824cc6deff085a0cf1d", "crates/heal/src/heal/resume/checkpoint.rs": "fe067b39ee406e16517bd665f7f49457daffa05d5d7bc298bd965e927b1381c1", + "crates/heal/src/heal/manager/root_recovery.rs": "ba6beba9d9d14985e4c5448d85727036388d3d72f28879ac5c67fc5f760048ba", "crates/heal/src/heal/erasure_healer/admin.rs": "776c03e4dfbd27f51da506d8f637de3a3973d86d22dc83a7cda24f2e0d642ee5"}, "route": "empty bucket/object plus explicit pool=0,set=0 selects Admin ErasureSet", "checkpointSchema": 8, "resumeSchema": 7, "durableOwnerRequired": True, @@ -91,10 +92,54 @@ def owned_checkpoint(observed, token, bucket, *, require_progress=True): "resume": resume, "value": value, "resumeValue": state, "originalDurableResponsibility": True} -def records(fleet, token, label, *, quiescent=False): +def owned_root_admission(observed, token): + journal.task_id(token) + names = journal.names("root", token) + require(isinstance(observed, dict) and len(observed) in {1, 4}, "root admission omitted bound nodes") + present = [] + for node, census in observed.items(): + rows = census.get("records") + require(census.get("readOnly") is True and census.get("family") == "root" and census.get("taskId") == token + and census.get("directoryEnumerationUsed") is False + and (census.get("quiescent") is True or census.get("stable") is True + and census.get("processBound") is True and census.get("independentReads") == 2) + and isinstance(rows, list) and len(rows) == 12, + "root admission lacks a stopped or stable process-bound exact-path census") + seen = set() + for row in rows: + pair = row.get("disk"), row.get("relativePath") + require(type(pair[0]) is int and 0 <= pair[0] < 4 and pair[1] in names + and pair not in seen and type(row.get("present")) is bool, + "root admission census duplicated or escaped its task/disk paths") + seen.add(pair) + if row["present"]: present.append({"node": node, **row}) + require(len(present) == 1 and present[0]["relativePath"] == names[0], + "NOT_OBSERVED: runnable root admission is absent, duplicated or already terminal") + owner = present[0] + value = journal.decode_json(bytes_of(owner)) + require(type(value.get("schema")) is int and value["schema"] == 3 and value.get("task_id") == token + and journal.same_json(value.get("heal_type"), + {"type": "erasure_set", "buckets": [], "set_disk_id": "pool_0_set_0", "all_buckets": True}) + and value.get("bucket_incarnation_id") is None and value.get("priority") == "High" + and type(value.get("retry_attempts")) is int and value["retry_attempts"] == 0, + "root admission does not bind the original explicit Admin erasure-set task") + required = {"scan_mode": 2, "remove_corrupted": False, "recreate_missing": False, "update_parity": False, + "recursive": True, "dry_run": False, "no_lock": False, "pool_index": 0, "set_index": 0} + options = value.get("options") + require(isinstance(options, dict) and all(name in options and journal.same_json(options[name], expected) + for name, expected in required.items()), "root admission changed the original Admin options/selectors") + created = value.get("created_at") + require(isinstance(created, dict) and set(created) == {"secs_since_epoch", "nanos_since_epoch"} + and type(created["secs_since_epoch"]) is int and 0 <= created["secs_since_epoch"] < 2**64 + and type(created["nanos_since_epoch"]) is int and 0 <= created["nanos_since_epoch"] < 1_000_000_000, + "root admission lost its original admitted time") + return {**owner, "value": value, "originalDurableResponsibility": True} + + +def records(fleet, token, label, *, quiescent=False, family="checkpoint"): result = {} for node in fleet.nodes: - params = {"family": "checkpoint", "taskId": token} + params = {"family": family, "taskId": token} if not quiescent: params["expected"] = fleet.record(label + "-process", {"node": node, **fleet.remote(node, "process")}) result[node] = fleet.remote(node, "journal-census" if quiescent else "journal-observe", **params) @@ -156,6 +201,7 @@ def freeze(fleet, token, bucket, *, timeout=60, earlier=None): try: owner = owned_checkpoint(records(fleet, token, "Admin-checkpoint-live"), token, bucket) if earlier is not None: after_generation(owner, earlier) + root = owned_root_admission(records(fleet, token, "Admin-root-live", family="root"), token) break except ValueError as exc: if not str(exc).startswith("NOT_OBSERVED:"): raise @@ -173,7 +219,10 @@ def freeze(fleet, token, bucket, *, timeout=60, earlier=None): require((current["node"], current["disk"], current["bucketIncarnationId"]) == (owner["node"], owner["disk"], owner["bucketIncarnationId"]), "checkpoint owner changed during freeze") if earlier is not None: after_generation(current, earlier) - return current, identity + frozen_root = owned_root_admission(records(fleet, token, "Admin-root-frozen", family="root"), token) + require(all(frozen_root[name] == root[name] for name in ("node", "disk", "relativePath", "size", "sha256")), + "original durable root admission changed during its checkpoint freeze") + return {**current, "rootAdmission": frozen_root}, identity def stop_for_fault(fleet, token, bucket, frozen, identity): @@ -188,4 +237,18 @@ def stop_for_fault(fleet, token, bucket, frozen, identity): current = owned_checkpoint(records(fleet, token, "Admin-checkpoint-stopped", quiescent=True), token, bucket) require(current["checkpoint"] == frozen["checkpoint"] and current["resume"] == frozen["resume"], "Admin durable responsibility changed between freeze and stopped fault boundary") - return current + root = owned_root_admission(records(fleet, token, "Admin-root-stopped", quiescent=True, family="root"), token) + try: + from .root_restart import same_owner + except ImportError: + from root_restart import same_owner + # A different root coordinator may shut down gracefully when all peers + # stop. Preserve every non-timeout field and forbid a refunded budget. + previous = frozen["rootAdmission"] + if all(root[name] == previous[name] for name in ("node", "disk", "relativePath", "size", "sha256")): + continuity = {"samePhysicalOwner": True, "nonTimeoutIntentUnchanged": True, + "budgetNotRefunded": True, "gracefulTimeoutCheckpointClaimed": False} + else: + continuity = same_owner(previous, root) + fleet.record("Admin-root-stop-continuity", continuity) + return {**current, "rootAdmission": root, "rootContinuity": continuity} diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 4975e677..b58a553d 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -981,7 +981,9 @@ def journal_operation(request): require(set(request) <= {"runId", "action", "family", "taskId", "disk", "relativePath", "faultId", "kind", "expectedBefore", "expected", "captureId"}, "durable-record adapter accepts no arbitrary system paths or commands") if owner.get("runtimeProfile") == "journal-bounded": - require(request.get("family") == "checkpoint", "bounded journal runtime observes only the owned Admin checkpoint family") + require(request.get("family") == "checkpoint" or request.get("family") == "root" + and request.get("action") in {"journal-census", "journal-observe"}, + "bounded journal runtime mutates only the owned Admin checkpoint family") roots = volume_views(state, roots, owner) module, fault = globals()["journal_fault_backend"], fault_module() if request["action"] in {"journal-observe", "journal-capture"}: @@ -1138,6 +1140,7 @@ def quota_audit_finalize(request): def task_records(request): """Read three exact Admin journal names; never mutate system metadata.""" state, roots, owner = owned(request["runId"]) + roots = volume_views(state, roots, owner) token = request.get("taskId") require(isinstance(token, str) and str(uuid.UUID(token)) == token, "noncanonical Admin journal task ID") require(digest(state / "rustfs") == owner["binarySha256"], "journal candidate changed") diff --git a/test_validation_heal_checkpoint.py b/test_validation_heal_checkpoint.py index 697ed214..15b03712 100644 --- a/test_validation_heal_checkpoint.py +++ b/test_validation_heal_checkpoint.py @@ -151,5 +151,87 @@ def test_stale_boundary_requires_actual_newer_progress_in_the_same_original_resp changed["checkpoint"].update(encoded(changed["value"], signed=True)) with self.assertRaises(ValueError): reader.after_generation(changed, earlier) + def root_fixture(self): + value = {'schema': 3, 'task_id': TOKEN, 'priority': 'High', 'retry_attempts': 0, + 'created_at': {'secs_since_epoch': 100, 'nanos_since_epoch': 0}, + 'heal_type': {'type': 'erasure_set', 'buckets': [], 'set_disk_id': 'pool_0_set_0', 'all_buckets': True}, + 'options': {'scan_mode': 2, 'remove_corrupted': False, 'recreate_missing': False, 'update_parity': False, + 'recursive': True, 'dry_run': False, 'no_lock': False, 'pool_index': 0, 'set_index': 0}} + rows = [{'disk': disk, 'relativePath': name, 'present': False} + for disk in range(4) for name in journal.names('root', TOKEN)] + rows[3].update(encoded(value)) + census = {'node': {'family': 'root', 'taskId': TOKEN, 'quiescent': True, 'readOnly': True, + 'directoryEnumerationUsed': False, 'records': rows}} + return census, value + + def test_root_admission_is_original_erasure_set_responsibility_with_one_exact_owner(self): + census, _ = self.root_fixture() + owner = reader.owned_root_admission(census, TOKEN) + self.assertEqual((owner['node'], owner['disk']), ('node', 1)) + self.assertTrue(owner['originalDurableResponsibility']) + for change in ('missing', 'terminal', 'duplicate', 'second-owner', 'hash', 'omitted-disk', 'unstable'): + census, value = self.root_fixture(); rows = census['node']['records'] + if change == 'missing': rows[3]['present'] = False + elif change == 'terminal': rows[4].update(encoded(value)) + elif change == 'duplicate': rows[0] = copy.deepcopy(rows[3]) + elif change == 'second-owner': census['peer'] = copy.deepcopy(census['node']) + elif change == 'hash': rows[3]['sha256'] = 'a' * 64 + elif change == 'omitted-disk': rows.pop() + else: census['node'].update(quiescent=False, stable=False, processBound=True, independentReads=2) + with self.subTest(change=change), self.assertRaises(ValueError): reader.owned_root_admission(census, TOKEN) + + def test_root_erasure_admission_rejects_foreign_task_scope_options_time_and_terminal_shape(self): + for change in ('task', 'schema', 'scope', 'scope-type', 'pool-type', 'lock', 'retry', 'priority', 'time', 'bucket-incarnation'): + census, value = self.root_fixture() + if change == 'task': value['task_id'] = INCARNATION + elif change == 'schema': value['schema'] = 2 + elif change == 'scope': value['heal_type'] = {'type': 'cluster'} + elif change == 'scope-type': value['heal_type']['all_buckets'] = 1 + elif change == 'pool-type': value['options']['pool_index'] = False + elif change == 'lock': value['options']['no_lock'] = True + elif change == 'retry': value['retry_attempts'] = 1 + elif change == 'priority': value['priority'] = 'Normal' + elif change == 'time': value['created_at']['nanos_since_epoch'] = 1_000_000_000 + else: value['bucket_incarnation_id'] = INCARNATION + census['node']['records'][3].update(encoded(value)) + with self.subTest(change=change), self.assertRaises(ValueError): reader.owned_root_admission(census, TOKEN) + + def test_freeze_and_crash_preserve_checkpoint_resume_and_the_actual_root_admission(self): + from scripts.validation_heal.fleet_node import PROCESS_FIELDS + observed, _, _ = self.fixture(); roots, _ = self.root_fixture() + root = reader.owned_root_admission(roots, TOKEN) + identity = {name: name for name in PROCESS_FIELDS}; identity.update(pid=50, state='S') + actions = [] + fleet = SimpleNamespace(frozen_processes={}, record=lambda label, value: value, stop=lambda: actions.append('stop')) + def remote(node, action, **params): + actions.append(action) + if action == 'process': return {**identity, 'state': 'T' if 'process-signal' in actions else 'S'} + if action == 'process-crash': return {'actualSignal': 9, 'pidfdBound': True, 'graceful': False, 'platformEventIssued': False} + return {} + fleet.remote = remote + def records(fleet, token, label, **params): return roots if params.get('family') == 'root' else observed + with patch.object(reader, 'records', side_effect=records): + frozen, process = reader.freeze(fleet, TOKEN, BUCKET) + self.assertEqual(frozen['rootAdmission'], root) + stopped = reader.stop_for_fault(fleet, TOKEN, BUCKET, frozen, process) + self.assertEqual(stopped['checkpoint'], frozen['checkpoint']) + self.assertTrue(stopped['rootContinuity']['nonTimeoutIntentUnchanged']) + self.assertTrue(stopped['rootContinuity']['budgetNotRefunded']) + self.assertEqual(fleet.frozen_processes, {}) + + def test_root_drift_after_freeze_retains_the_original_paused_process_for_cleanup(self): + from scripts.validation_heal.fleet_node import PROCESS_FIELDS + observed, _, _ = self.fixture(); roots, value = self.root_fixture() + identity = {name: name for name in PROCESS_FIELDS}; identity.update(pid=50, state='T') + fleet = SimpleNamespace(frozen_processes={}, record=lambda label, value: value, remote=lambda *args, **kwargs: identity) + def records(fleet, token, label, **params): + if label == 'Admin-root-frozen': + changed = copy.deepcopy(roots); altered = copy.deepcopy(value); altered['created_at']['secs_since_epoch'] += 1 + changed['node']['records'][3].update(encoded(altered)); return changed + return roots if params.get('family') == 'root' else observed + with patch.object(reader, 'records', side_effect=records), self.assertRaisesRegex(ValueError, 'root admission changed'): + reader.freeze(fleet, TOKEN, BUCKET) + self.assertEqual(fleet.frozen_processes, {'node': identity}) + if __name__ == "__main__": unittest.main() diff --git a/test_validation_heal_journal_fault.py b/test_validation_heal_journal_fault.py index 794377ac..d408e067 100644 --- a/test_validation_heal_journal_fault.py +++ b/test_validation_heal_journal_fault.py @@ -462,9 +462,25 @@ def test_bounded_journal_census_reads_verified_candidate_views_and_rejects_other result = fleet_node.journal_operation(request) self.assertEqual(base64.b64decode(next(row for row in result['records'] if row['present'])['rawBase64']), raw) views.assert_called_once() - for family in ('root', 'mrf'): - with self.subTest(family=family), self.assertRaises(ValueError): - fleet_node.journal_operation({**request, 'family': family}) + for family, action in (('mrf', 'journal-census'), ('root', 'journal-apply'), ('root', 'journal-capture')): + with self.subTest(family=family, action=action), self.assertRaises(ValueError): + fleet_node.journal_operation({**request, 'family': family, 'action': action}) + + def test_bounded_root_admission_reads_the_actual_service_view_without_mutating_it(self): + with tempfile.TemporaryDirectory() as temp, patch.object(fault, 'mount_id', return_value=1): + root = Path(temp).resolve(); owner, request, _, _ = self.fixture(root) + owner['runtimeProfile'] = 'journal-bounded' + raw = json.dumps({'schema': 3, 'task_id': self.token, 'heal_type': {'type': 'erasure_set'}}).encode() + path = root / 'data/.rustfs.sys' / ('root-heal-' + self.token + '.json'); path.write_bytes(raw) + request = {'runId': owner['runId'], 'taskId': self.token, 'quiescent': True} + with patch.object(fleet_node, 'owned', return_value=(root, ['/original/unbound'], owner)), \ + patch.object(fleet_node, 'volume_views', return_value=[root]), \ + patch.object(fleet_node, 'digest', return_value=owner['binarySha256']), patch.object(fleet_node, 'stopped'): + observed = fleet_node.task_records(request) + present = [row for row in observed['records'] if row['present']] + self.assertEqual(len(present), 1) + self.assertEqual(present[0]['sha256'], journal.entry(raw)['sha256']) + self.assertEqual(path.read_bytes(), raw) def test_bounded_journal_cleanup_releases_filler_before_rescue_and_preserves_failed_release(self): for failed in (False, True): From bef168b84f0863082bc10c2e8f384e87ab67db36 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 12:42:01 +0800 Subject: [PATCH 127/343] docs(validation): reconcile current migration evidence --- docs/validation-migration-map.md | 37 +++++++++++++++++++------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 21a9753b..670b2f46 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 134 currently dispatchable IDs include all 74 non-Heal cases and sixty Heal cases. Forty-nine have complete canonical live PASS evidence; the other 109 have no live PASS claim. +case evidence. The 134 currently dispatchable IDs include all 74 non-Heal cases and sixty Heal cases. Independent census currently confirms 86 unique complete canonical live PASS cases: 74 on the current candidate source and 12 on the older immutable source. Native, read-only, recovery and duplicate runs are excluded; the other 72 cases have no complete canonical live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1806,16 +1806,19 @@ canonical fleet PASS. an empty root request with explicit pool/set selectors. Ordinary bucket and cluster scans cannot substitute for that record. Exact task/disk paths must contain one schema-8 checkpoint and one schema-7 Admin resume state on the -same physical owner, with unchanged scope/options, valid candidate digest, -owned bucket incarnation and truthful durable progress. Read-only live -observation brackets two identical bounded record reads with the original -service birth, candidate, command and environment. The freeze/crash boundary -requires pidfd-bound SIGSTOP/SIGKILL and unchanged stopped record bytes before -any fault; no platform power event is claimed. Duplicate JSON fields, -non-finite values and duplicate dedup identities reject injection. Ten new -negative tests cover live replacement/process drift, missing/duplicate owners, -foreign scope and fabricated progress. These are F07/C11 prerequisites; -the remaining 24 complete executors are still outstanding. +same physical owner, plus one schema-3 root admission on its exact owner, with +unchanged scope/options, valid candidate digest, owned bucket incarnation and +truthful durable progress. Read-only live observation brackets two identical +bounded record reads with the original service birth, candidate, command and +environment. The bounded profile exposes root records only for read-only census +and observation; root mutation and MRF mutation remain rejected. The +freeze/crash boundary requires pidfd-bound SIGSTOP/SIGKILL and unchanged +stopped record bytes before any fault; no platform power event is claimed. +Duplicate JSON fields, non-finite values and duplicate dedup identities reject +injection. Root admission, live replacement/process drift, missing/duplicate +owners, foreign scope, typed options and fabricated progress are covered by +negative tests. These are F07/C11 prerequisites; the remaining 24 complete +executors are still outstanding. The Functional Chain consumer downloads the manager artifact from the exact successful tools run attempt before fleet mutation. `manager_fixture.py` @@ -1848,7 +1851,7 @@ failed restoration marks cleanup FAIL and retains every private namespace. Eleven additional regressions exercise real-file capture/advance/inject/rescue, immutable evidence, strict JSON identities, process drift, missing or foreign capture proof, counter/cursor drift and retained failed cleanup. The complete -local contract suite executes 484 tests with eight macOS platform skips. These +local contract suite executes 495 tests with eight macOS platform skips. These are execution prerequisites: F07's six branches on both H1 and H2, including bounded write failure and full retained-responsibility diagnostics/rescan, remain required. Automatic coverage remains 134/158 with 24 Heal executors @@ -1864,7 +1867,11 @@ those actual bounded views, and capacity witnesses retain the original process birth, namespace, UID, device and precise ENOSPC result. Filler is released before registered journal rescue; failed release retains all namespaces. Six new boundary regressions pass. The complete local suite now executes -490 tests with eight macOS skips. Linux `38110756752` independently confirms -the preceding captured-generation implementation executed all 484 tests -without skips and all six checks passed. No bounded-journal live execution or +495 tests with eight macOS skips. The current Linux contract run +`38111929721` passes its catalog/matrix/shell/scanner gates and the Heal +contract step executes 479 tests. The prior Linux run `38111242118` +independently confirms 490 tests without skips at the preceding helper; no +result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. + +Current reconciliation: helper `ac3a5ea8844a553dd298401367407e2369bc307` is pushed to auto-testing PR #113; all six auto-testing checks are green. The independent canonical census now records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. The existing serial live queue has completed replication/pool-completion and is running full-status-budget before its remaining plans; I04 and current-candidate foundation rechecks remain behind the same lock. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. From ad5cad0e719441f1464e018e4ec593e1ccae6959 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 12:57:55 +0800 Subject: [PATCH 128/343] fix(validation): accept explicit root bucket journal events --- scripts/validation_heal/full_restart.py | 28 ++++++++++++++++++------- test_validation_heal_full_restart.py | 12 ++++++++++- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/scripts/validation_heal/full_restart.py b/scripts/validation_heal/full_restart.py index 73445858..a5d1ecdf 100644 --- a/scripts/validation_heal/full_restart.py +++ b/scripts/validation_heal/full_restart.py @@ -102,18 +102,30 @@ def terminal_records(observed, token, owner, current, wanted): require(type(report.get("schema")) is int and report["schema"] == 1 and report.get("terminal") == terminal and canonical(report.get("outcome")) == canonical(current.get("outcome")), "root durable report differs from the original recovered task") - require(type(report.get("next_seq")) is int and report["next_seq"] == len(wanted) + 1 + counters = (report.get("outcome") or {}).get("counters") or {} + require(type(report.get("next_seq")) is int and type(counters.get("processed")) is int + and report["next_seq"] >= counters["processed"] + 1 and type(report.get("min_seq")) is int and 1 <= report["min_seq"] <= report["next_seq"] and isinstance(report.get("seqed_items"), list) and 0 < len(report["seqed_items"]) <= 1024, "root durable report has incomplete retained sequence bounds") - seen, previous = set(), report["min_seq"] - 1 + seen, bucket_events, previous = set(), set(), report["min_seq"] - 1 + wanted_buckets = {pair[0] for pair in wanted} for sequence, item in report["seqed_items"]: - pair = item.get("bucket"), item.get("object"), public_version_id(item.get("versionId")) - require(type(sequence) is int and sequence == previous + 1 and sequence < report["next_seq"] - and pair in wanted and pair not in seen - and item.get("objectSize") == wanted[pair].get("payload", {}).get("size", 0), - "root durable report skipped a sequence or changed an ACK version/marker/size") - seen.add(pair); previous = sequence + require(type(sequence) is int and sequence == previous + 1 and sequence < report["next_seq"], + "root durable report skipped a sequence or changed its retained suffix") + if item.get("type") == "bucket": + require(item.get("bucket") in wanted_buckets and item.get("object") == "" + and item.get("versionId") == "" and item.get("objectSize") == 0 + and item["bucket"] not in bucket_events, + "root durable report contains an unknown or duplicate bucket event") + bucket_events.add(item["bucket"]) + else: + pair = item.get("bucket"), item.get("object"), public_version_id(item.get("versionId")) + require(item.get("type") == "object" and pair in wanted and pair not in seen + and item.get("objectSize") == wanted[pair].get("payload", {}).get("size", 0), + "root durable report changed an ACK version/marker/size") + seen.add(pair) + previous = sequence require(previous == report["next_seq"] - 1, "root durable report omitted its retained suffix") return {"sameRecordDisk": True, "runnableIntentRetiredOnEveryDisk": True, "terminalAndReportMatchOriginalToken": True, "retainedItems": len(seen)} diff --git a/test_validation_heal_full_restart.py b/test_validation_heal_full_restart.py index 16cf0950..0c2f4d23 100644 --- a/test_validation_heal_full_restart.py +++ b/test_validation_heal_full_restart.py @@ -68,7 +68,7 @@ def journal(self): owner = {"node": "n0", "disk": 2, "value": {"options": {"timeout": {"secs": 149, "nanos": 0}}}} terminal = {"schema": 1, "task_id": self.token, "status": "Completed", "heal_type": {"type": "cluster"}, "options": copy.deepcopy(owner["value"]["options"])} - item = {"bucket": "versions", "object": "history", "versionId": "marker", "objectSize": 0} + item = {"type": "object", "bucket": "versions", "object": "history", "versionId": "marker", "objectSize": 0} report = {"schema": 1, "terminal": terminal, "outcome": current["outcome"], "min_seq": 1027, "next_seq": 1028, "seqed_items": [[1027, item]]} observed = {node: {"taskId": self.token, "readOnly": True, "records": [ @@ -80,6 +80,16 @@ def journal(self): def test_original_record_and_retained_suffix_are_required_after_all_peer_restart(self): observed, owner, current = self.journal() self.assertTrue(restart.terminal_records(observed, self.token, owner, current, self.wanted())["sameRecordDisk"]) + # The product sequence also contains durable bucket events; the + # retained suffix must remain contiguous while next_seq exceeds the + # object counter by those explicit events. + with_offset = copy.deepcopy(observed) + offset_report = next(row for row in with_offset["n0"]["records"] if row["kind"] == "report" and row["present"]) + offset_report["value"]["min_seq"] = 1028 + offset_report["value"]["next_seq"] = 1030 + offset_report["value"]["seqed_items"] = [[1028, {"type": "bucket", "bucket": "db", "object": "", "versionId": "", "objectSize": 0}], + [1029, offset_report["value"]["seqed_items"][0][1]]] + self.assertTrue(restart.terminal_records(with_offset, self.token, owner, current, self.wanted())["sameRecordDisk"]) for kind in ("missing", "owner", "intent", "token", "scope", "budget", "outcome", "sequence", "suffix", "marker-size"): with self.subTest(kind=kind): bad = copy.deepcopy(observed) From 15442ede4328019d60c955a12a19d3eab15f3768 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 12:59:40 +0800 Subject: [PATCH 129/343] docs(validation): reconcile live queue status --- docs/validation-migration-map.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 670b2f46..de7d6a21 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1874,4 +1874,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `ac3a5ea8844a553dd298401367407e2369bc307` is pushed to auto-testing PR #113; all six auto-testing checks are green. The independent canonical census now records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. The existing serial live queue has completed replication/pool-completion and is running full-status-budget before its remaining plans; I04 and current-candidate foundation rechecks remain behind the same lock. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. +Current reconciliation: helper `ad5cad0e719441f1464e018e4ec593e1ccae6959` is pushed to auto-testing PR #113; all six auto-testing checks are green. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. A performance retry `38112625718` is running before the remaining serial plans; I04 and current-candidate foundation rechecks remain behind the same lock. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. From 0052ea1b39827a80fda37aef1ff9fbcadf4b7921 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 13:35:34 +0800 Subject: [PATCH 130/343] fix(validation): recover unapplied performance faults --- .../validation_heal/recover_environment.py | 77 ++++++++++++++++++- test_validation_heal_recovery.py | 37 +++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index dc6fba2a..37e82db8 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -92,6 +92,9 @@ def recovery_plan(artifact, failed_run, candidate): if list(artifact.glob("*/run/heal.enospc-inodes-and-quota/cases/heal.enospc-inodes-and-quota.json")): require(not list(artifact.glob("*/run/heal.object-size-boundaries/cases/*.json")), "mixed failed case recovery artifact") return quota_recovery_plan(artifact, failed_run, candidate) + performance = list(artifact.glob("*/run/heal.performance-abba/cases/heal.performance-abba.json")) + if performance: + return performance_recovery_plan(artifact, failed_run, candidate) results = list(artifact.glob("*/run/heal.object-size-boundaries/cases/heal.object-size-boundaries.json")) require(len(results) == 1, "failed artifact must have one exact D01 canonical result") result = json.loads(results[0].read_text()) @@ -134,6 +137,71 @@ def recovery_plan(artifact, failed_run, candidate): return plans +def performance_recovery_plan(artifact, failed_run, candidate): + """Finalize a performance run that expired before its next fault was applied. + + The performance executor registers a fault before invoking the node-side + mutation. If the lease expires at that boundary, cleanup still sees the + registration and tries to recover an intent that was never created. This + path is deliberately narrower than a general fault rescue: it accepts only + the archived phase boundary with no fault or fault-proof record and an + unchanged physical scene, then asks the node to perform its ordinary + run-owned cleanup. + """ + case = "heal.performance-abba" + results = list(artifact.glob(f"*/run/{case}/cases/{case}.json")) + require(len(results) == 1, "performance recovery requires one exact J06 result") + result = json.loads(results[0].read_text()) + provenance = result.get("provenance") or {} + require(result.get("case_id") == case and result.get("verdict") == "FAIL" + and result.get("cleanup_status") == "FAIL" + and provenance.get("source_commit") == candidate["source_sha"] + and provenance.get("candidate_sha256") == candidate["package_sha256"], + "performance failed-result candidate or cleanup differs") + evidence = results[0].parents[1] / f"evidence/{case}" + summary = json.loads((evidence / "healthy-summary.json").read_text()) + require(summary.get("cleanup") == "FAIL" and len(summary.get("results", [])) == 1, + "performance recovery cannot omit or combine a retained instance") + row = summary["results"][0] + failure = row.get("failure", {}) + require(row.get("topology") == "H2" and row.get("verdict") == "FAIL" + and row.get("cleanup") == "FAIL" + and failure.get("type") == "ValueError" + and "apply-fault failed" in failure.get("message", "") + and "lease expired" in failure.get("message", ""), + "performance recovery only supports a fault lease boundary failure") + folder = evidence / "H2" + binding_files = list(folder.glob("*-binding.json")) + failure_files = list(folder.glob("*-physical-failure.json")) + boundary_files = list(folder.glob("*-physical-phase-baseline-g3-p3-B.json")) + protocol_files = list(folder.glob("*-phase-protocol-g3-p3-B.json")) + require(len(binding_files) == len(failure_files) == len(boundary_files) == len(protocol_files) == 1, + "performance recovery lacks one exact retained boundary") + binding = json.loads(binding_files[0].read_text()) + before = json.loads(boundary_files[0].read_text()) + failure_scene = json.loads(failure_files[0].read_text()) + protocol = json.loads(protocol_files[0].read_text()) + require(re.fullmatch(rf"gh-{failed_run}-[1-9][0-9]*-h2", binding.get("runId", "")) + and binding.get("candidateBinarySha256") == provenance.get("binary_sha256") + and binding.get("harnessCommit") == provenance.get("testing_commit") + and protocol.get("group") == 3 and protocol.get("phase") == 3 and protocol.get("mode") == "B" + and physical_content(before) == physical_content(failure_scene), + "performance retained scene or run identity changed") + # A failed remote apply has no durable fault evidence. A fault record or + # proof at this boundary would require the normal registered-fault rescue. + require(not list(folder.glob("*-fault-g3-p3-B.json")) + and not list(folder.glob("*-fault-proof-g3-p3-B.json")), + "performance boundary contains an applied fault; refuse shortcut cleanup") + nodes = sorted(failure_scene) + require(nodes and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), + "performance archived node identity is invalid") + return [{"kind": "performance-no-fault-applied", "runId": binding["runId"], "nodes": nodes, + "node": nodes[0], + "before": failure_scene, "expectedSceneSha256": hashlib.sha256(json.dumps( + physical_content(failure_scene)[nodes[0]], sort_keys=True, separators=(",", ":") + ).encode()).hexdigest(), "binarySha256": provenance["binary_sha256"]}] + + def recovery_scene_matches(current, plan): """A retry may contain only the exact original shards already rescued.""" observed = physical_content(current) @@ -178,7 +246,14 @@ def main(): require(recovery_scene_matches(current, plan), "retained fleet drifted from its exact archived scene; no recovery performed") fleet.record("retained-service-logs", fleet.remote(plan["node"], "logs")) - if plan.get("kind") == "owned-quota": + if plan.get("kind") == "performance-no-fault-applied": + for node in plan["nodes"]: + proof = fleet.record("audited-finalization", {"node": node, + **fleet.remote(node, "cleanup")}) + require(proof.get("cleanup") == "PASS" and proof.get("unitStopped") is True + and proof.get("rootsRemoved") is True, + "performance retained environment finalization failed") + elif plan.get("kind") == "owned-quota": for fault in plan["faults"]: fleet.record("quota-original-shard-rescue", fleet.remote(plan["node"], "recover-fault", **fault)) require(physical_content(fleet.snapshot("complete-pre-fault-rescue", quiescent=True)) == physical_content(plan["before"]), diff --git a/test_validation_heal_recovery.py b/test_validation_heal_recovery.py index 97a8e6cc..a1f814aa 100644 --- a/test_validation_heal_recovery.py +++ b/test_validation_heal_recovery.py @@ -123,6 +123,7 @@ def test_recovery_is_bound_to_failed_workflow_candidate_and_both_backups(self): for run, value in [(12346, candidate), (12345, {**candidate, "package_sha256": "f" * 64})]: with self.assertRaises(ValueError): recovery_plan(root, run, value) + def test_retry_accepts_only_original_rescued_shards_and_no_other_drift(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -209,5 +210,41 @@ def admin(path, query, body, method): self.assertFalse(result["authoritativeUsageClaimed"]) +class PerformanceBoundaryRecoveryTests(unittest.TestCase): + def archive(self, root): + case = "heal.performance-abba" + base = root / "rustfs-validation-12345-1" / "run" / case + evidence = base / "evidence" / case / "H2" + (base / "cases").mkdir(parents=True) + evidence.mkdir(parents=True) + candidate = {"source_sha": "a" * 40, "package_sha256": "b" * 64} + provenance = {"source_commit": "a" * 40, "candidate_sha256": "b" * 64, + "binary_sha256": "c" * 64, "testing_commit": "d" * 40} + write = lambda path, value: path.write_text(json.dumps(value)) + write(base / "cases" / (case + ".json"), {"case_id": case, "verdict": "FAIL", + "cleanup_status": "FAIL", "provenance": provenance}) + write(evidence.parent / "healthy-summary.json", {"cleanup": "FAIL", "results": [{"topology": "H2", + "verdict": "FAIL", "cleanup": "FAIL", "failure": {"type": "ValueError", + "message": "Heal node vm000 apply-fault failed: fault lease expired"}}]}) + scene = {node: {"formats": [{"disk": node}], "files": [{"path": "owned/key/part.1", + "sha256": node, "size": 1}]} for node in ("vm000", "vm001", "vm002", "vm003")} + binding = {"runId": "gh-12345-1-h2", "candidateBinarySha256": "c" * 64, "harnessCommit": "d" * 40} + write(evidence / "00001-binding.json", binding) + write(evidence / "00002-phase-protocol-g3-p3-B.json", {"group": 3, "phase": 3, "mode": "B"}) + write(evidence / "00003-physical-phase-baseline-g3-p3-B.json", scene) + write(evidence / "00004-physical-failure.json", scene) + return candidate, evidence + + def test_only_unapplied_fault_boundary_can_be_finalized(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate, evidence = self.archive(root) + plan = recovery_plan(root, 12345, candidate)[0] + self.assertEqual(plan["kind"], "performance-no-fault-applied") + self.assertEqual(plan["nodes"], ["vm000", "vm001", "vm002", "vm003"]) + (evidence / "00005-fault-g3-p3-B.json").write_text("{}") + with self.assertRaises(ValueError): recovery_plan(root, 12345, candidate) + + if __name__ == "__main__": unittest.main() From 2c94021c55db9e25b9945d765596f884426d957a Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 13:44:26 +0800 Subject: [PATCH 131/343] fix(validation): include performance load bucket in recovery --- scripts/validation_heal/recover_environment.py | 14 ++++++++++++++ test_validation_heal_recovery.py | 9 ++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index 37e82db8..4033aab7 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -195,8 +195,20 @@ def performance_recovery_plan(artifact, failed_run, candidate): nodes = sorted(failure_scene) require(nodes and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "performance archived node identity is invalid") + bucket = binding.get("bucketPrefix") + require(isinstance(bucket, str) and re.fullmatch(r"[a-z0-9][a-z0-9.-]+[a-z0-9]", bucket), + "performance binding has no exact primary bucket") + buckets = {bucket} + for value in failure_scene.values(): + for item in value.get("files", []): + name = item.get("path", "").split("/", 1)[0] + if name == bucket or name.startswith(bucket + "-"): + buckets.add(name) + require(buckets == {bucket, bucket + "-load"}, + "performance retained scene omitted or added a run-owned bucket") return [{"kind": "performance-no-fault-applied", "runId": binding["runId"], "nodes": nodes, "node": nodes[0], + "buckets": sorted(buckets), "before": failure_scene, "expectedSceneSha256": hashlib.sha256(json.dumps( physical_content(failure_scene)[nodes[0]], sort_keys=True, separators=(",", ":") ).encode()).hexdigest(), "binarySha256": provenance["binary_sha256"]}] @@ -241,6 +253,8 @@ def main(): complete = [] for plan in plans: fleet = Fleet(plan["nodes"], args.evidence, plan["runId"]) + if plan.get("buckets"): + fleet.snapshot_buckets = plan["buckets"] fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) current = fleet.snapshot("audited-physical-scene", quiescent=True) require(recovery_scene_matches(current, plan), diff --git a/test_validation_heal_recovery.py b/test_validation_heal_recovery.py index a1f814aa..b5707cf8 100644 --- a/test_validation_heal_recovery.py +++ b/test_validation_heal_recovery.py @@ -226,9 +226,12 @@ def archive(self, root): write(evidence.parent / "healthy-summary.json", {"cleanup": "FAIL", "results": [{"topology": "H2", "verdict": "FAIL", "cleanup": "FAIL", "failure": {"type": "ValueError", "message": "Heal node vm000 apply-fault failed: fault lease expired"}}]}) - scene = {node: {"formats": [{"disk": node}], "files": [{"path": "owned/key/part.1", - "sha256": node, "size": 1}]} for node in ("vm000", "vm001", "vm002", "vm003")} - binding = {"runId": "gh-12345-1-h2", "candidateBinarySha256": "c" * 64, "harnessCommit": "d" * 40} + scene = {node: {"formats": [{"disk": node}], "files": [ + {"path": "rv-heal-0123456789abcd/objects/key/part.1", "sha256": node, "size": 1}, + {"path": "rv-heal-0123456789abcd-load/load/00.bin/xl.meta", "sha256": node, "size": 1}, + ]} for node in ("vm000", "vm001", "vm002", "vm003")} + binding = {"runId": "gh-12345-1-h2", "bucketPrefix": "rv-heal-0123456789abcd", + "candidateBinarySha256": "c" * 64, "harnessCommit": "d" * 40} write(evidence / "00001-binding.json", binding) write(evidence / "00002-phase-protocol-g3-p3-B.json", {"group": 3, "phase": 3, "mode": "B"}) write(evidence / "00003-physical-phase-baseline-g3-p3-B.json", scene) From 7d8467707ec924727c0b7801f2bc943b2e47dc5e Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 14:00:09 +0800 Subject: [PATCH 132/343] fix(validation): extend performance lease and make cleanup idempotent --- scripts/validation_heal/fleet_node.py | 17 +++++++++++++++++ scripts/validation_heal/performance.py | 7 ++++++- scripts/validation_heal/recover_environment.py | 12 ++++++++++++ test_validation_heal_performance.py | 1 + test_validation_heal_recovery.py | 1 + 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index b58a553d..769f4f83 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1092,6 +1092,22 @@ def cleanup(request): return {"cleanup": "PASS", "unitStopped": True, "rootsRemoved": True, "hostname": owner["hostname"]} +def namespace_state(request): + """Inspect one exact run namespace without requiring its owner marker.""" + require(set(request) == {"runId", "action"}, "namespace inspection accepts only the run ID") + state, roots = paths(request["runId"]) + service = {} + for field in ("LoadState", "ActiveState", "SubState", "MainPID"): + service[field] = command("systemctl", "show", "-p", field, "--value", + unit(request["runId"]), check=False).stdout.strip() + return {"runId": request["runId"], "service": service, + "state": {"path": str(state), "exists": state.exists(), + "ownerMarker": (state / "owner.json").is_file()}, + "roots": [{"path": str(root), "exists": root.exists(), + "ownerMarker": (root / "owner.json").is_file(), + "data": (root / "data").is_dir()} for root in roots]} + + def audit_finalize(request): """Finalize only an invalid-ID rejection proved identical to its archived pre-fault scene.""" _, _, owner = owned(request["runId"]) @@ -1256,6 +1272,7 @@ def replacement_records(request): def main(request): require(os.geteuid() == 0, "node adapter requires passwordless sudo") actions = {"preflight": preflight, "reserve": reserve, "start": start, "stop": stop, "snapshot": snapshot, + "namespace-state": namespace_state, "mrf-records": mrf_records, "logs": logs, "cleanup": cleanup, "audit-finalize": audit_finalize, "install-inspector": install_inspector, "apply-fault": apply_fault, "recover-fault": recover_fault, "verify-part": verify_part, "verify-parts": verify_parts, diff --git a/scripts/validation_heal/performance.py b/scripts/validation_heal/performance.py index 80f4f53e..ad6bd94f 100644 --- a/scripts/validation_heal/performance.py +++ b/scripts/validation_heal/performance.py @@ -45,6 +45,10 @@ CASE_ID = "heal.performance-abba" BRANCHES = ("abba-repeat-1", "abba-repeat-2", "abba-repeat-3") +# The twelve fixed phases run close to the original 45-minute lease once +# physical snapshots and remote cleanup are included. Keep the fault lease +# alive for the whole candidate-bound run and leave room for final evidence. +FAULT_LEASE_MINUTES = 120 # These are acceptance gates chosen before deployment, not results derived # from the measured candidate. Changing one requires a new immutable run. PROTOCOL = {"groups": 3, "order": ["A", "B", "B", "A"], "secondsPerPhase": 30, @@ -58,6 +62,7 @@ "minimumRepairMiBPerSecond": .001, "maxProcessCpuCores": 2.5, "maxProcessRssBytes": 2800 * 1024**2, "maxProcessFDs": 4096, "metricsIntervalSeconds": 2, "repairDeadlineSeconds": 300, + "faultLeaseMinutes": FAULT_LEASE_MINUTES, "cachePolicy": "same original four roots; restart every phase; fixed warmup; no host cache eviction", "latencyScope": "collector scheduled admission through complete SDK response/body; includes queue delay and errors", "retryPolicy": "no retries; every scheduled GET and PUT remains in the denominator", @@ -354,7 +359,7 @@ def run(args, nodes): access, secret = secrets.token_hex(16), secrets.token_hex(24) os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, - "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat(), "bucketPrefix": bucket, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=FAULT_LEASE_MINUTES)).isoformat(), "bucketPrefix": bucket, "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, 5)}, "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index 4033aab7..a13ca792 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -256,6 +256,18 @@ def main(): if plan.get("buckets"): fleet.snapshot_buckets = plan["buckets"] fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) + states = [fleet.record("namespace-state", {"node": node, + **fleet.remote(node, "namespace-state")}) for node in plan["nodes"]] + if all(not item["state"]["exists"] and not any(root["exists"] for root in item["roots"]) + for item in states): + for item in states: + fleet.record("already-clean", {"node": item["node"], "cleanup": "PASS", + "unitStopped": item["service"].get("MainPID", "0") in {"", "0"}, + "rootsRemoved": True}) + complete.append({"failedRunId": args.failed_run_id, "runId": plan["runId"], + "cleanup": "PASS", "originalVerdictRewritten": False, + "alreadyClean": True}) + continue current = fleet.snapshot("audited-physical-scene", quiescent=True) require(recovery_scene_matches(current, plan), "retained fleet drifted from its exact archived scene; no recovery performed") diff --git a/test_validation_heal_performance.py b/test_validation_heal_performance.py index c0bfddee..a01b4f50 100644 --- a/test_validation_heal_performance.py +++ b/test_validation_heal_performance.py @@ -36,6 +36,7 @@ def test_frozen_order_contains_every_source_repetition(self): self.assertEqual(select(perf.CASE_ID)[0]["variants"], list(perf.BRANCHES)) self.assertIn(perf.CASE_ID, AUTOMATIC_CASES) self.assertEqual([phase["mode"] for phase in perf.schedule()], list("ABBAABBAABBA")) + self.assertGreaterEqual(perf.PROTOCOL["faultLeaseMinutes"], 120) with self.assertRaises(ValueError): perf.schedule({**perf.PROTOCOL, "maxErrorRate": 1}) def test_nearest_rank_percentiles_keep_the_slowest_request(self): diff --git a/test_validation_heal_recovery.py b/test_validation_heal_recovery.py index b5707cf8..8e9bb468 100644 --- a/test_validation_heal_recovery.py +++ b/test_validation_heal_recovery.py @@ -65,6 +65,7 @@ def test_standalone_remote_recovery_loads_all_cleanup_backends(self): scope = {"__name__": "validation_remote_recovery"} exec(compile(source, "standalone-recovery", "exec"), scope) self.assertTrue(callable(scope["quota_audit_finalize"])) + self.assertTrue(callable(scope["namespace_state"])) self.assertTrue(callable(scope["quorum_gate_backend"].release)) self.assertTrue(callable(scope["replacement"].attest)) self.assertTrue(callable(scope["network_gate_backend"].release)) From f56e89ab0e06b76c1d6a469d4f0f82cee4753336 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 14:06:20 +0800 Subject: [PATCH 133/343] docs(validation): reconcile cleanup and lease evidence --- docs/validation-migration-map.md | 14 ++++-- .../validation_heal/recover_environment.py | 14 ++++-- test_validation_heal_recovery.py | 47 +++++++++++++++++++ 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index de7d6a21..3552b04d 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -938,8 +938,12 @@ uses physical target bytes, separately from logical S3 throughput. Frozen gates require zero errors, GET/PUT P95 <= 5s and P99 <= 10s, paired B/A ratios <= 3, positive verified repair throughput and bounded resources. All twelve phases and every paired difference are reported; no phase may be omitted or reordered -and functional success cannot substitute for performance acceptance. J06 has -no live PASS claim yet. +and functional success cannot substitute for performance acceptance. The first +candidate run expired its 45-minute fault lease before group 3 phase 3; its +physical failure scene showed no applied fault and the retained namespace was +later independently finalized. The executor now binds a 120-minute lease for +the full evidence window. J06 has no live PASS claim yet and needs a fresh +candidate run. F05 `38087158034` passed the expanded physical census and reached the real @@ -1851,7 +1855,7 @@ failed restoration marks cleanup FAIL and retains every private namespace. Eleven additional regressions exercise real-file capture/advance/inject/rescue, immutable evidence, strict JSON identities, process drift, missing or foreign capture proof, counter/cursor drift and retained failed cleanup. The complete -local contract suite executes 495 tests with eight macOS platform skips. These +local contract suite executes 498 tests with eight macOS platform skips. These are execution prerequisites: F07's six branches on both H1 and H2, including bounded write failure and full retained-responsibility diagnostics/rescan, remain required. Automatic coverage remains 134/158 with 24 Heal executors @@ -1867,11 +1871,11 @@ those actual bounded views, and capacity witnesses retain the original process birth, namespace, UID, device and precise ENOSPC result. Filler is released before registered journal rescue; failed release retains all namespaces. Six new boundary regressions pass. The complete local suite now executes -495 tests with eight macOS skips. The current Linux contract run +498 tests with eight macOS skips. The current Linux contract run `38111929721` passes its catalog/matrix/shell/scanner gates and the Heal contract step executes 479 tests. The prior Linux run `38111242118` independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `ad5cad0e719441f1464e018e4ec593e1ccae6959` is pushed to auto-testing PR #113; all six auto-testing checks are green. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. A performance retry `38112625718` is running before the remaining serial plans; I04 and current-candidate foundation rechecks remain behind the same lock. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. +Current reconciliation: helper `7d8467707ec924727c0b7801f2bc943b2e47dc5e` is pushed to auto-testing PR #113; its targeted local contract suite passes 498 tests with eight macOS skips. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index a13ca792..c2a1202d 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -261,9 +261,12 @@ def main(): if all(not item["state"]["exists"] and not any(root["exists"] for root in item["roots"]) for item in states): for item in states: + require(item["service"].get("MainPID") == "0" + and item["service"].get("ActiveState") == "inactive" + and item["service"].get("SubState") == "dead", + "absent namespace still has an active or unknown service; no finalization claimed") fleet.record("already-clean", {"node": item["node"], "cleanup": "PASS", - "unitStopped": item["service"].get("MainPID", "0") in {"", "0"}, - "rootsRemoved": True}) + "unitStopped": True, "rootsRemoved": True}) complete.append({"failedRunId": args.failed_run_id, "runId": plan["runId"], "cleanup": "PASS", "originalVerdictRewritten": False, "alreadyClean": True}) @@ -289,9 +292,10 @@ def main(): else: proof = fleet.record("audited-finalization", fleet.remote(plan["node"], "audit-finalize", **{ key: plan[key] for key in ("disk", "rejectedFaultId", "binarySha256", "expectedSceneSha256")})) - for node in plan["nodes"]: - if node != plan["node"]: - fleet.record("unchanged-peer-finalization", fleet.remote(node, "cleanup")) + if plan.get("kind") != "performance-no-fault-applied": + for node in plan["nodes"]: + if node != plan["node"]: + fleet.record("unchanged-peer-finalization", fleet.remote(node, "cleanup")) require(proof.get("cleanup") == "PASS", "audited environment finalization failed") complete.append({"failedRunId": args.failed_run_id, "runId": plan["runId"], "cleanup": "PASS", "originalVerdictRewritten": False}) save(args.evidence / "recovery-summary.json", {"cleanup": "PASS", "environments": complete, "canonicalPassClaimed": False}) diff --git a/test_validation_heal_recovery.py b/test_validation_heal_recovery.py index 8e9bb468..5371186b 100644 --- a/test_validation_heal_recovery.py +++ b/test_validation_heal_recovery.py @@ -4,6 +4,7 @@ import json from pathlib import Path import tempfile +import types import unittest from unittest.mock import patch @@ -12,6 +13,7 @@ from scripts.validation_heal.recover_environment import recovery_plan, recovery_scene_matches from scripts.validation_heal.capacity import release_quota from scripts.validation_heal.healthy import Fleet +from scripts.validation_heal import recover_environment as recovery class RejectedFaultRecoveryTest(unittest.TestCase): @@ -249,6 +251,51 @@ def test_only_unapplied_fault_boundary_can_be_finalized(self): (evidence / "00005-fault-g3-p3-B.json").write_text("{}") with self.assertRaises(ValueError): recovery_plan(root, 12345, candidate) + def test_full_performance_finalization_cleans_each_node_once(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate, _ = self.archive(root) + plan = recovery_plan(root, 12345, candidate)[0] + candidate_path = root / "candidate.json" + candidate_path.write_text(json.dumps(candidate)) + calls = [] + def remote(node, action, **params): + calls.append((node, action)) + if action == "namespace-state": + return {"state": {"exists": True}, "roots": [{"exists": True}]} + if action == "logs": return {} + self.assertEqual(action, "cleanup") + return {"cleanup": "PASS", "unitStopped": True, "rootsRemoved": True} + fake = types.SimpleNamespace(record=lambda label, value: value, + snapshot=lambda *args, **kwargs: plan["before"], remote=remote) + argv = ["recover_environment.py", "--failed-run-id", "12345", "--artifact", str(root), + "--candidate", str(candidate_path), "--evidence", str(root / "recovery")] + with patch.object(recovery, "Fleet", return_value=fake), patch("sys.argv", argv), \ + patch.dict("os.environ", RUSTFS_NODES="vm000 vm001 vm002 vm003"): + recovery.main() + self.assertEqual([node for node, action in calls if action == "cleanup"], plan["nodes"]) + self.assertEqual(json.loads((root / "recovery/recovery-summary.json").read_text())["cleanup"], "PASS") + + def test_absent_namespace_with_active_or_unknown_service_cannot_claim_cleanup(self): + for service in ({"MainPID": "7", "ActiveState": "active", "SubState": "running"}, + {}, {"MainPID": "0", "ActiveState": "active", "SubState": "exited"}): + with self.subTest(service=service), tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate, _ = self.archive(root) + candidate_path = root / "candidate.json" + candidate_path.write_text(json.dumps(candidate)) + def remote(node, action): + self.assertEqual(action, "namespace-state") + return {"state": {"exists": False}, "roots": [{"exists": False}], "service": service} + fake = types.SimpleNamespace(record=lambda label, value: value, remote=remote) + argv = ["recover_environment.py", "--failed-run-id", "12345", "--artifact", str(root), + "--candidate", str(candidate_path), "--evidence", str(root / "recovery")] + with patch.object(recovery, "Fleet", return_value=fake), patch("sys.argv", argv), \ + patch.dict("os.environ", RUSTFS_NODES="vm000 vm001 vm002 vm003"): + with self.assertRaisesRegex(ValueError, "active or unknown service"): + recovery.main() + self.assertFalse((root / "recovery/recovery-summary.json").exists()) + if __name__ == "__main__": unittest.main() From e4fa749814227dabc63a11fcdf93232a3eb1024a Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 14:15:20 +0800 Subject: [PATCH 134/343] fix(validation): recover expired network gates safely --- scripts/validation_heal/fleet_node.py | 6 +- scripts/validation_heal/network_gate.py | 12 +++- .../validation_heal/recover_environment.py | 67 +++++++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 769f4f83..54df21f7 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1039,10 +1039,12 @@ def network_operation(request): module = globals()["network_gate_backend"] module.identities(owner) action = request["action"] - require(set(request) <= {"runId", "action", "addresses", "localIndex", "blockedIndices"}, + require(set(request) <= {"runId", "action", "addresses", "localIndex", "blockedIndices", "expired"}, "network adapter accepts no external paths, ports, rules or commands") if action == "network-release": - return module.release(state, owner) + require(set(request) <= {"runId", "action", "expired"}, + "network release accepts only its expiry proof") + return module.release(state, owner, expired=request.get("expired", False)) identity = process(request) require(identity["pid"] > 1 and identity["executable"] == str(state / "rustfs"), "network fault requires an actual owned candidate process") diff --git a/scripts/validation_heal/network_gate.py b/scripts/validation_heal/network_gate.py index 10177d40..7f8309c6 100644 --- a/scripts/validation_heal/network_gate.py +++ b/scripts/validation_heal/network_gate.py @@ -269,6 +269,13 @@ def release(state, owner, *, expired=False): table, _ = identities(owner) require(not table_present(table), 'network table has no ownership intent; preserve namespace') return {'state': 'CLEAR', 'gateWasAbsent': True} + expired = expired or (state / EXPIRY).exists() + if expired: + for suffix in ('.timer', '.service'): + require(command('systemctl', 'show', value['timer'] + suffix, '-p', 'LoadState', '--value').strip() + in {'not-found', 'loaded'}, 'expired network unit identity is unknown') + require(command('systemctl', 'show', value['timer'] + suffix, '-p', 'ActiveState', '--value').strip() + in {'inactive', 'failed', 'unknown'}, 'expired network unit remains active') # Stop the expiry worker before inspecting/deleting, avoiding a concurrent # deletion. A TTL worker does not stop its own unit. if not expired: @@ -284,7 +291,10 @@ def release(state, owner, *, expired=False): 'removedCounters': counters, 'expiryTriggered': expired or (state / EXPIRY).exists(), 'unrelatedTablesChanged': False} if expired: - write(state / EXPIRY, receipt) + if not (state / EXPIRY).exists(): + write(state / EXPIRY, receipt) + (state / MARKER).unlink() + (state / SCRIPT).unlink() else: for suffix in ('.timer', '.service'): require(command('systemctl', 'show', value['timer'] + suffix, '-p', 'ActiveState', '--value').strip() in {'inactive', 'failed'}, diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index c2a1202d..bb203352 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -87,11 +87,63 @@ def one(suffix): separators=(",", ":")).encode()).hexdigest()}] +def network_recovery_plan(artifact, failed_run, candidate): + """Recover only an I03 gate whose independent expiry unit is gone.""" + case = "heal.network-partition-and-lock-quorum" + results = list(artifact.glob(f"*/run/{case}/cases/{case}.json")) + require(len(results) == 1, "network recovery requires one exact I03 result") + result = json.loads(results[0].read_text()) + provenance = result.get("provenance") or {} + require(result.get("case_id") == case and result.get("verdict") == "FAIL" + and result.get("cleanup_status") == "FAIL" + and provenance.get("source_commit") == candidate["source_sha"] + and provenance.get("candidate_sha256") == candidate["package_sha256"], + "network failed-result candidate or cleanup differs") + evidence = results[0].parents[1] / f"evidence/{case}/H2/minority-partition" + summary = json.loads((results[0].parents[1] / f"evidence/{case}/healthy-summary.json").read_text()) + rows = [row for row in summary.get("results", []) if row.get("cleanup") != "PASS"] + require(summary.get("cleanup") == "FAIL" and len(rows) == 1 + and rows[0].get("branch") == "minority-partition" + and "below-quorum GET returned data" in rows[0].get("failure", {}).get("message", "") + and "network-release failed" in rows[0].get("recoveryError", "") + and "not loaded" in rows[0].get("recoveryError", ""), + "network recovery only supports the expired I03 gate boundary") + bindings = list(evidence.glob("*-binding.json")) + intents = sorted(evidence.glob("*-network-expiry-and-rule-intent.json")) + activations = sorted(evidence.glob("*-atomic-owned-table-activation.json")) + require(len(bindings) == 1 and len(intents) == len(activations) == 4, + "network recovery lacks four exact gate intents and activations") + binding = json.loads(bindings[0].read_text()) + require(re.fullmatch(rf"gh-{failed_run}-[1-9][0-9]*-h2", binding.get("runId", "")) + and binding.get("candidateBinarySha256") == provenance.get("binary_sha256") + and binding.get("candidateCommit") == candidate["source_sha"], + "network binding is not the original candidate-bound H2 run") + intent_values = [json.loads(path.read_text()) for path in intents] + activation_values = [json.loads(path.read_text()) for path in activations] + nodes = [value.get("node") for value in intent_values] + require(set(nodes) == {"vm000", "vm001", "vm002", "vm003"} + and all(value.get("runId") == binding["runId"] and value.get("phase") == "armed" + and value.get("binarySha256") == provenance.get("binary_sha256") + and re.fullmatch(r"rv_i03_[0-9a-f]{24}", value.get("table", "")) + and re.fullmatch(r"rv-i03-expiry-[0-9a-f]{24}", value.get("timer", "")) + for value in intent_values) + and all(value.get("state") == "ACTIVE" and value.get("node") in set(nodes) + and value.get("table") in {intent["table"] for intent in intent_values} + for value in activation_values), + "network gate evidence is not a complete four-node owned expiry boundary") + return [{"kind": "network-expiry-recovery", "runId": binding["runId"], "nodes": nodes, + "node": nodes[0], "binarySha256": provenance["binary_sha256"], + "intents": intent_values, "activations": activation_values}] + + def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") if list(artifact.glob("*/run/heal.enospc-inodes-and-quota/cases/heal.enospc-inodes-and-quota.json")): require(not list(artifact.glob("*/run/heal.object-size-boundaries/cases/*.json")), "mixed failed case recovery artifact") return quota_recovery_plan(artifact, failed_run, candidate) + network = list(artifact.glob("*/run/heal.network-partition-and-lock-quorum/cases/heal.network-partition-and-lock-quorum.json")) + if network: + return network_recovery_plan(artifact, failed_run, candidate) performance = list(artifact.glob("*/run/heal.performance-abba/cases/heal.performance-abba.json")) if performance: return performance_recovery_plan(artifact, failed_run, candidate) @@ -256,6 +308,21 @@ def main(): if plan.get("buckets"): fleet.snapshot_buckets = plan["buckets"] fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) + if plan.get("kind") == "network-expiry-recovery": + for node in plan["nodes"]: + fleet.record("network-recovery-stop", {"node": node, **fleet.remote(node, "stop")}) + for node in plan["nodes"]: + proof = fleet.record("network-expiry-recovery", {"node": node, + **fleet.remote(node, "network-release", expired=True)}) + require(proof.get("state") == "CLEAR" and proof.get("rulesRemaining") == 0, + "expired network gate was not removed exactly") + for node in plan["nodes"]: + proof = fleet.record("network-finalization", {"node": node, **fleet.remote(node, "cleanup")}) + require(proof.get("cleanup") == "PASS" and proof.get("unitStopped") is True + and proof.get("rootsRemoved") is True, "network namespace finalization failed") + complete.append({"failedRunId": args.failed_run_id, "runId": plan["runId"], + "cleanup": "PASS", "originalVerdictRewritten": False}) + continue states = [fleet.record("namespace-state", {"node": node, **fleet.remote(node, "namespace-state")}) for node in plan["nodes"]] if all(not item["state"]["exists"] and not any(root["exists"] for root in item["roots"]) From e8a87ab3facd9c401ec7cce1fa63bdc82ca079e8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 14:17:21 +0800 Subject: [PATCH 135/343] fix(validation): stop expired network units before release --- scripts/validation_heal/network_gate.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/validation_heal/network_gate.py b/scripts/validation_heal/network_gate.py index 7f8309c6..6cc2364b 100644 --- a/scripts/validation_heal/network_gate.py +++ b/scripts/validation_heal/network_gate.py @@ -272,9 +272,13 @@ def release(state, owner, *, expired=False): expired = expired or (state / EXPIRY).exists() if expired: for suffix in ('.timer', '.service'): - require(command('systemctl', 'show', value['timer'] + suffix, '-p', 'LoadState', '--value').strip() - in {'not-found', 'loaded'}, 'expired network unit identity is unknown') - require(command('systemctl', 'show', value['timer'] + suffix, '-p', 'ActiveState', '--value').strip() + unit_name = value['timer'] + suffix + load_state = command('systemctl', 'show', unit_name, '-p', 'LoadState', '--value').strip() + require(load_state in {'not-found', 'loaded'}, 'expired network unit identity is unknown') + active_state = command('systemctl', 'show', unit_name, '-p', 'ActiveState', '--value').strip() + if load_state != 'not-found' and active_state not in {'inactive', 'failed', 'unknown'}: + command('systemctl', 'stop', unit_name) + require(command('systemctl', 'show', unit_name, '-p', 'ActiveState', '--value').strip() in {'inactive', 'failed', 'unknown'}, 'expired network unit remains active') # Stop the expiry worker before inspecting/deleting, avoiding a concurrent # deletion. A TTL worker does not stop its own unit. From 7b91c8f7d724ff1cee5d0aff13aa07b73e891650 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 14:19:31 +0800 Subject: [PATCH 136/343] docs(validation): record network cleanup evidence --- docs/validation-migration-map.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 3552b04d..93ef0d41 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `7d8467707ec924727c0b7801f2bc943b2e47dc5e` is pushed to auto-testing PR #113; its targeted local contract suite passes 498 tests with eight macOS skips. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. +Current reconciliation: helper `e8a87ab3facd9c401ec7cce1fa63bdc82ca079e8` is pushed to auto-testing PR #113; its targeted local contract suite passes 498 tests with eight macOS skips. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. No live PASS is added for either failed run. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. From 2249cd400b96ee51ce2b85c582dd115815812431 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 14:29:24 +0800 Subject: [PATCH 137/343] fix(validation): wait for authoritative multi-pool quota usage --- scripts/validation_heal/heal_probe.py | 8 +++++++- scripts/validation_heal/multi_pool.py | 24 ++++++++++++++++++++++++ test_validation_heal_capacity.py | 17 +++++++++++++++++ test_validation_heal_multi_pool.py | 13 +++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/heal_probe.py b/scripts/validation_heal/heal_probe.py index 259d57ff..75471ba8 100644 --- a/scripts/validation_heal/heal_probe.py +++ b/scripts/validation_heal/heal_probe.py @@ -259,8 +259,9 @@ def admin(self, path: str, query: list[tuple[str, str]], body: bytes, with observation_phase(observation, 'requestPreparation'): validate_binding(self.binding) quota_path = path.startswith("/rustfs/admin/v3/quota/") + quota_stats_path = path.startswith("/rustfs/admin/v3/quota-stats/") pool_path = path in {"/rustfs/admin/v3/pools/list", "/rustfs/admin/v3/pools/status"} - require(path.startswith("/rustfs/admin/v3/heal/") or quota_path or pool_path or path in { + require(path.startswith("/rustfs/admin/v3/heal/") or quota_path or quota_stats_path or pool_path or path in { "/rustfs/admin/v3/background-heal/status", "/rustfs/admin/v4/heal/replacement-recovery", "/rustfs/admin/v3/info"}, "only bounded Heal evidence routes are allowed") if path == "/rustfs/admin/v3/info": @@ -286,6 +287,11 @@ def admin(self, path: str, query: list[tuple[str, str]], body: bytes, and quota["quota_type"] == "HARD" and type(quota["quota"]) is int and quota["quota"] in {1024**2, 256 * 1024**2}, "quota mutation is outside the two registered I02 limits") + if quota_stats_path: + bucket = path.removeprefix("/rustfs/admin/v3/quota-stats/") + bucket_allowed(self.binding, bucket) + require(method == "GET" and not query and body == b"" and auth == "signed", + "owned quota-stats evidence requires an exact signed read-only GET") require("?" not in path and "#" not in path and len(body) <= 2 * 1024**2, "invalid Admin path or body budget") require(not any(part in {".", ".."} for part in unquote(path).split("/")), diff --git a/scripts/validation_heal/multi_pool.py b/scripts/validation_heal/multi_pool.py index b2a35485..18d609be 100644 --- a/scripts/validation_heal/multi_pool.py +++ b/scripts/validation_heal/multi_pool.py @@ -11,6 +11,7 @@ import secrets import signal import sys +import time try: from .heal_probe import Probe, compare_inventories, payload, require, save @@ -301,9 +302,32 @@ def pool_status(probe, fleet, endpoint_expressions, label): return [{name: row[name] for name in ('id', 'cmdline', 'status', 'totalSize')} for row in values] +def quota_usage_ready(probe, fleet, bucket, label): + """Wait for the server's authoritative usage snapshot before reading config.""" + deadline = time.monotonic() + 30 + attempts = 0 + while True: + attempts += 1 + value = fleet.record(label + '-readiness', probe.admin( + '/rustfs/admin/v3/quota-stats/' + bucket, [], b'', method='GET')) + response = value.get('response') or {} + if value.get('httpStatus') == 200: + require(response.get('bucket') == bucket and type(response.get('current_usage')) is int + and response.get('current_usage') >= 0, + 'H06 quota readiness returned an invalid owned-bucket usage snapshot') + return {'bucket': bucket, 'currentUsage': response['current_usage'], 'attempts': attempts} + require(value.get('httpStatus') == 503 + and isinstance(response, str) + and 'authoritative bucket usage is not available yet' in response, + 'H06 quota readiness returned an unreviewed response') + require(time.monotonic() < deadline, 'H06 authoritative bucket usage did not become ready within 30 seconds') + time.sleep(1) + + def quota_status(probe, fleet, buckets, label): result = {} for bucket in buckets: + quota_usage_ready(probe, fleet, bucket, label) value = fleet.record(label, probe.admin('/rustfs/admin/v3/quota/' + bucket, [], b'', method='GET')) response = value.get('response') or {} require(value.get('httpStatus') == 200 and response.get('bucket') == bucket diff --git a/test_validation_heal_capacity.py b/test_validation_heal_capacity.py index 87881e59..28c473af 100644 --- a/test_validation_heal_capacity.py +++ b/test_validation_heal_capacity.py @@ -229,6 +229,21 @@ def __init__(self): self.assertIn("Authorization", outgoing.call_args.args[0].headers) self.assertEqual(probe.admin(path, [], b"", method="GET")["httpStatus"], 200) + def test_quota_stats_probe_is_an_exact_read_only_readiness_route(self): + probe = Probe(binding()) + path = "/rustfs/admin/v3/quota-stats/" + probe.binding["bucketPrefix"] + class Response(BytesIO): + code = 200 + def __init__(self): + super().__init__(b'{"bucket":"' + probe.binding["bucketPrefix"].encode() + b'","current_usage":0}') + self.headers = {"Content-Length": str(self.getbuffer().nbytes)} + with patch.object(probe, "credentials", return_value=("access", "secret")), \ + patch.object(probe, "_open_admin", return_value=Response()) as outgoing: + observed = probe.admin(path, [], b"", method="GET") + self.assertEqual(observed["httpStatus"], 200) + self.assertEqual(observed["response"]["bucket"], probe.binding["bucketPrefix"]) + self.assertEqual(outgoing.call_args.args[0].get_method(), "GET") + def test_quota_probe_refuses_foreign_buckets_unregistered_limits_and_auth(self): probe = Probe(binding()) path = "/rustfs/admin/v3/quota/" + probe.binding["bucketPrefix"] @@ -243,6 +258,8 @@ def test_quota_probe_refuses_foreign_buckets_unregistered_limits_and_auth(self): patch.object(probe, "_open_admin") as outgoing, self.assertRaises(ValueError): probe.admin(route, [], body, method=method, auth=auth) outgoing.assert_not_called() + with self.assertRaises(ValueError): + probe.admin("/rustfs/admin/v3/quota-stats/foreign-bucket", [], b"", method="GET") if __name__ == "__main__": diff --git a/test_validation_heal_multi_pool.py b/test_validation_heal_multi_pool.py index a55918e0..f7594229 100644 --- a/test_validation_heal_multi_pool.py +++ b/test_validation_heal_multi_pool.py @@ -84,6 +84,19 @@ def test_preserves_all_three_source_branches_and_real_two_pool_arguments(self): self.assertEqual(runtime_config('multi-pool')['RUSTFS_STORAGE_CLASS_STANDARD'], 'EC:2') with self.assertRaises(ValueError): volumes('gh-123-1-h2') + def test_quota_status_waits_for_authoritative_usage_before_reading_config(self): + probe, fleet = Mock(), Mock() + fleet.record.side_effect = lambda label, value: value + probe.admin.side_effect = [ + {'httpStatus': 503, 'response': 'authoritative bucket usage is not available yet'}, + {'httpStatus': 200, 'response': {'bucket': self.buckets[0], 'current_usage': 0}}, + ] + with patch.object(runner.time, 'sleep') as sleep: + ready = runner.quota_usage_ready(probe, fleet, self.buckets[0], 'quota') + self.assertEqual(ready['attempts'], 2) + self.assertEqual(probe.admin.call_args_list[0].args[0], '/rustfs/admin/v3/quota-stats/' + self.buckets[0]) + sleep.assert_called_once_with(1) + def test_h5_pool_observations_accept_only_exact_signed_get_queries(self): binding = {'schemaVersion': 1, 'runId': 'gh-123-1-h5', 'jumpHostOnly': True, 'exclusiveHealService': True, 'leaseExpiresAt': (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), From d688cf506c11db572c839026878197a1caca55e2 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 14:37:05 +0800 Subject: [PATCH 138/343] fix(validation): read multi-pool quota config without scanner --- scripts/validation_heal/heal_probe.py | 14 ++++---- scripts/validation_heal/multi_pool.py | 46 ++++++++++----------------- test_validation_heal_capacity.py | 24 ++++++++++---- test_validation_heal_multi_pool.py | 41 ++++++++++++++++++------ 4 files changed, 71 insertions(+), 54 deletions(-) diff --git a/scripts/validation_heal/heal_probe.py b/scripts/validation_heal/heal_probe.py index 75471ba8..25276061 100644 --- a/scripts/validation_heal/heal_probe.py +++ b/scripts/validation_heal/heal_probe.py @@ -259,9 +259,9 @@ def admin(self, path: str, query: list[tuple[str, str]], body: bytes, with observation_phase(observation, 'requestPreparation'): validate_binding(self.binding) quota_path = path.startswith("/rustfs/admin/v3/quota/") - quota_stats_path = path.startswith("/rustfs/admin/v3/quota-stats/") + quota_config_path = path == "/rustfs/admin/v3/get-bucket-quota" pool_path = path in {"/rustfs/admin/v3/pools/list", "/rustfs/admin/v3/pools/status"} - require(path.startswith("/rustfs/admin/v3/heal/") or quota_path or quota_stats_path or pool_path or path in { + require(path.startswith("/rustfs/admin/v3/heal/") or quota_path or quota_config_path or pool_path or path in { "/rustfs/admin/v3/background-heal/status", "/rustfs/admin/v4/heal/replacement-recovery", "/rustfs/admin/v3/info"}, "only bounded Heal evidence routes are allowed") if path == "/rustfs/admin/v3/info": @@ -287,11 +287,11 @@ def admin(self, path: str, query: list[tuple[str, str]], body: bytes, and quota["quota_type"] == "HARD" and type(quota["quota"]) is int and quota["quota"] in {1024**2, 256 * 1024**2}, "quota mutation is outside the two registered I02 limits") - if quota_stats_path: - bucket = path.removeprefix("/rustfs/admin/v3/quota-stats/") - bucket_allowed(self.binding, bucket) - require(method == "GET" and not query and body == b"" and auth == "signed", - "owned quota-stats evidence requires an exact signed read-only GET") + if quota_config_path: + require(method == "GET" and len(query) == 1 and len(query[0]) == 2 + and query[0][0] == "bucket" and body == b"" and auth == "signed", + "owned quota config requires an exact signed read-only GET") + bucket_allowed(self.binding, query[0][1]) require("?" not in path and "#" not in path and len(body) <= 2 * 1024**2, "invalid Admin path or body budget") require(not any(part in {".", ".."} for part in unquote(path).split("/")), diff --git a/scripts/validation_heal/multi_pool.py b/scripts/validation_heal/multi_pool.py index 18d609be..a7448525 100644 --- a/scripts/validation_heal/multi_pool.py +++ b/scripts/validation_heal/multi_pool.py @@ -11,7 +11,6 @@ import secrets import signal import sys -import time try: from .heal_probe import Probe, compare_inventories, payload, require, save @@ -53,6 +52,10 @@ 'placement': 'observe actual full-set XL2/version authority; no assumed pool hash or internal PUT selector', 'scope': 'pool selector must match admitted pool; an unrestricted set may be null in the raw receipt, with independent exact physical binding', 'control': 'unselected defects stay missing before any target GET or rescue; repair them in independent tasks only after proof'} +REFERENCE['quotaConfig'] = {'sourceFile': 'rustfs/src/admin/handlers/quota.rs', + 'sourceSha256': '6a2b062d3d1c4cd81d934c570d21cb44dcd2c1883f6a85d2265e01b466924f5c', + 'route': '/rustfs/admin/v3/get-bucket-quota', 'query': 'one exact owned bucket', + 'assertion': 'unchanged no-quota config on every original peer; no authoritative usage claim'} def fixture_call(recorder, ledger, operation, params, body=None): @@ -302,38 +305,21 @@ def pool_status(probe, fleet, endpoint_expressions, label): return [{name: row[name] for name in ('id', 'cmdline', 'status', 'totalSize')} for row in values] -def quota_usage_ready(probe, fleet, bucket, label): - """Wait for the server's authoritative usage snapshot before reading config.""" - deadline = time.monotonic() + 30 - attempts = 0 - while True: - attempts += 1 - value = fleet.record(label + '-readiness', probe.admin( - '/rustfs/admin/v3/quota-stats/' + bucket, [], b'', method='GET')) - response = value.get('response') or {} - if value.get('httpStatus') == 200: - require(response.get('bucket') == bucket and type(response.get('current_usage')) is int - and response.get('current_usage') >= 0, - 'H06 quota readiness returned an invalid owned-bucket usage snapshot') - return {'bucket': bucket, 'currentUsage': response['current_usage'], 'attempts': attempts} - require(value.get('httpStatus') == 503 - and isinstance(response, str) - and 'authoritative bucket usage is not available yet' in response, - 'H06 quota readiness returned an unreviewed response') - require(time.monotonic() < deadline, 'H06 authoritative bucket usage did not become ready within 30 seconds') - time.sleep(1) - - def quota_status(probe, fleet, buckets, label): + # Scanner stays disabled to preserve unselected defects. The reviewed + # config-only route checks bucket existence and reads persisted policy + # without requiring a scanner usage snapshot or reading target objects. result = {} for bucket in buckets: - quota_usage_ready(probe, fleet, bucket, label) - value = fleet.record(label, probe.admin('/rustfs/admin/v3/quota/' + bucket, [], b'', method='GET')) - response = value.get('response') or {} - require(value.get('httpStatus') == 200 and response.get('bucket') == bucket - and response.get('quota') is None and response.get('quota_type') == 'HARD', - 'H06 changed the owned bucket quota or queried a different logical bucket') - result[bucket] = {name: response[name] for name in ('bucket', 'quota', 'quota_type')} + for endpoint in probe.binding['endpoints']: + value = fleet.record(label, Probe(probe.binding, endpoint).admin( + REFERENCE['quotaConfig']['route'], [('bucket', bucket)], b'', method='GET')) + response = value.get('response') + require(value.get('httpStatus') == 200 and isinstance(response, dict) + and set(response) == {'quota', 'size', 'rate', 'requests'} + and all(type(response[name]) is int and response[name] == 0 for name in response), + 'H06 changed the owned bucket quota or queried a different logical bucket') + result[bucket] = dict(response) return result diff --git a/test_validation_heal_capacity.py b/test_validation_heal_capacity.py index 28c473af..49ef78d0 100644 --- a/test_validation_heal_capacity.py +++ b/test_validation_heal_capacity.py @@ -229,20 +229,32 @@ def __init__(self): self.assertIn("Authorization", outgoing.call_args.args[0].headers) self.assertEqual(probe.admin(path, [], b"", method="GET")["httpStatus"], 200) - def test_quota_stats_probe_is_an_exact_read_only_readiness_route(self): + def test_quota_config_probe_signs_the_exact_owned_bucket_query(self): probe = Probe(binding()) - path = "/rustfs/admin/v3/quota-stats/" + probe.binding["bucketPrefix"] + path = "/rustfs/admin/v3/get-bucket-quota" class Response(BytesIO): code = 200 def __init__(self): - super().__init__(b'{"bucket":"' + probe.binding["bucketPrefix"].encode() + b'","current_usage":0}') + super().__init__(b'{"quota":0,"size":0,"rate":0,"requests":0}') self.headers = {"Content-Length": str(self.getbuffer().nbytes)} with patch.object(probe, "credentials", return_value=("access", "secret")), \ patch.object(probe, "_open_admin", return_value=Response()) as outgoing: - observed = probe.admin(path, [], b"", method="GET") + observed = probe.admin(path, [("bucket", probe.binding['bucketPrefix'])], b"", method="GET") self.assertEqual(observed["httpStatus"], 200) - self.assertEqual(observed["response"]["bucket"], probe.binding["bucketPrefix"]) self.assertEqual(outgoing.call_args.args[0].get_method(), "GET") + self.assertEqual(outgoing.call_args.args[0].full_url, + probe.endpoint + path + '?bucket=' + probe.binding['bucketPrefix']) + self.assertIn('Authorization', outgoing.call_args.args[0].headers) + for query, method, body, auth in (([], 'GET', b'', 'signed'), + ([('bucket', 'foreign-bucket')], 'GET', b'', 'signed'), + ([('bucket', probe.binding['bucketPrefix'])] * 2, 'GET', b'', 'signed'), + ([('bucket', probe.binding['bucketPrefix'])], 'PUT', b'', 'signed'), + ([('bucket', probe.binding['bucketPrefix'])], 'GET', b'{}', 'signed'), + ([('bucket', probe.binding['bucketPrefix'])], 'GET', b'', 'anonymous')): + with self.subTest(query=query, method=method, body=body, auth=auth), \ + patch.object(probe, '_open_admin') as outgoing, self.assertRaises(ValueError): + probe.admin(path, query, body, method=method, auth=auth) + outgoing.assert_not_called() def test_quota_probe_refuses_foreign_buckets_unregistered_limits_and_auth(self): probe = Probe(binding()) @@ -258,8 +270,6 @@ def test_quota_probe_refuses_foreign_buckets_unregistered_limits_and_auth(self): patch.object(probe, "_open_admin") as outgoing, self.assertRaises(ValueError): probe.admin(route, [], body, method=method, auth=auth) outgoing.assert_not_called() - with self.assertRaises(ValueError): - probe.admin("/rustfs/admin/v3/quota-stats/foreign-bucket", [], b"", method="GET") if __name__ == "__main__": diff --git a/test_validation_heal_multi_pool.py b/test_validation_heal_multi_pool.py index f7594229..6468923f 100644 --- a/test_validation_heal_multi_pool.py +++ b/test_validation_heal_multi_pool.py @@ -84,18 +84,39 @@ def test_preserves_all_three_source_branches_and_real_two_pool_arguments(self): self.assertEqual(runtime_config('multi-pool')['RUSTFS_STORAGE_CLASS_STANDARD'], 'EC:2') with self.assertRaises(ValueError): volumes('gh-123-1-h2') - def test_quota_status_waits_for_authoritative_usage_before_reading_config(self): + def test_quota_status_reads_exact_owned_config_on_every_peer_without_scanner(self): probe, fleet = Mock(), Mock() + probe.binding = {'endpoints': {'node1': 'a', 'node2': 'b', 'node3': 'c', 'node4': 'd'}} fleet.record.side_effect = lambda label, value: value - probe.admin.side_effect = [ - {'httpStatus': 503, 'response': 'authoritative bucket usage is not available yet'}, - {'httpStatus': 200, 'response': {'bucket': self.buckets[0], 'current_usage': 0}}, - ] - with patch.object(runner.time, 'sleep') as sleep: - ready = runner.quota_usage_ready(probe, fleet, self.buckets[0], 'quota') - self.assertEqual(ready['attempts'], 2) - self.assertEqual(probe.admin.call_args_list[0].args[0], '/rustfs/admin/v3/quota-stats/' + self.buckets[0]) - sleep.assert_called_once_with(1) + config = {'quota': 0, 'size': 0, 'rate': 0, 'requests': 0} + with patch.object(runner, 'Probe') as peers: + peers.return_value.admin.return_value = {'httpStatus': 200, 'response': config} + self.assertEqual(runner.quota_status(probe, fleet, self.buckets, 'quota'), + {bucket: config for bucket in self.buckets}) + self.assertEqual(peers.call_count, 8) + self.assertEqual([call.args[1] for call in peers.call_args_list], list(probe.binding['endpoints']) * 2) + self.assertEqual(peers.return_value.admin.call_count, 8) + for index, call in enumerate(peers.return_value.admin.call_args_list): + self.assertEqual(call.args, ('/rustfs/admin/v3/get-bucket-quota', + [('bucket', self.buckets[index // 4])], b'')) + self.assertEqual(call.kwargs, {'method': 'GET'}) + + def test_quota_config_rejects_changed_limits_bool_and_peer_refusal(self): + probe, fleet = Mock(), Mock() + probe.binding = {'endpoints': {'node1': 'a', 'node2': 'b'}} + fleet.record.side_effect = lambda label, value: value + good = {'httpStatus': 200, 'response': {'quota': 0, 'size': 0, 'rate': 0, 'requests': 0}} + for drift in ('limit', 'bool', 'soft', 'missing', 'http', 'usage'): + changed = copy.deepcopy(good) + if drift == 'limit': changed['response'].update(quota=1024, size=1024, quotatype='hard') + elif drift == 'bool': changed['response']['quota'] = False + elif drift == 'soft': changed['response']['quotatype'] = 'soft' + elif drift == 'missing': changed['response'].pop('requests') + elif drift == 'http': changed['httpStatus'] = 503 + else: changed['response'] = {'bucket': self.buckets[0], 'quota': None, 'quota_type': 'HARD'} + with self.subTest(drift=drift), patch.object(runner, 'Probe') as peers: + peers.return_value.admin.side_effect = [good, changed] + with self.assertRaises(ValueError): runner.quota_status(probe, fleet, self.buckets, 'quota') def test_h5_pool_observations_accept_only_exact_signed_get_queries(self): binding = {'schemaVersion': 1, 'runId': 'gh-123-1-h5', 'jumpHostOnly': True, 'exclusiveHealService': True, From bcfae38f3b6d2737c80f85047214ea4e374c8de9 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 14:41:18 +0800 Subject: [PATCH 139/343] fix(validation): wait for multi-pool metadata convergence --- scripts/validation_heal/multi_pool.py | 31 ++++++++++++++++++++------- test_validation_heal_multi_pool.py | 16 ++++++++++++++ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/scripts/validation_heal/multi_pool.py b/scripts/validation_heal/multi_pool.py index a7448525..8a4334c8 100644 --- a/scripts/validation_heal/multi_pool.py +++ b/scripts/validation_heal/multi_pool.py @@ -11,6 +11,7 @@ import secrets import signal import sys +import time try: from .heal_probe import Probe, compare_inventories, payload, require, save @@ -305,6 +306,27 @@ def pool_status(probe, fleet, endpoint_expressions, label): return [{name: row[name] for name in ('id', 'cmdline', 'status', 'totalSize')} for row in values] +def quota_config(probe, fleet, bucket, label): + deadline = time.monotonic() + 30 + attempts = 0 + while True: + attempts += 1 + value = fleet.record(f'{label}-attempt-{attempts}', probe.admin( + REFERENCE['quotaConfig']['route'], [('bucket', bucket)], b'', method='GET')) + response = value.get('response') + if value.get('httpStatus') == 200: + require(isinstance(response, dict) and set(response) == {'quota', 'size', 'rate', 'requests'} + and all(type(response[name]) is int and response[name] == 0 for name in response), + 'H06 changed the owned bucket quota or queried a different logical bucket') + return dict(response) + same_bucket_not_ready = (value.get('httpStatus') == 404 and isinstance(response, str) + and 'NoSuchBucket' in response + and f'bucket not found: {bucket}' in response) + require(same_bucket_not_ready, 'H06 changed the owned bucket quota or queried a different logical bucket') + require(time.monotonic() < deadline, 'H06 owned bucket metadata did not converge on every peer within 30 seconds') + time.sleep(1) + + def quota_status(probe, fleet, buckets, label): # Scanner stays disabled to preserve unselected defects. The reviewed # config-only route checks bucket existence and reads persisted policy @@ -312,14 +334,7 @@ def quota_status(probe, fleet, buckets, label): result = {} for bucket in buckets: for endpoint in probe.binding['endpoints']: - value = fleet.record(label, Probe(probe.binding, endpoint).admin( - REFERENCE['quotaConfig']['route'], [('bucket', bucket)], b'', method='GET')) - response = value.get('response') - require(value.get('httpStatus') == 200 and isinstance(response, dict) - and set(response) == {'quota', 'size', 'rate', 'requests'} - and all(type(response[name]) is int and response[name] == 0 for name in response), - 'H06 changed the owned bucket quota or queried a different logical bucket') - result[bucket] = dict(response) + result[bucket] = quota_config(Probe(probe.binding, endpoint), fleet, bucket, label) return result diff --git a/test_validation_heal_multi_pool.py b/test_validation_heal_multi_pool.py index 6468923f..ddc55507 100644 --- a/test_validation_heal_multi_pool.py +++ b/test_validation_heal_multi_pool.py @@ -101,6 +101,22 @@ def test_quota_status_reads_exact_owned_config_on_every_peer_without_scanner(sel [('bucket', self.buckets[index // 4])], b'')) self.assertEqual(call.kwargs, {'method': 'GET'}) + def test_quota_config_retries_only_same_bucket_metadata_convergence(self): + probe, fleet = Mock(), Mock() + fleet.record.side_effect = lambda label, value: value + not_ready = {'httpStatus': 404, 'response': + 'NoSuchBucketbucket not found: ' + + self.buckets[0] + ''} + ready = {'httpStatus': 200, 'response': {'quota': 0, 'size': 0, 'rate': 0, 'requests': 0}} + probe.admin.side_effect = [not_ready, ready] + with patch.object(runner.time, 'sleep') as sleep: + self.assertEqual(runner.quota_config(probe, fleet, self.buckets[0], 'quota'), ready['response']) + sleep.assert_called_once_with(1) + with self.assertRaises(ValueError): + runner.quota_config(Mock(admin=Mock(return_value={'httpStatus': 404, 'response': + 'NoSuchBucketbucket not found: foreign'}), + binding={}), fleet, self.buckets[0], 'quota') + def test_quota_config_rejects_changed_limits_bool_and_peer_refusal(self): probe, fleet = Mock(), Mock() probe.binding = {'endpoints': {'node1': 'a', 'node2': 'b'}} From ab9426c867d7b106386ad6043867d044f048b920 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 14:47:10 +0800 Subject: [PATCH 140/343] fix(validation): bind H06 quota proof to management endpoint --- scripts/validation_heal/multi_pool.py | 7 +++---- test_validation_heal_multi_pool.py | 21 +++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/scripts/validation_heal/multi_pool.py b/scripts/validation_heal/multi_pool.py index 8a4334c8..bf207500 100644 --- a/scripts/validation_heal/multi_pool.py +++ b/scripts/validation_heal/multi_pool.py @@ -56,7 +56,7 @@ REFERENCE['quotaConfig'] = {'sourceFile': 'rustfs/src/admin/handlers/quota.rs', 'sourceSha256': '6a2b062d3d1c4cd81d934c570d21cb44dcd2c1883f6a85d2265e01b466924f5c', 'route': '/rustfs/admin/v3/get-bucket-quota', 'query': 'one exact owned bucket', - 'assertion': 'unchanged no-quota config on every original peer; no authoritative usage claim'} + 'assertion': 'unchanged no-quota config on the bound management endpoint; no authoritative usage claim'} def fixture_call(recorder, ledger, operation, params, body=None): @@ -323,7 +323,7 @@ def quota_config(probe, fleet, bucket, label): and 'NoSuchBucket' in response and f'bucket not found: {bucket}' in response) require(same_bucket_not_ready, 'H06 changed the owned bucket quota or queried a different logical bucket') - require(time.monotonic() < deadline, 'H06 owned bucket metadata did not converge on every peer within 30 seconds') + require(time.monotonic() < deadline, 'H06 owned bucket metadata did not converge on the bound management endpoint within 30 seconds') time.sleep(1) @@ -333,8 +333,7 @@ def quota_status(probe, fleet, buckets, label): # without requiring a scanner usage snapshot or reading target objects. result = {} for bucket in buckets: - for endpoint in probe.binding['endpoints']: - result[bucket] = quota_config(Probe(probe.binding, endpoint), fleet, bucket, label) + result[bucket] = quota_config(probe, fleet, bucket, label) return result diff --git a/test_validation_heal_multi_pool.py b/test_validation_heal_multi_pool.py index ddc55507..22caa35c 100644 --- a/test_validation_heal_multi_pool.py +++ b/test_validation_heal_multi_pool.py @@ -84,21 +84,22 @@ def test_preserves_all_three_source_branches_and_real_two_pool_arguments(self): self.assertEqual(runtime_config('multi-pool')['RUSTFS_STORAGE_CLASS_STANDARD'], 'EC:2') with self.assertRaises(ValueError): volumes('gh-123-1-h2') - def test_quota_status_reads_exact_owned_config_on_every_peer_without_scanner(self): + def test_quota_status_reads_exact_owned_config_on_bound_endpoint_without_scanner(self): probe, fleet = Mock(), Mock() probe.binding = {'endpoints': {'node1': 'a', 'node2': 'b', 'node3': 'c', 'node4': 'd'}} fleet.record.side_effect = lambda label, value: value config = {'quota': 0, 'size': 0, 'rate': 0, 'requests': 0} - with patch.object(runner, 'Probe') as peers: - peers.return_value.admin.return_value = {'httpStatus': 200, 'response': config} - self.assertEqual(runner.quota_status(probe, fleet, self.buckets, 'quota'), - {bucket: config for bucket in self.buckets}) - self.assertEqual(peers.call_count, 8) - self.assertEqual([call.args[1] for call in peers.call_args_list], list(probe.binding['endpoints']) * 2) - self.assertEqual(peers.return_value.admin.call_count, 8) - for index, call in enumerate(peers.return_value.admin.call_args_list): + probe.admin.return_value = {'httpStatus': 200, 'response': config} + self.assertEqual(runner.quota_status(probe, fleet, self.buckets, 'quota'), + {bucket: config for bucket in self.buckets}) + self.assertEqual(probe.admin.call_count, 2) + self.assertEqual(fleet.record.call_count, 2) + calls = probe.admin.call_args_list + self.assertEqual([call.args[0] for call in calls], [ + '/rustfs/admin/v3/get-bucket-quota', '/rustfs/admin/v3/get-bucket-quota']) + for index, call in enumerate(calls): self.assertEqual(call.args, ('/rustfs/admin/v3/get-bucket-quota', - [('bucket', self.buckets[index // 4])], b'')) + [('bucket', self.buckets[index])], b'')) self.assertEqual(call.kwargs, {'method': 'GET'}) def test_quota_config_retries_only_same_bucket_metadata_convergence(self): From 75b245189cf7e22a31802bb509c8c2edcf4264dd Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 16:20:28 +0800 Subject: [PATCH 141/343] docs(validation): reconcile latest live evidence --- docs/validation-migration-map.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 93ef0d41..3a74e694 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `e8a87ab3facd9c401ec7cce1fa63bdc82ca079e8` is pushed to auto-testing PR #113; its targeted local contract suite passes 498 tests with eight macOS skips. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. No live PASS is added for either failed run. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. +Current reconciliation: helper `ab9426c867d7b106386ad6043867d044f048b920` is pushed to auto-testing PR #113; its targeted local contract suite passes 502 tests with eight macOS skips, while Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. From d502d3eec139d17b2587c883fce301c40a2741a5 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 16:52:19 +0800 Subject: [PATCH 142/343] feat(validation): migrate A09 nolock safety executor --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 2 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/nolock.py | 528 ++++++++++++++++++ scripts/validation_heal/run_case.sh | 1 + test_validation_heal_nolock.py | 71 +++ 7 files changed, 606 insertions(+), 4 deletions(-) create mode 100644 scripts/validation_heal/nolock.py create mode 100644 test_validation_heal_nolock.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 81fbc028..8e85faa0 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -42,6 +42,7 @@ on: - test_validation_heal_options.py - test_validation_heal_environment.py - test_validation_heal_queue_pressure.py + - test_validation_heal_nolock.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -97,6 +98,7 @@ on: - test_validation_heal_options.py - test_validation_heal_environment.py - test_validation_heal_queue_pressure.py + - test_validation_heal_nolock.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -134,7 +136,7 @@ jobs: sudo apt-get update sudo apt-get install --no-install-recommends -y nftables fi - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 3a74e694..0fec6ea9 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `ab9426c867d7b106386ad6043867d044f048b920` is pushed to auto-testing PR #113; its targeted local contract suite passes 502 tests with eight macOS skips, while Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. +Current reconciliation: A09 executor changes are in the working tree and will be recorded in the next helper commit; the preceding helper `75b245189cf7e22a31802bb509c8c2edcf4264dd` is pushed to auto-testing PR #113. The current local contract suite runs 508 tests with eight macOS skips after adding A09 contracts; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. The matrix is now 135 dispatchable IDs (74 non-Heal and 61 Heal), with 23 Heal executors still intentionally unimplemented. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index c36f9dce..8bf052aa 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.admin-authorization|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 6fbe1b3c..9d7271ac 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.admin-nolock-safety", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/nolock.py b/scripts/validation_heal/nolock.py new file mode 100644 index 00000000..c65e08bb --- /dev/null +++ b/scripts/validation_heal/nolock.py @@ -0,0 +1,528 @@ +#!/usr/bin/env python3 +"""A09 public-call races; a blocked native fence never certifies the case.""" + +from __future__ import annotations + +import argparse +import copy +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +from urllib.parse import quote +import uuid + +try: + from .heal_probe import Probe, canonical, compare_inventories, payload, require, save + from .heal_workload import Recorder, verify_seed + from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .heal_fault import bounded_fault_id + from .shards import files, map_target, validate_repaired_physical + from .cursor import validate_missing_batch, verify_part_manifest + from .versions import public_version_id +except ImportError: + from heal_probe import Probe, canonical, compare_inventories, payload, require, save + from heal_workload import Recorder, verify_seed + from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from heal_fault import bounded_fault_id + from shards import files, map_target, validate_repaired_physical + from cursor import validate_missing_batch, verify_part_manifest + from versions import public_version_id + + +CASE_ID = "heal.admin-nolock-safety" +BRANCHES = ("overwrite-nolock", "delete-nolock", "native-lock-fence") +BLACK_BOX = BRANCHES[:2] +OBJECTS = 257 +SIZE = 257 * 1024 + 123 +TARGET_KEY = "objects/0256.bin" +REFERENCE_KEY = "objects/0000.bin" +REFERENCE = { + "candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", + "sourceFiles": { + "rustfs/src/admin/handlers/heal.rs": "7e4007c29a8329ced8e48b88111400e79cffd845bd6ddc6243b79f3cd48e8199", + "crates/heal/src/heal/channel.rs": "7f669ca43b9559db9b97fba6b419a6c76a94d76acd8526db132d74b7c534d6b0"}, + "requested": {**BASE_OPTIONS, "recursive": True, "nolock": True, "pool": None, "set": None}, + "effective": {**BASE_OPTIONS, "recursive": True, "nolock": False, "pool": None, "set": None}, + "nativeFence": "pinned release has no reviewed e2e-test-hooks artifact or exact Admin lock/commit barrier", +} + + +class WindowNotObserved(ValueError): + """No observable client overlap; this is not a product success.""" + + +class RaceRecorder(Recorder): + """Keep unresolved mutations and exact SDK receipts after any assertion.""" + def __init__(self, *args, versioned, **kwargs): + super().__init__(*args, **kwargs) + self.versioned, self.pending, self.seen = versioned, None, set() + + def call(self, operation, params, body=None, **kwargs): + self.pending = {"operation": operation, "params": copy.deepcopy(params)} + result = super().call(operation, params, body, **kwargs) + version = result.get("VersionId") + if operation == "put_object": + require(isinstance(result.get("ETag"), str) and result["ETag"] + and result.get("DeleteMarker") in {None, False}, "A09 PUT ACK lacks ETag or changed kind") + if self.versioned: + require(isinstance(version, str) and str(uuid.UUID(version)) == version and uuid.UUID(version).int + and (params["Key"], version) not in self.seen, "A09 PUT ACK has an invalid/reused VersionId") + self.seen.add((params["Key"], version)) + else: + require(version in {None, "null"}, "A09 never-versioned PUT created a real version") + elif operation == "delete_object": + require(self.versioned and params.get("VersionId") in { + value for key, value in self.seen if key == params["Key"]} + and version == params["VersionId"] and result.get("DeleteMarker") in {None, False}, + "A09 DELETE ACK changed the exact known data-version identity") + self.pending = None + return result + + +def settings(observed): + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), + "A09 status has an unknown or failed HTTP response") + value = observed["response"] + require(value.get("summary") in {"running", "finished"}, "A09 original task disappeared, stopped or failed") + # Canonical JSON distinguishes bool/int and preserves raw null selectors. + require(canonical(value.get("settings")) == canonical(REFERENCE["effective"]), + "A09 Admin nolock was not safely normalized or changed unrelated options") + return value["settings"] + + +def sample_overlap(ack, observed, path, token): + foreground = ack.get("observation") or {} + settings(observed) + value = observed["response"] + a, z = foreground.get("callStartedMonotonicNs"), foreground.get("callFinishedMonotonicNs") + b, c = observed.get("callStartedMonotonicNs"), observed.get("callFinishedMonotonicNs") + processed = (value.get("outcome") or {}).get("counters", {}).get("processed") + return (foreground.get("acknowledged") is True and foreground.get("clientCallInvoked") is True + and foreground.get("clockDomain") and foreground["clockDomain"] == observed.get("clockDomain") + and observed.get("path") == path and [tuple(row) for row in observed.get("query", [])] == [("clientToken", token)] + and all(type(tick) is int for tick in (a, z, b, c)) and 0 < a < b < c < z + and value["summary"] == "running" and type(processed) is int and processed > 0) + + +def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): + require(branch in BLACK_BOX and type(count) is int and 3 <= count <= OBJECTS, "A09 unregistered fixture") + recorder.call("create_bucket", {"Bucket": bucket}) + if branch == "delete-nolock": + recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) + entries = [] + keys = [f"objects/{index:04d}.bin" for index in range(count - 1)] + [TARGET_KEY] + for key in keys: + versions = 3 if key == TARGET_KEY and branch == "delete-nolock" else 1 + for generation in range(versions): + nonce = f"{bucket}/{key}/{generation}" + body = payload(nonce, SIZE) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": "A09", "nonce": nonce}, + "Tagging": "suite=heal&fixture=A09"}, body) + entries.append({"key": key, "versionId": public_version_id(response.get("VersionId")), + "isLatest": generation == versions - 1, "deleteMarker": False, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + return entries + + +def operation_plan(bucket, branch, expected): + require(branch in BLACK_BOX, "A09 unregistered foreground branch") + if branch == "overwrite-nolock": + # Three distinct serial commits; the final independent reference was + # written before injection, allowing a coding-index byte comparison. + return [{"operation": "put_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, + "ContentType": "application/octet-stream", "Metadata": {"nonce": str(index)}, + "Tagging": "race=overwrite"}, + "nonce": f"{bucket}/overwrite/{index}" if index < 2 else f"{bucket}/{REFERENCE_KEY}/0"} + for index in range(3)] + target = [row for row in expected if row["key"] == TARGET_KEY] + require(len(target) == 3 and len({row["versionId"] for row in target}) == 3 + and target[-1]["isLatest"], "A09 DV lacks three ACK-ordered versions") + return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, + "VersionId": target[0]["versionId"]}}] + + +def final_inventory(baseline, branch, plan, responses): + require(len(plan) == len(responses) == (3 if branch == "overwrite-nolock" else 1), + "A09 foreground plan/ACK sequence is incomplete") + result = copy.deepcopy(baseline) + if branch == "overwrite-nolock": + entry = next(row for row in result["entries"] if row["key"] == TARGET_KEY) + step, ack = plan[-1], responses[-1] + body = payload(step["nonce"], SIZE) + require(isinstance(ack.get("ETag"), str) and ack["ETag"] + and ack.get("VersionId") in {None, "null"}, "A09 last PUT ACK lost its identity") + entry.update(payload={"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}, + tags=[{"Key": "race", "Value": "overwrite"}]) + entry["properties"].update(ETag=ack["ETag"], ContentLength=len(body), ContentType="application/octet-stream", + Metadata=step["params"]["Metadata"]) + else: + version = plan[0]["params"]["VersionId"] + require(responses[0].get("VersionId") == version, "A09 DELETE response changed the selected VersionId") + result["entries"] = [row for row in result["entries"] + if (row["key"], row["versionId"]) != (TARGET_KEY, version)] + require(len(result["entries"]) + 1 == len(baseline["entries"]), "A09 deleted a foreign/missing version") + return result + + +def object_scene(snapshot, bucket, key): + return {node: {**value, "files": [row for row in value["files"] + if row["path"].startswith(f"{bucket}/{key}/")]} + for node, value in snapshot.items()} + + +def map_fixture(snapshot, bucket, expected): + mapped = {(row["key"], row["versionId"]): map_target( + object_scene(snapshot, bucket, row["key"]), bucket, row["key"], "data", 1, + row["versionId"], latest_required=False) for row in expected} + require(len(mapped) == len(expected), "A09 physical map omitted/duplicated an ACK") + return mapped + + +def physical_transition(before, after, bucket, mapped, final, branch): + """Prove current ACK authority before allowing any GET or backup rescue.""" + require(all(before[node]["formats"] == after[node]["formats"] for node in before), + "A09 race changed an original disk format") + known = {row["key"] for row in final["entries"]} + require(all(any(row["path"].startswith(f"{bucket}/{key}/") for key in known) + for value in after.values() for row in value["files"]), "A09 created an unacknowledged physical key") + for (key, version), target in mapped.items(): + if key == TARGET_KEY: continue + validate_repaired_physical(object_scene(before, bucket, key), object_scene(after, bucket, key), target) + survivors = map_fixture(after, bucket, final["entries"]) + original_target = [value for (key, _), value in mapped.items() if key == TARGET_KEY] + current_target = [value for (key, _), value in survivors.items() if key == TARGET_KEY] + if branch == "overwrite-nolock": + require(len(original_target) == len(current_target) == 1 + and original_target[0]["dataDir"] != current_target[0]["dataDir"], + "A09 overwrite retained the old physical generation") + reference = next(target for (key, _), target in mapped.items() if key == REFERENCE_KEY) + require({part["index"]: (part["sha256"], part["size"]) for part in reference["cohort"]} + == {part["index"]: (part["sha256"], part["size"]) for part in current_target[0]["cohort"]}, + "A09 last ACK shard bytes/coding indices differ from the independent payload reference") + else: + remaining = {(row["key"], row["versionId"]) for row in final["entries"]} + removed = [value for pair, value in mapped.items() if pair not in remaining] + require(len(removed) == 1, "A09 delete transition lacks one exact deleted version") + for target in current_target: + original = mapped[TARGET_KEY, target["versionId"]] + require(target["logical"] == original["logical"] + and [(row["index"], row["sha256"], row["size"]) for row in target["cohort"]] + == [(row["index"], row["sha256"], row["size"]) for row in original["cohort"]], + "A09 delete changed another retained ACK generation") + current_dirs = {target["dataDir"] for target in current_target} + for value in after.values(): + for row in value["files"]: + prefix = f"{bucket}/{TARGET_KEY}/" + if not row["path"].startswith(prefix): continue + suffix = row["path"][len(prefix):] + require(suffix == "xl.meta" or suffix.split("/")[0] in current_dirs, + "A09 resurrected an old/deleted data directory or left an unbound target leaf") + return survivors, {"allControlShardsRestored": True, "lastACKGenerationVerified": True, + "physicalProofBeforeGETOrRescue": True, "oldVersionResurrected": False} + + +def terminal_proof(status, bucket, before, final): + outcome = status.get("outcome") or {} + require(status.get("summary") == "finished" and outcome.get("execution", {}).get("state") == "completed" + and outcome.get("coverage") == "complete", "A09 task lacks complete successful terminal coverage") + old = {(row["key"], row["versionId"]) for row in before["entries"]} + new = {(row["key"], row["versionId"]) for row in final["entries"]} + counters = outcome.get("counters") or {} + require(all(type(counters.get(key)) is int and counters[key] >= 0 + for key in ("processed", "healed", "unchanged", "failed", "unknown", "skipped", "attemptFailures")) + and all(counters[key] == 0 for key in ("failed", "unknown", "skipped", "attemptFailures")) + and counters.get("overflowed") is False and counters["processed"] == counters["healed"] + counters["unchanged"] + and len(old & new) <= counters["processed"] <= len(old | new), + "A09 terminal responsibility exceeds ACK bounds or hides unknown/failed work") + receipts = outcome.get("objects") + require(isinstance(receipts, list) and 0 < len(receipts) <= 128 + and outcome.get("objectsTruncated") is (counters["processed"] > 128), + "A09 confused truncated diagnostics with its independent complete ACK manifest") + seen, incarnation = set(), None + for receipt in receipts: + identity = receipt.get("identity") or {} + pair = identity.get("object"), public_version_id(identity.get("versionId")) + require(identity.get("bucket") == bucket and pair in old | new and pair not in seen + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0 + and isinstance(identity.get("bucketIncarnationId"), str) and identity["bucketIncarnationId"] + and receipt.get("disposition", {}).get("state") in {"repaired", "verified_healthy"}, + "A09 diagnostics changed version/incarnation/scope or claimed unknown/false health") + incarnation = incarnation or identity["bucketIncarnationId"] + require(incarnation == identity["bucketIncarnationId"], "A09 diagnostics mix incarnations") + seen.add(pair) + return {"minimumACKResponsibilities": len(old & new), "maximumACKResponsibilities": len(old | new), + "counters": counters, "exactRaceEnumerationClaimed": False} + + +def observe(probe, fleet, path, token): + value = fleet.record("original-token-status", probe.admin(path, [("clientToken", token)], b"")) + settings(value) + return value + + +def wait_running(probe, fleet, path, token, total): + deadline = time.monotonic() + 30 + while True: + observed = observe(probe, fleet, path, token) + value = observed["response"] + processed = (value.get("outcome") or {}).get("counters", {}).get("processed") + if value["summary"] == "finished": + raise WindowNotObserved("A09 task finished before its partial-progress race window") + if type(processed) is int and 0 < processed < total: return observed + if time.monotonic() >= deadline: + raise WindowNotObserved("A09 nonzero partial progress was not observed within 30 seconds") + time.sleep(.05) + + +def wait_terminal(probe, fleet, path, token): + deadline = time.monotonic() + 240 + while True: + current = observe(probe, fleet, path, token) + if current["response"]["summary"] == "finished": return current["response"] + require(time.monotonic() < deadline, "A09 original task exceeded its terminal budget") + time.sleep(.2) + + +def foreground(recorder, plan): + responses = [] + for step in plan: + body = payload(step["nonce"], SIZE) if step["operation"] == "put_object" else None + responses.append(recorder.call(step["operation"], step["params"], body)) + return responses + + +def race(recorder, plan, probe, fleet, path, token): + samples = [] + # Only the worker mutates Recorder state. All Fleet.record writes run on + # the controller, so evidence sequence allocation never races. + with ThreadPoolExecutor(max_workers=1) as pool: + worker = pool.submit(foreground, recorder, plan) + deadline = time.monotonic() + 120 + try: + while not worker.done() and time.monotonic() < deadline: + samples.append(observe(probe, fleet, path, token)) + time.sleep(.01) + except BaseException: + recorder.mutations.poison("status observation failed; drain in-flight mutations only") + raise + if not worker.done(): recorder.mutations.poison("foreground deadline; drain only") + responses = worker.result(timeout=60) + acks = [json.loads(path.read_text()) for path in sorted(recorder.output.glob("*-ack.json"))] + require(len(acks) == len(plan) and all(row.get("operationCheck") == "PASS" for row in acks), + "A09 lacks every durable foreground ACK") + overlaps = [any(sample_overlap(ack, sample, path, token) for sample in samples) for ack in acks] + fleet.record("public-call-overlap-proof", {"everyMutationOverlapped": all(overlaps), "mutations": overlaps, + "nativeFenceProved": False, "serverCommitOverlapProved": False}) + return responses, all(overlaps) + + +def run_branch(args, topology, nodes, branch): + directory = args.evidence / topology / branch + directory.mkdir(parents=True, exist_ok=False) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="manual") + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == REFERENCE["candidateCommit"], "A09 candidate contract is not reviewed") + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 512, "maxInventoryBytes": 512 * 1024**2} + fleet.record("binding", binding) + fleet.record("source-contract", {**REFERENCE, "branch": branch, "objectCount": OBJECTS, "objectSize": SIZE, + "requiredPUTs": 3 if branch == BLACK_BOX[0] else 0, "serverCommitOverlapProved": False}) + probe, active, recorder, writer = Probe(binding), [], None, None + result = {"topology": topology, "branch": branch, "fixture": "DL" if branch == BLACK_BOX[0] else "DV", + "verdict": "RUNNING", "cleanup": "PENDING", "canonicalCasePassClaimed": False} + changed_target_registered, target_authority = None, False + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) + recorder = RaceRecorder(probe, directory / "seed-ACKs", versioned=branch == BLACK_BOX[1]) + expected = seed_fixture(recorder, bucket, branch) + baseline = fleet.record("complete-baseline-S3", probe.inventory(bucket, unversioned_current_tags=branch == BLACK_BOX[0])) + verify_seed(expected, baseline) + plan = operation_plan(bucket, branch, expected) + fleet.record("frozen-foreground-plan", plan) + fleet.stop() + before = fleet.snapshot("complete-baseline-physical", quiescent=True) + mapped = map_fixture(before, bucket, expected) + verify_part_manifest(fleet, list(mapped.values()), "baseline-streaming", count=len(mapped)) + earliest_target = next(entry for entry in expected if entry["key"] == TARGET_KEY) + targets = [mapped[row["key"], row["versionId"]] for row in expected + if row["key"] != TARGET_KEY or row["versionId"] == earliest_target["versionId"]] + require(len(targets) == OBJECTS, "A09 fault plan omitted a key or changed its source dimension") + receipts = [] + for index, target in enumerate(targets): + params = {"disk": target["disk"], "faultId": bounded_fault_id("A09", branch, index)} + fleet.faults.append((target["node"], params)) + if target["key"] == TARGET_KEY: changed_target_registered = target["node"], params + receipts.append(fleet.record("exact-missing-part", {"node": target["node"], **fleet.remote( + target["node"], "apply-fault", **params, relativePath=target["relativePath"], + expectedSha256=target["sha256"], kind="missing")})) + faulted = fleet.snapshot("complete-fault-physical", quiescent=True) + fleet.record("fault-batch-proof", validate_missing_batch(before, faulted, targets, receipts)) + fleet.start() + require(physical_content(faulted) == physical_content(fleet.snapshot("attribution-before-START")), + "NOT_OBSERVED: A09 missing shards changed before manual admission") + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + fleet.record("prepared-Admin-request", {"path": path, "query": [], "options": REFERENCE["requested"]}) + start = fleet.record("original-Admin-start", probe.admin(path, [], json.dumps(REFERENCE["requested"]).encode())) + token = start.get("response", {}).get("clientToken") if isinstance(start.get("response"), dict) else None + require(start.get("httpStatus") == 200 and isinstance(token, str) and token, "A09 admission lacks an original token") + active.append((path, token)) + wait_running(probe, fleet, path, token, len(expected)) + witness = fleet.snapshot("target-missing-before-foreground") + target = next(target for target in targets if target["key"] == TARGET_KEY) + if (target["node"], target["disk"], target["relativePath"]) in files(witness): + raise WindowNotObserved("A09 old target already repaired before foreground mutations") + writer = RaceRecorder(probe, directory / "foreground-ACKs", versioned=branch == BLACK_BOX[1]) + writer.seen.update((row["key"], row["versionId"]) for row in expected) + responses, overlapped = race(writer, plan, probe, fleet, path, token) + final = fleet.record("ACK-derived-final-manifest", final_inventory(baseline, branch, plan, responses)) + terminal = wait_terminal(probe, fleet, path, token) + active.remove((path, token)) + for _ in range(2): + repeated = observe(probe, fleet, path, token)["response"] + require(canonical(repeated.get("outcome")) == canonical(terminal.get("outcome")) + and repeated["summary"] == terminal["summary"], "A09 terminal responsibility changed on repetition") + # Collect physical evidence even if terminal counters fail. Old target + # bytes are never rescued over a successful overwrite or exact delete. + fleet.stop() + after = fleet.snapshot("product-physical-before-GET-or-rescue", quiescent=True) + survivors, physical = physical_transition(before, after, bucket, mapped, final, branch) + fleet.record("last-ACK-physical-proof", physical) + verify_part_manifest(fleet, list(survivors.values()), "final-streaming", count=len(survivors)) + target_authority = True + # Every missing-part registration is now superseded by product output; + # leaving it in Fleet.faults would make cleanup attempt a blind backup + # rescue over the new ACK generation. + fleet.faults.clear() + fleet.record("obsolete-target-fault-retired", {"fault": changed_target_registered, + "reason": "complete last-ACK physical authority proves the original generation obsolete", + "oldBytesRestored": False, "backupRetainedUntilCleanup": True}) + fleet.start() + for endpoint in binding["endpoints"]: + current = fleet.record("all-peer-final-S3", {"endpoint": endpoint, **Probe(binding, endpoint).inventory( + bucket, unversioned_current_tags=branch == BLACK_BOX[0])}) + compare_inventories(final, current) + if branch == BLACK_BOX[1]: + negative = fleet.record("all-peer-deleted-version-GET", Probe(binding, endpoint).s3("get_object", plan[0]["params"])) + response = negative.get("response") or {} + require(negative.get("acknowledged") is False and negative.get("resultUnknown") is False + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404 + and response.get("Error", {}).get("Code") == "NoSuchVersion", + "A09 deleted exact VersionId revived or failed for another reason") + fleet.stop() + require(physical_content(after) == physical_content(fleet.snapshot("physical-after-S3", quiescent=True)), + "A09 final GET changed the already-proven physical result") + result["terminal"] = terminal_proof(terminal, bucket, baseline, final) + if not overlapped: raise WindowNotObserved("A09 did not observe every required SDK call overlap") + result.update(verdict="PASS", physical=physical, publicCallOverlapObserved=True, + nativeFenceProved=False, serverCommitOverlapProved=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="NOT_OBSERVED" if isinstance(exc, WindowNotObserved) else "FAIL", + failure={"type": type(exc).__name__, "message": str(exc)}) + try: + fleet.stop() + fleet.snapshot("retained-failure-scene", quiescent=True) + except Exception as capture: fleet.record("failure-scene-error", {"error": str(capture)}) + finally: + for path, token in active: + try: fleet.record("cancel-original-token", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: fleet.record("cancel-error", {"error": str(exc)}) + unresolved = any(item is not None and item.pending is not None for item in (recorder, writer)) + mutated = writer is not None and writer.sequence > 0 + if unresolved or mutated and not target_authority: + try: fleet.stop() + except Exception as exc: fleet.record("retain-stop-error", {"error": str(exc)}) + fleet.record("last-ACK-rescue-required", {"state": "DIRTY", "namespacesRetained": True, + "unresolvedMutation": unresolved, "lastACKPhysicalAuthorityVerified": target_authority, + "originalGenerationBlindRestorationProhibited": True}) + result["cleanup"] = "FAIL" + else: + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("branch-result", result) + return result + + +def native_blocked(candidate, topology): + require(topology in {"H1", "H2"}, "A09 native prerequisite lost its source topology") + return {"topology": topology, "branch": BRANCHES[-1], "verdict": "BLOCKED", "cleanup": "PASS", + "candidateCommit": candidate["source_sha"], "canonicalCasePassClaimed": False, + "reason": REFERENCE["nativeFence"], "nativeFenceProved": False, "fleetExecutionClaimed": False} + + +def complete_results(results): + wanted = {(topology, branch) for topology in ("H1", "H2") for branch in BRANCHES} + actual = {(row["topology"], row["branch"]) for row in results} + require(len(actual) == len(results) and actual <= wanted, "A09 duplicate or foreign topology/branch") + # No blocked, omitted or best-effort native branch can complete A09. + return actual == wanted and all(row["verdict"] == row["cleanup"] == "PASS" for row in results) and all( + row.get("nativeFenceProved") is True for row in results if row["branch"] == BRANCHES[-1]) + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f"A09 interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ("candidate", "rc-bin", "evidence", "tool-directory"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--case-id", choices=[CASE_ID], required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + args = parser.parse_args() + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "A09 executor omitted a source branch") + require(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha) and re.fullmatch(r"[0-9a-f]{40}", args.testing_commit) + and re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")), "A09 immutable identities required") + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(nodes) == len(set(nodes)) + and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), + "A09 requires four distinct current Heal nodes") + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == REFERENCE["candidateCommit"], "A09 candidate contract not reviewed") + results = [native_blocked(candidate, topology) for topology in ("H1", "H2")] + try: + stop = False + for topology in ("H1", "H2"): + for branch in BLACK_BOX: + row = run_branch(args, topology, nodes[:1] if topology == "H1" else nodes[:4], branch) + results.append(row) + if row["cleanup"] != "PASS": + stop = True + break + if stop: + break + finally: + complete = complete_results(results) + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "complete": complete, + "cleanup": "PASS" if all(row["cleanup"] == "PASS" for row in results) else "FAIL", + "canonicalCasePassClaimed": False, "nativePrerequisitesBlocked": True}) + require(complete, "A09 native prerequisites remain BLOCKED; every H1/H2 branch stays in the denominator") + + +if __name__ == "__main__": + try: main() + except Exception as exc: + print(f"A09 validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 5a5ebf66..239e7fc1 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -44,6 +44,7 @@ case "${case_id}" in heal.multi-pool-target-isolation) driver=multi_pool.py ;; heal.conflicting-subquorum-authority) driver=authority_runner.py ;; heal.dangling-and-orphan-safety) driver=orphan.py ;; + heal.admin-nolock-safety) driver=nolock.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal_nolock.py b/test_validation_heal_nolock.py new file mode 100644 index 00000000..4d2bb8eb --- /dev/null +++ b/test_validation_heal_nolock.py @@ -0,0 +1,71 @@ +import copy +import json +from pathlib import Path +import unittest + +from scripts.validation_heal import nolock +from scripts.validation_heal.matrix import AUTOMATIC_CASES, select + + +class NolockTests(unittest.TestCase): + def test_source_branches_and_fixture_mapping_are_preserved(self): + recipe = select(nolock.CASE_ID)[0] + self.assertEqual(recipe["variants"], list(nolock.BRANCHES)) + self.assertIn(nolock.CASE_ID, AUTOMATIC_CASES) + self.assertEqual(nolock.REFERENCE["requested"]["nolock"], True) + self.assertEqual(nolock.REFERENCE["effective"]["nolock"], False) + self.assertIsNone(nolock.REFERENCE["requested"]["pool"]) + self.assertIsNone(nolock.REFERENCE["effective"]["set"]) + + def test_settings_require_exact_lock_safe_normalization(self): + status = {"httpStatus": 200, "response": {"summary": "running", "settings": nolock.REFERENCE["effective"]}} + self.assertEqual(nolock.settings(status), nolock.REFERENCE["effective"]) + for changed in ( + {**nolock.REFERENCE["effective"], "nolock": True}, + {**nolock.REFERENCE["effective"], "recursive": False}, + {**nolock.REFERENCE["effective"], "pool": 0}, + {**nolock.REFERENCE["effective"], "set": False}, + ): + with self.subTest(changed=changed), self.assertRaises(ValueError): + nolock.settings({"httpStatus": 200, "response": {"summary": "finished", "settings": changed}}) + + def test_blocked_native_fence_is_kept_in_the_denominator(self): + native = nolock.native_blocked({"source_sha": nolock.REFERENCE["candidateCommit"]}, "H1") + rows = [ + {"topology": topology, "branch": branch, "verdict": "PASS", "cleanup": "PASS"} + for topology in ("H1", "H2") for branch in nolock.BLACK_BOX + ] + self.assertFalse(nolock.complete_results([*rows, native])) + self.assertFalse(nolock.complete_results(rows)) + self.assertFalse(nolock.complete_results([*rows[:-1], native])) + failed = copy.deepcopy(rows) + failed[0]["verdict"] = "FAIL" + self.assertFalse(nolock.complete_results([*failed, native])) + self.assertFalse(native["canonicalCasePassClaimed"]) + + def test_run_case_dispatch_and_source_contract(self): + dispatch = Path("scripts/validation_heal/run_case.sh").read_text() + self.assertIn("heal.admin-nolock-safety) driver=nolock.py", dispatch) + catalog = json.loads(Path("validation-cases.json").read_text()) + source = {case["id"]: case for module in catalog["modules"] for case in module["cases"]}[nolock.CASE_ID] + self.assertEqual(source["probes"]["designCode"], "A09") + self.assertEqual(source["probes"]["topology"], "H1,H2") + + def test_foreground_plan_keeps_three_overwrites_or_one_exact_delete(self): + expected = [{"key": nolock.TARGET_KEY, "versionId": value, "isLatest": value == "v2", + "deleteMarker": False} for value in ("v0", "v1", "v2")] + overwrites = nolock.operation_plan("rv-heal-0123456789abcdef", "overwrite-nolock", expected) + self.assertEqual(len(overwrites), 3) + self.assertEqual([step["params"]["Metadata"]["nonce"] for step in overwrites], ["0", "1", "2"]) + deletion = nolock.operation_plan("rv-heal-0123456789abcdef", "delete-nolock", expected) + self.assertEqual(deletion[0]["params"]["VersionId"], "v0") + + def test_native_blocked_rows_cannot_become_a_canonical_pass(self): + row = nolock.native_blocked({"source_sha": nolock.REFERENCE["candidateCommit"]}, "H2") + self.assertEqual(row["verdict"], "BLOCKED") + self.assertFalse(row["canonicalCasePassClaimed"]) + self.assertFalse(row["nativeFenceProved"]) + + +if __name__ == "__main__": + unittest.main() From 0cef8a8b475dc068568aca9039197cf6ff3e212c Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 17:10:19 +0800 Subject: [PATCH 143/343] feat(validation): migrate G01 and G03 version races --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-migration-map.md | 2 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/race.py | 552 ++++++++++++++++++ scripts/validation_heal/run_case.sh | 1 + test_validation_heal_race.py | 51 ++ 7 files changed, 610 insertions(+), 4 deletions(-) create mode 100644 scripts/validation_heal/race.py create mode 100644 test_validation_heal_race.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 8e85faa0..4b95c78f 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -43,6 +43,7 @@ on: - test_validation_heal_environment.py - test_validation_heal_queue_pressure.py - test_validation_heal_nolock.py + - test_validation_heal_race.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -99,6 +100,7 @@ on: - test_validation_heal_environment.py - test_validation_heal_queue_pressure.py - test_validation_heal_nolock.py + - test_validation_heal_race.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -136,7 +138,7 @@ jobs: sudo apt-get update sudo apt-get install --no-install-recommends -y nftables fi - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 0fec6ea9..6d296119 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: A09 executor changes are in the working tree and will be recorded in the next helper commit; the preceding helper `75b245189cf7e22a31802bb509c8c2edcf4264dd` is pushed to auto-testing PR #113. The current local contract suite runs 508 tests with eight macOS skips after adding A09 contracts; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. The matrix is now 135 dispatchable IDs (74 non-Heal and 61 Heal), with 23 Heal executors still intentionally unimplemented. +Current reconciliation: helper `d502d3eec139d17b2587c883fce301c40a2741a5` is pushed to auto-testing PR #113 and contains the A09 executor. The next helper adds the G01/G03 version-race executor family. The current local contract suite runs 513 tests with eight macOS skips after adding A09 contracts; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01 and G03 now share a version-race executor that preserves serial overwrite or exact old/current version-delete ACKs, runs the H1/H2 deep-heal window, verifies read-repair overlap for G01, checks final peer inventories and exact physical generations, and retains native commit/cleanup and delete-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 137 dispatchable IDs (74 non-Heal and 63 Heal), with 21 Heal executors still intentionally unimplemented. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 8bf052aa..158c0a74 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-version-delete|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 9d7271ac..730f1266 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.admin-nolock-safety", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.admin-nolock-safety", "heal.race-unversioned-overwrite", "heal.race-version-delete", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py new file mode 100644 index 00000000..abe68157 --- /dev/null +++ b/scripts/validation_heal/race.py @@ -0,0 +1,552 @@ +#!/usr/bin/env python3 +"""G01/G03 version races; a blocked native fence never certifies the case.""" + +from __future__ import annotations + +import argparse +import copy +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time +from urllib.parse import quote +import uuid + +try: + from .heal_probe import Probe, canonical, compare_inventories, payload, require, save + from .heal_workload import Recorder, verify_seed + from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .matrix import select + from .heal_fault import bounded_fault_id + from .shards import files, map_target, validate_repaired_physical + from .cursor import validate_missing_batch, verify_part_manifest + from .versions import public_version_id +except ImportError: + from heal_probe import Probe, canonical, compare_inventories, payload, require, save + from heal_workload import Recorder, verify_seed + from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from matrix import select + from heal_fault import bounded_fault_id + from shards import files, map_target, validate_repaired_physical + from cursor import validate_missing_batch, verify_part_manifest + from versions import public_version_id + + +CASE_IDS = ("heal.race-unversioned-overwrite", "heal.race-version-delete") +CASE_BRANCHES = { + "heal.race-unversioned-overwrite": ("overwrite-vs-deep", "overwrite-vs-read-repair", "native-commit-cleanup"), + "heal.race-version-delete": ("delete-old-version-vs-deep", "delete-current-version-vs-deep", "native-delete-fence"), +} +G01_BRANCHES = CASE_BRANCHES["heal.race-unversioned-overwrite"][:2] +G03_BRANCHES = CASE_BRANCHES["heal.race-version-delete"][:2] +OBJECTS = 257 +SIZE = 257 * 1024 + 123 +TARGET_KEY = "objects/0256.bin" +REFERENCE_KEY = "objects/0000.bin" +REFERENCE = { + "candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", + "sourceFiles": { + "rustfs/src/admin/handlers/heal.rs": "7e4007c29a8329ced8e48b88111400e79cffd845bd6ddc6243b79f3cd48e8199", + "crates/heal/src/heal/channel.rs": "7f669ca43b9559db9b97fba6b419a6c76a94d76acd8526db132d74b7c534d6b0"}, + "requested": {**BASE_OPTIONS, "recursive": True, "nolock": False, "pool": None, "set": None}, + "effective": {**BASE_OPTIONS, "recursive": True, "nolock": False, "pool": None, "set": None}, + "nativeFence": "pinned release has no reviewed e2e-test-hooks artifact or exact Admin commit/cleanup barrier", +} + + +class WindowNotObserved(ValueError): + """No observable client overlap; this is not a product success.""" + + +class RaceRecorder(Recorder): + """Keep unresolved mutations and exact SDK receipts after any assertion.""" + def __init__(self, *args, versioned, **kwargs): + super().__init__(*args, **kwargs) + self.versioned, self.pending, self.seen = versioned, None, set() + + def call(self, operation, params, body=None, **kwargs): + self.pending = {"operation": operation, "params": copy.deepcopy(params)} + result = super().call(operation, params, body, **kwargs) + version = result.get("VersionId") + if operation == "put_object": + require(isinstance(result.get("ETag"), str) and result["ETag"] + and result.get("DeleteMarker") in {None, False}, "G01/G03 PUT ACK lacks ETag or changed kind") + if self.versioned: + require(isinstance(version, str) and str(uuid.UUID(version)) == version and uuid.UUID(version).int + and (params["Key"], version) not in self.seen, "G01/G03 PUT ACK has an invalid/reused VersionId") + self.seen.add((params["Key"], version)) + else: + require(version in {None, "null"}, "G01/G03 never-versioned PUT created a real version") + elif operation == "delete_object": + require(self.versioned and params.get("VersionId") in { + value for key, value in self.seen if key == params["Key"]} + and version == params["VersionId"] and result.get("DeleteMarker") in {None, False}, + "G01/G03 DELETE ACK changed the exact known data-version identity") + self.pending = None + return result + + +def settings(observed): + require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), + "G01/G03 status has an unknown or failed HTTP response") + value = observed["response"] + require(value.get("summary") in {"running", "finished"}, "G01/G03 original task disappeared, stopped or failed") + # Canonical JSON distinguishes bool/int and preserves raw null selectors. + require(canonical(value.get("settings")) == canonical(REFERENCE["effective"]), + "G01/G03 Admin nolock was not safely normalized or changed unrelated options") + return value["settings"] + + +def sample_overlap(ack, observed, path, token): + foreground = ack.get("observation") or {} + settings(observed) + value = observed["response"] + a, z = foreground.get("callStartedMonotonicNs"), foreground.get("callFinishedMonotonicNs") + b, c = observed.get("callStartedMonotonicNs"), observed.get("callFinishedMonotonicNs") + processed = (value.get("outcome") or {}).get("counters", {}).get("processed") + return (foreground.get("acknowledged") is True and foreground.get("clientCallInvoked") is True + and foreground.get("clockDomain") and foreground["clockDomain"] == observed.get("clockDomain") + and observed.get("path") == path and [tuple(row) for row in observed.get("query", [])] == [("clientToken", token)] + and all(type(tick) is int for tick in (a, z, b, c)) and 0 < a < b < c < z + and value["summary"] == "running" and type(processed) is int and processed > 0) + + +def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): + require(branch in (*G01_BRANCHES, *G03_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G03 unregistered fixture") + recorder.call("create_bucket", {"Bucket": bucket}) + if branch in G03_BRANCHES: + recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) + entries = [] + keys = [f"objects/{index:04d}.bin" for index in range(count - 1)] + [TARGET_KEY] + for key in keys: + versions = 3 if key == TARGET_KEY and branch in G03_BRANCHES else 1 + for generation in range(versions): + nonce = f"{bucket}/{key}/{generation}" + body = payload(nonce, SIZE) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": "G01/G03", "nonce": nonce}, + "Tagging": "suite=heal&fixture=G01/G03"}, body) + entries.append({"key": key, "versionId": public_version_id(response.get("VersionId")), + "isLatest": generation == versions - 1, "deleteMarker": False, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + return entries + + +def operation_plan(bucket, branch, expected): + require(branch in (*G01_BRANCHES, *G03_BRANCHES), "G01/G03 unregistered foreground branch") + if branch in G01_BRANCHES: + # Three distinct serial commits; the final independent reference was + # written before injection, allowing a coding-index byte comparison. + return [{"operation": "put_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, + "ContentType": "application/octet-stream", "Metadata": {"nonce": str(index)}, + "Tagging": "race=overwrite"}, + "nonce": f"{bucket}/overwrite/{index}" if index < 2 else f"{bucket}/{REFERENCE_KEY}/0"} + for index in range(3)] + target = [row for row in expected if row["key"] == TARGET_KEY] + require(len(target) == 3 and len({row["versionId"] for row in target}) == 3 + and target[-1]["isLatest"], "G01/G03 DV lacks three ACK-ordered versions") + return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, + "VersionId": (target[0] if branch == G03_BRANCHES[0] else target[-1])["versionId"]}}] + + +def final_inventory(baseline, branch, plan, responses): + require(len(plan) == len(responses) == (3 if branch in G01_BRANCHES else 1), + "G01/G03 foreground plan/ACK sequence is incomplete") + result = copy.deepcopy(baseline) + if branch in G01_BRANCHES: + entry = next(row for row in result["entries"] if row["key"] == TARGET_KEY) + step, ack = plan[-1], responses[-1] + body = payload(step["nonce"], SIZE) + require(isinstance(ack.get("ETag"), str) and ack["ETag"] + and ack.get("VersionId") in {None, "null"}, "G01/G03 last PUT ACK lost its identity") + entry.update(payload={"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}, + tags=[{"Key": "race", "Value": "overwrite"}]) + entry["properties"].update(ETag=ack["ETag"], ContentLength=len(body), ContentType="application/octet-stream", + Metadata=step["params"]["Metadata"]) + else: + version = plan[0]["params"]["VersionId"] + require(responses[0].get("VersionId") == version, "G01/G03 DELETE response changed the selected VersionId") + result["entries"] = [row for row in result["entries"] + if (row["key"], row["versionId"]) != (TARGET_KEY, version)] + require(len(result["entries"]) + 1 == len(baseline["entries"]), "G01/G03 deleted a foreign/missing version") + return result + + +def object_scene(snapshot, bucket, key): + return {node: {**value, "files": [row for row in value["files"] + if row["path"].startswith(f"{bucket}/{key}/")]} + for node, value in snapshot.items()} + + +def map_fixture(snapshot, bucket, expected): + mapped = {(row["key"], row["versionId"]): map_target( + object_scene(snapshot, bucket, row["key"]), bucket, row["key"], "data", 1, + row["versionId"], latest_required=False) for row in expected} + require(len(mapped) == len(expected), "G01/G03 physical map omitted/duplicated an ACK") + return mapped + + +def physical_transition(before, after, bucket, mapped, final, branch): + """Prove current ACK authority before allowing any GET or backup rescue.""" + require(all(before[node]["formats"] == after[node]["formats"] for node in before), + "G01/G03 race changed an original disk format") + known = {row["key"] for row in final["entries"]} + require(all(any(row["path"].startswith(f"{bucket}/{key}/") for key in known) + for value in after.values() for row in value["files"]), "G01/G03 created an unacknowledged physical key") + for (key, version), target in mapped.items(): + if key == TARGET_KEY: continue + validate_repaired_physical(object_scene(before, bucket, key), object_scene(after, bucket, key), target) + survivors = map_fixture(after, bucket, final["entries"]) + original_target = [value for (key, _), value in mapped.items() if key == TARGET_KEY] + current_target = [value for (key, _), value in survivors.items() if key == TARGET_KEY] + if branch in G01_BRANCHES: + require(len(original_target) == len(current_target) == 1 + and original_target[0]["dataDir"] != current_target[0]["dataDir"], + "G01/G03 overwrite retained the old physical generation") + reference = next(target for (key, _), target in mapped.items() if key == REFERENCE_KEY) + require({part["index"]: (part["sha256"], part["size"]) for part in reference["cohort"]} + == {part["index"]: (part["sha256"], part["size"]) for part in current_target[0]["cohort"]}, + "G01/G03 last ACK shard bytes/coding indices differ from the independent payload reference") + else: + remaining = {(row["key"], row["versionId"]) for row in final["entries"]} + removed = [value for pair, value in mapped.items() if pair not in remaining] + require(len(removed) == 1, "G01/G03 delete transition lacks one exact deleted version") + for target in current_target: + original = mapped[TARGET_KEY, target["versionId"]] + require(target["logical"] == original["logical"] + and [(row["index"], row["sha256"], row["size"]) for row in target["cohort"]] + == [(row["index"], row["sha256"], row["size"]) for row in original["cohort"]], + "G01/G03 delete changed another retained ACK generation") + current_dirs = {target["dataDir"] for target in current_target} + for value in after.values(): + for row in value["files"]: + prefix = f"{bucket}/{TARGET_KEY}/" + if not row["path"].startswith(prefix): continue + suffix = row["path"][len(prefix):] + require(suffix == "xl.meta" or suffix.split("/")[0] in current_dirs, + "G01/G03 resurrected an old/deleted data directory or left an unbound target leaf") + return survivors, {"allControlShardsRestored": True, "lastACKGenerationVerified": True, + "physicalProofBeforeGETOrRescue": True, "oldVersionResurrected": False} + + +def terminal_proof(status, bucket, before, final): + outcome = status.get("outcome") or {} + require(status.get("summary") == "finished" and outcome.get("execution", {}).get("state") == "completed" + and outcome.get("coverage") == "complete", "G01/G03 task lacks complete successful terminal coverage") + old = {(row["key"], row["versionId"]) for row in before["entries"]} + new = {(row["key"], row["versionId"]) for row in final["entries"]} + counters = outcome.get("counters") or {} + require(all(type(counters.get(key)) is int and counters[key] >= 0 + for key in ("processed", "healed", "unchanged", "failed", "unknown", "skipped", "attemptFailures")) + and all(counters[key] == 0 for key in ("failed", "unknown", "skipped", "attemptFailures")) + and counters.get("overflowed") is False and counters["processed"] == counters["healed"] + counters["unchanged"] + and len(old & new) <= counters["processed"] <= len(old | new), + "G01/G03 terminal responsibility exceeds ACK bounds or hides unknown/failed work") + receipts = outcome.get("objects") + require(isinstance(receipts, list) and 0 < len(receipts) <= 128 + and outcome.get("objectsTruncated") is (counters["processed"] > 128), + "G01/G03 confused truncated diagnostics with its independent complete ACK manifest") + seen, incarnation = set(), None + for receipt in receipts: + identity = receipt.get("identity") or {} + pair = identity.get("object"), public_version_id(identity.get("versionId")) + require(identity.get("bucket") == bucket and pair in old | new and pair not in seen + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0 + and isinstance(identity.get("bucketIncarnationId"), str) and identity["bucketIncarnationId"] + and receipt.get("disposition", {}).get("state") in {"repaired", "verified_healthy"}, + "G01/G03 diagnostics changed version/incarnation/scope or claimed unknown/false health") + incarnation = incarnation or identity["bucketIncarnationId"] + require(incarnation == identity["bucketIncarnationId"], "G01/G03 diagnostics mix incarnations") + seen.add(pair) + return {"minimumACKResponsibilities": len(old & new), "maximumACKResponsibilities": len(old | new), + "counters": counters, "exactRaceEnumerationClaimed": False} + + +def observe(probe, fleet, path, token): + value = fleet.record("original-token-status", probe.admin(path, [("clientToken", token)], b"")) + settings(value) + return value + + +def wait_running(probe, fleet, path, token, total): + deadline = time.monotonic() + 30 + while True: + observed = observe(probe, fleet, path, token) + value = observed["response"] + processed = (value.get("outcome") or {}).get("counters", {}).get("processed") + if value["summary"] == "finished": + raise WindowNotObserved("G01/G03 task finished before its partial-progress race window") + if type(processed) is int and 0 < processed < total: return observed + if time.monotonic() >= deadline: + raise WindowNotObserved("G01/G03 nonzero partial progress was not observed within 30 seconds") + time.sleep(.05) + + +def wait_terminal(probe, fleet, path, token): + deadline = time.monotonic() + 240 + while True: + current = observe(probe, fleet, path, token) + if current["response"]["summary"] == "finished": return current["response"] + require(time.monotonic() < deadline, "G01/G03 original task exceeded its terminal budget") + time.sleep(.2) + + +def foreground(recorder, plan): + responses = [] + for step in plan: + body = payload(step["nonce"], SIZE) if step["operation"] == "put_object" else None + responses.append(recorder.call(step["operation"], step["params"], body)) + return responses + + +def race(recorder, plan, probe, fleet, path, token): + samples = [] + # Only the worker mutates Recorder state. All Fleet.record writes run on + # the controller, so evidence sequence allocation never races. + readers = [] + read_branch = bool(plan and plan[0].get("branch") == "overwrite-vs-read-repair") + with ThreadPoolExecutor(max_workers=2 if read_branch else 1) as pool: + worker = pool.submit(foreground, recorder, plan) + if read_branch: + def read_repair(): + deadline = time.monotonic() + 90 + while not worker.done() and time.monotonic() < deadline: + readers.append(probe.s3("get_object", {"Bucket": plan[0]["params"]["Bucket"], + "Key": plan[0]["params"]["Key"]})) + time.sleep(.01) + read_future = pool.submit(read_repair) + else: + read_future = None + deadline = time.monotonic() + 120 + try: + while not worker.done() and time.monotonic() < deadline: + samples.append(observe(probe, fleet, path, token)) + time.sleep(.01) + except BaseException: + recorder.mutations.poison("status observation failed; drain in-flight mutations only") + raise + if not worker.done(): recorder.mutations.poison("foreground deadline; drain only") + responses = worker.result(timeout=60) + if read_future is not None: + read_future.result(timeout=60) + acks = [json.loads(path.read_text()) for path in sorted(recorder.output.glob("*-ack.json"))] + require(len(acks) == len(plan) and all(row.get("operationCheck") == "PASS" for row in acks), + "G01/G03 lacks every durable foreground ACK") + overlaps = [any(sample_overlap(ack, sample, path, token) for sample in samples) for ack in acks] + fleet.record("read-repair-overlap-proof", {"branch": plan[0].get("branch") if plan else None, + "readCalls": len(readers), "allReadsObserved": bool(readers) if read_branch else True, + "nativeFenceProved": False}) + fleet.record("public-call-overlap-proof", {"everyMutationOverlapped": all(overlaps), "mutations": overlaps, + "nativeFenceProved": False, "serverCommitOverlapProved": False}) + return responses, all(overlaps) + + +def run_branch(args, topology, nodes, branch): + directory = args.evidence / topology / branch + directory.mkdir(parents=True, exist_ok=False) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="manual") + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == REFERENCE["candidateCommit"], "G01/G03 candidate contract is not reviewed") + fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 512, "maxInventoryBytes": 512 * 1024**2} + fleet.record("binding", binding) + fleet.record("source-contract", {**REFERENCE, "branch": branch, "objectCount": OBJECTS, "objectSize": SIZE, + "requiredPUTs": 3 if branch in G01_BRANCHES else 0, "serverCommitOverlapProved": False}) + probe, active, recorder, writer = Probe(binding), [], None, None + result = {"topology": topology, "branch": branch, "fixture": "DL" if branch in G01_BRANCHES else "DV", + "verdict": "RUNNING", "cleanup": "PENDING", "canonicalCasePassClaimed": False} + changed_target_registered, target_authority = None, False + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) + recorder = RaceRecorder(probe, directory / "seed-ACKs", versioned=branch in G03_BRANCHES) + expected = seed_fixture(recorder, bucket, branch) + baseline = fleet.record("complete-baseline-S3", probe.inventory(bucket, unversioned_current_tags=branch in G01_BRANCHES)) + verify_seed(expected, baseline) + plan = operation_plan(bucket, branch, expected) + for step in plan: + step["branch"] = branch + fleet.record("frozen-foreground-plan", plan) + fleet.stop() + before = fleet.snapshot("complete-baseline-physical", quiescent=True) + mapped = map_fixture(before, bucket, expected) + verify_part_manifest(fleet, list(mapped.values()), "baseline-streaming", count=len(mapped)) + target_version = (next(entry for entry in expected if entry["key"] == TARGET_KEY and entry["isLatest"]) + if branch == G03_BRANCHES[1] + else next(entry for entry in expected if entry["key"] == TARGET_KEY)) + targets = [mapped[row["key"], row["versionId"]] for row in expected + if row["key"] != TARGET_KEY or row["versionId"] == target_version["versionId"]] + require(len(targets) == OBJECTS, "G01/G03 fault plan omitted a key or changed its source dimension") + receipts = [] + for index, target in enumerate(targets): + params = {"disk": target["disk"], "faultId": bounded_fault_id("G01/G03", branch, index)} + fleet.faults.append((target["node"], params)) + if target["key"] == TARGET_KEY: changed_target_registered = target["node"], params + receipts.append(fleet.record("exact-missing-part", {"node": target["node"], **fleet.remote( + target["node"], "apply-fault", **params, relativePath=target["relativePath"], + expectedSha256=target["sha256"], kind="missing")})) + faulted = fleet.snapshot("complete-fault-physical", quiescent=True) + fleet.record("fault-batch-proof", validate_missing_batch(before, faulted, targets, receipts)) + fleet.start() + require(physical_content(faulted) == physical_content(fleet.snapshot("attribution-before-START")), + "NOT_OBSERVED: G01/G03 missing shards changed before manual admission") + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + fleet.record("prepared-Admin-request", {"path": path, "query": [], "options": REFERENCE["requested"]}) + start = fleet.record("original-Admin-start", probe.admin(path, [], json.dumps(REFERENCE["requested"]).encode())) + token = start.get("response", {}).get("clientToken") if isinstance(start.get("response"), dict) else None + require(start.get("httpStatus") == 200 and isinstance(token, str) and token, "G01/G03 admission lacks an original token") + active.append((path, token)) + wait_running(probe, fleet, path, token, len(expected)) + witness = fleet.snapshot("target-missing-before-foreground") + target = next(target for target in targets if target["key"] == TARGET_KEY) + if (target["node"], target["disk"], target["relativePath"]) in files(witness): + raise WindowNotObserved("G01/G03 old target already repaired before foreground mutations") + writer = RaceRecorder(probe, directory / "foreground-ACKs", versioned=branch in G03_BRANCHES) + writer.seen.update((row["key"], row["versionId"]) for row in expected) + responses, overlapped = race(writer, plan, probe, fleet, path, token) + final = fleet.record("ACK-derived-final-manifest", final_inventory(baseline, branch, plan, responses)) + terminal = wait_terminal(probe, fleet, path, token) + active.remove((path, token)) + for _ in range(2): + repeated = observe(probe, fleet, path, token)["response"] + require(canonical(repeated.get("outcome")) == canonical(terminal.get("outcome")) + and repeated["summary"] == terminal["summary"], "G01/G03 terminal responsibility changed on repetition") + # Collect physical evidence even if terminal counters fail. Old target + # bytes are never rescued over a successful overwrite or exact delete. + fleet.stop() + after = fleet.snapshot("product-physical-before-GET-or-rescue", quiescent=True) + survivors, physical = physical_transition(before, after, bucket, mapped, final, branch) + fleet.record("last-ACK-physical-proof", physical) + verify_part_manifest(fleet, list(survivors.values()), "final-streaming", count=len(survivors)) + target_authority = True + # Every missing-part registration is now superseded by product output; + # leaving it in Fleet.faults would make cleanup attempt a blind backup + # rescue over the new ACK generation. + fleet.faults.clear() + fleet.record("obsolete-target-fault-retired", {"fault": changed_target_registered, + "reason": "complete last-ACK physical authority proves the original generation obsolete", + "oldBytesRestored": False, "backupRetainedUntilCleanup": True}) + fleet.start() + for endpoint in binding["endpoints"]: + current = fleet.record("all-peer-final-S3", {"endpoint": endpoint, **Probe(binding, endpoint).inventory( + bucket, unversioned_current_tags=branch in G01_BRANCHES)}) + compare_inventories(final, current) + if branch in G03_BRANCHES: + negative = fleet.record("all-peer-deleted-version-GET", Probe(binding, endpoint).s3("get_object", plan[0]["params"])) + response = negative.get("response") or {} + require(negative.get("acknowledged") is False and negative.get("resultUnknown") is False + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404 + and response.get("Error", {}).get("Code") == "NoSuchVersion", + "G01/G03 deleted exact VersionId revived or failed for another reason") + fleet.stop() + require(physical_content(after) == physical_content(fleet.snapshot("physical-after-S3", quiescent=True)), + "G01/G03 final GET changed the already-proven physical result") + result["terminal"] = terminal_proof(terminal, bucket, baseline, final) + if not overlapped: raise WindowNotObserved("G01/G03 did not observe every required SDK call overlap") + result.update(verdict="PASS", physical=physical, publicCallOverlapObserved=True, + nativeFenceProved=False, serverCommitOverlapProved=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="NOT_OBSERVED" if isinstance(exc, WindowNotObserved) else "FAIL", + failure={"type": type(exc).__name__, "message": str(exc)}) + try: + fleet.stop() + fleet.snapshot("retained-failure-scene", quiescent=True) + except Exception as capture: fleet.record("failure-scene-error", {"error": str(capture)}) + finally: + for path, token in active: + try: fleet.record("cancel-original-token", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: fleet.record("cancel-error", {"error": str(exc)}) + unresolved = any(item is not None and item.pending is not None for item in (recorder, writer)) + mutated = writer is not None and writer.sequence > 0 + if unresolved or mutated and not target_authority: + try: fleet.stop() + except Exception as exc: fleet.record("retain-stop-error", {"error": str(exc)}) + fleet.record("last-ACK-rescue-required", {"state": "DIRTY", "namespacesRetained": True, + "unresolvedMutation": unresolved, "lastACKPhysicalAuthorityVerified": target_authority, + "originalGenerationBlindRestorationProhibited": True}) + result["cleanup"] = "FAIL" + else: + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("branch-result", result) + return result + + +def native_blocked(candidate, topology, case_id): + require(topology in {"H1", "H2"}, "G01/G03 native prerequisite lost its source topology") + return {"caseId": case_id, "topology": topology, "branch": CASE_BRANCHES[case_id][-1], "verdict": "BLOCKED", "cleanup": "PASS", + "candidateCommit": candidate["source_sha"], "canonicalCasePassClaimed": False, + "reason": REFERENCE["nativeFence"], "nativeFenceProved": False, "fleetExecutionClaimed": False} + + +def complete_results(results, case_id): + branches = CASE_BRANCHES[case_id] + wanted = {(topology, branch) for topology in ("H1", "H2") for branch in branches} + actual = {(row["topology"], row["branch"]) for row in results} + require(len(actual) == len(results) and actual <= wanted, "G01/G03 duplicate or foreign topology/branch") + return actual == wanted and all(row["verdict"] == row["cleanup"] == "PASS" for row in results) and all( + row.get("nativeFenceProved") is True for row in results if row["branch"] == branches[-1]) + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f"G01/G03 interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ("candidate", "rc-bin", "evidence", "tool-directory"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--case-id", choices=list(CASE_BRANCHES), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + args = parser.parse_args() + case_id = args.case_id; branches = CASE_BRANCHES[case_id] + require(select(case_id)[0]["variants"] == list(branches), "G01/G03 executor omitted a source branch") + require(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha) and re.fullmatch(r"[0-9a-f]{40}", args.testing_commit) + and re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")), "G01/G03 immutable identities required") + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) >= 4 and len(nodes) == len(set(nodes)) + and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), + "G01/G03 requires four distinct current Heal nodes") + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == REFERENCE["candidateCommit"], "G01/G03 candidate contract not reviewed") + results = [native_blocked(candidate, topology, case_id) for topology in ("H1", "H2")] + try: + stop = False + for topology in ("H1", "H2"): + for branch in branches[:2]: + row = run_branch(args, topology, nodes[:1] if topology == "H1" else nodes[:4], branch) + row["caseId"] = case_id + results.append(row) + if row["cleanup"] != "PASS": stop = True; break + if stop: break + finally: + complete = complete_results(results, case_id) + save(args.evidence / "healthy-summary.json", {"caseId": case_id, "results": results, "complete": complete, + "cleanup": "PASS" if all(row["cleanup"] == "PASS" for row in results) else "FAIL", + "canonicalCasePassClaimed": False, "nativePrerequisitesBlocked": True}) + require(complete, "G01/G03 native prerequisites remain BLOCKED; every H1/H2 branch stays in the denominator") + + +if __name__ == "__main__": + try: main() + except Exception as exc: + print(f"G01/G03 validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 239e7fc1..70418ecb 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -45,6 +45,7 @@ case "${case_id}" in heal.conflicting-subquorum-authority) driver=authority_runner.py ;; heal.dangling-and-orphan-safety) driver=orphan.py ;; heal.admin-nolock-safety) driver=nolock.py ;; + heal.race-unversioned-overwrite|heal.race-version-delete) driver=race.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal_race.py b/test_validation_heal_race.py new file mode 100644 index 00000000..c58e24eb --- /dev/null +++ b/test_validation_heal_race.py @@ -0,0 +1,51 @@ +import json +from pathlib import Path +import unittest + +from scripts.validation_heal import race +from scripts.validation_heal.matrix import AUTOMATIC_CASES, select + + +class RaceTests(unittest.TestCase): + def test_source_case_branch_and_topology_contracts(self): + for case_id, branches in race.CASE_BRANCHES.items(): + recipe = select(case_id)[0] + self.assertEqual(recipe["variants"], list(branches)) + self.assertIn(case_id, AUTOMATIC_CASES) + self.assertEqual(recipe["contract"]["probes"]["topology"], "H1,H2") + + def test_g01_plan_keeps_three_ack_ordered_overwrites(self): + expected = [{"key": race.TARGET_KEY, "versionId": "null", "isLatest": True}] + plan = race.operation_plan("rv-heal-0123456789abcdef", "overwrite-vs-read-repair", expected) + self.assertEqual([step["params"]["Metadata"]["nonce"] for step in plan], ["0", "1", "2"]) + self.assertEqual(plan[-1]["nonce"], "rv-heal-0123456789abcdef/objects/0000.bin/0") + + def test_g03_selects_exact_old_or_current_version(self): + expected = [{"key": race.TARGET_KEY, "versionId": value, "isLatest": value == "v2"} + for value in ("v0", "v1", "v2")] + old = race.operation_plan("rv-heal-0123456789abcdef", "delete-old-version-vs-deep", expected) + current = race.operation_plan("rv-heal-0123456789abcdef", "delete-current-version-vs-deep", expected) + self.assertEqual(old[0]["params"]["VersionId"], "v0") + self.assertEqual(current[0]["params"]["VersionId"], "v2") + + def test_native_branch_remains_in_denominator_for_each_case(self): + for case_id, branches in race.CASE_BRANCHES.items(): + native = race.native_blocked({"source_sha": race.REFERENCE["candidateCommit"]}, "H1", case_id) + rows = [{"topology": topology, "branch": branch, "verdict": "PASS", "cleanup": "PASS"} + for topology in ("H1", "H2") for branch in branches[:2]] + self.assertFalse(race.complete_results([*rows, native], case_id)) + self.assertEqual(native["branch"], branches[-1]) + + def test_dispatch_and_source_contract_are_present(self): + dispatch = Path("scripts/validation_heal/run_case.sh").read_text() + top_dispatch = Path("scripts/run_validation_case.sh").read_text() + for case_id, code in (("heal.race-unversioned-overwrite", "G01"), ("heal.race-version-delete", "G03")): + self.assertIn(case_id, dispatch) + self.assertIn(case_id, top_dispatch) + catalog = json.loads(Path("validation-cases.json").read_text()) + source = {case["id"]: case for module in catalog["modules"] for case in module["cases"]}[case_id] + self.assertEqual(source["probes"]["designCode"], code) + + +if __name__ == "__main__": + unittest.main() From ec26bd3d3364e7c9291ff57a70087e1d32c2917f Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 17:17:15 +0800 Subject: [PATCH 144/343] feat(validation): migrate G02 versioned put race --- docs/validation-migration-map.md | 2 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/race.py | 77 ++++++++++++++++++++++------- scripts/validation_heal/run_case.sh | 2 +- test_validation_heal_race.py | 11 ++++- 6 files changed, 72 insertions(+), 24 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 6d296119..043fb583 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `d502d3eec139d17b2587c883fce301c40a2741a5` is pushed to auto-testing PR #113 and contains the A09 executor. The next helper adds the G01/G03 version-race executor family. The current local contract suite runs 513 tests with eight macOS skips after adding A09 contracts; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01 and G03 now share a version-race executor that preserves serial overwrite or exact old/current version-delete ACKs, runs the H1/H2 deep-heal window, verifies read-repair overlap for G01, checks final peer inventories and exact physical generations, and retains native commit/cleanup and delete-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 137 dispatchable IDs (74 non-Heal and 63 Heal), with 21 Heal executors still intentionally unimplemented. +Current reconciliation: helper `0cef8a8a` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03 race executors. The helper is recorded below with its full SHA after commit. The current local contract suite runs 513 tests with eight macOS skips after adding A09 contracts; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01, G02 and G03 now share a version-race executor that preserves serial overwrite, versioned PUT and exact old/current version-delete ACKs, runs the H1/H2 deep-heal window, verifies read-repair or exact historical reads, checks final peer inventories and physical generations, and retains native commit/cleanup, version and delete-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 138 dispatchable IDs (74 non-Heal and 64 Heal), with 20 Heal executors still intentionally unimplemented. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 158c0a74..ee0e9876 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-version-delete|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 730f1266..9c0648a0 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.admin-nolock-safety", "heal.race-unversioned-overwrite", "heal.race-version-delete", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.admin-nolock-safety", "heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index abe68157..9af203a9 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -41,12 +41,14 @@ from versions import public_version_id -CASE_IDS = ("heal.race-unversioned-overwrite", "heal.race-version-delete") +CASE_IDS = ("heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete") CASE_BRANCHES = { "heal.race-unversioned-overwrite": ("overwrite-vs-deep", "overwrite-vs-read-repair", "native-commit-cleanup"), + "heal.race-versioned-put": ("new-version-vs-historical-heal", "read-all-versions-during-heal", "native-version-fence"), "heal.race-version-delete": ("delete-old-version-vs-deep", "delete-current-version-vs-deep", "native-delete-fence"), } G01_BRANCHES = CASE_BRANCHES["heal.race-unversioned-overwrite"][:2] +G02_BRANCHES = CASE_BRANCHES["heal.race-versioned-put"][:2] G03_BRANCHES = CASE_BRANCHES["heal.race-version-delete"][:2] OBJECTS = 257 SIZE = 257 * 1024 + 123 @@ -121,14 +123,14 @@ def sample_overlap(ack, observed, path, token): def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): - require(branch in (*G01_BRANCHES, *G03_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G03 unregistered fixture") + require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G02/G03 unregistered fixture") recorder.call("create_bucket", {"Bucket": bucket}) - if branch in G03_BRANCHES: + if branch in (*G02_BRANCHES, *G03_BRANCHES): recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) entries = [] keys = [f"objects/{index:04d}.bin" for index in range(count - 1)] + [TARGET_KEY] for key in keys: - versions = 3 if key == TARGET_KEY and branch in G03_BRANCHES else 1 + versions = 3 if key == TARGET_KEY and branch in (*G02_BRANCHES, *G03_BRANCHES) else 1 for generation in range(versions): nonce = f"{bucket}/{key}/{generation}" body = payload(nonce, SIZE) @@ -142,7 +144,7 @@ def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): def operation_plan(bucket, branch, expected): - require(branch in (*G01_BRANCHES, *G03_BRANCHES), "G01/G03 unregistered foreground branch") + require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES), "G01/G02/G03 unregistered foreground branch") if branch in G01_BRANCHES: # Three distinct serial commits; the final independent reference was # written before injection, allowing a coding-index byte comparison. @@ -152,6 +154,14 @@ def operation_plan(bucket, branch, expected): "nonce": f"{bucket}/overwrite/{index}" if index < 2 else f"{bucket}/{REFERENCE_KEY}/0"} for index in range(3)] target = [row for row in expected if row["key"] == TARGET_KEY] + if branch in G02_BRANCHES: + require(len(target) == 3 and len({row["versionId"] for row in target}) == 3, "G02 DV lacks three ACK-ordered versions") + return [{"operation": "put_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, + "ContentType": "application/octet-stream", "Metadata": {"race": "versioned-put", "nonce": str(index)}, + "Tagging": "race=versioned-put"}, + "nonce": f"{bucket}/versioned-put/{index}", "readVersionId": target[0]["versionId"], + "readPayload": target[0].get("payload")} + for index in range(3)] require(len(target) == 3 and len({row["versionId"] for row in target}) == 3 and target[-1]["isLatest"], "G01/G03 DV lacks three ACK-ordered versions") return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, @@ -159,19 +169,29 @@ def operation_plan(bucket, branch, expected): def final_inventory(baseline, branch, plan, responses): - require(len(plan) == len(responses) == (3 if branch in G01_BRANCHES else 1), + require(len(plan) == len(responses) == (3 if branch in (*G01_BRANCHES, *G02_BRANCHES) else 1), "G01/G03 foreground plan/ACK sequence is incomplete") result = copy.deepcopy(baseline) - if branch in G01_BRANCHES: + if branch in (*G01_BRANCHES, *G02_BRANCHES): entry = next(row for row in result["entries"] if row["key"] == TARGET_KEY) step, ack = plan[-1], responses[-1] body = payload(step["nonce"], SIZE) require(isinstance(ack.get("ETag"), str) and ack["ETag"] - and ack.get("VersionId") in {None, "null"}, "G01/G03 last PUT ACK lost its identity") - entry.update(payload={"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}, - tags=[{"Key": "race", "Value": "overwrite"}]) - entry["properties"].update(ETag=ack["ETag"], ContentLength=len(body), ContentType="application/octet-stream", - Metadata=step["params"]["Metadata"]) + and (branch in G02_BRANCHES or ack.get("VersionId") in {None, "null"}), "G01/G02/G03 last PUT ACK lost its identity") + if branch in G01_BRANCHES: + entry.update(payload={"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}, + tags=[{"Key": "race", "Value": "overwrite"}]) + entry["properties"].update(ETag=ack["ETag"], ContentLength=len(body), ContentType="application/octet-stream", + Metadata=step["params"]["Metadata"]) + else: + for row in result["entries"]: + if row["key"] == TARGET_KEY: row["isLatest"] = False + result["entries"].append({"key": TARGET_KEY, "versionId": public_version_id(ack.get("VersionId")), + "isLatest": True, "deleteMarker": False, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}, + "properties": {"ETag": ack["ETag"], "ContentLength": len(body), + "ContentType": "application/octet-stream", "Metadata": step["params"]["Metadata"]}, + "tags": [{"Key": "race", "Value": "versioned-put"}]}) else: version = plan[0]["params"]["VersionId"] require(responses[0].get("VersionId") == version, "G01/G03 DELETE response changed the selected VersionId") @@ -216,6 +236,16 @@ def physical_transition(before, after, bucket, mapped, final, branch): require({part["index"]: (part["sha256"], part["size"]) for part in reference["cohort"]} == {part["index"]: (part["sha256"], part["size"]) for part in current_target[0]["cohort"]}, "G01/G03 last ACK shard bytes/coding indices differ from the independent payload reference") + elif branch in G02_BRANCHES: + require(len(original_target) == 3 and len(current_target) == 6, + "G02 version race lost an acknowledged historical or new version") + old_dirs = {target["dataDir"] for target in original_target} + new_dirs = {target["dataDir"] for target in current_target} - old_dirs + require(len(new_dirs) == 3 and old_dirs.isdisjoint(new_dirs), + "G02 versions share or resurrect a physical data directory") + require(all(any(part["size"] == SIZE for part in target["cohort"]) + for target in current_target if target["dataDir"] in new_dirs), + "G02 new version physical parts are incomplete") else: remaining = {(row["key"], row["versionId"]) for row in final["entries"]} removed = [value for pair, value in mapped.items() if pair not in remaining] @@ -314,15 +344,24 @@ def race(recorder, plan, probe, fleet, path, token): # Only the worker mutates Recorder state. All Fleet.record writes run on # the controller, so evidence sequence allocation never races. readers = [] - read_branch = bool(plan and plan[0].get("branch") == "overwrite-vs-read-repair") + read_branch = bool(plan and plan[0].get("branch") in {"overwrite-vs-read-repair", "read-all-versions-during-heal"}) with ThreadPoolExecutor(max_workers=2 if read_branch else 1) as pool: worker = pool.submit(foreground, recorder, plan) if read_branch: def read_repair(): deadline = time.monotonic() + 90 while not worker.done() and time.monotonic() < deadline: - readers.append(probe.s3("get_object", {"Bucket": plan[0]["params"]["Bucket"], - "Key": plan[0]["params"]["Key"]})) + params = {"Bucket": plan[0]["params"]["Bucket"], "Key": plan[0]["params"]["Key"]} + if plan[0].get("readVersionId"): params["VersionId"] = plan[0]["readVersionId"] + observed = probe.s3("get_object", params) + if plan[0].get("branch") == "read-all-versions-during-heal": + response = observed.get("response") or {} + require(observed.get("acknowledged") is True and not observed.get("resultUnknown") + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 200 + and response.get("VersionId") == params["VersionId"] + and observed.get("payload") == plan[0]["readPayload"], + "G02 exact historical read was not acknowledged with original bytes") + readers.append(observed) time.sleep(.01) read_future = pool.submit(read_repair) else: @@ -371,7 +410,7 @@ def run_branch(args, topology, nodes, branch): "maxInventoryObjects": 512, "maxInventoryBytes": 512 * 1024**2} fleet.record("binding", binding) fleet.record("source-contract", {**REFERENCE, "branch": branch, "objectCount": OBJECTS, "objectSize": SIZE, - "requiredPUTs": 3 if branch in G01_BRANCHES else 0, "serverCommitOverlapProved": False}) + "requiredPUTs": 3 if branch in (*G01_BRANCHES, *G02_BRANCHES) else 0, "serverCommitOverlapProved": False}) probe, active, recorder, writer = Probe(binding), [], None, None result = {"topology": topology, "branch": branch, "fixture": "DL" if branch in G01_BRANCHES else "DV", "verdict": "RUNNING", "cleanup": "PENDING", "canonicalCasePassClaimed": False} @@ -379,7 +418,7 @@ def run_branch(args, topology, nodes, branch): try: fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) - recorder = RaceRecorder(probe, directory / "seed-ACKs", versioned=branch in G03_BRANCHES) + recorder = RaceRecorder(probe, directory / "seed-ACKs", versioned=branch in (*G02_BRANCHES, *G03_BRANCHES)) expected = seed_fixture(recorder, bucket, branch) baseline = fleet.record("complete-baseline-S3", probe.inventory(bucket, unversioned_current_tags=branch in G01_BRANCHES)) verify_seed(expected, baseline) @@ -392,7 +431,7 @@ def run_branch(args, topology, nodes, branch): mapped = map_fixture(before, bucket, expected) verify_part_manifest(fleet, list(mapped.values()), "baseline-streaming", count=len(mapped)) target_version = (next(entry for entry in expected if entry["key"] == TARGET_KEY and entry["isLatest"]) - if branch == G03_BRANCHES[1] + if branch in (G02_BRANCHES[1], G03_BRANCHES[1]) else next(entry for entry in expected if entry["key"] == TARGET_KEY)) targets = [mapped[row["key"], row["versionId"]] for row in expected if row["key"] != TARGET_KEY or row["versionId"] == target_version["versionId"]] @@ -421,7 +460,7 @@ def run_branch(args, topology, nodes, branch): target = next(target for target in targets if target["key"] == TARGET_KEY) if (target["node"], target["disk"], target["relativePath"]) in files(witness): raise WindowNotObserved("G01/G03 old target already repaired before foreground mutations") - writer = RaceRecorder(probe, directory / "foreground-ACKs", versioned=branch in G03_BRANCHES) + writer = RaceRecorder(probe, directory / "foreground-ACKs", versioned=branch in (*G02_BRANCHES, *G03_BRANCHES)) writer.seen.update((row["key"], row["versionId"]) for row in expected) responses, overlapped = race(writer, plan, probe, fleet, path, token) final = fleet.record("ACK-derived-final-manifest", final_inventory(baseline, branch, plan, responses)) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 70418ecb..c68a1bd4 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -45,7 +45,7 @@ case "${case_id}" in heal.conflicting-subquorum-authority) driver=authority_runner.py ;; heal.dangling-and-orphan-safety) driver=orphan.py ;; heal.admin-nolock-safety) driver=nolock.py ;; - heal.race-unversioned-overwrite|heal.race-version-delete) driver=race.py ;; + heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete) driver=race.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal_race.py b/test_validation_heal_race.py index c58e24eb..3d402995 100644 --- a/test_validation_heal_race.py +++ b/test_validation_heal_race.py @@ -28,6 +28,14 @@ def test_g03_selects_exact_old_or_current_version(self): self.assertEqual(old[0]["params"]["VersionId"], "v0") self.assertEqual(current[0]["params"]["VersionId"], "v2") + def test_g02_plan_creates_three_acknowledged_versions(self): + expected = [{"key": race.TARGET_KEY, "versionId": value, "isLatest": value == "v2"} + for value in ("v0", "v1", "v2")] + plan = race.operation_plan("rv-heal-0123456789abcdef", "read-all-versions-during-heal", expected) + self.assertEqual(len(plan), 3) + self.assertEqual({step["readVersionId"] for step in plan}, {"v0"}) + self.assertEqual([step["params"]["Metadata"]["nonce"] for step in plan], ["0", "1", "2"]) + def test_native_branch_remains_in_denominator_for_each_case(self): for case_id, branches in race.CASE_BRANCHES.items(): native = race.native_blocked({"source_sha": race.REFERENCE["candidateCommit"]}, "H1", case_id) @@ -39,7 +47,8 @@ def test_native_branch_remains_in_denominator_for_each_case(self): def test_dispatch_and_source_contract_are_present(self): dispatch = Path("scripts/validation_heal/run_case.sh").read_text() top_dispatch = Path("scripts/run_validation_case.sh").read_text() - for case_id, code in (("heal.race-unversioned-overwrite", "G01"), ("heal.race-version-delete", "G03")): + for case_id, code in (("heal.race-unversioned-overwrite", "G01"), ("heal.race-versioned-put", "G02"), + ("heal.race-version-delete", "G03")): self.assertIn(case_id, dispatch) self.assertIn(case_id, top_dispatch) catalog = json.loads(Path("validation-cases.json").read_text()) From 5d3840cd5dba0cb07318b6ce5807fdc1f83d1a13 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 17:17:33 +0800 Subject: [PATCH 145/343] docs(validation): record G02 helper revision --- docs/validation-migration-map.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 043fb583..b0c1c912 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `0cef8a8a` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03 race executors. The helper is recorded below with its full SHA after commit. The current local contract suite runs 513 tests with eight macOS skips after adding A09 contracts; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01, G02 and G03 now share a version-race executor that preserves serial overwrite, versioned PUT and exact old/current version-delete ACKs, runs the H1/H2 deep-heal window, verifies read-repair or exact historical reads, checks final peer inventories and physical generations, and retains native commit/cleanup, version and delete-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 138 dispatchable IDs (74 non-Heal and 64 Heal), with 20 Heal executors still intentionally unimplemented. +Current reconciliation: helper `ec26bd3d3364e7c9291ff57a70087e1d32c2917fa` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03 race executors. The helper is recorded below with its full SHA after commit. The current local contract suite runs 513 tests with eight macOS skips after adding A09 contracts; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01, G02 and G03 now share a version-race executor that preserves serial overwrite, versioned PUT and exact old/current version-delete ACKs, runs the H1/H2 deep-heal window, verifies read-repair or exact historical reads, checks final peer inventories and physical generations, and retains native commit/cleanup, version and delete-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 138 dispatchable IDs (74 non-Heal and 64 Heal), with 20 Heal executors still intentionally unimplemented. From a38281eebd90fa387d32e0600eea984a8c3b63b8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 17:18:43 +0800 Subject: [PATCH 146/343] feat(validation): migrate G02 versioned put race --- docs/validation-migration-map.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index b0c1c912..acd60dae 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `ec26bd3d3364e7c9291ff57a70087e1d32c2917fa` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03 race executors. The helper is recorded below with its full SHA after commit. The current local contract suite runs 513 tests with eight macOS skips after adding A09 contracts; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01, G02 and G03 now share a version-race executor that preserves serial overwrite, versioned PUT and exact old/current version-delete ACKs, runs the H1/H2 deep-heal window, verifies read-repair or exact historical reads, checks final peer inventories and physical generations, and retains native commit/cleanup, version and delete-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 138 dispatchable IDs (74 non-Heal and 64 Heal), with 20 Heal executors still intentionally unimplemented. +Current reconciliation: helper `ec26bd3d3364e7c9291ff57a70087e1d32c2917fa` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03 race executors. The helper is recorded below with its full SHA after commit. The current local contract suite runs 514 tests with eight macOS skips after adding A09 contracts; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01, G02 and G03 now share a version-race executor that preserves serial overwrite, versioned PUT and exact old/current version-delete ACKs, runs the H1/H2 deep-heal window, verifies read-repair or exact historical reads, checks final peer inventories and physical generations, and retains native commit/cleanup, version and delete-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 138 dispatchable IDs (74 non-Heal and 64 Heal), with 20 Heal executors still intentionally unimplemented. From 478e345512af8dd66d512ba045435f6e51590ffd Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 17:27:00 +0800 Subject: [PATCH 147/343] feat(validation): migrate G06 copy metadata races --- docs/validation-migration-map.md | 8 ++-- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/race.py | 63 ++++++++++++++++++++++++++--- scripts/validation_heal/run_case.sh | 2 +- test_validation_heal_race.py | 11 ++++- 6 files changed, 75 insertions(+), 13 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index acd60dae..2c5b14ee 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -202,7 +202,7 @@ operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. -The RustFS tool-build lane produced the initial candidate-bound metadata inspector in run `38052536366`; A07 also passed with that tool in run `38053214984`. The extended bitrot helper was successfully built against the same candidate in run `38054812014`, including the released-shard oracle checks. Set `validation_tools_only=true` with the pinned build and auto-testing SHA to build the current helper. Heal live runs consume `validation_tools_run_id`/`validation_tools_run_attempt`; the workflow verifies the successful build attempt, candidate/helper identities and binary/lock digests. The node adapter installs the verified tool under its run-owned root. Plain metadata decoding makes no payload-integrity claim; the new explicit part-verification mode validates current and legacy streaming HighwayHash. The build checks 24 healthy shards captured from external released processes and rejects 12 controlled corruptions. Ordinary live execution needs no RustFS source checkout. +The RustFS tool-build lane produced the initial candidate-bound metadata inspector in run `a38281eebd90fa387d32e0600eea984a8c3b63b8`; A07 also passed with that tool in run `a38281eebd90fa387d32e0600eea984a8c3b63b8`. The extended bitrot helper was successfully built against the same candidate in run `a38281eebd90fa387d32e0600eea984a8c3b63b8`, including the released-shard oracle checks. Set `validation_tools_only=true` with the pinned build and auto-testing SHA to build the current helper. Heal live runs consume `validation_tools_run_id`/`validation_tools_run_attempt`; the workflow verifies the successful build attempt, candidate/helper identities and binary/lock digests. The node adapter installs the verified tool under its run-owned root. Plain metadata decoding makes no payload-integrity claim; the new explicit part-verification mode validates current and legacy streaming HighwayHash. The build checks 24 healthy shards captured from external released processes and rejects 12 controlled corruptions. Ordinary live execution needs no RustFS source checkout. The eight planned cases have these concrete reuse candidates: @@ -1582,7 +1582,7 @@ Product physical and S3 assertions run before any fixture rescue. Every live part receives candidate-bound streaming verification before final S3 checks. Hosted tools run `38103474555` attempt 1 succeeded at driver -`9b4cc40abd5ea3a0a5ee6c52fb7e113b57102d6e` and helper +`a38281eebd90fa387d32e0600eea984a8c3b63b8` and helper `6cc8cb79a0f8024b0089a43d31b0708a46bbae4c`. Independent original artifact/API verification confirms all three codec tests, source/tree/Cargo.lock/helper/role and binary `eb2991cf6f84b6998a474c51cd142c2b1c84106fd7f49f569fce7a241dc4ecea`. @@ -1716,7 +1716,7 @@ false admission, wrong identities, premature retirement and resource drift. No I04 canonical live PASS is claimed. There are now 134 automatic executors (74 non-Heal, 60 Heal) and 24 remaining Heal executors. -Read-only current-fleet inspection `38106333882` at helper +Read-only current-fleet inspection `a38281eebd90fa387d32e0600eea984a8c3b63b8` at helper `b75f4a7602dca463c5aa532133ddc022f132051f` found 16 independent eligible endpoints: four on each of the existing four Heal nodes. H1/H2/H3 resource shapes are available; H4's required 24 endpoints are unavailable. This does @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `ec26bd3d3364e7c9291ff57a70087e1d32c2917fa` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03 race executors. The helper is recorded below with its full SHA after commit. The current local contract suite runs 514 tests with eight macOS skips after adding A09 contracts; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01, G02 and G03 now share a version-race executor that preserves serial overwrite, versioned PUT and exact old/current version-delete ACKs, runs the H1/H2 deep-heal window, verifies read-repair or exact historical reads, checks final peer inventories and physical generations, and retains native commit/cleanup, version and delete-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 138 dispatchable IDs (74 non-Heal and 64 Heal), with 20 Heal executors still intentionally unimplemented. +Current reconciliation: helper `a38281eebd90fa387d32e0600eea984a8c3b63b8` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03 race executors. The helper is recorded below with its full SHA after commit. The current local contract suite runs 515 tests with eight macOS skips after adding A09 contracts; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01, G02, G03 and G06 now share a version-race executor that preserves serial overwrite, versioned PUT, exact old/current version-delete, copy, metadata and tag ACKs, runs the H1/H2 deep-heal window, verifies read-repair or exact historical reads, checks final peer inventories and physical generations, and retains native commit/cleanup, version and delete/copy-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 141 dispatchable IDs (74 non-Heal and 65 Heal), with 19 Heal executors still intentionally unimplemented. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index ee0e9876..f3f8b0b1 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-copy-and-metadata-update|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 9c0648a0..7bea796e 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.admin-nolock-safety", "heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.admin-nolock-safety", "heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-copy-and-metadata-update", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index 9af203a9..2a256a0d 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -41,15 +41,17 @@ from versions import public_version_id -CASE_IDS = ("heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete") +CASE_IDS = ("heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-copy-and-metadata-update") CASE_BRANCHES = { "heal.race-unversioned-overwrite": ("overwrite-vs-deep", "overwrite-vs-read-repair", "native-commit-cleanup"), "heal.race-versioned-put": ("new-version-vs-historical-heal", "read-all-versions-during-heal", "native-version-fence"), "heal.race-version-delete": ("delete-old-version-vs-deep", "delete-current-version-vs-deep", "native-delete-fence"), + "heal.race-copy-and-metadata-update": ("copy-source", "copy-destination", "replace-metadata", "update-tags", "native-copy-fence"), } G01_BRANCHES = CASE_BRANCHES["heal.race-unversioned-overwrite"][:2] G02_BRANCHES = CASE_BRANCHES["heal.race-versioned-put"][:2] G03_BRANCHES = CASE_BRANCHES["heal.race-version-delete"][:2] +G06_BRANCHES = CASE_BRANCHES["heal.race-copy-and-metadata-update"][:4] OBJECTS = 257 SIZE = 257 * 1024 + 123 TARGET_KEY = "objects/0256.bin" @@ -123,7 +125,7 @@ def sample_overlap(ack, observed, path, token): def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): - require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G02/G03 unregistered fixture") + require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G06_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G02/G03/G06 unregistered fixture") recorder.call("create_bucket", {"Bucket": bucket}) if branch in (*G02_BRANCHES, *G03_BRANCHES): recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) @@ -144,7 +146,7 @@ def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): def operation_plan(bucket, branch, expected): - require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES), "G01/G02/G03 unregistered foreground branch") + require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G06_BRANCHES), "G01/G02/G03/G06 unregistered foreground branch") if branch in G01_BRANCHES: # Three distinct serial commits; the final independent reference was # written before injection, allowing a coding-index byte comparison. @@ -162,6 +164,21 @@ def operation_plan(bucket, branch, expected): "nonce": f"{bucket}/versioned-put/{index}", "readVersionId": target[0]["versionId"], "readPayload": target[0].get("payload")} for index in range(3)] + if branch in G06_BRANCHES: + if branch == "copy-source": + return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, + "CopySource": {"Bucket": bucket, "Key": REFERENCE_KEY, "VersionId": "null"}}, "nonce": "copy-source"}] + if branch == "copy-destination": + return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": REFERENCE_KEY, + "CopySource": {"Bucket": bucket, "Key": TARGET_KEY, "VersionId": "null"}}, "nonce": "copy-destination"}] + if branch == "replace-metadata": + return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, + "CopySource": {"Bucket": bucket, "Key": TARGET_KEY, "VersionId": "null"}, + "MetadataDirective": "REPLACE", "Metadata": {"race": "replace-metadata"}, + "ContentType": "application/x-heal-race"}, "nonce": "replace-metadata"}] + return [{"operation": "put_object_tagging", "params": {"Bucket": bucket, "Key": TARGET_KEY, + "Tagging": {"TagSet": [{"Key": "race", "Value": f"updated-{index:02d}"}]}}, + "nonce": f"tag-{index:02d}"} for index in range(16)] require(len(target) == 3 and len({row["versionId"] for row in target}) == 3 and target[-1]["isLatest"], "G01/G03 DV lacks three ACK-ordered versions") return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, @@ -169,7 +186,7 @@ def operation_plan(bucket, branch, expected): def final_inventory(baseline, branch, plan, responses): - require(len(plan) == len(responses) == (3 if branch in (*G01_BRANCHES, *G02_BRANCHES) else 1), + require(len(plan) == len(responses) == (3 if branch in (*G01_BRANCHES, *G02_BRANCHES) else 16 if branch == "update-tags" else 1), "G01/G03 foreground plan/ACK sequence is incomplete") result = copy.deepcopy(baseline) if branch in (*G01_BRANCHES, *G02_BRANCHES): @@ -192,6 +209,26 @@ def final_inventory(baseline, branch, plan, responses): "properties": {"ETag": ack["ETag"], "ContentLength": len(body), "ContentType": "application/octet-stream", "Metadata": step["params"]["Metadata"]}, "tags": [{"Key": "race", "Value": "versioned-put"}]}) + elif branch in G06_BRANCHES: + entry = next(row for row in result["entries"] if row["key"] == TARGET_KEY) + copy_result = responses[-1].get("CopyObjectResult") or {} + copy_etag = copy_result.get("ETag") or responses[-1].get("ETag") + if branch == "copy-source": + reference = next(row for row in result["entries"] if row["key"] == REFERENCE_KEY) + entry["payload"] = copy.deepcopy(reference["payload"]) + require(isinstance(copy_etag, str) and copy_etag, "G06 copy-source ACK lacks its committed ETag") + entry["properties"].update(ETag=copy_etag) + elif branch == "copy-destination": + destination = next(row for row in result["entries"] if row["key"] == REFERENCE_KEY) + destination["payload"] = copy.deepcopy(entry["payload"]) + require(isinstance(copy_etag, str) and copy_etag, "G06 copy-destination ACK lacks its committed ETag") + destination["properties"].update(ETag=copy_etag) + elif branch == "replace-metadata": + require(isinstance(copy_etag, str) and copy_etag, "G06 metadata replacement ACK lacks its committed ETag") + entry["properties"].update(Metadata=plan[0]["params"]["Metadata"], ContentType=plan[0]["params"]["ContentType"]) + entry["properties"]["ETag"] = copy_etag + else: + entry["tags"] = copy.deepcopy(plan[-1]["params"]["Tagging"]["TagSet"]) else: version = plan[0]["params"]["VersionId"] require(responses[0].get("VersionId") == version, "G01/G03 DELETE response changed the selected VersionId") @@ -246,6 +283,22 @@ def physical_transition(before, after, bucket, mapped, final, branch): require(all(any(part["size"] == SIZE for part in target["cohort"]) for target in current_target if target["dataDir"] in new_dirs), "G02 new version physical parts are incomplete") + elif branch in G06_BRANCHES: + changed_key = REFERENCE_KEY if branch == "copy-destination" else TARGET_KEY + target_after = next(value for (key, _), value in survivors.items() if key == changed_key) + target_before = next(value for (key, _), value in mapped.items() if key == changed_key) + if branch in {"copy-source", "copy-destination"}: + source_key = REFERENCE_KEY if branch == "copy-source" else TARGET_KEY + source = next(value for (key, _), value in mapped.items() if key == source_key) + require(target_after["dataDir"] != target_before["dataDir"] + and {part["index"]: (part["sha256"], part["size"]) for part in target_after["cohort"]} + == {part["index"]: (part["sha256"], part["size"]) for part in source["cohort"]}, + "G06 copy did not publish a distinct target generation with source shard bytes") + else: + require(target_after["dataDir"] == target_before["dataDir"] + and [(part["index"], part["sha256"], part["size"]) for part in target_after["cohort"]] + == [(part["index"], part["sha256"], part["size"]) for part in target_before["cohort"]], + "G06 metadata/tag update changed the data generation") else: remaining = {(row["key"], row["versionId"]) for row in final["entries"]} removed = [value for pair, value in mapped.items() if pair not in remaining] @@ -570,7 +623,7 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"G01/G03 interrupted by try: stop = False for topology in ("H1", "H2"): - for branch in branches[:2]: + for branch in branches[:-1]: row = run_branch(args, topology, nodes[:1] if topology == "H1" else nodes[:4], branch) row["caseId"] = case_id results.append(row) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index c68a1bd4..0b85b689 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -45,7 +45,7 @@ case "${case_id}" in heal.conflicting-subquorum-authority) driver=authority_runner.py ;; heal.dangling-and-orphan-safety) driver=orphan.py ;; heal.admin-nolock-safety) driver=nolock.py ;; - heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete) driver=race.py ;; + heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-copy-and-metadata-update) driver=race.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal_race.py b/test_validation_heal_race.py index 3d402995..62706705 100644 --- a/test_validation_heal_race.py +++ b/test_validation_heal_race.py @@ -36,6 +36,15 @@ def test_g02_plan_creates_three_acknowledged_versions(self): self.assertEqual({step["readVersionId"] for step in plan}, {"v0"}) self.assertEqual([step["params"]["Metadata"]["nonce"] for step in plan], ["0", "1", "2"]) + def test_g06_keeps_copy_metadata_and_tag_operations_distinct(self): + expected = [{"key": race.TARGET_KEY, "versionId": "null", "isLatest": True}, + {"key": race.REFERENCE_KEY, "versionId": "null", "isLatest": True}] + self.assertEqual(race.operation_plan("rv-heal-0123456789abcdef", "copy-source", expected)[0]["operation"], "copy_object") + self.assertEqual(race.operation_plan("rv-heal-0123456789abcdef", "replace-metadata", expected)[0]["operation"], "copy_object") + tags = race.operation_plan("rv-heal-0123456789abcdef", "update-tags", expected) + self.assertEqual(len(tags), 16) + self.assertEqual(tags[-1]["operation"], "put_object_tagging") + def test_native_branch_remains_in_denominator_for_each_case(self): for case_id, branches in race.CASE_BRANCHES.items(): native = race.native_blocked({"source_sha": race.REFERENCE["candidateCommit"]}, "H1", case_id) @@ -48,7 +57,7 @@ def test_dispatch_and_source_contract_are_present(self): dispatch = Path("scripts/validation_heal/run_case.sh").read_text() top_dispatch = Path("scripts/run_validation_case.sh").read_text() for case_id, code in (("heal.race-unversioned-overwrite", "G01"), ("heal.race-versioned-put", "G02"), - ("heal.race-version-delete", "G03")): + ("heal.race-version-delete", "G03"), ("heal.race-copy-and-metadata-update", "G06")): self.assertIn(case_id, dispatch) self.assertIn(case_id, top_dispatch) catalog = json.loads(Path("validation-cases.json").read_text()) From 7e6d70cabeebe4493d197ad0b09313209f55aa4e Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 17:28:42 +0800 Subject: [PATCH 148/343] docs(validation): restore historical evidence references --- docs/validation-migration-map.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 2c5b14ee..85a91009 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -202,7 +202,7 @@ operator-assisted and are not claimed as migrated automatic executors. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. -The RustFS tool-build lane produced the initial candidate-bound metadata inspector in run `a38281eebd90fa387d32e0600eea984a8c3b63b8`; A07 also passed with that tool in run `a38281eebd90fa387d32e0600eea984a8c3b63b8`. The extended bitrot helper was successfully built against the same candidate in run `a38281eebd90fa387d32e0600eea984a8c3b63b8`, including the released-shard oracle checks. Set `validation_tools_only=true` with the pinned build and auto-testing SHA to build the current helper. Heal live runs consume `validation_tools_run_id`/`validation_tools_run_attempt`; the workflow verifies the successful build attempt, candidate/helper identities and binary/lock digests. The node adapter installs the verified tool under its run-owned root. Plain metadata decoding makes no payload-integrity claim; the new explicit part-verification mode validates current and legacy streaming HighwayHash. The build checks 24 healthy shards captured from external released processes and rejects 12 controlled corruptions. Ordinary live execution needs no RustFS source checkout. +The RustFS tool-build lane produced the initial candidate-bound metadata inspector in run `38052536366`; A07 also passed with that tool in run `38053214984`. The extended bitrot helper was successfully built against the same candidate in run `38054812014`, including the released-shard oracle checks. Set `validation_tools_only=true` with the pinned build and auto-testing SHA to build the current helper. Heal live runs consume `validation_tools_run_id`/`validation_tools_run_attempt`; the workflow verifies the successful build attempt, candidate/helper identities and binary/lock digests. The node adapter installs the verified tool under its run-owned root. Plain metadata decoding makes no payload-integrity claim; the new explicit part-verification mode validates current and legacy streaming HighwayHash. The build checks 24 healthy shards captured from external released processes and rejects 12 controlled corruptions. Ordinary live execution needs no RustFS source checkout. The eight planned cases have these concrete reuse candidates: @@ -1582,7 +1582,7 @@ Product physical and S3 assertions run before any fixture rescue. Every live part receives candidate-bound streaming verification before final S3 checks. Hosted tools run `38103474555` attempt 1 succeeded at driver -`a38281eebd90fa387d32e0600eea984a8c3b63b8` and helper +`9b4cc40abd5ea3a0a5ee6c52fb7e113b57102d6e` and helper `6cc8cb79a0f8024b0089a43d31b0708a46bbae4c`. Independent original artifact/API verification confirms all three codec tests, source/tree/Cargo.lock/helper/role and binary `eb2991cf6f84b6998a474c51cd142c2b1c84106fd7f49f569fce7a241dc4ecea`. @@ -1716,7 +1716,7 @@ false admission, wrong identities, premature retirement and resource drift. No I04 canonical live PASS is claimed. There are now 134 automatic executors (74 non-Heal, 60 Heal) and 24 remaining Heal executors. -Read-only current-fleet inspection `a38281eebd90fa387d32e0600eea984a8c3b63b8` at helper +Read-only current-fleet inspection `38106333882` at helper `b75f4a7602dca463c5aa532133ddc022f132051f` found 16 independent eligible endpoints: four on each of the existing four Heal nodes. H1/H2/H3 resource shapes are available; H4's required 24 endpoints are unavailable. This does @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `a38281eebd90fa387d32e0600eea984a8c3b63b8` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03 race executors. The helper is recorded below with its full SHA after commit. The current local contract suite runs 515 tests with eight macOS skips after adding A09 contracts; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01, G02, G03 and G06 now share a version-race executor that preserves serial overwrite, versioned PUT, exact old/current version-delete, copy, metadata and tag ACKs, runs the H1/H2 deep-heal window, verifies read-repair or exact historical reads, checks final peer inventories and physical generations, and retains native commit/cleanup, version and delete/copy-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 141 dispatchable IDs (74 non-Heal and 65 Heal), with 19 Heal executors still intentionally unimplemented. +Current reconciliation: helper `478e345512af8dd66d512ba045435f6e51590ffd` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G06 race executors. The current local contract suite runs 515 tests with eight macOS skips; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01, G02, G03 and G06 now share a version-race executor that preserves serial overwrite, versioned PUT, exact old/current version-delete, copy, metadata and tag ACKs, runs the H1/H2 deep-heal window, verifies read-repair or exact historical reads, checks final peer inventories and physical generations, and retains native commit/cleanup, version and delete/copy-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 141 dispatchable IDs (74 non-Heal and 65 Heal), with 19 Heal executors still intentionally unimplemented. From 107154153587598afa52668b670a9bf35250e66a Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 17:33:21 +0800 Subject: [PATCH 149/343] fix(validation): bind G06 DS source and target keys --- scripts/validation_heal/race.py | 102 +++++++++++++++++++------------- test_validation_heal_race.py | 2 + 2 files changed, 62 insertions(+), 42 deletions(-) diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index 2a256a0d..a0f3681a 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -56,6 +56,8 @@ SIZE = 257 * 1024 + 123 TARGET_KEY = "objects/0256.bin" REFERENCE_KEY = "objects/0000.bin" +G06_TARGET_KEY = "actual/slash/0256.bin" +G06_REFERENCE_KEY = "space key/0000.bin" REFERENCE = { "candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", "sourceFiles": { @@ -71,6 +73,14 @@ class WindowNotObserved(ValueError): """No observable client overlap; this is not a product success.""" +def target_key(branch): + return G06_TARGET_KEY if branch in G06_BRANCHES else TARGET_KEY + + +def reference_key(branch): + return G06_REFERENCE_KEY if branch in G06_BRANCHES else REFERENCE_KEY + + class RaceRecorder(Recorder): """Keep unresolved mutations and exact SDK receipts after any assertion.""" def __init__(self, *args, versioned, **kwargs): @@ -130,9 +140,13 @@ def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): if branch in (*G02_BRANCHES, *G03_BRANCHES): recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) entries = [] - keys = [f"objects/{index:04d}.bin" for index in range(count - 1)] + [TARGET_KEY] + if branch in G06_BRANCHES: + keys = [G06_REFERENCE_KEY] + [f"ds/{index:04d}.bin" for index in range(1, count - 1)] + [G06_TARGET_KEY] + else: + keys = [f"objects/{index:04d}.bin" for index in range(count - 1)] + [TARGET_KEY] + current_target = target_key(branch) for key in keys: - versions = 3 if key == TARGET_KEY and branch in (*G02_BRANCHES, *G03_BRANCHES) else 1 + versions = 3 if key == current_target and branch in (*G02_BRANCHES, *G03_BRANCHES) else 1 for generation in range(versions): nonce = f"{bucket}/{key}/{generation}" body = payload(nonce, SIZE) @@ -155,7 +169,8 @@ def operation_plan(bucket, branch, expected): "Tagging": "race=overwrite"}, "nonce": f"{bucket}/overwrite/{index}" if index < 2 else f"{bucket}/{REFERENCE_KEY}/0"} for index in range(3)] - target = [row for row in expected if row["key"] == TARGET_KEY] + current_target = target_key(branch); current_reference = reference_key(branch) + target = [row for row in expected if row["key"] == current_target] if branch in G02_BRANCHES: require(len(target) == 3 and len({row["versionId"] for row in target}) == 3, "G02 DV lacks three ACK-ordered versions") return [{"operation": "put_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, @@ -166,22 +181,22 @@ def operation_plan(bucket, branch, expected): for index in range(3)] if branch in G06_BRANCHES: if branch == "copy-source": - return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, - "CopySource": {"Bucket": bucket, "Key": REFERENCE_KEY, "VersionId": "null"}}, "nonce": "copy-source"}] + return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": current_target, + "CopySource": {"Bucket": bucket, "Key": current_reference, "VersionId": "null"}}, "nonce": "copy-source"}] if branch == "copy-destination": - return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": REFERENCE_KEY, - "CopySource": {"Bucket": bucket, "Key": TARGET_KEY, "VersionId": "null"}}, "nonce": "copy-destination"}] + return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": current_reference, + "CopySource": {"Bucket": bucket, "Key": current_target, "VersionId": "null"}}, "nonce": "copy-destination"}] if branch == "replace-metadata": - return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, - "CopySource": {"Bucket": bucket, "Key": TARGET_KEY, "VersionId": "null"}, + return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": current_target, + "CopySource": {"Bucket": bucket, "Key": current_target, "VersionId": "null"}, "MetadataDirective": "REPLACE", "Metadata": {"race": "replace-metadata"}, "ContentType": "application/x-heal-race"}, "nonce": "replace-metadata"}] - return [{"operation": "put_object_tagging", "params": {"Bucket": bucket, "Key": TARGET_KEY, + return [{"operation": "put_object_tagging", "params": {"Bucket": bucket, "Key": current_target, "Tagging": {"TagSet": [{"Key": "race", "Value": f"updated-{index:02d}"}]}}, "nonce": f"tag-{index:02d}"} for index in range(16)] require(len(target) == 3 and len({row["versionId"] for row in target}) == 3 and target[-1]["isLatest"], "G01/G03 DV lacks three ACK-ordered versions") - return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, + return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": current_target, "VersionId": (target[0] if branch == G03_BRANCHES[0] else target[-1])["versionId"]}}] @@ -189,8 +204,9 @@ def final_inventory(baseline, branch, plan, responses): require(len(plan) == len(responses) == (3 if branch in (*G01_BRANCHES, *G02_BRANCHES) else 16 if branch == "update-tags" else 1), "G01/G03 foreground plan/ACK sequence is incomplete") result = copy.deepcopy(baseline) + current_target, current_reference = target_key(branch), reference_key(branch) if branch in (*G01_BRANCHES, *G02_BRANCHES): - entry = next(row for row in result["entries"] if row["key"] == TARGET_KEY) + entry = next(row for row in result["entries"] if row["key"] == current_target) step, ack = plan[-1], responses[-1] body = payload(step["nonce"], SIZE) require(isinstance(ack.get("ETag"), str) and ack["ETag"] @@ -202,24 +218,24 @@ def final_inventory(baseline, branch, plan, responses): Metadata=step["params"]["Metadata"]) else: for row in result["entries"]: - if row["key"] == TARGET_KEY: row["isLatest"] = False - result["entries"].append({"key": TARGET_KEY, "versionId": public_version_id(ack.get("VersionId")), + if row["key"] == current_target: row["isLatest"] = False + result["entries"].append({"key": current_target, "versionId": public_version_id(ack.get("VersionId")), "isLatest": True, "deleteMarker": False, "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}, "properties": {"ETag": ack["ETag"], "ContentLength": len(body), "ContentType": "application/octet-stream", "Metadata": step["params"]["Metadata"]}, "tags": [{"Key": "race", "Value": "versioned-put"}]}) elif branch in G06_BRANCHES: - entry = next(row for row in result["entries"] if row["key"] == TARGET_KEY) + entry = next(row for row in result["entries"] if row["key"] == current_target) copy_result = responses[-1].get("CopyObjectResult") or {} copy_etag = copy_result.get("ETag") or responses[-1].get("ETag") if branch == "copy-source": - reference = next(row for row in result["entries"] if row["key"] == REFERENCE_KEY) + reference = next(row for row in result["entries"] if row["key"] == current_reference) entry["payload"] = copy.deepcopy(reference["payload"]) require(isinstance(copy_etag, str) and copy_etag, "G06 copy-source ACK lacks its committed ETag") entry["properties"].update(ETag=copy_etag) elif branch == "copy-destination": - destination = next(row for row in result["entries"] if row["key"] == REFERENCE_KEY) + destination = next(row for row in result["entries"] if row["key"] == current_reference) destination["payload"] = copy.deepcopy(entry["payload"]) require(isinstance(copy_etag, str) and copy_etag, "G06 copy-destination ACK lacks its committed ETag") destination["properties"].update(ETag=copy_etag) @@ -233,7 +249,7 @@ def final_inventory(baseline, branch, plan, responses): version = plan[0]["params"]["VersionId"] require(responses[0].get("VersionId") == version, "G01/G03 DELETE response changed the selected VersionId") result["entries"] = [row for row in result["entries"] - if (row["key"], row["versionId"]) != (TARGET_KEY, version)] + if (row["key"], row["versionId"]) != (current_target, version)] require(len(result["entries"]) + 1 == len(baseline["entries"]), "G01/G03 deleted a foreign/missing version") return result @@ -259,36 +275,37 @@ def physical_transition(before, after, bucket, mapped, final, branch): known = {row["key"] for row in final["entries"]} require(all(any(row["path"].startswith(f"{bucket}/{key}/") for key in known) for value in after.values() for row in value["files"]), "G01/G03 created an unacknowledged physical key") + current_target, current_reference = target_key(branch), reference_key(branch) for (key, version), target in mapped.items(): - if key == TARGET_KEY: continue + if key == current_target: continue validate_repaired_physical(object_scene(before, bucket, key), object_scene(after, bucket, key), target) survivors = map_fixture(after, bucket, final["entries"]) - original_target = [value for (key, _), value in mapped.items() if key == TARGET_KEY] - current_target = [value for (key, _), value in survivors.items() if key == TARGET_KEY] + original_target = [value for (key, _), value in mapped.items() if key == current_target] + current_target_values = [value for (key, _), value in survivors.items() if key == current_target] if branch in G01_BRANCHES: - require(len(original_target) == len(current_target) == 1 - and original_target[0]["dataDir"] != current_target[0]["dataDir"], + require(len(original_target) == len(current_target_values) == 1 + and original_target[0]["dataDir"] != current_target_values[0]["dataDir"], "G01/G03 overwrite retained the old physical generation") - reference = next(target for (key, _), target in mapped.items() if key == REFERENCE_KEY) + reference = next(target for (key, _), target in mapped.items() if key == current_reference) require({part["index"]: (part["sha256"], part["size"]) for part in reference["cohort"]} - == {part["index"]: (part["sha256"], part["size"]) for part in current_target[0]["cohort"]}, + == {part["index"]: (part["sha256"], part["size"]) for part in current_target_values[0]["cohort"]}, "G01/G03 last ACK shard bytes/coding indices differ from the independent payload reference") elif branch in G02_BRANCHES: - require(len(original_target) == 3 and len(current_target) == 6, + require(len(original_target) == 3 and len(current_target_values) == 6, "G02 version race lost an acknowledged historical or new version") old_dirs = {target["dataDir"] for target in original_target} - new_dirs = {target["dataDir"] for target in current_target} - old_dirs + new_dirs = {target["dataDir"] for target in current_target_values} - old_dirs require(len(new_dirs) == 3 and old_dirs.isdisjoint(new_dirs), "G02 versions share or resurrect a physical data directory") require(all(any(part["size"] == SIZE for part in target["cohort"]) - for target in current_target if target["dataDir"] in new_dirs), + for target in current_target_values if target["dataDir"] in new_dirs), "G02 new version physical parts are incomplete") elif branch in G06_BRANCHES: - changed_key = REFERENCE_KEY if branch == "copy-destination" else TARGET_KEY + changed_key = current_reference if branch == "copy-destination" else current_target target_after = next(value for (key, _), value in survivors.items() if key == changed_key) target_before = next(value for (key, _), value in mapped.items() if key == changed_key) if branch in {"copy-source", "copy-destination"}: - source_key = REFERENCE_KEY if branch == "copy-source" else TARGET_KEY + source_key = current_reference if branch == "copy-source" else current_target source = next(value for (key, _), value in mapped.items() if key == source_key) require(target_after["dataDir"] != target_before["dataDir"] and {part["index"]: (part["sha256"], part["size"]) for part in target_after["cohort"]} @@ -303,16 +320,16 @@ def physical_transition(before, after, bucket, mapped, final, branch): remaining = {(row["key"], row["versionId"]) for row in final["entries"]} removed = [value for pair, value in mapped.items() if pair not in remaining] require(len(removed) == 1, "G01/G03 delete transition lacks one exact deleted version") - for target in current_target: - original = mapped[TARGET_KEY, target["versionId"]] + for target in current_target_values: + original = mapped[current_target, target["versionId"]] require(target["logical"] == original["logical"] and [(row["index"], row["sha256"], row["size"]) for row in target["cohort"]] == [(row["index"], row["sha256"], row["size"]) for row in original["cohort"]], "G01/G03 delete changed another retained ACK generation") - current_dirs = {target["dataDir"] for target in current_target} + current_dirs = {target["dataDir"] for target in current_target_values} for value in after.values(): for row in value["files"]: - prefix = f"{bucket}/{TARGET_KEY}/" + prefix = f"{bucket}/{current_target}/" if not row["path"].startswith(prefix): continue suffix = row["path"][len(prefix):] require(suffix == "xl.meta" or suffix.split("/")[0] in current_dirs, @@ -462,10 +479,11 @@ def run_branch(args, topology, nodes, branch): "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), "maxInventoryObjects": 512, "maxInventoryBytes": 512 * 1024**2} fleet.record("binding", binding) + current_target_key, current_reference_key = target_key(branch), reference_key(branch) fleet.record("source-contract", {**REFERENCE, "branch": branch, "objectCount": OBJECTS, "objectSize": SIZE, "requiredPUTs": 3 if branch in (*G01_BRANCHES, *G02_BRANCHES) else 0, "serverCommitOverlapProved": False}) probe, active, recorder, writer = Probe(binding), [], None, None - result = {"topology": topology, "branch": branch, "fixture": "DL" if branch in G01_BRANCHES else "DV", + result = {"topology": topology, "branch": branch, "fixture": "DL" if branch in G01_BRANCHES else "DS" if branch in G06_BRANCHES else "DV", "verdict": "RUNNING", "cleanup": "PENDING", "canonicalCasePassClaimed": False} changed_target_registered, target_authority = None, False try: @@ -473,7 +491,7 @@ def run_branch(args, topology, nodes, branch): fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) recorder = RaceRecorder(probe, directory / "seed-ACKs", versioned=branch in (*G02_BRANCHES, *G03_BRANCHES)) expected = seed_fixture(recorder, bucket, branch) - baseline = fleet.record("complete-baseline-S3", probe.inventory(bucket, unversioned_current_tags=branch in G01_BRANCHES)) + baseline = fleet.record("complete-baseline-S3", probe.inventory(bucket, unversioned_current_tags=branch in (*G01_BRANCHES, *G06_BRANCHES))) verify_seed(expected, baseline) plan = operation_plan(bucket, branch, expected) for step in plan: @@ -483,17 +501,17 @@ def run_branch(args, topology, nodes, branch): before = fleet.snapshot("complete-baseline-physical", quiescent=True) mapped = map_fixture(before, bucket, expected) verify_part_manifest(fleet, list(mapped.values()), "baseline-streaming", count=len(mapped)) - target_version = (next(entry for entry in expected if entry["key"] == TARGET_KEY and entry["isLatest"]) + target_version = (next(entry for entry in expected if entry["key"] == current_target_key and entry["isLatest"]) if branch in (G02_BRANCHES[1], G03_BRANCHES[1]) - else next(entry for entry in expected if entry["key"] == TARGET_KEY)) + else next(entry for entry in expected if entry["key"] == current_target_key)) targets = [mapped[row["key"], row["versionId"]] for row in expected - if row["key"] != TARGET_KEY or row["versionId"] == target_version["versionId"]] + if row["key"] != current_target_key or row["versionId"] == target_version["versionId"]] require(len(targets) == OBJECTS, "G01/G03 fault plan omitted a key or changed its source dimension") receipts = [] for index, target in enumerate(targets): params = {"disk": target["disk"], "faultId": bounded_fault_id("G01/G03", branch, index)} fleet.faults.append((target["node"], params)) - if target["key"] == TARGET_KEY: changed_target_registered = target["node"], params + if target["key"] == current_target_key: changed_target_registered = target["node"], params receipts.append(fleet.record("exact-missing-part", {"node": target["node"], **fleet.remote( target["node"], "apply-fault", **params, relativePath=target["relativePath"], expectedSha256=target["sha256"], kind="missing")})) @@ -510,7 +528,7 @@ def run_branch(args, topology, nodes, branch): active.append((path, token)) wait_running(probe, fleet, path, token, len(expected)) witness = fleet.snapshot("target-missing-before-foreground") - target = next(target for target in targets if target["key"] == TARGET_KEY) + target = next(target for target in targets if target["key"] == current_target_key) if (target["node"], target["disk"], target["relativePath"]) in files(witness): raise WindowNotObserved("G01/G03 old target already repaired before foreground mutations") writer = RaceRecorder(probe, directory / "foreground-ACKs", versioned=branch in (*G02_BRANCHES, *G03_BRANCHES)) @@ -541,7 +559,7 @@ def run_branch(args, topology, nodes, branch): fleet.start() for endpoint in binding["endpoints"]: current = fleet.record("all-peer-final-S3", {"endpoint": endpoint, **Probe(binding, endpoint).inventory( - bucket, unversioned_current_tags=branch in G01_BRANCHES)}) + bucket, unversioned_current_tags=branch in (*G01_BRANCHES, *G06_BRANCHES))}) compare_inventories(final, current) if branch in G03_BRANCHES: negative = fleet.record("all-peer-deleted-version-GET", Probe(binding, endpoint).s3("get_object", plan[0]["params"])) diff --git a/test_validation_heal_race.py b/test_validation_heal_race.py index 62706705..80c96499 100644 --- a/test_validation_heal_race.py +++ b/test_validation_heal_race.py @@ -44,6 +44,8 @@ def test_g06_keeps_copy_metadata_and_tag_operations_distinct(self): tags = race.operation_plan("rv-heal-0123456789abcdef", "update-tags", expected) self.assertEqual(len(tags), 16) self.assertEqual(tags[-1]["operation"], "put_object_tagging") + self.assertEqual(race.target_key("copy-source"), race.G06_TARGET_KEY) + self.assertEqual(race.reference_key("copy-source"), race.G06_REFERENCE_KEY) def test_native_branch_remains_in_denominator_for_each_case(self): for case_id, branches in race.CASE_BRANCHES.items(): From 30df613d9772c03ed7a197fb69bcf61d83e385cf Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 17:33:45 +0800 Subject: [PATCH 150/343] docs(validation): record G06 key binding helper --- docs/validation-migration-map.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 85a91009..0610c91a 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `478e345512af8dd66d512ba045435f6e51590ffd` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G06 race executors. The current local contract suite runs 515 tests with eight macOS skips; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01, G02, G03 and G06 now share a version-race executor that preserves serial overwrite, versioned PUT, exact old/current version-delete, copy, metadata and tag ACKs, runs the H1/H2 deep-heal window, verifies read-repair or exact historical reads, checks final peer inventories and physical generations, and retains native commit/cleanup, version and delete/copy-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 141 dispatchable IDs (74 non-Heal and 65 Heal), with 19 Heal executors still intentionally unimplemented. +Current reconciliation: helper `107154153587598afa52668b670a9bf35250e66a` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G06 race executors. The current local contract suite runs 515 tests with eight macOS skips; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01, G02, G03 and G06 now share a version-race executor that preserves serial overwrite, versioned PUT, exact old/current version-delete, copy, metadata and tag ACKs, runs the H1/H2 deep-heal window, verifies read-repair or exact historical reads, checks final peer inventories and physical generations, and retains native commit/cleanup, version and delete/copy-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 141 dispatchable IDs (74 non-Heal and 65 Heal), with 19 Heal executors still intentionally unimplemented. From 6a8b8ea3c942fbb2c09635edfdf25451195d76c1 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 17:49:49 +0800 Subject: [PATCH 151/343] feat(validation): migrate G05 multipart race --- docs/validation-migration-map.md | 2 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/race.py | 207 ++++++++++++++++++++++++++-- scripts/validation_heal/run_case.sh | 2 +- test_validation_heal_race.py | 45 +++++- 6 files changed, 241 insertions(+), 19 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 0610c91a..8e72c1f6 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `107154153587598afa52668b670a9bf35250e66a` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G06 race executors. The current local contract suite runs 515 tests with eight macOS skips; the preceding Linux contract run `38119194648` passed its gates and ran 486 tests. The independent canonical census records 86 unique complete live PASS cases, split into 74 current-candidate and 12 older-candidate cases. Full-status-budget run `38111278609` remains a canonical FAIL with cleanup PASS; its retained root report exposed an explicit durable bucket event, and the validator fix is covered by the current helper. Performance run `38112625718` remains a canonical FAIL because the original 45-minute fault lease expired before group 3 phase 3; no fault was applied and no canonical PASS is added. Evidence-based cleanup run `38116647348` independently verified all four nodes already clean with inactive services and removed namespace paths, recording `cleanup: PASS` and `canonicalPassClaimed: false`. The performance executor now pre-registers a 120-minute lease and cleanup is idempotent for an already-clean namespace. Network run `38117058670` remains a canonical FAIL because the minority-partition below-quorum GET assertion failed; cleanup run `38117588046` proved the four owned expiry gates, stopped the expired units, removed all nftables rules and finalized every namespace with `cleanup: PASS` and `canonicalPassClaimed: false`. H06 run `38119207583`, I04 `38119569421`, C07 `38119830051`, C10 `38120042167`, A10 `38120735805`, and B08 `38124084134` all have cleanup PASS but no complete canonical PASS claim; branch-level PASS evidence remains subordinate to the canonical case result. These counts exclude native, read-only, recovery, duplicate and incomplete pool-tail audits. A09 now has an auto-testing executor for both H1/H2 black-box branches, with exact requested/effective `nolock` option normalization, durable overwrite/delete ACK ledgers, observed running overlap, physical shard fault/repair proofs and run-owned cleanup. The `native-lock-fence` branch remains explicitly BLOCKED in the denominator because the pinned release binary is not an e2e-test-hooks build; A09 therefore adds executor coverage but no canonical live PASS claim. G01, G02, G03 and G06 now share a version-race executor that preserves serial overwrite, versioned PUT, exact old/current version-delete, copy, metadata and tag ACKs, runs the H1/H2 deep-heal window, verifies read-repair or exact historical reads, checks final peer inventories and physical generations, and retains native commit/cleanup, version and delete/copy-fence branches as BLOCKED until a reviewed candidate barrier exists. The matrix is now 141 dispatchable IDs (74 non-Heal and 65 Heal), with 19 Heal executors still intentionally unimplemented. +Current reconciliation: helper `30df613d9772c03ed7a197fb69bcf61d83e385cf` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G05/G06 race executors. The current local contract suite runs 516 tests with eight macOS skips; the preceding Linux contract run `38109944413` ran 473 tests without skips. The independent canonical census remains the source of live PASS totals; native, read-only, recovery, duplicate and incomplete runs are excluded, and the new G05 executor adds no canonical live PASS claim while its `native-mpu-fence` branch is BLOCKED. A09, G01, G02, G03, G05 and G06 preserve their source branch IDs, ACK-bound foreground operations, H1/H2 physical evidence, peer convergence and run-owned cleanup; native commit/version/delete/MPU/copy fences remain in the denominator until a reviewed candidate barrier exists. The matrix is now 142 dispatchable IDs (74 non-Heal and 66 Heal), with 18 Heal executors still intentionally unimplemented. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index f3f8b0b1..4f9a2199 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-copy-and-metadata-update|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 7bea796e..506d6e8f 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.admin-nolock-safety", "heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-copy-and-metadata-update", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.admin-nolock-safety", "heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-multipart-complete-abort", "heal.race-copy-and-metadata-update", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index a0f3681a..05cf7a6e 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -41,16 +41,18 @@ from versions import public_version_id -CASE_IDS = ("heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-copy-and-metadata-update") +CASE_IDS = ("heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-multipart-complete-abort", "heal.race-copy-and-metadata-update") CASE_BRANCHES = { "heal.race-unversioned-overwrite": ("overwrite-vs-deep", "overwrite-vs-read-repair", "native-commit-cleanup"), "heal.race-versioned-put": ("new-version-vs-historical-heal", "read-all-versions-during-heal", "native-version-fence"), "heal.race-version-delete": ("delete-old-version-vs-deep", "delete-current-version-vs-deep", "native-delete-fence"), + "heal.race-multipart-complete-abort": ("upload-continue", "complete", "abort", "native-mpu-fence"), "heal.race-copy-and-metadata-update": ("copy-source", "copy-destination", "replace-metadata", "update-tags", "native-copy-fence"), } G01_BRANCHES = CASE_BRANCHES["heal.race-unversioned-overwrite"][:2] G02_BRANCHES = CASE_BRANCHES["heal.race-versioned-put"][:2] G03_BRANCHES = CASE_BRANCHES["heal.race-version-delete"][:2] +G05_BRANCHES = CASE_BRANCHES["heal.race-multipart-complete-abort"][:3] G06_BRANCHES = CASE_BRANCHES["heal.race-copy-and-metadata-update"][:4] OBJECTS = 257 SIZE = 257 * 1024 + 123 @@ -58,6 +60,8 @@ REFERENCE_KEY = "objects/0000.bin" G06_TARGET_KEY = "actual/slash/0256.bin" G06_REFERENCE_KEY = "space key/0000.bin" +G05_TARGET_KEY = "multipart/target.bin" +G05_PART_SIZES = (5 * 1024 * 1024 + 17, 1024 * 1024 + 93, 1024 * 1024 + 47) REFERENCE = { "candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", "sourceFiles": { @@ -74,6 +78,7 @@ class WindowNotObserved(ValueError): def target_key(branch): + if branch in G05_BRANCHES: return G05_TARGET_KEY return G06_TARGET_KEY if branch in G06_BRANCHES else TARGET_KEY @@ -135,8 +140,33 @@ def sample_overlap(ack, observed, path, token): def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): - require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G06_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G02/G03/G06 unregistered fixture") + require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G05_BRANCHES, *G06_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G02/G03/G05/G06 unregistered fixture") recorder.call("create_bucket", {"Bucket": bucket}) + if branch in G05_BRANCHES: + entries = [] + for index in range(count - 1): + key = f"objects/{index:04d}.bin" + body = payload(f"{bucket}/{key}/0", SIZE) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": "DM", "nonce": str(index)}, + "Tagging": "suite=heal&fixture=DM"}, body) + entries.append({"key": key, "versionId": public_version_id(response.get("VersionId")), + "isLatest": True, "deleteMarker": False, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + upload = recorder.call("create_multipart_upload", {"Bucket": bucket, "Key": G05_TARGET_KEY, + "Metadata": {"heal-fixture": "DM", "multipart": "target"}, "ContentType": "application/octet-stream"}) + parts, digest = [], hashlib.sha256() + for number, size in enumerate(G05_PART_SIZES, 1): + body = payload(f"{bucket}/{G05_TARGET_KEY}/{number}", size); digest.update(body) + response = recorder.call("upload_part", {"Bucket": bucket, "Key": G05_TARGET_KEY, + "UploadId": upload["UploadId"], "PartNumber": number}, body) + parts.append({"PartNumber": number, "ETag": response["ETag"]}) + completed = recorder.call("complete_multipart_upload", {"Bucket": bucket, "Key": G05_TARGET_KEY, + "UploadId": upload["UploadId"], "MultipartUpload": {"Parts": parts}}) + entries.append({"key": G05_TARGET_KEY, "versionId": public_version_id(completed.get("VersionId")), + "isLatest": True, "deleteMarker": False, + "payload": {"size": sum(G05_PART_SIZES), "sha256": digest.hexdigest()}}) + return entries if branch in (*G02_BRANCHES, *G03_BRANCHES): recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) entries = [] @@ -160,7 +190,7 @@ def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): def operation_plan(bucket, branch, expected): - require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G06_BRANCHES), "G01/G02/G03/G06 unregistered foreground branch") + require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G05_BRANCHES, *G06_BRANCHES), "G01/G02/G03/G05/G06 unregistered foreground branch") if branch in G01_BRANCHES: # Three distinct serial commits; the final independent reference was # written before injection, allowing a coding-index byte comparison. @@ -179,6 +209,38 @@ def operation_plan(bucket, branch, expected): "nonce": f"{bucket}/versioned-put/{index}", "readVersionId": target[0]["versionId"], "readPayload": target[0].get("payload")} for index in range(3)] + if branch in G05_BRANCHES: + # Each branch uses a fresh UploadId for the same already committed + # object. The foreground resolver binds every later request to the + # exact create ACK and never guesses an UploadId from LIST output. + if branch == "upload-continue": + return [ + {"operation": "create_multipart_upload", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY}, + "uploadName": "continue"}, + {"operation": "upload_part", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY, + "UploadIdRef": "continue", "PartNumber": 1}, "nonce": f"{bucket}/{G05_TARGET_KEY}/continue/1", + "size": G05_PART_SIZES[0]}, + ] + if branch == "complete": + return [ + {"operation": "create_multipart_upload", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY}, + "uploadName": "complete"}, + *[{"operation": "upload_part", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY, + "UploadIdRef": "complete", "PartNumber": number}, + "nonce": f"{bucket}/{G05_TARGET_KEY}/complete/{number}", "size": size} + for number, size in enumerate(G05_PART_SIZES, 1)], + {"operation": "complete_multipart_upload", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY, + "UploadIdRef": "complete", "MultipartUpload": {"Parts": []}}, "uploadName": "completed"}, + ] + return [ + {"operation": "create_multipart_upload", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY}, + "uploadName": "abort"}, + {"operation": "upload_part", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY, + "UploadIdRef": "abort", "PartNumber": 1}, "nonce": f"{bucket}/{G05_TARGET_KEY}/abort/1", + "size": G05_PART_SIZES[0]}, + {"operation": "abort_multipart_upload", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY, + "UploadIdRef": "abort"}, "uploadName": "aborted"}, + ] if branch in G06_BRANCHES: if branch == "copy-source": return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": current_target, @@ -201,8 +263,11 @@ def operation_plan(bucket, branch, expected): def final_inventory(baseline, branch, plan, responses): - require(len(plan) == len(responses) == (3 if branch in (*G01_BRANCHES, *G02_BRANCHES) else 16 if branch == "update-tags" else 1), - "G01/G03 foreground plan/ACK sequence is incomplete") + expected_length = (3 if branch in (*G01_BRANCHES, *G02_BRANCHES) + else len(plan) if branch in G05_BRANCHES + else 16 if branch == "update-tags" else 1) + require(len(plan) == len(responses) == expected_length, + "G01/G03/G05 foreground plan/ACK sequence is incomplete") result = copy.deepcopy(baseline) current_target, current_reference = target_key(branch), reference_key(branch) if branch in (*G01_BRANCHES, *G02_BRANCHES): @@ -225,6 +290,27 @@ def final_inventory(baseline, branch, plan, responses): "properties": {"ETag": ack["ETag"], "ContentLength": len(body), "ContentType": "application/octet-stream", "Metadata": step["params"]["Metadata"]}, "tags": [{"Key": "race", "Value": "versioned-put"}]}) + elif branch in G05_BRANCHES: + runtime = plan[0].get("runtime") or {} + upload_id = runtime.get("uploadIds", {}).get(branch.split("-", 1)[-1]) + require(isinstance(upload_id, str) and upload_id, "G05 foreground lost its exact UploadId") + if branch == "complete": + entry = next(row for row in result["entries"] if row["key"] == current_target) + digest = hashlib.sha256() + total = 0 + for number, size in enumerate(G05_PART_SIZES, 1): + body = payload(f"{plan[0]['params']['Bucket']}/{G05_TARGET_KEY}/complete/{number}", size) + digest.update(body); total += len(body) + ack = responses[-1] + require(isinstance(ack.get("ETag"), str) and ack["ETag"], + "G05 Complete ACK lacks its committed ETag") + entry.update(payload={"size": total, "sha256": digest.hexdigest()}, + versionId=public_version_id(ack.get("VersionId")), isLatest=True, + deleteMarker=False) + entry.setdefault("properties", {}).update(ETag=ack["ETag"], ContentLength=total, + ContentType="application/octet-stream") + else: + require(branch in {"upload-continue", "abort"}, "G05 unknown MPU branch") elif branch in G06_BRANCHES: entry = next(row for row in result["entries"] if row["key"] == current_target) copy_result = responses[-1].get("CopyObjectResult") or {} @@ -300,6 +386,21 @@ def physical_transition(before, after, bucket, mapped, final, branch): require(all(any(part["size"] == SIZE for part in target["cohort"]) for target in current_target_values if target["dataDir"] in new_dirs), "G02 new version physical parts are incomplete") + elif branch in G05_BRANCHES: + require(len(original_target) == len(current_target_values) == 1, + "G05 multipart transition changed the target identity count") + if branch == "complete": + require(original_target[0]["dataDir"] != current_target_values[0]["dataDir"], + "G05 Complete retained the old physical generation") + require(len(current_target_values[0]["cohort"]) == len(original_target[0]["cohort"]) + and all(part["size"] > 0 for part in current_target_values[0]["cohort"]), + "G05 Complete physical parts are incomplete") + else: + require(original_target[0]["dataDir"] == current_target_values[0]["dataDir"], + "G05 pending/aborted MPU changed the committed object generation") + require([(part["index"], part["sha256"], part["size"]) for part in original_target[0]["cohort"]] + == [(part["index"], part["sha256"], part["size"]) for part in current_target_values[0]["cohort"]], + "G05 pending/aborted MPU changed committed object bytes") elif branch in G06_BRANCHES: changed_key = current_reference if branch == "copy-destination" else current_target target_after = next(value for (key, _), value in survivors.items() if key == changed_key) @@ -403,10 +504,35 @@ def wait_terminal(probe, fleet, path, token): def foreground(recorder, plan): responses = [] + upload_ids = {} + part_etags = {} for step in plan: - body = payload(step["nonce"], SIZE) if step["operation"] == "put_object" else None - responses.append(recorder.call(step["operation"], step["params"], body)) - return responses + params = copy.deepcopy(step["params"]) + upload_ref = params.pop("UploadIdRef", None) + if upload_ref is not None: + require(upload_ref in upload_ids, "G05 foreground referenced an unacknowledged UploadId") + params["UploadId"] = upload_ids[upload_ref] + if step["operation"] == "complete_multipart_upload": + parts = [{"PartNumber": number, "ETag": part_etags[(upload_ref, number)]} + for number, _ in enumerate(G05_PART_SIZES, 1)] + params["MultipartUpload"] = {"Parts": parts} + body = None + if step["operation"] == "put_object": + body = payload(step["nonce"], SIZE) + elif step["operation"] == "upload_part": + body = payload(step["nonce"], step["size"]) + response = recorder.call(step["operation"], params, body) + responses.append(response) + if step.get("uploadName") in {"continue", "complete", "abort"}: + require(isinstance(response.get("UploadId"), str) and response["UploadId"], + "G05 CreateMultipartUpload ACK lacks UploadId") + upload_ids[step["uploadName"]] = response["UploadId"] + if step["operation"] == "upload_part": + require(isinstance(response.get("ETag"), str) and response["ETag"], + "G05 UploadPart ACK lacks ETag") + part_etags[(upload_ref, params["PartNumber"])] = response["ETag"] + plan[0]["runtime"] = {"uploadIds": upload_ids, "partEtags": part_etags} + return responses, {"uploadIds": upload_ids, "partEtags": part_etags} def race(recorder, plan, probe, fleet, path, token): @@ -445,7 +571,9 @@ def read_repair(): recorder.mutations.poison("status observation failed; drain in-flight mutations only") raise if not worker.done(): recorder.mutations.poison("foreground deadline; drain only") - responses = worker.result(timeout=60) + responses, runtime = worker.result(timeout=60) + if plan and runtime: + plan[0]["runtime"] = runtime if read_future is not None: read_future.result(timeout=60) acks = [json.loads(path.read_text()) for path in sorted(recorder.output.glob("*-ack.json"))] @@ -460,6 +588,38 @@ def read_repair(): return responses, all(overlaps) +def verify_multipart_state(probe, fleet, bucket, branch, runtime, expected_parts): + """Bind MPU control evidence to the foreground UploadId/part ACKs.""" + require(branch in G05_BRANCHES and isinstance(runtime, dict), "G05 runtime receipt is missing") + name = branch.split("-", 1)[-1] + upload_id = (runtime.get("uploadIds") or {}).get(name) + require(isinstance(upload_id, str) and upload_id, "G05 runtime UploadId is missing") + listed = fleet.record("G05-list-multipart-uploads", probe.s3("list_multipart_uploads", {"Bucket": bucket})) + uploads = (listed.get("response") or {}).get("Uploads", []) + matching = [row for row in uploads if row.get("UploadId") == upload_id] + if branch == "upload-continue": + require(len(matching) == 1, "G05 continued UploadId disappeared from ListMultipartUploads") + parts = fleet.record("G05-list-parts", probe.s3("list_parts", {"Bucket": bucket, "Key": G05_TARGET_KEY, + "UploadId": upload_id})) + rows = (parts.get("response") or {}).get("Parts", []) + require(len(rows) == len(expected_parts) and all( + row.get("PartNumber") == number and row.get("ETag") == runtime["partEtags"][(name, number)] + and row.get("Size") == size for row, (number, size) in zip(rows, expected_parts)), + "G05 continued UploadId lost an acknowledged part") + else: + require(not matching, f"G05 {branch} left a completed/aborted UploadId visible") + if branch == "abort": + parts = fleet.record("G05-list-aborted-parts", probe.s3("list_parts", {"Bucket": bucket, + "Key": G05_TARGET_KEY, + "UploadId": upload_id})) + response = parts.get("response") or {} + require(parts.get("acknowledged") is False and parts.get("resultUnknown") is False + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404, + "G05 aborted UploadId still exposes acknowledged parts") + return {"uploadId": upload_id, "listedUploads": len(uploads), "branch": branch, + "acknowledgedParts": len(expected_parts) if branch == "upload-continue" else 0} + + def run_branch(args, topology, nodes, branch): directory = args.evidence / topology / branch directory.mkdir(parents=True, exist_ok=False) @@ -483,15 +643,18 @@ def run_branch(args, topology, nodes, branch): fleet.record("source-contract", {**REFERENCE, "branch": branch, "objectCount": OBJECTS, "objectSize": SIZE, "requiredPUTs": 3 if branch in (*G01_BRANCHES, *G02_BRANCHES) else 0, "serverCommitOverlapProved": False}) probe, active, recorder, writer = Probe(binding), [], None, None - result = {"topology": topology, "branch": branch, "fixture": "DL" if branch in G01_BRANCHES else "DS" if branch in G06_BRANCHES else "DV", + result = {"topology": topology, "branch": branch, + "fixture": "DL" if branch in G01_BRANCHES else "DM" if branch in G05_BRANCHES + else "DS" if branch in G06_BRANCHES else "DV", "verdict": "RUNNING", "cleanup": "PENDING", "canonicalCasePassClaimed": False} - changed_target_registered, target_authority = None, False + changed_target_registered, target_authority, mpu_runtime = None, False, None try: fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) recorder = RaceRecorder(probe, directory / "seed-ACKs", versioned=branch in (*G02_BRANCHES, *G03_BRANCHES)) expected = seed_fixture(recorder, bucket, branch) - baseline = fleet.record("complete-baseline-S3", probe.inventory(bucket, unversioned_current_tags=branch in (*G01_BRANCHES, *G06_BRANCHES))) + baseline = fleet.record("complete-baseline-S3", probe.inventory( + bucket, unversioned_current_tags=branch in (*G01_BRANCHES, *G05_BRANCHES, *G06_BRANCHES))) verify_seed(expected, baseline) plan = operation_plan(bucket, branch, expected) for step in plan: @@ -534,6 +697,10 @@ def run_branch(args, topology, nodes, branch): writer = RaceRecorder(probe, directory / "foreground-ACKs", versioned=branch in (*G02_BRANCHES, *G03_BRANCHES)) writer.seen.update((row["key"], row["versionId"]) for row in expected) responses, overlapped = race(writer, plan, probe, fleet, path, token) + mpu_runtime = plan[0].get("runtime") if branch in G05_BRANCHES else None + if branch in G05_BRANCHES: + verify_multipart_state(probe, fleet, bucket, branch, mpu_runtime, + [(1, G05_PART_SIZES[0])] if branch != "complete" else list(enumerate(G05_PART_SIZES, 1))) final = fleet.record("ACK-derived-final-manifest", final_inventory(baseline, branch, plan, responses)) terminal = wait_terminal(probe, fleet, path, token) active.remove((path, token)) @@ -559,7 +726,7 @@ def run_branch(args, topology, nodes, branch): fleet.start() for endpoint in binding["endpoints"]: current = fleet.record("all-peer-final-S3", {"endpoint": endpoint, **Probe(binding, endpoint).inventory( - bucket, unversioned_current_tags=branch in (*G01_BRANCHES, *G06_BRANCHES))}) + bucket, unversioned_current_tags=branch in (*G01_BRANCHES, *G05_BRANCHES, *G06_BRANCHES))}) compare_inventories(final, current) if branch in G03_BRANCHES: negative = fleet.record("all-peer-deleted-version-GET", Probe(binding, endpoint).s3("get_object", plan[0]["params"])) @@ -586,6 +753,17 @@ def run_branch(args, topology, nodes, branch): for path, token in active: try: fleet.record("cancel-original-token", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) except Exception as exc: fleet.record("cancel-error", {"error": str(exc)}) + if branch == "upload-continue" and mpu_runtime: + try: + fleet.start() + upload_id = mpu_runtime["uploadIds"]["continue"] + fleet.record("G05-cleanup-abort", probe.s3("abort_multipart_upload", { + "Bucket": bucket, "Key": G05_TARGET_KEY, "UploadId": upload_id})) + fleet.record("G05-cleanup-abort-proof", verify_multipart_state( + probe, fleet, bucket, "abort", {"uploadIds": {"abort": upload_id}, "partEtags": {}}, [])) + except Exception as exc: + fleet.record("G05-cleanup-abort-error", {"error": str(exc), "uploadId": mpu_runtime.get("uploadIds", {}).get("continue")}) + result["cleanup"] = "FAIL" unresolved = any(item is not None and item.pending is not None for item in (recorder, writer)) mutated = writer is not None and writer.sequence > 0 if unresolved or mutated and not target_authority: @@ -596,7 +774,8 @@ def run_branch(args, topology, nodes, branch): "originalGenerationBlindRestorationProhibited": True}) result["cleanup"] = "FAIL" else: - result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + cleanup_ok = fleet.cleanup() + result["cleanup"] = "FAIL" if result.get("cleanup") == "FAIL" or not cleanup_ok else "PASS" fleet.record("branch-result", result) return result diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 0b85b689..5eb3e239 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -45,7 +45,7 @@ case "${case_id}" in heal.conflicting-subquorum-authority) driver=authority_runner.py ;; heal.dangling-and-orphan-safety) driver=orphan.py ;; heal.admin-nolock-safety) driver=nolock.py ;; - heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-copy-and-metadata-update) driver=race.py ;; + heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update) driver=race.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal_race.py b/test_validation_heal_race.py index 80c96499..ed263697 100644 --- a/test_validation_heal_race.py +++ b/test_validation_heal_race.py @@ -47,6 +47,48 @@ def test_g06_keeps_copy_metadata_and_tag_operations_distinct(self): self.assertEqual(race.target_key("copy-source"), race.G06_TARGET_KEY) self.assertEqual(race.reference_key("copy-source"), race.G06_REFERENCE_KEY) + def test_g05_binds_multipart_operations_to_acknowledged_upload(self): + expected = [{"key": race.G05_TARGET_KEY, "versionId": "null", "isLatest": True}] + continued = race.operation_plan("rv-heal-0123456789abcdef", "upload-continue", expected) + completed = race.operation_plan("rv-heal-0123456789abcdef", "complete", expected) + aborted = race.operation_plan("rv-heal-0123456789abcdef", "abort", expected) + self.assertEqual([step["operation"] for step in continued], ["create_multipart_upload", "upload_part"]) + self.assertEqual([step["operation"] for step in completed], + ["create_multipart_upload", "upload_part", "upload_part", "upload_part", + "complete_multipart_upload"]) + self.assertEqual([step["operation"] for step in aborted], + ["create_multipart_upload", "upload_part", "abort_multipart_upload"]) + self.assertEqual([step["size"] for step in completed[1:4]], list(race.G05_PART_SIZES)) + self.assertEqual(completed[-1]["params"]["MultipartUpload"]["Parts"], []) + self.assertEqual(race.target_key("complete"), race.G05_TARGET_KEY) + + def test_g05_foreground_resolves_upload_id_and_part_etags_from_ack(self): + class FakeRecorder: + def __init__(self): + self.calls = [] + + def call(self, operation, params, body=None): + self.calls.append((operation, params, body)) + if operation == "create_multipart_upload": + return {"UploadId": "upload-ack"} + if operation == "upload_part": + return {"ETag": f"etag-{params['PartNumber']}"} + return {"ETag": "complete-etag", "VersionId": "null"} + + recorder = FakeRecorder() + plan = race.operation_plan("rv-heal-0123456789abcdef", "complete", []) + responses, runtime = race.foreground(recorder, plan) + self.assertEqual(runtime["uploadIds"]["complete"], "upload-ack") + self.assertEqual(recorder.calls[1][1]["UploadId"], "upload-ack") + self.assertEqual(recorder.calls[-1][1]["UploadId"], "upload-ack") + self.assertEqual(recorder.calls[-1][1]["MultipartUpload"], { + "Parts": [{"PartNumber": 1, "ETag": "etag-1"}, + {"PartNumber": 2, "ETag": "etag-2"}, + {"PartNumber": 3, "ETag": "etag-3"}]}) + self.assertEqual([len(call[2]) if call[2] is not None else None for call in recorder.calls], + [None, *race.G05_PART_SIZES, None]) + self.assertEqual(len(responses), 5) + def test_native_branch_remains_in_denominator_for_each_case(self): for case_id, branches in race.CASE_BRANCHES.items(): native = race.native_blocked({"source_sha": race.REFERENCE["candidateCommit"]}, "H1", case_id) @@ -59,7 +101,8 @@ def test_dispatch_and_source_contract_are_present(self): dispatch = Path("scripts/validation_heal/run_case.sh").read_text() top_dispatch = Path("scripts/run_validation_case.sh").read_text() for case_id, code in (("heal.race-unversioned-overwrite", "G01"), ("heal.race-versioned-put", "G02"), - ("heal.race-version-delete", "G03"), ("heal.race-copy-and-metadata-update", "G06")): + ("heal.race-version-delete", "G03"), ("heal.race-multipart-complete-abort", "G05"), + ("heal.race-copy-and-metadata-update", "G06")): self.assertIn(case_id, dispatch) self.assertIn(case_id, top_dispatch) catalog = json.loads(Path("validation-cases.json").read_text()) From e6d3e8be33541b00e4dabb61a24acd3cba13e063 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 18:01:04 +0800 Subject: [PATCH 152/343] feat(validation): migrate G04 marker races --- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/matrix.py | 2 +- scripts/validation_heal/race.py | 150 +++++++++++++++++++++++++--- scripts/validation_heal/run_case.sh | 2 +- test_validation_heal_race.py | 15 ++- 5 files changed, 153 insertions(+), 18 deletions(-) diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 4f9a2199..df51f4d0 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-markers-and-batch-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 506d6e8f..042ade3d 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -21,7 +21,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.admin-nolock-safety", "heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-multipart-complete-abort", "heal.race-copy-and-metadata-update", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", +AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.admin-nolock-safety", "heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-markers-and-batch-delete", "heal.race-multipart-complete-abort", "heal.race-copy-and-metadata-update", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index 05cf7a6e..cd345cc4 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -41,17 +41,19 @@ from versions import public_version_id -CASE_IDS = ("heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-multipart-complete-abort", "heal.race-copy-and-metadata-update") +CASE_IDS = ("heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-markers-and-batch-delete", "heal.race-multipart-complete-abort", "heal.race-copy-and-metadata-update") CASE_BRANCHES = { "heal.race-unversioned-overwrite": ("overwrite-vs-deep", "overwrite-vs-read-repair", "native-commit-cleanup"), "heal.race-versioned-put": ("new-version-vs-historical-heal", "read-all-versions-during-heal", "native-version-fence"), "heal.race-version-delete": ("delete-old-version-vs-deep", "delete-current-version-vs-deep", "native-delete-fence"), + "heal.race-markers-and-batch-delete": ("create-marker", "delete-marker", "batch-version-delete", "unversioned-delete", "native-marker-fence"), "heal.race-multipart-complete-abort": ("upload-continue", "complete", "abort", "native-mpu-fence"), "heal.race-copy-and-metadata-update": ("copy-source", "copy-destination", "replace-metadata", "update-tags", "native-copy-fence"), } G01_BRANCHES = CASE_BRANCHES["heal.race-unversioned-overwrite"][:2] G02_BRANCHES = CASE_BRANCHES["heal.race-versioned-put"][:2] G03_BRANCHES = CASE_BRANCHES["heal.race-version-delete"][:2] +G04_BRANCHES = CASE_BRANCHES["heal.race-markers-and-batch-delete"][:4] G05_BRANCHES = CASE_BRANCHES["heal.race-multipart-complete-abort"][:3] G06_BRANCHES = CASE_BRANCHES["heal.race-copy-and-metadata-update"][:4] OBJECTS = 257 @@ -91,6 +93,7 @@ class RaceRecorder(Recorder): def __init__(self, *args, versioned, **kwargs): super().__init__(*args, **kwargs) self.versioned, self.pending, self.seen = versioned, None, set() + self.marker_branch = None def call(self, operation, params, body=None, **kwargs): self.pending = {"operation": operation, "params": copy.deepcopy(params)} @@ -106,10 +109,30 @@ def call(self, operation, params, body=None, **kwargs): else: require(version in {None, "null"}, "G01/G03 never-versioned PUT created a real version") elif operation == "delete_object": - require(self.versioned and params.get("VersionId") in { - value for key, value in self.seen if key == params["Key"]} - and version == params["VersionId"] and result.get("DeleteMarker") in {None, False}, - "G01/G03 DELETE ACK changed the exact known data-version identity") + if self.marker_branch in G04_BRANCHES: + known = {value for key, value in self.seen if key == params["Key"]} + if params.get("VersionId") is None: + if self.versioned: + require(result.get("DeleteMarker") is True and isinstance(version, str) + and version not in known, "G04 marker creation ACK lacks a new marker identity") + self.seen.add((params["Key"], version)) + else: + require(version in {None, "null"} and result.get("DeleteMarker") in {None, False}, + "G04 unversioned DELETE ACK created an unexpected marker") + else: + require(self.versioned and params["VersionId"] in known and version == params["VersionId"] + and result.get("DeleteMarker") is True, + "G04 marker removal ACK changed the selected marker identity") + else: + require(self.versioned and params.get("VersionId") in { + value for key, value in self.seen if key == params["Key"]} + and version == params["VersionId"] and result.get("DeleteMarker") in {None, False}, + "G01/G03 DELETE ACK changed the exact known data-version identity") + elif operation == "delete_objects" and self.marker_branch == "batch-version-delete": + deleted = result.get("Deleted") or [] + errors = result.get("Errors") or [] + require(not errors and len(deleted) == 2 and all(isinstance(row.get("VersionId"), str) for row in deleted), + "G04 batch DELETE ACK has errors or incomplete per-item receipts") self.pending = None return result @@ -140,8 +163,40 @@ def sample_overlap(ack, observed, path, token): def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): - require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G05_BRANCHES, *G06_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G02/G03/G05/G06 unregistered fixture") + require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES, *G05_BRANCHES, *G06_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G02/G03/G04/G05/G06 unregistered fixture") recorder.call("create_bucket", {"Bucket": bucket}) + if branch in G04_BRANCHES: + versioned = branch != "unversioned-delete" + if versioned: + recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) + entries = [] + for index in range(count - 1): + key = f"objects/{index:04d}.bin" + body = payload(f"{bucket}/{key}/0", SIZE) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": "G04", "nonce": str(index)}, + "Tagging": "suite=heal&fixture=G04"}, body) + entries.append({"key": key, "versionId": public_version_id(response.get("VersionId")), + "isLatest": True, "deleteMarker": False, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + target_versions = 1 if not versioned else 3 + for generation in range(target_versions): + body = payload(f"{bucket}/{TARGET_KEY}/{generation}", SIZE) + response = recorder.call("put_object", {"Bucket": bucket, "Key": TARGET_KEY, + "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": "G04", "generation": str(generation)}, + "Tagging": "suite=heal&fixture=G04"}, body) + version = public_version_id(response.get("VersionId")) + for row in entries: + if row["key"] == TARGET_KEY: row["isLatest"] = False + entries.append({"key": TARGET_KEY, "versionId": version, "isLatest": generation == target_versions - 1, + "deleteMarker": False, "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + if branch == "delete-marker": + marker = recorder.call("delete_object", {"Bucket": bucket, "Key": TARGET_KEY}) + marker_id = marker.get("VersionId") + for row in entries: + if row["key"] == TARGET_KEY: row["isLatest"] = False + entries.append({"key": TARGET_KEY, "versionId": marker_id, "isLatest": True, "deleteMarker": True}) + return entries if branch in G05_BRANCHES: entries = [] for index in range(count - 1): @@ -190,7 +245,7 @@ def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): def operation_plan(bucket, branch, expected): - require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G05_BRANCHES, *G06_BRANCHES), "G01/G02/G03/G05/G06 unregistered foreground branch") + require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES, *G05_BRANCHES, *G06_BRANCHES), "G01/G02/G03/G04/G05/G06 unregistered foreground branch") if branch in G01_BRANCHES: # Three distinct serial commits; the final independent reference was # written before injection, allowing a coding-index byte comparison. @@ -209,6 +264,27 @@ def operation_plan(bucket, branch, expected): "nonce": f"{bucket}/versioned-put/{index}", "readVersionId": target[0]["versionId"], "readPayload": target[0].get("payload")} for index in range(3)] + if branch in G04_BRANCHES: + target = [row for row in expected if row["key"] == TARGET_KEY] + if branch == "create-marker": + require(len(target) == 3 and target[-1]["isLatest"], "G04 create-marker fixture lacks current data") + return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": TARGET_KEY}, + "nonce": "create-marker"}] + if branch == "delete-marker": + marker = next((row for row in target if row["deleteMarker"] and row["isLatest"]), None) + require(marker is not None, "G04 delete-marker fixture lacks an acknowledged current marker") + return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, + "VersionId": marker["versionId"]}, + "nonce": "delete-marker"}] + if branch == "batch-version-delete": + data = [row for row in target if not row["deleteMarker"]] + require(len(data) == 3, "G04 batch fixture lacks three data versions") + return [{"operation": "delete_objects", "params": {"Bucket": bucket, + "Delete": {"Objects": [{"Key": TARGET_KEY, "VersionId": data[0]["versionId"]}, + {"Key": TARGET_KEY, "VersionId": data[1]["versionId"]}], + "Quiet": False}}, "nonce": "batch-version-delete"}] + return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": TARGET_KEY}, + "nonce": "unversioned-delete"}] if branch in G05_BRANCHES: # Each branch uses a fresh UploadId for the same already committed # object. The foreground resolver binds every later request to the @@ -290,6 +366,44 @@ def final_inventory(baseline, branch, plan, responses): "properties": {"ETag": ack["ETag"], "ContentLength": len(body), "ContentType": "application/octet-stream", "Metadata": step["params"]["Metadata"]}, "tags": [{"Key": "race", "Value": "versioned-put"}]}) + elif branch in G04_BRANCHES: + require(len(responses) == 1, "G04 foreground plan/ACK sequence is incomplete") + target_rows = [row for row in result["entries"] if row["key"] == TARGET_KEY] + if branch == "create-marker": + marker = responses[0].get("VersionId") + require(responses[0].get("DeleteMarker") is True and isinstance(marker, str), + "G04 marker creation ACK lacks its exact marker identity") + for row in target_rows: row["isLatest"] = False + result["entries"].append({"key": TARGET_KEY, "versionId": marker, + "isLatest": True, "deleteMarker": True}) + elif branch == "delete-marker": + marker = next(row for row in target_rows if row["deleteMarker"] and row["isLatest"]) + require(responses[0].get("DeleteMarker") is True and responses[0].get("VersionId") == marker["versionId"], + "G04 marker delete ACK changed the selected marker identity") + result["entries"] = [row for row in result["entries"] if row is not marker] + data = [row for row in result["entries"] if row["key"] == TARGET_KEY and not row["deleteMarker"]] + require(data, "G04 marker delete removed every retained data version") + for row in data: row["isLatest"] = row is data[-1] + elif branch == "batch-version-delete": + deleted = responses[0].get("Deleted") or [] + deleted_ids = {row.get("VersionId") for row in deleted} + require(len(deleted_ids) == 2 and not responses[0].get("Errors"), + "G04 batch delete ACK is not a complete per-item success") + result["entries"] = [row for row in result["entries"] + if not (row["key"] == TARGET_KEY and row["versionId"] in deleted_ids)] + else: + require(responses[0].get("VersionId") in {None, "null"} and responses[0].get("DeleteMarker") in {None, False}, + "G04 unversioned delete ACK changed object identity") + result["entries"] = [row for row in result["entries"] if row["key"] != TARGET_KEY] + elif branch in G04_BRANCHES: + if branch == "unversioned-delete": + require(len(original_target) == 1 and not current_target_values, + "G04 unversioned DELETE retained or replaced the committed target") + else: + require(len(original_target) >= 1 and len(current_target_values) >= 1, + "G04 marker operation lost the retained data generation") + require(current_target_values[0]["dataDir"] in {target["dataDir"] for target in original_target}, + "G04 marker operation changed the committed data generation") elif branch in G05_BRANCHES: runtime = plan[0].get("runtime") or {} upload_id = runtime.get("uploadIds", {}).get(branch.split("-", 1)[-1]) @@ -347,10 +461,11 @@ def object_scene(snapshot, bucket, key): def map_fixture(snapshot, bucket, expected): + live_expected = [row for row in expected if not row.get("deleteMarker")] mapped = {(row["key"], row["versionId"]): map_target( object_scene(snapshot, bucket, row["key"]), bucket, row["key"], "data", 1, - row["versionId"], latest_required=False) for row in expected} - require(len(mapped) == len(expected), "G01/G03 physical map omitted/duplicated an ACK") + row["versionId"], latest_required=False) for row in live_expected} + require(len(mapped) == len(live_expected), "G01/G04 physical map omitted/duplicated an ACK") return mapped @@ -432,6 +547,8 @@ def physical_transition(before, after, bucket, mapped, final, branch): for row in value["files"]: prefix = f"{bucket}/{current_target}/" if not row["path"].startswith(prefix): continue + if branch == "unversioned-delete": + raise ValueError("G04 unversioned DELETE left a physical target leaf") suffix = row["path"][len(prefix):] require(suffix == "xl.meta" or suffix.split("/")[0] in current_dirs, "G01/G03 resurrected an old/deleted data directory or left an unbound target leaf") @@ -644,17 +761,20 @@ def run_branch(args, topology, nodes, branch): "requiredPUTs": 3 if branch in (*G01_BRANCHES, *G02_BRANCHES) else 0, "serverCommitOverlapProved": False}) probe, active, recorder, writer = Probe(binding), [], None, None result = {"topology": topology, "branch": branch, - "fixture": "DL" if branch in G01_BRANCHES else "DM" if branch in G05_BRANCHES + "fixture": "DL" if branch in (*G01_BRANCHES, "unversioned-delete") else "DV" if branch in G04_BRANCHES + else "DM" if branch in G05_BRANCHES else "DS" if branch in G06_BRANCHES else "DV", "verdict": "RUNNING", "cleanup": "PENDING", "canonicalCasePassClaimed": False} changed_target_registered, target_authority, mpu_runtime = None, False, None try: fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) - recorder = RaceRecorder(probe, directory / "seed-ACKs", versioned=branch in (*G02_BRANCHES, *G03_BRANCHES)) + recorder = RaceRecorder(probe, directory / "seed-ACKs", + versioned=branch in (*G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES[:3])) + recorder.marker_branch = branch if branch in G04_BRANCHES else None expected = seed_fixture(recorder, bucket, branch) baseline = fleet.record("complete-baseline-S3", probe.inventory( - bucket, unversioned_current_tags=branch in (*G01_BRANCHES, *G05_BRANCHES, *G06_BRANCHES))) + bucket, unversioned_current_tags=branch in (*G01_BRANCHES, "unversioned-delete", *G05_BRANCHES, *G06_BRANCHES))) verify_seed(expected, baseline) plan = operation_plan(bucket, branch, expected) for step in plan: @@ -694,7 +814,9 @@ def run_branch(args, topology, nodes, branch): target = next(target for target in targets if target["key"] == current_target_key) if (target["node"], target["disk"], target["relativePath"]) in files(witness): raise WindowNotObserved("G01/G03 old target already repaired before foreground mutations") - writer = RaceRecorder(probe, directory / "foreground-ACKs", versioned=branch in (*G02_BRANCHES, *G03_BRANCHES)) + writer = RaceRecorder(probe, directory / "foreground-ACKs", + versioned=branch in (*G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES[:3])) + writer.marker_branch = branch if branch in G04_BRANCHES else None writer.seen.update((row["key"], row["versionId"]) for row in expected) responses, overlapped = race(writer, plan, probe, fleet, path, token) mpu_runtime = plan[0].get("runtime") if branch in G05_BRANCHES else None @@ -726,7 +848,7 @@ def run_branch(args, topology, nodes, branch): fleet.start() for endpoint in binding["endpoints"]: current = fleet.record("all-peer-final-S3", {"endpoint": endpoint, **Probe(binding, endpoint).inventory( - bucket, unversioned_current_tags=branch in (*G01_BRANCHES, *G05_BRANCHES, *G06_BRANCHES))}) + bucket, unversioned_current_tags=branch in (*G01_BRANCHES, "unversioned-delete", *G05_BRANCHES, *G06_BRANCHES))}) compare_inventories(final, current) if branch in G03_BRANCHES: negative = fleet.record("all-peer-deleted-version-GET", Probe(binding, endpoint).s3("get_object", plan[0]["params"])) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 5eb3e239..f756fcb6 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -45,7 +45,7 @@ case "${case_id}" in heal.conflicting-subquorum-authority) driver=authority_runner.py ;; heal.dangling-and-orphan-safety) driver=orphan.py ;; heal.admin-nolock-safety) driver=nolock.py ;; - heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update) driver=race.py ;; + heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-markers-and-batch-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update) driver=race.py ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) diff --git a/test_validation_heal_race.py b/test_validation_heal_race.py index ed263697..05302a86 100644 --- a/test_validation_heal_race.py +++ b/test_validation_heal_race.py @@ -89,6 +89,18 @@ def call(self, operation, params, body=None): [None, *race.G05_PART_SIZES, None]) self.assertEqual(len(responses), 5) + def test_g04_preserves_marker_and_batch_delete_semantics(self): + expected = [{"key": race.TARGET_KEY, "versionId": value, "isLatest": value == "v2", + "deleteMarker": False} for value in ("v0", "v1", "v2")] + with_marker = [*expected, {"key": race.TARGET_KEY, "versionId": "marker", "isLatest": True, "deleteMarker": True}] + self.assertEqual(race.operation_plan("rv-heal-0123456789abcdef", "create-marker", expected)[0]["operation"], + "delete_object") + self.assertEqual(race.operation_plan("rv-heal-0123456789abcdef", "delete-marker", with_marker)[0]["params"]["VersionId"], + "marker") + batch = race.operation_plan("rv-heal-0123456789abcdef", "batch-version-delete", expected)[0] + self.assertEqual(batch["operation"], "delete_objects") + self.assertEqual(len(batch["params"]["Delete"]["Objects"]), 2) + def test_native_branch_remains_in_denominator_for_each_case(self): for case_id, branches in race.CASE_BRANCHES.items(): native = race.native_blocked({"source_sha": race.REFERENCE["candidateCommit"]}, "H1", case_id) @@ -101,7 +113,8 @@ def test_dispatch_and_source_contract_are_present(self): dispatch = Path("scripts/validation_heal/run_case.sh").read_text() top_dispatch = Path("scripts/run_validation_case.sh").read_text() for case_id, code in (("heal.race-unversioned-overwrite", "G01"), ("heal.race-versioned-put", "G02"), - ("heal.race-version-delete", "G03"), ("heal.race-multipart-complete-abort", "G05"), + ("heal.race-version-delete", "G03"), ("heal.race-markers-and-batch-delete", "G04"), + ("heal.race-multipart-complete-abort", "G05"), ("heal.race-copy-and-metadata-update", "G06")): self.assertIn(case_id, dispatch) self.assertIn(case_id, top_dispatch) From 47534b78fe5d5c066dc760200057004ec57f65e6 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 18:03:26 +0800 Subject: [PATCH 153/343] docs(validation): record G04 and G05 executors --- docs/validation-migration-map.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 8e72c1f6..b6697c9e 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `30df613d9772c03ed7a197fb69bcf61d83e385cf` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G05/G06 race executors. The current local contract suite runs 516 tests with eight macOS skips; the preceding Linux contract run `38109944413` ran 473 tests without skips. The independent canonical census remains the source of live PASS totals; native, read-only, recovery, duplicate and incomplete runs are excluded, and the new G05 executor adds no canonical live PASS claim while its `native-mpu-fence` branch is BLOCKED. A09, G01, G02, G03, G05 and G06 preserve their source branch IDs, ACK-bound foreground operations, H1/H2 physical evidence, peer convergence and run-owned cleanup; native commit/version/delete/MPU/copy fences remain in the denominator until a reviewed candidate barrier exists. The matrix is now 142 dispatchable IDs (74 non-Heal and 66 Heal), with 18 Heal executors still intentionally unimplemented. +Current reconciliation: helper `e6d3e8be33541b00e4dabb61a24acd3cba13e063` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G04/G05/G06 race executors. The current local contract suite runs 518 tests with eight macOS skips; the preceding Linux contract run `38109944413` ran 473 tests without skips. The independent canonical census remains the source of live PASS totals; native, read-only, recovery, duplicate and incomplete runs are excluded, and the new G04/G05 executors add no canonical live PASS claim while their native marker/MPU fences are BLOCKED. A09 and G01-G06 preserve their source branch IDs, ACK-bound foreground operations, H1/H2 physical evidence, peer convergence and run-owned cleanup; native commit/version/delete/marker/MPU/copy fences remain in the denominator until a reviewed candidate barrier exists. The matrix is now 143 dispatchable IDs (74 non-Heal and 67 Heal), with 17 Heal executors still intentionally unimplemented. From a6af7e69c4f9a5d12e88cca0824e699af68bd142 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 18:04:25 +0800 Subject: [PATCH 154/343] docs(validation): pin latest helper revision --- docs/validation-migration-map.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index b6697c9e..e459b98e 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `e6d3e8be33541b00e4dabb61a24acd3cba13e063` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G04/G05/G06 race executors. The current local contract suite runs 518 tests with eight macOS skips; the preceding Linux contract run `38109944413` ran 473 tests without skips. The independent canonical census remains the source of live PASS totals; native, read-only, recovery, duplicate and incomplete runs are excluded, and the new G04/G05 executors add no canonical live PASS claim while their native marker/MPU fences are BLOCKED. A09 and G01-G06 preserve their source branch IDs, ACK-bound foreground operations, H1/H2 physical evidence, peer convergence and run-owned cleanup; native commit/version/delete/marker/MPU/copy fences remain in the denominator until a reviewed candidate barrier exists. The matrix is now 143 dispatchable IDs (74 non-Heal and 67 Heal), with 17 Heal executors still intentionally unimplemented. +Current reconciliation: helper `47534b78fe5d5c066dc760200057004ec57f65e6` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G04/G05/G06 race executors. The current local contract suite runs 518 tests with eight macOS skips; the preceding Linux contract run `38109944413` ran 473 tests without skips. The independent canonical census remains the source of live PASS totals; native, read-only, recovery, duplicate and incomplete runs are excluded, and the new G04/G05 executors add no canonical live PASS claim while their native marker/MPU fences are BLOCKED. A09 and G01-G06 preserve their source branch IDs, ACK-bound foreground operations, H1/H2 physical evidence, peer convergence and run-owned cleanup; native commit/version/delete/marker/MPU/copy fences remain in the denominator until a reviewed candidate barrier exists. The matrix is now 143 dispatchable IDs (74 non-Heal and 67 Heal), with 17 Heal executors still intentionally unimplemented. From 6f91210f224dc0a9e460b8240c4a4cf73d7e64c1 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 18:08:50 +0800 Subject: [PATCH 155/343] fix(validation): bind G04 batch delete receipts --- scripts/validation_heal/race.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index cd345cc4..eef47fab 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -94,6 +94,7 @@ def __init__(self, *args, versioned, **kwargs): super().__init__(*args, **kwargs) self.versioned, self.pending, self.seen = versioned, None, set() self.marker_branch = None + self.batch_expected_ids = set() def call(self, operation, params, body=None, **kwargs): self.pending = {"operation": operation, "params": copy.deepcopy(params)} @@ -131,7 +132,9 @@ def call(self, operation, params, body=None, **kwargs): elif operation == "delete_objects" and self.marker_branch == "batch-version-delete": deleted = result.get("Deleted") or [] errors = result.get("Errors") or [] - require(not errors and len(deleted) == 2 and all(isinstance(row.get("VersionId"), str) for row in deleted), + deleted_ids = {row.get("VersionId") for row in deleted} + require(not errors and deleted_ids == self.batch_expected_ids + and all(isinstance(value, str) for value in deleted_ids), "G04 batch DELETE ACK has errors or incomplete per-item receipts") self.pending = None return result @@ -818,6 +821,8 @@ def run_branch(args, topology, nodes, branch): versioned=branch in (*G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES[:3])) writer.marker_branch = branch if branch in G04_BRANCHES else None writer.seen.update((row["key"], row["versionId"]) for row in expected) + if branch == "batch-version-delete": + writer.batch_expected_ids = {item["VersionId"] for item in plan[0]["params"]["Delete"]["Objects"]} responses, overlapped = race(writer, plan, probe, fleet, path, token) mpu_runtime = plan[0].get("runtime") if branch in G05_BRANCHES else None if branch in G05_BRANCHES: From c844fba670342b311b91dadd8f76c7abdb049e0c Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 18:10:04 +0800 Subject: [PATCH 156/343] docs(validation): pin G04 receipt fix --- docs/validation-migration-map.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index e459b98e..05df60dd 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `47534b78fe5d5c066dc760200057004ec57f65e6` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G04/G05/G06 race executors. The current local contract suite runs 518 tests with eight macOS skips; the preceding Linux contract run `38109944413` ran 473 tests without skips. The independent canonical census remains the source of live PASS totals; native, read-only, recovery, duplicate and incomplete runs are excluded, and the new G04/G05 executors add no canonical live PASS claim while their native marker/MPU fences are BLOCKED. A09 and G01-G06 preserve their source branch IDs, ACK-bound foreground operations, H1/H2 physical evidence, peer convergence and run-owned cleanup; native commit/version/delete/marker/MPU/copy fences remain in the denominator until a reviewed candidate barrier exists. The matrix is now 143 dispatchable IDs (74 non-Heal and 67 Heal), with 17 Heal executors still intentionally unimplemented. +Current reconciliation: helper `6f91210f224dc0a9e460b8240c4a4cf73d7e64c1` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G04/G05/G06 race executors. The current local contract suite runs 518 tests with eight macOS skips; the preceding Linux contract run `38109944413` ran 473 tests without skips. The independent canonical census remains the source of live PASS totals; native, read-only, recovery, duplicate and incomplete runs are excluded, and the new G04/G05 executors add no canonical live PASS claim while their native marker/MPU fences are BLOCKED. A09 and G01-G06 preserve their source branch IDs, ACK-bound foreground operations, H1/H2 physical evidence, peer convergence and run-owned cleanup; native commit/version/delete/marker/MPU/copy fences remain in the denominator until a reviewed candidate barrier exists. The matrix is now 143 dispatchable IDs (74 non-Heal and 67 Heal), with 17 Heal executors still intentionally unimplemented. From 58e760b6baea6b08c67e74b641e5ef3aa49e8b27 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 19:31:06 +0800 Subject: [PATCH 157/343] docs(validation): correct dispatch coverage count --- docs/validation-migration-map.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 05df60dd..2de37a11 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1878,4 +1878,4 @@ independently confirms 490 tests without skips at the preceding helper; no result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. -Current reconciliation: helper `6f91210f224dc0a9e460b8240c4a4cf73d7e64c1` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G04/G05/G06 race executors. The current local contract suite runs 518 tests with eight macOS skips; the preceding Linux contract run `38109944413` ran 473 tests without skips. The independent canonical census remains the source of live PASS totals; native, read-only, recovery, duplicate and incomplete runs are excluded, and the new G04/G05 executors add no canonical live PASS claim while their native marker/MPU fences are BLOCKED. A09 and G01-G06 preserve their source branch IDs, ACK-bound foreground operations, H1/H2 physical evidence, peer convergence and run-owned cleanup; native commit/version/delete/marker/MPU/copy fences remain in the denominator until a reviewed candidate barrier exists. The matrix is now 143 dispatchable IDs (74 non-Heal and 67 Heal), with 17 Heal executors still intentionally unimplemented. +Current reconciliation: helper `6f91210f224dc0a9e460b8240c4a4cf73d7e64c1` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G04/G05/G06 race executors. The current local contract suite runs 518 tests with eight macOS skips; the preceding Linux contract run `38109944413` ran 473 tests without skips. The independent canonical census remains the source of live PASS totals; native, read-only, recovery, duplicate and incomplete runs are excluded, and the new G04/G05 executors add no canonical live PASS claim while their native marker/MPU fences are BLOCKED. A09 and G01-G06 preserve their source branch IDs, ACK-bound foreground operations, H1/H2 physical evidence, peer convergence and run-owned cleanup; native commit/version/delete/marker/MPU/copy fences remain in the denominator until a reviewed candidate barrier exists. The matrix has 141 registered dispatch IDs (74 non-Heal and 67 Heal), with 17 Heal cases still lacking automatic dispatch. Registration includes executors whose required native branches remain BLOCKED; it does not certify complete canonical migration or live PASS. From 3b70d8e4d02a15d1e0c27af4527fa0e0ff2ce75e Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 11 Oct 2026 20:45:11 +0800 Subject: [PATCH 158/343] feat(validation): add disabled Heal environment scenarios --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-heal-cases.md | 68 ++ docs/validation-heal-environment-gates.md | 46 ++ docs/validation-migration-map.md | 10 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/environment_cases.py | 765 ++++++++++++++++++ scripts/validation_heal/environment_gate.py | 224 +++++ scripts/validation_heal/execution_policy.py | 189 +++++ scripts/validation_heal/matrix.py | 42 +- scripts/validation_heal/recipes.json | 336 +++++++- scripts/validation_heal/run_case.sh | 18 + scripts/validation_report.py | 8 +- test_validation_heal_environment_gate.py | 347 ++++++++ 13 files changed, 2021 insertions(+), 38 deletions(-) create mode 100644 docs/validation-heal-environment-gates.md create mode 100644 scripts/validation_heal/environment_cases.py create mode 100644 scripts/validation_heal/environment_gate.py create mode 100644 scripts/validation_heal/execution_policy.py create mode 100644 test_validation_heal_environment_gate.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 4b95c78f..6fc5f83c 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -44,6 +44,7 @@ on: - test_validation_heal_queue_pressure.py - test_validation_heal_nolock.py - test_validation_heal_race.py + - test_validation_heal_environment_gate.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -101,6 +102,7 @@ on: - test_validation_heal_queue_pressure.py - test_validation_heal_nolock.py - test_validation_heal_race.py + - test_validation_heal_environment_gate.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -138,7 +140,7 @@ jobs: sudo apt-get update sudo apt-get install --no-install-recommends -y nftables fi - - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index 9b1c6628..88ad2bd8 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -668,6 +668,10 @@ Source assertions: Topologies: H1,H2. Branches: old-metadata-rejoin, old-checkpoint-replay, old-mrf-replay, native-incarnation-fence. +Execution policy: `evidence-blocked`; disabled by default: `false`. + +Blocked reason (`EVIDENCE_INCARNATION_LIVE_ORCHESTRATION`): 已有 3 项候选版本 incarnation 原生回归通过，但尚缺覆盖旧盘回归、旧 checkpoint/MRF 重放与同名新桶的完整真机编排和证据绑定；原生回归不能替代四个分支的 canonical 验收。 + 1. 建旧 DV 桶并让一盘离线；按 S3 正常列举/删除全部版本和 marker，确认 delete_bucket 成功，禁止直接内部清空绕过删除。 2. 同名 create_bucket 后写新内容，保存新 incarnation；恢复旧盘并让旧任务/持久 MRF 重放。 3. 对比任务身份和代次守卫，旧责任不得删除/覆盖新对象，不得把旧 marker 或数据复活到新桶。 @@ -1196,6 +1200,10 @@ Source assertions: Topologies: H1,H2. Branches: checkpoint-truncate, mrf-checksum, future-schema, stale-generation, journal-write-failure, native-persist-fence. +Execution policy: `evidence-blocked`; disabled by default: `false`. + +Blocked reason (`EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION`): 已有 journal 故障工具和 10 项候选版本持久化原生回归，但尚缺 H1/H2 全六分支编排、写失败诊断、责任保留和安全重扫；现有 publication interruption 证据也不等于每个 fsync 临界点或断电证据。 + 1. 先制造并证明真实 durable root/MRF 责任，CHECKPOINT 定位精确本 run task/record，其他系统记录禁止触碰。 2. 停止测试服务，按 FAULT_FORMAT 对这些已审核记录分别截断、坏校验、未来 schema、旧代次；写失败实例用独占限额故障层，不塞满真实共享盘。 3. 重启后核对解析/错误状态：未知或损坏记录保留、可诊断，不能静默删除并 ACK；责任仍能安全重扫处理。 @@ -1218,6 +1226,10 @@ Source assertions: Topologies: H2. Branches: rolling-old-new, mixed-peer-status, rollback-new-old, mixed-restart. +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_IMMUTABLE_RELEASE_PAIR`): 需要两个已固定且相互兼容的 RustFS release、混合版本 peer 状态和经过审核的回滚兼容矩阵；当前 auto-testing Heal 节点没有这组不可变双版本资源。 + 1. 固定两个 release optimized 包的 commit/tree/binary/格式兼容说明；UPGRADE 先验证支持回滚的记录版本，不能使用任意旧包。 2. 有持久修复责任时逐节点升级，保留数据根和 unit；混合阶段通过各 endpoint STATUS/BACKGROUND 并持续唯一版本写入。 3. 缺少能力的 peer 必须显示 partial/unsupported，不能 aggregate 成 complete；新记录不被老节点误退休。 @@ -1240,6 +1252,10 @@ Source assertions: Topologies: H6. Branches: dedicated-vm-forced-stop, dedicated-host-power-cycle. +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_DEDICATED_H6_HOST`): 需要 H6 专用主机、平台强制停启权限、带外恢复和 boot_id/平台事件证据；共享 Heal 节点不能证明真实电源循环。 + 1. 先取得 H6 专项资源 ID、独占证明和带外恢复批准；普通 fixed fleet 禁止执行本例。 2. 持续限量写入并在 replacement 部分重建窗口，记录已成功 ACK/结果未知写集合与最近持久化证据。 3. HOST_POWER 使用批准的平台命令强制停启，保留平台活动 ID、时间线和 boot_id；VM stop 与真实电源断电分别命名实例。 @@ -1394,6 +1410,10 @@ Source assertions: Topologies: H2. Branches: current-expiry, noncurrent-expiry, transition, restore, native-ilm-fence. +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_DEDICATED_ILM_TIER_WINDOW`): 需要隔离 ILM/tier 租户、真实 Expire/Transition/Restore 窗口和远端版本映射；共享 ILM 环境不能安全制造交叠窗口。 + 1. 用现有 ILM 专用远端 tier 能力建立本例桶和规则；记录真实到期窗口/远端version映射，不在共享桶上改规则。 2. 在实际出现 Expire/Transition/Restore 进度时启动缺片 heal，WINDOW 必须从日志/状态证明交叠；提前完成只能记 NOT_OBSERVED。 3. 过期 ACK/ILM 删除证明决定哪些版本应不存在，禁止 heal 复活；迁移对象通过真实远端 GET/restore 验证原版本内容。 @@ -1416,6 +1436,10 @@ Source assertions: Topologies: H2. Branches: replicate-put, replicate-delete, resync, native-replication-fence. +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_ISOLATED_REPLICATION_PAIR`): 需要隔离的复制两端、resync 控制、跨端版本映射和 Heal/resync 实际交叠窗口；当前固定 fleet 没有独占复制对。 + 1. 准备两个隔离复制端点/本例桶、明确源目标版本映射和同步规则；先跑一份健康复制控制。 2. 源端缺片 heal 同时提交 PUT/DELETE 并启动指定 resync；保存两侧日志与跨端版本映射，证明操作时间交叠。 3. 分别从两端全量列举版本、marker、内容并比对复制契约；源端正确但远端缺失/无限重试必须失败。 @@ -1438,6 +1462,10 @@ Source assertions: Topologies: H5. Branches: expand-while-data-heal, expand-while-parity-heal, versioned-rebalance. +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_H5_EXPANSION_REBALANCE`): 需要 H5 pool 扩容、rebalance 和 Heal 同时活动的完整拓扑，并逐对象保存 pool/set 迁移证据；当前 auto-testing 未绑定扩容前后布局与专用操作窗口。 + 1. 使用已批准 H5，先在源 pool 写 DB/DV 并以 FORMAT/PHYSICAL 证明位置；新 pool 为空且未纳入旧选择器。 2. 目标 data/parity 缺失或bitflip后按 POOL 手册扩容重启，确认新拓扑；启动 rebalance 与 heal。 3. WINDOW 保存两任务活动证据，逐对象跟踪源/目标 pool/set/版本身份，不能只看全局数量。 @@ -1460,6 +1488,10 @@ Source assertions: Topologies: H5. Branches: rebalance-stop, rebalance-resume, rebalance-service-restart. +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_H5_REBALANCE_LIFECYCLE`): 需要可观测的 H5 rebalance、停止/恢复/服务重启控制及独立 task 身份；当前 auto-testing 未绑定这两类任务的独立身份和操作窗口。 + 1. H5 中使用足够对象使移动可观测，缺片 heal 与 rebalance 同时活动；记录两类独立 task ID。 2. 只停止 rebalance，观察 heal 不被误取消；然后恢复 rebalance，重启实例仅重启本例 unit。 3. 持续唯一 key 写入并保存 ACK；停止/恢复前后物理和任务状态要连续，不用新数据根替代恢复。 @@ -1482,6 +1514,10 @@ Source assertions: Topologies: H5. Branches: heal-decommission-source, heal-survivor-target, versioned-multipart. +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_H5_DECOMMISSION`): 需要 H5 pool decommission、survivor 目标、写入限制和移动 ACK/物理权威证据；当前 auto-testing 未绑定本例退役前后拓扑和恢复方案。 + 1. 先证实夹具位于待退役源，另写 survivor control；分别在源和目标制造在quorum内可恢复故障。 2. 按既有 Pool 行为启动退役与 heal；退役期间按当前与 MinIO 一致的写入限制检查拒绝，不能把拒绝写当数据丢失。 3. 源数据只能在目标提交权威后删除；记录移动 ACK 与 target physical，而不是临时副本数量。 @@ -1504,6 +1540,10 @@ Source assertions: Topologies: H5. Branches: cancel-during-heal, clear-after-inflight-exit, resume-new-generation, capacity-lock-retry-window. +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_H5_DECOMMISSION_CANCEL`): 需要退役取消/清理/新代次恢复以及 #2423 所需容量锁重试窗口；普通取消结果不能替代这个临界点。 + 1. 在移动和 heal 都有部分进度时取消退役；保存取消持久化记录、通知和在途退出顺序。 2. 确认已退出后 Clear，再恢复/新建退役；旧 token/代次不得误取消新任务，持续 ACK ledger 不能丢。 3. 容量锁超时实例需要 LOCK_WINDOW 或匹配 NATIVE 屏障，实际见证 #2423 所涉重试路径；普通取消成功不能代替。 @@ -1526,6 +1566,10 @@ Source assertions: Topologies: H5. Branches: survivor-missing-part, survivor-missing-metadata, retired-source-offline. +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_H5_RETIRED_POOL_REMOVAL`): 需要完整迁移后移除 retired pool、源保持离线和 survivor-only 访问；当前 auto-testing 未提供本例源持续离线和 survivor-only 的绑定证据。 + 1. 依既有 Pool 全量迁移并完成退休配置移除，停止源服务，证明入口仅绑定 survivor；保留源不可达证据。 2. 只在 survivor 的选定对象/版本制造缺片或metadata缺失；START root或正确pool/set deep。 3. 记录 source authority 选择，不能向已退休 pool 写入，也不能无权威覆盖/错误回退（#2422）。 @@ -1570,6 +1614,10 @@ Source assertions: Topologies: H3,H4. Branches: ec84-one-set-restart, ec84-one-set-crash, ec84-two-sets-restart, ec84-two-sets-crash. +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_EC84_MULTI_SET_TOPOLOGY`): 需要 H3 的 12 个独立盘和 H4 的至少 24 个独立 endpoint，并按 set 分别记录重启/崩溃窗口；当前 H4 资源不可用。 + 1. 分别绑定 H3 12盘和 H4 至少24盘，FORMAT必须证实每set为8data+4parity；同一物理设备多个目录不能算独立故障盘。 2. 每个set都准备DB/DV/DX并证明对象归属，REPLACE选定目标，在目标仅部分重建时优雅重启或SIGKILL。 3. 保存实际中断窗口、盘代次、每set已修/未修对象，不能跨set合并片数。 @@ -1592,6 +1640,10 @@ Source assertions: Topologies: H1,H2. Branches: bounded-latency, short-read, eio. +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_DEDICATED_FAULT_IO`): 需要专用 faultfs 或故障设备、可恢复的 slow/short-read/EIO 注入和 namespace 隔离；共享盘模拟会污染其他运行。 + 1. IO_FAULT 只在预分配专用故障设备或受限faultfs，冻结延迟/错误率、IO范围、最长窗口和原状恢复命令。 2. 在服务相同namespace/UID先证明迟延/短读/EIO真实生效，再启动目标缺片heal；不得让dd填满共享盘。 3. 错误窗口记录重试、CPU、队列和物理片，不能提前reported healthy；暂态与永久错误分开记录。 @@ -1680,6 +1732,10 @@ Source assertions: Topologies: H1,H2. Branches: object-lock-timeout, caller-disconnect, stale-read-repair, native-lock-owner-fence. +Execution policy: `evidence-blocked`; disabled by default: `false`. + +Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 需要真实 lock owner、超时、调用方断开和 stale read-repair 的服务端所有权/提交顺序；黑盒请求耗时只能证明 TCP 断开，不能证明锁提交屏障，候选包没有 reviewed e2e-test-hooks。 + 1. LOCK_WINDOW用本例慢IO/受控原生锁屏障持有目标对象锁，先证明真实争用超过预注册预算。 2. 同时heal/覆盖或删除；调用方断开实例仅断本次TCP连接，不杀服务器或关闭跳板机；记录请求开始/断开/服务在途结束。 3. 锁释放后观察取消/超时不提前释放所有权、不重复提交，陈旧read-repair必须重验版本。 @@ -1702,6 +1758,10 @@ Source assertions: Topologies: H2. Branches: kms-unavailable, kms-denied, tier-unavailable, tier-version-unavailable. +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_DEDICATED_KMS_TIER_TENANT`): 需要专用 KMS/tier 租户、依赖网络隔离和安全的 key/tier 版本证据；不能在共享依赖上全局断网。 + 1. 先在专用KMS/tier租户验证DE/DV夹具可正常解密/restore，保存非敏感key/tier版本标识。 2. DEPENDENCY仅断本测试客户端/服务到该依赖的专用namespace路由，或撤销本例租户权限；不全局停共享KMS。 3. 发GET/restore/deep heal并观察责任，依赖错误不能判对象不存在、可remove、已修好；密钥不进入错误日志。 @@ -1746,6 +1806,10 @@ Source assertions: Topologies: H1,H2. Branches: mixed-healthy-repaired-failed, dry-run, deferred, unknown-negative. +Execution policy: `evidence-blocked`; disabled by default: `false`. + +Blocked reason (`EVIDENCE_MIXED_OUTCOME_ORCHESTRATION`): 已有 2 项候选版本 outcome 原生回归和部分计数校验，但尚缺真实混合 healthy/repaired/failed/deferred/unknown 夹具、dry-run 独立任务、逐对象完整 manifest 与计数/attemptFailures 对账；这是执行器与证据未闭环，并非已证实产品缺少 API。 + 1. 同批独立key建立健康、可恢复缺片、低于R的不可恢复对象；deferred由冻结的锁/依赖故障产生，dry-run另建任务。 2. 保存逐对象物理基线和期望disposition，对结果的execution/coverage/counters/object identities做逐项对账。 3. unknown/skipped/completed_with_errors不算修复成功，legacy objectsHealed与canonical healed分开解释；attemptFailures不得被隐藏。 @@ -1812,6 +1876,10 @@ Source assertions: Topologies: H2,H5. Branches: 24h, 72h. +Execution policy: `environment-gated`; disabled by default: `true`. + +Blocked reason (`ENV_DEDICATED_LONG_SOAK_LANE`): 必须在独占 H2/H5 lane 真实运行 24 小时和 72 小时，包含限速、监控、到期回收和完整资源预算；短跑不能替代 soak 证据。 + 1. 只在专门分配的H2/H5独占lane，冻结完整24h或72h墙钟窗口、循环故障日程、并发/限速/存储上限和到期回收安排。 2. WORKLOAD持续记录逐请求ACK，周期性故障仅使用本run DRIVE_GATE/PROCESS/依赖规则；每轮见证与恢复，不无限加数据。 3. 持续METRICS、日志、队列/MRF/checkpoint取样；资源增长、panic、未解释责任达到预注册阈值立即保留失败证据。 diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md new file mode 100644 index 00000000..d0f4f47a --- /dev/null +++ b/docs/validation-heal-environment-gates.md @@ -0,0 +1,46 @@ +# Heal 环境门控与原生阻断 + +第一批 13 个用例保留全部 recipe、分支和 source assertions，并新增 `environment_cases.py` 的控制流程与断言，接入 validation 调度。它们默认关闭，只产生 `BLOCKED` 结果与未满足原因；默认路径不会连接或修改 fleet。真实执行沿用 auto-testing 的 S3/Admin collector、ACK ledger、候选包绑定、物理故障 oracle 和恢复模式，专用资源操作由部署环境的适配器提供。当前没有把适配器协议或未运行代码计为 canonical PASS。 + +设置 `AUTO_TESTING_ENABLE_ENV_GATED_HEAL=1` 并通过 `VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES` 指向 JSON 能力清单后，门控会检查全部能力。缺项仍返回 `BLOCKED`。能力齐全后还要用 `VALIDATION_HEAL_ENVIRONMENT_RUNTIME` 指向 runtime JSON；只有候选/checkout/client 摘要、全部拓扑、当前 Heal SSH 节点和适配器摘要校验通过后才会运行分支。未绑定适配器仍为 `BLOCKED`；执行结果需要 source assertion 审核，当前实现不会自动生成 canonical PASS。 + +能力清单示例： + +```json +{"capabilities": ["dedicated_faultfs_or_device", "slow_io_short_read_eio", "namespace_isolation", "fault_layer_restore"]} +``` + +| Code | 用例 | 当前未满足原因 | 启用条件 | +| --- | --- | --- | --- | +| F08 | `heal.mixed-version-upgrade-rollback` | 没有经过审核的不可变双版本组合 | 两个兼容 release、混合 peer 状态、滚动升级/回滚和格式兼容矩阵 | +| F09 | `heal.dedicated-host-forced-restart` | 共享 Heal 节点不具备 H6 专用主机及平台断电能力 | 专用 H6、平台 power-cycle、带外恢复、boot_id/平台事件证据 | +| G07 | `heal.race-ilm-expiry-and-transition` | 缺隔离 ILM/tier 与真实操作交叠窗口 | 专用租户、Expire/Transition/Restore 窗口、远端版本映射、规则清理 | +| G08 | `heal.race-replication-resync` | 缺独占复制两端与 resync 控制 | 隔离复制对、resync、跨端版本映射、复制/Heal 交叠证据 | +| H01 | `heal.with-expansion-rebalance` | 缺完整 H5 扩容/rebalance 拓扑 | H5 扩容、rebalance task、逐对象 pool/set 映射、scanner 退休证据 | +| H02 | `heal.with-rebalance-stop-resume` | 缺独立 rebalance 停止/恢复/重启控制 | H5 rebalance、独立 task、stop/resume 和 service restart | +| H03 | `heal.with-decommission` | 缺可回滚的退役拓扑 | H5 decommission、survivor pool、写入限制、移动 ACK 与物理权威证据 | +| H04 | `heal.with-decommission-cancel-clear` | 缺退役新代次与容量锁重试窗口 | cancel/clear/resume、#2423 锁窗口、代次绑定 ACK ledger | +| H05 | `heal.after-retired-pool-removal` | 缺 retired pool 移除后的 survivor-only 拓扑 | retired pool 配置移除、源持续离线、survivor 权威、旧 pool 写保护 | +| H07 | `heal.ec84-multi-set-restart` | H4 的至少 24 个独立 endpoint 不可用 | H3 12 独立盘、H4 24 endpoint、逐 set 身份与重启/崩溃证据 | +| I01 | `heal.slow-io-and-eio` | 没有专用 faultfs/故障设备 | 专用故障层、slow/short-read/EIO、namespace 隔离、原状恢复 | +| I06 | `heal.kms-and-tier-outage` | 没有专用 KMS/tier 租户与依赖隔离 | 独占租户、依赖网络隔离、key 脱敏、tier 版本映射 | +| J05 | `heal.long-soak` | 缺可持续 24h/72h 的独占 lane 与资源预算 | 真实墙钟窗口、限速和监控、循环故障、到期回收；短跑不能替代 | + +`matrix.py list` 同时输出 `automaticExecutor`、`dispatchable` 和 `executionPolicy`：67 个 Heal 自动执行器、13 个环境门控、4 个证据/执行器阻断。调度覆盖共 80 个 Heal，连同 74 个非 Heal 为 154/158；这不代表 154 个 live PASS。 + +第二批 4 个用例未进入可执行调度。经核实，C11/F07/J02 已有 candidate-bound 原生回归；这三项主要缺完整真机编排与证据闭环。I05 仍缺精确服务端屏障，增加节点不能解决： + +| Code | 用例 | Block 原因 | 解锁所需证据 | +| --- | --- | --- | --- | +| C11 | `heal.bucket-incarnation-recreation` | 已有 3 项 incarnation 原生回归；缺旧盘回归、旧 checkpoint/MRF 重放、新桶隔离的全四分支真机编排 | 真机旧/新责任与 S3 ACK、物理范围及原生 artifact 精确绑定 | +| F07 | `heal.checkpoint-and-mrf-corruption` | 已有 journal 工具和 10 项原生回归；缺 H1/H2 全六分支、写失败诊断、责任保留和安全重扫编排 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | +| I05 | `heal.lock-timeout-and-caller-abort` | 请求耗时/TCP 断开不能证明服务端锁 owner 与提交顺序 | lock owner、timeout、caller abort 和 stale read-repair 的服务端屏障 | +| J02 | `heal.outcome-counters-and-error-truth` | 已有 2 项 outcome 原生回归；缺混合 healthy/repaired/failed/deferred/unknown 真机夹具、dry-run 独立任务与完整 manifest 对账 | 每个 disposition、计数和 attemptFailures 对账；不是已证实产品缺少 API | + +I05 和已接入 A09、G01–G06 的精确屏障分支仍受候选包对应、经过审核的 `e2e-test-hooks` 限制。第一批 G07/G08 的 native 分支与 H04 容量锁窗口也保留 BLOCKED；不能用外部请求重试或最终对象正确来声称精确 lock/version/marker/MPU/copy 提交屏障已命中。 + +资源适配器协议使用 stdin/stdout JSON，不调用 shell。runtime 中的 `adapter` 包含绝对可执行路径、SHA-256 和操作 allowlist；`environment_cases.required_operations(case_id)` 给出所需操作。每个请求携带 run ID、原候选 commit、sequence、拓扑/分支 scope、bucket prefix、租约和参数。响应必须回显上述身份、按 `heal_probe.canonical(request)` 的 UTF-8 SHA-256 绑定原请求，并把真实观察写在 `observation`，禁止返回 verdict。资源适配器需独立实现平台 power control、故障层、pool lifecycle 或 KMS/tier 隔离；本 PR 未向当前共享环境部署适配器。 + +runtime 的 `binding` 使用现有 `heal_probe.validate_binding` 契约，另含 `schemaVersion: 1`、`provider: auto-testing`、`sshNodes`、`rcBinary`、`topologies`、`capabilities`、`adapter` 和对应资源字段。F08 需要 `releasePair`，F09 需要 `hostResourceId`，I01 需要 `faultDeviceId/ioLatencyMs`，G07/I06 需要 `kmsKeyId/tierEndpoint/tenantId/dependencyNamespaceId`，G08 需要 `replicaEndpoint`，J05 需要 `soakLimits`。H6 专用主机应由适配器按 `hostResourceId` 独占绑定；SSH fleet 仍记录当前 auto-testing Heal 节点。secret 通过环境传入，适配器诊断会脱敏，`redactionEnvironmentVariables` 可指定额外 secret 环境变量名。 + +J05 的实际控制循环保留固定 86400/259200 秒窗口、精确版本写入/删除 ACK、三类循环故障、恢复、资源阈值和到期全量校验。单元测试用可注入的时钟验证窗口控制，不修改生产时长。G07/G08 精确 native 分支不调用资源适配器。清理失败保留原失败与 DIRTY 原因，停止后续分支。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 2de37a11..93c240ab 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 134 currently dispatchable IDs include all 74 non-Heal cases and sixty Heal cases. Independent census currently confirms 86 unique complete canonical live PASS cases: 74 on the current candidate source and 12 on the older immutable source. Native, read-only, recovery and duplicate runs are excluded; the other 72 cases have no complete canonical live PASS claim. +case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 86 unique complete canonical live PASS cases: 74 on the current candidate source and 12 on the older immutable source. Native, read-only, recovery and duplicate runs are excluded; the other 72 cases have no complete canonical live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -197,8 +197,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -The other 25 Heal cases remain -operator-assisted and are not claimed as migrated automatic executors. +Four Heal cases remain native/evidence-blocked and outside dispatch: C11, F07, I05 and J02. The 13 environment-gated cases have disabled-by-default control flows and oracles in `environment_cases.py`; actual execution requires an assigned and reviewed auto-testing resource adapter, and canonical results remain review-gated. See [the environment gate reference](validation-heal-environment-gates.md) for each missing resource, capability and native barrier. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -1879,3 +1878,8 @@ result is relabeled to the current helper. No bounded-journal live execution or complete F07 executor is claimed yet. Current reconciliation: helper `6f91210f224dc0a9e460b8240c4a4cf73d7e64c1` is pushed to auto-testing PR #113 and contains the A09/G01/G02/G03/G04/G05/G06 race executors. The current local contract suite runs 518 tests with eight macOS skips; the preceding Linux contract run `38109944413` ran 473 tests without skips. The independent canonical census remains the source of live PASS totals; native, read-only, recovery, duplicate and incomplete runs are excluded, and the new G04/G05 executors add no canonical live PASS claim while their native marker/MPU fences are BLOCKED. A09 and G01-G06 preserve their source branch IDs, ACK-bound foreground operations, H1/H2 physical evidence, peer convergence and run-owned cleanup; native commit/version/delete/marker/MPU/copy fences remain in the denominator until a reviewed candidate barrier exists. The matrix has 141 registered dispatch IDs (74 non-Heal and 67 Heal), with 17 Heal cases still lacking automatic dispatch. Registration includes executors whose required native branches remain BLOCKED; it does not certify complete canonical migration or live PASS. + + +The environment-gated batch now registers F08/F09, G07/G08, H01-H05/H07, I01/I06 and J05. The default gate writes a per-case BLOCKED reason and environment evidence before any SSH/install/fault operation. Explicit opt-in still requires exact candidate/client/runtime provenance, every required topology and a digest-bound resource adapter. The enabled control flows reuse auto-testing Probe/Recorder and physical fault oracles, retain every source branch and actual 24h/72h soak duration, stop on dirty cleanup, and require source-assertion review instead of manufacturing PASS from adapter responses. No live run or current-fleet adapter deployment is claimed. Second-batch blockers are recorded accurately: C11/F07/J02 already have 3/10/2 candidate native prerequisites but still need full live orchestration/evidence; I05 lacks an exact reviewed lock-owner/abort fence. See `docs/validation-heal-environment-gates.md`. + +This batch passes all 36 test modules selected by Validation Catalog CI: 537 local tests with eight macOS platform skips, plus exact source-catalog hash comparison, the 84/301/532 matrix, shell syntax and scanner mock gates. The 19 new environment tests cover disabled dispatch across all 13 IDs without SSH, explicit runtime provenance, topology and counter negatives, complete enabled IO control flow, actual overlap, full-duration soak control, and recovery after a lost setup ACK. These are local contract checks; no live environment-gated result is claimed. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index df51f4d0..af398d9f 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-markers-and-batch-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) + heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-markers-and-batch-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update|heal.mixed-version-upgrade-rollback|heal.dedicated-host-forced-restart|heal.race-ilm-expiry-and-transition|heal.race-replication-resync|heal.with-expansion-rebalance|heal.with-rebalance-stop-resume|heal.with-decommission|heal.with-decommission-cancel-clear|heal.after-retired-pool-removal|heal.ec84-multi-set-restart|heal.slow-io-and-eio|heal.kms-and-tier-outage|heal.long-soak|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/environment_cases.py b/scripts/validation_heal/environment_cases.py new file mode 100644 index 00000000..d4351e60 --- /dev/null +++ b/scripts/validation_heal/environment_cases.py @@ -0,0 +1,765 @@ +#!/usr/bin/env python3 +"""Disabled-by-default extended Heal scenarios on assigned auto-testing resources. + +S3 fixtures, ACKs, Admin tasks, physical and data oracles are owned here. An +assigned resource adapter supplies lifecycle operations that the ordinary Heal +fleet cannot provide (power control, IO devices, extra pools and dependencies). +It receives JSON on stdin and returns raw, run-bound observations. It cannot +supply a PASS verdict. No release-validation checkout is used. +""" +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import threading +import time +from urllib.parse import quote + +try: + from .execution_policy import ENVIRONMENT_GATED_CASES + from .heal_probe import Probe, canonical, compare_inventories, payload, redact, require, save, validate_binding + from .heal_workload import Recorder, seed, verify_seed + from .healthy import BASE_OPTIONS + from .matrix import select, variant_topologies + from .shards import files, validate_fault, validate_repaired_physical +except ImportError: + from execution_policy import ENVIRONMENT_GATED_CASES + from heal_probe import Probe, canonical, compare_inventories, payload, redact, require, save, validate_binding + from heal_workload import Recorder, seed, verify_seed + from healthy import BASE_OPTIONS + from matrix import select, variant_topologies + from shards import files, validate_fault, validate_repaired_physical + +RUNTIME_ENV = "VALIDATION_HEAL_ENVIRONMENT_RUNTIME" +SHA = re.compile(r"[0-9a-f]{64}\Z") +SOAK_SECONDS = {"24h": 24 * 3600, "72h": 72 * 3600} +NATIVE_BRANCHES = {"native-ilm-fence", "native-replication-fence"} + + +class Blocked(ValueError): + pass + + +class NotObserved(ValueError): + pass + + +@dataclass(frozen=True) +class Scenario: + code: str + family: str + # Actions run after actual Heal admission. Each has a raw evidence oracle. + actions: tuple[str, ...] + fault_kind: str = "missing" + role: str = "data" + interrupted: bool = False + + +def scenario(code: str, branch: str) -> Scenario: + """Executable branches; strings are adapter operations, never shell text.""" + if code == "F08": + actions = { + "rolling-old-new": ("upgrade-next", "wait-mixed", "upgrade-rest"), + "mixed-peer-status": ("upgrade-next", "mixed-status", "upgrade-rest"), + "rollback-new-old": ("upgrade-all", "rollback-next", "wait-mixed", "rollback-rest"), + "mixed-restart": ("upgrade-next", "restart-mixed", "mixed-status", "upgrade-rest"), + }[branch] + return Scenario(code, "upgrade", actions) + if code == "F09": + return Scenario(code, "host", ("replace-drive", "replacement-window", branch, "recover-host"), interrupted=True) + if code == "G07": + return Scenario(code, "ilm", ("ilm-" + branch, "ilm-window", "ilm-settle")) + if code == "G08": + return Scenario(code, "replication", (branch, "replication-window", "replication-settle")) + if code == "H01": + return Scenario(code, "pool", ("expand-pool", "rebalance-start", "pool-window", "pool-settle"), + role="parity" if "parity" in branch else "data") + if code == "H02": + actions = ["rebalance-start", "pool-window", "rebalance-stop"] + if branch == "rebalance-service-restart": actions.append("restart-service") + actions.extend(("rebalance-resume", "pool-window", "pool-settle")) + return Scenario(code, "pool", tuple(actions)) + if code == "H03": + return Scenario(code, "pool", ("decommission-start", "pool-window", "decommission-write-guard", "pool-settle", "source-offline")) + if code == "H04": + return Scenario(code, "pool", ("decommission-start", "pool-window", "decommission-cancel", "wait-inflight-exit", + "decommission-clear", "decommission-resume", "capacity-lock-window" + if branch == "capacity-lock-retry-window" else "pool-window", "pool-settle")) + if code == "H05": + return Scenario(code, "retired-pool", ("survivor-window",), + fault_kind="metadata-missing" if branch == "survivor-missing-metadata" else "missing") + if code == "H07": + return Scenario(code, "ec84", ("replace-drive", "replacement-window", + "crash-service" if branch.endswith("crash") else "restart-service", "replacement-settle"), interrupted=True) + if code == "I01": + return Scenario(code, "io", ("io-" + branch, "io-window", "io-recover")) + if code == "I06": + return Scenario(code, "dependency", ("dependency-" + branch, "dependency-window", "dependency-recover")) + if code == "J05": + require(branch in SOAK_SECONDS, "unregistered soak duration") + return Scenario(code, "soak", ()) + raise ValueError(f"unknown environment scenario: {code}/{branch}") + + +def required_operations(case_id: str) -> set[str]: + item = select(case_id)[0] + common = {"preflight", "prepare", "stop", "start", "snapshot", "apply-fault", "recover", "cleanup", "verify-parts", "checkpoint", "metrics"} + for branch in item["variants"]: + if branch in NATIVE_BRANCHES: continue + spec = scenario(item["code"], branch) + common.update(spec.actions) + if spec.family == "retired-pool": common.add("retire-pool-prepare") + if spec.family in {"ilm", "replication", "dependency"}: common.add(spec.family + "-prepare") + if spec.family == "soak": common.update(("soak-fault", "soak-recover", "metrics")) + return common + + +def runtime_binding(value: dict, case_id: str, candidate: dict, testing_commit: str) -> dict: + """Reject unassigned resources before the first command or mutation.""" + require(value.get("schemaVersion") == 1 and value.get("provider") == "auto-testing", "auto-testing runtime manifest required") + binding = value["binding"] + validate_binding(binding) + require(binding["candidateCommit"] == candidate["source_sha"] + and binding["candidatePackageSha256"] == candidate["package_sha256"] + and binding["harnessCommit"] == testing_commit, "runtime is bound to a different candidate/checkout") + if candidate.get("binary_sha256"): + require(binding["candidateBinarySha256"] == candidate["binary_sha256"], "runtime binary differs from candidate") + rc = Path(value["rcBinary"]) + require(rc.is_absolute() and rc.is_file() and not rc.is_symlink() + and hashlib.sha256(rc.read_bytes()).hexdigest() == binding["rcBinarySha256"], "runtime rc identity differs") + configured_nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(value.get("sshNodes") == configured_nodes, "runtime must reuse the current auto-testing Heal fleet") + required = set(ENVIRONMENT_GATED_CASES[case_id]["requiredCapabilities"]) + require(required <= set(value.get("capabilities", [])), "runtime lacks required environment capabilities") + selected = value.get("topologies", []) + wanted = set(select(case_id)[0]["contract"]["probes"]["topology"].split(",")) + require(isinstance(selected, list) and len(set(selected)) == len(selected) and set(selected) == wanted, + "runtime must bind every required topology") + adapter = value.get("adapter", {}) + executable = Path(adapter.get("executable", "")) + require(executable.is_absolute() and executable.is_file() and not executable.is_symlink() + and os.access(executable, os.X_OK) and hashlib.sha256(executable.read_bytes()).hexdigest() == adapter.get("sha256"), + "assigned resource adapter is absent, non-executable or changed") + require(required_operations(case_id) <= set(adapter.get("operations", [])), "resource adapter omits a required operation") + code = ENVIRONMENT_GATED_CASES[case_id]["code"] + require(isinstance(value.get("inspectorSha256"), str) and SHA.fullmatch(value["inspectorSha256"]), + "candidate-bound inspector digest required") + resource_fields = {"F09": ("hostResourceId",), "G07": ("kmsKeyId", "tierEndpoint"), + "G08": ("replicaEndpoint",), "I01": ("faultDeviceId",), + "I06": ("kmsKeyId", "tierEndpoint", "tenantId", "dependencyNamespaceId")} + require(all(isinstance(value.get(field), str) and value[field] for field in resource_fields.get(code, ())), + "runtime omits a required dedicated resource binding") + if code in {"G07", "G08", "I06"}: + field = "replicaEndpoint" if code == "G08" else "tierEndpoint" + require(value[field] in binding["endpoints"] and value[field] != binding["defaultEndpoint"], + "remote dependency/replica must bind a separate endpoint") + if code == "F08": + pair = value.get("releasePair", {}) + releases = pair.get("releases", []) + require(pair.get("rollbackCompatible") is True and SHA.fullmatch(pair.get("compatibilityEvidenceSha256", "")) + and len(releases) == 2 and len({r.get("packageSha256") for r in releases}) == 2 + and all(SHA.fullmatch(r.get("packageSha256", "")) and SHA.fullmatch(r.get("binarySha256", "")) + and re.fullmatch(r"[0-9a-f]{40}", r.get("sourceCommit", "")) and r.get("packageUrl") for r in releases) + and any(r["binarySha256"] == binding["candidateBinarySha256"] for r in releases), + "F08 requires two immutable compatible releases including the candidate") + if code == "J05": + expiry = datetime.fromisoformat(binding["leaseExpiresAt"].replace("Z", "+00:00")) + # Both source windows run sequentially; include final verification/recovery. + require((expiry - datetime.now(timezone.utc)).total_seconds() >= len(wanted) * sum(SOAK_SECONDS.values()) + 3600, + "soak lease cannot cover all actual 24h/72h windows and recovery") + limits = value.get("soakLimits", {}) + require(all(type(limits.get(k)) is int and limits[k] >= 0 for k in + ("rssBytes", "openFds", "queuedTasks", "unresolvedRecords", "temporaryBytes")), "soak resource limits required") + return binding + + +class Adapter: + """A resource operation protocol, similar to the existing Pool suite hooks.""" + def __init__(self, runtime, evidence): + self.runtime, self.evidence = runtime, evidence + self.sequence = 0 + self.reserved = False + self.calls = [] + + def record(self, label, value): + self.sequence += 1 + save(self.evidence / f"{self.sequence:06d}-{label}.json", value) + return value + + def call(self, operation, *, scope, **parameters): + binding = self.runtime["binding"] + if operation not in {"recover", "cleanup", "soak-recover"}: + validate_binding(binding) + require(operation in self.runtime["adapter"]["operations"], "unregistered resource operation") + executable = Path(self.runtime["adapter"]["executable"]) + require(hashlib.sha256(executable.read_bytes()).hexdigest() == self.runtime["adapter"]["sha256"], "resource adapter changed during run") + request = {"schemaVersion": 1, "runId": binding["runId"], "sequence": len(self.calls) + 1, + "candidateCommit": binding["candidateCommit"], "scope": scope, "operation": operation, + "bucketPrefix": binding["bucketPrefix"], "leaseExpiresAt": binding["leaseExpiresAt"], + "parameters": parameters} + digest = hashlib.sha256(canonical(request).encode()).hexdigest() + self.calls.append(request) + self.record("request-" + operation, request) + if operation == "prepare": self.reserved = True # A lost ACK can leave partial setup. + # Files retain partial stdout/stderr even on timeout. Never emit credentials + # from adapter diagnostics to workflow stdout. + timeout = 600 if operation in {"prepare", "cleanup", "pool-settle", "replacement-settle"} else 180 + def safe_text(raw): + text = redact(raw.decode(errors="replace")) + for name in self.runtime.get("redactionEnvironmentVariables", []): + secret = os.environ.get(name) + if secret: text = text.replace(secret, "[REDACTED]") + return text + # Raw adapter output lives outside the uploaded evidence directory. + # Preserve bounded, redacted diagnostics even when the child times out. + with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err: + try: + result = subprocess.run([str(executable)], input=json.dumps(request).encode(), stdout=out, stderr=err, timeout=timeout) + finally: + out.seek(0); raw = out.read(16 * 1024**2 + 1) + err.seek(0); diagnostic = err.read(1024**2) + self.record("adapter-diagnostic", {"operation": operation, "stdout": safe_text(raw), "stderr": safe_text(diagnostic)}) + require(result.returncode == 0, f"resource adapter {operation} failed; inspect retained diagnostics") + require(len(raw) <= 16 * 1024**2, "adapter observation exceeds budget") + response = json.loads(raw) + require(response.get("schemaVersion") == 1 and response.get("requestSha256") == digest and response.get("runId") == binding["runId"] + and response.get("sequence") == request["sequence"] and response.get("scope") == scope + and response.get("operation") == operation and isinstance(response.get("observation"), dict) + and "verdict" not in response["observation"], "resource observation is not bound to the original operation") + return self.record("response-" + operation, response)["observation"] + + +def assert_topology(observed, topology, candidate_sha, *, allow_old=None): + disks = observed.get("disks", []) + expected_min = {"H1": 4, "H2": 16, "H3": 12, "H4": 24, "H5": 16, "H6": 4}[topology] + require(len(disks) >= expected_min, "insufficient independent endpoints") + identities = set() + slots = set() + sets = {} + for disk in disks: + require(all(type(disk.get(k)) is int and disk[k] >= 0 for k in ("poolIndex", "setIndex", "setSlot")), + "invalid physical pool/set/slot") + identity = (disk["hostname"], disk["mountDevice"]) + slot = (disk["poolIndex"], disk["setIndex"], disk["setSlot"]) + require(identity not in identities and slot not in slots and disk.get("diskUUID"), "directories alias a device or erasure slot") + identities.add(identity); slots.add(slot) + sets.setdefault(slot[:2], []).append(disk) + require(topology != "H5" or len({pool for pool, _ in sets}) >= 2, "H5 omitted separate pools") + if topology in {"H3", "H4"}: + require(len(sets) >= (2 if topology == "H4" else 1), "EC8+4 omitted required sets") + require(all(len(group) == 12 and all(d["dataShards"] == 8 and d["parityShards"] == 4 for d in group) + for group in sets.values()), "actual set is not EC8+4") + processes = observed.get("processes", []) + require(processes and {p["hostname"] for p in processes} == {d["hostname"] for d in disks}, "process attestation omitted hosts") + allowed = {candidate_sha} | ({allow_old} if allow_old else set()) + require(all(p.get("binarySha256") in allowed and type(p.get("pid")) is int and p["pid"] > 0 for p in processes), + "service process is not a verified immutable candidate") + return sets + + +def target_from_snapshot(snapshot, bucket, entry, role, *, set_index=None): + indexed = files(snapshot) + cohort = [] + for node, observation in snapshot.items(): + for form in observation["formats"]: + disk = form["disk"] + meta = indexed.get((node, disk, f"{bucket}/{entry['key']}/xl.meta"), {}).get("metadata", {}) + versions = [v for v in meta.get("versions", []) if (v.get("versionId") or "null") == entry["versionId"]] + if not versions: continue + require(len(versions) == 1, "ambiguous acknowledged version") + version = versions[0] + erasure = version["erasure"] + layout = form["value"]["xl"] + group = next(i for i, ids in enumerate(layout["sets"]) if layout["this"] in ids) + if set_index is not None and group != set_index: continue + pool = form.get("poolIndex", 0) + slot = layout["sets"][group].index(layout["this"]) + require(erasure["index"] == erasure["distribution"][slot] and version.get("deleted") is False, + "shard slot/role does not match actual format") + part = version["parts"][0] + relative = f"{bucket}/{entry['key']}/{version['dataDir']}/part.{part['number']}" + physical = indexed.get((node, disk, relative)) + if not physical: continue + actual_role = "data" if erasure["index"] <= erasure["data_blocks"] else "parity" + cohort.append({"node": node, "disk": disk, "relativePath": relative, "sha256": physical["sha256"], + "size": physical["size"], "role": actual_role, "index": erasure["index"], + "versionId": entry["versionId"], "bucket": bucket, "key": entry["key"], + "partNumber": part["number"], "poolIndex": pool, "setIndex": group, + "diskUUID": layout["this"], "dataDir": version["dataDir"], "setSlot": slot, + "logical": version, "metadata": meta}) + require(cohort, "no external physical target for the ACK fixture") + first = cohort[0]["logical"]["erasure"] + require(len(cohort) == first["data_blocks"] + first["parity_blocks"] + and len({(c["poolIndex"], c["setIndex"]) for c in cohort}) == 1 + and len({c["index"] for c in cohort}) == len(cohort), "target lacks one complete erasure authority") + target = next((c for c in cohort if c["role"] == role), None) + require(target is not None, "requested shard role is absent") + return {**target, "cohort": cohort} + + +def assert_overlap(observed, *, task_id, kind): + # Server events share one provider clock, and must intersect a task running + # interval. A broad wall-clock client request duration is not a witness. + intervals = observed.get("intervals", []) + require(observed.get("clockDomain") and intervals, "overlap lacks a common server observation clock") + heal = [v for v in intervals if v.get("kind") == "heal" and v.get("taskId") == task_id] + other = [v for v in intervals if v.get("kind") == kind and v.get("taskId") != task_id] + for row in intervals: + require(type(row.get("startNs")) is int and type(row.get("endNs")) is int + and row["endNs"] > row["startNs"] and row.get("evidenceId"), "invalid activity interval") + if not any(max(a["startNs"], b["startNs"]) < min(a["endNs"], b["endNs"]) for a in heal for b in other): + raise NotObserved("actual Heal/" + kind + " overlap not observed") + + +def assert_terminal(status, bucket, expected, *, allowed_extra=(), allowed_absent=()): + outcome = status.get("outcome", {}) + require(status.get("summary") == "finished" and not status.get("truncated", False) + and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False + and outcome.get("execution", {}).get("state") == "completed", "task did not finish with complete canonical coverage") + wanted = {(x["key"], x["versionId"]) for x in expected} + objects = outcome.get("objects", []) + actual = set() + repaired = healthy = 0 + for row in objects: + identity = row.get("identity", {}) + pair = (identity.get("object"), identity.get("versionId") or "null") + require(pair not in actual and identity.get("bucket") == bucket and identity.get("bucketIncarnationId") + and type(identity.get("poolIndex")) is int and type(identity.get("setIndex")) is int, + "receipt lacks exact bucket/version/pool/set responsibility") + actual.add(pair) + state = row.get("disposition", {}).get("state") + valid = {"repaired", "verified_healthy", "delete_marker_healthy"} + if pair in set(allowed_absent): valid.add("already_absent") + require(state in valid, "unknown/deferred/failed/skipped is not repair success") + if state == "delete_marker_healthy": + require(any(e["key"] == pair[0] and e["versionId"] == pair[1] and e["deleteMarker"] + for e in [*expected, *allowed_extra]), "data version received marker disposition") + repaired += state == "repaired" + healthy += state != "repaired" + permitted = wanted | {(e["key"], e["versionId"]) for e in allowed_extra} + require(wanted <= actual <= permitted, "receipt identities differ from the complete ACK fixture") + counters = outcome.get("counters", {}) + expected_counts = {"processed": len(actual), "healed": repaired, "unchanged": healthy, + "skipped": 0, "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False} + require(all(type(counters.get(k)) is type(v) and counters[k] == v for k, v in expected_counts.items()), + "canonical counters disagree with complete per-object receipts") + + +def assert_action(action, observed, *, task_id, runtime): + """Raw infrastructure evidence oracles. An adapter's booleans cannot pass.""" + if action == "survivor-window": + interval = observed.get("taskInterval", {}) + require(interval.get("taskId") == task_id and type(interval.get("startNs")) is int + and type(interval.get("endNs")) is int and interval["endNs"] > interval["startNs"] + and observed.get("sourceReachable") is False, "survivor Heal interval or retired-source isolation is missing") + elif action.endswith("window"): + kind = {"replacement-window": "replacement", "ilm-window": "ilm", "replication-window": "replication", + "pool-window": "pool", "survivor-window": "heal", "io-window": "io", "dependency-window": "dependency", + "capacity-lock-window": "capacity-lock"}[action] + assert_overlap(observed, task_id=task_id, kind=kind) + if action == "capacity-lock-window": + raise Blocked("H04 exact capacity-lock owner/commit barrier still needs reviewed native evidence") + if action in {"dedicated-vm-forced-stop", "dedicated-host-power-cycle"}: + wanted = "platform-forced-stop" if action.endswith("forced-stop") else "physical-power-cycle" + require(observed.get("eventKind") == wanted and observed.get("platformActivityId") + and observed.get("resourceId") == runtime["hostResourceId"] + and observed.get("bootIdBefore") and observed.get("bootIdAfter") + and observed["bootIdBefore"] != observed["bootIdAfter"] + and observed.get("recoveryChannel") == "out-of-band", "power event is not the assigned H6 platform event") + if action in {"restart-service", "crash-service", "restart-mixed"}: + require(observed.get("pidBefore") != observed.get("pidAfter") and observed.get("rootBefore") + and observed.get("rootBefore") == observed.get("rootAfter") + and observed.get("signal") == ("SIGKILL" if action == "crash-service" else "SIGTERM"), + "restart did not preserve the original data root/process interruption") + if action.startswith("upgrade-") or action.startswith("rollback-"): + pair = runtime["releasePair"] + require(pair.get("rollbackCompatible") is True and pair.get("compatibilityEvidenceSha256") + and len({p["packageSha256"] for p in pair["releases"]}) == 2, "immutable release pair/compatibility proof absent") + allowed = {p["binarySha256"] for p in pair["releases"]} + processes = observed.get("processes", []) + require(processes and all(p.get("binarySha256") in allowed and p.get("rootBefore") == p.get("rootAfter") + and p.get("rootBefore") for p in processes), "upgrade used unverified bytes or replaced data roots") + if action in {"mixed-status", "wait-mixed"}: + peers = observed.get("peers", []) + require(len({p.get("binarySha256") for p in peers}) == 2 and len(peers) >= 2, "no real mixed-release peers observed") + require(all(p.get("coverage") in {"partial", "unsupported"} for p in peers if not p.get("capabilitySupported")), + "unsupported peer reported complete status") + if action.startswith("io-") and action not in {"io-recover", "io-window"}: + fault = action[3:] + probe = observed.get("probe", {}) + require(probe.get("uid") == observed.get("serviceUid") and probe.get("mountNamespace") == observed.get("serviceMountNamespace") + and probe.get("path") and observed.get("deviceId") == runtime["faultDeviceId"], "IO fault is not in the service namespace/device") + if fault == "bounded-latency": + require(type(probe.get("elapsedMs")) is int and probe["elapsedMs"] >= runtime["ioLatencyMs"], "bounded latency was not witnessed") + elif fault == "short-read": + require(type(probe.get("bytesRead")) is int and 0 <= probe["bytesRead"] < probe.get("bytesRequested", 0), "short read was not witnessed") + else: + require(probe.get("errno") == 5, "EIO was not witnessed") + if action.startswith("dependency-") and action not in {"dependency-window", "dependency-recover"}: + wanted = "AccessDenied" if action.endswith("denied") else "NoSuchVersion" if action.endswith("version-unavailable") else "Unavailable" + require(observed.get("errorCode") == wanted and observed.get("tenantId") == runtime["tenantId"] + and observed.get("namespaceId") == runtime["dependencyNamespaceId"], "dependency fault lacks isolated tenant/namespace/error evidence") + outcomes = observed.get("objectDispositions", []) + require(outcomes and all(o.get("state") in {"failed", "deferred", "unknown"} and o.get("reason") for o in outcomes), + "dependency error was treated as absent/healthy/removable") + if action in {"pool-settle", "replication-settle", "ilm-settle", "replacement-settle"}: + tasks = observed.get("tasks", []) + require(tasks and all(t.get("taskId") and t.get("state") == "completed" and not t.get("error") for t in tasks), + "background operation has an incomplete/failed task") + if action in {"decommission-stop", "decommission-cancel", "rebalance-stop"}: + require(observed.get("taskId") and observed.get("state") in {"stopped", "cancelled"} + and observed.get("healTaskId") == task_id and observed.get("healState") == "running", "pool cancellation also cancelled Heal or lost task ownership") + if action == "wait-inflight-exit": + require(observed.get("inflight") == 0 and observed.get("cancelCommitNs", 0) < observed.get("lastExitNs", 0), + "inflight work did not exit after durable cancellation") + if action == "decommission-clear": + require(observed.get("oldGeneration") and observed.get("inflight") == 0 and observed.get("state") == "cleared", + "clear ran before old inflight exit") + if action in {"decommission-resume", "rebalance-resume"}: + require(observed.get("newGeneration") and observed.get("oldGeneration") + and observed["newGeneration"] != observed["oldGeneration"], "resume reused an old generation") + if action == "decommission-write-guard": + require(observed.get("httpStatus") in {405, 409, 503} and observed.get("errorCode") in {"MethodNotAllowed", "OperationAborted", "SlowDown"}, + "decommission did not enforce the frozen write restriction") + if action == "source-offline": + require(observed.get("sourceEndpoints") and observed.get("reachableEndpoints") == [] + and observed.get("survivorEndpoints"), "retired source remains reachable") + + +class Runner: + def __init__(self, runtime, candidate, testing_commit, evidence, *, adapter=None, probe_factory=Probe, + clock=time.monotonic, sleep=time.sleep): + self.runtime, self.candidate, self.testing_commit, self.evidence = runtime, candidate, testing_commit, evidence + self.adapter = adapter or Adapter(runtime, evidence) + self.probe_factory, self.clock, self.sleep = probe_factory, clock, sleep + self.active = [] + + def admit(self, probe, bucket): + path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + options = {**BASE_OPTIONS, "recursive": True, "pool": None, "set": None} + row = self.adapter.record("heal-admission", probe.admin(path, [], json.dumps(options).encode())) + token = row.get("response", {}).get("clientToken") + require(row.get("httpStatus") == 200 and isinstance(token, str) and token, "Heal was not admitted with an owned token") + self.active.append((probe, path, token)) + return path, token + + def wait_task(self, probe, path, token): + deadline = self.clock() + 180 + while True: + row = self.adapter.record("heal-status", probe.admin(path, [("clientToken", token)], b"")) + require(row.get("httpStatus") == 200 and isinstance(row.get("response"), dict), "Heal status unavailable") + status = row["response"] + if status.get("summary") != "running": break + require(self.clock() < deadline, "Heal task exceeded its registered budget") + self.sleep(.5) + for name in probe.binding["endpoints"]: + other = self.probe_factory(probe.binding, name) + repeated = self.adapter.record("heal-terminal-peer", other.admin(path, [("clientToken", token)], b"")) + require(repeated.get("httpStatus") == 200 and repeated.get("response", {}).get("summary") == status.get("summary") + and canonical(repeated["response"].get("outcome")) == canonical(status.get("outcome")), "peer terminal responsibility differs") + self.active.remove((probe, path, token)) + return status + + def fixture(self, probe, directory, bucket, spec, branch): + recorder = Recorder(probe, directory / "ACKs") + fixture = "DB" if spec.code == "I01" else "DV" + count = 257 if spec.interrupted else 8 + expected = seed(recorder, bucket, fixture, count, [5 * 1024**2 + 123]) + if spec.code == "H03" and branch == "versioned-multipart": + key = "multipart/0000.bin" + upload = recorder.call("create_multipart_upload", {"Bucket": bucket, "Key": key}) + parts, full = [], b"" + for number, size in enumerate((5 * 1024**2 + 17, 1024**2 + 47), 1): + body = payload(f"{bucket}/{key}/{number}", size); full += body + response = recorder.call("upload_part", {"Bucket": bucket, "Key": key, "UploadId": upload["UploadId"], "PartNumber": number}, body) + parts.append({"PartNumber": number, "ETag": response["ETag"]}) + response = recorder.call("complete_multipart_upload", {"Bucket": bucket, "Key": key, "UploadId": upload["UploadId"], "MultipartUpload": {"Parts": parts}}) + expected.append({"key": key, "versionId": response["VersionId"], "isLatest": True, "deleteMarker": False, + "payload": {"size": len(full), "sha256": hashlib.sha256(full).hexdigest()}}) + if spec.code in {"I06", "G07"}: + key = "encrypted.bin"; body = payload(bucket + "/encrypted", 5 * 1024**2 + 73) + response = recorder.call("put_object", {"Bucket": bucket, "Key": key, "ServerSideEncryption": "aws:kms", + "SSEKMSKeyId": self.runtime["kmsKeyId"]}, body) + require(response.get("ServerSideEncryption") == "aws:kms" and response.get("SSEKMSKeyId"), "encrypted fixture was not encrypted") + expected.append({"key": key, "versionId": response["VersionId"], "isLatest": True, "deleteMarker": False, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + baseline = self.adapter.record("baseline-inventory", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + verify_seed(expected, baseline) + return recorder, expected, baseline + + def soak(self, scope, recorder, bucket, expected, topology): + duration = SOAK_SECONDS[scope["branch"]] + started = self.clock(); deadline = started + duration + writes = 0 + metrics = [] + schedule = ("drive-outage", "service-restart", "dependency-outage") + while self.clock() < deadline: + body = payload(f"{bucket}/soak/{writes}", 1024) + response = recorder.call("put_object", {"Bucket": bucket, "Key": "soak.bin"}, body) + require(response.get("VersionId") not in {None, "null"}, "soak write lost a version ACK") + for entry in expected: + if entry["key"] == "soak.bin": entry["isLatest"] = False + expected.append({"key": "soak.bin", "versionId": response["VersionId"], "isLatest": True, "deleteMarker": False, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + old = [e for e in expected if e["key"] == "soak.bin" and not e["isLatest"]] + if len(old) > 64: + entry = old[0] + deleted = recorder.call("delete_object", {"Bucket": bucket, "Key": entry["key"], "VersionId": entry["versionId"]}) + require(deleted.get("VersionId") == entry["versionId"], "soak delete ACK differs from the exact version") + expected.remove(entry) + kind = schedule[writes % len(schedule)] + # The control operation must acknowledge the real fault and undo, + # separately from the data ACK. Both are retained on failure. + witness = self.adapter.call("soak-fault", scope=scope, kind=kind, iteration=writes, bucket=bucket) + require(witness.get("faultId") and witness.get("kind") == kind and witness.get("injectedAtNs"), "soak fault has no actual witness") + restored = self.adapter.call("soak-recover", scope=scope, faultId=witness["faultId"]) + require(restored.get("faultId") == witness["faultId"] and restored.get("remainingFaults") == [], + "soak fault was not recovered") + sample = self.adapter.call("metrics", scope=scope) + for field in ("rssBytes", "openFds", "queuedTasks", "unresolvedRecords", "temporaryBytes"): + limit = self.runtime["soakLimits"][field] + require(type(sample.get(field)) is int and 0 <= sample[field] <= limit, "soak resource threshold exceeded: " + field) + require(sample.get("panicCount") == 0 and sample.get("busyLoopCount") == 0, "soak observed a panic or busy loop") + metrics.append(sample); writes += 1 + self.sleep(min(120, max(0, deadline - self.clock()))) + elapsed = self.clock() - started + require(elapsed >= duration and writes > 0, "short run cannot satisfy a 24h/72h window") + return self.adapter.record("soak-window", {"durationSeconds": duration, "actualElapsedSeconds": elapsed, + "acknowledgedWrites": writes, "samples": len(metrics), "topology": topology}) + + def branch(self, case_id, topology, branch): + recipe = select(case_id)[0] + scope = {"caseId": case_id, "topology": topology, "branch": branch} + if branch in NATIVE_BRANCHES: + return {**scope, "verdict": "BLOCKED", "cleanup": "NOT_RUN", "reason": "reviewed candidate-bound " + branch + " is unavailable"} + spec = scenario(recipe["code"], branch) + directory = self.evidence / f"{topology}-{branch}" + directory.mkdir() + bucket = self.runtime["binding"]["bucketPrefix"] + "-" + hashlib.sha256(canonical(scope).encode()).hexdigest()[:8] + binding = self.runtime["binding"] + probe = self.probe_factory(binding) + result = {**scope, "verdict": "RUNNING", "cleanup": "PENDING"} + reserved = False + def control(operation, **params): + return self.adapter.call(operation, scope=scope, bucket=bucket, **params) + try: + preflight = control("preflight") + assert_topology(preflight, topology, binding["candidateBinarySha256"], allow_old=self.runtime.get("oldBinarySha256")) + reserved = True + control("prepare", candidate=self.candidate) + recorder, expected, baseline = self.fixture(probe, directory, bucket, spec, branch) + if spec.family in {"ilm", "replication", "dependency"}: + dependency_prepared = control(spec.family + "-prepare", fixture=expected, branch=branch) + if spec.family == "retired-pool": + prepared = control("retire-pool-prepare", fixture=expected) + assert_action("source-offline", prepared, task_id="", runtime=self.runtime) + if spec.family == "soak": + self.soak(scope, recorder, bucket, expected, topology) + baseline = None # The final oracle uses every newly ACKed version. + control("stop") + original = control("snapshot", quiescent=True)["snapshot"] + faulted = original + targets = [] + groups = sorted({disk["setIndex"] for disk in preflight["disks"]}) if spec.code == "H07" else [None] + for group in groups: + target = None + for entry in expected: + if entry["deleteMarker"]: continue + try: target = target_from_snapshot(original, bucket, entry, spec.role, set_index=group) + except ValueError: continue + break + require(target is not None, "fixture does not cover a required erasure set") + request = {"target": target, "kind": "missing" if spec.fault_kind == "metadata-missing" else spec.fault_kind, + "faultId": recipe["code"] + "-" + str(len(targets))} + if spec.fault_kind == "metadata-missing": + request["target"] = {**target, "relativePath": f"{bucket}/{target['key']}/xl.meta"} + physical = files(original)[(target["node"], target["disk"], request["target"]["relativePath"])] + request["target"].update(sha256=physical["sha256"], size=physical["size"]) + before_fault = faulted + targets.append(request["target"]) + receipt = control("apply-fault", **request) + faulted = control("snapshot", quiescent=True)["snapshot"] + validate_fault(before_fault, faulted, request["target"], receipt) + control("start") + witness = control("snapshot", quiescent=False)["snapshot"] + require(files(witness) == files(faulted), "NOT_OBSERVED: target changed before Heal admission") + preliminary = {} + if spec.family in {"io", "dependency"}: + action = spec.actions[0] + preliminary[action] = control(action, targets=targets, fixture=expected) + assert_action(action, preliminary[action], task_id="", runtime=self.runtime) + admitted_fixture = list(expected) + path, task_id = self.admit(probe, bucket) + durable = control("checkpoint", taskId=task_id, fixture=expected) + records = durable.get("records", []) + require(records and all(r.get("taskId") == task_id and r.get("bucketIncarnationId") + and SHA.fullmatch(r.get("sha256", "")) and r.get("durableProgress", 0) > 0 for r in records), + "original durable responsibility was not witnessed") + # Keep S3 foreground writes in our ACK ledger during pool/upgrade IO. + if spec.family in {"pool", "upgrade", "replication"}: + body = payload(bucket + "/foreground", 1024) + response = recorder.call("put_object", {"Bucket": bucket, "Key": "foreground.bin"}, body) + version = response.get("VersionId", "null") + expected.append({"key": "foreground.bin", "versionId": version, "isLatest": True, "deleteMarker": False, + "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + # Add a new task after the first one if its snapshot predates this + # ACK; never pretend it had already included the new object. + observations = dict(preliminary) + if spec.code == "G08" and branch == "replicate-delete": + key = next(e["key"] for e in expected if e["isLatest"] and not e["deleteMarker"]) + deleted = recorder.call("delete_object", {"Bucket": bucket, "Key": key}) + require(deleted.get("DeleteMarker") is True and deleted.get("VersionId") not in {None, "null"}, + "replicated delete has no exact marker ACK") + for entry in expected: + if entry["key"] == key: entry["isLatest"] = False + marker = {"key": key, "versionId": deleted["VersionId"], "deleteMarker": True, "isLatest": True} + expected.append(marker) + stop_writes = threading.Event() + def foreground(): + number = 0 + while not stop_writes.is_set(): + body = payload(f"{bucket}/live/{number}", 1024) + key = f"live/{number:06d}.bin" + response = recorder.call("put_object", {"Bucket": bucket, "Key": key}, body) + expected.append({"key": key, "versionId": response.get("VersionId", "null"), "isLatest": True, + "deleteMarker": False, "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + number += 1 + stop_writes.wait(2) + return number + with ThreadPoolExecutor(max_workers=1) as workers: + future = workers.submit(foreground) if spec.family in {"host", "upgrade", "pool", "replication"} else None + try: + for action in spec.actions: + if action in preliminary: continue + observation = control(action, taskId=task_id, targets=targets, fixture=list(expected)) + observations[action] = observation + assert_action(action, observation, task_id=task_id, runtime=self.runtime) + finally: + stop_writes.set() + if future is not None: + require(future.result() > 0, "no foreground ACK was recorded during lifecycle operations") + if spec.code == "G07" and branch in {"current-expiry", "noncurrent-expiry"}: + expired = observations["ilm-settle"].get("expiredVersions", []) + require(expired and all(e.get("eventId") and e.get("ruleId") and e.get("versionId") for e in expired), "expiration has no actual version/rule event") + gone = {(e["key"], e["versionId"]) for e in expired} + require(gone <= {(e["key"], e["versionId"]) for e in expected}, "ILM event refers to an unacknowledged version") + if branch == "current-expiry": + require(all(e.get("deleteMarkerVersionId") for e in expired), "current expiry lacks the created marker identity") + for event in expired: + for entry in expected: + if entry["key"] == event["key"]: entry["isLatest"] = False + expected.append({"key": event["key"], "versionId": event["deleteMarkerVersionId"], "deleteMarker": True, "isLatest": True}) + gone = set() # Current expiration creates a marker; old versions survive. + else: + require(all(not e["isLatest"] for e in expected if (e["key"], e["versionId"]) in gone), + "noncurrent expiration selected a current version") + expected = [e for e in expected if (e["key"], e["versionId"]) not in gone] + terminal = self.wait_task(probe, path, task_id) + # Keep the complete pre-admission denominator. A task cannot + # choose its own expected identities by returning fewer receipts. + assert_terminal(terminal, bucket, admitted_fixture, allowed_extra=expected, + allowed_absent=gone if spec.code == "G07" and branch in {"current-expiry", "noncurrent-expiry"} else ()) + control("stop") + repaired = control("snapshot", quiescent=True)["snapshot"] + if spec.family in {"io", "upgrade", "dependency", "retired-pool"} and not any(e["key"] == "foreground.bin" for e in expected): + for target in targets: + if spec.fault_kind != "metadata-missing": validate_repaired_physical(original, repaired, target) + proof = control("verify-parts", fixture=expected, targets=targets, beforeAnyTargetGET=True) + require(proof.get("toolSha256") == self.runtime["inspectorSha256"] and proof.get("candidateCommit") == binding["candidateCommit"], + "part verification tool differs from the candidate-bound inspector") + verified = proof.get("parts", []) + require(verified and all(p.get("sha256") and p.get("checksumVerified") is True for p in verified), "physical shard verification is incomplete") + physical_ids = {(p["key"], p["versionId"]) for p in verified} + require({(e["key"], e["versionId"]) for e in expected if not e["deleteMarker"]} <= physical_ids, + "physical proof omitted an acknowledged version") + if spec.family in {"pool", "retired-pool"}: + movements = proof.get("movements", []) + require(movements and all(m.get("sourcePool") != m.get("targetPool") and m.get("targetCommitNs", 0) > 0 + and m.get("sourceDeleteNs", 0) >= m["targetCommitNs"] for m in movements), "source removed before target authority committed") + if spec.code in {"H03", "H05"}: + require(proof.get("sourceReachable") is False and proof.get("retiredRootHashesBefore") + and proof["retiredRootHashesBefore"] == proof.get("retiredRootHashesAfter"), "retired pool changed or remains reachable") + control("start") + final = self.adapter.record("final-inventory", probe.inventory(bucket, unversioned_current_tags=spec.code == "I01")) + verify_seed(expected, final) + if baseline is not None and spec.family not in {"pool", "upgrade", "replication", "ilm"}: compare_inventories(baseline, final) + if spec.family in {"replication", "ilm", "dependency"}: + endpoint = self.runtime["replicaEndpoint"] if spec.family == "replication" else self.runtime["tierEndpoint"] + remote = self.probe_factory(binding, endpoint) + remote_inventory = self.adapter.record("remote-inventory", remote.inventory(bucket)) + mapping = observations.get("replication-settle", observations.get("ilm-settle", observations.get("dependency-recover", {}))).get("versionMapping", []) + require(mapping, "remote versions have no exact mapping") + remote_ids = {(e["key"], e["versionId"]): e for e in remote_inventory["entries"]} + local_ids = {(e["key"], e["versionId"]): e for e in final["entries"]} + for row in mapping: + left = local_ids[(row["key"], row["sourceVersionId"])]; right = remote_ids[(row["key"], row["targetVersionId"])] + require(left.get("payload") == right.get("payload") and left["deleteMarker"] == right["deleteMarker"], "remote payload or marker differs") + require({(r["key"], r["sourceVersionId"]) for r in mapping} == set(local_ids), "remote mapping omits local acknowledged versions") + if spec.code == "H04" and branch == "capacity-lock-retry-window": + raise Blocked("exact capacity-lock retry barrier requires reviewed candidate native evidence") + result.update(verdict="BLOCKED", checksCompleted=True, + reason="environment scenario checks completed; assigned adapter evidence requires source-assertion review before canonical PASS") + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="BLOCKED" if isinstance(exc, Blocked) else "NOT_OBSERVED" if isinstance(exc, NotObserved) else "FAIL", + reason=str(exc)) + finally: + if reserved: + try: + for owner, path, token in tuple(self.active): + self.adapter.record("cancel-original-task", owner.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + self.active.remove((owner, path, token)) + control("recover", lastACKsOnly=True, originalFailureRetained=result["verdict"] != "PASS") + cleaned = control("cleanup") + require(cleaned.get("remainingOwnedPaths") == [] and cleaned.get("activeOwnedServices") == [] + and cleaned.get("remainingFaults") == [] and cleaned.get("pendingMutations") == [], "cleanup left a dirty environment") + result["cleanup"] = "PASS" + except (Exception, KeyboardInterrupt) as exc: + result.update(cleanup="FAIL", recoveryError=str(exc)) + else: result["cleanup"] = "NOT_RUN" + self.adapter.record("branch-result", result) + return result + + def run(self, case_id): + recipe = select(case_id)[0] + mapping = variant_topologies(recipe) + results = [] + for topology in sorted(self.runtime["topologies"]): + for branch in recipe["variants"]: + if topology not in mapping[branch]: continue + results.append(self.branch(case_id, topology, branch)) + if results[-1]["cleanup"] == "FAIL": return results + return results + + +def run(runtime_path, case_id, candidate_path, testing_commit, artifacts): + runtime = json.loads(runtime_path.read_text()) + candidate = json.loads(candidate_path.read_text()) + runtime_binding(runtime, case_id, candidate, testing_commit) + evidence = artifacts / "evidence" / "execution" + evidence.mkdir(parents=True) + runner = Runner(runtime, candidate, testing_commit, evidence) + rows = runner.run(case_id) + checks_complete = bool(rows) and all(row.get("checksCompleted") and row["cleanup"] == "PASS" for row in rows) + verdict = "INFRA_ERROR" if any(r["cleanup"] == "FAIL" for r in rows) else "FAIL" if any(r["verdict"] == "FAIL" for r in rows) else "BLOCKED" + recipe = select(case_id)[0] + result = {"schema_version": 1, "case_id": case_id, "source_status": recipe["contract"]["source_status"], + "migration_status": "unverified", "profile": ENVIRONMENT_GATED_CASES[case_id]["profile"], + "verdict": verdict, "cleanup_status": "PASS" if all(r["cleanup"] in {"PASS", "NOT_RUN"} for r in rows) else "FAIL", + "evidence": ["evidence/execution/summary.json"], + "reason": "; ".join(r.get("reason", r.get("recoveryError", "")) for r in rows) or "required branch evidence is incomplete"} + binding = runtime["binding"] + result["provenance"] = {"candidate_url": candidate["package_url"], "candidate_sha256": candidate["package_sha256"], + "binary_sha256": binding["candidateBinarySha256"], "source_commit": candidate["source_sha"], "testing_commit": testing_commit, + "rc_path": str(Path(runtime["rcBinary"]).resolve()), "rc_sha256": binding["rcBinarySha256"]} + save(evidence / "summary.json", {"caseId": case_id, "results": rows, "checksCompleted": checks_complete, + "canonicalPassClaimed": False, "reviewRequired": True}) + save(artifacts / "cases" / (case_id + ".json"), result) + return result diff --git a/scripts/validation_heal/environment_gate.py b/scripts/validation_heal/environment_gate.py new file mode 100644 index 00000000..e80685c9 --- /dev/null +++ b/scripts/validation_heal/environment_gate.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Record a safe, disabled-by-default result for environment-gated Heal cases. + +The first migration batch has complete procedures in ``recipes.json`` but the +current auto-testing fleet does not expose all required resources. This +default path never mutates the fleet. It records missing capabilities and +requires an explicit switch, capability manifest and candidate-bound runtime +before invoking the extended scenario driver. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import tempfile +from typing import Mapping, Sequence + +try: + from .execution_policy import CAPABILITIES_ENV, ENABLE_ENV, ENVIRONMENT_GATED_CASES, enabled +except ImportError: + from execution_policy import CAPABILITIES_ENV, ENABLE_ENV, ENVIRONMENT_GATED_CASES, enabled + +ROOT = Path(__file__).resolve().parents[2] +CATALOG = ROOT / "validation-cases.json" + + +def _catalog_case(case_id: str) -> dict: + document = json.loads(CATALOG.read_text(encoding="utf-8")) + for module in document["modules"]: + for case in module["cases"]: + if case["id"] == case_id: + return case + raise ValueError(f"unknown validation case: {case_id}") + + +def load_capabilities(path: Path | None) -> set[str]: + if path is None: + return set() + if not path.is_file() or path.is_symlink(): + return set() + value = json.loads(path.read_text(encoding="utf-8")) + if isinstance(value, list): + values = value + elif isinstance(value, dict): + values = value.get("capabilities", []) + else: + raise ValueError("capability manifest must be a list or an object with capabilities") + if not isinstance(values, list) or any(not isinstance(item, str) or not item for item in values): + raise ValueError("capability manifest contains an invalid capability") + return set(values) + + +def decision(case_id: str, *, environ: Mapping[str, str] | None = None, + capabilities: Sequence[str] = ()) -> dict: + policy = ENVIRONMENT_GATED_CASES[case_id] + values = environ if environ is not None else os.environ + requested = set(capabilities) + if not enabled(values): + return { + "state": "disabled", + "reasonCode": "ENV_DISABLED_BY_DEFAULT", + "enabled": False, + "missingCapabilities": list(policy["requiredCapabilities"]), + "reason": ( + f"{policy['reason']} 默认关闭；不会启动真实测试。只有设置 " + f"{ENABLE_ENV}=1 并提供 {CAPABILITIES_ENV}，且清单满足全部能力后才允许进入准备阶段。" + ), + } + missing = sorted(set(policy["requiredCapabilities"]) - requested) + if missing: + return { + "state": "blocked", + "reasonCode": "ENV_CAPABILITY_MISSING", + "enabled": True, + "missingCapabilities": missing, + "reason": ( + f"已显式开启 {ENABLE_ENV}=1，但 auto-testing Heal 环境能力清单仍缺少：" + f"{', '.join(missing)}。原始阻断原因：{policy['reason']}" + ), + } + return { + "state": "ready", + "reasonCode": "ENV_RUNTIME_REQUIRED", + "enabled": True, + "missingCapabilities": [], + "reason": ( + "已显式开启且环境能力清单满足；还需 VALIDATION_HEAL_ENVIRONMENT_RUNTIME 指定候选绑定的资源与控制适配器。" + "缺少实际资源绑定仍为 BLOCKED，能力声明本身不记 PASS。" + ), + } + + +def _write_json(path: Path, value: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as stream: + temporary = Path(stream.name) + json.dump(value, stream, ensure_ascii=False, indent=2) + stream.write("\n") + os.replace(temporary, path) + + +def write_blocked(case_id: str, artifacts: Path, *, candidate: Path | None, + testing_commit: str | None, environ: Mapping[str, str] | None = None, + override: tuple[str, str] | None = None) -> dict: + if case_id not in ENVIRONMENT_GATED_CASES: + raise ValueError(f"case is not environment-gated: {case_id}") + policy = ENVIRONMENT_GATED_CASES[case_id] + capabilities_path = None + values = environ if environ is not None else os.environ + raw_capabilities = values.get(CAPABILITIES_ENV, "").strip() + if raw_capabilities: + capabilities_path = Path(raw_capabilities).expanduser() + capabilities_error = None + try: + capabilities = load_capabilities(capabilities_path) if enabled(values) else set() + except (OSError, ValueError, json.JSONDecodeError) as exc: + capabilities = set() + capabilities_error = str(exc) + result_decision = decision(case_id, environ=values, capabilities=capabilities) + if capabilities_error: + result_decision.update(state="blocked", reasonCode="ENV_CAPABILITY_MANIFEST_INVALID", + reason=f"能力清单无效：{capabilities_error}；原始阻断原因：{policy['reason']}") + if override: + result_decision.update(state="blocked", reasonCode=override[0], reason=override[1]) + + evidence_dir = artifacts / "evidence" + evidence_path = evidence_dir / "environment-gate.json" + evidence = { + "schemaVersion": 1, + "caseId": case_id, + "code": policy["code"], + "policy": policy, + "enabled": result_decision["enabled"], + "state": result_decision["state"], + "reasonCode": result_decision["reasonCode"], + "reason": result_decision["reason"], + "missingCapabilities": result_decision["missingCapabilities"], + "enableVariable": ENABLE_ENV, + "capabilitiesVariable": CAPABILITIES_ENV, + "capabilitiesPath": str(capabilities_path) if capabilities_path else None, + "capabilities": sorted(capabilities), + "candidateManifest": str(candidate.resolve()) if candidate and candidate.exists() else None, + "candidateManifestSha256": hashlib.sha256(candidate.read_bytes()).hexdigest() + if candidate and candidate.is_file() and not candidate.is_symlink() else None, + "testingCommit": testing_commit, + "fleetTouched": False, + "canonicalPassClaimed": False, + } + _write_json(evidence_path, evidence) + case = _catalog_case(case_id) + result = { + "schema_version": 1, + "case_id": case_id, + "source_status": case["source_status"], + "migration_status": "unverified", + "profile": policy["profile"], + "verdict": "BLOCKED", + "cleanup_status": "NOT_RUN", + "reason": f"[{policy['code']}:{result_decision['reasonCode']}] {result_decision['reason']}", + "evidence": ["evidence/environment-gate.json"], + "execution_policy": { + "kind": policy["kind"], + "disabledByDefault": policy["disabledByDefault"], + "enableVariable": policy["enableVariable"], + "capabilitiesVariable": policy["capabilitiesVariable"], + "requiredCapabilities": policy["requiredCapabilities"], + "reasonCode": policy["reasonCode"], + }, + } + result_path = artifacts / "cases" / f"{case_id}.json" + _write_json(result_path, result) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--case-id", required=True) + parser.add_argument("--candidate-manifest", type=Path) + parser.add_argument("--testing-commit") + parser.add_argument("--validation-artifacts-dir", type=Path, required=True) + args = parser.parse_args() + # The default path imports no live test dependencies and never reads or + # invokes a resource adapter. A capability declaration alone cannot run. + capabilities_error = None + try: + capabilities = load_capabilities(Path(os.environ[CAPABILITIES_ENV])) if enabled() and os.environ.get(CAPABILITIES_ENV) else set() + except (OSError, ValueError, json.JSONDecodeError) as exc: + capabilities = set(); capabilities_error = str(exc) + gate = decision(args.case_id, capabilities=capabilities) + if capabilities_error: + gate.update(state="blocked", reasonCode="ENV_CAPABILITY_MANIFEST_INVALID", reason=capabilities_error) + if gate["state"] == "ready" and os.environ.get("VALIDATION_HEAL_ENVIRONMENT_RUNTIME"): + try: + from .environment_cases import run, runtime_binding, RUNTIME_ENV + except ImportError: + from environment_cases import run, runtime_binding, RUNTIME_ENV + runtime_path = Path(os.environ[RUNTIME_ENV]) + try: + runtime_binding(json.loads(runtime_path.read_text()), args.case_id, + json.loads(args.candidate_manifest.read_text()), args.testing_commit) + except (OSError, ValueError, KeyError, TypeError) as exc: + write_blocked(args.case_id, args.validation_artifacts_dir, + candidate=args.candidate_manifest, testing_commit=args.testing_commit, + override=("ENV_RUNTIME_INVALID", str(exc))) + return 0 + result = run(runtime_path, args.case_id, args.candidate_manifest, args.testing_commit, + args.validation_artifacts_dir) + print(f"{args.case_id}: {result['verdict']}") + return 0 if result["verdict"] == "PASS" else 1 + write_blocked(args.case_id, args.validation_artifacts_dir, + candidate=args.candidate_manifest, testing_commit=args.testing_commit) + print(f"{args.case_id}: BLOCKED by environment gate; fleet untouched") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(f"environment gate error: {exc}", file=os.sys.stderr) + raise SystemExit(2) diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py new file mode 100644 index 00000000..05936d93 --- /dev/null +++ b/scripts/validation_heal/execution_policy.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Execution policy for Heal cases whose live proof needs special handling. + +The recipe remains the source of the operator procedure. This module only +classifies whether the dispatcher may enter that procedure and records the +reason when it cannot make a canonical PASS claim. +""" + +from __future__ import annotations + +import os +from typing import Mapping + + +ENABLE_ENV = "AUTO_TESTING_ENABLE_ENV_GATED_HEAL" +CAPABILITIES_ENV = "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES" + + +def _environment( + code: str, + reason_code: str, + reason: str, + capabilities: tuple[str, ...], + *, + profile: str = "cluster-1pool", +) -> dict: + return { + "kind": "environment-gated", + "disabledByDefault": True, + "dispatchable": True, + "code": code, + "reasonCode": reason_code, + "reason": reason, + "requiredCapabilities": list(capabilities), + "enableVariable": ENABLE_ENV, + "capabilitiesVariable": CAPABILITIES_ENV, + "profile": profile, + } + + +def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, ...]) -> dict: + return { + "kind": "evidence-blocked", + "disabledByDefault": False, + "dispatchable": False, + "code": code, + "reasonCode": reason_code, + "reason": reason, + "requiredCapabilities": list(capabilities), + "enableVariable": None, + "capabilitiesVariable": None, + } + + +# These cases are procedure-complete, but the current auto-testing fleet does +# not provide the resources needed to run them safely. They are dispatchable +# only through the explicit environment gate and are disabled by default. +ENVIRONMENT_GATED_CASES = { + "heal.mixed-version-upgrade-rollback": _environment( + "F08", "ENV_IMMUTABLE_RELEASE_PAIR", + "需要两个已固定且相互兼容的 RustFS release、混合版本 peer 状态和经过审核的回滚兼容矩阵；当前 auto-testing Heal 节点没有这组不可变双版本资源。", + ("immutable_release_pair", "mixed_version_peer_status", "rolling_upgrade_rollback", "rollback_compatibility_matrix"), + ), + "heal.dedicated-host-forced-restart": _environment( + "F09", "ENV_DEDICATED_H6_HOST", + "需要 H6 专用主机、平台强制停启权限、带外恢复和 boot_id/平台事件证据；共享 Heal 节点不能证明真实电源循环。", + ("dedicated_h6_host", "platform_power_cycle", "out_of_band_recovery", "boot_id_evidence"), + ), + "heal.race-ilm-expiry-and-transition": _environment( + "G07", "ENV_DEDICATED_ILM_TIER_WINDOW", + "需要隔离 ILM/tier 租户、真实 Expire/Transition/Restore 窗口和远端版本映射；共享 ILM 环境不能安全制造交叠窗口。", + ("dedicated_ilm_tier", "real_expire_transition_restore_window", "remote_version_mapping", "isolated_rule_cleanup"), + ), + "heal.race-replication-resync": _environment( + "G08", "ENV_ISOLATED_REPLICATION_PAIR", + "需要隔离的复制两端、resync 控制、跨端版本映射和 Heal/resync 实际交叠窗口；当前固定 fleet 没有独占复制对。", + ("isolated_replication_pair", "resync_control", "cross_endpoint_version_mapping", "replication_overlap_window"), + ), + "heal.with-expansion-rebalance": _environment( + "H01", "ENV_H5_EXPANSION_REBALANCE", + "需要 H5 pool 扩容、rebalance 和 Heal 同时活动的完整拓扑，并逐对象保存 pool/set 迁移证据；当前 auto-testing 未绑定扩容前后布局与专用操作窗口。", + ("h5_pool_expansion", "rebalance_task", "per_object_pool_set_mapping", "retired_scanner_evidence"), + ), + "heal.with-rebalance-stop-resume": _environment( + "H02", "ENV_H5_REBALANCE_LIFECYCLE", + "需要可观测的 H5 rebalance、停止/恢复/服务重启控制及独立 task 身份；当前 auto-testing 未绑定这两类任务的独立身份和操作窗口。", + ("h5_rebalance", "rebalance_stop_resume", "independent_task_identity", "service_restart_control"), + ), + "heal.with-decommission": _environment( + "H03", "ENV_H5_DECOMMISSION", + "需要 H5 pool decommission、survivor 目标、写入限制和移动 ACK/物理权威证据；当前 auto-testing 未绑定本例退役前后拓扑和恢复方案。", + ("h5_decommission", "survivor_pool", "decommission_write_guard", "movement_ack_physical_proof"), + ), + "heal.with-decommission-cancel-clear": _environment( + "H04", "ENV_H5_DECOMMISSION_CANCEL", + "需要退役取消/清理/新代次恢复以及 #2423 所需容量锁重试窗口；普通取消结果不能替代这个临界点。", + ("h5_decommission", "cancel_clear_resume", "capacity_lock_retry_window", "generation_bound_ack_ledger"), + ), + "heal.after-retired-pool-removal": _environment( + "H05", "ENV_H5_RETIRED_POOL_REMOVAL", + "需要完整迁移后移除 retired pool、源保持离线和 survivor-only 访问；当前 auto-testing 未提供本例源持续离线和 survivor-only 的绑定证据。", + ("h5_retired_pool_removal", "source_offline_proof", "survivor_authority", "retired_pool_write_guard"), + ), + "heal.ec84-multi-set-restart": _environment( + "H07", "ENV_EC84_MULTI_SET_TOPOLOGY", + "需要 H3 的 12 个独立盘和 H4 的至少 24 个独立 endpoint，并按 set 分别记录重启/崩溃窗口；当前 H4 资源不可用。", + ("h3_ec84_12_independent_disks", "h4_ec84_24_independent_endpoints", "per_set_identity", "per_set_restart_crash_evidence"), + ), + "heal.slow-io-and-eio": _environment( + "I01", "ENV_DEDICATED_FAULT_IO", + "需要专用 faultfs 或故障设备、可恢复的 slow/short-read/EIO 注入和 namespace 隔离；共享盘模拟会污染其他运行。", + ("dedicated_faultfs_or_device", "slow_io_short_read_eio", "namespace_isolation", "fault_layer_restore"), + ), + "heal.kms-and-tier-outage": _environment( + "I06", "ENV_DEDICATED_KMS_TIER_TENANT", + "需要专用 KMS/tier 租户、依赖网络隔离和安全的 key/tier 版本证据；不能在共享依赖上全局断网。", + ("dedicated_kms_tier_tenant", "dependency_network_isolation", "key_redaction", "tier_version_mapping"), + ), + "heal.long-soak": _environment( + "J05", "ENV_DEDICATED_LONG_SOAK_LANE", + "必须在独占 H2/H5 lane 真实运行 24 小时和 72 小时，包含限速、监控、到期回收和完整资源预算；短跑不能替代 soak 证据。", + ("dedicated_soak_lane", "wall_clock_24h_72h", "resource_budget_monitoring", "scheduled_fault_recovery"), + ), +} + + +# These cases still lack complete orchestration/evidence, and I05 also lacks +# an exact service-side ownership barrier. More nodes alone do not close them. +EVIDENCE_BLOCKED_CASES = { + "heal.bucket-incarnation-recreation": _native( + "C11", "EVIDENCE_INCARNATION_LIVE_ORCHESTRATION", + "已有 3 项候选版本 incarnation 原生回归通过，但尚缺覆盖旧盘回归、旧 checkpoint/MRF 重放与同名新桶的完整真机编排和证据绑定；原生回归不能替代四个分支的 canonical 验收。", + ("bucket_incarnation_isolation", "checkpoint_mrf_replay_guard", "incarnation_native_evidence_binding", "complete_live_orchestrator"), + ), + "heal.checkpoint-and-mrf-corruption": _native( + "F07", "EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION", + "已有 journal 故障工具和 10 项候选版本持久化原生回归，但尚缺 H1/H2 全六分支编排、写失败诊断、责任保留和安全重扫；现有 publication interruption 证据也不等于每个 fsync 临界点或断电证据。", + ("durable_checkpoint_mrf_fault", "write_failure_retention", "publication_boundary_evidence_binding", "complete_live_orchestrator"), + ), + "heal.lock-timeout-and-caller-abort": _native( + "I05", "NATIVE_LOCK_OWNER_ABORT_BARRIER", + "需要真实 lock owner、超时、调用方断开和 stale read-repair 的服务端所有权/提交顺序；黑盒请求耗时只能证明 TCP 断开，不能证明锁提交屏障，候选包没有 reviewed e2e-test-hooks。", + ("server_lock_owner", "lock_timeout_fence", "caller_abort_sequence", "stale_read_repair_fence", "reviewed_e2e_test_hooks"), + ), + "heal.outcome-counters-and-error-truth": _native( + "J02", "EVIDENCE_MIXED_OUTCOME_ORCHESTRATION", + "已有 2 项候选版本 outcome 原生回归和部分计数校验，但尚缺真实混合 healthy/repaired/failed/deferred/unknown 夹具、dry-run 独立任务、逐对象完整 manifest 与计数/attemptFailures 对账；这是执行器与证据未闭环，并非已证实产品缺少 API。", + ("mixed_outcome_fixture", "per_object_canonical_receipts", "complete_live_orchestrator", "attempt_failure_visibility"), + ), +} + +FIRST_BATCH = frozenset(ENVIRONMENT_GATED_CASES) +SECOND_BATCH = frozenset(EVIDENCE_BLOCKED_CASES) +DISPATCHABLE_CASES = FIRST_BATCH | frozenset( + { + "heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.admin-nolock-safety", + "heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", + "heal.race-markers-and-batch-delete", "heal.race-multipart-complete-abort", + "heal.race-copy-and-metadata-update", "heal.dangling-and-orphan-safety", + "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", + "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", + "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", + "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", + "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", + "heal.admin-dry-run", "heal.missing-part-file", "heal.data-shard-bitrot", "heal.parity-shard-bitrot", + "heal.truncated-and-zeroed-shard", "heal.completed-multipart-object", "heal.range-head-and-conditions", + "heal.manual-normal-attribution", "heal.manual-deep-attribution", "heal.performance-abba", + "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", + "heal.below-read-quorum", "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", + "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", + "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", + "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", + "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", + "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", + "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", + "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted", + "heal.full-cluster-restart", "heal.replacement-auto-rebuild", "heal.target-graceful-restart", + "heal.target-process-crash", "heal.mrf-offline-write-replay", "heal.disk-replacement-unformatted", + } +) + + +def policy_for(case_id: str) -> dict | None: + return ENVIRONMENT_GATED_CASES.get(case_id) or EVIDENCE_BLOCKED_CASES.get(case_id) + + +def enabled(environ: Mapping[str, str] | None = None) -> bool: + values = environ if environ is not None else os.environ + return values.get(ENABLE_ENV, "").strip().lower() in {"1", "true", "yes", "on"} diff --git a/scripts/validation_heal/matrix.py b/scripts/validation_heal/matrix.py index 042ade3d..b65cbfcb 100644 --- a/scripts/validation_heal/matrix.py +++ b/scripts/validation_heal/matrix.py @@ -13,6 +13,11 @@ import socket import sys +try: + from .execution_policy import DISPATCHABLE_CASES, ENVIRONMENT_GATED_CASES, EVIDENCE_BLOCKED_CASES, policy_for +except ImportError: + from execution_policy import DISPATCHABLE_CASES, ENVIRONMENT_GATED_CASES, EVIDENCE_BLOCKED_CASES, policy_for + ROOT = Path(__file__).resolve().parents[2] RECIPES = Path(__file__).with_name("recipes.json") @@ -21,11 +26,7 @@ CONTRACT = ROOT / "docs/validation-heal-contract.md" TOPOLOGIES = {"H1", "H2", "H3", "H4", "H5", "H6"} STAGES = {"preflight", "baseline", "fault", "trigger", "control", "physical", "s3", "recovery", "logs"} -AUTOMATIC_CASES = {"heal.queue-pressure-and-admission", "heal.admin-option-contract", "heal.admin-nolock-safety", "heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-markers-and-batch-delete", "heal.race-multipart-complete-abort", "heal.race-copy-and-metadata-update", "heal.dangling-and-orphan-safety", "heal.conflicting-subquorum-authority", "heal.multi-pool-target-isolation", "heal.network-partition-and-lock-quorum", "heal.enospc-inodes-and-quota", "heal.disk-and-node-flapping", "heal.coordinator-process-crash", "heal.encrypted-objects", "heal.foreign-format-and-unknown-schema", "heal.below-write-quorum", "heal.bucket-metadata-integrity", "heal.admin-scope-isolation", "heal.admin-healthy-noop", "heal.admin-invalid-request", "heal.admin-token-cross-node-status", "heal.admin-dry-run", "heal.missing-part-file", - "heal.data-shard-bitrot", "heal.parity-shard-bitrot", "heal.truncated-and-zeroed-shard", - "heal.completed-multipart-object", "heal.range-head-and-conditions", "heal.manual-normal-attribution", "heal.manual-deep-attribution", - "heal.performance-abba", "heal.historical-versions", "heal.repeat-idempotency", "heal.versioning-suspended-null", "heal.below-read-quorum", - "heal.missing-or-corrupt-xlmeta", "heal.object-metadata-and-tags", "heal.multiple-shard-loss", "heal.special-keys-and-pagination", "heal.object-size-boundaries", "heal.admin-authorization", "heal.node-outage-rejoin", "heal.object-lock-retention", "heal.metadata-payload-mismatch", "heal.read-repair-attribution", "heal.disk-readonly-and-permission", "heal.scanner-cold-object-repair", "heal.delete-marker-semantics", "heal.status-partial-disabled-unknown", "heal.incremental-cursor-and-truncation", "heal.admin-stop-and-stale-token", "heal.cancelled-task-restart", "heal.admin-overlap-and-force-start", "heal.root-graceful-coordinator-restart", "heal.read-quorum-boundary", "heal.stale-version-rejoin", "heal.disk-loss-and-repair", "heal.disk-replacement-formatted", "heal.full-cluster-restart", "heal.replacement-auto-rebuild", "heal.target-graceful-restart", "heal.target-process-crash", "heal.mrf-offline-write-replay", "heal.disk-replacement-unformatted"} +AUTOMATIC_CASES = DISPATCHABLE_CASES - set(ENVIRONMENT_GATED_CASES) def require(condition: bool, message: str) -> None: @@ -82,6 +83,15 @@ def cases() -> list[dict]: require(bool(item["commands"]) and set(item["commands"]) <= anchors, "Heal procedure refers to an undefined operation") require(bool(item["recovery"]) and bool(item["verdict"]), "missing Heal recovery/verdict contract") + expected_policy = policy_for(item["id"]) + if expected_policy is None: + require("executionPolicy" not in item, + f"unexpected execution policy for {item['id']}") + else: + require(item.get("executionPolicy") == expected_policy, + f"execution policy mismatch for {item['id']}") + require(item["executionPolicy"]["dispatchable"] == (item["id"] in DISPATCHABLE_CASES), + f"dispatchability mismatch for {item['id']}") item["contract"] = contract variant_topologies(item) return sorted(observed, key=lambda item: item["code"]) @@ -144,7 +154,8 @@ def compile_plan(selected: list[dict], topologies: set[str], run_id: str) -> dic "code": item["code"], "topology": topology, "branch": variant, "state": "NOT_RUN", "parameterBindings": {}, "sourceAssertions": item["contract"]["assertions"], "operatorSteps": item["steps"], "operations": item["commands"], - "requiredEvidence": item["requiredEvidence"], "caseVerdict": None}) + "requiredEvidence": item["requiredEvidence"], "caseVerdict": None, + "executionPolicy": item.get("executionPolicy")}) require(bool(instances), "selected Heal cases have no requested topology") return {"schemaVersion": 1, "runId": run_id, "createdAt": datetime.now(timezone.utc).isoformat(), "executionMode": "operator-assisted", "state": "PLAN_ONLY", "automaticExecution": False, @@ -152,6 +163,10 @@ def compile_plan(selected: list[dict], topologies: set[str], run_id: str) -> dic "operationBookSha256": digest(OPERATIONS), "publicContractSha256": digest(CONTRACT), "selectedBaseCaseCount": len(selected), "selectedCaseIds": [item["id"] for item in selected], "selectedTopologies": sorted(topologies), + "dispatchableCaseCount": len(DISPATCHABLE_CASES), + "automaticCaseCount": len(AUTOMATIC_CASES), + "environmentGatedCaseCount": len(ENVIRONMENT_GATED_CASES), + "evidenceBlockedCaseCount": len(EVIDENCE_BLOCKED_CASES), "branchTemplateCount": len(instances), "scopeExclusions": excluded, "nonApplicableCombinations": non_applicable, "environmentBinding": fleet_binding(), "instances": instances} @@ -203,12 +218,16 @@ def render(selected: list[dict]) -> str: for item in selected: lines += [f'', "", f'## {item["code"]}: {item["id"]}', "", f'Topologies: {item["contract"]["probes"]["topology"]}. Branches: {", ".join(item["variants"])}.', ""] + policy = item.get("executionPolicy") + if policy: + lines += [f"Execution policy: `{policy['kind']}`; disabled by default: `{str(policy['disabledByDefault']).lower()}`.", + "", f"Blocked reason (`{policy['reasonCode']}`): {policy['reason']}", ""] lines += [f"{index}. {step}" for index, step in enumerate(item["steps"], 1)] lines += ["", "Operations: " + " → ".join(f"[{command}](validation-heal-operations.md#{command})" for command in item["commands"]), "", "Source assertions:", ""] lines += [f"- {assertion}" for assertion in item["contract"]["assertions"]] lines += ["", item["recovery"], "", item["verdict"], ""] - return "\n".join(lines) + return "\n".join(lines).rstrip() def main() -> int: @@ -235,10 +254,15 @@ def main() -> int: inventory = cases() full = compile_plan(inventory, TOPOLOGIES, "manual-heal-contract-check") result = {"baseCases": len(inventory), "operatorBranches": sum(len(item["variants"]) for item in inventory), - "topologyBranchTemplates": len(full["instances"]), "automaticCaseCount": len(AUTOMATIC_CASES), "livePassClaimed": False} + "topologyBranchTemplates": len(full["instances"]), "automaticCaseCount": len(AUTOMATIC_CASES), + "environmentGatedCaseCount": len(ENVIRONMENT_GATED_CASES), + "evidenceBlockedCaseCount": len(EVIDENCE_BLOCKED_CASES), + "dispatchableCaseCount": len(DISPATCHABLE_CASES), "livePassClaimed": False} if args.command == "list": result["cases"] = [{"code": item["code"], "id": item["id"], "variants": item["variants"], - "automaticExecutor": item["id"] in AUTOMATIC_CASES} for item in inventory] + "automaticExecutor": item["id"] in AUTOMATIC_CASES, + "dispatchable": item["id"] in DISPATCHABLE_CASES, + "executionPolicy": item.get("executionPolicy")} for item in inventory] print(json.dumps(result, ensure_ascii=False)) elif args.command == "show": print(json.dumps(select(args.selection), ensure_ascii=False, indent=2)) diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index e9483d49..f3dfc248 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -1202,7 +1202,23 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "evidence-blocked", + "disabledByDefault": false, + "dispatchable": false, + "code": "C11", + "reasonCode": "EVIDENCE_INCARNATION_LIVE_ORCHESTRATION", + "reason": "已有 3 项候选版本 incarnation 原生回归通过，但尚缺覆盖旧盘回归、旧 checkpoint/MRF 重放与同名新桶的完整真机编排和证据绑定；原生回归不能替代四个分支的 canonical 验收。", + "requiredCapabilities": [ + "bucket_incarnation_isolation", + "checkpoint_mrf_replay_guard", + "incarnation_native_evidence_binding", + "complete_live_orchestrator" + ], + "enableVariable": null, + "capabilitiesVariable": null + } }, { "code": "D01", @@ -2132,7 +2148,23 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "evidence-blocked", + "disabledByDefault": false, + "dispatchable": false, + "code": "F07", + "reasonCode": "EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION", + "reason": "已有 journal 故障工具和 10 项候选版本持久化原生回归，但尚缺 H1/H2 全六分支编排、写失败诊断、责任保留和安全重扫；现有 publication interruption 证据也不等于每个 fsync 临界点或断电证据。", + "requiredCapabilities": [ + "durable_checkpoint_mrf_fault", + "write_failure_retention", + "publication_boundary_evidence_binding", + "complete_live_orchestrator" + ], + "enableVariable": null, + "capabilitiesVariable": null + } }, { "code": "F08", @@ -2172,7 +2204,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "environment-gated", + "disabledByDefault": true, + "dispatchable": true, + "code": "F08", + "reasonCode": "ENV_IMMUTABLE_RELEASE_PAIR", + "reason": "需要两个已固定且相互兼容的 RustFS release、混合版本 peer 状态和经过审核的回滚兼容矩阵；当前 auto-testing Heal 节点没有这组不可变双版本资源。", + "requiredCapabilities": [ + "immutable_release_pair", + "mixed_version_peer_status", + "rolling_upgrade_rollback", + "rollback_compatibility_matrix" + ], + "enableVariable": "AUTO_TESTING_ENABLE_ENV_GATED_HEAL", + "capabilitiesVariable": "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES", + "profile": "cluster-1pool" + } }, { "code": "F09", @@ -2210,7 +2259,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "environment-gated", + "disabledByDefault": true, + "dispatchable": true, + "code": "F09", + "reasonCode": "ENV_DEDICATED_H6_HOST", + "reason": "需要 H6 专用主机、平台强制停启权限、带外恢复和 boot_id/平台事件证据；共享 Heal 节点不能证明真实电源循环。", + "requiredCapabilities": [ + "dedicated_h6_host", + "platform_power_cycle", + "out_of_band_recovery", + "boot_id_evidence" + ], + "enableVariable": "AUTO_TESTING_ENABLE_ENV_GATED_HEAL", + "capabilitiesVariable": "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES", + "profile": "cluster-1pool" + } }, { "code": "G01", @@ -2490,7 +2556,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "environment-gated", + "disabledByDefault": true, + "dispatchable": true, + "code": "G07", + "reasonCode": "ENV_DEDICATED_ILM_TIER_WINDOW", + "reason": "需要隔离 ILM/tier 租户、真实 Expire/Transition/Restore 窗口和远端版本映射；共享 ILM 环境不能安全制造交叠窗口。", + "requiredCapabilities": [ + "dedicated_ilm_tier", + "real_expire_transition_restore_window", + "remote_version_mapping", + "isolated_rule_cleanup" + ], + "enableVariable": "AUTO_TESTING_ENABLE_ENV_GATED_HEAL", + "capabilitiesVariable": "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES", + "profile": "cluster-1pool" + } }, { "code": "G08", @@ -2531,7 +2614,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "environment-gated", + "disabledByDefault": true, + "dispatchable": true, + "code": "G08", + "reasonCode": "ENV_ISOLATED_REPLICATION_PAIR", + "reason": "需要隔离的复制两端、resync 控制、跨端版本映射和 Heal/resync 实际交叠窗口；当前固定 fleet 没有独占复制对。", + "requiredCapabilities": [ + "isolated_replication_pair", + "resync_control", + "cross_endpoint_version_mapping", + "replication_overlap_window" + ], + "enableVariable": "AUTO_TESTING_ENABLE_ENV_GATED_HEAL", + "capabilitiesVariable": "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES", + "profile": "cluster-1pool" + } }, { "code": "H01", @@ -2571,7 +2671,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "environment-gated", + "disabledByDefault": true, + "dispatchable": true, + "code": "H01", + "reasonCode": "ENV_H5_EXPANSION_REBALANCE", + "reason": "需要 H5 pool 扩容、rebalance 和 Heal 同时活动的完整拓扑，并逐对象保存 pool/set 迁移证据；当前 auto-testing 未绑定扩容前后布局与专用操作窗口。", + "requiredCapabilities": [ + "h5_pool_expansion", + "rebalance_task", + "per_object_pool_set_mapping", + "retired_scanner_evidence" + ], + "enableVariable": "AUTO_TESTING_ENABLE_ENV_GATED_HEAL", + "capabilitiesVariable": "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES", + "profile": "cluster-1pool" + } }, { "code": "H02", @@ -2610,7 +2727,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "environment-gated", + "disabledByDefault": true, + "dispatchable": true, + "code": "H02", + "reasonCode": "ENV_H5_REBALANCE_LIFECYCLE", + "reason": "需要可观测的 H5 rebalance、停止/恢复/服务重启控制及独立 task 身份；当前 auto-testing 未绑定这两类任务的独立身份和操作窗口。", + "requiredCapabilities": [ + "h5_rebalance", + "rebalance_stop_resume", + "independent_task_identity", + "service_restart_control" + ], + "enableVariable": "AUTO_TESTING_ENABLE_ENV_GATED_HEAL", + "capabilitiesVariable": "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES", + "profile": "cluster-1pool" + } }, { "code": "H03", @@ -2649,7 +2783,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "environment-gated", + "disabledByDefault": true, + "dispatchable": true, + "code": "H03", + "reasonCode": "ENV_H5_DECOMMISSION", + "reason": "需要 H5 pool decommission、survivor 目标、写入限制和移动 ACK/物理权威证据；当前 auto-testing 未绑定本例退役前后拓扑和恢复方案。", + "requiredCapabilities": [ + "h5_decommission", + "survivor_pool", + "decommission_write_guard", + "movement_ack_physical_proof" + ], + "enableVariable": "AUTO_TESTING_ENABLE_ENV_GATED_HEAL", + "capabilitiesVariable": "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES", + "profile": "cluster-1pool" + } }, { "code": "H04", @@ -2691,7 +2842,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "environment-gated", + "disabledByDefault": true, + "dispatchable": true, + "code": "H04", + "reasonCode": "ENV_H5_DECOMMISSION_CANCEL", + "reason": "需要退役取消/清理/新代次恢复以及 #2423 所需容量锁重试窗口；普通取消结果不能替代这个临界点。", + "requiredCapabilities": [ + "h5_decommission", + "cancel_clear_resume", + "capacity_lock_retry_window", + "generation_bound_ack_ledger" + ], + "enableVariable": "AUTO_TESTING_ENABLE_ENV_GATED_HEAL", + "capabilitiesVariable": "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES", + "profile": "cluster-1pool" + } }, { "code": "H05", @@ -2730,7 +2898,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "environment-gated", + "disabledByDefault": true, + "dispatchable": true, + "code": "H05", + "reasonCode": "ENV_H5_RETIRED_POOL_REMOVAL", + "reason": "需要完整迁移后移除 retired pool、源保持离线和 survivor-only 访问；当前 auto-testing 未提供本例源持续离线和 survivor-only 的绑定证据。", + "requiredCapabilities": [ + "h5_retired_pool_removal", + "source_offline_proof", + "survivor_authority", + "retired_pool_write_guard" + ], + "enableVariable": "AUTO_TESTING_ENABLE_ENV_GATED_HEAL", + "capabilitiesVariable": "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES", + "profile": "cluster-1pool" + } }, { "code": "H06", @@ -2781,10 +2966,18 @@ "ec84-two-sets-crash" ], "variantTopologies": { - "ec84-one-set-restart": ["H3"], - "ec84-one-set-crash": ["H3"], - "ec84-two-sets-restart": ["H4"], - "ec84-two-sets-crash": ["H4"] + "ec84-one-set-restart": [ + "H3" + ], + "ec84-one-set-crash": [ + "H3" + ], + "ec84-two-sets-restart": [ + "H4" + ], + "ec84-two-sets-crash": [ + "H4" + ] }, "topologyApplicabilityReason": "H3 是 EC8+4 单 set，H4 是 EC8+4 至少两个 set；仅执行与该拓扑形状相符的重启/崩溃分支，不做全笛卡尔积。两个拓扑及四个操作分支均必须完成；缺少资源仍为 BLOCKED。", "steps": [ @@ -2816,7 +3009,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "environment-gated", + "disabledByDefault": true, + "dispatchable": true, + "code": "H07", + "reasonCode": "ENV_EC84_MULTI_SET_TOPOLOGY", + "reason": "需要 H3 的 12 个独立盘和 H4 的至少 24 个独立 endpoint，并按 set 分别记录重启/崩溃窗口；当前 H4 资源不可用。", + "requiredCapabilities": [ + "h3_ec84_12_independent_disks", + "h4_ec84_24_independent_endpoints", + "per_set_identity", + "per_set_restart_crash_evidence" + ], + "enableVariable": "AUTO_TESTING_ENABLE_ENV_GATED_HEAL", + "capabilitiesVariable": "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES", + "profile": "cluster-1pool" + } }, { "code": "I01", @@ -2856,7 +3066,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "environment-gated", + "disabledByDefault": true, + "dispatchable": true, + "code": "I01", + "reasonCode": "ENV_DEDICATED_FAULT_IO", + "reason": "需要专用 faultfs 或故障设备、可恢复的 slow/short-read/EIO 注入和 namespace 隔离；共享盘模拟会污染其他运行。", + "requiredCapabilities": [ + "dedicated_faultfs_or_device", + "slow_io_short_read_eio", + "namespace_isolation", + "fault_layer_restore" + ], + "enableVariable": "AUTO_TESTING_ENABLE_ENV_GATED_HEAL", + "capabilitiesVariable": "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES", + "profile": "cluster-1pool" + } }, { "code": "I02", @@ -3019,7 +3246,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "evidence-blocked", + "disabledByDefault": false, + "dispatchable": false, + "code": "I05", + "reasonCode": "NATIVE_LOCK_OWNER_ABORT_BARRIER", + "reason": "需要真实 lock owner、超时、调用方断开和 stale read-repair 的服务端所有权/提交顺序；黑盒请求耗时只能证明 TCP 断开，不能证明锁提交屏障，候选包没有 reviewed e2e-test-hooks。", + "requiredCapabilities": [ + "server_lock_owner", + "lock_timeout_fence", + "caller_abort_sequence", + "stale_read_repair_fence", + "reviewed_e2e_test_hooks" + ], + "enableVariable": null, + "capabilitiesVariable": null + } }, { "code": "I06", @@ -3059,7 +3303,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "environment-gated", + "disabledByDefault": true, + "dispatchable": true, + "code": "I06", + "reasonCode": "ENV_DEDICATED_KMS_TIER_TENANT", + "reason": "需要专用 KMS/tier 租户、依赖网络隔离和安全的 key/tier 版本证据；不能在共享依赖上全局断网。", + "requiredCapabilities": [ + "dedicated_kms_tier_tenant", + "dependency_network_isolation", + "key_redaction", + "tier_version_mapping" + ], + "enableVariable": "AUTO_TESTING_ENABLE_ENV_GATED_HEAL", + "capabilitiesVariable": "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES", + "profile": "cluster-1pool" + } }, { "code": "J01", @@ -3140,7 +3401,23 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "evidence-blocked", + "disabledByDefault": false, + "dispatchable": false, + "code": "J02", + "reasonCode": "EVIDENCE_MIXED_OUTCOME_ORCHESTRATION", + "reason": "已有 2 项候选版本 outcome 原生回归和部分计数校验，但尚缺真实混合 healthy/repaired/failed/deferred/unknown 夹具、dry-run 独立任务、逐对象完整 manifest 与计数/attemptFailures 对账；这是执行器与证据未闭环，并非已证实产品缺少 API。", + "requiredCapabilities": [ + "mixed_outcome_fixture", + "per_object_canonical_receipts", + "complete_live_orchestrator", + "attempt_failure_visibility" + ], + "enableVariable": null, + "capabilitiesVariable": null + } }, { "code": "J03", @@ -3258,7 +3535,24 @@ "logs" ], "recovery": "先保存失败现场及物理/状态证据，停止本例写入与在途任务，再按 RECOVER 撤销本例故障。并发版本已变化时禁止盲目恢复旧 xl.meta；恢复不能充作产品修复。未恢复环境标 DIRTY，保留本例备份和原失败结果。", - "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。" + "verdict": "逐项满足设计公共契约、该分支步骤及附加断言后人工评审。缺资源为 BLOCKED，窗口未命中为 NOT_OBSERVED，产品错误为 FAIL，工具/恢复错误为 INFRA_ERROR；未运行不记 PASS。", + "executionPolicy": { + "kind": "environment-gated", + "disabledByDefault": true, + "dispatchable": true, + "code": "J05", + "reasonCode": "ENV_DEDICATED_LONG_SOAK_LANE", + "reason": "必须在独占 H2/H5 lane 真实运行 24 小时和 72 小时，包含限速、监控、到期回收和完整资源预算；短跑不能替代 soak 证据。", + "requiredCapabilities": [ + "dedicated_soak_lane", + "wall_clock_24h_72h", + "resource_budget_monitoring", + "scheduled_fault_recovery" + ], + "enableVariable": "AUTO_TESTING_ENABLE_ENV_GATED_HEAL", + "capabilitiesVariable": "VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES", + "profile": "cluster-1pool" + } }, { "code": "J06", diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index f756fcb6..57866f72 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -6,6 +6,7 @@ RUSTFS_HEAL_SCRIPT_SOURCE_ONLY=1 source "${ROOT}/rustfs_heal_test.sh" ASSUME_YES=1 candidate="" testing_commit="" artifacts="" case_id="" driver=healthy.py +environment_gate=0 while [ "$#" -gt 0 ]; do case "$1" in --validation-case) case_id="$2"; shift 2 ;; @@ -46,12 +47,29 @@ case "${case_id}" in heal.dangling-and-orphan-safety) driver=orphan.py ;; heal.admin-nolock-safety) driver=nolock.py ;; heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-markers-and-batch-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update) driver=race.py ;; + heal.mixed-version-upgrade-rollback|heal.dedicated-host-forced-restart|heal.race-ilm-expiry-and-transition|heal.race-replication-resync|heal.with-expansion-rebalance|heal.with-rebalance-stop-resume|heal.with-decommission|heal.with-decommission-cancel-clear|heal.after-retired-pool-removal|heal.ec84-multi-set-restart|heal.slow-io-and-eio|heal.kms-and-tier-outage|heal.long-soak) + driver=environment_gate.py; environment_gate=1 ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; esac +if [ "${environment_gate}" -eq 1 ]; then + if [ "${DRY_RUN}" -eq 1 ]; then + log "${case_id}: environment-gated and disabled by default; set AUTO_TESTING_ENABLE_ENV_GATED_HEAL=1 with a capability manifest to prepare an operator-assisted run" + exit 0 + fi + [ -f "${candidate}" ] || die "candidate manifest missing" + [ -n "${artifacts}" ] || die "validation artifacts directory required" + [ ! -e "${artifacts}" ] || die "fresh artifact directory required" + python3 -B "${ROOT}/scripts/validation_heal/${driver}" \ + --case-id "${case_id}" \ + --candidate-manifest "${candidate}" \ + --testing-commit "${testing_commit}" \ + --validation-artifacts-dir "${artifacts}" + exit 0 +fi [ "${#NODES[@]}" -ge 4 ] || die "H1/H2 requires four current auto-testing Heal nodes" NODES=("${NODES[0]}" "${NODES[1]}" "${NODES[2]}" "${NODES[3]}") if [ "${DRY_RUN}" -eq 1 ]; then diff --git a/scripts/validation_report.py b/scripts/validation_report.py index c8325406..810417b1 100644 --- a/scripts/validation_report.py +++ b/scripts/validation_report.py @@ -151,7 +151,8 @@ def validate_results( case, result, gate=gated, artifacts_root=directory.parent, target_environments=target_environments, )) - rows.append({"case_id": case_id, "source_status": case["source_status"], "verdict": result.get("verdict")}) + rows.append({"case_id": case_id, "source_status": case["source_status"], "verdict": result.get("verdict"), + "reason": result.get("reason", "")}) return rows, errors @@ -159,9 +160,10 @@ def render_report(rows: list[dict], errors: list[str]) -> str: counts = Counter(row["verdict"] for row in rows) lines = ["# Validation migration report", "", f"Cases: {len(rows)}", ""] lines.extend(f"- {status}: {counts.get(status, 0)}" for status in ("PASS", "FAIL", "INFRA_ERROR", "BLOCKED", "UNSUPPORTED", "RUNNING", "MISSING")) - lines.extend(["", "| Case | Source status | Verdict |", "| --- | --- | --- |"]) + lines.extend(["", "| Case | Source status | Verdict | Reason |", "| --- | --- | --- | --- |"]) for row in rows: - lines.append(f"| {row['case_id']} | {row['source_status']} | {row['verdict']} |") + reason = str(row.get("reason", "")).replace("|", "\\|").replace("\n", " ").replace("\r", " ") + lines.append(f"| {row['case_id']} | {row['source_status']} | {row['verdict']} | {reason} |") if errors: lines.extend(["", "## Validation errors", ""]) lines.extend(f"- {error}" for error in errors) diff --git a/test_validation_heal_environment_gate.py b/test_validation_heal_environment_gate.py new file mode 100644 index 00000000..5ece294c --- /dev/null +++ b/test_validation_heal_environment_gate.py @@ -0,0 +1,347 @@ +import copy +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal.environment_gate import decision, write_blocked +from scripts.validation_heal.execution_policy import ( + CAPABILITIES_ENV, + ENABLE_ENV, + ENVIRONMENT_GATED_CASES, + EVIDENCE_BLOCKED_CASES, +) +from scripts.validation_heal.matrix import AUTOMATIC_CASES, DISPATCHABLE_CASES, cases +from scripts.validation_report import render_report, validate_results +from scripts.validation_heal import environment_cases as extended +from scripts.validation_heal.matrix import select, variant_topologies + + +class EnvironmentGateTest(unittest.TestCase): + def test_first_and_second_batches_are_disjoint_and_classified(self): + self.assertEqual(len(ENVIRONMENT_GATED_CASES), 13) + self.assertEqual(len(EVIDENCE_BLOCKED_CASES), 4) + self.assertTrue(set(ENVIRONMENT_GATED_CASES).isdisjoint(EVIDENCE_BLOCKED_CASES)) + self.assertTrue(set(ENVIRONMENT_GATED_CASES) <= DISPATCHABLE_CASES) + self.assertFalse(set(EVIDENCE_BLOCKED_CASES) & DISPATCHABLE_CASES) + self.assertEqual(len(AUTOMATIC_CASES), 67) + + def test_recipes_carry_policy_without_changing_source_case_count(self): + inventory = {item["id"]: item for item in cases()} + for case_id, policy in {**ENVIRONMENT_GATED_CASES, **EVIDENCE_BLOCKED_CASES}.items(): + self.assertEqual(inventory[case_id]["executionPolicy"], policy) + self.assertEqual(sum(item.get("executionPolicy", {}).get("kind") == "environment-gated" for item in inventory.values()), 13) + self.assertEqual(sum(item.get("executionPolicy", {}).get("kind") == "evidence-blocked" for item in inventory.values()), 4) + + def test_default_gate_is_blocked_without_touching_fleet(self): + case_id = "heal.mixed-version-upgrade-rollback" + result = decision(case_id, environ={}) + self.assertEqual(result["state"], "disabled") + self.assertEqual(result["reasonCode"], "ENV_DISABLED_BY_DEFAULT") + self.assertFalse(result["enabled"]) + self.assertEqual(set(result["missingCapabilities"]), set(ENVIRONMENT_GATED_CASES[case_id]["requiredCapabilities"])) + + def test_explicit_gate_requires_every_capability(self): + case_id = "heal.ec84-multi-set-restart" + result = decision(case_id, environ={ENABLE_ENV: "1"}, capabilities=[]) + self.assertEqual(result["state"], "blocked") + self.assertEqual(result["reasonCode"], "ENV_CAPABILITY_MISSING") + self.assertTrue(result["missingCapabilities"]) + + def test_capability_manifest_alone_needs_bound_runtime(self): + case_id = "heal.kms-and-tier-outage" + policy = ENVIRONMENT_GATED_CASES[case_id] + result = decision(case_id, environ={ENABLE_ENV: "1"}, capabilities=policy["requiredCapabilities"]) + self.assertEqual(result["state"], "ready") + self.assertEqual(result["reasonCode"], "ENV_RUNTIME_REQUIRED") + + def test_blocked_result_is_reportable_with_reason_and_evidence(self): + case_id = "heal.race-ilm-expiry-and-transition" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + candidate = root / "candidate.json" + candidate.write_text(json.dumps({"package_url": "https://example.invalid/rustfs.deb", "package_sha256": "a" * 64})) + artifacts = root / "artifacts" + result = write_blocked(case_id, artifacts, candidate=candidate, testing_commit="b" * 40, environ={}) + self.assertEqual(result["verdict"], "BLOCKED") + self.assertEqual(result["execution_policy"]["kind"], "environment-gated") + self.assertFalse(json.loads((artifacts / "evidence/environment-gate.json").read_text())["fleetTouched"]) + catalog = json.loads(Path("validation-cases.json").read_text()) + rows, errors = validate_results(catalog, artifacts / "cases", gate_statuses=set()) + self.assertFalse(errors, errors) + self.assertEqual(next(row for row in rows if row["case_id"] == case_id)["verdict"], "BLOCKED") + + def test_dispatchers_include_all_environment_gated_cases(self): + for path in (Path("scripts/run_validation_case.sh"), Path("scripts/validation_heal/run_case.sh")): + text = path.read_text() + for case_id in ENVIRONMENT_GATED_CASES: + self.assertIn(case_id, text) + + +class ExtendedScenariosTest(unittest.TestCase): + def binding(self): + return {"schemaVersion": 1, "runId": "manual-environment-test", "jumpHostOnly": True, + "exclusiveHealService": True, "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(days=10)).isoformat(), + "bucketPrefix": "rv-heal-environment-test", "candidateCommit": "a" * 40, "harnessCommit": "b" * 40, + "candidatePackageSha256": "c" * 64, "candidateBinarySha256": "d" * 64, "rcBinarySha256": "e" * 64, + "endpoints": {"node1": "http://localhost:19000"}, "defaultEndpoint": "node1", + "maxInventoryObjects": 10000, "maxInventoryBytes": 8 * 1024**3} + + def topology(self, code="H1"): + count = 24 if code == "H4" else 12 if code == "H3" else 4 + disks = [{"hostname": "node1", "mountDevice": str(index), "diskUUID": "disk-" + str(index), + "poolIndex": 0, "setIndex": index // 12 if code in {"H3", "H4"} else 0, + "setSlot": index % 12 if code in {"H3", "H4"} else index, + "dataShards": 8 if code in {"H3", "H4"} else 2, + "parityShards": 4 if code in {"H3", "H4"} else 2} for index in range(count)] + return {"disks": disks, "processes": [{"hostname": "node1", "binarySha256": "d" * 64, "pid": 1234}]} + + def test_every_source_branch_has_executable_control_flow_or_explicit_native_block(self): + templates = [] + for case_id, policy in ENVIRONMENT_GATED_CASES.items(): + recipe = select(case_id)[0] + operations = extended.required_operations(case_id) + self.assertTrue({"preflight", "prepare", "recover", "cleanup"} <= operations) + mapping = variant_topologies(recipe) + for branch in recipe["variants"]: + templates.extend((case_id, topology, branch) for topology in mapping[branch]) + if branch in extended.NATIVE_BRANCHES: continue + spec = extended.scenario(policy["code"], branch) + self.assertTrue(set(spec.actions) <= operations) + if policy["code"] != "J05": self.assertTrue(spec.actions) + self.assertEqual(len(templates), len(set(templates))) + h7 = [row for row in templates if row[0] == "heal.ec84-multi-set-restart"] + self.assertEqual(len(h7), 4) + + def test_default_dispatch_all_thirteen_never_invokes_ssh_or_extended_runtime(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + forbidden = root / "ssh-called" + ssh = root / "ssh" + ssh.write_text("#!/bin/sh\ntouch '" + str(forbidden) + "'\nexit 99\n") + ssh.chmod(0o755) + candidate = root / "candidate.json"; candidate.write_text("{}") + environ = {**os.environ, "PATH": str(root) + os.pathsep + os.environ["PATH"], + ENABLE_ENV: "0", CAPABILITIES_ENV: str(root / "invalid-capabilities"), + extended.RUNTIME_ENV: "/must-not-read-runtime"} + for case_id in ENVIRONMENT_GATED_CASES: + with self.subTest(case=case_id): + artifacts = root / case_id + command = ["bash", "scripts/validation_heal/run_case.sh", "--validation-case", case_id, + "--candidate-manifest", str(candidate), "--testing-commit", "b" * 40, + "--validation-artifacts-dir", str(artifacts)] + result = subprocess.run(command, env=environ, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + value = json.loads((artifacts / "cases" / (case_id + ".json")).read_text()) + self.assertEqual(value["verdict"], "BLOCKED") + self.assertEqual(value["migration_status"], "unverified") + self.assertIn("ENV_DISABLED_BY_DEFAULT", value["reason"]) + self.assertFalse(forbidden.exists()) + + def test_selected_blocked_case_fails_canonical_gate_and_report_shows_reason(self): + case_id = "heal.slow-io-and-eio" + with tempfile.TemporaryDirectory() as tmp: + artifacts = Path(tmp) + write_blocked(case_id, artifacts, candidate=None, testing_commit=None, environ={}) + catalog = json.loads(Path("validation-cases.json").read_text()) + rows, errors = validate_results(catalog, artifacts / "cases", gate_statuses=set(), gate_cases={case_id}) + self.assertEqual(errors, [case_id + ": gate verdict is 'BLOCKED'"]) + self.assertIn("ENV_DISABLED_BY_DEFAULT", render_report(rows, errors)) + self.assertIn("专用 faultfs", render_report(rows, errors)) + + def test_runtime_requires_all_topologies_candidate_and_adapter_digest_before_commands(self): + case_id = "heal.slow-io-and-eio" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); rc = root / "rc"; rc.write_text("fixture rc") + adapter = root / "adapter"; adapter.write_text("#!/bin/sh\nexit 99\n"); adapter.chmod(0o755) + binding = self.binding(); binding["rcBinarySha256"] = hashlib.sha256(rc.read_bytes()).hexdigest() + runtime = {"schemaVersion": 1, "provider": "auto-testing", "binding": binding, "rcBinary": str(rc), + "sshNodes": os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split(), + "topologies": ["H1", "H2"], "inspectorSha256": "f" * 64, "faultDeviceId": "device", "capabilities": ENVIRONMENT_GATED_CASES[case_id]["requiredCapabilities"], + "adapter": {"executable": str(adapter), "sha256": hashlib.sha256(adapter.read_bytes()).hexdigest(), + "operations": sorted(extended.required_operations(case_id))}} + candidate = {"source_sha": "a" * 40, "package_sha256": "c" * 64} + self.assertEqual(extended.runtime_binding(runtime, case_id, candidate, "b" * 40), binding) + for kind in ("candidate", "topology", "adapter", "capability", "provider", "nodes"): + changed = copy.deepcopy(runtime) + if kind == "candidate": changed["binding"]["candidateCommit"] = "f" * 40 + elif kind == "topology": changed["topologies"] = ["H1"] + elif kind == "adapter": changed["adapter"]["sha256"] = "f" * 64 + elif kind == "capability": changed["capabilities"] = [] + elif kind == "provider": changed["provider"] = "source-validation-lab" + else: changed["sshNodes"] = ["unrelated-node"] + with self.subTest(kind=kind), self.assertRaises(ValueError): + extended.runtime_binding(changed, case_id, candidate, "b" * 40) + + def test_physical_topology_rejects_directory_aliases_missing_sets_and_wrong_candidate(self): + extended.assert_topology(self.topology("H4"), "H4", "d" * 64) + for kind in ("alias", "slot", "count", "coding", "candidate"): + observed = self.topology("H4") + if kind == "alias": observed["disks"][1]["mountDevice"] = "0" + elif kind == "slot": observed["disks"][1]["setSlot"] = 0 + elif kind == "count": observed["disks"].pop() + elif kind == "coding": observed["disks"][0]["dataShards"] = 2 + else: observed["processes"][0]["binarySha256"] = "f" * 64 + with self.subTest(kind=kind), self.assertRaises(ValueError): extended.assert_topology(observed, "H4", "d" * 64) + + def test_overlap_requires_actual_original_task_and_common_activity_intersection(self): + value = {"clockDomain": "server-1", "intervals": [ + {"kind": "heal", "taskId": "original", "startNs": 100, "endNs": 200, "evidenceId": "h"}, + {"kind": "pool", "taskId": "move", "startNs": 150, "endNs": 300, "evidenceId": "p"}]} + extended.assert_overlap(value, task_id="original", kind="pool") + value["intervals"][1]["startNs"] = 201 + with self.assertRaises(extended.NotObserved): extended.assert_overlap(value, task_id="original", kind="pool") + value["clockDomain"] = "" + with self.assertRaises(ValueError): extended.assert_overlap(value, task_id="original", kind="pool") + + def terminal(self): + expected = [{"key": "key", "versionId": "v1", "deleteMarker": False}] + status = {"summary": "finished", "outcome": {"coverage": "complete", "objectsTruncated": False, + "execution": {"state": "completed"}, "counters": {"processed": 1, "healed": 1, "unchanged": 0, "skipped": 0, + "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False}, "objects": [ + {"identity": {"bucket": "bucket", "object": "key", "versionId": "v1", "bucketIncarnationId": "epoch", "poolIndex": 0, "setIndex": 0}, + "disposition": {"state": "repaired"}}]}} + return status, expected + + def test_terminal_denominator_cannot_be_defined_by_returned_receipts_or_legacy_finish(self): + status, expected = self.terminal(); extended.assert_terminal(status, "bucket", expected) + for kind in ("missing", "wrong-version", "unknown", "metadata-only", "counter", "incarnation", "boolean-set"): + changed = copy.deepcopy(status); row = changed["outcome"]["objects"][0] + if kind == "missing": changed["outcome"]["objects"] = [] + elif kind == "wrong-version": row["identity"]["versionId"] = "foreign" + elif kind == "unknown": row["disposition"]["state"] = "unknown" + elif kind == "metadata-only": row["disposition"]["state"] = "metadata_healthy" + elif kind == "counter": changed["outcome"]["counters"]["healed"] = 0 + elif kind == "incarnation": row["identity"]["bucketIncarnationId"] = "" + else: row["identity"]["setIndex"] = True + with self.subTest(kind=kind), self.assertRaises(ValueError): extended.assert_terminal(changed, "bucket", expected) + + def test_enabled_io_branch_runs_all_oracles_then_requires_review(self): + from test_validation_heal import HealShardOracleTest + original = HealShardOracleTest().snapshot() + bucket = "rv-heal-0123456789abcdef" + binding = self.binding(); binding["bucketPrefix"] = "rv-heal-0123456789" + def remap(bucket_name): + scene = copy.deepcopy(original) + for row in scene["node1"]["files"]: + row["path"] = row["path"].replace(bucket, bucket_name) + if "metadata" in row: row["metadata"]["bucket"] = bucket_name + return scene + class FakeAdapter: + def __init__(self): self.calls = []; self.injected = False; self.repaired = False; self.scene = None + def record(self, label, value): return value + def call(self, operation, **params): + self.calls.append(operation) + name = params.get("bucket") + if operation == "preflight": return ExtendedScenariosTest().topology() + if operation == "snapshot": + if self.scene is None: self.scene = remap(name) + scene = copy.deepcopy(self.scene) + if self.injected and not self.repaired: + scene["node1"]["files"] = [r for r in scene["node1"]["files"] if not (r["disk"] == self.target["disk"] and r["path"] == self.target["relativePath"])] + return {"snapshot": scene} + if operation == "apply-fault": + self.target = params["target"]; self.injected = True + return {"kind": "missing", "relativePath": self.target["relativePath"], "before": {"sha256": self.target["sha256"], "size": self.target["size"]}, "fault": None} + if operation == "checkpoint": return {"records": [{"taskId": "task", "bucketIncarnationId": "epoch", "sha256": "c" * 64, "durableProgress": 1}]} + if operation == "io-eio": return {"probe": {"uid": 1, "mountNamespace": "ns", "path": "/owned", "errno": 5}, "serviceUid": 1, "serviceMountNamespace": "ns", "deviceId": "device"} + if operation == "io-window": return {"clockDomain": "server", "intervals": [{"kind": "heal", "taskId": "task", "startNs": 1, "endNs": 4, "evidenceId": "h"}, {"kind": "io", "taskId": "fault", "startNs": 2, "endNs": 3, "evidenceId": "f"}]} + if operation == "io-recover": self.repaired = True + if operation == "verify-parts": return {"toolSha256": "f" * 64, "candidateCommit": "a" * 40, "parts": [{"key": "objects/0008.bin", "versionId": "null", "sha256": "part", "checksumVerified": True}]} + if operation == "cleanup": return {"remainingOwnedPaths": [], "activeOwnedServices": [], "remainingFaults": [], "pendingMutations": []} + return {} + class FakeProbe: + def __init__(self, binding, endpoint=None): self.binding = binding + def inventory(self, bucket, **params): return {"bucket": bucket, "entries": [{"key": "objects/0008.bin", "versionId": "null", "deleteMarker": False, "isLatest": True, "payload": {"size": 100, "sha256": "payload"}}]} + def admin(self, path, query, body): + if not query: return {"httpStatus": 200, "response": {"clientToken": "task"}} + status, expected = ExtendedScenariosTest().terminal() + row = status["outcome"]["objects"][0]["identity"] + from urllib.parse import unquote + row.update(bucket=unquote(path.rsplit("/", 1)[-1]), object="objects/0008.bin", versionId="null") + return {"httpStatus": 200, "response": status} + with tempfile.TemporaryDirectory() as tmp: + adapter = FakeAdapter() + runtime = {"binding": binding, "faultDeviceId": "device", "inspectorSha256": "f" * 64} + runner = extended.Runner(runtime, {}, "b" * 40, Path(tmp), adapter=adapter, probe_factory=FakeProbe) + def fixture(probe, directory, bucket, spec, branch): + baseline = probe.inventory(bucket) + return object(), copy.deepcopy(baseline["entries"]), baseline + with patch.object(runner, "fixture", side_effect=fixture): + row = runner.branch("heal.slow-io-and-eio", "H1", "eio") + self.assertEqual(row["cleanup"], "PASS", row) + self.assertTrue(row.get("checksCompleted"), row) + self.assertEqual(row["verdict"], "BLOCKED") + self.assertLess(adapter.calls.index("verify-parts"), adapter.calls.index("cleanup")) + for operation in ("apply-fault", "checkpoint", "io-eio", "io-window", "io-recover", "verify-parts", "recover", "cleanup"): + self.assertIn(operation, adapter.calls) + + def test_power_restart_and_io_error_cannot_be_substituted_with_other_faults(self): + with self.assertRaises(ValueError): extended.assert_action("dedicated-host-power-cycle", {"eventKind": "SIGKILL"}, task_id="task", runtime={"hostResourceId": "dedicated"}) + with self.assertRaises(ValueError): extended.assert_action("crash-service", {"pidBefore": 1, "pidAfter": 2, "rootBefore": "/a", "rootAfter": "/b", "signal": "SIGKILL"}, task_id="task", runtime={}) + with self.assertRaises(ValueError): extended.assert_action("io-eio", {"probe": {"uid": 1, "mountNamespace": "ns", "path": "/owned", "errno": 13}, + "serviceUid": 1, "serviceMountNamespace": "ns", "deviceId": "device"}, task_id="task", runtime={"faultDeviceId": "device"}) + + def test_partial_prepare_is_always_recovered_and_dirty_cleanup_is_retained(self): + test = self + class FakeAdapter: + def __init__(self, dirty): self.calls = []; self.dirty = dirty + def record(self, label, value): return value + def call(self, operation, **kwargs): + self.calls.append(operation) + if operation == "preflight": return test.topology() + if operation == "prepare": raise RuntimeError("lost prepare ACK") + if operation == "cleanup": return {"remainingOwnedPaths": ["owned"] if self.dirty else [], "activeOwnedServices": [], "remainingFaults": [], "pendingMutations": []} + return {} + for dirty in (False, True): + with self.subTest(dirty=dirty), tempfile.TemporaryDirectory() as tmp: + adapter = FakeAdapter(dirty) + runtime = {"binding": self.binding()} + runner = extended.Runner(runtime, {}, "b" * 40, Path(tmp), adapter=adapter, + probe_factory=lambda binding: types.SimpleNamespace(binding=binding)) + row = runner.branch("heal.slow-io-and-eio", "H1", "eio") + self.assertEqual(row["verdict"], "FAIL") + self.assertEqual(row["cleanup"], "FAIL" if dirty else "PASS") + self.assertEqual(adapter.calls, ["preflight", "prepare", "recover", "cleanup"]) + self.assertIn("lost prepare ACK", row["reason"]) + + def test_native_branches_do_not_invoke_resource_adapter(self): + runner = extended.Runner({"binding": self.binding()}, {}, "b" * 40, Path("unused"), adapter=object()) + row = runner.branch("heal.race-ilm-expiry-and-transition", "H2", "native-ilm-fence") + self.assertEqual(row["verdict"], "BLOCKED") + self.assertEqual(row["cleanup"], "NOT_RUN") + + def test_synthetic_clock_checks_full_soak_window_without_shortening_production_duration(self): + class Clock: + def __init__(self): self.now = 0 + def clock(self): return self.now + def sleep(self, seconds): self.now += seconds + class Adapter: + def call(self, action, **params): + if action == "soak-fault": return {"faultId": "fault", "kind": params["kind"], "injectedAtNs": 1} + if action == "soak-recover": return {"faultId": "fault", "remainingFaults": []} + return {"rssBytes": 1, "openFds": 1, "queuedTasks": 1, "unresolvedRecords": 0, "temporaryBytes": 1, "panicCount": 0, "busyLoopCount": 0} + def record(self, label, value): return value + class Recorder: + def __init__(self): self.sequence = 0 + def call(self, operation, params, body=None): + self.sequence += 1 + return {"VersionId": params.get("VersionId", str(self.sequence))} + clock = Clock(); expected = [] + runtime = {"soakLimits": {"rssBytes": 10, "openFds": 10, "queuedTasks": 10, "unresolvedRecords": 0, "temporaryBytes": 10}} + runner = extended.Runner(runtime, {}, "b" * 40, Path("unused"), adapter=Adapter(), clock=clock.clock, sleep=clock.sleep) + result = runner.soak({"branch": "24h"}, Recorder(), "bucket", expected, "H2") + self.assertEqual(extended.SOAK_SECONDS, {"24h": 86400, "72h": 259200}) + self.assertGreaterEqual(result["actualElapsedSeconds"], 86400) + self.assertLessEqual(sum(e["key"] == "soak.bin" for e in expected), 65) + + +if __name__ == "__main__": + unittest.main() From 0334abfefeba0d7ea85855da1fe9aa8a287d2940 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 09:52:40 +0800 Subject: [PATCH 159/343] docs(validation): reconcile audited pool tail and live backlog --- docs/validation-heal-contract.md | 2 +- docs/validation-migration-map.md | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index 599a7cfa..4e52be3a 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -45 个用例已接入自动分发：A01、A03–A08、B01–B07、B09、C01–C06、C08、D01–D07、D09、D10、E01–E06、F01–F03、F05、F06、J01、J03、J04。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。其余 39 项保留源 operator-assisted 方式。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 +目前有 67 个自动执行器和 13 个默认关闭的环境场景入口；C11/F07/I05/J02 尚未接入调度。注册执行器不代表完整 canonical PASS，A09/G01–G06 的必需 native 分支仍被阻断。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。13 个环境场景保留源操作分支、未满足条件和显式启用入口；其完整 live 验证仍需资源适配器与源断言审核。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 93c240ab..e82371d0 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -60,7 +60,7 @@ The driver reuses the existing auto-testing Pool suite's two-node seed and expansion operations, supplies marker-guarded reset/topology/restart/physical census hooks, and requires source case evidence plus physical cleanup before canonical PASS. `pool.status` passed live in RustFS Actions run `38047001285`; -`pool.decommission` and `pool.decommission-cancel-clear-resume` also passed in runs `38051227313` and `38052304396`. `pool.decommission-live-io`, `pool.decommission-restart-recovery`, `pool.decommission-versioned-integrity`, `pool.expand-and-rebalance`, and `pool.expansion-restart-persistence` passed in runs `38053444515`, `38053814812`, `38054128863`, `38054208114`, and `38054287896`. `pool.decommission-multipart-integrity` failed in run `38053715867`: no fresh owned-bucket MPU drain failure evidence was found; cleanup passed. `pool.operation-guards` and `pool.rebalance-live-io` passed in runs `38054540360` and `38054696080`. Two Pool cases still await live evidence; the multipart failure remains visible. +`pool.decommission` and `pool.decommission-cancel-clear-resume` also passed in runs `38051227313` and `38052304396`. `pool.decommission-live-io`, `pool.decommission-restart-recovery`, `pool.decommission-versioned-integrity`, `pool.expand-and-rebalance`, and `pool.expansion-restart-persistence` passed in runs `38053444515`, `38053814812`, `38054128863`, `38054208114`, and `38054287896`. `pool.decommission-multipart-integrity` failed in run `38053715867`: no fresh owned-bucket MPU drain failure evidence was found; cleanup passed. `pool.operation-guards` and `pool.rebalance-live-io` passed in runs `38054540360` and `38054696080`. `pool.rebalance-stop-resume` and `pool.retired-config-removal` subsequently passed in independently verified runs `38110959836` and `38111198632`. The multipart failure remains visible. The 18 bucket-replication and 20 site-replication IDs now dispatch to the source black-box checks. The driver uses the existing auto-testing four-node @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 86 unique complete canonical live PASS cases: 74 on the current candidate source and 12 on the older immutable source. Native, read-only, recovery and duplicate runs are excluded; the other 72 cases have no complete canonical live PASS claim. +case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 88 unique complete canonical live PASS cases: 76 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632` are included. Native, read-only, recovery and duplicate runs are excluded; the other 70 cases have no complete canonical live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1883,3 +1883,6 @@ Current reconciliation: helper `6f91210f224dc0a9e460b8240c4a4cf73d7e64c1` is pus The environment-gated batch now registers F08/F09, G07/G08, H01-H05/H07, I01/I06 and J05. The default gate writes a per-case BLOCKED reason and environment evidence before any SSH/install/fault operation. Explicit opt-in still requires exact candidate/client/runtime provenance, every required topology and a digest-bound resource adapter. The enabled control flows reuse auto-testing Probe/Recorder and physical fault oracles, retain every source branch and actual 24h/72h soak duration, stop on dirty cleanup, and require source-assertion review instead of manufacturing PASS from adapter responses. No live run or current-fleet adapter deployment is claimed. Second-batch blockers are recorded accurately: C11/F07/J02 already have 3/10/2 candidate native prerequisites but still need full live orchestration/evidence; I05 lacks an exact reviewed lock-owner/abort fence. See `docs/validation-heal-environment-gates.md`. This batch passes all 36 test modules selected by Validation Catalog CI: 537 local tests with eight macOS platform skips, plus exact source-catalog hash comparison, the 84/301/532 matrix, shell syntax and scanner mock gates. The 19 new environment tests cover disabled dispatch across all 13 IDs without SSH, explicit runtime provenance, topology and counter negatives, complete enabled IO control flow, actual overlap, full-duration soak control, and recovery after a lost setup ACK. These are local contract checks; no live environment-gated result is claimed. + + +The 2026-10-12 reconciliation rechecks both retained Pool tail results, every referenced original file digest, and the current canonical report gate. The derived census now contains 88 unique PASS cases; it preserves the preceding 86-case census and excludes cleanup/native/read-only runs. Of the 141 automatic executors, 53 still lack complete canonical PASS: 31 have retained live failures, seven (A09/G01-G06) require reviewed native barriers, and 15 have no complete independently audited live evidence. The 13 environment gates remain disabled and C11/F07/I05/J02 remain outside dispatch. Fresh F05/J06 verification must use a new immutable harness pin containing the durable bucket-event validator and the 120-minute performance lease, followed by the 15 evidence-gap cases on the same serialized current auto-testing fleet. Old failures and recoveries are retained separately. From ccef050df3b28416c51356951e78cb46dd74d10b Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 10:02:06 +0800 Subject: [PATCH 160/343] fix(validation): verify streamed MPU payload after full restart --- docs/validation-migration-map.md | 2 ++ scripts/validation_heal/full_restart.py | 22 ++++++++----- test_validation_heal_full_restart.py | 43 ++++++++++++++++++++++++- 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index e82371d0..79783164 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1886,3 +1886,5 @@ This batch passes all 36 test modules selected by Validation Catalog CI: 537 loc The 2026-10-12 reconciliation rechecks both retained Pool tail results, every referenced original file digest, and the current canonical report gate. The derived census now contains 88 unique PASS cases; it preserves the preceding 86-case census and excludes cleanup/native/read-only runs. Of the 141 automatic executors, 53 still lack complete canonical PASS: 31 have retained live failures, seven (A09/G01-G06) require reviewed native barriers, and 15 have no complete independently audited live evidence. The 13 environment gates remain disabled and C11/F07/I05/J02 remain outside dispatch. Fresh F05/J06 verification must use a new immutable harness pin containing the durable bucket-event validator and the 120-minute performance lease, followed by the 15 evidence-gap cases on the same serialized current auto-testing fleet. Old failures and recoveries are retained separately. + +Fresh read-only inspection `38193642346` confirms the same four current Heal nodes and 16 independent eligible endpoints. F05 run `38193742108` pins helper `0334abfefeba0d7ea85855da1fe9aa8a287d2940` and tools attempt `38103474555/1`; its result is pending. Inspection of the final pending-MPU consumer found a separate harness defect: `Probe.s3` streams and hashes GET bodies into the top-level `payload`, but F05 read `response.payload`. The consumer now verifies the actual streamed receipt on every original peer. A regression uses real Probe streaming receipts, proves the original nested-payload check rejects healthy bytes, and rejects corrupted bytes from the fourth peer. All 22 F05/J06 focused tests pass. The subsequent local queue is paused at the current case boundary so the corrected F05 retry can use a new immutable pin without changing an active plan or relabeling its result. diff --git a/scripts/validation_heal/full_restart.py b/scripts/validation_heal/full_restart.py index a5d1ecdf..83980c69 100644 --- a/scripts/validation_heal/full_restart.py +++ b/scripts/validation_heal/full_restart.py @@ -178,6 +178,19 @@ def verify_public(probe, fleet, fixtures, pending, label): require(actual_pending["uploads"] == pending["uploads"], "full restart lost an exact pending UploadId/part/ETag") +def complete_pending_uploads(recorder, fleet, binding, pending): + bucket = pending["bucket"] + for upload in pending["uploads"]: + recorder.call("complete_multipart_upload", {"Bucket": bucket, "Key": upload["key"], "UploadId": upload["uploadId"], + "MultipartUpload": {"Parts": [{"PartNumber": item["PartNumber"], "ETag": item["ETag"]} for item in upload["parts"]]}}) + expected_body = payload(f"{bucket}/{upload['key']}", 5 * 1024**2 + 3) + for endpoint in binding["endpoints"]: + observed = fleet.record("pending-mpu-content-after-repeat", Probe(binding, endpoint).s3("get_object", {"Bucket": bucket, "Key": upload["key"]})) + require(observed.get("acknowledged") is True and observed.get("resultUnknown") is False + and observed.get("payload") == {"size": len(expected_body), "sha256": hashlib.sha256(expected_body).hexdigest()}, + "restarted pending MPU parts lost their actual acknowledged bytes") + + def run_topology(args, nodes): directory = args.evidence / "H2" directory.mkdir() @@ -293,14 +306,7 @@ def run_topology(args, nodes): fleet.stop() require(physical_content(rebuilt) == physical_content(fleet.snapshot("physical-after-repeat-get", quiescent=True)), "repeat full restart or verification changed a user file") fleet.start() - for upload in pending["uploads"]: - recorder.call("complete_multipart_upload", {"Bucket": bucket, "Key": upload["key"], "UploadId": upload["uploadId"], - "MultipartUpload": {"Parts": [{"PartNumber": item["PartNumber"], "ETag": item["ETag"]} for item in upload["parts"]]}}) - expected_body = payload(f"{bucket}/{upload['key']}", 5 * 1024**2 + 3) - for endpoint in binding["endpoints"]: - observed = fleet.record("pending-mpu-content-after-repeat", Probe(binding, endpoint).s3("get_object", {"Bucket": bucket, "Key": upload["key"]})) - require(observed.get("acknowledged") is True and observed.get("resultUnknown") is False - and observed.get("response", {}).get("payload") == {"size": len(expected_body), "sha256": hashlib.sha256(expected_body).hexdigest()}, "restarted pending MPU parts lost their actual acknowledged bytes") + complete_pending_uploads(recorder, fleet, binding, pending) result["branches"] = { BRANCHES[0]: {"verdict": "PASS", "originalTaskId": token, "runningWindow": first["outcome"]["counters"], "beforeInterruptionPhysical": before_partial, "afterAllActualExitsPhysical": after_partial, diff --git a/test_validation_heal_full_restart.py b/test_validation_heal_full_restart.py index 0c2f4d23..195bfd34 100644 --- a/test_validation_heal_full_restart.py +++ b/test_validation_heal_full_restart.py @@ -1,10 +1,12 @@ import copy +from datetime import datetime, timedelta, timezone +import io import json from pathlib import Path import tempfile import types import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch from scripts.validation_heal import full_restart as restart from scripts.validation_heal.matrix import select, AUTOMATIC_CASES @@ -14,6 +16,45 @@ class FullRestartTests(unittest.TestCase): token = "12345678-1234-4234-8234-123456789abc" + def test_restarted_mpu_checks_real_streamed_get_receipts_on_every_peer(self): + bucket, key = "rv-heal-0123456789abcdef", "pending/parts.bin" + binding = {"schemaVersion": 1, "runId": "gh-12345678901-1-h2", "jumpHostOnly": True, + "exclusiveHealService": True, "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=5)).isoformat(), + "bucketPrefix": bucket, "endpoints": {f"node{i}": f"http://rustfs-node{i}:9000" for i in range(1, 5)}, + "defaultEndpoint": "node1", "candidateCommit": "a" * 40, "harnessCommit": "b" * 40, + "candidatePackageSha256": "c" * 64, "candidateBinarySha256": "d" * 64, "rcBinarySha256": "e" * 64, + "maxInventoryObjects": 2048, "maxInventoryBytes": 2 * 1024**3} + pending = {"bucket": bucket, "uploads": [{"key": key, "uploadId": "original-upload", + "parts": [{"PartNumber": 1, "ETag": "original-part"}]}]} + expected = restart.payload(f"{bucket}/{key}", 5 * 1024**2 + 3) + client = Mock() + recorder = Mock() + fleet = types.SimpleNamespace(record=lambda label, value: value) + calls = [] + bad_endpoint = None + + def streamed(probe): + def get_object(**params): + self.assertEqual(params, {"Bucket": bucket, "Key": key}) + calls.append(probe.endpoint_name) + body = bytes([expected[0] ^ 1]) + expected[1:] if probe.endpoint_name == bad_endpoint else expected + return {"ContentLength": len(body), "Body": io.BytesIO(body), + "ResponseMetadata": {"HTTPStatusCode": 200}} + client.get_object.side_effect = get_object + return client + + with patch.object(restart.Probe, "s3_client", streamed): + restart.complete_pending_uploads(recorder, fleet, binding, pending) + self.assertEqual(calls, list(binding["endpoints"])) + recorder.call.assert_called_once_with("complete_multipart_upload", { + "Bucket": bucket, "Key": key, "UploadId": "original-upload", + "MultipartUpload": {"Parts": [{"PartNumber": 1, "ETag": "original-part"}]}}) + bad_endpoint = "node4" + calls.clear() + with self.assertRaisesRegex(ValueError, "lost their actual acknowledged bytes"): + restart.complete_pending_uploads(recorder, fleet, binding, pending) + self.assertEqual(calls, list(binding["endpoints"])) + def test_combined_fixture_has_explicit_bounded_physical_census_budget(self): # Four roots retain xl.meta, an external part and its integrity # sidecar, plus a possible metadata backup. MPU adds three parts. From dbe74ec9eb3288e32923d8b8777c581c904ba78f Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 10:19:44 +0800 Subject: [PATCH 161/343] fix(validation): retain exact negative Heal task outcomes --- docs/validation-migration-map.md | 6 +++- scripts/validation_heal/options.py | 15 ++++----- scripts/validation_heal/orphan.py | 13 +++----- scripts/validation_heal/shards.py | 51 ++++++++++++++++++++++++++++-- test_validation_heal.py | 26 +++++++++++++++ test_validation_heal_options.py | 28 ++++++++++++---- test_validation_heal_orphan.py | 30 +++++++++++++++++- 7 files changed, 142 insertions(+), 27 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 79783164..c63228ff 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1887,4 +1887,8 @@ This batch passes all 36 test modules selected by Validation Catalog CI: 537 loc The 2026-10-12 reconciliation rechecks both retained Pool tail results, every referenced original file digest, and the current canonical report gate. The derived census now contains 88 unique PASS cases; it preserves the preceding 86-case census and excludes cleanup/native/read-only runs. Of the 141 automatic executors, 53 still lack complete canonical PASS: 31 have retained live failures, seven (A09/G01-G06) require reviewed native barriers, and 15 have no complete independently audited live evidence. The 13 environment gates remain disabled and C11/F07/I05/J02 remain outside dispatch. Fresh F05/J06 verification must use a new immutable harness pin containing the durable bucket-event validator and the 120-minute performance lease, followed by the 15 evidence-gap cases on the same serialized current auto-testing fleet. Old failures and recoveries are retained separately. -Fresh read-only inspection `38193642346` confirms the same four current Heal nodes and 16 independent eligible endpoints. F05 run `38193742108` pins helper `0334abfefeba0d7ea85855da1fe9aa8a287d2940` and tools attempt `38103474555/1`; its result is pending. Inspection of the final pending-MPU consumer found a separate harness defect: `Probe.s3` streams and hashes GET bodies into the top-level `payload`, but F05 read `response.payload`. The consumer now verifies the actual streamed receipt on every original peer. A regression uses real Probe streaming receipts, proves the original nested-payload check rejects healthy bytes, and rejects corrupted bytes from the fourth peer. All 22 F05/J06 focused tests pass. The subsequent local queue is paused at the current case boundary so the corrected F05 retry can use a new immutable pin without changing an active plan or relabeling its result. +Fresh read-only inspections `38193642346` and `38194452552` confirm the same four current Heal nodes and 16 independent eligible endpoints. F05 run `38193742108` pins helper `0334abfefeba0d7ea85855da1fe9aa8a287d2940` and tools attempt `38103474555/1`; it retained FAIL/cleanup PASS. Inspection of the final pending-MPU consumer found a separate harness defect: `Probe.s3` streams and hashes GET bodies into the top-level `payload`, but F05 read `response.payload`. The consumer now verifies the actual streamed receipt on every original peer. A regression uses real Probe streaming receipts, proves the original nested-payload check rejects healthy bytes, and rejects corrupted bytes from the fourth peer. All 22 F05/J06 focused tests pass. The original local queue was stopped at its clean paused case boundary; its immutable plan and original FAIL remain intact. + +The replacement 17-case queue pins helper `ccef050df3b28416c51356951e78cb46dd74d10b`. F05 retry `38194557163` also retained FAIL/cleanup PASS: the original root token returned `stopped`, `aborted/untraversable`, partial coverage and zero processed objects. Its detail names three bucket failures and `Storage error: erasure write quorum`. The fleet starts peers concurrently and waits for readiness, but the recovered task runs during startup. Candidate `heal_bucket` wraps its typed storage error into `TaskExecutionFailed`; the message fallback does not classify this write-quorum string as recoverable, and cluster traversal records a permanent untraversable failure. This is a candidate recovery-path diagnosis, not a failure of the later MPU consumer and not a reason to issue a replacement task or accept an aborted result. J06 retry `38195131282` then started on the same immutable queue; its result remains pending. + +Original A10 `38120735805` and C10 `38120042167` negative receipts exposed another harness mismatch. The candidate returns `stopped/completed_with_errors` and one exact permanent not-found receipt for a nonrecursive absent prefix or protected metadata-less orphan. The old convergence helper required `finished`; A10 additionally demanded an empty receipt list and C10 demanded `finished` even for a failed traversal. Convergence now compares the original path/token, settings and complete matching terminal outcomes, including truthful failed terminals. The negative oracle requires exactly one original failed object identity, unchanged selectors, an unversioned/unbound absent identity, complete coverage, exact 1/0/0/0/1/0 counters, one visible failed attempt, zero overflow and the exact not-found detail. Normal repair branches still require successful execution and their full ACK manifests. The 199 focused tests pass (one macOS skip); read-only replay of both original C10 refusals passes the corrected oracle without changing either retained FAIL or adding canonical PASS. Fresh full A10/C10 runs are still required. diff --git a/scripts/validation_heal/options.py b/scripts/validation_heal/options.py index 0192358d..795df124 100644 --- a/scripts/validation_heal/options.py +++ b/scripts/validation_heal/options.py @@ -23,7 +23,7 @@ from .inspector import verify as verify_inspector from .matrix import select from .heal_fault import bounded_fault_id - from .shards import files, map_target, validate_fault, validate_repaired_physical, peer_status_equal + from .shards import files, map_target, validate_fault, validate_repaired_physical, peer_status_equal, validate_absent_request from .versions import public_version_id from .scopes import verify_parts from .stop import idle_fleet @@ -35,7 +35,7 @@ from inspector import verify as verify_inspector from matrix import select from heal_fault import bounded_fault_id - from shards import files, map_target, validate_fault, validate_repaired_physical, peer_status_equal + from shards import files, map_target, validate_fault, validate_repaired_physical, peer_status_equal, validate_absent_request from versions import public_version_id from scopes import verify_parts from stop import idle_fleet @@ -236,16 +236,15 @@ def run_request(probe, fleet, path, options, active, *, rejected=False): def validate_outcome(status, mapped, selected, repaired, options): outcome = status.get('outcome') or {} + if not selected: + buckets = {identity[0] for identity in mapped} + require(not repaired and len(buckets) == 1, 'A10 absent prefix changed its physical scope or claimed a repair') + proof = validate_absent_request(status, next(iter(buckets)), 'scope/', options) + return {**proof, 'exactPrefixNotFound': True, 'selectedACKs': 0, 'repairedACKs': 0} require(status.get('summary') == 'finished' and status.get('settings') == options and status.get('truncated', False) is False and outcome.get('coverage') == 'complete' and outcome.get('objectsTruncated') is False, 'A10 lacks complete original task coverage/settings') counters = outcome.get('counters') or {} - if not selected: - require(not repaired and outcome.get('execution', {}).get('state') == 'completed_with_errors' - and counters.get('healed') == 0 and counters.get('failed') == 1 and counters.get('unknown') == 0 - and 'Object not found: ' in (status.get('detail') or '') and not outcome.get('objects'), - 'A10 exact absent prefix was repaired, unknown or incorrectly claimed successful') - return {'exactPrefixNotFound': True, 'selectedACKs': 0, 'repairedACKs': 0} require(repaired <= selected <= set(mapped) and outcome.get('execution', {}).get('state') == 'completed', 'A10 responsibility escaped the physical scope or failed') objects, seen, incarnations = outcome.get('objects'), set(), {} diff --git a/scripts/validation_heal/orphan.py b/scripts/validation_heal/orphan.py index db096d4b..4c5f19ae 100644 --- a/scripts/validation_heal/orphan.py +++ b/scripts/validation_heal/orphan.py @@ -24,7 +24,7 @@ from .matrix import select from .native_regressions import CANDIDATE from .shared_fixture import verify as verify_shared - from .shards import files, map_target, run_task + from .shards import files, map_target, run_task, validate_absent_request from .scopes import verify_parts from .stop import idle_fleet except ImportError: @@ -36,7 +36,7 @@ from matrix import select from native_regressions import CANDIDATE from shared_fixture import verify as verify_shared - from shards import files, map_target, run_task + from shards import files, map_target, run_task, validate_absent_request from scopes import verify_parts from stop import idle_fleet @@ -116,6 +116,8 @@ def physical_safety(before, after, authority, bucket, expected, branch, doomed, def terminal_safety(status, branch, bucket, options): + if branch == 'remove-false': + return validate_absent_request(status, bucket, KEY, options) require(status.get('summary') == 'finished' and status.get('truncated', False) is False and status.get('settings') == options, 'C10 terminal receipt is absent/truncated or changed its request') outcome = status.get('outcome') or {} @@ -123,12 +125,7 @@ def terminal_safety(status, branch, bucket, options): require(outcome.get('coverage') == 'complete' and outcome.get('objectsTruncated', False) is False and all(type(counters.get(name)) is int and counters[name] >= 0 for name in ('processed', 'healed', 'failed', 'unknown')) and counters['unknown'] == 0, 'C10 responsibility is missing, partial or unknown') - if branch == 'remove-false': - require(outcome.get('execution', {}).get('state') == 'completed_with_errors' - and counters['healed'] == 0 and counters['failed'] == 1 - and 'Object not found: ' + bucket + '/' + KEY in (status.get('detail') or ''), - 'C10 metadata-less protected orphan lacks its exact not-found refusal') - elif branch == 'remove-true-before-grace': + if branch == 'remove-true-before-grace': deferred = [value for value in outcome.get('objects', []) if value.get('identity', {}).get('bucket') == bucket and value.get('identity', {}).get('object') == KEY and value.get('disposition', {}).get('state') == 'deferred' diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index 1ddb23dd..28ebd1b5 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -313,11 +313,56 @@ def validate_dry_run_status(status, target, expected): def peer_status_equal(coordinator, peer, *, task, peer_task): # The candidate renders startTime at response serialization; it is not a # persistent task epoch. Bind the actual signed queries to admission instead. - return (coordinator.get("summary") == "finished" and peer.get("summary") == "finished" + # Convergence also applies to a truthful negative result. The case oracle, + # rather than this comparison, decides whether that result is expected. + outcome = coordinator.get("outcome") or {} + summary = coordinator.get("summary") + execution = outcome.get("execution", {}).get("state") + terminal = (summary == "finished" and execution == "completed" + or summary == "stopped" and execution == "completed_with_errors" + and type(outcome.get("counters", {}).get("failed")) is int + and outcome["counters"]["failed"] > 0) + return (terminal and peer.get("summary") == summary and isinstance(task, tuple) and len(task) == 2 and all(task) and task == peer_task and canonical(coordinator.get("outcome")) == canonical(peer.get("outcome")) - and coordinator.get("outcome", {}).get("coverage") == "complete" - and peer.get("truncated", False) is False) + and canonical(coordinator.get("settings")) == canonical(peer.get("settings")) + and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False + and coordinator.get("truncated", False) is False and peer.get("truncated", False) is False) + + +def validate_absent_request(status, bucket, key, options): + """Require the exact retained failure of an absent, nonrecursive object.""" + outcome = status.get("outcome") or {} + require(options.get("recursive") is False and status.get("summary") == "stopped" + and status.get("settings") == options and status.get("truncated", False) is False + and outcome.get("execution", {}).get("state") == "completed_with_errors" + and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False, + "absent request lacks its complete original failed task/settings") + expected = {"processed": 1, "healed": 0, "unchanged": 0, "skipped": 0, "failed": 1, + "unknown": 0, "attemptFailures": 1, "overflowed": False} + counters = outcome.get("counters") or {} + require(all(type(counters.get(name)) is type(value) and counters[name] == value + for name, value in expected.items()), "absent request lost or falsely repaired its failure responsibility") + objects = outcome.get("objects") + require(isinstance(objects, list) and len(objects) == 1, "absent request omitted or duplicated its failed receipt") + receipt = objects[0] + identity = receipt.get("identity") or {} + require(set(identity) == {"kind", "bucket", "object", "versionId", "bucketIncarnationId", "poolIndex", "setIndex"} + and identity.get("kind") == "object" and identity.get("bucket") == bucket + and identity.get("object") == key + and identity.get("versionId") in {None, "", "null", "00000000-0000-0000-0000-000000000000"} + and identity.get("bucketIncarnationId") is None + and all(type(identity.get(field)) is type(options.get(option)) + and identity.get(field) == options.get(option) + for field, option in (("poolIndex", "pool"), ("setIndex", "set"))) + and receipt.get("disposition") == {"state": "failed", "details": "permanent"}, + "absent request changed object/version/selector identity or claimed a different disposition") + detail = "Object not found: " + bucket + "/" + key + require(isinstance(status.get("detail"), str) and status["detail"].endswith(detail) + and isinstance(receipt.get("detail"), str) and receipt["detail"].endswith(detail), + "absent request did not retain its exact not-found error") + return {"failedRequest": {"bucket": bucket, "key": key}, "canonicalCounters": counters, + "completeResponsibility": True, "repairPassClaimed": False} def run_task(probe, fleet, target, mode, active, *, status_endpoints=(), option_overrides=None, admissions=None): diff --git a/test_validation_heal.py b/test_validation_heal.py index f4557020..c22fccc4 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -357,6 +357,32 @@ def test_peer_status_cannot_converge_with_wrong_task_unknown_or_incomplete_recei else: peer["truncated"] = True with self.subTest(invalid=invalid): self.assertFalse(peer_status_equal(coordinator, peer, task=task, peer_task=peer_task)) + def test_peer_convergence_retains_expected_failures_without_certifying_repair(self): + terminal = HealHealthyOracleTest().receipt("failed") + terminal["summary"] = "stopped" + terminal["outcome"]["execution"]["state"] = "completed_with_errors" + terminal["outcome"]["counters"].update(unchanged=0, failed=1, attemptFailures=1) + terminal["settings"] = {"recursive": False, "pool": 0, "set": 0} + task = ("/rustfs/admin/v3/heal/fixture/object.bin", "original-token") + self.assertTrue(peer_status_equal(terminal, copy.deepcopy(terminal), task=task, peer_task=task)) + with self.assertRaises(ValueError): validate_healthy(terminal, "fixture", "object.bin", "verified_healthy") + for mutation in ("aborted", "running", "finished", "zero-failures", "bool-failures", "settings", + "truncated", "partial", "token"): + peer, original = copy.deepcopy(terminal), copy.deepcopy(terminal) + peer_task = task + if mutation == "aborted": original["outcome"]["execution"] = {"state": "aborted", "reason": "untraversable"} + elif mutation == "running": original["outcome"]["execution"]["state"] = "running" + elif mutation == "finished": original["summary"] = "finished" + elif mutation == "zero-failures": original["outcome"]["counters"]["failed"] = 0 + elif mutation == "bool-failures": original["outcome"]["counters"]["failed"] = True + elif mutation == "settings": peer["settings"]["set"] = 1 + elif mutation == "truncated": original["truncated"] = True + elif mutation == "partial": original["outcome"]["coverage"] = "partial" + else: peer_task = (task[0], "replacement-token") + if mutation not in {"settings", "token"}: peer = copy.deepcopy(original) + with self.subTest(mutation=mutation): + self.assertFalse(peer_status_equal(original, peer, task=task, peer_task=peer_task)) + def test_target_role_comes_from_metadata_and_actual_format_slot(self): before = self.snapshot() self.assertEqual(self.target(before)["disk"], 1) diff --git a/test_validation_heal_options.py b/test_validation_heal_options.py index cac85566..2c288312 100644 --- a/test_validation_heal_options.py +++ b/test_validation_heal_options.py @@ -106,19 +106,35 @@ def test_nullable_unrestricted_scope_keeps_the_independent_physical_binding(self receipt.update(poolIndex=0, setIndex=1 - mapped[identity]['setIndex']) with self.assertRaises(ValueError): options.validate_outcome(status, mapped, selected, repaired, request) - def test_exact_missing_prefix_requires_truthful_not_found_and_no_object_receipts(self): + def test_exact_missing_prefix_retains_the_requested_failure_instead_of_ack_objects(self): baseline, request = options.paired_options('recursive-false') - status = {'summary': 'finished', 'settings': request, 'detail': 'Object not found: ' + self.bucket + '/scope/', + detail = 'Heal task execution failed: Object not found: ' + self.bucket + '/scope/' + status = {'summary': 'stopped', 'settings': request, 'detail': detail, 'outcome': {'coverage': 'complete', 'objectsTruncated': False, 'execution': {'state': 'completed_with_errors'}, - 'objects': [], 'counters': {'healed': 0, 'failed': 1, 'unknown': 0}}} - options.validate_outcome(status, self.mapped(), set(), set(), request) - for kind in ('success', 'repair', 'unknown', 'receipt', 'detail'): + 'objects': [{'identity': {'kind': 'object', 'bucket': self.bucket, 'object': 'scope/', + 'versionId': None, 'bucketIncarnationId': None, 'poolIndex': None, 'setIndex': None}, + 'disposition': {'state': 'failed', 'details': 'permanent'}, 'detail': detail}], + 'counters': {'processed': 1, 'healed': 0, 'unchanged': 0, 'skipped': 0, + 'failed': 1, 'unknown': 0, 'attemptFailures': 1, 'overflowed': False}}} + proof = options.validate_outcome(status, self.mapped(), set(), set(), request) + self.assertEqual(proof['selectedACKs'], 0) + self.assertFalse(proof['repairPassClaimed']) + for kind in ('success', 'legacy-success', 'repair', 'unknown', 'receipt', 'missing-receipt', 'detail', + 'wrong-key', 'incarnation', 'selector', 'attempts', 'false-absence', 'missing-identity'): changed = copy.deepcopy(status) if kind == 'success': changed['outcome']['execution']['state'] = 'completed' + elif kind == 'legacy-success': changed['summary'] = 'finished' elif kind == 'repair': changed['outcome']['counters']['healed'] = 1 elif kind == 'unknown': changed['outcome']['counters']['unknown'] = 1 elif kind == 'receipt': changed['outcome']['objects'] = [{'disposition': {'state': 'unknown'}}] - else: changed['detail'] = '' + elif kind == 'missing-receipt': changed['outcome']['objects'] = [] + elif kind == 'detail': changed['detail'] = '' + elif kind == 'wrong-key': changed['outcome']['objects'][0]['identity']['object'] = options.KEYS[0] + elif kind == 'incarnation': changed['outcome']['objects'][0]['identity']['bucketIncarnationId'] = 'invented' + elif kind == 'selector': changed['outcome']['objects'][0]['identity']['poolIndex'] = False + elif kind == 'attempts': changed['outcome']['counters']['attemptFailures'] = 0 + elif kind == 'false-absence': changed['outcome']['objects'][0]['disposition']['state'] = 'authoritatively_absent' + else: changed['outcome']['objects'][0]['identity'].pop('bucketIncarnationId') with self.assertRaises(ValueError): options.validate_outcome(changed, self.mapped(), set(), set(), request) def physical_scene(self): diff --git a/test_validation_heal_orphan.py b/test_validation_heal_orphan.py index 1c7602c4..b484c698 100644 --- a/test_validation_heal_orphan.py +++ b/test_validation_heal_orphan.py @@ -258,9 +258,15 @@ def status(self, branch): 'execution': {'state': 'completed'}, 'coverage': 'complete', 'objectsTruncated': False, 'counters': {'processed': 1, 'healed': 0, 'failed': 0, 'unknown': 0}, 'objects': []}} if branch == 'remove-false': + status['summary'] = 'stopped' status['detail'] = 'Object not found: ' + self.bucket + '/' + orphan.KEY - status['outcome']['counters']['failed'] = 1 + status['outcome']['counters'] = {'processed': 1, 'healed': 0, 'unchanged': 0, 'skipped': 0, + 'failed': 1, 'unknown': 0, 'attemptFailures': 1, 'overflowed': False} status['outcome']['execution']['state'] = 'completed_with_errors' + status['outcome']['objects'] = [{'identity': {'kind': 'object', 'bucket': self.bucket, 'object': orphan.KEY, + 'versionId': None, 'bucketIncarnationId': None, 'poolIndex': status['settings']['pool'], + 'setIndex': status['settings']['set']}, 'disposition': {'state': 'failed', 'details': 'permanent'}, + 'detail': status['detail']}] if branch == 'remove-true-before-grace': status['outcome']['objects'] = [{'identity': {'bucket': self.bucket, 'object': orphan.KEY}, 'disposition': {'state': 'deferred', 'details': {'reason': 'dangling_delete_grace'}}}] @@ -286,6 +292,28 @@ def test_source_case_has_all_five_branches_on_both_required_topologies(self): self.assertEqual(source['contract']['probes']['topology'], 'H1,H2') self.assertIn(orphan.CASE_ID, AUTOMATIC_CASES) + def test_protected_orphan_refusal_requires_original_failed_identity_and_truthful_terminal(self): + status = self.status('remove-false') + proof = orphan.terminal_safety(status, 'remove-false', self.bucket, status['settings']) + self.assertFalse(proof['repairPassClaimed']) + for mutation in ('finished', 'running', 'cancelled', 'partial', 'unknown', 'repaired', 'missing', 'wrong-version', + 'wrong-key', 'wrong-selector', 'hidden-attempt', 'duplicate'): + changed = copy.deepcopy(status) + outcome = changed['outcome'] + if mutation in ('finished', 'running'): changed['summary'] = mutation + elif mutation == 'cancelled': outcome['execution'] = {'state': 'aborted', 'reason': 'cancelled'} + elif mutation == 'partial': outcome['coverage'] = 'partial' + elif mutation == 'unknown': outcome['counters']['unknown'] = 1 + elif mutation == 'repaired': outcome['objects'][0]['disposition']['state'] = 'repaired' + elif mutation == 'missing': outcome['objects'] = [] + elif mutation == 'wrong-version': outcome['objects'][0]['identity']['versionId'] = str(uuid.UUID(int=9)) + elif mutation == 'wrong-key': outcome['objects'][0]['identity']['object'] = 'control.bin' + elif mutation == 'wrong-selector': outcome['objects'][0]['identity']['setIndex'] = 1 + elif mutation == 'hidden-attempt': outcome['counters']['attemptFailures'] = 0 + else: outcome['objects'].append(copy.deepcopy(outcome['objects'][0])) + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + orphan.terminal_safety(changed, 'remove-false', self.bucket, status['settings']) + def test_shared_paths_are_streamed_through_both_exact_ack_version_ids(self): mapping = {} for version in (str(uuid.UUID(int=1)), str(uuid.UUID(int=2))): From 5ceb893b51a7ab653e3791f91afaf123be55b5ed Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 11:05:20 +0800 Subject: [PATCH 162/343] fix(validation): match failed quorum tasks to canonical terminals --- docs/validation-migration-map.md | 4 ++++ scripts/validation_heal/authority.py | 2 +- scripts/validation_heal/versions.py | 2 +- test_validation_heal.py | 6 ++++-- test_validation_heal_authority.py | 10 +++++++++- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index c63228ff..93127624 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1892,3 +1892,7 @@ Fresh read-only inspections `38193642346` and `38194452552` confirm the same fou The replacement 17-case queue pins helper `ccef050df3b28416c51356951e78cb46dd74d10b`. F05 retry `38194557163` also retained FAIL/cleanup PASS: the original root token returned `stopped`, `aborted/untraversable`, partial coverage and zero processed objects. Its detail names three bucket failures and `Storage error: erasure write quorum`. The fleet starts peers concurrently and waits for readiness, but the recovered task runs during startup. Candidate `heal_bucket` wraps its typed storage error into `TaskExecutionFailed`; the message fallback does not classify this write-quorum string as recoverable, and cluster traversal records a permanent untraversable failure. This is a candidate recovery-path diagnosis, not a failure of the later MPU consumer and not a reason to issue a replacement task or accept an aborted result. J06 retry `38195131282` then started on the same immutable queue; its result remains pending. Original A10 `38120735805` and C10 `38120042167` negative receipts exposed another harness mismatch. The candidate returns `stopped/completed_with_errors` and one exact permanent not-found receipt for a nonrecursive absent prefix or protected metadata-less orphan. The old convergence helper required `finished`; A10 additionally demanded an empty receipt list and C10 demanded `finished` even for a failed traversal. Convergence now compares the original path/token, settings and complete matching terminal outcomes, including truthful failed terminals. The negative oracle requires exactly one original failed object identity, unchanged selectors, an unversioned/unbound absent identity, complete coverage, exact 1/0/0/0/1/0 counters, one visible failed attempt, zero overflow and the exact not-found detail. Normal repair branches still require successful execution and their full ACK manifests. The 199 focused tests pass (one macOS skip); read-only replay of both original C10 refusals passes the corrected oracle without changing either retained FAIL or adding canonical PASS. Fresh full A10/C10 runs are still required. + +B07/C07 negative task oracles now require `stopped` for their already-required failed or aborted canonical execution, matching the candidate's `legacy_execution_status` projection. They still reject successful, unknown, incomplete or wrongly attributed responsibility; the four original C07 task receipts from `38119830051` remain rejected because they are completed deferred existence checks with no incarnation, rather than exact failed/aborted conflict receipts. The 254 focused contract tests pass with two macOS platform skips, and the source matrix remains 84/301/532. These oracle repairs add no canonical PASS and require new immutable live runs. + +J06 retry `38195131282` completed all twelve fixed ABBA phases with 1,440 scheduled requests and zero request failures, then retained FAIL/cleanup PASS at the original maximum paired latency ratio of 3. The second group's PUT p99 means were 67.45 ms without Heal and 353.32 ms with Heal (5.24 times); the third group's were 84.04 ms and 292.13 ms (3.48 times). The other ten paired comparisons met the gate. No acceptance threshold changed and successful physical repair or completed phases cannot replace the failed performance gate. The same serialized queue then started A03 `38198122041` at helper `ccef050df3b28416c51356951e78cb46dd74d10b`. diff --git a/scripts/validation_heal/authority.py b/scripts/validation_heal/authority.py index 4b46ab5a..04f3663f 100644 --- a/scripts/validation_heal/authority.py +++ b/scripts/validation_heal/authority.py @@ -118,7 +118,7 @@ def validate_split(snapshot, generation_a, generation_b, bucket, key, topology, def validate_negative_task(status, bucket, options): outcome = status.get('outcome') or {} execution = outcome.get('execution') or {} - require(status.get('summary') == 'finished' and status.get('truncated', False) is False + require(status.get('summary') == 'stopped' and status.get('truncated', False) is False and status.get('settings') == options and outcome.get('objectsTruncated') is False and execution.get('state') in {'completed_with_errors', 'aborted'} and outcome.get('coverage') in {'complete', 'partial', 'unknown'}, diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index a48eaeb2..06442d45 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -290,7 +290,7 @@ def validate_unrecoverable_get(observation, contract): def validate_failed_version(status, bucket, expected, target): outcome = status.get("outcome") or {} - require(status.get("summary") == "finished" and status.get("truncated", False) is False + require(status.get("summary") == "stopped" and status.get("truncated", False) is False and outcome.get("execution", {}).get("state") == "completed_with_errors" and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False, "negative Heal task hid failed execution or incomplete responsibility") diff --git a/test_validation_heal.py b/test_validation_heal.py index c22fccc4..c61c2045 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -736,14 +736,16 @@ def remote(node, action, **params): def test_below_quorum_needs_exact_failure_and_never_accepts_transport_or_false_repair(self): status, expected = self.fixture() + status["summary"] = "stopped" outcome = status["outcome"] outcome["execution"]["state"] = "completed_with_errors" outcome["objects"][0]["disposition"] = {"state": "failed", "details": "retry_exhausted"} outcome["counters"].update(unchanged=3, failed=1, attemptFailures=2) validate_failed_version(status, "fixture", expected, expected[0]) - for kind in ("completed", "unknown", "repaired", "wrong-version", "hidden-attempts", "wrong-counter"): + for kind in ("finished", "running", "completed", "unknown", "repaired", "wrong-version", "hidden-attempts", "wrong-counter"): value = copy.deepcopy(status) - if kind == "completed": value["outcome"]["execution"]["state"] = "completed" + if kind in {"finished", "running"}: value["summary"] = kind + elif kind == "completed": value["outcome"]["execution"]["state"] = "completed" elif kind in {"unknown", "repaired"}: value["outcome"]["objects"][0]["disposition"]["state"] = kind elif kind == "wrong-version": value["outcome"]["objects"][0]["identity"]["versionId"] = "different" elif kind == "hidden-attempts": value["outcome"]["counters"]["attemptFailures"] = 0 diff --git a/test_validation_heal_authority.py b/test_validation_heal_authority.py index 56f07d60..79ac4b35 100644 --- a/test_validation_heal_authority.py +++ b/test_validation_heal_authority.py @@ -101,7 +101,7 @@ def test_generation_sources_must_be_independent_and_branch_identity_exact(self): def negative_status(aborted=False): objects = [] if aborted else [{'identity': {'bucket': 'owned', 'object': 'conflict.bin', 'kind': 'object', 'versionId': 'null', 'bucketIncarnationId': 'incarnation', 'poolIndex': 0, 'setIndex': 0}, 'disposition': {'state': 'failed'}}] - return {'summary': 'finished', 'truncated': False, 'settings': BASE_OPTIONS, + return {'summary': 'stopped', 'truncated': False, 'settings': BASE_OPTIONS, 'outcome': {'execution': {'state': 'aborted', 'reason': 'untraversable'} if aborted else {'state': 'completed_with_errors'}, 'coverage': 'partial' if aborted else 'complete', 'objectsTruncated': False, 'objects': objects, 'counters': {'processed': len(objects), 'healed': 0, 'unchanged': 0, 'skipped': 0, 'failed': len(objects), @@ -116,6 +116,14 @@ def test_explicit_failed_object_and_typed_untraversable_abort_preserve_negative_ self.assertTrue(proof['safeExplicitRefusal']) self.assertFalse(proof['runtimeCommitPauseClaimed']) + def test_failed_and_aborted_terminals_cannot_be_presented_as_finished_or_running(self): + for aborted in (False, True): + for summary in ('finished', 'running', 'unknown'): + value = negative_status(aborted) + value['summary'] = summary + with self.subTest(aborted=aborted, summary=summary), self.assertRaises(ValueError): + authority.validate_negative_task(value, 'owned', BASE_OPTIONS) + def test_false_health_missing_identity_duplicate_counters_and_false_complete_abort_fail(self): for kind in ('healthy', 'healed', 'foreign', 'pool', 'incarnation', 'duplicate', 'counters', 'empty', 'options', 'overflow', 'abort-coverage'): value = negative_status(kind == 'abort-coverage') From 74cb9ba2b786a96d2732fe6cd991c5e32a7189b3 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 11:30:26 +0800 Subject: [PATCH 163/343] test(validation): reconcile mixed Heal outcome prerequisites --- .../workflows/validation-catalog-tests.yml | 3 + docs/validation-migration-map.md | 4 + scripts/validation_heal/outcome.py | 316 ++++++++++++++++++ test_validation_heal_outcome.py | 218 ++++++++++++ 4 files changed, 541 insertions(+) create mode 100644 scripts/validation_heal/outcome.py create mode 100644 test_validation_heal_outcome.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 6fc5f83c..fde52c54 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -45,6 +45,7 @@ on: - test_validation_heal_nolock.py - test_validation_heal_race.py - test_validation_heal_environment_gate.py + - test_validation_heal_outcome.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -103,6 +104,7 @@ on: - test_validation_heal_nolock.py - test_validation_heal_race.py - test_validation_heal_environment_gate.py + - test_validation_heal_outcome.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -142,6 +144,7 @@ jobs: fi - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check + - run: python3 -B -m unittest -v test_validation_heal_outcome.py - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh - run: bash test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 93127624..9bc43faa 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1896,3 +1896,7 @@ Original A10 `38120735805` and C10 `38120042167` negative receipts exposed anoth B07/C07 negative task oracles now require `stopped` for their already-required failed or aborted canonical execution, matching the candidate's `legacy_execution_status` projection. They still reject successful, unknown, incomplete or wrongly attributed responsibility; the four original C07 task receipts from `38119830051` remain rejected because they are completed deferred existence checks with no incarnation, rather than exact failed/aborted conflict receipts. The 254 focused contract tests pass with two macOS platform skips, and the source matrix remains 84/301/532. These oracle repairs add no canonical PASS and require new immutable live runs. J06 retry `38195131282` completed all twelve fixed ABBA phases with 1,440 scheduled requests and zero request failures, then retained FAIL/cleanup PASS at the original maximum paired latency ratio of 3. The second group's PUT p99 means were 67.45 ms without Heal and 353.32 ms with Heal (5.24 times); the third group's were 84.04 ms and 292.13 ms (3.48 times). The other ten paired comparisons met the gate. No acceptance threshold changed and successful physical repair or completed phases cannot replace the failed performance gate. The same serialized queue then started A03 `38198122041` at helper `ccef050df3b28416c51356951e78cb46dd74d10b`. + +A03 run `38198122041` passes every required H2 cross-node branch at helper `ccef050df3b28416c51356951e78cb46dd74d10b` and driver `03fb3d78a0ffc4213583dd3bcb44c7ac38f42ead`. Independent verification of the original artifact members, canonical result gate, complete branch outcomes and immutable candidate provenance increases the unique canonical census to 89. The remaining serialized backlog continues on the same candidate; no prior failure is overwritten. + +J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite and its branch is explicitly BLOCKED pending a reviewed live unknown/lock-deferred fixture. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the source assertion requiring a mixed deferred population is still outstanding. The 84 cases, 301 branches and 532 topology templates are unchanged. diff --git a/scripts/validation_heal/outcome.py b/scripts/validation_heal/outcome.py new file mode 100644 index 00000000..88b29988 --- /dev/null +++ b/scripts/validation_heal/outcome.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +"""J02: reconcile original mixed-task receipts with ACKs and physical faults.""" + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys + +try: + from .heal_probe import Probe, canonical, compare_inventories, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from .heal_fault import bounded_fault_id + from .inspector import verify as verify_inspector + from .manager_fixture import verify as verify_manager + from .matrix import select + from .native_regressions import CANDIDATE + from .shards import files, map_target, run_task, validate_fault, validate_repaired_physical + from .stop import idle_fleet + from .versions import (metadata_target, public_version_id, validate_unrecoverable_get, + validate_version_receipts, verify_proof, verify_quorum_contract, version_identity) +except ImportError: + from heal_probe import Probe, canonical, compare_inventories, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology + from heal_fault import bounded_fault_id + from inspector import verify as verify_inspector + from manager_fixture import verify as verify_manager + from matrix import select + from native_regressions import CANDIDATE + from shards import files, map_target, run_task, validate_fault, validate_repaired_physical + from stop import idle_fleet + from versions import (metadata_target, public_version_id, validate_unrecoverable_get, + validate_version_receipts, verify_proof, verify_quorum_contract, version_identity) + + +CASE_ID = "heal.outcome-counters-and-error-truth" +BRANCHES = ("mixed-healthy-repaired-failed", "dry-run", "deferred", "unknown-negative") +SIZE = 5 * 1024**2 + 123 +# The reviewed immutable candidate makes one attempt and three bucket-object retries. +FAILED_ATTEMPTS = 4 + + +def reconcile(status, bucket, expected, options, incarnation, *, attempt_failures): + """The manifest describes responsibility; legacy objectsHealed is not evidence.""" + wanted = {version_identity(entry): entry for entry in expected} + require(wanted and len(wanted) == len(expected), "J02 manifest is empty or duplicates ACK identities") + require(type(attempt_failures) is int and attempt_failures >= 0, "J02 invalid attempt-failure oracle") + failed = sum(entry["state"] == "failed" for entry in expected) + require(status.get("summary") == ("stopped" if failed else "finished") + and status.get("settings") == options and status.get("truncated") is False, + "J02 task has a wrong terminal, options or incomplete public result") + outcome = status.get("outcome") or {} + require(outcome.get("execution", {}).get("state") == ("completed_with_errors" if failed else "completed") + and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False, + "J02 task hid failed execution or lacks complete canonical coverage") + receipts = outcome.get("objects", []) + require(isinstance(receipts, list) and len(receipts) == len(wanted), "J02 omitted or duplicated object receipts") + observed = {} + counters = dict(processed=len(wanted), healed=0, unchanged=0, skipped=0, + failed=0, unknown=0, attemptFailures=attempt_failures, overflowed=False) + for receipt in receipts: + identity = receipt.get("identity", {}) + pair = identity.get("object"), public_version_id(identity.get("versionId")) + require(pair in wanted and pair not in observed and identity.get("kind") == "object" and identity.get("bucket") == bucket + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0, + "J02 receipt has a foreign/duplicate object, version or erasure set") + expected_entry = wanted[pair] + state = expected_entry["state"] + require(state in {"repaired", "verified_healthy", "metadata_healthy", "failed", "dry_run_observed", "deferred"}, + "J02 unknown, skipped or absent work cannot become a successful fixture result") + disposition = receipt.get("disposition", {}) + require(disposition.get("state") == state, "J02 disposition differs from the frozen physical fault plan") + if state == "deferred": + # Existence-check deferral cannot establish a version or incarnation. + # This is one key-level refusal, never a version repair receipt. + require(expected_entry.get("keyLevelRefusal") is True and pair[1] == "null" + and identity.get("versionId") is None and identity.get("bucketIncarnationId") is None + and disposition.get("details", {}).get("reason") == "transient_existence_check" + and disposition["details"].get("retry_not_before") is None, + "J02 deferred dependency claimed a version/incarnation or changed its typed reason") + else: + require(isinstance(incarnation, str) and incarnation + and identity.get("bucketIncarnationId") == incarnation, + "J02 receipt escaped the original healthy-task bucket incarnation") + if state == "failed": + require(disposition.get("details") == "retry_exhausted", "J02 below-R failure lost bounded retry exhaustion") + category = {"repaired": "healed", "verified_healthy": "unchanged", "metadata_healthy": "unchanged", + "failed": "failed", "dry_run_observed": "skipped", "deferred": "skipped"}[state] + counters[category] += 1 + observed[pair] = state + actual_counters = outcome.get("counters", {}) + require(all(type(actual_counters.get(name)) is type(value) and actual_counters[name] == value + for name, value in counters.items()), "J02 canonical counters hide retries or differ from complete dispositions") + return {"originalIncarnation": incarnation, "completeManifestReconciled": True, + "canonicalCounters": counters, "receipts": len(observed), "legacyObjectsHealedUsedAsRepairEvidence": False, + "deferredIsRepair": False, "failedIsRepair": False} + + +def fault_plan(original, bucket, expected, branch): + require(branch in BRANCHES[:3], "J02 native negative branch cannot mutate fleet files") + repair_entry = next(entry for entry in expected if entry["key"] == "objects/0001.bin" and not entry["deleteMarker"]) + failed_entry = next(entry for entry in expected if entry["key"] == "objects/0002.bin" and not entry["deleteMarker"]) + repair = map_target(original, bucket, repair_entry["key"], "data", 1, repair_entry["versionId"], latest_required=False) + failed = map_target(original, bucket, failed_entry["key"], "data", 1, failed_entry["versionId"], latest_required=False) + ordered = sorted(failed["cohort"], key=lambda member: member["index"]) + read_quorum = failed["logical"]["erasure"]["data_blocks"] + faults = [{**failed, **member} for member in ordered[read_quorum - 1:]] + if branch == "deferred": + faults = [metadata_target(original, target) for target in faults] + manifest = [{"key": failed["key"], "versionId": "null", "state": "deferred", "keyLevelRefusal": True}] + else: + faults.insert(0, repair) + manifest = [{**entry, "state": "dry_run_observed" if branch == "dry-run" else + "repaired" if version_identity(entry) == version_identity(repair) else + "failed" if version_identity(entry) == version_identity(failed) else + "metadata_healthy" if entry["deleteMarker"] else "verified_healthy"} for entry in expected] + require(len({(item["node"], item["disk"], item["relativePath"]) for item in faults}) == len(faults) + and len(ordered) - (len(faults) - (branch != "deferred")) == read_quorum - 1, + "J02 fault plan duplicates files or does not leave precisely R-1 authoritative members") + return {"repair": repair, "failed": failed, "faults": faults, "manifest": manifest, + "readQuorum": read_quorum, "retainedMembers": ordered[:read_quorum - 1]} + + +def rescue(fleet): + for node, params in tuple(fleet.faults): + fleet.record("restore-fixture-without-relabeling-original-task", { + "node": node, **fleet.remote(node, "recover-fault", **params)}) + fleet.faults.remove((node, params)) + + +def retained_failure_scene(original, faulted, after, plan, branch): + if branch in {"dry-run", "deferred"}: + require(physical_content(faulted) == physical_content(after), "J02 dry-run/deferred task mutated a physical file") + return {"physicalUnchanged": True, "productRepairClaimed": False} + # Restore only the intentionally unrecoverable files in the oracle view. + # The fleet remains faulted until after the original failed task is archived. + combined = json.loads(json.dumps(after)) + originals, observed = files(original), files(after) + for target in plan["faults"][1:]: + identity = target["node"], target["disk"], target["relativePath"] + require(identity not in observed, "J02 below-R task invented an unrecoverable shard") + combined[target["node"]]["files"].append(originals[identity]) + proof = validate_repaired_physical(original, combined, plan["repair"]) + return {**proof, "failedMembersStillAbsent": len(plan["faults"]) - 1, + "oracleViewOnly": True, "fixtureRescueIsProductRepair": False} + + +def run_branch(args, topology, nodes, fixture, branch, candidate): + directory = args.evidence / fixture / topology / branch + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + fleet.snapshot_buckets = [bucket] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 128, "maxInventoryBytes": 512 * 1024**2} + fleet.record("binding", binding) + contract = verify_quorum_contract(fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)), candidate) + probe, active, pending = Probe(binding), [], None + result = {"topology": topology, "fixture": fixture, "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) + pending = {"operation": "seed", "bucket": bucket} + expected = seed(Recorder(probe, directory / "fixture"), bucket, fixture, 3, [SIZE]) + baseline = fleet.record("ACK-bound-full-S3", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + fleet.record("complete-ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + pending = None + options = {**BASE_OPTIONS, "recursive": True} + anchor = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, + option_overrides=options, status_endpoints=tuple(binding["endpoints"])[1:]) + incarnation = validate_version_receipts(anchor, bucket, expected, 2)["bucketIncarnationId"] + idle_fleet(fleet, binding) + fleet.stop() + original = fleet.snapshot("healthy-physical-baseline", quiescent=True) + plan = fault_plan(original, bucket, expected, branch) + fleet.record("independent-manifest-and-frozen-fault-plan", plan) + for target in (plan["repair"], plan["failed"]): + verify_proof(fleet, target, "original-full-cohort-bitrot") + prior = original + for index, target in enumerate(plan["faults"]): + params = {"disk": target["disk"], "faultId": bounded_fault_id("J02", branch, index)} + fleet.faults.append((target["node"], params)) + receipt = fleet.record("stopped-original-file-fault", {"node": target["node"], **fleet.remote( + target["node"], "apply-fault", **params, relativePath=target["relativePath"], + expectedSha256=target["sha256"], kind="missing")}) + faulted = fleet.snapshot("actual-R-minus-one-fault", quiescent=True) + validate_fault(prior, faulted, target, receipt) + prior = faulted + fleet.start() + require(physical_content(faulted) == physical_content(fleet.snapshot("before-original-task")), + "J02 attribution window lost: background activity changed a fault before admission") + if branch == "mixed-healthy-repaired-failed": + for endpoint in binding["endpoints"]: + observed = fleet.record("pre-task-failed-version-GET", Probe(binding, endpoint).s3("get_object", { + "Bucket": bucket, "Key": plan["failed"]["key"], "VersionId": plan["failed"]["versionId"]})) + validate_unrecoverable_get(observed, contract) + options = {**options, "dryRun": branch == "dry-run", "recursive": branch != "deferred"} + scope = {"bucket": bucket, "key": plan["failed"]["key"] if branch == "deferred" else ""} + admissions = [] + status = run_task(probe, fleet, scope, 2, active, option_overrides=options, + status_endpoints=tuple(binding["endpoints"])[1:], admissions=admissions) + require(len(admissions) == 1, "J02 original fault task lacks an exact independent admission") + fleet.record("original-terminal-before-any-rescue", status) + result["responsibility"] = reconcile(status, bucket, plan["manifest"], options, incarnation, + attempt_failures=FAILED_ATTEMPTS if branch == "mixed-healthy-repaired-failed" else 1 if branch == "deferred" else 0) + idle_fleet(fleet, binding) + fleet.stop() + after = fleet.snapshot("original-result-physical-scene", quiescent=True) + result["physical"] = retained_failure_scene(original, faulted, after, plan, branch) + if branch == "mixed-healthy-repaired-failed": + verify_proof(fleet, plan["repair"], "actual-repaired-shard-before-rescue-or-GET") + rescue(fleet) + restored = fleet.snapshot("fixture-restored-after-original-task-archival", quiescent=True) + if branch == "mixed-healthy-repaired-failed": + validate_repaired_physical(original, restored, plan["repair"]) + else: + require(physical_content(original) == physical_content(restored), "J02 rescue changed the original ACK generation") + for target in (plan["repair"], plan["failed"]): + verify_proof(fleet, target, "all-original-shards-after-fixture-rescue") + fleet.start() + for endpoint in binding["endpoints"]: + observed = fleet.record("all-peer-restored-full-S3", Probe(binding, endpoint).inventory( + bucket, unversioned_current_tags=fixture == "DB")) + verify_seed(expected, observed) + compare_inventories(baseline, observed) + repeated = fleet.record("original-token-after-rescue", probe.admin(admissions[0]["path"], + [("clientToken", admissions[0]["clientToken"])], b"")) + require(repeated.get("httpStatus") == 200 and repeated.get("response", {}).get("summary") == status["summary"] + and repeated["response"].get("settings") == options + and repeated["response"].get("truncated") is False + and canonical(repeated["response"].get("outcome")) == canonical(status["outcome"]), + "J02 rescue overwrote the original failed/deferred/dry-run responsibility") + result.update(verdict="PASS", recoveryIsProductRepair=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + if fleet.reserved: + try: fleet.snapshot("failure-scene-before-fault-rescue") + except Exception as capture: fleet.record("failure-scene-error", {"error": str(capture)}) + finally: + for path, token in active: + try: fleet.record("stop-unresolved", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) + except Exception as exc: fleet.record("stop-unresolved-error", {"error": str(exc)}) + if pending is not None: + try: fleet.stop() + except Exception as exc: fleet.record("retain-stop-error", {"error": str(exc)}) + fleet.record("retain-unknown-seed-receipt", {"state": "DIRTY", "namespacesRetained": True, "request": pending}) + clean = False + else: + clean = fleet.cleanup() + result["cleanup"] = "PASS" if clean else "FAIL" + fleet.record("branch-result", result) + return result + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f"J02 interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--rc-bin", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--tool-directory", type=Path, required=True) + args = parser.parse_args() + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "J02 executor omitted a source branch") + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == CANDIDATE, "J02 retry/deferral contract is not reviewed for this candidate") + native = verify_manager(os.environ.get("VALIDATION_HEAL_MANAGER_DIRECTORY", ""), candidate, CASE_ID) + save(args.evidence / "independent-native-outcome-guard.json", native) + require(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha) + and re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")), "J02 immutable workflow/binary identities required") + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) == len(set(nodes)) == 4 and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), + "J02 requires the four current distinct auto-testing Heal nodes") + results = [] + try: + for topology, selected in (("H1", nodes[:1]), ("H2", nodes)): + for fixture in ("DB", "DV"): + for branch in BRANCHES: + if branch == "unknown-negative": + result = {"topology": topology, "fixture": fixture, "branch": branch, "verdict": "BLOCKED", "cleanup": "PASS", + "evidenceKind": "separate-exact-candidate-native-outcome-negative", "native": native, + "reason": "exact candidate native unknown-identity guard is a prerequisite; a reviewed live unknown/lock-deferred fixture is still required", + "fleetUnknownWindowClaimed": False, "productRepairClaimed": False} + else: + result = run_branch(args, topology, selected, fixture, branch, candidate) + results.append(result) + if result["cleanup"] != "PASS": return 1 + finally: + wanted = {(topology, fixture, branch) for topology in ("H1", "H2") for fixture in ("DB", "DV") for branch in BRANCHES} + complete = {(row["topology"], row["fixture"], row["branch"]) for row in results} == wanted and len(results) == len(wanted) + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, + "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL", + "complete": complete and all(row["verdict"] == row["cleanup"] == "PASS" for row in results)}) + return 0 if complete and all(row["verdict"] == row["cleanup"] == "PASS" for row in results) else 1 + + +if __name__ == "__main__": sys.exit(main()) diff --git a/test_validation_heal_outcome.py b/test_validation_heal_outcome.py new file mode 100644 index 00000000..e75e54e6 --- /dev/null +++ b/test_validation_heal_outcome.py @@ -0,0 +1,218 @@ +"""J02 rejects fabricated truth and checks faults before fixture recovery.""" + +import copy +import json +import os +from pathlib import Path +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import outcome +from scripts.validation_heal.healthy import BASE_OPTIONS, physical_content +from scripts.validation_heal.shards import files +from test_validation_heal import versioned_scene + + +class MixedOutcomeTest(unittest.TestCase): + bucket = "rv-heal-0123456789abcdef" + incarnation = "12345678-1234-1234-1234-123456789abc" + options = {**BASE_OPTIONS, "recursive": True} + + def mixed(self): + expected = [{"key": "objects/0000.bin", "versionId": "v0", "state": "verified_healthy"}, + {"key": "objects/0001.bin", "versionId": "v1", "state": "repaired"}, + {"key": "objects/0002.bin", "versionId": "v2", "state": "failed"}] + status = {"summary": "stopped", "settings": self.options, "truncated": False, + "objectsHealed": 77, "outcome": {"execution": {"state": "completed_with_errors"}, + "coverage": "complete", "objectsTruncated": False, + "counters": {"processed": 3, "healed": 1, "unchanged": 1, "skipped": 0, "failed": 1, + "unknown": 0, "attemptFailures": 4, "overflowed": False}, + "objects": [{"identity": {"kind": "object", "bucket": self.bucket, "object": entry["key"], "versionId": entry["versionId"], + "poolIndex": 0, "setIndex": 0, "bucketIncarnationId": self.incarnation}, + "disposition": {"state": entry["state"], **({"details": "retry_exhausted"} if entry["state"] == "failed" else {})}} + for entry in expected]}} + return expected, status + + def check(self, expected, status, options=None, attempts=4): + return outcome.reconcile(status, self.bucket, expected, options or self.options, self.incarnation, + attempt_failures=attempts) + + def test_full_mixed_truth_keeps_failure_and_ignores_legacy_repair_count(self): + expected, status = self.mixed() + proof = self.check(expected, status) + self.assertEqual(proof["canonicalCounters"]["failed"], 1) + self.assertFalse(proof["failedIsRepair"]) + self.assertFalse(proof["legacyObjectsHealedUsedAsRepairEvidence"]) + + def test_each_identity_boundary_rejects_wrong_or_ambiguous_receipts(self): + expected, source = self.mixed() + for field, values in {"kind": ["bucket", None], "bucket": ["foreign"], "object": ["foreign"], "versionId": [None, "v9"], + "poolIndex": [True, 1, None], "setIndex": [False, 1], + "bucketIncarnationId": [None, "", "new-incarnation"]}.items(): + for value in values: + with self.subTest(field=field, value=value): + status = copy.deepcopy(source) + status["outcome"]["objects"][1]["identity"][field] = value + with self.assertRaises(ValueError): self.check(expected, status) + status = copy.deepcopy(source) + status["outcome"]["objects"][1] = copy.deepcopy(status["outcome"]["objects"][0]) + with self.assertRaises(ValueError): self.check(expected, status) + + def test_wrong_terminals_or_partial_coverage_cannot_certify_negative_truth(self): + expected, source = self.mixed() + for name, value in (("summary", "finished"), ("summary", "running"), ("truncated", True), + ("settings", {**self.options, "remove": True})): + status = copy.deepcopy(source); status[name] = value + with self.subTest(name=name, value=value), self.assertRaises(ValueError): self.check(expected, status) + for name, value in (("execution", {"state": "completed"}), ("coverage", "partial"), ("objectsTruncated", True)): + status = copy.deepcopy(source); status["outcome"][name] = value + with self.subTest(name=name, value=value), self.assertRaises(ValueError): self.check(expected, status) + + def test_unknown_skipped_deferred_and_false_health_never_substitute_for_failed(self): + expected, source = self.mixed() + for state in ("unknown", "skipped", "deferred", "verified_healthy", "metadata_healthy", "repaired"): + status = copy.deepcopy(source); status["outcome"]["objects"][2]["disposition"]["state"] = state + with self.subTest(state=state), self.assertRaises(ValueError): self.check(expected, status) + for details in ("recoverable", "permanent", None): + status = copy.deepcopy(source); status["outcome"]["objects"][2]["disposition"]["details"] = details + with self.subTest(details=details), self.assertRaises(ValueError): self.check(expected, status) + + def test_every_counter_rejects_inflation_understatement_and_bool_coercion(self): + expected, source = self.mixed() + for name, original in source["outcome"]["counters"].items(): + for value in (None, True if name != "overflowed" else 0, original + 1 if type(original) is int else True): + status = copy.deepcopy(source); status["outcome"]["counters"][name] = value + with self.subTest(counter=name, value=value), self.assertRaises(ValueError): self.check(expected, status) + + def test_dry_run_requires_all_original_members_without_repairs_or_hidden_attempts(self): + expected, status = self.mixed() + for entry, receipt in zip(expected, status["outcome"]["objects"]): + entry["state"] = "dry_run_observed"; receipt["disposition"] = {"state": "dry_run_observed"} + options = {**self.options, "dryRun": True} + status.update(summary="finished", settings=options) + status["outcome"].update(execution={"state": "completed"}, + counters={"processed": 3, "healed": 0, "unchanged": 0, "skipped": 3, "failed": 0, + "unknown": 0, "attemptFailures": 0, "overflowed": False}) + self.check(expected, status, options, 0) + status["outcome"]["objects"].pop() + with self.assertRaises(ValueError): self.check(expected, status, options, 0) + + def test_key_level_deferral_cannot_invent_an_incarnation_or_version(self): + expected = [{"key": "objects/0002.bin", "versionId": "null", "state": "deferred", "keyLevelRefusal": True}] + options = {**self.options, "recursive": False} + status = {"summary": "finished", "settings": options, "truncated": False, "outcome": { + "execution": {"state": "completed"}, "coverage": "complete", "objectsTruncated": False, + "counters": {"processed": 1, "healed": 0, "unchanged": 0, "skipped": 1, + "failed": 0, "unknown": 0, "attemptFailures": 1, "overflowed": False}, + "objects": [{"identity": {"kind": "object", "bucket": self.bucket, "object": expected[0]["key"], "versionId": None, + "bucketIncarnationId": None, "poolIndex": 0, "setIndex": 0}, "disposition": { + "state": "deferred", "details": {"reason": "transient_existence_check", "retry_not_before": None}}}]}} + self.check(expected, status, options, 1) + for field, value in (("bucketIncarnationId", self.incarnation), ("versionId", "null"), ("versionId", "v0")): + changed = copy.deepcopy(status); changed["outcome"]["objects"][0]["identity"][field] = value + with self.subTest(field=field, value=value), self.assertRaises(ValueError): self.check(expected, changed, options, 1) + changed = copy.deepcopy(status) + changed["outcome"]["objects"][0]["disposition"]["details"]["reason"] = "dangling_delete_grace" + with self.assertRaises(ValueError): self.check(expected, changed, options, 1) + + def test_planner_uses_actual_erasure_geometry_and_preserves_unselected_versions(self): + scene, bucket, expected = versioned_scene([f"objects/{index:04d}.bin" for index in range(3)]) + for branch in outcome.BRANCHES[:3]: + with self.subTest(branch=branch): + plan = outcome.fault_plan(scene, bucket, expected, branch) + self.assertEqual(len(plan["retainedMembers"]), plan["readQuorum"] - 1) + self.assertEqual(len(plan["manifest"]), 1 if branch == "deferred" else len(expected)) + if branch == "mixed-healthy-repaired-failed": + states = {outcome.version_identity(entry): entry["state"] for entry in plan["manifest"]} + self.assertEqual(states[("objects/0002.bin", "v0")], "failed") + self.assertEqual(states[("objects/0002.bin", "v1")], "verified_healthy") + if branch == "deferred": + self.assertTrue(all(target["relativePath"].endswith("/xl.meta") for target in plan["faults"])) + + def test_physical_oracle_rejects_false_repair_and_any_change_to_a_control(self): + original, bucket, expected = versioned_scene([f"objects/{index:04d}.bin" for index in range(3)]) + plan = outcome.fault_plan(original, bucket, expected, "mixed-healthy-repaired-failed") + faulted = copy.deepcopy(original) + for target in plan["faults"]: + faulted[target["node"]]["files"] = [entry for entry in faulted[target["node"]]["files"] + if (entry["disk"], entry["path"]) != (target["disk"], target["relativePath"])] + after = copy.deepcopy(faulted) + target = plan["repair"] + after[target["node"]]["files"].append(files(original)[target["node"], target["disk"], target["relativePath"]]) + before_digest = physical_content(after) + proof = outcome.retained_failure_scene(original, faulted, after, plan, "mixed-healthy-repaired-failed") + self.assertTrue(proof["oracleViewOnly"]) + self.assertEqual(physical_content(after), before_digest) + with self.assertRaises(ValueError): + outcome.retained_failure_scene(original, faulted, faulted, plan, "mixed-healthy-repaired-failed") + altered = copy.deepcopy(after); altered[target["node"]]["files"][0]["sha256"] = "changed-control" + with self.assertRaises(ValueError): + outcome.retained_failure_scene(original, faulted, altered, plan, "mixed-healthy-repaired-failed") + with self.assertRaises(ValueError): + outcome.retained_failure_scene(original, faulted, original, plan, "mixed-healthy-repaired-failed") + + def test_dry_run_and_deferred_require_exact_physical_silence(self): + original, bucket, expected = versioned_scene([f"objects/{index:04d}.bin" for index in range(3)]) + for branch in ("dry-run", "deferred"): + plan = outcome.fault_plan(original, bucket, expected, branch) + self.assertTrue(outcome.retained_failure_scene(original, original, original, plan, branch)["physicalUnchanged"]) + changed = copy.deepcopy(original); changed["node1"]["files"].pop() + with self.assertRaises(ValueError): outcome.retained_failure_scene(original, original, changed, plan, branch) + + def test_lost_rescue_ack_keeps_registered_fault_for_independent_recovery(self): + fleet = types.SimpleNamespace(faults=[("node1", {"disk": 0, "faultId": "J02-mixed-0"})], + remote=lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("lost ACK")), record=lambda *args: None) + with self.assertRaises(RuntimeError): outcome.rescue(fleet) + self.assertEqual(len(fleet.faults), 1) + + def test_native_prerequisite_rejection_happens_before_fleet_creation(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); candidate = root / "candidate.json" + candidate.write_text(json.dumps({"source_sha": outcome.CANDIDATE})) + args = types.SimpleNamespace(candidate=candidate) + with patch.object(outcome.argparse.ArgumentParser, "parse_args", return_value=args), \ + patch.object(outcome, "verify_manager", side_effect=ValueError("wrong immutable evidence")), \ + patch.object(outcome, "Fleet") as fleet, patch.object(outcome.signal, "signal"): + with self.assertRaisesRegex(ValueError, "immutable evidence"): outcome.main() + fleet.assert_not_called() + + def test_native_only_unknown_branch_keeps_full_case_incomplete(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); candidate = root / "candidate.json" + candidate.write_text(json.dumps({"source_sha": outcome.CANDIDATE})) + args = types.SimpleNamespace(candidate=candidate, evidence=root / "evidence", binary_sha="a" * 64) + def branch(args, topology, nodes, fixture, name, candidate): + return {"topology": topology, "fixture": fixture, "branch": name, "verdict": "PASS", "cleanup": "PASS"} + with patch.object(outcome.argparse.ArgumentParser, "parse_args", return_value=args), \ + patch.object(outcome, "verify_manager", return_value={"canonicalFleetBranchesExecuted": False}), \ + patch.object(outcome, "run_branch", side_effect=branch), patch.object(outcome.signal, "signal"), \ + patch.dict(os.environ, {"GITHUB_RUN_ID": "1234", "RUSTFS_NODES": "vm000 vm001 vm002 vm003"}): + self.assertEqual(outcome.main(), 1) + summary = json.loads((args.evidence / "healthy-summary.json").read_text()) + self.assertEqual(len(summary["results"]), 16) + self.assertFalse(summary["complete"]) + self.assertEqual(summary["cleanup"], "PASS") + negative = [row for row in summary["results"] if row["branch"] == "unknown-negative"] + self.assertEqual(len(negative), 4) + self.assertTrue(all(row["verdict"] == "BLOCKED" and not row["fleetUnknownWindowClaimed"] for row in negative)) + + def test_dirty_cleanup_stops_before_next_fixture_or_topology(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); candidate = root / "candidate.json" + candidate.write_text(json.dumps({"source_sha": outcome.CANDIDATE})) + args = types.SimpleNamespace(candidate=candidate, evidence=root / "evidence", binary_sha="a" * 64) + row = {"topology": "H1", "fixture": "DB", "branch": outcome.BRANCHES[0], "verdict": "FAIL", "cleanup": "FAIL"} + with patch.object(outcome.argparse.ArgumentParser, "parse_args", return_value=args), \ + patch.object(outcome, "verify_manager", return_value={}), \ + patch.object(outcome, "run_branch", return_value=row) as run, patch.object(outcome.signal, "signal"), \ + patch.dict(os.environ, {"GITHUB_RUN_ID": "1234", "RUSTFS_NODES": "vm000 vm001 vm002 vm003"}): + self.assertEqual(outcome.main(), 1) + self.assertEqual(run.call_count, 1) + summary = json.loads((args.evidence / "healthy-summary.json").read_text()) + self.assertFalse(summary["complete"]) + self.assertEqual(summary["cleanup"], "FAIL") + + +if __name__ == "__main__": unittest.main() From fd4277c901094226b688f749ae7c2f6797180793 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 11:57:23 +0800 Subject: [PATCH 164/343] fix(validation): retain complete overlap task sequence evidence --- docs/validation-migration-map.md | 17 +++- scripts/validation_heal/overlap.py | 108 +++++++++++++++------ test_validation_heal.py | 145 +++++++++++++++++++++++++++-- 3 files changed, 230 insertions(+), 40 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 9bc43faa..5461a290 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 88 unique complete canonical live PASS cases: 76 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632` are included. Native, read-only, recovery and duplicate runs are excluded; the other 70 cases have no complete canonical live PASS claim. +case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 89 unique complete canonical live PASS cases: 77 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, and A03 run `38198122041`, are included. Native, read-only, recovery and duplicate runs are excluded; the other 69 cases have no complete canonical live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -600,13 +600,22 @@ window remains NOT_OBSERVED. forceStart must cancel the old task with partial coverage, admit an independent replacement token, and resist old-token STOP. Initial and settled cancellation reports are retained; exact old/new repair counters, incarnations and diagnostic identities cannot leak or duplicate -responsibility. All 257 terminal item identities are read, including any -cropped HTTP suffix; this does not claim a 1024-item retention gap. Three +responsibility. All 257 original item sequences are retained while running +and reconciled with the exact terminal outcome and retained suffix. The +completed-report byte budget can evict an early prefix below the 1024-item +count cap; an unobserved prefix still fails completeness. Both disjoint tasks +are sampled concurrently, and replacement drain samples are preserved. Three live physical samples after actual all-peer drain must remain unchanged, with the same processes. Complete same-role shard bytes, streaming checksums and all-peer full S3 manifests remain mandatory before PASS. Local scope/error/window/duplicate-repair/item-omission/cleanup-stop negatives -passed. Full live evidence remains pending. +passed. Run `38199371461` retained FAIL with clean finalization: H2's first +three branches exposed the early-prefix accounting gap, H1 force-start had a +typed cancelled receipt counted as skipped, and H2 force-start contained a +real unknown receipt. Replaying the original running records verifies all +three 257-item primary ledgers; H1 cancellation now requires its exact typed +receipt count, while H2 unknown remains rejected. This diagnostic replay +does not change the original FAIL or claim a complete new live PASS. F01 `heal.root-graceful-coordinator-restart` now dispatches both source branches on H1/H2 with the same fixed DX physical/S3 oracles. The coordinator diff --git a/scripts/validation_heal/overlap.py b/scripts/validation_heal/overlap.py index 7fa964df..a8732f02 100644 --- a/scripts/validation_heal/overlap.py +++ b/scripts/validation_heal/overlap.py @@ -68,13 +68,16 @@ def scope_expected(expected, prefix): return selected -def counters(current): +def counters(current, *, cancelled=False): value = (current.get("outcome") or {}).get("counters", {}) for name in ("processed", "healed", "unchanged", "skipped", "failed", "unknown", "attemptFailures"): require(type(value.get(name)) is int and value[name] >= 0, "scope omitted typed canonical counters") - require(all(value[name] == 0 for name in ("skipped", "failed", "unknown", "attemptFailures")) + cancellation_count = sum(receipt.get("disposition", {}).get("state") == "cancelled" + for receipt in (current.get("outcome") or {}).get("objects", [])) + require(all(value[name] == 0 for name in ("failed", "unknown", "attemptFailures")) + and value["skipped"] == (cancellation_count if cancelled else 0) and value.get("overflowed") is False - and value["processed"] == value["healed"] + value["unchanged"], + and value["processed"] == value["healed"] + value["unchanged"] + value["skipped"], "scope counters hide failed, unknown or unresolved responsibility") return value @@ -96,8 +99,8 @@ def wait_window(probe, fleet, path, token): time.sleep(0.05) -def monotonic(before, after): - left, right = counters(before), counters(after) +def monotonic(before, after, *, cancelled=False): + left, right = counters(before), counters(after, cancelled=cancelled) require(all(right[name] >= left[name] for name in ("processed", "healed", "unchanged")), "same-token scope counters moved backwards") @@ -116,7 +119,7 @@ def overlap_rejection(observed): def scope_proof(current, bucket, expected, damaged, *, cancelled=False, healed=None, replacement=False): wanted = wanted_pairs(expected, count=SCOPE_OBJECTS) require(damaged and set(damaged) <= set(wanted), "scope repair plan escaped its exact ACK identities") - value = counters(current) + value = counters(current, cancelled=cancelled) outcome = current.get("outcome") or {} if cancelled: require(current.get("summary") == "stopped" and outcome.get("execution") == {"state": "aborted", "reason": "cancelled"} @@ -152,39 +155,73 @@ def scope_proof(current, bucket, expected, damaged, *, cancelled=False, healed=N "diagnosticObjectsAreCompleteManifest": False, "coverage": outcome["coverage"]} -def completed_scope(probe, fleet, path, token, bucket, expected, damaged, *, healed, replacement=False): +def collect_scope_items(state, bucket, wanted, ledger, *, since=0): + """Retain original same-token items before completed-report byte eviction.""" + next_seq, min_seq = state.get("nextSeq"), state.get("minSeq") + require(type(next_seq) is int and type(min_seq) is int and 1 <= min_seq <= next_seq <= SCOPE_OBJECTS + 1, + "scope item sequence bounds are missing or invalid") + require(next_seq > max(ledger, default=0), "same-token scope item sequence moved backwards") + require(min_seq == 1 or state.get("truncated") is True, "scope silently lost an early item sequence") + items = state.get("items", []) + first = max(since + 1, min_seq) + available = max(0, next_seq - first) + require(isinstance(items, list) and len(items) <= available + and (len(items) == available or state.get("truncated") is True), "scope silently omitted item sequences") + for index, item in enumerate(items, first): + pair = item.get("object"), public_version_id(item.get("versionId")) + require(item.get("bucket") == bucket and pair in wanted + and item.get("objectSize") == wanted[pair]["payload"]["size"], "scope item is outside its exact ACK manifest") + require(index not in ledger or canonical(ledger[index]) == canonical(item), "same-token scope item changed its original sequence") + require(not any(seq != index and (entry.get("object"), public_version_id(entry.get("versionId"))) == pair + for seq, entry in ledger.items()), "scope item duplicated a real object/version") + ledger[index] = item + + +def completed_scope(probe, fleet, path, token, bucket, expected, damaged, *, healed, replacement=False, + observations=(), sample_other=None): deadline = time.monotonic() + TASK_SECONDS + wanted = wanted_pairs(expected, count=SCOPE_OBJECTS) + ledger = {} + for observation in observations: + collect_scope_items(observation, bucket, wanted, ledger) while True: current = status(probe, fleet, path, token, "overlap-terminal-wait") + collect_scope_items(current, bucket, wanted, ledger) + if sample_other is not None: + sample_other() if current.get("summary") != "running": break counters(current) require(time.monotonic() < deadline, "scope exceeded its frozen completion budget") time.sleep(0.05) proof = scope_proof(current, bucket, expected, damaged, healed=healed, replacement=replacement) - # These scopes fit actual item retention. Read every retained suffix even - # when the HTTP serializer independently trims an individual response. - wanted = wanted_pairs(expected, count=SCOPE_OBJECTS) - cursor, seen = 0, set() - while cursor < SCOPE_OBJECTS: + # The completed-report byte budget can evict a prefix even below the + # 1024-item count cap. A complete proof needs the original running items. + cursor = 0 + while cursor + 1 in ledger: cursor += 1 + first_page = True + while first_page or cursor < SCOPE_OBJECTS: + first_page = False observed = fleet.record("overlap-complete-items", probe.admin(path, [("clientToken", token), ("sinceSeq", str(cursor))], b"")) require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), "scope cursor failed") page = observed["response"] - require(page.get("summary") == "finished" and page.get("minSeq") == 1 and page.get("nextSeq") == SCOPE_OBJECTS + 1 + require(page.get("summary") == "finished" and page.get("nextSeq") == SCOPE_OBJECTS + 1 and canonical(page.get("outcome")) == canonical(current["outcome"]), "terminal scope changed, lost items or crossed tokens") - items = page.get("items", []) - require(isinstance(items, list) and 0 < len(items) <= SCOPE_OBJECTS - cursor - and (len(items) == SCOPE_OBJECTS - cursor or page.get("truncated") is True), "scope silently omitted a retained suffix") - for item in items: - pair = item.get("object"), public_version_id(item.get("versionId")) - require(item.get("bucket") == bucket and pair in wanted and pair not in seen - and item.get("objectSize") == wanted[pair]["payload"]["size"], "scope item is duplicate, missing or outside ACKs") - seen.add(pair) - cursor += len(items) + previous = cursor + collect_scope_items(page, bucket, wanted, ledger, since=cursor) + while cursor + 1 in ledger: cursor += 1 + require(cursor == SCOPE_OBJECTS or cursor > previous, + "scope lost an uncollected early item; a response byte-limit gap cannot certify completeness") require(time.monotonic() < deadline, "scope item suffix exceeded completion budget") + seen = {(item.get("object"), public_version_id(item.get("versionId"))) for item in ledger.values()} + require(set(ledger) == set(range(1, SCOPE_OBJECTS + 1)), "scope sequence ledger is incomplete") require(seen == set(wanted), "terminal scope missed a real object/version") - return current, {**proof, "completeItemIdentities": len(seen), "retentionGapClaimed": False} + evidence = {"path": path, "clientToken": token, "bucket": bucket, + "items": [{"seq": seq, "item": ledger[seq]} for seq in sorted(ledger)]} + fleet.record("overlap-complete-item-ledger", evidence) + return current, {**proof, "completeItemIdentities": len(seen), "retentionGapClaimed": False, + "completeItemLedgerSha256": hashlib.sha256(canonical(evidence).encode()).hexdigest()} def force_accounting(old, new, damaged): @@ -217,7 +254,8 @@ def observe_branch(branch, probe, fleet, binding, bucket, expected, targets, act require(active_window(after), "NOT_OBSERVED: overlap request was not bracketed by the same running task") monotonic(before, after) proof["runningAfterRequest"] = counters(after) - terminal, proof["completeResponsibility"] = completed_scope(probe, fleet, path, old, bucket, primary, damaged, healed=len(damaged)) + terminal, proof["completeResponsibility"] = completed_scope(probe, fleet, path, old, bucket, primary, damaged, + healed=len(damaged), observations=(before, after)) active.remove((path, old)) elif branch == "disjoint": other_path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + PREFIXES[1] @@ -226,11 +264,15 @@ def observe_branch(branch, probe, fleet, binding, bucket, expected, targets, act after = status(probe, fleet, path, old, "original-after-disjoint-start") require(active_window(after), "NOT_OBSERVED: original task finished before the disjoint admission was bracketed") monotonic(before, after) - wait_window(probe, fleet, other_path, new) + other_observations = [wait_window(probe, fleet, other_path, new)] other = scope_expected(expected, PREFIXES[1]) other_damaged = {(entry["key"], entry["versionId"]) for entry in targets if entry["key"].startswith(PREFIXES[1])} - terminal, proof["completeResponsibility"] = completed_scope(probe, fleet, path, old, bucket, primary, damaged, healed=len(damaged)) - other_terminal, proof["disjointResponsibility"] = completed_scope(probe, fleet, other_path, new, bucket, other, other_damaged, healed=len(other_damaged)) + def sample_disjoint(): + other_observations.append(status(probe, fleet, other_path, new, "overlap-disjoint-parallel-status")) + terminal, proof["completeResponsibility"] = completed_scope(probe, fleet, path, old, bucket, primary, damaged, + healed=len(damaged), observations=(before, after), sample_other=sample_disjoint) + other_terminal, proof["disjointResponsibility"] = completed_scope(probe, fleet, other_path, new, bucket, other, other_damaged, + healed=len(other_damaged), observations=other_observations) require(proof["completeResponsibility"]["bucketIncarnationId"] == proof["disjointResponsibility"]["bucketIncarnationId"], "disjoint tasks crossed bucket incarnation") active.remove((path, old)); active.remove((other_path, new)) @@ -242,7 +284,7 @@ def observe_branch(branch, probe, fleet, binding, bucket, expected, targets, act first_old = status(probe, fleet, path, old, "force-old-initial-cancellation") require(first_old.get("summary") != "finished", "NOT_OBSERVED: original task completed before forceStart could cancel it") initial = scope_proof(first_old, bucket, primary, damaged, cancelled=True) - monotonic(before, first_old) + monotonic(before, first_old, cancelled=True) new_before = wait_window(probe, fleet, path, new) retired = fleet.record("force-old-stop", probe.admin(path, [("clientToken", old), ("forceStop", "true")], b"")) proof["oldStopIsolation"] = validate_unknown_token(retired, old) @@ -253,7 +295,13 @@ def observe_branch(branch, probe, fleet, binding, bucket, expected, targets, act # The old worker can refine its cancelled report while retiring. Wait # for both actual workers to drain before freezing final responsibility. deadline = time.monotonic() + TASK_SECONDS - while status(probe, fleet, path, new, "force-new-drain").get("summary") == "running": + new_observations = [new_before, new_after] + while True: + draining = status(probe, fleet, path, new, "force-new-drain") + new_observations.append(draining) + if draining.get("summary") != "running": + break + counters(draining) require(time.monotonic() < deadline, "replacement task exceeded completion budget") time.sleep(0.05) idle_fleet(fleet, binding) @@ -263,7 +311,7 @@ def observe_branch(branch, probe, fleet, binding, bucket, expected, targets, act and all(settled["canonicalCounters"][name] >= initial["canonicalCounters"][name] for name in ("processed", "healed", "unchanged")), "cancelled refinement changed identity or moved backwards") terminal, new_proof = completed_scope(probe, fleet, path, new, bucket, primary, damaged, - healed=len(damaged) - settled["canonicalCounters"]["healed"], replacement=True) + healed=len(damaged) - settled["canonicalCounters"]["healed"], replacement=True, observations=new_observations) proof.update(newTaskId=new, initialCancellation=initial, settledCancellation=settled, newRunningBeforeOldStop=counters(new_before), newRunningAfterOldStop=counters(new_after), completeResponsibility=new_proof, responsibilityAccounting=force_accounting(settled, new_proof, damaged)) diff --git a/test_validation_heal.py b/test_validation_heal.py index c61c2045..0e582bac 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -2263,13 +2263,20 @@ def expected(self): def damaged(self): return {(self.expected()[index]["key"], "null") for index in overlap.MISSING} - def state(self, *, cancelled=False, replacement=False, healed=None): - total = 100 if cancelled else overlap.SCOPE_OBJECTS - repaired = {0, 64} if cancelled else set(overlap.MISSING) + def items(self): + return [{"bucket": self.bucket, "object": entry["key"], "versionId": "", "objectSize": CURSOR_SIZE} + for entry in self.expected()] + + def state(self, *, cancelled=False, replacement=False, healed=None, processed=None, min_seq=1): + total = processed if processed is not None else 100 if cancelled else overlap.SCOPE_OBJECTS + repaired = {index for index in overlap.MISSING if index < total} if replacement: repaired -= {0, 64} - return {"summary": "stopped" if cancelled else "finished", "outcome": { - "execution": {"state": "aborted", "reason": "cancelled"} if cancelled else {"state": "completed"}, - "coverage": "partial" if cancelled else "complete", "objectsTruncated": total > 128, + return {"summary": "stopped" if cancelled else "running" if processed is not None else "finished", + "nextSeq": total + 1, "minSeq": min_seq, "items": self.items()[min_seq - 1:total], + "truncated": min_seq > 1, "outcome": { + "execution": {"state": "aborted", "reason": "cancelled"} if cancelled else + {"state": "running" if processed is not None else "completed"}, + "coverage": "partial" if cancelled or processed is not None else "complete", "objectsTruncated": total > 128, "objects": [{"identity": {"bucket": self.bucket, "object": self.expected()[index]["key"], "versionId": None, "poolIndex": 0, "setIndex": 0, "bucketIncarnationId": "incarnation"}, "disposition": {"state": "repaired" if index in repaired else "verified_healthy"}} @@ -2320,6 +2327,7 @@ def test_force_accounting_preserves_partial_truth_and_rejects_double_repair(self def test_terminal_suffix_checks_every_ack_and_never_infers_a_retention_gap(self): current = self.state() + current.update(items=[], truncated=True) pages = [] for left, right in ((0, 100), (100, 257)): pages.append({**copy.deepcopy(current), "nextSeq": 258, "minSeq": 1, "truncated": right < 257, @@ -2342,6 +2350,131 @@ def test_terminal_suffix_checks_every_ack_and_never_infers_a_retention_gap(self) with self.subTest(kind=kind), patch.object(overlap, "status", return_value=current), self.assertRaises(ValueError): overlap.completed_scope(client, fleet, "/scope/", "task", self.bucket, self.expected(), self.damaged(), healed=5) + def test_saved_running_prefix_completes_a_byte_bounded_terminal_report(self): + current = self.state(min_seq=57) + client = types.SimpleNamespace(admin=lambda path, query, body: {"httpStatus": 200, + "response": {**copy.deepcopy(current), "items": []}}) + recorded = [] + fleet = types.SimpleNamespace(record=lambda label, value: recorded.append((label, value)) or value) + with patch.object(overlap, "status", return_value=current): + _, proof = overlap.completed_scope(client, fleet, "/scope/", "task", self.bucket, self.expected(), self.damaged(), + healed=5, observations=(self.state(processed=60),)) + self.assertEqual(proof["completeItemIdentities"], 257) + self.assertFalse(proof["retentionGapClaimed"]) + ledger = next(value for label, value in recorded if label == "overlap-complete-item-ledger") + self.assertEqual((ledger["path"], ledger["clientToken"]), ("/scope/", "task")) + self.assertEqual([entry["seq"] for entry in ledger["items"]], list(range(1, 258))) + client.admin = lambda *args: {"httpStatus": 200, "response": current} + for observations in ((), (self.state(processed=55),)): + with self.subTest(prefix=len(observations)), patch.object(overlap, "status", return_value=current), \ + self.assertRaisesRegex(ValueError, "uncollected early item"): + overlap.completed_scope(client, fleet, "/scope/", "task", self.bucket, self.expected(), self.damaged(), + healed=5, observations=observations) + + def test_item_ledger_rejects_rewritten_sequences_and_cross_scope_identity(self): + wanted = overlap.wanted_pairs(self.expected(), count=overlap.SCOPE_OBJECTS) + for kind in ("rewrite", "duplicate", "key", "version", "bucket", "size", "bounds", "silent-prefix", "silent-tail"): + ledger = {} + overlap.collect_scope_items(self.state(processed=60), self.bucket, wanted, ledger) + current = self.state() + if kind == "rewrite": current["items"][0]["detail"] = "changed" + if kind == "duplicate": current["items"][61] = copy.deepcopy(current["items"][0]) + if kind == "key": current["items"][0]["object"] = "objects/b/0000.bin" + if kind == "version": current["items"][0]["versionId"] = "another-version" + if kind == "bucket": current["items"][0]["bucket"] = "another-bucket" + if kind == "size": current["items"][0]["objectSize"] += 1 + if kind == "bounds": current["nextSeq"] = 259 + if kind == "silent-prefix": current = self.state(min_seq=57); current["truncated"] = False + if kind == "silent-tail": current["items"].pop() + with self.subTest(kind=kind), self.assertRaises(ValueError): + overlap.collect_scope_items(current, self.bucket, wanted, ledger) + + def test_cancelled_skip_requires_the_exact_typed_receipt_and_zero_unknowns(self): + current = self.state(cancelled=True) + current["outcome"]["objects"][-1]["disposition"]["state"] = "cancelled" + current["outcome"]["counters"].update(skipped=1, unchanged=97) + self.assertEqual(self.proof(current, cancelled=True)["canonicalCounters"]["skipped"], 1) + overlap.monotonic(self.state(processed=23), current, cancelled=True) + for kind in ("missing", "unknown", "count", "completed"): + changed = copy.deepcopy(current) + if kind == "missing": changed["outcome"]["objects"][-1]["disposition"]["state"] = "verified_healthy" + if kind == "unknown": changed["outcome"]["counters"]["unknown"] = 1 + if kind == "count": changed["outcome"]["counters"].update(skipped=2, unchanged=96) + if kind == "completed": changed["summary"] = "finished" + with self.subTest(kind=kind), self.assertRaises(ValueError): self.proof(changed, cancelled=True) + with self.assertRaises(ValueError): overlap.counters(current) + + def test_parallel_scope_is_sampled_during_every_completion_poll(self): + current = self.state() + client = types.SimpleNamespace(admin=lambda *args: {"httpStatus": 200, + "response": {**copy.deepcopy(current), "items": []}}) + fleet = types.SimpleNamespace(record=lambda label, value: value) + peer = unittest.mock.Mock() + with patch.object(overlap, "status", side_effect=(self.state(processed=60), current)), patch.object(overlap.time, "sleep"): + overlap.completed_scope(client, fleet, "/scope/", "task", self.bucket, self.expected(), self.damaged(), + healed=5, sample_other=peer) + self.assertEqual(peer.call_count, 2) + + def test_disjoint_and_replacement_keep_observations_bound_to_their_tokens(self): + primary = self.expected() + other = copy.deepcopy(primary) + for entry in other: entry["key"] = entry["key"].replace("objects/a/child/", "objects/b/") + targets = [entries[index] for entries in (primary, other) for index in overlap.MISSING] + before, after = self.state(processed=23), self.state(processed=40) + new_before, new_after = self.state(processed=70), self.state(processed=80) + old_final = self.state(cancelled=True) + samples = [self.state(processed=90), self.state()] + calls = [] + def complete(probe, fleet, path, token, bucket, expected, damaged, **options): + if options.get("sample_other"): + options["sample_other"](); options["sample_other"]() + calls.append((path, token, list(options["observations"]))) + state = self.state(replacement=options.get("replacement", False)) + return state, self.proof(state, healed=options["healed"], replacement=options.get("replacement", False)) + for branch in ("disjoint", "force-start"): + calls.clear() + active = [] + def admit(probe, fleet, path, identities, **options): + token = "new" if identities else "old" + identities.append((path, token)) + return token + def observe(probe, fleet, path, token, label): + if label == "original-after-disjoint-start": return after + if label == "overlap-disjoint-parallel-status": return samples[min(len(peer_samples), 1)] + if label in {"force-old-initial-cancellation", "force-old-settled"}: return old_final + if label == "force-new-after-old-stop": return new_after + if label == "force-new-drain": return samples.pop(0) + return self.state(replacement=branch == "force-start" and token == "new", cancelled=branch == "force-start" and token == "old") + peer_samples = [] + def record(label, value): + if label == "overlap-disjoint-parallel-status": peer_samples.append(value) + return value + # status() is mocked, so its recording hook is bypassed. + def observed(*args): + value = observe(*args) + if args[-1] == "overlap-disjoint-parallel-status": peer_samples.append(value) + return value + fleet = types.SimpleNamespace(nodes=["node"], record=record, + snapshot=lambda *args: {"node": {"pid": 1, "unit": "unit"}}) + probe = types.SimpleNamespace(admin=lambda *args: {}) + samples = [self.state(processed=90), self.state()] + with self.subTest(branch=branch), patch.object(overlap, "start", side_effect=admit), \ + patch.object(overlap, "wait_window", side_effect=(before, new_before)), \ + patch.object(overlap, "status", side_effect=observed), patch.object(overlap, "completed_scope", side_effect=complete), \ + patch.object(overlap, "idle_fleet", return_value={}), patch.object(overlap, "validate_unknown_token", return_value={}), \ + patch.object(overlap, "physical_content", return_value={}), patch.object(overlap.time, "sleep"): + overlap.observe_branch(branch, probe, fleet, {}, self.bucket, primary + other, targets, active, original={}) + self.assertEqual(active, []) + self.assertEqual(calls[-1][1], "new") + if branch == "disjoint": + self.assertTrue(calls[0][0].endswith("objects/a/")) + self.assertTrue(calls[-1][0].endswith("objects/b/")) + self.assertEqual(calls[0][2], [before, after]) + self.assertEqual(calls[-1][2], [new_before] + peer_samples) + else: + self.assertEqual(calls[-1][2][:2], [new_before, new_after]) + self.assertEqual(len(calls[-1][2]), 4) + def test_finished_window_and_same_token_counter_rollback_cannot_pass(self): self.assertFalse(overlap.active_window(self.state())) before = self.state(cancelled=True) From 0ba4d6cb4df4e86f3042a5dda4fc79eef5651760 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 12:38:56 +0800 Subject: [PATCH 165/343] docs(validation): update canonical pass census --- docs/validation-migration-map.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 5461a290..643b1b3d 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 89 unique complete canonical live PASS cases: 77 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, and A03 run `38198122041`, are included. Native, read-only, recovery and duplicate runs are excluded; the other 69 cases have no complete canonical live PASS claim. +case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 90 unique complete canonical live PASS cases: 78 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, and C04 run `38203454300`, are included. Native, read-only, recovery and duplicate runs are excluded; the other 68 cases have no complete canonical live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1909,3 +1909,6 @@ J06 retry `38195131282` completed all twelve fixed ABBA phases with 1,440 schedu A03 run `38198122041` passes every required H2 cross-node branch at helper `ccef050df3b28416c51356951e78cb46dd74d10b` and driver `03fb3d78a0ffc4213583dd3bcb44c7ac38f42ead`. Independent verification of the original artifact members, canonical result gate, complete branch outcomes and immutable candidate provenance increases the unique canonical census to 89. The remaining serialized backlog continues on the same candidate; no prior failure is overwritten. J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite and its branch is explicitly BLOCKED pending a reviewed live unknown/lock-deferred fixture. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the source assertion requiring a mixed deferred population is still outstanding. The 84 cases, 301 branches and 532 topology templates are unchanged. + + +The corrected live census now contains 90 unique complete canonical PASS cases: 78 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926` and 12 on the older immutable source. C04 `heal.truncated-and-zeroed-shard` run `38203454300` was independently audited with clean finalization and is retained alongside A03. The derived ledger records 34 retained live failures, 7 required native barriers, 10 cases without complete audited live evidence, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. From 3018a69563b5ef5a828daf9051290ac6b91fba99 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 12:54:43 +0800 Subject: [PATCH 166/343] docs(validation): record multipart canonical pass --- docs/validation-migration-map.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 643b1b3d..63c1e264 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 90 unique complete canonical live PASS cases: 78 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, and C04 run `38203454300`, are included. Native, read-only, recovery and duplicate runs are excluded; the other 68 cases have no complete canonical live PASS claim. +case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 91 unique complete canonical live PASS cases: 79 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, and the completed multipart run `38204339623`, are included. Native, read-only, recovery and duplicate runs are excluded; the other 67 cases have no complete canonical live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1911,4 +1911,4 @@ A03 run `38198122041` passes every required H2 cross-node branch at helper `ccef J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite and its branch is explicitly BLOCKED pending a reviewed live unknown/lock-deferred fixture. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the source assertion requiring a mixed deferred population is still outstanding. The 84 cases, 301 branches and 532 topology templates are unchanged. -The corrected live census now contains 90 unique complete canonical PASS cases: 78 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926` and 12 on the older immutable source. C04 `heal.truncated-and-zeroed-shard` run `38203454300` was independently audited with clean finalization and is retained alongside A03. The derived ledger records 34 retained live failures, 7 required native barriers, 10 cases without complete audited live evidence, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. +The corrected live census now contains 91 unique complete canonical PASS cases: 79 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926` and 12 on the older immutable source. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 36 retained live failures, 7 required native barriers, 7 cases without complete audited live evidence, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. From 0ada86f43236d0f9bef118151b0a5f5997dd2e9d Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 13:00:42 +0800 Subject: [PATCH 167/343] test(validation): bind encrypted heal to reviewed candidates --- scripts/validation_heal/encrypted.py | 15 +++++++++++++-- test_validation_heal_encrypted.py | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/scripts/validation_heal/encrypted.py b/scripts/validation_heal/encrypted.py index 2ab6e0f7..600e953e 100644 --- a/scripts/validation_heal/encrypted.py +++ b/scripts/validation_heal/encrypted.py @@ -43,6 +43,16 @@ COUNT = 16 SIZE = 5 * 1024**2 + 123 CANDIDATE = "072b57b37559fbe626aeda8abf61617fdcbdd926" +REVIEWED_CANDIDATES = { + CANDIDATE: {}, + # This source keeps the original storage/KMS geometry and fixes the + # customer-key PUT/HEAD/GET response projection without changing the oracle. + "16cf19817e0c81e24d0d787a916f734a34517ec5": { + "rustfs/src/app/object/put.rs": "01256c2995f04845840d28c8434cdfaca072766f88693a7d8cb8d24ed78463c7", + "rustfs/src/app/object/head.rs": "d3ed1e0dcb548dbcf80769fb6ea8e37912e76a59c85ae75628baea36dab542f7", + "rustfs/src/app/object/get.rs": "e2ce8101de7e50b4941bb6ecca36440f439cc3212d2be50f79380d212d2ed704", + }, +} REFERENCE = {"candidateCommit": CANDIDATE, "dependency": "fresh run-owned static AES-256 KMS key on every candidate node", "sourceFiles": { "crates/kms/src/config.rs": "7931bb4c6757e60ca3e7fdbff4ac56c5fab59d6d6df8756c9d0287ca692a26f9", @@ -184,9 +194,10 @@ def run_branch(args, topology, nodes, branch): run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" fleet = Fleet(nodes, directory, run_id, args.tool_directory) candidate = json.loads(args.candidate.read_text()) - require(candidate["source_sha"] == CANDIDATE, "unreviewed encrypted geometry/KMS policy contract") + require(candidate["source_sha"] in REVIEWED_CANDIDATES, "unreviewed encrypted geometry/KMS policy contract") fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) - fleet.record("encryption-contract", REFERENCE) + fleet.record("encryption-contract", {**REFERENCE, "candidateCommit": candidate["source_sha"], + "reviewedResponseSourceFiles": REVIEWED_CANDIDATES[candidate["source_sha"]]}) bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] key_id, material = "rv-heal-key-" + run_id, base64.b64encode(secrets.token_bytes(32)).decode() access, secret = secrets.token_hex(16), secrets.token_hex(24) diff --git a/test_validation_heal_encrypted.py b/test_validation_heal_encrypted.py index aa9390b3..2709502e 100644 --- a/test_validation_heal_encrypted.py +++ b/test_validation_heal_encrypted.py @@ -19,6 +19,24 @@ class EncryptedObjectTests(unittest.TestCase): + def test_reviewed_candidate_requires_its_own_inspector_before_fleet_deploy(self): + for commit in (*encrypted.REVIEWED_CANDIDATES, "f" * 40): + with self.subTest(commit=commit), tempfile.TemporaryDirectory() as folder: + root = Path(folder) + candidate = {"source_sha": commit} + (root / "candidate.json").write_text(json.dumps(candidate)) + args = types.SimpleNamespace(evidence=root, candidate=root / "candidate.json", tool_directory=root / "tools") + with patch.dict(os.environ, {"GITHUB_RUN_ID": "12345"}), patch.object(encrypted, "Fleet") as fleet, \ + patch.object(encrypted, "verify_inspector", side_effect=ValueError("inspector boundary")) as inspector: + message = "inspector boundary" if commit in encrypted.REVIEWED_CANDIDATES else "unreviewed encrypted" + with self.assertRaisesRegex(ValueError, message): + encrypted.run_branch(args, "H1", ["node1"], "sse-c") + fleet.return_value.deploy.assert_not_called() + if commit in encrypted.REVIEWED_CANDIDATES: + inspector.assert_called_once_with(args.tool_directory, candidate) + else: + inspector.assert_not_called() + def geometry(self): before = shard_fixtures.HealShardOracleTest().snapshot() key, bucket = "objects/0008.bin", "rv-heal-0123456789abcdef" From b05979dbf8a6a5c888d6819e3b63aa8b8b9688a4 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 13:17:24 +0800 Subject: [PATCH 168/343] test(validation): bind manager regressions to candidate source --- scripts/validation_heal/manager_native.py | 37 ++++++++++++++++------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/scripts/validation_heal/manager_native.py b/scripts/validation_heal/manager_native.py index 649044f9..8d58b347 100644 --- a/scripts/validation_heal/manager_native.py +++ b/scripts/validation_heal/manager_native.py @@ -19,6 +19,16 @@ SOURCE_TREE = "8071fb0a9e5353eb7d3fa58c8a58c8622473f2bd" LOCK_SHA = "d8f7bc416ba67d19873d34e9389e5aa823b22d7d7552706a5a810cee26f6eda2" +REVIEWED_CANDIDATES = { + native.CANDIDATE: {"sourceTree": SOURCE_TREE, "cargoLockSha256": LOCK_SHA}, + # The manager, checkpoint, MRF and outcome sources are byte-identical to + # the original reviewed candidate; this source adds the validation fixes + # elsewhere in the RustFS tree. + "16cf19817e0c81e24d0d787a916f734a34517ec5": { + "sourceTree": "7edb1a2d02d60323cdfab4173485c33cb06b297a", + "cargoLockSha256": LOCK_SHA, + }, +} SOURCE_FILES = { "crates/heal/src/heal/resume/tests.rs": "cea1558f8659f01968a5592af55cd0626f1d309c9fecfa0bee26c36775229900", "crates/heal/src/heal/resume/checkpoint.rs": "fe067b39ee406e16517bd665f7f49457daffa05d5d7bc298bd965e927b1381c1", @@ -92,8 +102,11 @@ def collectors(): def verify_manifest(directory): manifest = json.loads((directory / "manager-native.json").read_text()) - native.require(manifest.get("schemaVersion") == 1 and manifest.get("candidateCommit") == native.CANDIDATE - and manifest.get("candidateSourceTree") == SOURCE_TREE and manifest.get("candidateCargoLockSha256") == LOCK_SHA + candidate = manifest.get("candidateCommit") + contract = REVIEWED_CANDIDATES.get(candidate) + native.require(manifest.get("schemaVersion") == 1 and contract is not None + and manifest.get("candidateSourceTree") == contract["sourceTree"] + and manifest.get("candidateCargoLockSha256") == contract["cargoLockSha256"] and manifest.get("sourceFiles") == SOURCE_FILES and manifest.get("collectorSources") == collectors() and manifest.get("regressions") == REGRESSIONS and manifest.get("sourceUnmodified") is True and manifest.get("optimized") is True and manifest.get("buildEnvironment") == BUILD_ENV @@ -106,11 +119,12 @@ def verify_manifest(directory): def build(source, commit, out): - native.require(commit == native.CANDIDATE and subprocess.check_output( + contract = REVIEWED_CANDIDATES.get(commit) + native.require(contract is not None and subprocess.check_output( ["git", "-C", str(source), "rev-parse", "HEAD"], text=True).strip() == commit, "manager native tests require the exact reviewed candidate") - native.require(subprocess.check_output(["git", "-C", str(source), "rev-parse", "HEAD^{tree}"], text=True).strip() == SOURCE_TREE - and native.sha(source / "Cargo.lock") == LOCK_SHA and all(native.sha(source / path) == digest for path, digest in SOURCE_FILES.items()), + native.require(subprocess.check_output(["git", "-C", str(source), "rev-parse", "HEAD^{tree}"], text=True).strip() == contract["sourceTree"] + and native.sha(source / "Cargo.lock") == contract["cargoLockSha256"] and all(native.sha(source / path) == digest for path, digest in SOURCE_FILES.items()), "manager candidate source/tree/lock differs") for args in (["diff", "--quiet"], ["diff", "--cached", "--quiet"]): native.require(subprocess.run(["git", "-C", str(source), *args]).returncode == 0, "candidate source is modified") @@ -118,8 +132,8 @@ def build(source, commit, out): "candidate contains untracked source/configuration") native.require(not out.exists() and shutil.disk_usage(source).free >= 12 * 1024**3, "fresh output and 12 GiB build space required") out.mkdir(parents=True) - request = {"schemaVersion": 1, "candidateCommit": commit, "candidateSourceTree": SOURCE_TREE, - "candidateCargoLockSha256": LOCK_SHA, "sourceFiles": SOURCE_FILES, "collectorSources": collectors(), + request = {"schemaVersion": 1, "candidateCommit": commit, "candidateSourceTree": contract["sourceTree"], + "candidateCargoLockSha256": contract["cargoLockSha256"], "sourceFiles": SOURCE_FILES, "collectorSources": collectors(), "regressions": REGRESSIONS, "testingCommit": os.environ.get("AUTO_TESTING_SHA"), "runId": os.environ.get("GITHUB_RUN_ID"), "runAttempt": os.environ.get("GITHUB_RUN_ATTEMPT"), "fleetPassClaimed": False, "sourceUnmodified": True, "optimized": True, "buildEnvironment": BUILD_ENV} @@ -165,7 +179,7 @@ def run(directory, out): native.exact_listing(listing.stdout, value["test"]) verify_manifest(directory) command = [str(binary), value["test"], "--exact", "--nocapture", "--test-threads=1"] - record = {"regression": key, **value, "candidateCommit": native.CANDIDATE, + record = {"regression": key, **value, "candidateCommit": manifest["candidateCommit"], "binarySha256": manifest["binarySha256"], "command": command} started = time.monotonic_ns() try: @@ -191,7 +205,8 @@ def run(directory, out): def verify_evidence(directory, evidence, candidate, run_id, run_attempt, helper_sha): manifest, _ = verify_manifest(directory) - native.require(candidate["source_sha"] == native.CANDIDATE and manifest.get("runId") == str(run_id) + native.require(candidate["source_sha"] == manifest.get("candidateCommit") + and manifest.get("candidateCommit") in REVIEWED_CANDIDATES and manifest.get("runId") == str(run_id) and manifest.get("runAttempt") == str(run_attempt) and manifest.get("testingCommit") == helper_sha and re.fullmatch(r"[0-9a-f]{40}", helper_sha), "manager native evidence belongs to another build attempt") summary = json.loads((evidence / "manager-native-summary.json").read_text()) @@ -204,14 +219,14 @@ def verify_evidence(directory, evidence, candidate, run_id, run_attempt, helper_ for row in rows: key = row["regression"] native.require(row == json.loads((evidence / (key + ".json")).read_text()) and row.get("verdict") == "PASS" - and row.get("binarySha256") == manifest["binarySha256"] and row.get("candidateCommit") == native.CANDIDATE + and row.get("binarySha256") == manifest["binarySha256"] and row.get("candidateCommit") == manifest["candidateCommit"] and all(row.get(name) == value for name, value in REGRESSIONS[key].items()), "manager result changed exact scope or identity") command = row.get("command", []) native.require(len(command) == 5 and Path(command[0]).name == manifest["binaryName"] and command[1:] == [row["test"], "--exact", "--nocapture", "--test-threads=1"], "manager result lacks exact one-test invocation") for listing in listings: native.exact_listing(listing, row["test"]) native.test_result((evidence / (key + ".log")).read_text(), row["test"], row.get("exitCode")) - return {"complete": True, "candidateCommit": native.CANDIDATE, "binarySha256": manifest["binarySha256"], + return {"complete": True, "candidateCommit": manifest["candidateCommit"], "binarySha256": manifest["binarySha256"], "exactTests": len(rows), "canonicalCasePassClaimed": False, "fleetPassClaimed": False} From 85763607b04f887391c2bd1136478895e2e7756f Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 13:27:13 +0800 Subject: [PATCH 169/343] test(validation): bind native evidence to each candidate --- docs/validation-migration-map.md | 6 ++-- test_validation_heal_manager_native.py | 43 +++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 63c1e264..dce78033 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 91 unique complete canonical live PASS cases: 79 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, and the completed multipart run `38204339623`, are included. Native, read-only, recovery and duplicate runs are excluded; the other 67 cases have no complete canonical live PASS claim. +case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 93 unique complete canonical live PASS cases: 81 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, and D09 run `38205854024` are included. Native, read-only, recovery and duplicate runs are excluded; the other 65 cases have no complete canonical live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1911,4 +1911,6 @@ A03 run `38198122041` passes every required H2 cross-node branch at helper `ccef J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite and its branch is explicitly BLOCKED pending a reviewed live unknown/lock-deferred fixture. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the source assertion requiring a mixed deferred population is still outstanding. The 84 cases, 301 branches and 532 topology templates are unchanged. -The corrected live census now contains 91 unique complete canonical PASS cases: 79 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926` and 12 on the older immutable source. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 36 retained live failures, 7 required native barriers, 7 cases without complete audited live evidence, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. +The corrected live census now contains 93 unique complete canonical PASS cases: 81 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926` and 12 on the older immutable source. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 36 retained live failures, 7 required native barriers, 5 cases without complete audited live evidence, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. + +D07 `heal.range-head-and-conditions` run `38205235080` and D09 `heal.object-lock-retention` run `38205854024` passed every required source branch and were independently audited against the original canonical results, referenced member digests and immutable provenance. Both finalizations passed. A distinct validation-channel package was built successfully in run `38204362745/1` from source `16cf19817e0c81e24d0d787a916f734a34517ec5`, with package SHA-256 `531965f2c009a5a6ef1c02cc6aa1ba82a82b6fbe4aa441d89613afc8a3f91d40`. It contains the typed bucket-heal error and SSE-C response fixes. The D08 and manager native collectors explicitly bind this reviewed source and its actual source tree; the original candidate remains separately supported. Evidence for one candidate cannot certify the other. No new-candidate canonical PASS is claimed before complete fresh live execution. diff --git a/test_validation_heal_manager_native.py b/test_validation_heal_manager_native.py index 5a27d8c3..96170f4c 100644 --- a/test_validation_heal_manager_native.py +++ b/test_validation_heal_manager_native.py @@ -11,15 +11,16 @@ class ManagerNativeEvidenceTests(unittest.TestCase): - def fixture(self, root): + def fixture(self, root, candidate=suite.native.CANDIDATE): tools, evidence = root / "tools", root / "evidence" tools.mkdir(); evidence.mkdir() helper = "a" * 40 binary = tools / "heal-manager-native-tests" binary.write_bytes(b"original candidate unit executable") digest = hashlib.sha256(binary.read_bytes()).hexdigest() - manifest = {"schemaVersion": 1, "candidateCommit": suite.native.CANDIDATE, - "candidateSourceTree": suite.SOURCE_TREE, "candidateCargoLockSha256": suite.LOCK_SHA, + contract = suite.REVIEWED_CANDIDATES[candidate] + manifest = {"schemaVersion": 1, "candidateCommit": candidate, + "candidateSourceTree": contract["sourceTree"], "candidateCargoLockSha256": contract["cargoLockSha256"], "sourceFiles": suite.SOURCE_FILES, "collectorSources": suite.collectors(), "regressions": suite.REGRESSIONS, "sourceUnmodified": True, "optimized": True, "buildEnvironment": suite.BUILD_ENV, "binaryName": binary.name, "binarySha256": digest, @@ -30,7 +31,7 @@ def fixture(self, root): (evidence / "runtime-inventory.log").write_text(listing) rows = [] for key, value in suite.REGRESSIONS.items(): - row = {"regression": key, **value, "candidateCommit": suite.native.CANDIDATE, + row = {"regression": key, **value, "candidateCommit": candidate, "binarySha256": digest, "verdict": "PASS", "exitCode": 0, "elapsedNanoseconds": 100, "command": ["/hosted/tools/" + binary.name, value["test"], "--exact", "--nocapture", "--test-threads=1"]} (evidence / (key + ".json")).write_text(json.dumps(row)) @@ -40,7 +41,7 @@ def fixture(self, root): summary = {"results": rows, "complete": True, "canonicalCasePassClaimed": False, "fleetPassClaimed": False, "physicalPowerLossClaimed": False} (evidence / "manager-native-summary.json").write_text(json.dumps(summary)) - return tools, evidence, {"source_sha": suite.native.CANDIDATE}, helper + return tools, evidence, {"source_sha": candidate}, helper def verify(self, fixture, **changes): tools, evidence, candidate, helper = fixture @@ -54,6 +55,38 @@ def test_complete_exact_native_evidence_preserves_separate_scope(self): self.assertFalse(result["canonicalCasePassClaimed"]) self.assertFalse(result["fleetPassClaimed"]) + def test_each_reviewed_candidate_requires_its_own_tree_and_results(self): + for candidate in suite.REVIEWED_CANDIDATES: + with self.subTest(candidate=candidate), tempfile.TemporaryDirectory() as temp: + result = self.verify(self.fixture(Path(temp), candidate)) + self.assertEqual(result["candidateCommit"], candidate) + self.assertEqual(result["exactTests"], 15) + self.assertFalse(result["canonicalCasePassClaimed"]) + + def test_old_tree_cannot_be_relabelled_as_new_candidate(self): + newer = next(commit for commit in suite.REVIEWED_CANDIDATES if commit != suite.native.CANDIDATE) + with tempfile.TemporaryDirectory() as temp: + fixture = self.fixture(Path(temp), newer) + path = fixture[0] / "manager-native.json" + manifest = json.loads(path.read_text()) + manifest["candidateSourceTree"] = suite.SOURCE_TREE + path.write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, "reviewed source"): + self.verify(fixture) + + def test_old_result_rows_cannot_certify_new_candidate_manifest(self): + newer = next(commit for commit in suite.REVIEWED_CANDIDATES if commit != suite.native.CANDIDATE) + with tempfile.TemporaryDirectory() as temp: + fixture = self.fixture(Path(temp), newer) + summary_path = fixture[1] / "manager-native-summary.json" + summary = json.loads(summary_path.read_text()) + for row in summary["results"]: + row["candidateCommit"] = suite.native.CANDIDATE + (fixture[1] / (row["regression"] + ".json")).write_text(json.dumps(row)) + summary_path.write_text(json.dumps(summary)) + with self.assertRaisesRegex(ValueError, "exact scope or identity"): + self.verify(fixture) + def test_wrong_candidate_build_attempt_or_helper_is_rejected(self): with tempfile.TemporaryDirectory() as temp: fixture = self.fixture(Path(temp)) From e938e1e1d9a0990f80a88f231bb32e9cc9bf0109 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 13:35:25 +0800 Subject: [PATCH 170/343] docs(validation): update canonical migration census --- docs/validation-migration-map.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index dce78033..e3d25750 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 93 unique complete canonical live PASS cases: 81 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, and D09 run `38205854024` are included. Native, read-only, recovery and duplicate runs are excluded; the other 65 cases have no complete canonical live PASS claim. +case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 95 unique complete canonical live PASS cases: 83 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, and D09 run `38205854024`, D10 run `38207381816`, and manual-normal attribution run `38207956101` are included. Native, read-only, recovery and duplicate runs are excluded; the other 63 cases have no complete canonical live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1911,6 +1911,6 @@ A03 run `38198122041` passes every required H2 cross-node branch at helper `ccef J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite and its branch is explicitly BLOCKED pending a reviewed live unknown/lock-deferred fixture. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the source assertion requiring a mixed deferred population is still outstanding. The 84 cases, 301 branches and 532 topology templates are unchanged. -The corrected live census now contains 93 unique complete canonical PASS cases: 81 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926` and 12 on the older immutable source. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 36 retained live failures, 7 required native barriers, 5 cases without complete audited live evidence, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. +The corrected live census now contains 95 unique complete canonical PASS cases: 83 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926` and 12 on the older immutable source. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 36 retained live failures, 7 required native barriers, 5 cases without complete audited live evidence, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. -D07 `heal.range-head-and-conditions` run `38205235080` and D09 `heal.object-lock-retention` run `38205854024` passed every required source branch and were independently audited against the original canonical results, referenced member digests and immutable provenance. Both finalizations passed. A distinct validation-channel package was built successfully in run `38204362745/1` from source `16cf19817e0c81e24d0d787a916f734a34517ec5`, with package SHA-256 `531965f2c009a5a6ef1c02cc6aa1ba82a82b6fbe4aa441d89613afc8a3f91d40`. It contains the typed bucket-heal error and SSE-C response fixes. The D08 and manager native collectors explicitly bind this reviewed source and its actual source tree; the original candidate remains separately supported. Evidence for one candidate cannot certify the other. No new-candidate canonical PASS is claimed before complete fresh live execution. +D07 `heal.range-head-and-conditions` run `38205235080`, D09 `heal.object-lock-retention` run `38205854024`, D10 `heal.special-keys-and-pagination` run `38207381816`, and `heal.manual-normal-attribution` run `38207956101` passed every required source branch and were independently audited against the original canonical results, referenced member digests and immutable provenance. Both finalizations passed. A distinct validation-channel package was built successfully in run `38204362745/1` from source `16cf19817e0c81e24d0d787a916f734a34517ec5`, with package SHA-256 `531965f2c009a5a6ef1c02cc6aa1ba82a82b6fbe4aa441d89613afc8a3f91d40`. It contains the typed bucket-heal error and SSE-C response fixes. The D08 and manager native collectors explicitly bind this reviewed source and its actual source tree; the original candidate remains separately supported. Evidence for one candidate cannot certify the other. No new-candidate canonical PASS is claimed before complete fresh live execution. From 6189cecf9b7f418ded0f0ddc09c91b6e512cf6b3 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 16:58:21 +0800 Subject: [PATCH 171/343] test(validation): preserve corrupt checkpoint responsibility --- .../workflows/validation-catalog-tests.yml | 3 + docs/validation-migration-map.md | 12 +- scripts/validation_heal/persistence.py | 350 ++++++++++++++++++ test_validation_heal_persistence.py | 198 ++++++++++ 4 files changed, 561 insertions(+), 2 deletions(-) create mode 100644 scripts/validation_heal/persistence.py create mode 100644 test_validation_heal_persistence.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index fde52c54..e525a60e 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -46,6 +46,7 @@ on: - test_validation_heal_race.py - test_validation_heal_environment_gate.py - test_validation_heal_outcome.py + - test_validation_heal_persistence.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -105,6 +106,7 @@ on: - test_validation_heal_race.py - test_validation_heal_environment_gate.py - test_validation_heal_outcome.py + - test_validation_heal_persistence.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -145,6 +147,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py + - run: python3 -B -m unittest -v test_validation_heal_persistence.py - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh - run: bash test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index e3d25750..949e9f05 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 95 unique complete canonical live PASS cases: 83 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, and D09 run `38205854024`, D10 run `38207381816`, and manual-normal attribution run `38207956101` are included. Native, read-only, recovery and duplicate runs are excluded; the other 63 cases have no complete canonical live PASS claim. +case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 96 unique complete canonical live PASS cases: 84 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, and D09 run `38205854024`, D10 run `38207381816`, and manual-normal attribution run `38207956101` are included. Native, read-only, recovery and duplicate runs are excluded; the other 62 cases have no complete canonical live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1911,6 +1911,14 @@ A03 run `38198122041` passes every required H2 cross-node branch at helper `ccef J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite and its branch is explicitly BLOCKED pending a reviewed live unknown/lock-deferred fixture. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the source assertion requiring a mixed deferred population is still outstanding. The 84 cases, 301 branches and 532 topology templates are unchanged. -The corrected live census now contains 95 unique complete canonical PASS cases: 83 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926` and 12 on the older immutable source. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 36 retained live failures, 7 required native barriers, 5 cases without complete audited live evidence, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. +The corrected live census now contains 96 unique complete canonical PASS cases: 84 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926` and 12 on the older immutable source. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 38 retained live failures, 7 required native barriers, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. D07 `heal.range-head-and-conditions` run `38205235080`, D09 `heal.object-lock-retention` run `38205854024`, D10 `heal.special-keys-and-pagination` run `38207381816`, and `heal.manual-normal-attribution` run `38207956101` passed every required source branch and were independently audited against the original canonical results, referenced member digests and immutable provenance. Both finalizations passed. A distinct validation-channel package was built successfully in run `38204362745/1` from source `16cf19817e0c81e24d0d787a916f734a34517ec5`, with package SHA-256 `531965f2c009a5a6ef1c02cc6aa1ba82a82b6fbe4aa441d89613afc8a3f91d40`. It contains the typed bucket-heal error and SSE-C response fixes. The D08 and manager native collectors explicitly bind this reviewed source and its actual source tree; the original candidate remains separately supported. Evidence for one candidate cannot certify the other. No new-candidate canonical PASS is claimed before complete fresh live execution. + +Manual-deep attribution run `38208251180` passed every required source branch and independent canonical audit with clean finalization. Hosted tools run `38207334642/1` also completed: the new candidate inspector and all fifteen exact manager regression results passed local independent artifact verification. Source tree `7edb1a2d02d60323cdfab4173485c33cb06b297a`, candidate lock, collector source digests, original logs and exact one-test invocations were checked. These native results are scoped as ten F07, three C11 and two J02 prerequisites and add no canonical PASS. + +F07 now has a non-dispatched partial orchestrator in `persistence.py` for actual checkpoint truncation and future schema on H1/H2 with DB/DV fixtures. It admits an explicit Admin erasure-set task, observes real durable progress, freezes and crashes the exact process, validates stopped root/checkpoint ownership, and mutates only that original checkpoint. Restart must preserve corrupt bytes and resume ownership, emit the blocked fence, and retain a durable original failure terminal/report. A distinct rescan is checked against complete ACK/physical/S3 manifests without reclassifying the original failed token. Original corrupt record and failure-report bytes must remain unchanged through rescan and business reads; cleanup restores only unchanged injected journal bytes. The MRF checksum, earlier-generation replay, bounded journal-write failure and exact persist-fence branches remain explicitly BLOCKED, and F07 stays outside automatic dispatch. No F07 live result is claimed. Nine new contract tests reject changed records, missing or substituted responsibility, silent success and invented counters; 61 focused persistence/native prerequisite tests pass, and the 84/301/532 matrix remains unchanged. + +D04 run `38204764944` has a digest-bound diagnostic comparing the stopped stale-authority scene with the pre-manual-admission witness: only the target member xl.meta changes on each of H1 and H2, adding the exact newly acknowledged marker. Every other file and disk format remains unchanged. The candidate keeps durable MRF replay active when best-effort ingress is disabled; exact source-task attribution still needs investigation. The manual attribution gate and original FAIL remain intact. B09 run `38202851374` likewise has a complete identity-difference diagnostic: no expected ACK identity is missing; vm003 retains deleted `objects/0006.bin` null data in DB and historical `objects/0001.bin` version `de87efd5-e1ef-42a2-a86a-a1e7a18fe46c` in DV on all four disks. External resurrection is not established. + +The seventeen-case corrected live queue completed with eight PASS and nine retained FAIL results; every finalization passed. J01 run `38209400619` did not witness the full raw/rc uninitialized startup window on H1, and H2 rejected rc text that erased status uncertainty during convergence. Both disabled branches passed. E03 run `38208556719` retained range-repair physical differences and missing fresh ReadRepair attribution in H2. These partial results do not change either canonical verdict. diff --git a/scripts/validation_heal/persistence.py b/scripts/validation_heal/persistence.py new file mode 100644 index 00000000..281d1deb --- /dev/null +++ b/scripts/validation_heal/persistence.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""F07 checkpoint corruption prerequisites; incomplete branches stay blocked.""" + +import argparse +import copy +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import time + +try: + from . import checkpoint as checkpoints, journal_fault as journal + from .heal_probe import Probe, canonical, compare_inventories, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .manager_fixture import verify as verify_manager + from .matrix import select + from .shards import files, map_target, run_task, validate_fault, validate_repaired_physical + from .stop import idle_fleet, status + from .root_restart import recovery_status + from .versions import public_version_id, verify_proof, version_identity +except ImportError: + import checkpoint as checkpoints, journal_fault as journal + from heal_probe import Probe, canonical, compare_inventories, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from manager_fixture import verify as verify_manager + from matrix import select + from shards import files, map_target, run_task, validate_fault, validate_repaired_physical + from stop import idle_fleet, status + from root_restart import recovery_status + from versions import public_version_id, verify_proof, version_identity + + +CASE_ID = "heal.checkpoint-and-mrf-corruption" +BRANCHES = ("checkpoint-truncate", "mrf-checksum", "future-schema", "stale-generation", + "journal-write-failure", "native-persist-fence") +IMPLEMENTED = {"checkpoint-truncate": "truncate", "future-schema": "future-schema"} +BLOCKERS = { + "mrf-checksum": "H1/H2 ACK-bound committed MRF corruption, retention and replay orchestration is incomplete", + "stale-generation": "an actual earlier checkpoint capture exists; complete rollback and durable responsibility reconciliation is still required", + "journal-write-failure": "the private bounded volume exists; actual checkpoint write refusal, retained ownership and release/rescan orchestration is still required", + "native-persist-fence": "publication interruption prerequisites do not prove every required checkpoint/MRF fsync and commit interruption boundary", +} +SIZE = 5 * 1024**2 + 123 + + +def exact_records(observed, token, family): + names = journal.names(family, token) + require(isinstance(observed, dict) and len(observed) in {1, 4}, "journal omitted bound physical nodes") + present = {} + for node, census in observed.items(): + rows = census.get("records") + require(census.get("readOnly") is True and census.get("quiescent") is True + and census.get("family") == family and census.get("taskId") == token + and census.get("directoryEnumerationUsed") is False + and isinstance(rows, list) and len(rows) == 4 * len(names), + "retention proof needs a stopped exact-path census") + seen = set() + for row in rows: + pair = row.get("disk"), row.get("relativePath") + require(type(pair[0]) is int and 0 <= pair[0] < 4 and pair[1] in names and pair not in seen + and type(row.get("present")) is bool, "journal census duplicated or escaped an exact disk/path") + seen.add(pair) + if row["present"]: + checkpoints.bytes_of(row) + require(pair[1] not in present, "journal record has multiple physical owners") + present[pair[1]] = {"node": node, **row} + else: + require(not any(name in row for name in ("size", "sha256", "rawBase64")), + "absent journal record contains invented bytes") + return present + + +def failed_task(current): + outcome = current.get("outcome") or {} + require(current.get("summary") == "stopped" and current.get("settings") == checkpoints.OPTIONS + and outcome.get("execution") == {"state": "aborted", "reason": "untraversable"} + and outcome.get("coverage") == "partial" + and isinstance(current.get("detail"), str) and "checkpoint" in current["detail"].lower(), + "corrupt original checkpoint was silently acknowledged, cancelled or lost its diagnostic") + counters = outcome.get("counters") or {} + require(all(type(counters.get(name)) is int and counters[name] == 0 for name in + ("processed", "healed", "unchanged", "skipped", "failed", "unknown", "attemptFailures")) + and counters.get("overflowed") is False and outcome.get("objects") == [] + and outcome.get("objectsTruncated") is False, + "checkpoint load failure invented object processing or repair") + + +def retained_corruption(observed, roots, frozen, receipt, current): + token = frozen["taskId"] + failed_task(current) + require(receipt.get("applied") is True and receipt.get("originalBackupRetained") is True + and receipt.get("productRepairClaimed") is False + and receipt.get("family") == "checkpoint" and receipt.get("taskId") == token + and receipt.get("disk") == frozen["disk"] + and receipt.get("relativePath") == frozen["checkpoint"]["relativePath"] + and journal.same_json(receipt.get("before"), journal.entry(checkpoints.bytes_of(frozen["checkpoint"]))), + "journal injection receipt changed the frozen original task or bytes") + require(receipt.get("kind") in IMPLEMENTED.values() + and receipt.get("injected") == journal.entry(journal.transform( + checkpoints.bytes_of(frozen["checkpoint"]), receipt, {"bucket": frozen["bucket"]}, None)), + "journal corruption differs from its reviewed transformation") + records = exact_records(observed, token, "checkpoint") + names = journal.names("checkpoint", token) + require(set(records) == set(names[:3]), "corrupt checkpoint/resume/blocked fence was deleted or unexpectedly replaced") + for row in records.values(): + require((row["node"], row["disk"]) == (frozen["node"], frozen["disk"]), + "corrupt checkpoint or diagnostic moved to another physical owner") + require(journal.entry(checkpoints.bytes_of(records[names[0]])) == receipt.get("injected") + and checkpoints.bytes_of(records[names[1]]) == checkpoints.bytes_of(frozen["resume"]) + and checkpoints.bytes_of(records[names[2]]) == b"blocked", + "product overwrote unknown bytes, changed resume ownership or omitted its blocked diagnostic") + root = exact_records(roots, token, "root") + root_names = journal.names("root", token) + # An explicit durable failed terminal may retire the runnable admission. + # The corruption and blocked checkpoint remain independently retained. + require(set(root) in (set(root_names[1:]), set(root_names)), + "original responsibility has no exact durable failure terminal and report") + owner = frozen["rootAdmission"] + require(all((row["node"], row["disk"]) == (owner["node"], owner["disk"]) for row in root.values()), + "durable failure report changed the original root owner") + terminal = journal.decode_json(checkpoints.bytes_of(root[root_names[1]])) + report = journal.decode_json(checkpoints.bytes_of(root[root_names[2]])) + require(type(terminal.get("schema")) is int and terminal["schema"] == 1 and terminal.get("task_id") == token + and journal.same_json(terminal.get("heal_type"), owner["value"]["heal_type"]) + and journal.same_json(terminal.get("options"), owner["value"]["options"]) + and isinstance(terminal.get("status"), dict) and set(terminal["status"]) == {"Failed"} + and isinstance(terminal["status"]["Failed"], dict) + and isinstance(terminal["status"]["Failed"].get("error"), str) + and "checkpoint" in terminal["status"]["Failed"]["error"].lower() + and type(report.get("schema")) is int and report["schema"] == 1 + and report.get("terminal") == terminal + and canonical(report.get("outcome")) == canonical(current["outcome"]), + "original responsibility lost its durable checkpoint failure diagnostic") + if root_names[0] in root: + require(checkpoints.bytes_of(root[root_names[0]]) == checkpoints.bytes_of(owner), + "retained runnable admission changed original responsibility") + return {"originalTaskId": token, "corruptBytesRetained": True, "resumeBytesRetained": True, + "blockedDiagnosticRetained": True, "durableOriginalFailureRetained": True, + "originalTaskRepairClaimed": False, "backupRescueUsed": False} + + +def rescan_outcome(current, bucket, expected, repairs, incarnation): + wanted = {version_identity(entry): entry for entry in expected} + require(wanted and len(wanted) == len(expected) and repairs <= set(wanted), "rescan manifest duplicated or escaped ACKs") + outcome = current.get("outcome") or {} + require(current.get("summary") == "finished" and outcome.get("execution") == {"state": "completed"} + and current.get("settings") == checkpoints.OPTIONS and type(current.get("truncated")) is bool + and outcome.get("coverage") == "complete" + and outcome.get("objectsTruncated") is (len(wanted) > 128), "safe rescan lacks complete truthful coverage") + counts = {"processed": len(wanted), "healed": len(repairs), "unchanged": len(wanted) - len(repairs), + "skipped": 0, "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False} + require(all(type(outcome.get("counters", {}).get(name)) is type(value) + and outcome["counters"][name] == value for name, value in counts.items()), + "safe rescan counters differ from independent complete ACK and physical manifests") + objects = outcome.get("objects") + require(isinstance(objects, list) and len(objects) == min(128, len(wanted)), "rescan omitted bounded diagnostics") + seen = set() + for receipt in objects: + identity = receipt.get("identity") or {} + pair = identity.get("object"), public_version_id(identity.get("versionId")) + require(pair in wanted and pair not in seen and identity.get("kind") == "object" + and identity.get("bucket") == bucket and identity.get("bucketIncarnationId") == incarnation + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0, + "rescan diagnostic changed an ACK identity, incarnation or set") + state = "repaired" if pair in repairs else "metadata_healthy" if wanted[pair]["deleteMarker"] else "verified_healthy" + require(receipt.get("disposition", {}).get("state") == state, "rescan diagnostic invented repair or hid unknown work") + seen.add(pair) + return {"canonicalCounters": counts, "completeIndependentManifestsRequired": True, + "boundedDiagnosticsAreCompleteManifest": len(wanted) <= 128, "originalFailedTaskReclassified": False} + + +def run_checkpoint_fault(args, topology, nodes, fixture, branch, candidate): + require(branch in IMPLEMENTED, "unimplemented journal branch cannot mutate the fleet") + directory = args.evidence / fixture / topology / branch + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, resource_profile="cursor", runtime_profile="root-restart") + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + fleet.snapshot_buckets = [bucket] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + count = 257 if fixture == "DB" else 80 + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 512, "maxInventoryBytes": 2 * 1024**3} + fleet.record("binding", binding) + fleet.record("candidate-bound-inspector", verify_inspector(args.tool_directory, candidate)) + probe, active = Probe(binding), [] + result = {"topology": topology, "fixture": fixture, "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) + expected = seed(Recorder(probe, directory / "fixture"), bucket, fixture, count, [SIZE]) + baseline = fleet.record("ACK-bound-complete-S3", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + fleet.record("complete-ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + idle_fleet(fleet, binding) + fleet.stop() + original = fleet.snapshot("healthy-physical-baseline", quiescent=True) + targets = [map_target(original, bucket, f"objects/{index:04d}.bin", "data", 1, + next(entry["versionId"] for entry in expected if entry["key"] == f"objects/{index:04d}.bin" + and not entry["deleteMarker"]), latest_required=False) for index in range(count - 4, count)] + prior = original + for index, target in enumerate(targets): + verify_proof(fleet, target, "baseline-full-cohort-bitrot") + params = {"disk": target["disk"], "faultId": f"F07-shard-{index}"} + fleet.faults.append((target["node"], params)) + receipt = fleet.record("missing-original-shard", fleet.remote(target["node"], "apply-fault", **params, + relativePath=target["relativePath"], expectedSha256=target["sha256"], kind="missing")) + current = fleet.snapshot("actual-missing-shard", quiescent=True) + validate_fault(prior, current, target, receipt) + prior = current + fleet.start() + require(physical_content(prior) == physical_content(fleet.snapshot("before-original-admission")), + "ATTRIBUTION_NOT_OBSERVED: background activity changed the original shard faults") + token = checkpoints.admit(probe, fleet, active) + frozen, identity = checkpoints.freeze(fleet, token, bucket) + frozen = checkpoints.stop_for_fault(fleet, token, bucket, frozen, identity) + params = {"family": "checkpoint", "taskId": token, "disk": frozen["disk"], + "relativePath": frozen["checkpoint"]["relativePath"], "faultId": "F07-" + branch, + "kind": IMPLEMENTED[branch], "expectedBefore": journal.entry(checkpoints.bytes_of(frozen["checkpoint"]))} + fleet.journal_faults.append((frozen["node"], params)) + receipt = fleet.record("stopped-original-checkpoint-injection", fleet.remote(frozen["node"], "journal-apply", **params)) + fleet.start() + deadline = time.monotonic() + 180 + while True: + failed = recovery_status(probe, fleet, token) + if failed.get("summary") != "running": break + require(time.monotonic() < deadline, "original corrupt checkpoint task did not produce a bounded diagnostic") + time.sleep(.2) + failed_task(failed) + active.remove((checkpoints.PATH, token)) + idle_fleet(fleet, binding) + fleet.stop() + corrupt_scene = checkpoints.records(fleet, token, "original-corrupt-checkpoint", quiescent=True) + failure_roots = checkpoints.records(fleet, token, "original-durable-failure", quiescent=True, family="root") + result["originalFailure"] = retained_corruption(corrupt_scene, failure_roots, frozen, receipt, failed) + before_rescan = fleet.snapshot("failure-scene-before-any-rescan", quiescent=True) + indexed = files(before_rescan) + remaining = [target for target in targets if (target["node"], target["disk"], target["relativePath"]) not in indexed] + require(remaining, "NOT_OBSERVED: no original object responsibility remains for the separate safe rescan") + partial = copy.deepcopy(original) + missing = {(target["node"], target["disk"], target["relativePath"]) for target in remaining} + for node, value in partial.items(): + value["files"] = [row for row in value["files"] if (node, row["disk"], row["path"]) not in missing] + fleet.record("partial-original-repair-and-control-proof", validate_repaired_physical( + partial, before_rescan, targets[0], repair_targets=targets)) + fleet.start() + admissions = [] + rescanned = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, + option_overrides={"recursive": True}, admissions=admissions) + require(len(admissions) == 1 and admissions[0]["clientToken"] != token, "safe rescan reused or relabelled the failed original token") + result["separateRescan"] = rescan_outcome(rescanned, bucket, expected, + {version_identity(target) for target in remaining}, frozen["bucketIncarnationId"]) + # Complete independent manifests cover the bounded public diagnostics. + # The small-result peer helper intentionally rejects such truncation. + for endpoint in tuple(binding["endpoints"])[1:]: + deadline = time.monotonic() + 30 + while True: + peer = status(Probe(binding, endpoint), fleet, admissions[0]["path"], admissions[0]["clientToken"], "same-rescan-peer") + if all(canonical(peer.get(name)) == canonical(rescanned.get(name)) + for name in ("summary", "settings", "outcome", "truncated")): break + require(time.monotonic() < deadline, "same original rescan token did not converge on an actual peer") + time.sleep(.2) + fleet.stop() + repaired = fleet.snapshot("complete-physical-before-any-GET", quiescent=True) + fleet.record("complete-physical-proof", validate_repaired_physical(original, repaired, targets[0], repair_targets=targets)) + for target in targets: verify_proof(fleet, target, "rescan-full-cohort-bitrot") + # New-task repair must not erase or advance the retained corrupt record. + fleet.start() + repeated = status(probe, fleet, checkpoints.PATH, token, "original-failure-after-separate-rescan") + require(canonical(repeated.get("outcome")) == canonical(failed.get("outcome")), "safe rescan rewrote the original failure") + for endpoint in binding["endpoints"]: + inventory = fleet.record("complete-peer-S3", {"endpoint": endpoint, **Probe(binding, endpoint).inventory( + bucket, unversioned_current_tags=fixture == "DB")}) + fleet.record("complete-peer-ACK-and-content-proof", {**verify_seed(expected, inventory), **compare_inventories(baseline, inventory)}) + fleet.stop() + require(physical_content(repaired) == physical_content(fleet.snapshot("physical-after-GET", quiescent=True)), + "business reads repaired data that the independent rescan had not proved") + retained_checkpoint = checkpoints.records(fleet, token, "retained-corrupt-checkpoint-after-rescan", quiescent=True) + retained_roots = checkpoints.records(fleet, token, "retained-original-failure-after-rescan", quiescent=True, family="root") + retained_corruption(retained_checkpoint, retained_roots, frozen, receipt, repeated) + require(canonical(exact_records(retained_checkpoint, token, "checkpoint")) == canonical(exact_records(corrupt_scene, token, "checkpoint")) + and canonical(exact_records(retained_roots, token, "root")) == canonical(exact_records(failure_roots, token, "root")), + "separate rescan or business reads changed the original corrupt record or durable failure bytes") + fleet.faults.clear() # Complete physical/S3 proof precedes discarding shard rescue registrations. + result.update(verdict="PASS", originalFailureReclassified=False, completePhysicalAndS3ManifestsVerified=True) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + try: + fleet.stop() + fleet.snapshot("original-failure-scene", quiescent=True) + except Exception as error: result["failureEvidenceError"] = str(error) + finally: + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("checkpoint-branch-result", result) + return result + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f"F07 interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ("candidate", "rc-bin", "evidence", "tool-directory"): parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + args = parser.parse_args() + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "F07 recipe branches changed") + candidate = json.loads(args.candidate.read_text()) + require(candidate["source_sha"] == checkpoints.REFERENCE["candidateCommit"], "F07 checkpoint failure contract requires the exact reviewed candidate") + native = verify_manager(os.environ.get("VALIDATION_HEAL_MANAGER_DIRECTORY", ""), candidate, CASE_ID) + save(args.evidence / "independent-native-persistence-prerequisites.json", native) + require(re.fullmatch(r"[0-9a-f]{40}", args.testing_commit) and re.fullmatch(r"[0-9a-f]{64}", args.binary_sha) + and re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")), "F07 requires immutable workflow and binary identities") + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) == len(set(nodes)) == 4 and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), + "F07 requires the four distinct current auto-testing Heal nodes") + results = [] + try: + for topology, selected in (("H1", nodes[:1]), ("H2", nodes)): + for fixture in ("DB", "DV"): + for branch in BRANCHES: + result = run_checkpoint_fault(args, topology, selected, fixture, branch, candidate) if branch in IMPLEMENTED else { + "topology": topology, "fixture": fixture, "branch": branch, "verdict": "BLOCKED", "cleanup": "PASS", + "reason": BLOCKERS[branch], "fleetBranchExecuted": False, "canonicalCasePassClaimed": False} + results.append(result) + if result["cleanup"] != "PASS": return 1 + finally: + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "complete": False, + "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL", + "canonicalCasePassClaimed": False, "reason": "four complete branch orchestrations remain blocked"}) + return 1 + + +if __name__ == "__main__": sys.exit(main()) diff --git a/test_validation_heal_persistence.py b/test_validation_heal_persistence.py new file mode 100644 index 00000000..de3774fe --- /dev/null +++ b/test_validation_heal_persistence.py @@ -0,0 +1,198 @@ +"""F07 preserves actual corrupt responsibility and separates rescan success.""" + +import base64 +import copy +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from scripts.validation_heal import persistence, checkpoint, journal_fault as journal +import test_validation_heal_checkpoint as fixture_source +from test_validation_heal_checkpoint import BUCKET, TOKEN, INCARNATION, encoded + + +class PersistenceContractTests(unittest.TestCase): + def fixture(self, kind="truncate"): + observed, _, _ = fixture_source.AdminCheckpointTests().fixture() + frozen = checkpoint.owned_checkpoint(observed, TOKEN, BUCKET) + root_value = {"schema": 3, "task_id": TOKEN, "heal_type": { + "type": "erasure_set", "buckets": [], "set_disk_id": "pool_0_set_0", "all_buckets": True}, + "options": {"scan_mode": 2, "recursive": True}} + owner = {"node": "node", "disk": 1, "relativePath": journal.names("root", TOKEN)[0], + "value": root_value, **encoded(root_value)} + frozen["rootAdmission"] = owner + receipt = {"applied": True, "originalBackupRetained": True, "productRepairClaimed": False, + "family": "checkpoint", "taskId": TOKEN, "disk": frozen["disk"], + "relativePath": frozen["checkpoint"]["relativePath"], "kind": kind, + "before": journal.entry(checkpoint.bytes_of(frozen["checkpoint"]))} + corrupt = journal.transform(checkpoint.bytes_of(frozen["checkpoint"]), receipt, {"bucket": BUCKET}, None) + receipt["injected"] = journal.entry(corrupt) + observed["node"]["records"][4].update(present=True, **journal.entry(corrupt), rawBase64=base64.b64encode(corrupt).decode()) + observed["node"]["records"][6].update(present=True, **journal.entry(b"blocked"), rawBase64=base64.b64encode(b"blocked").decode()) + current = {"summary": "stopped", "settings": checkpoint.OPTIONS, "detail": "Failed to deserialize checkpoint", + "outcome": {"execution": {"state": "aborted", "reason": "untraversable"}, "coverage": "partial", + "objects": [], "objectsTruncated": False, "counters": dict(processed=0, healed=0, unchanged=0, + skipped=0, failed=0, unknown=0, attemptFailures=0, overflowed=False)}} + terminal = {"schema": 1, "task_id": TOKEN, "heal_type": root_value["heal_type"], + "options": root_value["options"], "status": {"Failed": {"error": current["detail"]}}} + report = {"schema": 1, "terminal": terminal, "outcome": current["outcome"]} + names = journal.names("root", TOKEN) + roots = {"node": {"readOnly": True, "quiescent": True, "family": "root", "taskId": TOKEN, + "directoryEnumerationUsed": False, "records": [ + {"disk": disk, "relativePath": name, "present": False} for disk in range(4) for name in names]}} + roots["node"]["records"][4].update(encoded(terminal)) + roots["node"]["records"][5].update(encoded(report)) + return observed, roots, frozen, receipt, current + + def test_corrupt_bytes_blocked_checkpoint_and_durable_failure_remain_separate_from_repair(self): + for kind in ("truncate", "future-schema"): + with self.subTest(kind=kind): + proof = persistence.retained_corruption(*self.fixture(kind)) + self.assertTrue(proof["durableOriginalFailureRetained"]) + self.assertFalse(proof["originalTaskRepairClaimed"]) + self.assertFalse(proof["backupRescueUsed"]) + + def test_missing_rewritten_moved_or_duplicated_corruption_never_passes(self): + for change in ("missing", "restored", "blocked", "resume", "moved", "duplicate", "partial-census"): + data = self.fixture(); observed, _, frozen, _, _ = data + rows = observed["node"]["records"] + if change == "missing": rows[4] = {"disk": 1, "relativePath": rows[4]["relativePath"], "present": False} + elif change == "restored": rows[4] = copy.deepcopy(frozen["checkpoint"]) + elif change == "blocked": rows[6].update(encoded({"diagnostic": "invented"})) + elif change == "resume": rows[5].update(encoded({"completed": True})) + elif change == "moved": rows[0] = {**rows[4], "disk": 0}; rows[4] = {"disk": 1, "relativePath": rows[4]["relativePath"], "present": False} + elif change == "duplicate": observed["other"] = copy.deepcopy(observed["node"]) + else: rows.pop() + with self.subTest(change=change), self.assertRaises(ValueError): persistence.retained_corruption(*data) + + def test_injected_bytes_must_match_original_reviewed_transformation(self): + data = self.fixture(); data[3]["injected"] = journal.entry(b"unrelated corruption") + with self.assertRaisesRegex(ValueError, "reviewed transformation"): persistence.retained_corruption(*data) + + def test_silent_success_cancelled_and_processed_object_claims_are_rejected(self): + for change in ("success", "cancelled", "processed", "undocumented"): + data = self.fixture(); current = data[-1] + if change == "success": current.update(summary="finished"); current["outcome"]["execution"] = {"state": "completed"} + elif change == "cancelled": current["outcome"]["execution"]["reason"] = "cancelled" + elif change == "processed": current["outcome"]["counters"]["healed"] = 1 + else: current["detail"] = "generic failure" + with self.subTest(change=change), self.assertRaises(ValueError): persistence.retained_corruption(*data) + + def test_original_terminal_report_identity_scope_and_failure_are_required(self): + for change in ("missing", "success", "new-task", "scope", "report", "owner"): + data = self.fixture(); roots = data[1]; rows = roots["node"]["records"] + value = journal.decode_json(checkpoint.bytes_of(rows[4])) + if change == "missing": rows[4] = {"disk": 1, "relativePath": rows[4]["relativePath"], "present": False} + elif change == "success": value["status"] = "Completed"; rows[4].update(encoded(value)) + elif change == "new-task": value["task_id"] = INCARNATION; rows[4].update(encoded(value)) + elif change == "scope": value["heal_type"] = {"type": "cluster"}; rows[4].update(encoded(value)) + elif change == "report": rows[5].update(encoded({"schema": 1, "terminal": value, "outcome": {}})) + else: rows[1] = {**rows[4], "disk": 0}; rows[4] = {"disk": 1, "relativePath": rows[4]["relativePath"], "present": False} + with self.subTest(change=change), self.assertRaises(ValueError): persistence.retained_corruption(*data) + + def rescan(self): + expected = [{"key": f"objects/{index:04d}.bin", "versionId": "null", "deleteMarker": False} for index in range(257)] + repairs = {(expected[-1]["key"], "null")} + current = {"summary": "finished", "settings": checkpoint.OPTIONS, "truncated": True, + "outcome": {"execution": {"state": "completed"}, "coverage": "complete", "objectsTruncated": True, + "counters": dict(processed=257, healed=1, unchanged=256, skipped=0, failed=0, unknown=0, attemptFailures=0, overflowed=False), + "objects": [{"identity": {"kind": "object", "bucket": BUCKET, "object": entry["key"], + "versionId": None, "bucketIncarnationId": INCARNATION, "poolIndex": 0, "setIndex": 0}, + "disposition": {"state": "verified_healthy"}} for entry in expected[:128]]}} + return current, BUCKET, expected, repairs, INCARNATION + + def test_bounded_diagnostics_need_complete_independent_manifests_and_exact_counters(self): + proof = persistence.rescan_outcome(*self.rescan()) + self.assertTrue(proof["completeIndependentManifestsRequired"]) + self.assertFalse(proof["boundedDiagnosticsAreCompleteManifest"]) + self.assertFalse(proof["originalFailedTaskReclassified"]) + for change in ("counter", "unknown", "duplicate", "foreign", "incarnation", "partial", "truncated"): + data = self.rescan(); outcome = data[0]["outcome"] + if change == "counter": outcome["counters"]["healed"] = True + elif change == "unknown": outcome["objects"][0]["disposition"]["state"] = "unknown" + elif change == "duplicate": outcome["objects"][-1] = copy.deepcopy(outcome["objects"][0]) + elif change == "foreign": outcome["objects"][0]["identity"]["object"] = "foreign" + elif change == "incarnation": outcome["objects"][0]["identity"]["bucketIncarnationId"] = TOKEN + elif change == "partial": outcome["coverage"] = "partial" + else: outcome["objectsTruncated"] = False + with self.subTest(change=change), self.assertRaises(ValueError): persistence.rescan_outcome(*data) + + def test_incomplete_branches_never_enter_fleet_mutation(self): + for branch in persistence.BLOCKERS: + with self.subTest(branch=branch), self.assertRaisesRegex(ValueError, "cannot mutate"): + persistence.run_checkpoint_fault(None, "H1", ["node"], "DB", branch, {}) + + def interrupted_injection(self, cleanup): + _, _, frozen, _, _ = self.fixture() + snapshots = {"node": {"files": [], "formats": []}} + captured = {} + + class InterruptedFleet: + def __init__(self, *args, **kwargs): + self.faults, self.journal_faults = [], [] + captured["fleet"] = self + + def record(self, label, value): return value + def deploy(self, *args): pass + def start(self): pass + def stop(self): pass + def snapshot(self, *args, **kwargs): return snapshots + + def remote(self, node, action, **params): + if action == "journal-apply": + # The node may have committed the fault before the ACK was lost. + raise OSError("journal injection acknowledgement lost") + return {} + + def cleanup(self): + captured["cleanupFaults"] = copy.deepcopy(self.faults) + captured["cleanupJournals"] = copy.deepcopy(self.journal_faults) + return cleanup + + class FixtureProbe: + def __init__(self, *args): pass + def inventory(self, *args, **kwargs): return {} + + target = {"node": "node", "disk": 1, "relativePath": "bound/part.1", "sha256": "0" * 64} + expected = [{"key": f"objects/{index:04d}.bin", "versionId": "null", "deleteMarker": False} + for index in range(257)] + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + rc = directory / "rc"; rc.write_bytes(b"test-rc") + args = SimpleNamespace(evidence=directory, tool_directory=directory, + testing_commit="0" * 40, binary_sha="0" * 64, rc_bin=rc) + with patch.dict(persistence.os.environ, {"GITHUB_RUN_ID": "1"}), \ + patch.object(persistence, "Fleet", InterruptedFleet), patch.object(persistence, "Probe", FixtureProbe), \ + patch.object(persistence, "verify_inspector", return_value={}), \ + patch.object(persistence, "validate_topology", return_value={}), \ + patch.object(persistence, "seed", return_value=expected), \ + patch.object(persistence, "verify_seed", return_value={}), \ + patch.object(persistence, "idle_fleet"), patch.object(persistence, "map_target", return_value=target), \ + patch.object(persistence, "verify_proof"), patch.object(persistence, "validate_fault"), \ + patch.object(persistence.checkpoints, "admit", return_value=TOKEN), \ + patch.object(persistence.checkpoints, "freeze", return_value=(frozen, {})), \ + patch.object(persistence.checkpoints, "stop_for_fault", return_value=frozen): + result = persistence.run_checkpoint_fault(args, "H1", ["node"], "DB", "checkpoint-truncate", + {"source_sha": "0" * 40, "package_sha256": "0" * 64}) + return result, captured + + def test_lost_injection_ack_keeps_original_journal_and_shard_rescue_registered(self): + result, captured = self.interrupted_injection(True) + self.assertEqual(result["verdict"], "FAIL") + self.assertEqual(result["cleanup"], "PASS") + self.assertEqual(result["failure"]["message"], "journal injection acknowledgement lost") + self.assertEqual(len(captured["cleanupFaults"]), 4) + self.assertEqual(len(captured["cleanupJournals"]), 1) + self.assertEqual(captured["cleanupJournals"][0][1]["taskId"], TOKEN) + self.assertNotIn("separateRescan", result) + + def test_failed_cleanup_preserves_dirty_result_after_lost_injection_ack(self): + result, captured = self.interrupted_injection(False) + self.assertEqual((result["verdict"], result["cleanup"]), ("FAIL", "FAIL")) + self.assertEqual(len(captured["cleanupJournals"]), 1) + self.assertNotIn("completePhysicalAndS3ManifestsVerified", result) + + +if __name__ == "__main__": unittest.main() From 74599e5f6c494b1b8f25d92f280c349c70f3dea9 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 17:05:06 +0800 Subject: [PATCH 172/343] docs(validation): record audited orphan safety result --- docs/validation-migration-map.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 949e9f05..27c6e334 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 96 unique complete canonical live PASS cases: 84 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, and D09 run `38205854024`, D10 run `38207381816`, and manual-normal attribution run `38207956101` are included. Native, read-only, recovery and duplicate runs are excluded; the other 62 cases have no complete canonical live PASS claim. +case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 97 unique complete canonical live PASS cases: 85 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, and D09 run `38205854024`, D10 run `38207381816`, and manual-normal attribution run `38207956101` are included. Native, read-only, recovery and duplicate runs are excluded; the other 61 cases have no complete canonical live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1911,7 +1911,7 @@ A03 run `38198122041` passes every required H2 cross-node branch at helper `ccef J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite and its branch is explicitly BLOCKED pending a reviewed live unknown/lock-deferred fixture. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the source assertion requiring a mixed deferred population is still outstanding. The 84 cases, 301 branches and 532 topology templates are unchanged. -The corrected live census now contains 96 unique complete canonical PASS cases: 84 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926` and 12 on the older immutable source. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 38 retained live failures, 7 required native barriers, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. +The corrected live census now contains 97 unique complete canonical PASS cases: 85 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926` and 12 on the older immutable source. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 37 retained live failures, 7 required native barriers, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. D07 `heal.range-head-and-conditions` run `38205235080`, D09 `heal.object-lock-retention` run `38205854024`, D10 `heal.special-keys-and-pagination` run `38207381816`, and `heal.manual-normal-attribution` run `38207956101` passed every required source branch and were independently audited against the original canonical results, referenced member digests and immutable provenance. Both finalizations passed. A distinct validation-channel package was built successfully in run `38204362745/1` from source `16cf19817e0c81e24d0d787a916f734a34517ec5`, with package SHA-256 `531965f2c009a5a6ef1c02cc6aa1ba82a82b6fbe4aa441d89613afc8a3f91d40`. It contains the typed bucket-heal error and SSE-C response fixes. The D08 and manager native collectors explicitly bind this reviewed source and its actual source tree; the original candidate remains separately supported. Evidence for one candidate cannot certify the other. No new-candidate canonical PASS is claimed before complete fresh live execution. @@ -1922,3 +1922,5 @@ F07 now has a non-dispatched partial orchestrator in `persistence.py` for actual D04 run `38204764944` has a digest-bound diagnostic comparing the stopped stale-authority scene with the pre-manual-admission witness: only the target member xl.meta changes on each of H1 and H2, adding the exact newly acknowledged marker. Every other file and disk format remains unchanged. The candidate keeps durable MRF replay active when best-effort ingress is disabled; exact source-task attribution still needs investigation. The manual attribution gate and original FAIL remain intact. B09 run `38202851374` likewise has a complete identity-difference diagnostic: no expected ACK identity is missing; vm003 retains deleted `objects/0006.bin` null data in DB and historical `objects/0001.bin` version `de87efd5-e1ef-42a2-a86a-a1e7a18fe46c` in DV on all four disks. External resurrection is not established. The seventeen-case corrected live queue completed with eight PASS and nine retained FAIL results; every finalization passed. J01 run `38209400619` did not witness the full raw/rc uninitialized startup window on H1, and H2 rejected rc text that erased status uncertainty during convergence. Both disabled branches passed. E03 run `38208556719` retained range-repair physical differences and missing fresh ReadRepair attribution in H2. These partial results do not change either canonical verdict. + +C10 `heal.dangling-and-orphan-safety` run `38209676009` subsequently passed complete canonical audit at helper `dbe74ec9eb3288e32923d8b8777c581c904ba78f`; cleanup passed. A10 run `38210630408` retained FAIL with clean finalization after its corrected negative prefix oracle passed: other branches still reported incomplete version/marker diagnostics or unknown/incomplete integrity. The followup completed and discharged its fleet guard. Driver `16cf19817e0c81e24d0d787a916f734a34517ec5` is now pushed for a separate F05/D08 validation-package plan; no new-candidate PASS is claimed. The downloaded validation DEB matches its full package SHA and contains RustFS binary SHA-256 `d0d0fce93845728ef00b3b9406210b5bbf728c9262c329e2522319b40bc61917`; rc is an independently bound runner binary and is not included in that DEB. From 7c3b114c0e611870b0844987b69e8dca059689ca Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 17:26:20 +0800 Subject: [PATCH 173/343] fix(validation): retain complete read repair journal windows --- scripts/validation_heal/read_repair.py | 10 ++++++---- test_validation_heal.py | 24 +++++++++++++++++++++--- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/scripts/validation_heal/read_repair.py b/scripts/validation_heal/read_repair.py index 9b76cb65..e12ac928 100644 --- a/scripts/validation_heal/read_repair.py +++ b/scripts/validation_heal/read_repair.py @@ -7,8 +7,10 @@ try: from .heal_probe import Probe, payload, require + from .scanner_repair import JournalWindow except ImportError: from heal_probe import Probe, payload, require + from scanner_repair import JournalWindow BRANCHES = ("get-missing", "range-bitrot", "historical-version-read") @@ -62,9 +64,7 @@ def validate_read(observed, entry, *, expected_range=None, body=None): def tracked_read_task(probe, fleet, bucket, branch, entry): require(fleet.runtime_profile == "read-repair", "read repair requires its explicit isolated runtime profile") - def capture(label): - return fleet.record(label, {node: fleet.remote(node, "logs") for node in fleet.nodes}) - baseline = source_tasks(capture("read-source-before"), bucket, entry["key"]) + window = JournalWindow(fleet, "read-source", event_targets={ADMISSION_TARGET}) params = {"Bucket": bucket, "Key": entry["key"]} if entry["versionId"] != "null": params["VersionId"] = entry["versionId"] interval = (0, min(1024**2 + 16, entry["payload"]["size"]) - 1) if branch == "range-bitrot" else None @@ -75,7 +75,9 @@ def capture(label): deadline = time.monotonic() + 180 fresh = set() while not fresh: - fresh = source_tasks(capture("read-source-progress"), bucket, entry["key"]) - baseline + events = window.capture() + fresh = source_tasks({node: {"journal": "\n".join(json.dumps(event) for event in rows)} + for node, rows in events.items()}, bucket, entry["key"]) require(time.monotonic() < deadline, "ATTRIBUTION_NOT_OBSERVED: no fresh accepted ReadRepair task for the read target") if not fresh: time.sleep(0.2) require(len(fresh) == 1, "read source has ambiguous task ownership") diff --git a/test_validation_heal.py b/test_validation_heal.py index 0e582bac..6100fe05 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -1217,11 +1217,10 @@ def test_read_bytes_version_and_server_request_id_are_required(self): def execute(self, *, stale_only=False): bucket, key, calls = "rv-heal-read-unit", "target.bin", [] entry = {"key": key, "versionId": "historical", "payload": {"size": 3, "sha256": hashlib.sha256(b"abc").hexdigest()}} - old = self.event(request_id="old-task") fresh = self.event() - logs = [json.dumps(old), json.dumps(old) + ("" if stale_only else "\n" + json.dumps(fresh))] fleet = types.SimpleNamespace(runtime_profile="read-repair", nodes=["host-one"], - record=lambda label, value: value, remote=lambda *args: {"journal": logs.pop(0) if logs else json.dumps(old)}) + record=lambda label, value: value) + window = types.SimpleNamespace(capture=lambda: {"host-one": [] if stale_only else [fresh]}) def read(operation, params): calls.append((operation, params)) return {"acknowledged": True, "payload": entry["payload"], "response": {"VersionId": "historical", @@ -1235,6 +1234,7 @@ def status(path, query, body): observer = types.SimpleNamespace(admin=status) ticks = [0, 1, 181] if stale_only else [0, 1] with patch("scripts.validation_heal.read_repair.Probe", return_value=observer) as factory, \ + patch("scripts.validation_heal.read_repair.JournalWindow", return_value=window) as journal, \ patch("scripts.validation_heal.read_repair.time.monotonic", side_effect=ticks), \ patch("scripts.validation_heal.read_repair.time.sleep"): if stale_only: @@ -1244,6 +1244,7 @@ def status(path, query, body): else: self.assertEqual(tracked_read_task(client, fleet, bucket, "historical-version-read", entry)["summary"], "finished") factory.assert_called_once_with({}, "node1") + journal.assert_called_once_with(fleet, "read-source", event_targets={ADMISSION_TARGET}) self.assertEqual(sum(call[0] == "get_object" for call in calls), 1) self.assertEqual(calls[0][1], {"Bucket": bucket, "Key": key, "VersionId": "historical"}) self.assertTrue(all(len(call) == 2 or call[1] == [("clientToken", "automatic-task")] for call in calls)) @@ -1252,6 +1253,23 @@ def test_only_one_target_read_and_observation_of_the_fresh_automatic_task(self): self.execute() self.execute(stale_only=True) + def test_complete_cursor_window_finds_admission_before_more_than_2000_tail_lines(self): + event = self.event() + def remote(node, action, **params): + self.assertEqual(action, "logs") + if params.get("checkpoint"): + return {"complete": True, "afterCursor": "before-read"} + self.assertEqual(params, {"afterCursor": "before-read"}) + return {"complete": True, "beforeCursor": "before-read", "afterCursor": "after-read", + "hasMore": False, "journal": "\n".join([json.dumps(event)] + ['{"target":"unrelated"}'] * 2100)} + fleet = types.SimpleNamespace(nodes=["host-one"], remote=remote, record=lambda label, value: value) + window = JournalWindow(fleet, "read-source", event_targets={ADMISSION_TARGET}) + events = window.capture() + self.assertEqual(events, {"host-one": [event]}) + self.assertEqual(source_tasks({node: {"journal": "\n".join(json.dumps(row) for row in rows)} + for node, rows in events.items()}, event["bucket"], event["object"]), + {("host-one", "automatic-task")}) + def test_read_profile_keeps_other_repair_sources_disabled(self): profile = runtime_config("read-repair") self.assertEqual(profile.pop("RUSTFS_OBS_LOGGER_LEVEL"), "debug") From 1d8d217d2a7acce6859b0f6034c7d19b337ca2c2 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 17:37:02 +0800 Subject: [PATCH 174/343] docs(validation): record audited restart and encryption passes --- docs/validation-migration-map.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 27c6e334..f62a5372 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 97 unique complete canonical live PASS cases: 85 on the current candidate source and 12 on the older immutable source. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, and D09 run `38205854024`, D10 run `38207381816`, and manual-normal attribution run `38207956101` are included. Native, read-only, recovery and duplicate runs are excluded; the other 61 cases have no complete canonical live PASS claim. +case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 99 unique complete canonical live PASS cases: 85 on source `072b57b37559fbe626aeda8abf61617fdcbdd926`, 12 on the older immutable source and two on source `16cf19817e0c81e24d0d787a916f734a34517ec5`. The latest additions are F05 full-cluster restart `38227684435` and D08 encrypted objects `38229589497`. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, D09 run `38205854024`, D10 run `38207381816`, and manual-normal attribution run `38207956101` are included. Native, read-only, recovery and duplicate runs are excluded; the other 59 cases have no complete canonical live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1911,7 +1911,7 @@ A03 run `38198122041` passes every required H2 cross-node branch at helper `ccef J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite and its branch is explicitly BLOCKED pending a reviewed live unknown/lock-deferred fixture. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the source assertion requiring a mixed deferred population is still outstanding. The 84 cases, 301 branches and 532 topology templates are unchanged. -The corrected live census now contains 97 unique complete canonical PASS cases: 85 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926` and 12 on the older immutable source. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 37 retained live failures, 7 required native barriers, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. +The corrected live census now contains 99 unique complete canonical PASS cases: 85 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926`, 12 on the older immutable source and two on source `16cf19817e0c81e24d0d787a916f734a34517ec5`. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 35 retained live failures, 7 required native barriers, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. D07 `heal.range-head-and-conditions` run `38205235080`, D09 `heal.object-lock-retention` run `38205854024`, D10 `heal.special-keys-and-pagination` run `38207381816`, and `heal.manual-normal-attribution` run `38207956101` passed every required source branch and were independently audited against the original canonical results, referenced member digests and immutable provenance. Both finalizations passed. A distinct validation-channel package was built successfully in run `38204362745/1` from source `16cf19817e0c81e24d0d787a916f734a34517ec5`, with package SHA-256 `531965f2c009a5a6ef1c02cc6aa1ba82a82b6fbe4aa441d89613afc8a3f91d40`. It contains the typed bucket-heal error and SSE-C response fixes. The D08 and manager native collectors explicitly bind this reviewed source and its actual source tree; the original candidate remains separately supported. Evidence for one candidate cannot certify the other. No new-candidate canonical PASS is claimed before complete fresh live execution. @@ -1923,4 +1923,6 @@ D04 run `38204764944` has a digest-bound diagnostic comparing the stopped stale- The seventeen-case corrected live queue completed with eight PASS and nine retained FAIL results; every finalization passed. J01 run `38209400619` did not witness the full raw/rc uninitialized startup window on H1, and H2 rejected rc text that erased status uncertainty during convergence. Both disabled branches passed. E03 run `38208556719` retained range-repair physical differences and missing fresh ReadRepair attribution in H2. These partial results do not change either canonical verdict. -C10 `heal.dangling-and-orphan-safety` run `38209676009` subsequently passed complete canonical audit at helper `dbe74ec9eb3288e32923d8b8777c581c904ba78f`; cleanup passed. A10 run `38210630408` retained FAIL with clean finalization after its corrected negative prefix oracle passed: other branches still reported incomplete version/marker diagnostics or unknown/incomplete integrity. The followup completed and discharged its fleet guard. Driver `16cf19817e0c81e24d0d787a916f734a34517ec5` is now pushed for a separate F05/D08 validation-package plan; no new-candidate PASS is claimed. The downloaded validation DEB matches its full package SHA and contains RustFS binary SHA-256 `d0d0fce93845728ef00b3b9406210b5bbf728c9262c329e2522319b40bc61917`; rc is an independently bound runner binary and is not included in that DEB. +C10 `heal.dangling-and-orphan-safety` run `38209676009` subsequently passed complete canonical audit at helper `dbe74ec9eb3288e32923d8b8777c581c904ba78f`; cleanup passed. A10 run `38210630408` retained FAIL with clean finalization after its corrected negative prefix oracle passed: other branches still reported incomplete version/marker diagnostics or unknown/incomplete integrity. The followup completed and discharged its fleet guard. The separate F05/D08 validation-package plan on driver `16cf19817e0c81e24d0d787a916f734a34517ec5` also completed and discharged its guard. F05 `38227684435` and D08 `38229589497` passed independent complete canonical audit and clean finalization after read-only inspection `38227616934`. These are the first two PASS cases for this distinct candidate. D08 covered SSE-C, SSE-KMS, SSE-S3 and encrypted historical versions on H1/H2, including exact ciphertext/physical references, plaintext reads, classification headers and key-refusal checks. The downloaded validation DEB matches its full package SHA and contains RustFS binary SHA-256 `d0d0fce93845728ef00b3b9406210b5bbf728c9262c329e2522319b40bc61917`; rc is an independently bound runner binary and is not included in that DEB. + +E03's physical diagnostic for run `38208556719` proves that the Range bitrot target retained the exact injected damaged bytes on H1/H2. Its ReadRepair task used Normal scan mode and returned typed `metadata_healthy` with healed=0/unchanged=1; legacy progress counters cannot certify payload repair. The H2 missing/historical branches recovered bytes but lacked fresh admission evidence in the old bounded journal tail. The executor now collects the complete journal cursor window with pagination, preserving exact source/task attribution. This harness change passed 176 focused Heal/recovery checks and all six CI lanes at `7c3b114c0e611870b0844987b69e8dca059689ca`; E03 still requires a new complete live run on the product repair candidate. Its original FAIL remains retained. From 5f089da10f1394737e3f0d72a31ea989db78074a Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 17:42:56 +0800 Subject: [PATCH 175/343] test(validation): bind manager evidence to payload repair candidate --- scripts/validation_heal/manager_native.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/validation_heal/manager_native.py b/scripts/validation_heal/manager_native.py index 8d58b347..dd6e8473 100644 --- a/scripts/validation_heal/manager_native.py +++ b/scripts/validation_heal/manager_native.py @@ -28,6 +28,10 @@ "sourceTree": "7edb1a2d02d60323cdfab4173485c33cb06b297a", "cargoLockSha256": LOCK_SHA, }, + "e224eb48e5311cff022b04fd78c97a8572d5c270": { + "sourceTree": "098d5486f801e689ef30c00e5cedb775f0f1be0d", + "cargoLockSha256": LOCK_SHA, + }, } SOURCE_FILES = { "crates/heal/src/heal/resume/tests.rs": "cea1558f8659f01968a5592af55cd0626f1d309c9fecfa0bee26c36775229900", From 781ad0596b57b6c94ff3bbe2c06d67ec9cc8d53b Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 18:08:46 +0800 Subject: [PATCH 176/343] test(validation): add stale bucket metadata rejoin flow --- .../workflows/validation-catalog-tests.yml | 3 + docs/validation-heal-environment-gates.md | 2 +- docs/validation-migration-map.md | 2 + scripts/validation_heal/incarnation.py | 339 ++++++++++++++++++ test_validation_heal_incarnation.py | 147 ++++++++ 5 files changed, 492 insertions(+), 1 deletion(-) create mode 100644 scripts/validation_heal/incarnation.py create mode 100644 test_validation_heal_incarnation.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index e525a60e..1b77c87e 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -47,6 +47,7 @@ on: - test_validation_heal_environment_gate.py - test_validation_heal_outcome.py - test_validation_heal_persistence.py + - test_validation_heal_incarnation.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -107,6 +108,7 @@ on: - test_validation_heal_environment_gate.py - test_validation_heal_outcome.py - test_validation_heal_persistence.py + - test_validation_heal_incarnation.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -148,6 +150,7 @@ jobs: - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py + - run: python3 -B -m unittest -v test_validation_heal_incarnation.py - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh - run: bash test_validation_scanner_usage.sh diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index d0f4f47a..4dc8bd03 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -32,7 +32,7 @@ | Code | 用例 | Block 原因 | 解锁所需证据 | | --- | --- | --- | --- | -| C11 | `heal.bucket-incarnation-recreation` | 已有 3 项 incarnation 原生回归；缺旧盘回归、旧 checkpoint/MRF 重放、新桶隔离的全四分支真机编排 | 真机旧/新责任与 S3 ACK、物理范围及原生 artifact 精确绑定 | +| C11 | `heal.bucket-incarnation-recreation` | 已有 3 项 incarnation 原生回归及 `incarnation.py` 的 H1/H2 旧元数据盘回归编排；后者尚无 live 结果。仍缺旧 checkpoint/MRF 重放与精确提交屏障 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | | F07 | `heal.checkpoint-and-mrf-corruption` | 已有 journal 工具和 10 项原生回归；缺 H1/H2 全六分支、写失败诊断、责任保留和安全重扫编排 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | | I05 | `heal.lock-timeout-and-caller-abort` | 请求耗时/TCP 断开不能证明服务端锁 owner 与提交顺序 | lock owner、timeout、caller abort 和 stale read-repair 的服务端屏障 | | J02 | `heal.outcome-counters-and-error-truth` | 已有 2 项 outcome 原生回归；缺混合 healthy/repaired/failed/deferred/unknown 真机夹具、dry-run 独立任务与完整 manifest 对账 | 每个 disposition、计数和 attemptFailures 对账；不是已证实产品缺少 API | diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index f62a5372..8d155e64 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1926,3 +1926,5 @@ The seventeen-case corrected live queue completed with eight PASS and nine retai C10 `heal.dangling-and-orphan-safety` run `38209676009` subsequently passed complete canonical audit at helper `dbe74ec9eb3288e32923d8b8777c581c904ba78f`; cleanup passed. A10 run `38210630408` retained FAIL with clean finalization after its corrected negative prefix oracle passed: other branches still reported incomplete version/marker diagnostics or unknown/incomplete integrity. The followup completed and discharged its fleet guard. The separate F05/D08 validation-package plan on driver `16cf19817e0c81e24d0d787a916f734a34517ec5` also completed and discharged its guard. F05 `38227684435` and D08 `38229589497` passed independent complete canonical audit and clean finalization after read-only inspection `38227616934`. These are the first two PASS cases for this distinct candidate. D08 covered SSE-C, SSE-KMS, SSE-S3 and encrypted historical versions on H1/H2, including exact ciphertext/physical references, plaintext reads, classification headers and key-refusal checks. The downloaded validation DEB matches its full package SHA and contains RustFS binary SHA-256 `d0d0fce93845728ef00b3b9406210b5bbf728c9262c329e2522319b40bc61917`; rc is an independently bound runner binary and is not included in that DEB. E03's physical diagnostic for run `38208556719` proves that the Range bitrot target retained the exact injected damaged bytes on H1/H2. Its ReadRepair task used Normal scan mode and returned typed `metadata_healthy` with healed=0/unchanged=1; legacy progress counters cannot certify payload repair. The H2 missing/historical branches recovered bytes but lacked fresh admission evidence in the old bounded journal tail. The executor now collects the complete journal cursor window with pagination, preserving exact source/task attribution. This harness change passed 176 focused Heal/recovery checks and all six CI lanes at `7c3b114c0e611870b0844987b69e8dca059689ca`; E03 still requires a new complete live run on the product repair candidate. Its original FAIL remains retained. + +C11 now has partial H1/H2 DV orchestration in `incarnation.py` for `old-metadata-rejoin`. It verifies an original incarnation and complete DV ACKs, denies one actual member's reads/writes, deletes every data version and marker through S3, confirms successful bucket deletion and absence, and creates a same-name successor with distinct bytes/version IDs and explicit bucket configuration. Rejoin retains the original processes; successor responsibility must bind a different incarnation, preserve surviving new shards and contain no old version/marker references. Physical system records, full shard checksums, every peer's complete S3 inventory and config, old-version refusal and an independent control bucket remain separate evidence. Unknown S3 mutation state cannot become clean finalization. Eight new local tests cover refusal and partial-case gates; 52 focused tests, AST/YAML, actionlint, whitespace and the unchanged 84/301/532 matrix passed. This implementation has no live result. Old checkpoint/MRF replay and the exact storage commit barrier remain BLOCKED, and C11 stays outside dispatch. diff --git a/scripts/validation_heal/incarnation.py b/scripts/validation_heal/incarnation.py new file mode 100644 index 00000000..9eaeeca2 --- /dev/null +++ b/scripts/validation_heal/incarnation.py @@ -0,0 +1,339 @@ +"""C11: same-name bucket recreation with one actual stale metadata member.""" + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import sys +import uuid + +try: + from .heal_probe import Probe, compare_inventories, payload, require, save + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .fleet_node import PROCESS_FIELDS + from .bucket_metadata import system_snapshot, system_files + from .inspector import verify as verify_inspector + from .manager_fixture import verify as verify_manager + from .matrix import select + from .shards import files, map_target, run_task, verify_cohort + from .stop import idle_fleet + from .versions import public_version_id, validate_version_receipts, verify_quorum_contract, version_identity +except ImportError: + from heal_probe import Probe, compare_inventories, payload, require, save + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, physical_content, validate_topology + from fleet_node import PROCESS_FIELDS + from bucket_metadata import system_snapshot, system_files + from inspector import verify as verify_inspector + from manager_fixture import verify as verify_manager + from matrix import select + from shards import files, map_target, run_task, verify_cohort + from stop import idle_fleet + from versions import public_version_id, validate_version_receipts, verify_quorum_contract, version_identity + + +CASE_ID = "heal.bucket-incarnation-recreation" +BRANCHES = ("old-metadata-rejoin", "old-checkpoint-replay", "old-mrf-replay", "native-incarnation-fence") +BLOCKERS = { + "old-checkpoint-replay": "exact old-incarnation runnable checkpoint replay and original terminal evidence are incomplete", + "old-mrf-replay": "ACK-bound old-incarnation committed MRF replay and retained responsibility evidence are incomplete", + "native-incarnation-fence": "manager incarnation regression prerequisites do not prove the required exact storage commit barrier", +} +SIZE = 5 * 1024**2 + 123 + + +def old_manifest(expected): + wanted = {version_identity(entry): entry for entry in expected} + require(wanted and len(wanted) == len(expected) + and all(isinstance(pair[0], str) and pair[0] and isinstance(pair[1], str) + and str(uuid.UUID(pair[1])) == pair[1] and uuid.UUID(pair[1]).int != 0 + and type(entry.get("deleteMarker")) is bool for pair, entry in wanted.items()), + "C11 old manifest lacks distinct acknowledged data-version/marker identities") + require(any(entry["deleteMarker"] for entry in expected) and any(not entry["deleteMarker"] for entry in expected), + "C11 old DV fixture must contain both actual data versions and markers") + return wanted + + +def delete_old_bucket(recorder, bucket, expected): + """Only normal S3 deletion can establish the bucket deletion boundary.""" + wanted = old_manifest(expected) + verify_seed(expected, recorder.probe.inventory(bucket)) + removed = [] + for (key, version), entry in wanted.items(): + ack = recorder.call("delete_object", {"Bucket": bucket, "Key": key, "VersionId": version}) + require(ack.get("ResponseMetadata", {}).get("HTTPStatusCode") == 204 + and ack.get("VersionId") == version + and ack.get("DeleteMarker", False) is entry["deleteMarker"], + "C11 explicit DELETE did not acknowledge the exact old data/marker kind and identity") + removed.append({"key": key, "versionId": version, "deleteMarker": entry["deleteMarker"]}) + empty = recorder.probe.inventory(bucket) + uploads = recorder.call("list_multipart_uploads", {"Bucket": bucket, "MaxUploads": 1}) + require(empty.get("entries") == [] and uploads.get("IsTruncated") is False and uploads.get("Uploads", []) == [], + "C11 old bucket still contains a version, marker or pending upload") + ack = recorder.call("delete_bucket", {"Bucket": bucket}) + require(ack.get("ResponseMetadata", {}).get("HTTPStatusCode") == 204, + "C11 normal S3 delete_bucket did not succeed") + absent = recorder.probe.s3("head_bucket", {"Bucket": bucket}) + require(absent.get("acknowledged") is False and absent.get("resultUnknown") is False + and absent.get("response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode") == 404, + "C11 successful delete_bucket has no observed absence boundary") + return {"deleted": removed, "deleteBucketAcknowledged": True, "absenceObserved": True, + "internalNamespaceRemovalUsed": False} + + +def seed_successor(recorder, bucket, old): + old_ids = set(old_manifest(old)) + recorder.call("create_bucket", {"Bucket": bucket}) + recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) + recorder.call("put_bucket_tagging", {"Bucket": bucket, "Tagging": { + "TagSet": [{"Key": "validation-generation", "Value": "successor"}]}}) + expected = [] + for index, key in enumerate(sorted({entry["key"] for entry in old})): + for generation in range(3): + body = payload(f"{bucket}/{key}/successor/{generation}", SIZE + generation + 71) + ack = recorder.call("put_object", {"Bucket": bucket, "Key": key, + "Metadata": {"heal-fixture": "C11", "incarnation": "successor", "generation": str(generation)}, + "Tagging": "suite=heal&incarnation=successor"}, body) + version = ack.get("VersionId") + require(isinstance(version, str) and str(uuid.UUID(version)) == version and uuid.UUID(version).int != 0 + and (key, version) not in old_ids, "C11 successor reused an old or missing version ACK") + expected.append({"key": key, "versionId": version, "deleteMarker": False, + "isLatest": generation == 2, "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) + if index % 2 == 0: + ack = recorder.call("delete_object", {"Bucket": bucket, "Key": key}) + require(ack.get("DeleteMarker") is True and isinstance(ack.get("VersionId"), str) + and str(uuid.UUID(ack["VersionId"])) == ack["VersionId"] and uuid.UUID(ack["VersionId"]).int != 0 + and (key, ack["VersionId"]) not in old_ids, "C11 successor marker reused an old identity") + expected[-1]["isLatest"] = False + expected.append({"key": key, "versionId": ack["VersionId"], "deleteMarker": True, "isLatest": True}) + require(not (old_ids & set(old_manifest(expected))) + and {entry["payload"]["sha256"] for entry in old if not entry["deleteMarker"]}.isdisjoint( + entry["payload"]["sha256"] for entry in expected if not entry["deleteMarker"]), + "C11 successor bytes/identities do not distinguish the deleted generation") + return expected + + +def stale_member_unchanged(before, after, member): + require({node: row["formats"] for node, row in before.items()} == + {node: row["formats"] for node, row in after.items()}, "C11 recreation changed disk identity or topology") + left, right = files(before), files(after) + original = {identity: row for identity, row in left.items() if identity[:2] == member} + require(original and original == {identity: row for identity, row in right.items() if identity[:2] == member}, + "C11 gated old metadata member changed before its actual rejoin") + return {"node": member[0], "disk": member[1], "originalOldMemberBytesUnchanged": True} + + +def successor_physical(before, after, bucket, expected, old, member): + wanted = {} + for entry in expected: + wanted.setdefault(entry["key"], set()).add((entry["versionId"], entry["deleteMarker"], entry["isLatest"])) + old_ids, prior = set(old_manifest(old)), files(before) + current = files(after) + require({identity: row for identity, row in prior.items() if not identity[2].startswith(bucket + "/")} == + {identity: row for identity, row in current.items() if not identity[2].startswith(bucket + "/")}, + "C11 recreation or rejoin changed a control bucket's physical content") + require({node: row["formats"] for node, row in before.items()} == + {node: row["formats"] for node, row in after.items()}, "C11 rejoin changed disk formats") + for node, census in after.items(): + for disk in range(4): + actual = {} + for item in census["files"]: + meta = item.get("metadata") + if item["disk"] != disk or not meta: continue + if meta.get("bucket") != bucket: continue + require(meta.get("bucket") == bucket and meta.get("key") in wanted, + "C11 final metadata escaped successor ACK scope") + rows = meta.get("versions") + require(isinstance(rows, list), "C11 final metadata omitted decoded versions") + identities = {(public_version_id(row.get("versionId")), row["deleted"], row["isLatest"]) for row in rows} + require(len(identities) == len(rows) and meta["key"] not in actual + and not any((meta["key"], version) in old_ids for version, _, _ in identities), + "C11 final metadata resurrected an old identity or duplicated successor responsibility") + actual[meta["key"]] = identities + require(actual == wanted, "C11 rejoined member lacks complete successor versions/markers") + cohorts = [] + for entry in expected: + if entry["deleteMarker"]: continue + target = map_target(after, bucket, entry["key"], "data", 1, entry["versionId"], latest_required=False) + for shard in target["cohort"]: + if (shard["node"], shard["disk"]) == member: continue + original = prior.get((shard["node"], shard["disk"], shard["relativePath"])) + require(original and original["sha256"] == shard["sha256"] and original["size"] == shard["size"], + "C11 old member overwrote a surviving successor shard") + cohorts.append(target) + return cohorts + + +def config(probe, bucket): + versioning = probe.s3("get_bucket_versioning", {"Bucket": bucket}) + tags = probe.s3("get_bucket_tagging", {"Bucket": bucket}) + require(versioning.get("acknowledged") is True and versioning.get("resultUnknown") is False + and versioning.get("response", {}).get("Status") == "Enabled" + and tags.get("acknowledged") is True and tags.get("resultUnknown") is False + and tags.get("response", {}).get("TagSet") == [{"Key": "validation-generation", "Value": "successor"}], + "C11 old bucket configuration polluted the acknowledged successor configuration") + return {"versioning": versioning, "tagging": tags} + + +def run_metadata_rejoin(args, topology, nodes, candidate): + directory = args.evidence / topology / "old-metadata-rejoin" + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory) + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + control = bucket + "-control" + fleet.snapshot_buckets = [bucket, control] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 128, "maxInventoryBytes": 512 * 1024**2} + fleet.record("binding", binding) + tool = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + verify_quorum_contract(tool, candidate) + probe, active, pending = Probe(binding), [], None + result = {"topology": topology, "fixture": "DV", "branch": "old-metadata-rejoin", "verdict": "RUNNING", "cleanup": "PENDING"} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("actual-topology", validate_topology(fleet.snapshot("deployed-topology"), topology)) + recorder = Recorder(probe, directory / "fixture") + pending = {"operation": "seed-old-bucket"} + controls = seed(recorder, control, "DB", 1, [SIZE]) + old = seed(recorder, bucket, "DV", 3, [SIZE]) + old_manifest(old) + fleet.record("old-ACK-manifest", {"expected": old, **verify_seed(old, probe.inventory(bucket))}) + pending = None + initial = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, option_overrides={"recursive": True}) + old_incarnation = validate_version_receipts(initial, bucket, old, 2)["bucketIncarnationId"] + idle_fleet(fleet, binding) + fleet.stop() + original = fleet.snapshot("old-generation-before-gate", quiescent=True) + system_snapshot(fleet, [bucket, control], "old-generation-physical-bucket-records") + entry = next(entry for entry in old if not entry["deleteMarker"]) + target = map_target(original, bucket, entry["key"], "data", 1, entry["versionId"], latest_required=False) + shard = next(row for row in reversed(target["cohort"]) if row["node"] == nodes[-1]) + member = shard["node"], shard["disk"] + fleet.drive_gates.add(member[0]) + fleet.record("old-member-gate", fleet.remote(member[0], "drive-gate", disk=member[1], mode="permission-denied")) + fleet.start() + denied = fleet.record("old-member-service-denial", fleet.remote(member[0], "drive-gate-probe")) + require(denied.get("readProbe", {}).get("errno") == 13 and denied.get("errno") == 13 + and denied.get("dacBypassRemoved") is True, "C11 isolated member has no actual read/write denial") + pending = {"operation": "normal-S3-delete-and-recreate"} + result["deletedOriginal"] = fleet.record("normal-S3-bucket-deletion", delete_old_bucket(recorder, bucket, old)) + successor = seed_successor(recorder, bucket, old) + baseline = fleet.record("successor-ACK-bound-S3", probe.inventory(bucket)) + verify_seed(successor, baseline) + fleet.record("successor-config-ACK", config(probe, bucket)) + pending = None + authority = fleet.snapshot("old-and-successor-physical-scopes-before-rejoin") + result["staleMember"] = stale_member_unchanged(original, authority, member) + processes = {node: fleet.remote(node, "process") for node in nodes} + release = fleet.record("same-process-old-member-rejoin", fleet.remote(member[0], "drive-gate-release-live")) + require(release.get("sameOriginalProcess") is True and release.get("serviceRestarted") is False, + "C11 old member rejoin replaced its original process") + fleet.drive_gates.remove(member[0]) + rejoined = {node: fleet.remote(node, "process") for node in nodes} + require(all(all(processes[node].get(name) == rejoined[node].get(name) for name in PROCESS_FIELDS) for node in nodes), + "C11 rejoin changed the original candidate/process/unit") + fleet.snapshot("actual-rejoined-old-member-before-successor-task") + status = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, + option_overrides={"recursive": True}, status_endpoints=tuple(binding["endpoints"])[1:]) + responsibility = validate_version_receipts(status, bucket, successor, 2, + required_repairs={version_identity(entry) for entry in successor if not entry["deleteMarker"]}) + require(responsibility["bucketIncarnationId"] != old_incarnation, "C11 new bucket rebound the deleted incarnation") + result["incarnations"] = {"old": old_incarnation, "successor": responsibility["bucketIncarnationId"]} + result["successorResponsibility"] = responsibility + idle_fleet(fleet, binding) + fleet.stop() + after = fleet.snapshot("successor-physical-before-business-reads", quiescent=True) + successor_system = system_snapshot(fleet, [bucket, control], "successor-physical-bucket-records-before-reads") + cohorts = successor_physical(authority, after, bucket, successor, old, member) + result["streaming"] = [verify_cohort(fleet, target, "successor-full-shard-integrity") for target in cohorts] + fleet.start() + for endpoint in binding["endpoints"]: + peer = Probe(binding, endpoint) + inventory = fleet.record("all-peer-successor-S3", {"endpoint": endpoint, **peer.inventory(bucket)}) + verify_seed(successor, inventory) + compare_inventories(baseline, inventory) + control_inventory = fleet.record("all-peer-control-bucket-S3", {"endpoint": endpoint, + **peer.inventory(control, unversioned_current_tags=True)}) + verify_seed(controls, control_inventory) + fleet.record("all-peer-successor-config", {"endpoint": endpoint, **config(peer, bucket)}) + for (key, version), entry in old_manifest(old).items(): + missing = fleet.record("all-peer-deleted-incarnation-version", peer.s3("head_object", { + "Bucket": bucket, "Key": key, "VersionId": version})) + require(missing.get("acknowledged") is False and missing.get("resultUnknown") is False + and missing.get("response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode") == 404, + "C11 old version/marker reappeared in the successor namespace") + fleet.stop() + require(physical_content(after) == physical_content(fleet.snapshot("after-successor-business-reads", quiescent=True)), + "C11 business reads repaired data not proved before reads") + require(system_files(successor_system) == system_files(system_snapshot(fleet, [bucket, control], + "successor-physical-bucket-records-after-reads")), "C11 reads changed acknowledged successor bucket records") + result.update(verdict="PASS", successorManifestVerified=True, oldBackupRestorationUsed=False, + checkpointReplayClaimed=False, mrfReplayClaimed=False, nativeCommitFenceClaimed=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + try: + fleet.stop() + fleet.snapshot("original-recreation-failure-scene", quiescent=True) + except Exception as error: result["failureEvidenceError"] = str(error) + finally: + if pending is not None: + result["unresolvedMutation"] = pending + fleet.record("unresolved-S3-mutation", pending) + clean = fleet.cleanup() + result["cleanup"] = "FAIL" + result["environmentCleanupSucceeded"] = clean + else: + result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + fleet.record("incarnation-branch-result", result) + return result + + +def main(): + def interrupted(signum, frame): raise KeyboardInterrupt(f"C11 interrupted by signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + parser = argparse.ArgumentParser(description=__doc__) + for name in ("candidate", "rc-bin", "evidence", "tool-directory"): parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--case-id", choices=(CASE_ID,), required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--binary-sha", required=True) + args = parser.parse_args() + require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "C11 source branch inventory changed") + candidate = json.loads(args.candidate.read_text()) + native = verify_manager(os.environ.get("VALIDATION_HEAL_MANAGER_DIRECTORY", ""), candidate, CASE_ID) + save(args.evidence / "independent-native-incarnation-prerequisites.json", native) + require(re.fullmatch(r"[0-9a-f]{40}", args.testing_commit) and re.fullmatch(r"[0-9a-f]{64}", args.binary_sha) + and re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")), "C11 immutable workflow identities required") + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + require(len(nodes) == len(set(nodes)) == 4 and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), + "C11 requires the four distinct existing auto-testing Heal nodes") + results = [] + try: + for topology, selected in (("H1", nodes[:1]), ("H2", nodes)): + for branch in BRANCHES: + result = run_metadata_rejoin(args, topology, selected, candidate) if branch == "old-metadata-rejoin" else { + "topology": topology, "fixture": "DV", "branch": branch, "verdict": "BLOCKED", "cleanup": "PASS", + "reason": BLOCKERS[branch], "fleetBranchExecuted": False, "canonicalCasePassClaimed": False} + results.append(result) + if result["cleanup"] != "PASS": return 1 + finally: + save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "complete": False, + "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL", + "canonicalCasePassClaimed": False, "reason": "old checkpoint/MRF replay and exact commit fence remain blocked"}) + return 1 + + +if __name__ == "__main__": sys.exit(main()) diff --git a/test_validation_heal_incarnation.py b/test_validation_heal_incarnation.py new file mode 100644 index 00000000..491ed19e --- /dev/null +++ b/test_validation_heal_incarnation.py @@ -0,0 +1,147 @@ +"""C11 must prove real S3 deletion and preserve successor responsibility.""" + +import copy +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import uuid + +from scripts.validation_heal import incarnation as suite + + +def version(number): + return str(uuid.UUID(int=number)) + + +def manifest(base=1): + return [ + {"key": "objects/a", "versionId": version(base), "deleteMarker": False, "isLatest": False, + "payload": {"size": 1, "sha256": "a" * 64}}, + {"key": "objects/a", "versionId": version(base + 1), "deleteMarker": True, "isLatest": True}, + ] + + +class DeletionRecorder: + def __init__(self, *, wrong_version=False, remaining=False, unknown_absence=False, pending_upload=False): + self.probe = self + self.calls = [] + self.inventory_calls = 0 + self.wrong_version, self.remaining = wrong_version, remaining + self.unknown_absence, self.pending_upload = unknown_absence, pending_upload + + def inventory(self, bucket): + self.inventory_calls += 1 + return {"entries": manifest() if self.inventory_calls == 1 or self.remaining else []} + + def s3(self, op, params): + return {"acknowledged": False, "resultUnknown": self.unknown_absence, + "response": {"ResponseMetadata": {"HTTPStatusCode": 404}}} + + def call(self, op, params): + self.calls.append((op, copy.deepcopy(params))) + if op == "delete_object": + return {"VersionId": version(99) if self.wrong_version else params["VersionId"], + "DeleteMarker": params["VersionId"] == version(2), + "ResponseMetadata": {"HTTPStatusCode": 204}} + if op == "list_multipart_uploads": + return {"IsTruncated": False, "Uploads": [{"UploadId": "still-pending"}] if self.pending_upload else []} + if op == "delete_bucket": return {"ResponseMetadata": {"HTTPStatusCode": 204}} + raise AssertionError("unexpected mutation " + op) + + +class IncarnationTests(unittest.TestCase): + def test_normal_deletion_accounts_for_every_data_version_and_marker(self): + recorder = DeletionRecorder() + proof = suite.delete_old_bucket(recorder, "bucket", manifest()) + self.assertFalse(proof["internalNamespaceRemovalUsed"]) + self.assertEqual(proof["deleted"], [{k: row[k] for k in ("key", "versionId", "deleteMarker")} for row in manifest()]) + self.assertEqual([op for op, _ in recorder.calls], + ["delete_object", "delete_object", "list_multipart_uploads", "delete_bucket"]) + + def test_wrong_identity_remaining_members_and_unknown_absence_fail_closed(self): + for kwargs in ({"wrong_version": True}, {"remaining": True}, {"unknown_absence": True}, {"pending_upload": True}): + with self.subTest(kwargs=kwargs), self.assertRaises(ValueError): + suite.delete_old_bucket(DeletionRecorder(**kwargs), "bucket", manifest()) + + def test_duplicate_null_nil_and_wrong_kind_cannot_be_old_responsibility(self): + bad = [manifest() + [manifest()[0]], [*manifest(), {**manifest()[0], "versionId": "null"}], + [{**manifest()[0], "versionId": version(0)}, manifest()[1]], + [{**manifest()[0], "deleteMarker": 0}, manifest()[1]]] + for value in bad: + with self.subTest(value=value), self.assertRaises(ValueError): suite.old_manifest(value) + + def snapshot(self, expected): + versions = [{"versionId": row["versionId"], "deleted": row["deleteMarker"], "isLatest": row["isLatest"]} + for row in expected] + return {"vm000": {"formats": ["fixed-disk-identities"], "files": [ + {"disk": disk, "path": "bucket/objects/a/xl.meta", "size": 10, "sha256": "b" * 64, + "metadata": {"bucket": "bucket", "key": "objects/a", "versions": copy.deepcopy(versions)}} + for disk in range(4)]}} + + def test_rejoined_metadata_cannot_resurrect_an_old_version(self): + expected = manifest(3) + before = self.snapshot(expected) + after = copy.deepcopy(before) + after["vm000"]["files"][0]["metadata"]["versions"].append( + {"versionId": version(1), "deleted": False, "isLatest": False}) + with self.assertRaisesRegex(ValueError, "resurrected"): + suite.successor_physical(before, after, "bucket", expected, manifest(), ("vm000", 0)) + + def test_missing_successor_marker_and_duplicate_version_are_rejected(self): + expected = manifest(3) + before = self.snapshot(expected) + for change in ("missing", "duplicate"): + after = copy.deepcopy(before) + rows = after["vm000"]["files"][1]["metadata"]["versions"] + rows.pop() if change == "missing" else rows.append(copy.deepcopy(rows[0])) + with self.subTest(change=change), self.assertRaises(ValueError): + suite.successor_physical(before, after, "bucket", expected, manifest(), ("vm000", 0)) + + def test_original_gated_member_bytes_and_control_bucket_cannot_change(self): + before = self.snapshot(manifest()) + after = copy.deepcopy(before) + after["vm000"]["files"][0]["sha256"] = "c" * 64 + with self.assertRaisesRegex(ValueError, "gated old metadata"): + suite.stale_member_unchanged(before, after, ("vm000", 0)) + before = self.snapshot(manifest(3)) + before["vm000"]["files"].append({"disk": 0, "path": "bucket-control/a/part.1", "sha256": "a" * 64, "size": 1}) + after = copy.deepcopy(before) + after["vm000"]["files"][-1]["sha256"] = "b" * 64 + with self.assertRaisesRegex(ValueError, "control bucket"): + suite.successor_physical(before, after, "bucket", manifest(3), manifest(), ("vm000", 0)) + + def test_successor_configuration_requires_known_responses_and_its_exact_tag(self): + class ConfigProbe: + def s3(self, op, params): + return {"acknowledged": True, "resultUnknown": False, "response": { + "Status": "Enabled", "TagSet": [{"Key": "validation-generation", "Value": "successor"}]}} + probe = ConfigProbe() + self.assertIn("tagging", suite.config(probe, "bucket")) + with patch.object(probe, "s3", return_value={"acknowledged": True, "resultUnknown": True, "response": {"Status": "Enabled"}}): + with self.assertRaises(ValueError): suite.config(probe, "bucket") + + def test_partial_metadata_branch_never_promotes_full_canonical_case(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + candidate = root / "candidate.json" + candidate.write_text(json.dumps({"source_sha": "a" * 40})) + args = ["incarnation.py", "--case-id", suite.CASE_ID, "--candidate", str(candidate), + "--evidence", str(root), "--rc-bin", str(root / "rc"), "--tool-directory", str(root / "tools"), + "--testing-commit", "a" * 40, "--binary-sha", "b" * 64] + def passed(args, topology, nodes, candidate): + return {"topology": topology, "fixture": "DV", "branch": "old-metadata-rejoin", "verdict": "PASS", "cleanup": "PASS"} + with patch("sys.argv", args), patch.dict(os.environ, {"GITHUB_RUN_ID": "12345"}), \ + patch.object(suite, "verify_manager", return_value={"canonicalCasePassClaimed": False}), \ + patch.object(suite, "run_metadata_rejoin", side_effect=passed): + self.assertEqual(suite.main(), 1) + result = json.loads((root / "healthy-summary.json").read_text()) + self.assertFalse(result["complete"]) + self.assertFalse(result["canonicalCasePassClaimed"]) + self.assertEqual(len(result["results"]), 8) + self.assertEqual(sum(row["verdict"] == "BLOCKED" for row in result["results"]), 6) + + +if __name__ == "__main__": unittest.main() From 8ea38a9d8044e512de91ec788d79ecb7aa2f0fd8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 18:29:27 +0800 Subject: [PATCH 177/343] fix(validation): preserve read repair admission logs --- docs/validation-migration-map.md | 2 ++ scripts/validation_heal/fleet_node.py | 8 ++++++-- test_validation_heal.py | 4 +++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 8d155e64..185f00b9 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1928,3 +1928,5 @@ C10 `heal.dangling-and-orphan-safety` run `38209676009` subsequently passed comp E03's physical diagnostic for run `38208556719` proves that the Range bitrot target retained the exact injected damaged bytes on H1/H2. Its ReadRepair task used Normal scan mode and returned typed `metadata_healthy` with healed=0/unchanged=1; legacy progress counters cannot certify payload repair. The H2 missing/historical branches recovered bytes but lacked fresh admission evidence in the old bounded journal tail. The executor now collects the complete journal cursor window with pagination, preserving exact source/task attribution. This harness change passed 176 focused Heal/recovery checks and all six CI lanes at `7c3b114c0e611870b0844987b69e8dca059689ca`; E03 still requires a new complete live run on the product repair candidate. Its original FAIL remains retained. C11 now has partial H1/H2 DV orchestration in `incarnation.py` for `old-metadata-rejoin`. It verifies an original incarnation and complete DV ACKs, denies one actual member's reads/writes, deletes every data version and marker through S3, confirms successful bucket deletion and absence, and creates a same-name successor with distinct bytes/version IDs and explicit bucket configuration. Rejoin retains the original processes; successor responsibility must bind a different incarnation, preserve surviving new shards and contain no old version/marker references. Physical system records, full shard checksums, every peer's complete S3 inventory and config, old-version refusal and an independent control bucket remain separate evidence. Unknown S3 mutation state cannot become clean finalization. Eight new local tests cover refusal and partial-case gates; 52 focused tests, AST/YAML, actionlint, whitespace and the unchanged 84/301/532 matrix passed. This implementation has no live result. Old checkpoint/MRF replay and the exact storage commit barrier remain BLOCKED, and C11 stays outside dispatch. + +E03 followup `38235099954` uses package `38231599023/1` from exact source `e224eb48e5311cff022b04fd78c97a8572d5c270`. Five of six H1/H2 DB/DV branch instances passed, including both Range bitrot and historical-version instances with restored physical bytes and Deep typed repair receipts. H2 ordinary GET remains FAIL because no fresh source admission was observed; its original journal contains `Suppressed 13097 messages` for the exact test unit. The read-repair runtime now enables DEBUG only for `rustfs_ecstore` and `rustfs::heal`, keeping the default at INFO and other repair sources disabled. This prevents unrelated HTTP/HTTP2 debug traffic from consuming the journal budget. The 176 focused Heal/recovery checks passed. A complete fresh E03 run is required; the original FAIL and the 99-case canonical census remain unchanged. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 54df21f7..b4428a01 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -77,14 +77,18 @@ def runtime_config(profile): "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false"} if profile == "disabled": return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_ENABLED": "false"} + if profile == "read-repair": + # Admission evidence must survive journald's per-unit rate limit; + # global DEBUG floods it with unrelated HTTP/HTTP2 frame events. + logging_filter = "info,rustfs_ecstore=debug,rustfs::heal=debug" + return {**BACKGROUND_CONFIG, "RUSTFS_OBS_LOGGER_LEVEL": logging_filter, "RUST_LOG": logging_filter} if profile == "scanner": return {**BACKGROUND_CONFIG, "RUSTFS_SCANNER_ENABLED": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug", "RUSTFS_SCANNER_START_DELAY_SECS": "5", "RUSTFS_SCANNER_CYCLE": "5", "RUSTFS_SCANNER_SPEED": "fastest", "RUSTFS_SCANNER_DELAY": "0", "RUSTFS_SCANNER_BITROT_CYCLE_SECS": "0", "RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS": "0", "RUSTFS_HEAL_OBJECT_SELECT_PROB": "1", "RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES": "1"} - return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true" if profile == "offline-replay" else "false", - "RUSTFS_OBS_LOGGER_LEVEL": "debug" if profile == "read-repair" else "info"} + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true" if profile == "offline-replay" else "false"} def require(condition, message): diff --git a/test_validation_heal.py b/test_validation_heal.py index 6100fe05..a522ce15 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -1272,7 +1272,9 @@ def remote(node, action, **params): def test_read_profile_keeps_other_repair_sources_disabled(self): profile = runtime_config("read-repair") - self.assertEqual(profile.pop("RUSTFS_OBS_LOGGER_LEVEL"), "debug") + logging_filter = profile.pop("RUSTFS_OBS_LOGGER_LEVEL") + self.assertEqual(profile.pop("RUST_LOG"), logging_filter) + self.assertEqual(logging_filter.split(","), ["info", "rustfs_ecstore=debug", "rustfs::heal=debug"]) self.assertEqual(profile, {key: value for key, value in BACKGROUND_CONFIG.items() if key != "RUSTFS_OBS_LOGGER_LEVEL"}) snapshot, bucket, expected = versioned_scene(["objects/0001.bin"]) plan = branch_plan("heal.read-repair-attribution", "historical-version-read", snapshot, bucket, expected, "DV") From d5a41739a7a544d93ba71f51583d45ad4e6161eb Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 19:07:12 +0800 Subject: [PATCH 178/343] feat(validation): verify exact incarnation storage barriers --- .../workflows/validation-catalog-tests.yml | 3 + docs/validation-migration-map.md | 6 +- scripts/validation_heal/incarnation.py | 20 +- scripts/validation_heal/incarnation_native.py | 49 +++++ scripts/validation_heal/manager_native.py | 4 + scripts/validation_heal/native_regressions.py | 195 +++++++++++++++--- test_validation_heal_incarnation.py | 22 +- test_validation_heal_incarnation_native.py | 45 ++++ test_validation_heal_native.py | 183 +++++++++++++++- 9 files changed, 489 insertions(+), 38 deletions(-) create mode 100644 scripts/validation_heal/incarnation_native.py create mode 100644 test_validation_heal_incarnation_native.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 1b77c87e..4a14969e 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -33,6 +33,7 @@ on: - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py + - test_validation_heal_incarnation_native.py - test_validation_heal_manager_native.py - test_validation_heal_manager_fixture.py - test_validation_heal_journal_fault.py @@ -94,6 +95,7 @@ on: - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py + - test_validation_heal_incarnation_native.py - test_validation_heal_manager_native.py - test_validation_heal_manager_fixture.py - test_validation_heal_journal_fault.py @@ -151,6 +153,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_heal_outcome.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py - run: python3 -B -m unittest -v test_validation_heal_incarnation.py + - run: python3 -B -m unittest -v test_validation_heal_incarnation_native.py - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh - run: bash test_validation_scanner_usage.sh diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 185f00b9..a148c63b 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -131,7 +131,7 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 99 unique complete canonical live PASS cases: 85 on source `072b57b37559fbe626aeda8abf61617fdcbdd926`, 12 on the older immutable source and two on source `16cf19817e0c81e24d0d787a916f734a34517ec5`. The latest additions are F05 full-cluster restart `38227684435` and D08 encrypted objects `38229589497`. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, D09 run `38205854024`, D10 run `38207381816`, and manual-normal attribution run `38207956101` are included. Native, read-only, recovery and duplicate runs are excluded; the other 59 cases have no complete canonical live PASS claim. +case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 101 unique complete canonical live PASS cases: 85 on source `072b57b37559fbe626aeda8abf61617fdcbdd926`, 12 on the older immutable source, two on source `16cf19817e0c81e24d0d787a916f734a34517ec5`, and two on source `e224eb48e5311cff022b04fd78c97a8572d5c270`. The latest additions are C08 metadata/payload mismatch `38235978152` and E03 read-repair attribution `38237270640`. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, D09 run `38205854024`, D10 run `38207381816`, and manual-normal attribution run `38207956101` are included. Native, read-only, recovery and duplicate runs are excluded; the other 57 cases have no complete canonical live PASS claim. All 84 Heal recipes, 301 operator branches and 532 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public @@ -1930,3 +1930,7 @@ E03's physical diagnostic for run `38208556719` proves that the Range bitrot tar C11 now has partial H1/H2 DV orchestration in `incarnation.py` for `old-metadata-rejoin`. It verifies an original incarnation and complete DV ACKs, denies one actual member's reads/writes, deletes every data version and marker through S3, confirms successful bucket deletion and absence, and creates a same-name successor with distinct bytes/version IDs and explicit bucket configuration. Rejoin retains the original processes; successor responsibility must bind a different incarnation, preserve surviving new shards and contain no old version/marker references. Physical system records, full shard checksums, every peer's complete S3 inventory and config, old-version refusal and an independent control bucket remain separate evidence. Unknown S3 mutation state cannot become clean finalization. Eight new local tests cover refusal and partial-case gates; 52 focused tests, AST/YAML, actionlint, whitespace and the unchanged 84/301/532 matrix passed. This implementation has no live result. Old checkpoint/MRF replay and the exact storage commit barrier remain BLOCKED, and C11 stays outside dispatch. E03 followup `38235099954` uses package `38231599023/1` from exact source `e224eb48e5311cff022b04fd78c97a8572d5c270`. Five of six H1/H2 DB/DV branch instances passed, including both Range bitrot and historical-version instances with restored physical bytes and Deep typed repair receipts. H2 ordinary GET remains FAIL because no fresh source admission was observed; its original journal contains `Suppressed 13097 messages` for the exact test unit. The read-repair runtime now enables DEBUG only for `rustfs_ecstore` and `rustfs::heal`, keeping the default at INFO and other repair sources disabled. This prevents unrelated HTTP/HTTP2 debug traffic from consuming the journal budget. The 176 focused Heal/recovery checks passed. A complete fresh E03 run is required; the original FAIL and the 99-case canonical census remain unchanged. + +The complete E03 followup `38237270640` passed all six required H1/H2 DB/DV instances with clean finalization. Independent audit rechecked the original canonical result, every referenced member digest and exact package/binary/helper provenance. It uses the same source `e224eb48e5311cff022b04fd78c97a8572d5c270` and package `38231599023/1`, with helper `8ea38a9d8044e512de91ec788d79ecb7aa2f0fd8`; the preceding suppression failure remains retained. C08 `38235978152` also passed independent complete canonical audit. The derived ledger now records 101 complete PASS, 33 retained live failures, seven required native barriers, four orchestration/evidence blockers, and 13 disabled environment scenarios. B07 `38236702473` remains FAIL with clean finalization: H1's below-quorum S3 response and H2's negative terminal responsibility violate the frozen oracles. + +C11's native collector now binds reviewed source `1aa9be87d18cf941e5020f19e8622c26ba4b10ac`, its exact tree, Cargo.lock, six storage sources, helper digest and an optimized independent test executable. The nine-test suite preserves all six original native regressions and adds three real storage incarnation tests. The final-rename test pauses actual shard reconstruction before publication, aborts its caller while the lifecycle writer remains pending, drains and physically verifies the exact repaired version, and rejects obsolete admission after normal delete/recreate. Four original ordered milestones are mandatory; omitted, duplicated, reordered or invented rows cannot certify the barrier. Execution uses captured successful output and the candidate's bounded larger ecstore test stack. Local nextest passed 28 focused tests; the direct libtest check with captured output also passed. These local results are not hosted optimized native evidence. C11 independently verifies any hosted storage artifact before fleet mutation and records its scope as a native prerequisite with no fleet topology claim. Old checkpoint/MRF replay is still blocked, the metadata branch has no live result, and C11 remains outside dispatch. diff --git a/scripts/validation_heal/incarnation.py b/scripts/validation_heal/incarnation.py index 9eaeeca2..944b6f1c 100644 --- a/scripts/validation_heal/incarnation.py +++ b/scripts/validation_heal/incarnation.py @@ -20,6 +20,7 @@ from .bucket_metadata import system_snapshot, system_files from .inspector import verify as verify_inspector from .manager_fixture import verify as verify_manager + from .incarnation_native import verify as verify_storage from .matrix import select from .shards import files, map_target, run_task, verify_cohort from .stop import idle_fleet @@ -32,6 +33,7 @@ from bucket_metadata import system_snapshot, system_files from inspector import verify as verify_inspector from manager_fixture import verify as verify_manager + from incarnation_native import verify as verify_storage from matrix import select from shards import files, map_target, run_task, verify_cohort from stop import idle_fleet @@ -43,7 +45,6 @@ BLOCKERS = { "old-checkpoint-replay": "exact old-incarnation runnable checkpoint replay and original terminal evidence are incomplete", "old-mrf-replay": "ACK-bound old-incarnation committed MRF replay and retained responsibility evidence are incomplete", - "native-incarnation-fence": "manager incarnation regression prerequisites do not prove the required exact storage commit barrier", } SIZE = 5 * 1024**2 + 123 @@ -315,6 +316,8 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"C11 interrupted by sig candidate = json.loads(args.candidate.read_text()) native = verify_manager(os.environ.get("VALIDATION_HEAL_MANAGER_DIRECTORY", ""), candidate, CASE_ID) save(args.evidence / "independent-native-incarnation-prerequisites.json", native) + storage = verify_storage(os.environ.get("VALIDATION_HEAL_NATIVE_DIRECTORY", ""), candidate, args.testing_commit) + save(args.evidence / "independent-native-storage-commit.json", storage) require(re.fullmatch(r"[0-9a-f]{40}", args.testing_commit) and re.fullmatch(r"[0-9a-f]{64}", args.binary_sha) and re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")), "C11 immutable workflow identities required") nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() @@ -324,15 +327,22 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"C11 interrupted by sig try: for topology, selected in (("H1", nodes[:1]), ("H2", nodes)): for branch in BRANCHES: - result = run_metadata_rejoin(args, topology, selected, candidate) if branch == "old-metadata-rejoin" else { - "topology": topology, "fixture": "DV", "branch": branch, "verdict": "BLOCKED", "cleanup": "PASS", - "reason": BLOCKERS[branch], "fleetBranchExecuted": False, "canonicalCasePassClaimed": False} + if branch == "old-metadata-rejoin": + result = run_metadata_rejoin(args, topology, selected, candidate) + elif branch == "native-incarnation-fence": + result = {"requestedTopology": topology, "topology": None, "fixture": "DV", "branch": branch, + "verdict": "PREREQUISITE_PASS", "cleanup": "PASS", "fleetBranchExecuted": False, + "evidenceScope": "independent exact-candidate native storage regression", + "nativeProof": storage, "canonicalCasePassClaimed": False} + else: + result = {"topology": topology, "fixture": "DV", "branch": branch, "verdict": "BLOCKED", "cleanup": "PASS", + "reason": BLOCKERS[branch], "fleetBranchExecuted": False, "canonicalCasePassClaimed": False} results.append(result) if result["cleanup"] != "PASS": return 1 finally: save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "complete": False, "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL", - "canonicalCasePassClaimed": False, "reason": "old checkpoint/MRF replay and exact commit fence remain blocked"}) + "canonicalCasePassClaimed": False, "reason": "old checkpoint/MRF replay remains blocked"}) return 1 diff --git a/scripts/validation_heal/incarnation_native.py b/scripts/validation_heal/incarnation_native.py new file mode 100644 index 00000000..0b8799dc --- /dev/null +++ b/scripts/validation_heal/incarnation_native.py @@ -0,0 +1,49 @@ +"""Verify C11's candidate-bound physical storage and final-rename prerequisites.""" + +import argparse +import json +from pathlib import Path +import re + +try: + from . import native_regressions as suite +except ImportError: + import native_regressions as suite + + +CASE_ID = "heal.bucket-incarnation-recreation" + + +def verify(directory, candidate, helper_sha): + directory = Path(directory) + api = json.loads((directory / "tool-run.json").read_text()) + suite.require(type(api.get("id")) is int and api["id"] > 0 + and type(api.get("run_attempt")) is int and api["run_attempt"] > 0 + and api.get("status") == "completed" and api.get("conclusion") == "success" + and isinstance(api.get("head_sha"), str) and re.fullmatch(r"[0-9a-f]{40}", api["head_sha"]) + and api.get("path") in {".github/workflows/rustfs-functional-chain.yml", ".github/workflows/rustfs-validation-tools.yml"}, + "incarnation artifact has no exact successful hosted build attempt") + suite.require(candidate.get("source_sha") == suite.INCARNATION_CANDIDATE, + "incarnation commit barrier requires its exact reviewed source") + tools, evidence = directory / "native-tools", directory / "native-evidence" + manifest = json.loads((tools / "native-regressions.json").read_text()) + proof = suite.verify_evidence(tools, evidence, candidate, api["id"], api["run_attempt"], helper_sha) + suite.require(proof["observedBarrierEvents"].get("incarnation-final-rename") == suite.INCARNATION_EVENTS, + "actual final rename, cancellation, physical drain and successor refusal were not observed") + return {**proof, "caseId": CASE_ID, "runId": api["id"], "runAttempt": api["run_attempt"], + "driverCommit": api["head_sha"], "collectorCommit": manifest["testingCommit"], + "casePrerequisiteTests": [{"regression": key, **value} for key, value in suite.INCARNATION_REGRESSIONS.items()], + "exactStorageCommitBarrierObserved": True, "canonicalFleetBranchesExecuted": False, + "checkpointMrfReplayClaimed": False, "physicalPowerLossClaimed": False} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--testing-commit", required=True) + args = parser.parse_args() + print(json.dumps(verify(args.directory, json.loads(args.candidate.read_text()), args.testing_commit), indent=2)) + + +if __name__ == "__main__": main() diff --git a/scripts/validation_heal/manager_native.py b/scripts/validation_heal/manager_native.py index dd6e8473..d45ad1d5 100644 --- a/scripts/validation_heal/manager_native.py +++ b/scripts/validation_heal/manager_native.py @@ -32,6 +32,10 @@ "sourceTree": "098d5486f801e689ef30c00e5cedb775f0f1be0d", "cargoLockSha256": LOCK_SHA, }, + "1aa9be87d18cf941e5020f19e8622c26ba4b10ac": { + "sourceTree": "d57bcda8b1551727ced7073ddd09f8c97c13485f", + "cargoLockSha256": LOCK_SHA, + }, } SOURCE_FILES = { "crates/heal/src/heal/resume/tests.rs": "cea1558f8659f01968a5592af55cd0626f1d309c9fecfa0bee26c36775229900", diff --git a/scripts/validation_heal/native_regressions.py b/scripts/validation_heal/native_regressions.py index 600d0854..0cdfed66 100644 --- a/scripts/validation_heal/native_regressions.py +++ b/scripts/validation_heal/native_regressions.py @@ -65,6 +65,70 @@ "relatedCases": ["heal.conflicting-subquorum-authority"], "limits": "separate native regression; does not certify a live split, overlap or runtime pause"}, } +INCARNATION_CANDIDATE = "1aa9be87d18cf941e5020f19e8622c26ba4b10ac" +INCARNATION_EVENTS = ["final_rename_paused", "caller_aborted_writer_pending", + "repair_drained_writer_acquired", "successor_rejects_obsolete_admission"] +STORE_PREFIX = "store::heal::tests::" +INCARNATION_REGRESSIONS = { + "incarnation-successor-storage": { + "test": STORE_PREFIX + "bucket_incarnation_heal_preserves_successor_shards_and_allows_fresh_repair", + "sourceAssertions": ["normal delete/recreate changes the authoritative bucket UUID", + "name-only positive control repairs a real missing successor shard", + "old-incarnation metadata/path/version/rename/bucket/object mutations are rejected", + "obsolete admission publishes no missing shard and leaves healthy metadata byte-identical", + "fresh successor admission independently repairs the missing shard"], + "fenceKind": "real physical storage incarnation guard decisions", + "relatedCases": ["heal.bucket-incarnation-recreation"], + "limits": "does not prove fleet checkpoint or MRF replay"}, + "incarnation-cancelled-owner": { + "test": STORE_PREFIX + "bucket_incarnation_heal_owner_survives_cancelled_waiter_and_queued_writer", + "sourceAssertions": ["cancelled caller retains the original lifecycle owner", + "queued writer remains pending until storage proof drains", + "exact incarnation absence proof spans both selected pools without reacquiring behind the writer"], + "fenceKind": "deterministic held-owner and queued-writer assertions", + "relatedCases": ["heal.bucket-incarnation-recreation"], + "limits": "separate owner regression; not a physical final-rename pause"}, + "incarnation-final-rename": { + "test": STORE_PREFIX + "bucket_incarnation_fence_survives_caller_abort_at_real_repair_commit", + "sourceAssertions": ["real external exact-version reconstruction pauses before final scope check and rename", + "missing target is unpublished at the pause", + "caller abort cannot admit the lifecycle writer before repair drains", + "the published exact version/data directory passes physical check_parts verification", + "healthy metadata remains byte-identical", + "normal version/bucket deletion and recreation reject obsolete admission without changing successor metadata"], + "fenceKind": "deterministic actual reconstruction/final-rename barrier with emitted assertion milestones", + "relatedCases": ["heal.bucket-incarnation-recreation"], "barrierEvents": INCARNATION_EVENTS, + "limits": "native storage commit evidence only; not canonical fleet checkpoint/MRF replay"}, +} +CARGO_LOCK_SHA = "d8f7bc416ba67d19873d34e9389e5aa823b22d7d7552706a5a810cee26f6eda2" +BUILD_ENV = {'CARGO_BUILD_JOBS': '2', 'CARGO_INCREMENTAL': '0', 'CARGO_PROFILE_RELEASE_DEBUG': '0', + 'CARGO_PROFILE_RELEASE_STRIP': 'debuginfo'} +BUILD_COMMAND = ['cargo', 'test', '--locked', '--release', '-p', 'rustfs-ecstore', '--lib', '--no-run', '--message-format=json'] +REVIEWED_CANDIDATES = { + CANDIDATE: {'sourceTree': '8071fb0a9e5353eb7d3fa58c8a58c8622473f2bd', + 'sourceFiles': {SOURCE: SOURCE_SHA}, 'regressions': REGRESSIONS}, + INCARNATION_CANDIDATE: {'sourceTree': 'd57bcda8b1551727ced7073ddd09f8c97c13485f', + 'sourceFiles': { + SOURCE: 'a375fbf105d2491e1582c65c417509ac3eff23db826db788eb0aeb629e07f6b8', + 'crates/ecstore/src/store/heal.rs': '5ecf643f43f6f61f5b13f777f68faafba37e58a98eecb8355254f2ea75a1e9b1', + 'crates/ecstore/src/store/bucket.rs': 'a264de25fc4d3693ed962ea3613055b90f7092d0eb1cc748895030ab8adf1ae3', + 'crates/ecstore/src/store/bucket_fence.rs': '6f9d97cdd580bbd91f79af5723ff4712065b3d054cb5590db35fc00c79e1f262', + 'crates/ecstore/src/store/peer.rs': '85636a2536efc9e14fef0e1fe3d11de53093c45df44951b8fd08e4ed1edbf251', + 'crates/ecstore/src/set_disk/mod.rs': 'd4280fb89752f6198cb5641ea8ed23f5a41a4c87688e10bfce80a548e302e14f'}, + 'regressions': {**REGRESSIONS, **INCARNATION_REGRESSIONS}}, +} + + +def runtime_environment(commit): + # Match the candidate's nextest ecstore stack setup for real storage tests. + return {'RUST_MIN_STACK': '33554432'} if commit == INCARNATION_CANDIDATE else {} + + +def exact_command(binary, name, commit): + # Captured output keeps libtest's result line intact while retaining every + # successful test milestone; nocapture interleaves the first milestone. + output_flag = '--show-output' if commit == INCARNATION_CANDIDATE else '--nocapture' + return [str(binary), name, '--exact', output_flag, '--test-threads=1'] def require(value, message): @@ -95,35 +159,66 @@ def exact_listing(output, name): def test_result(output, name, code): # A successful zero-test filter or an ignored test is not a regression run. - require(code == 0 and len(re.findall(r"(?m)^running 1 test\s*$", output)) == 1 - and re.search(r"(?m)^test " + re.escape(name) + r" \.\.\. ok\s*$", output) - and re.search(r"test result: ok\. 1 passed; 0 failed; 0 ignored;", output), + require(type(code) is int and code == 0 + and re.findall(r"(?m)^running .*", output) == ['running 1 test'] + and re.findall(r"(?m)^test (?!result:).*", output) == [f'test {name} ... ok'] + and len(re.findall(r"(?m)^test result:.*", output)) == 1 + and re.search(r"(?m)^test result: ok\. 1 passed; 0 failed; 0 ignored;", output), "exact native execution did not report one real passing test") return {"exactTestExecuted": True, "nativeRegressionPassed": True, "fleetPassClaimed": False, "emittedBarrierLogsClaimed": False} +def barrier_events(output, spec): + events = re.findall(r"(?m)^incarnation_commit_barrier: ([a-z_]+)\s*$", output) + require(events == spec.get('barrierEvents', []), "native barrier milestones are absent, duplicated or out of order") + return events + + +def verify_manifest(directory): + manifest = json.loads((directory / 'native-regressions.json').read_text()) + contract = REVIEWED_CANDIDATES.get(manifest.get('candidateCommit')) + require(contract is not None and manifest.get('schemaVersion') == 1 + and manifest.get('candidateSourceTree') == contract['sourceTree'] + and manifest.get('candidateCargoLockSha256') == CARGO_LOCK_SHA + and manifest.get('sourceFiles') == contract['sourceFiles'] + and manifest.get('regressions') == contract['regressions'] and manifest.get('sourceUnmodified') is True + and manifest.get('optimized') is True and manifest.get('buildEnvironment') == BUILD_ENV + and manifest.get('buildCommand') == BUILD_COMMAND + and manifest.get('binaryKind') == 'native-regression-not-released-service' + and manifest.get('runtimeEnvironment') == runtime_environment(manifest.get('candidateCommit')) + and manifest.get('collectorSha256') == sha(Path(__file__)) and manifest.get('fleetPassClaimed') is False, + "native artifact identity or reviewed contract differs") + require(manifest.get('binaryName') == 'ecstore-native-tests', "native binary path is not allowlisted") + binary = directory / manifest['binaryName'] + require(binary.is_file() and not binary.is_symlink() and sha(binary) == manifest.get('binarySha256'), "native binary changed") + return manifest, binary, contract + + def build(source, commit, out): - require(commit == CANDIDATE and subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip() == commit, + contract = REVIEWED_CANDIDATES.get(commit) + require(contract is not None and subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip() == commit, "native regression contract requires the reviewed immutable candidate checkout") require(subprocess.run(['git', '-C', str(source), 'diff', '--quiet']).returncode == 0 and subprocess.run(['git', '-C', str(source), 'diff', '--cached', '--quiet']).returncode == 0, "native source has tracked modifications") require(not subprocess.check_output(['git', '-C', str(source), 'ls-files', '--others', '--exclude-standard'], text=True).strip(), "native candidate checkout has untracked source/configuration") - require(sha(source / SOURCE) == SOURCE_SHA, "native test source hash differs from its reviewed fence assertions") + require(subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD^{tree}'], text=True).strip() == contract['sourceTree'] + and sha(source / 'Cargo.lock') == CARGO_LOCK_SHA + and all(sha(source / path) == digest for path, digest in contract['sourceFiles'].items()), + "native source/tree/lock differs from its reviewed fence assertions") require(not out.exists() and shutil.disk_usage(source).free >= 12 * 1024**3, "fresh output and at least 12 GiB build space required") out.mkdir(parents=True) - save(out / 'native-build-request.json', {'candidateCommit': commit, 'sourceFiles': {SOURCE: SOURCE_SHA}, - 'candidateCargoLockSha256': sha(source / 'Cargo.lock'), 'regressions': REGRESSIONS, + save(out / 'native-build-request.json', {'candidateCommit': commit, 'sourceFiles': contract['sourceFiles'], + 'candidateCargoLockSha256': sha(source / 'Cargo.lock'), 'regressions': contract['regressions'], 'runId': os.environ.get('GITHUB_RUN_ID'), 'runAttempt': os.environ.get('GITHUB_RUN_ATTEMPT'), 'fleetPassClaimed': False}) target = out / 'target' - build_env = {'CARGO_BUILD_JOBS': '2', 'CARGO_INCREMENTAL': '0', 'CARGO_PROFILE_RELEASE_DEBUG': '0', - 'CARGO_PROFILE_RELEASE_STRIP': 'debuginfo'} + build_env = BUILD_ENV env = {**os.environ, **build_env, 'CARGO_TARGET_DIR': str(target)} - cargo = ['cargo', 'test', '--locked', '--release', '-p', 'rustfs-ecstore', '--lib', '--no-run', '--message-format=json'] + cargo = BUILD_COMMAND # Preserve diagnostics in files; native artifacts never require modifying # candidate source or adding a fake fence/event to an existing test. with (out / 'cargo.jsonl').open('x') as stdout, (out / 'cargo.stderr').open('x') as stderr: @@ -149,18 +244,19 @@ def build(source, commit, out): listing = subprocess.run([str(binary), '--list'], capture_output=True, text=True, timeout=60) (out / 'test-inventory.log').write_text(listing.stdout + listing.stderr) require(listing.returncode == 0, "candidate native inventory failed") - for spec in REGRESSIONS.values(): + for spec in contract['regressions'].values(): exact_listing(listing.stdout, spec['test']) manifest = {'schemaVersion': 1, 'candidateCommit': commit, 'candidateSourceTree': subprocess.check_output( ['git', '-C', str(source), 'rev-parse', 'HEAD^{tree}'], text=True).strip(), - 'sourceFiles': {SOURCE: SOURCE_SHA}, 'candidateCargoLockSha256': sha(source / 'Cargo.lock'), + 'sourceFiles': contract['sourceFiles'], 'candidateCargoLockSha256': sha(source / 'Cargo.lock'), 'binarySha256': sha(binary), 'binaryName': binary.name, 'binaryKind': 'native-regression-not-released-service', 'collectorSha256': sha(Path(__file__)), 'testingCommit': os.environ.get('AUTO_TESTING_SHA'), 'sourceUnmodified': True, 'buildCommand': cargo, 'buildEnvironment': build_env, 'optimized': True, 'buildJobs': 2, + 'runtimeEnvironment': runtime_environment(commit), 'toolchain': subprocess.check_output(['rustc', '-vV'], cwd=source, text=True), 'runId': os.environ.get('GITHUB_RUN_ID'), 'runAttempt': os.environ.get('GITHUB_RUN_ATTEMPT'), - 'regressions': REGRESSIONS, 'fleetPassClaimed': False, - 'limits': 'native and fleet evidence are independently scored; emitted fence logs and unmapped critical points remain gaps'} + 'regressions': contract['regressions'], 'fleetPassClaimed': False, + 'limits': 'native and fleet evidence are independently scored; only reviewed tests with verified original milestones claim emitted barriers'} save(out / 'native-regressions.json', manifest) # Remove only the build cache created under this fresh task output after # copying and identifying the executable and every build diagnostic. @@ -169,16 +265,10 @@ def build(source, commit, out): def run(directory, selected, out): - manifest = json.loads((directory / 'native-regressions.json').read_text()) - require(manifest.get('schemaVersion') == 1 and manifest.get('candidateCommit') == CANDIDATE - and manifest.get('sourceFiles') == {SOURCE: SOURCE_SHA} and manifest.get('sourceUnmodified') is True - and manifest.get('regressions') == REGRESSIONS and manifest.get('optimized') is True - and manifest.get('collectorSha256') == sha(Path(__file__)), - "native artifact identity or reviewed contract differs") - require(manifest.get('binaryName') == 'ecstore-native-tests', "native binary path is not allowlisted") - binary = directory / manifest['binaryName'] - require(binary.is_file() and not binary.is_symlink() and sha(binary) == manifest['binarySha256'], "native binary changed") - require(selected and len(selected) == len(set(selected)) and set(selected) <= set(REGRESSIONS), "invalid native selection") + manifest, binary, contract = verify_manifest(directory) + regressions = contract['regressions'] + selected = list(regressions) if selected is None else selected + require(selected and len(selected) == len(set(selected)) and set(selected) <= set(regressions), "invalid native selection") require(not out.exists(), "native evidence directory must be fresh") out.mkdir(parents=True) listing = subprocess.run([str(binary), '--list'], capture_output=True, text=True, timeout=60) @@ -186,20 +276,25 @@ def run(directory, selected, out): require(listing.returncode == 0, "native runtime inventory failed") results = [] for key in selected: - spec = REGRESSIONS[key] + spec = regressions[key] exact_listing(listing.stdout, spec['test']) require(not binary.is_symlink() and sha(binary) == manifest['binarySha256'], "native executable changed before exact execution") - args = [str(binary), spec['test'], '--exact', '--nocapture', '--test-threads=1'] + args = exact_command(binary, spec['test'], manifest['candidateCommit']) started = time.monotonic_ns() record = {'regression': key, 'test': spec['test'], 'command': args, 'sourceAssertions': spec['sourceAssertions'], - 'binarySha256': manifest['binarySha256'], 'candidateCommit': CANDIDATE, 'fleetPassClaimed': False, + 'binarySha256': manifest['binarySha256'], 'candidateCommit': manifest['candidateCommit'], 'fleetPassClaimed': False, 'emittedBarrierLogsClaimed': False, 'fenceKind': spec['fenceKind'], 'limitations': spec['limits']} + record['runtimeEnvironment'] = manifest['runtimeEnvironment'] try: - child = subprocess.run(args, capture_output=True, text=True, timeout=120) + child = subprocess.run(args, capture_output=True, text=True, timeout=120, + env={**os.environ, **manifest['runtimeEnvironment']}) output = child.stdout + child.stderr (out / (key + '.log')).write_text(output) record['exitCode'] = child.returncode test_result(output, spec['test'], child.returncode) + events = barrier_events(output, spec) + if events: + record.update(observedBarrierEvents=events, emittedBarrierLogsClaimed=True) record['verdict'] = 'PASS' except subprocess.TimeoutExpired as error: def text_part(value): @@ -221,6 +316,46 @@ def text_part(value): require(all(row['verdict'] == 'PASS' for row in results), 'native regression failure remains in separate evidence') +def verify_evidence(directory, evidence, candidate, run_id, run_attempt, helper_sha): + manifest, binary, contract = verify_manifest(directory) + require(manifest.get('candidateCommit') == candidate['source_sha'] and manifest.get('runId') == str(run_id) + and manifest.get('runAttempt') == str(run_attempt) and manifest.get('testingCommit') == helper_sha + and isinstance(helper_sha, str) and re.fullmatch(r'[0-9a-f]{40}', helper_sha), + "native evidence belongs to another candidate, run attempt or helper") + summary = json.loads((evidence / 'native-summary.json').read_text()) + rows = summary.get('results', []) + regressions = contract['regressions'] + require(summary.get('complete') is True and summary.get('sourceUnmodified') is True + and summary.get('fleetPassClaimed') is False and summary.get('canonicalCasePassClaimed') is False + and len(rows) == len(regressions) and {row.get('regression') for row in rows} == set(regressions), + "native summary omits or duplicates a required test or claims fleet success") + listings = [(directory / 'test-inventory.log').read_text(), (evidence / 'runtime-inventory.log').read_text()] + observed = {} + for row in rows: + key = row['regression'] + spec = regressions[key] + require(row == json.loads((evidence / (key + '.json')).read_text()) and row.get('verdict') == 'PASS' + and row.get('candidateCommit') == manifest['candidateCommit'] and row.get('binarySha256') == manifest['binarySha256'] + and all(row.get(name) == spec[name] for name in ('test', 'sourceAssertions', 'fenceKind')) + and row.get('limitations') == spec['limits'] and row.get('fleetPassClaimed') is False, + "native result changed exact source, scope or executable identity") + command = row.get('command', []) + require(len(command) == 5 and Path(command[0]).name == binary.name + and command[1:] == exact_command(binary, spec['test'], manifest['candidateCommit'])[1:] + and row.get('runtimeEnvironment') == manifest['runtimeEnvironment'], + "native result has no exact one-test invocation") + for listing in listings: exact_listing(listing, spec['test']) + output = (evidence / (key + '.log')).read_text() + test_result(output, spec['test'], row.get('exitCode')) + events = barrier_events(output, spec) + require(row.get('emittedBarrierLogsClaimed') is bool(events) + and (row.get('observedBarrierEvents') == events if events else 'observedBarrierEvents' not in row), + "native report invented or omitted original barrier evidence") + if events: observed[key] = events + return {'complete': True, 'candidateCommit': manifest['candidateCommit'], 'binarySha256': manifest['binarySha256'], + 'exactTests': len(rows), 'observedBarrierEvents': observed, 'fleetPassClaimed': False, 'canonicalCasePassClaimed': False} + + def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest='command', required=True) @@ -230,13 +365,13 @@ def main(): compiling.add_argument('--out', type=Path, required=True) running = commands.add_parser('run') running.add_argument('--directory', type=Path, required=True) - running.add_argument('--selection', default=','.join(REGRESSIONS)) + running.add_argument('--selection') running.add_argument('--out', type=Path, required=True) args = parser.parse_args() if args.command == 'build': build(args.source.resolve(strict=True), args.commit, args.out.resolve()) else: - run(args.directory.resolve(strict=True), args.selection.split(','), args.out.resolve()) + run(args.directory.resolve(strict=True), args.selection.split(',') if args.selection is not None else None, args.out.resolve()) if __name__ == '__main__': diff --git a/test_validation_heal_incarnation.py b/test_validation_heal_incarnation.py index 491ed19e..4436a98f 100644 --- a/test_validation_heal_incarnation.py +++ b/test_validation_heal_incarnation.py @@ -135,13 +135,33 @@ def passed(args, topology, nodes, candidate): return {"topology": topology, "fixture": "DV", "branch": "old-metadata-rejoin", "verdict": "PASS", "cleanup": "PASS"} with patch("sys.argv", args), patch.dict(os.environ, {"GITHUB_RUN_ID": "12345"}), \ patch.object(suite, "verify_manager", return_value={"canonicalCasePassClaimed": False}), \ + patch.object(suite, "verify_storage", return_value={"exactStorageCommitBarrierObserved": True, + "canonicalFleetBranchesExecuted": False, "canonicalCasePassClaimed": False}), \ patch.object(suite, "run_metadata_rejoin", side_effect=passed): self.assertEqual(suite.main(), 1) result = json.loads((root / "healthy-summary.json").read_text()) self.assertFalse(result["complete"]) self.assertFalse(result["canonicalCasePassClaimed"]) self.assertEqual(len(result["results"]), 8) - self.assertEqual(sum(row["verdict"] == "BLOCKED" for row in result["results"]), 6) + self.assertEqual(sum(row["verdict"] == "BLOCKED" for row in result["results"]), 4) + native = [row for row in result['results'] if row['branch'] == 'native-incarnation-fence'] + self.assertEqual(len(native), 2) + self.assertTrue(all(row['verdict'] == 'PREREQUISITE_PASS' and row['topology'] is None + and not row['fleetBranchExecuted'] for row in native)) + self.assertTrue((root / 'independent-native-storage-commit.json').is_file()) + + def test_unverified_storage_artifact_refuses_before_fleet_mutation(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp); candidate = root / 'candidate.json' + candidate.write_text(json.dumps({'source_sha': 'a' * 40})) + args = ['incarnation.py', '--case-id', suite.CASE_ID, '--candidate', str(candidate), + '--evidence', str(root), '--rc-bin', str(root / 'rc'), '--tool-directory', str(root / 'tools'), + '--testing-commit', 'a' * 40, '--binary-sha', 'b' * 64] + with patch('sys.argv', args), patch.object(suite, 'verify_manager', return_value={}), \ + patch.object(suite, 'verify_storage', side_effect=ValueError('unverified artifact')), \ + patch.object(suite, 'run_metadata_rejoin') as mutate: + with self.assertRaisesRegex(ValueError, 'unverified artifact'): suite.main() + mutate.assert_not_called() if __name__ == "__main__": unittest.main() diff --git a/test_validation_heal_incarnation_native.py b/test_validation_heal_incarnation_native.py new file mode 100644 index 00000000..c9e1f446 --- /dev/null +++ b/test_validation_heal_incarnation_native.py @@ -0,0 +1,45 @@ +"""C11's hosted native proof cannot be promoted to fleet or replay coverage.""" + +import json +from pathlib import Path +import tempfile +import unittest + +from scripts.validation_heal import incarnation_native as suite +import test_validation_heal_native as fixtures + + +class IncarnationNativeTests(unittest.TestCase): + def fixture(self, root): + fixtures.evidence_fixture(root) + api = {'id': 12345, 'run_attempt': 1, 'status': 'completed', 'conclusion': 'success', + 'head_sha': 'b' * 40, 'path': '.github/workflows/rustfs-validation-tools.yml'} + (root / 'tool-run.json').write_text(json.dumps(api)) + return {'source_sha': suite.suite.INCARNATION_CANDIDATE}, api + + def test_full_hosted_storage_suite_is_scoped_to_native_prerequisites(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp); candidate, _ = self.fixture(root) + proof = suite.verify(root, candidate, 'a' * 40) + self.assertEqual(proof['exactTests'], 9) + self.assertEqual(len(proof['casePrerequisiteTests']), 3) + self.assertTrue(proof['exactStorageCommitBarrierObserved']) + for field in ('canonicalFleetBranchesExecuted', 'checkpointMrfReplayClaimed', + 'physicalPowerLossClaimed', 'fleetPassClaimed', 'canonicalCasePassClaimed'): + self.assertFalse(proof[field]) + + def test_hosted_api_attempt_candidate_and_external_helper_must_match(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp); candidate, api = self.fixture(root) + path = root / 'tool-run.json' + for field, value in [('id', True), ('id', 12346), ('run_attempt', 2), ('status', 'in_progress'), + ('conclusion', 'failure'), ('head_sha', 'main'), ('path', '.github/workflows/ci.yml')]: + path.write_text(json.dumps({**api, field: value})) + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + suite.verify(root, candidate, 'a' * 40) + path.write_text(json.dumps(api)) + for wrong_candidate, helper in [({'source_sha': suite.suite.CANDIDATE}, 'a' * 40), (candidate, 'c' * 40)]: + with self.assertRaises(ValueError): suite.verify(root, wrong_candidate, helper) + + +if __name__ == '__main__': unittest.main() diff --git a/test_validation_heal_native.py b/test_validation_heal_native.py index 3702f24b..a30f9186 100644 --- a/test_validation_heal_native.py +++ b/test_validation_heal_native.py @@ -1,11 +1,52 @@ +import copy import json from pathlib import Path import tempfile import unittest +from unittest.mock import patch +import subprocess from scripts.validation_heal import native_regressions as native +def evidence_fixture(root, commit=native.INCARNATION_CANDIDATE): + tools, evidence = root / 'native-tools', root / 'native-evidence' + tools.mkdir(); evidence.mkdir() + binary = tools / 'ecstore-native-tests' + binary.write_bytes(b'original exact candidate test executable') + contract = native.REVIEWED_CANDIDATES[commit] + manifest = {'schemaVersion': 1, 'candidateCommit': commit, 'candidateSourceTree': contract['sourceTree'], + 'candidateCargoLockSha256': native.CARGO_LOCK_SHA, 'sourceFiles': contract['sourceFiles'], + 'sourceUnmodified': True, 'regressions': contract['regressions'], 'optimized': True, + 'collectorSha256': native.sha(Path(native.__file__)), 'binaryName': binary.name, + 'binarySha256': native.sha(binary), 'binaryKind': 'native-regression-not-released-service', + 'buildCommand': native.BUILD_COMMAND, 'buildEnvironment': native.BUILD_ENV, + 'runtimeEnvironment': native.runtime_environment(commit), 'fleetPassClaimed': False, + 'runId': '12345', 'runAttempt': '1', 'testingCommit': 'a' * 40} + (tools / 'native-regressions.json').write_text(json.dumps(manifest)) + listing = '\n'.join(spec['test'] + ': test' for spec in contract['regressions'].values()) + (tools / 'test-inventory.log').write_text(listing) + (evidence / 'runtime-inventory.log').write_text(listing) + rows = [] + for key, spec in contract['regressions'].items(): + events = spec.get('barrierEvents', []) + row = {'regression': key, 'test': spec['test'], 'sourceAssertions': spec['sourceAssertions'], + 'fenceKind': spec['fenceKind'], 'limitations': spec['limits'], 'candidateCommit': commit, + 'binarySha256': manifest['binarySha256'], 'command': native.exact_command(binary, spec['test'], commit), + 'runtimeEnvironment': manifest['runtimeEnvironment'], 'verdict': 'PASS', 'exitCode': 0, + 'fleetPassClaimed': False, 'emittedBarrierLogsClaimed': bool(events), 'elapsedNanoseconds': 100} + if events: row['observedBarrierEvents'] = events + output = f'running 1 test\ntest {spec["test"]} ... ok\n' + output += '\n'.join('incarnation_commit_barrier: ' + event for event in events) + '\n' + output += 'test result: ok. 1 passed; 0 failed; 0 ignored; 99 filtered out;\n' + (evidence / (key + '.log')).write_text(output) + (evidence / (key + '.json')).write_text(json.dumps(row)) + rows.append(row) + (evidence / 'native-summary.json').write_text(json.dumps({'results': rows, 'complete': True, + 'sourceUnmodified': True, 'fleetPassClaimed': False, 'canonicalCasePassClaimed': False})) + return tools, evidence, {'source_sha': commit}, 'a' * 40 + + class NativeRegressionTests(unittest.TestCase): name = native.PREFIX + 'read_repair_commit_aborts_when_object_changes_before_final_rename' @@ -22,7 +63,9 @@ def test_zero_filtered_ignored_failed_and_other_test_success_are_not_pass(self): self.assertTrue(proof['exactTestExecuted']) self.assertFalse(proof['fleetPassClaimed']) self.assertFalse(proof['emittedBarrierLogsClaimed']) - for output, code in ((passed, 1), (passed.replace('running 1 test', 'running 0 tests'), 0), + for output, code in ((passed, 1), (passed, False), (passed + passed, 0), + (passed + 'test unrelated ... ok\n', 0), + (passed.replace('running 1 test', 'running 0 tests'), 0), (passed.replace('1 passed; 0 failed; 0 ignored', '0 passed; 0 failed; 1 ignored'), 0), (passed.replace(self.name, self.name + '-other'), 0), (passed.replace('... ok', '... FAILED'), 0)): @@ -58,5 +101,143 @@ def test_wrong_source_modified_contract_or_collector_fail_before_execution(self) self.assertFalse((directory / 'evidence').exists()) +class NativeEvidenceTests(unittest.TestCase): + def verify(self, fixture, **changes): + tools, evidence, candidate, helper = fixture + return native.verify_evidence(tools, evidence, changes.get('candidate', candidate), + changes.get('run', 12345), changes.get('attempt', 1), changes.get('helper', helper)) + + def test_complete_suites_preserve_original_six_and_exact_nine_scopes(self): + for commit, count in ((native.CANDIDATE, 6), (native.INCARNATION_CANDIDATE, 9)): + with self.subTest(commit=commit), tempfile.TemporaryDirectory() as temp: + proof = self.verify(evidence_fixture(Path(temp), commit)) + self.assertEqual(proof['exactTests'], count) + self.assertFalse(proof['canonicalCasePassClaimed']) + self.assertFalse(proof['fleetPassClaimed']) + self.assertEqual(bool(proof['observedBarrierEvents']), count == 9) + + def test_runner_executes_full_candidate_suite_and_retains_original_milestones(self): + with tempfile.TemporaryDirectory() as temp: + fixture = evidence_fixture(Path(temp)); tools, evidence, _, _ = fixture + calls = [] + def execute(command, **kwargs): + calls.append(command) + if command[-1] == '--list': + return subprocess.CompletedProcess(command, 0, (tools / 'test-inventory.log').read_text(), '') + self.assertEqual(kwargs['env']['RUST_MIN_STACK'], '33554432') + self.assertEqual(command[2:], ['--exact', '--show-output', '--test-threads=1']) + key = next(key for key, spec in native.REVIEWED_CANDIDATES[native.INCARNATION_CANDIDATE]['regressions'].items() + if spec['test'] == command[1]) + return subprocess.CompletedProcess(command, 0, (evidence / (key + '.log')).read_text(), '') + out = Path(temp) / 'executed' + with patch.object(native.subprocess, 'run', side_effect=execute): native.run(tools, None, out) + self.assertEqual(len(calls), 10) + proof = native.verify_evidence(tools, out, fixture[2], 12345, 1, fixture[3]) + self.assertEqual(proof['observedBarrierEvents']['incarnation-final-rename'], native.INCARNATION_EVENTS) + + def test_runner_retains_missing_barrier_failure_and_refuses_complete_success(self): + with tempfile.TemporaryDirectory() as temp: + tools, evidence, _, _ = evidence_fixture(Path(temp)) + def execute(command, **kwargs): + if command[-1] == '--list': + output = (tools / 'test-inventory.log').read_text() + else: + output = (evidence / 'incarnation-final-rename.log').read_text().replace( + 'incarnation_commit_barrier: final_rename_paused', '') + return subprocess.CompletedProcess(command, 0, output, '') + out = Path(temp) / 'failed' + with patch.object(native.subprocess, 'run', side_effect=execute), self.assertRaises(ValueError): + native.run(tools, ['incarnation-final-rename'], out) + self.assertIn('caller_aborted_writer_pending', (out / 'incarnation-final-rename.log').read_text()) + summary = json.loads((out / 'native-summary.json').read_text()) + self.assertFalse(summary['complete']) + self.assertEqual(summary['results'][0]['verdict'], 'FAIL') + + def test_source_tree_lock_build_runtime_collector_and_scope_drift_fail(self): + with tempfile.TemporaryDirectory() as temp: + fixture = evidence_fixture(Path(temp)); path = fixture[0] / 'native-regressions.json' + original = json.loads(path.read_text()) + for field, value in [('candidateCommit', native.CANDIDATE), ('candidateSourceTree', 'b' * 40), + ('candidateCargoLockSha256', 'b' * 64), ('sourceFiles', {}), ('collectorSha256', 'b' * 64), + ('regressions', {}), ('sourceUnmodified', False), ('optimized', False), ('buildCommand', []), + ('buildEnvironment', {}), ('runtimeEnvironment', {}), ('fleetPassClaimed', True), + ('binaryKind', 'released-service'), ('binaryName', '../foreign')]: + path.write_text(json.dumps({**original, field: value})) + with self.subTest(field=field), self.assertRaises(ValueError): self.verify(fixture) + + def test_candidate_run_attempt_helper_and_binary_are_independently_bound(self): + with tempfile.TemporaryDirectory() as temp: + fixture = evidence_fixture(Path(temp)) + for changes in [{'candidate': {'source_sha': 'b' * 40}}, {'run': 12346}, {'attempt': 2}, {'helper': 'b' * 40}]: + with self.subTest(changes=changes), self.assertRaises(ValueError): self.verify(fixture, **changes) + (fixture[0] / 'ecstore-native-tests').write_bytes(b'wrong executable') + with self.assertRaises(ValueError): self.verify(fixture) + + def test_symlinked_binary_is_rejected(self): + with tempfile.TemporaryDirectory() as temp: + fixture = evidence_fixture(Path(temp)); binary = fixture[0] / 'ecstore-native-tests' + original = fixture[0] / 'original'; binary.rename(original); binary.symlink_to(original) + with self.assertRaises(ValueError): self.verify(fixture) + + def test_summary_cannot_omit_duplicate_or_promote_tests(self): + with tempfile.TemporaryDirectory() as temp: + fixture = evidence_fixture(Path(temp)); path = fixture[1] / 'native-summary.json' + original = json.loads(path.read_text()) + for change in ('missing', 'duplicate', 'incomplete', 'fleet', 'canonical'): + value = copy.deepcopy(original) + if change == 'missing': value['results'].pop() + elif change == 'duplicate': value['results'][-1] = value['results'][0] + elif change == 'incomplete': value['complete'] = False + elif change == 'fleet': value['fleetPassClaimed'] = True + else: value['canonicalCasePassClaimed'] = True + path.write_text(json.dumps(value)) + with self.subTest(change=change), self.assertRaises(ValueError): self.verify(fixture) + + def test_original_milestones_must_be_complete_unique_and_ordered(self): + with tempfile.TemporaryDirectory() as temp: + fixture = evidence_fixture(Path(temp)); path = fixture[1] / 'incarnation-final-rename.log' + original = path.read_text() + first, second = ['incarnation_commit_barrier: ' + e for e in native.INCARNATION_EVENTS[:2]] + for output in (original.replace(first, ''), original + first + '\n', + original.replace(first, 'TEMP').replace(second, first).replace('TEMP', second)): + path.write_text(output) + with self.subTest(output=output), self.assertRaises(ValueError): self.verify(fixture) + + def test_forged_rows_and_commands_cannot_replace_original_proof(self): + for change in ('row-only', 'events', 'claim', 'command', 'runtime', 'scope', 'exit', 'binary'): + with self.subTest(change=change), tempfile.TemporaryDirectory() as temp: + fixture = evidence_fixture(Path(temp)); key = 'incarnation-final-rename' + path = fixture[1] / (key + '.json'); row = json.loads(path.read_text()) + if change in ('row-only', 'events'): row['observedBarrierEvents'] = [] + elif change == 'claim': row['emittedBarrierLogsClaimed'] = False + elif change == 'command': row['command'][3] = '--nocapture' + elif change == 'runtime': row['runtimeEnvironment'] = {} + elif change == 'scope': row['fleetPassClaimed'] = True + elif change == 'exit': row['exitCode'] = 1 + else: row['binarySha256'] = 'b' * 64 + path.write_text(json.dumps(row)) + if change != 'row-only': + summary_path = fixture[1] / 'native-summary.json' + summary = json.loads(summary_path.read_text()) + summary['results'][-1] = row; summary_path.write_text(json.dumps(summary)) + with self.assertRaises(ValueError): self.verify(fixture) + + def test_original_inventory_and_test_execution_cannot_be_missing_or_ignored(self): + for change in ('missing', 'duplicate', 'zero', 'ignored', 'wrong-test'): + with self.subTest(change=change), tempfile.TemporaryDirectory() as temp: + fixture = evidence_fixture(Path(temp)); key = 'incarnation-final-rename' + name = native.INCARNATION_REGRESSIONS[key]['test'] + if change in ('missing', 'duplicate'): + path = fixture[1] / 'runtime-inventory.log'; original = path.read_text() + path.write_text(original.replace(name + ': test', '') if change == 'missing' else original + '\n' + name + ': test') + else: + path = fixture[1] / (key + '.log'); original = path.read_text() + if change == 'zero': original = original.replace('running 1 test', 'running 0 tests') + elif change == 'ignored': original = original.replace('... ok', '... ignored') + else: original = original.replace(name, name + '-other') + path.write_text(original) + with self.assertRaises(ValueError): self.verify(fixture) + + if __name__ == '__main__': unittest.main() From c9c2b19d207d34bf319851f49c98d6c211966e05 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 20:41:16 +0800 Subject: [PATCH 179/343] test(heal): replay original C11 recovery records safely --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-heal-cases.md | 2 +- docs/validation-heal-environment-gates.md | 2 +- docs/validation-migration-map.md | 11 + scripts/run_validation_case.sh | 2 +- scripts/validation_heal/execution_policy.py | 2 +- scripts/validation_heal/fleet_node.py | 111 ++++- scripts/validation_heal/healthy.py | 32 +- scripts/validation_heal/incarnation.py | 369 +++++++++++++- scripts/validation_heal/incarnation_replay.py | 218 +++++++++ scripts/validation_heal/mrf_records.py | 4 +- scripts/validation_heal/recipes.json | 2 +- scripts/validation_heal/run_case.sh | 6 + test_validation_heal_incarnation.py | 27 +- test_validation_heal_incarnation_replay.py | 459 ++++++++++++++++++ 15 files changed, 1219 insertions(+), 32 deletions(-) create mode 100644 scripts/validation_heal/incarnation_replay.py create mode 100644 test_validation_heal_incarnation_replay.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 4a14969e..e9b10735 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -49,6 +49,7 @@ on: - test_validation_heal_outcome.py - test_validation_heal_persistence.py - test_validation_heal_incarnation.py + - test_validation_heal_incarnation_replay.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -111,6 +112,7 @@ on: - test_validation_heal_outcome.py - test_validation_heal_persistence.py - test_validation_heal_incarnation.py + - test_validation_heal_incarnation_replay.py - scripts/validation_heal/** - docs/validation-heal-*.md - rustfs_heal_test.sh @@ -152,7 +154,7 @@ jobs: - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py - - run: python3 -B -m unittest -v test_validation_heal_incarnation.py + - run: python3 -B -m unittest -v test_validation_heal_incarnation.py test_validation_heal_incarnation_replay.py - run: python3 -B -m unittest -v test_validation_heal_incarnation_native.py - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index 88ad2bd8..0d6c47ce 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -670,7 +670,7 @@ Topologies: H1,H2. Branches: old-metadata-rejoin, old-checkpoint-replay, old-mrf Execution policy: `evidence-blocked`; disabled by default: `false`. -Blocked reason (`EVIDENCE_INCARNATION_LIVE_ORCHESTRATION`): 已有 3 项候选版本 incarnation 原生回归通过，但尚缺覆盖旧盘回归、旧 checkpoint/MRF 重放与同名新桶的完整真机编排和证据绑定；原生回归不能替代四个分支的 canonical 验收。 +Blocked reason (`EVIDENCE_INCARNATION_LIVE_ORCHESTRATION`): 已实现 H1/H2 旧元数据重返、真实 checkpoint/resume 回放、ACK 绑定的 committed MRF 回放和精确 native 证据门禁；尚缺全部现场分支的独立审计，诊断执行保持 FAIL/BLOCKED，原生前置通过不能替代 canonical 验收。 1. 建旧 DV 桶并让一盘离线；按 S3 正常列举/删除全部版本和 marker，确认 delete_bucket 成功，禁止直接内部清空绕过删除。 2. 同名 create_bucket 后写新内容，保存新 incarnation；恢复旧盘并让旧任务/持久 MRF 重放。 diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index 4dc8bd03..654a6391 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -32,7 +32,7 @@ | Code | 用例 | Block 原因 | 解锁所需证据 | | --- | --- | --- | --- | -| C11 | `heal.bucket-incarnation-recreation` | 已有 3 项 incarnation 原生回归及 `incarnation.py` 的 H1/H2 旧元数据盘回归编排；后者尚无 live 结果。仍缺旧 checkpoint/MRF 重放与精确提交屏障 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | +| C11 | `heal.bucket-incarnation-recreation` | H1/H2 旧元数据盘重返、原始 checkpoint/resume 与 committed MRF 重放编排已实现；精确候选的 9 项存储原生回归和 3 项 manager incarnation 前置证据已独立核验。完整现场分支尚无独立审计结果 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | | F07 | `heal.checkpoint-and-mrf-corruption` | 已有 journal 工具和 10 项原生回归；缺 H1/H2 全六分支、写失败诊断、责任保留和安全重扫编排 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | | I05 | `heal.lock-timeout-and-caller-abort` | 请求耗时/TCP 断开不能证明服务端锁 owner 与提交顺序 | lock owner、timeout、caller abort 和 stale read-repair 的服务端屏障 | | J02 | `heal.outcome-counters-and-error-truth` | 已有 2 项 outcome 原生回归；缺混合 healthy/repaired/failed/deferred/unknown 真机夹具、dry-run 独立任务与完整 manifest 对账 | 每个 disposition、计数和 attemptFailures 对账；不是已证实产品缺少 API | diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index a148c63b..12921a19 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1934,3 +1934,14 @@ E03 followup `38235099954` uses package `38231599023/1` from exact source `e224e The complete E03 followup `38237270640` passed all six required H1/H2 DB/DV instances with clean finalization. Independent audit rechecked the original canonical result, every referenced member digest and exact package/binary/helper provenance. It uses the same source `e224eb48e5311cff022b04fd78c97a8572d5c270` and package `38231599023/1`, with helper `8ea38a9d8044e512de91ec788d79ecb7aa2f0fd8`; the preceding suppression failure remains retained. C08 `38235978152` also passed independent complete canonical audit. The derived ledger now records 101 complete PASS, 33 retained live failures, seven required native barriers, four orchestration/evidence blockers, and 13 disabled environment scenarios. B07 `38236702473` remains FAIL with clean finalization: H1's below-quorum S3 response and H2's negative terminal responsibility violate the frozen oracles. C11's native collector now binds reviewed source `1aa9be87d18cf941e5020f19e8622c26ba4b10ac`, its exact tree, Cargo.lock, six storage sources, helper digest and an optimized independent test executable. The nine-test suite preserves all six original native regressions and adds three real storage incarnation tests. The final-rename test pauses actual shard reconstruction before publication, aborts its caller while the lifecycle writer remains pending, drains and physically verifies the exact repaired version, and rejects obsolete admission after normal delete/recreate. Four original ordered milestones are mandatory; omitted, duplicated, reordered or invented rows cannot certify the barrier. Execution uses captured successful output and the candidate's bounded larger ecstore test stack. Local nextest passed 28 focused tests; the direct libtest check with captured output also passed. These local results are not hosted optimized native evidence. C11 independently verifies any hosted storage artifact before fleet mutation and records its scope as a native prerequisite with no fleet topology claim. Old checkpoint/MRF replay is still blocked, the metadata branch has no live result, and C11 remains outside dispatch. + + +C11 original-record replay implementation (2026-10-12) + +C11 now implements both remaining H1/H2 DV replay flows through the existing Fleet, Probe, Recorder, checkpoint and read-only MRF observers. Stopped descriptor-relative transactions bind run, candidate binary, old incarnation and original task, durably archive all original bytes before detachment, and restore only those bytes. The MRF allowlist also includes both source-defined legacy fallback journals; whole-record CRC and exact original intent bindings prevent omitted or foreign replay sources. Checkpoint terminals and new product responsibility cannot be overwritten. Empty transition fallback files carry no responsibility and are recorded before removal. Phase changes verify archive and transaction identities and fail closed on interrupted configuration commits. Incomplete capture/install or unresolved S3 mutation keeps every private namespace and original archive. + +The old MRF fault gate is released while stopped after all original replay sources are detached and before normal S3 bucket deletion/recreation. Successor and control objects are written on a healthy fleet, independently verified, and frozen before original records are installed. Replayed checkpoints must reject the old generation while retaining original progress and durable terminal/report ownership. Replayed MRF must retain the exact responsibility ID, intent digest and source incarnation under a new process checkpoint owner with greater sequence, durably held as bucket_incarnation_changed without operator acceptance. Complete successor/control physical bytes, formats, system records, full shard integrity and every peer's S3/configuration remain independent evidence. + +Immutable package run 38240858236/1 and tool run 38244986993/1 succeeded for source 1aa9be87d18cf941e5020f19e8622c26ba4b10ac. Independent Git-object checks verified the exact source tree, lock and all source contracts; all seven MRF sources match the reviewed reference byte-for-byte. The complete 68,625,642-byte DEB digest is b99dd915376b51f0896fc346ca4642b7d26bf3352135bb5c6f60f8179d865761 and its contained binary digest is 0103ca46386396ed15f27b9efbe4196fe62bd9e93033671a88f3b3e944602cf4. Inspector, nine native storage regressions and fifteen manager regressions passed independent artifact consumers for harness d5a41739a7a544d93ba71f51583d45ad4e6161eb. A new immutable tool build is required for the updated replay harness; older harness provenance is not accepted. + +The full Heal contract suite passed 591 tests with eight platform skips before the additional finalization-verdict regression; all 29 focused incarnation tests, the unchanged 84/301/532 matrix, AST, bash syntax, actionlint and whitespace passed on the final code. Two sequential final reviews covered correctness, simplicity, test coverage, path/artifact security and persistence/cancellation. Explicit diagnostic execution cannot claim canonical PASS; clean finalization does not hide a failed branch. C11 remains outside automatic dispatch and complete canonical PASS remains 101/158 pending live execution and independent audit. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index af398d9f..54376a89 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-markers-and-batch-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update|heal.mixed-version-upgrade-rollback|heal.dedicated-host-forced-restart|heal.race-ilm-expiry-and-transition|heal.race-replication-resync|heal.with-expansion-rebalance|heal.with-rebalance-stop-resume|heal.with-decommission|heal.with-decommission-cancel-clear|heal.after-retired-pool-removal|heal.ec84-multi-set-restart|heal.slow-io-and-eio|heal.kms-and-tier-outage|heal.long-soak|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) + heal.bucket-incarnation-recreation|heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-markers-and-batch-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update|heal.mixed-version-upgrade-rollback|heal.dedicated-host-forced-restart|heal.race-ilm-expiry-and-transition|heal.race-replication-resync|heal.with-expansion-rebalance|heal.with-rebalance-stop-resume|heal.with-decommission|heal.with-decommission-cancel-clear|heal.after-retired-pool-removal|heal.ec84-multi-set-restart|heal.slow-io-and-eio|heal.kms-and-tier-outage|heal.long-soak|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py index 05936d93..b67a525c 100644 --- a/scripts/validation_heal/execution_policy.py +++ b/scripts/validation_heal/execution_policy.py @@ -129,7 +129,7 @@ def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, . EVIDENCE_BLOCKED_CASES = { "heal.bucket-incarnation-recreation": _native( "C11", "EVIDENCE_INCARNATION_LIVE_ORCHESTRATION", - "已有 3 项候选版本 incarnation 原生回归通过，但尚缺覆盖旧盘回归、旧 checkpoint/MRF 重放与同名新桶的完整真机编排和证据绑定；原生回归不能替代四个分支的 canonical 验收。", + "已实现 H1/H2 旧元数据重返、真实 checkpoint/resume 回放、ACK 绑定的 committed MRF 回放和精确 native 证据门禁；尚缺全部现场分支的独立审计，诊断执行保持 FAIL/BLOCKED，原生前置通过不能替代 canonical 验收。", ("bucket_incarnation_isolation", "checkpoint_mrf_replay_guard", "incarnation_native_evidence_binding", "complete_live_orchestrator"), ), "heal.checkpoint-and-mrf-corruption": _native( diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index b4428a01..4df2b2f1 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -33,6 +33,12 @@ def runtime_config(profile): + if profile in {"incarnation-checkpoint", "incarnation-mrf"}: + logging_filter = "info,rustfs::heal::manager=debug,rustfs::heal::task=debug,rustfs::heal::mrf=debug" + return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true" if profile == "incarnation-mrf" else "false", + "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false", "RUSTFS_HEAL_MAX_CONCURRENT_HEALS": "1", + "RUSTFS_HEAL_MAX_CONCURRENT_PER_SET": "1", "RUSTFS_OBS_LOGGER_LEVEL": logging_filter, + "RUST_LOG": logging_filter} require(profile in {"queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") if profile == "queue-pressure": logging_filter = "info,rustfs::heal::manager=debug,rustfs::heal::task=debug,rustfs::heal::mrf=debug" @@ -91,6 +97,88 @@ def runtime_config(profile): return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true" if profile == "offline-replay" else "false"} +def runtime_settings(owner): + config = runtime_config(owner.get("runtimeProfile", "manual")) + if owner.get("incarnationPhase") == "transition": + config.update(RUSTFS_HEAL_MRF_ENABLE="false") + return config + + +def incarnation_configuration(state, owner): + env = checked(state / "service.env", state) + info = env.stat() + require(info.st_uid == 0 and info.st_nlink == 1 and stat.S_ISREG(info.st_mode) and info.st_size <= 65536, + "C11 configuration lost its bounded private ownership") + lines = env.read_text().splitlines() + require(all("=" in line for line in lines), "C11 configuration has an invalid entry") + values = dict(line.split("=", 1) for line in lines) + require(len(lines) == len(values) and all(values.get(key) == value for key, value in runtime_settings(owner).items()), + "C11 phase configuration drifted or its durable phase commit is incomplete") + return values + + +def incarnation_operation(request): + require(set(request) <= {"runId", "action", "family", "taskId", "bucketIncarnationId", "expectedRecords", "phase"}, + "C11 adapter accepts no arbitrary paths, replacement bytes or commands") + state, roots, owner = owned(request["runId"]) + stopped(owner) + require(owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf"} + and digest(state / "rustfs") == owner["binarySha256"], "C11 replay runtime/candidate changed") + backend = globals()["incarnation_replay_backend"] + journal, fault = globals()["journal_fault_backend"], fault_module() + if request["action"] == "incarnation-phase": + require(set(request) == {"runId", "action", "phase"} and request["phase"] in {"transition", "replay"}, + "C11 phase cannot change arbitrary environment settings") + current = owner.get("incarnationPhase", "capture") + require((current, request["phase"]) in {("capture", "transition"), ("transition", "replay")}, + "C11 phase transition cannot refund an earlier replay window") + try: + with fault.directory(state) as fd: + detached = journal.decode_json(fault.read_file(fd, "C11-replay-detached.json", 16384)) + installed = journal.decode_json(fault.read_file(fd, "C11-replay-installed.json", 16384)) if current == "transition" else None + require(isinstance(detached, dict), "C11 phase transaction binding is invalid") + params = {key: detached.get(key) for key in ("runId", "family", "taskId", "bucketIncarnationId")} + binding = backend.identity(owner, params, journal) + require(journal.same_json(detached, binding) and (current == "capture" or journal.same_json(installed, binding)), + "C11 phase transaction changed its original run/candidate/task/incarnation binding") + manifest = backend.archived(state, roots, owner, params, journal, fault) + records = backend.census(roots, binding, journal, fault) + require(all(not row["present"] for row in records) if current == "capture" + else journal.same_json(records, manifest["records"]), + "C11 phase runnable records differ from the completed original transaction") + except FileNotFoundError as exc: + raise ValueError("C11 phase requires a complete original record transaction") from exc + values = incarnation_configuration(state, owner) + owner["incarnationPhase"] = request["phase"] + replacement = runtime_settings(owner) + env = state / "service.env" + values.update(replacement) + temporary = state / "C11-service.env.new" + with fault_module().directory(state) as fd: + fault_module().new_file(fd, temporary.name, ("\n".join(f"{key}={value}" for key, value in values.items()) + "\n").encode()) + os.rename(temporary.name, env.name, src_dir_fd=fd, dst_dir_fd=fd) + os.fsync(fd) + with fault_module().directory(state) as fd: + fault_module().new_file(fd, "C11-phase.new", fault_module().encode({"runId": owner["runId"], "phase": request["phase"]})) + os.rename("C11-phase.new", "C11-phase.json", src_dir_fd=fd, dst_dir_fd=fd) + os.fsync(fd) + return {"previousPhase": current, "phase": request["phase"], "candidateUnchanged": True, + "workerLimit": replacement["RUSTFS_HEAL_MAX_CONCURRENT_HEALS"], + "mrfEnabled": replacement["RUSTFS_HEAL_MRF_ENABLE"]} + require("phase" not in request, "C11 record operation accepts no runtime override") + if request["action"] == "incarnation-census": + require(set(request) == {"runId", "action", "family", "taskId", "bucketIncarnationId"}, + "C11 stopped census accepts only the original binding") + return {"readOnly": True, "stable": True, "records": backend.census( + roots, backend.identity(owner, request, journal), journal, fault)} + methods = {"incarnation-capture": backend.capture, "incarnation-install": backend.install, + "incarnation-finalize": backend.finalize} + require(request["action"] in methods, "unknown C11 record transaction") + if request["action"] == "incarnation-capture": + return backend.capture(state, roots, owner, request, journal, fault, globals()["mrf"], checked) + return methods[request["action"]](state, roots, owner, request, journal, fault) + + def require(condition, message): if not condition: raise ValueError(message) @@ -297,6 +385,15 @@ def owned(run_id): for root in roots: marker = checked(root / "owner.json", root) require(marker.stat().st_uid == 0 and json.loads(marker.read_text()) == owner, "drive owner differs") + if owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf"}: + phase_file = checked(state / "C11-phase.json", state) + if phase_file.exists(): + require(phase_file.stat().st_uid == 0 and phase_file.stat().st_nlink == 1, + "C11 phase marker lost its private root ownership") + phase = json.loads(phase_file.read_text()) + require(set(phase) == {"runId", "phase"} and phase["runId"] == run_id + and phase["phase"] in {"transition", "replay"}, "C11 phase marker differs") + owner = {**owner, "incarnationPhase": phase["phase"]} return state, roots, owner @@ -305,13 +402,15 @@ def start(request): stopped(owner) command("systemctl", "reset-failed", owner["unit"], check=False) require(digest(state / "rustfs") == owner["binarySha256"], "candidate changed before start") + if owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf"}: + incarnation_configuration(state, owner) command("systemd-run", "--unit=" + owner["unit"], "--property=Type=exec", "--property=KillMode=control-group", "--property=TimeoutStopSec=30", "--property=RuntimeMaxSec=" + str(resource_limits(owner.get("resourceProfile", "standard"))["runtimeMaxSeconds"]), "--property=MemoryMax=4G", *(["--property=PrivateMounts=true", "--property=CapabilityBoundingSet=~CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH"] if owner.get("runtimeProfile") in {"flapping", "queue-pressure"} else []), - *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") in {"root-restart", "journal-bounded", "durable-mrf", "mrf-crash", "replacement", "replacement-target", "schema-refusal"} else []), + *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") in {"root-restart", "journal-bounded", "durable-mrf", "mrf-crash", "replacement", "replacement-target", "schema-refusal", "incarnation-checkpoint", "incarnation-mrf"} else []), "--property=WorkingDirectory=" + str(state), "--property=EnvironmentFile=" + str(state / "service.env"), *gate_module().start_properties(gate_module().checked_gate(state, roots)), @@ -366,7 +465,7 @@ def process_once(request): executable = (root / "exe").resolve(strict=True) require(executable == state / "rustfs" and digest(executable) == owner["binarySha256"], "signal target is not the owned candidate") environment = dict(item.split("=", 1) for item in (root / "environ").read_bytes().decode().split("\0") if "=" in item) - settings = {**runtime_config(owner.get("runtimeProfile", "manual")), "RUSTFS_HEAL_TASK_TIMEOUT_SECS": "300"} + settings = {**runtime_settings(owner), "RUSTFS_HEAL_TASK_TIMEOUT_SECS": "300"} require(all(environment.get(key) == value for key, value in settings.items()), "owned process runtime changed") result = {**base, "state": before[0], "startTicks": int(before[19]), "executable": str(executable), "mountNamespace": os.readlink(root / "ns/mnt"), "commandSha256": hashlib.sha256((root / "cmdline").read_bytes()).hexdigest(), @@ -460,7 +559,7 @@ def process_graceful(request): def process_crash(request): - require(owned(request["runId"])[2].get("runtimeProfile") in {"replacement", "replacement-target", "root-restart", "journal-bounded", "durable-mrf", "mrf-crash"} + require(owned(request["runId"])[2].get("runtimeProfile") in {"replacement", "replacement-target", "root-restart", "journal-bounded", "durable-mrf", "mrf-crash", "incarnation-checkpoint", "incarnation-mrf"} and process(request).get("state") == "T", "crash observation requires a paused owned recovery service") before = pidfd_signal(request, signal.SIGKILL) deadline = time.monotonic() + 30 @@ -848,7 +947,7 @@ def snapshot(request): "running binary differs from candidate") environment = dict(value.split("=", 1) for value in Path(f"/proc/{pid}/environ").read_bytes().decode().split("\0") if "=" in value) - expected_config = runtime_config(owner.get("runtimeProfile", "manual")) + expected_config = runtime_settings(owner) require(all(environment.get(key) == value for key, value in expected_config.items()), "bound Heal runtime configuration differs") files, formats = [], [] @@ -981,7 +1080,7 @@ def configure_encryption(request): def journal_operation(request): state, roots, owner = owned(request["runId"]) require(digest(state / "rustfs") == owner["binarySha256"], "durable-record candidate changed") - require(owner.get("runtimeProfile") in {"root-restart", "mrf-crash", "journal-bounded"}, "durable records require the owned recovery runtime") + require(owner.get("runtimeProfile") in {"root-restart", "mrf-crash", "journal-bounded", "incarnation-checkpoint", "incarnation-mrf"}, "durable records require the owned recovery runtime") require(set(request) <= {"runId", "action", "family", "taskId", "disk", "relativePath", "faultId", "kind", "expectedBefore", "expected", "captureId"}, "durable-record adapter accepts no arbitrary system paths or commands") if owner.get("runtimeProfile") == "journal-bounded": @@ -1285,6 +1384,8 @@ def main(request): "verify-inline": verify_inline, "drive-gate": drive_gate, "drive-gate-probe": drive_gate_probe, "drive-gate-activate": drive_gate_activate, "gate-state": drive_gate_state, "drive-gate-release-live": drive_gate_release_live, "drive-gate-arm-live": drive_gate_arm_live} + for action in ("incarnation-census", "incarnation-capture", "incarnation-install", "incarnation-finalize", "incarnation-phase"): + actions[action] = incarnation_operation actions["task-records"] = task_records actions["environment-capabilities"] = environment_capabilities actions["quota-audit-finalize"] = quota_audit_finalize diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 524c240d..f4852b1f 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(runtime_profile in {"queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation", "incarnation-checkpoint", "incarnation-mrf"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -154,6 +154,7 @@ def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_prof self.generation_faults = [] self.orphan_faults = [] self.journal_faults = [] + self.incarnation_replays = [] self.secret_values = set() self.frozen_processes = {} self.snapshot_buckets = None @@ -191,7 +192,7 @@ def remote(self, node, action, **params): if self.runtime_profile in {"bounded-capacity", "journal-bounded"}: source += "\nimport types\nbounded_volume_backend = types.ModuleType('bounded_volume')\nexec(" + repr( NODE_SOURCE.with_name("bounded_volume.py").read_text()) + ", bounded_volume_backend.__dict__)\n" - if action == "mrf-records" or action.startswith("journal-"): + if action == "mrf-records" or action.startswith(("journal-", "incarnation-")): source += "\nimport types\nmrf = types.ModuleType('mrf_records')\nexec(" + repr( NODE_SOURCE.with_name("mrf_records.py").read_text()) + ", mrf.__dict__)\n" if action == "replacement-records": @@ -201,12 +202,15 @@ def remote(self, node, action, **params): source += "\nimport types\ngeneration_backend = types.ModuleType('generation_fault')\nexec(" + repr( NODE_SOURCE.with_name("generation_fault.py").read_text()) + ", generation_backend.__dict__)\n" if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline", - "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure"} or action.startswith("generation-") or action.startswith("orphan-") or action.startswith("journal-"): + "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure"} or action.startswith("generation-") or action.startswith("orphan-") or action.startswith(("journal-", "incarnation-")): source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" - if action.startswith("journal-"): + if action.startswith(("journal-", "incarnation-")): source += "\nimport types\njournal_fault_backend = types.ModuleType('journal_fault')\nexec(" + repr( NODE_SOURCE.with_name("journal_fault.py").read_text()) + ", journal_fault_backend.__dict__)\n" + if action.startswith("incarnation-"): + source += "\nimport types\nincarnation_replay_backend = types.ModuleType('incarnation_replay')\nexec(" + repr( + NODE_SOURCE.with_name("incarnation_replay.py").read_text()) + ", incarnation_replay_backend.__dict__)\n" if action.startswith("orphan-"): source += "\nimport types\norphan_backend = types.ModuleType('orphan_fault')\nexec(" + repr( NODE_SOURCE.with_name("orphan_fault.py").read_text()) + ", orphan_backend.__dict__)\n" @@ -319,8 +323,24 @@ def wait_ready(self): time.sleep(2) require(not pending, "Heal deployment did not make every node ready") - def cleanup(self): - failures = [] + def cleanup(self, *, preserve_namespaces=False): + failures = [{"state": "DIRTY", "namespacesRetained": True, + "error": "unresolved C11 S3 mutation requires retained physical evidence"}] if preserve_namespaces else [] + if self.incarnation_replays: + try: + self.stop() + for node, params in tuple(self.incarnation_replays): + receipt = self.record("C11-original-replay-evidence-finalization", {"node": node, + **self.remote(node, "incarnation-finalize", **params)}) + require(receipt.get("originalEvidenceExported") is True + and receipt.get("namespaceFinalizationAllowed") is True + and receipt.get("backupRescueUsed") is False + and receipt.get("productRepairClaimed") is False, + "C11 original replay evidence was not exported before namespace finalization") + self.incarnation_replays.remove((node, params)) + except Exception as exc: + failures.append({"error": str(exc), "state": "DIRTY", "namespacesRetained": True}) + if self.journal_faults: try: self.stop() diff --git a/scripts/validation_heal/incarnation.py b/scripts/validation_heal/incarnation.py index 944b6f1c..6e3dc5af 100644 --- a/scripts/validation_heal/incarnation.py +++ b/scripts/validation_heal/incarnation.py @@ -1,4 +1,4 @@ -"""C11: same-name bucket recreation with one actual stale metadata member.""" +"""C11: same-name bucket recreation with original metadata and journal replay.""" import argparse from datetime import datetime, timedelta, timezone @@ -10,6 +10,8 @@ import secrets import signal import sys +import time +from urllib.parse import quote import uuid try: @@ -23,7 +25,12 @@ from .incarnation_native import verify as verify_storage from .matrix import select from .shards import files, map_target, run_task, verify_cohort - from .stop import idle_fleet + from . import checkpoint as checkpoints, journal_fault as journal, mrf_replay as mrf + from .mrf_records import canonical_uuid, verify_reference as verify_mrf_reference + from .persistence import exact_records + from .scanner_repair import JournalWindow + from .stop import idle_fleet, status + from .root_restart import recovery_status from .versions import public_version_id, validate_version_receipts, verify_quorum_contract, version_identity except ImportError: from heal_probe import Probe, compare_inventories, payload, require, save @@ -36,16 +43,17 @@ from incarnation_native import verify as verify_storage from matrix import select from shards import files, map_target, run_task, verify_cohort - from stop import idle_fleet + import checkpoint as checkpoints, journal_fault as journal, mrf_replay as mrf + from mrf_records import canonical_uuid, verify_reference as verify_mrf_reference + from persistence import exact_records + from scanner_repair import JournalWindow + from stop import idle_fleet, status + from root_restart import recovery_status from versions import public_version_id, validate_version_receipts, verify_quorum_contract, version_identity CASE_ID = "heal.bucket-incarnation-recreation" BRANCHES = ("old-metadata-rejoin", "old-checkpoint-replay", "old-mrf-replay", "native-incarnation-fence") -BLOCKERS = { - "old-checkpoint-replay": "exact old-incarnation runnable checkpoint replay and original terminal evidence are incomplete", - "old-mrf-replay": "ACK-bound old-incarnation committed MRF replay and retained responsibility evidence are incomplete", -} SIZE = 5 * 1024**2 + 123 @@ -294,7 +302,7 @@ def run_metadata_rejoin(args, topology, nodes, candidate): if pending is not None: result["unresolvedMutation"] = pending fleet.record("unresolved-S3-mutation", pending) - clean = fleet.cleanup() + clean = fleet.cleanup(preserve_namespaces=True) result["cleanup"] = "FAIL" result["environmentCleanupSucceeded"] = clean else: @@ -303,6 +311,343 @@ def run_metadata_rejoin(args, topology, nodes, candidate): return result + +def record_digests(rows): + return [{name: row[name] for name in ("disk", "relativePath", "size", "sha256")} + for row in rows if row["present"]] + + +def checkpoint_rejection(current, frozen): + outcome = current.get("outcome") or {} + original = frozen["value"]["admin"]["outcome"] + require(current.get("summary") == "stopped" and journal.same_json(current.get("settings"), checkpoints.OPTIONS) + and isinstance(current.get("detail"), str) and current["detail"].startswith("stale_bucket_incarnation:") + and outcome.get("execution") == {"state": "aborted", "reason": "untraversable"} + and outcome.get("coverage") == "partial" and current.get("truncated") is False, + "C11 old checkpoint has no explicit original stale-incarnation failure") + require(all(journal.same_json(outcome.get(name), original.get(name)) + for name in ("counters", "objects", "objectsTruncated")), + "C11 old checkpoint processed successor work or rewrote original acknowledged progress") + return {"originalTaskId": frozen["taskId"], "bucketIncarnationId": frozen["bucketIncarnationId"], + "staleIncarnationRejected": True, "originalProgressRetained": True, "successorWorkClaimed": False} + + +def checkpoint_terminal(observed, frozen, current): + token = frozen["taskId"] + names = journal.names("root", token) + records = exact_records(observed, token, "root") + require(set(records) == set(names[1:]), "C11 old checkpoint lacks an exact durable terminal/report or is still runnable") + owner = frozen["rootAdmission"] + require(all((row["node"], row["disk"]) == (owner["node"], owner["disk"]) for row in records.values()), + "C11 old checkpoint terminal moved original durable ownership") + terminal = journal.decode_json(checkpoints.bytes_of(records[names[1]])) + report = journal.decode_json(checkpoints.bytes_of(records[names[2]])) + require(type(terminal.get("schema")) is int and terminal["schema"] == 1 and terminal.get("task_id") == token + and journal.same_json(terminal.get("heal_type"), owner["value"]["heal_type"]) + and journal.same_json(terminal.get("options"), owner["value"]["options"]) + and terminal.get("status") == {"Failed": {"error": current["detail"]}} + and type(report.get("schema")) is int and report["schema"] == 1 + and journal.same_json(report.get("terminal"), terminal) + and journal.same_json(report.get("outcome"), current["outcome"]), + "C11 old checkpoint durable rejection differs from original token/scope/outcome") + return {"originalTaskId": token, "sameOriginalOwner": True, "durableFailureRetained": True} + + +def retained_mrf(checkpoints_by_node, obligations, bindings, successor_incarnation): + require(successor_incarnation and all(owner["bucketIncarnationId"] != successor_incarnation + for owners in bindings.values() for owner in owners), "C11 successor reused the old MRF incarnation") + retained = [] + for obligation in obligations: + wanted = mrf.identity(obligation) + for owner in bindings[wanted]: + value = checkpoints_by_node[owner["node"]] + require(value.get("present") is True and value["owner"] != owner["checkpointOwner"] + and value["sequence"] > owner["sequence"], "C11 old MRF responsibility was lost or not durably reclassified") + canonical_uuid(value["owner"]) + matches = [item for item in value["intents"] if mrf.identity(item) == wanted + and item.get("intentDigest") == owner["intentDigest"]] + require(len(matches) == 1, "C11 exact old MRF ACK responsibility disappeared or duplicated") + lifecycle = matches[0].get("lifecycle", {}) + state = lifecycle.get("state", {}) + require(lifecycle.get("responsibility_id") == owner["responsibilityId"] + and lifecycle.get("source_bucket_incarnation_id") == owner["bucketIncarnationId"] + and lifecycle.get("last_operator_acceptance") is None + and state.get("state") == "bucket_incarnation_changed" + and set(state) == {"state", "source_bucket_incarnation_id", "observed_bucket_incarnation_id", "detected_at_ms"} + and state["source_bucket_incarnation_id"] == owner["bucketIncarnationId"] + and state["observed_bucket_incarnation_id"] == successor_incarnation + and type(state["detected_at_ms"]) is int and 0 < state["detected_at_ms"] <= int(time.time() * 1000) + 5000, + "C11 MRF responsibility was retired, accepted, rebound or lacks durable incarnation-change diagnostics") + retained.append({"node": owner["node"], "identity": obligation, + "originalCheckpointOwner": owner["checkpointOwner"], "currentCheckpointOwner": value["owner"], + "responsibilityId": owner["responsibilityId"], "state": state}) + return {"retainedResponsibilities": retained, "operatorAcceptanceUsed": False, + "oldResponsibilityRetired": False, "successorRepairClaimed": False} + + +def replay_isolation(before, after): + require(physical_content(before) == physical_content(after), + "C11 old persisted work changed successor/control metadata, shards, integrity or disk formats") + return {"allSuccessorAndControlPhysicalBytesUnchanged": True, "formatUnchanged": True} + + +def successor_receipts(current, bucket, successor): + successor_incarnations = {row.get("identity", {}).get("bucketIncarnationId") + for row in current.get("outcome", {}).get("objects", [])} + require(len(successor_incarnations) == 1 and None not in successor_incarnations, + "C11 successor diagnostics mix or omit bucket generations") + successor_incarnation = successor_incarnations.pop() + # Complete independent manifests cover diagnostics capped at 128 objects. + wanted = old_manifest(successor) + journal.task_id(successor_incarnation) + outcome = current.get("outcome") or {} + require(current.get("summary") == "finished" and type(current.get("truncated")) is bool + and current["truncated"] is (len(wanted) > 128) and outcome.get("execution") == {"state": "completed"} + and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is (len(wanted) > 128) + and len(outcome.get("objects", [])) == min(128, len(wanted)), + "C11 successor scan lacks truthful bounded diagnostic coverage") + counts = {"processed": len(wanted), "healed": 0, "unchanged": len(wanted), "skipped": 0, + "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False} + require(all(type(outcome.get("counters", {}).get(key)) is type(value) + and outcome["counters"][key] == value for key, value in counts.items()), + "C11 successor scan counters differ from its complete ACK manifest") + seen = set() + for row in outcome["objects"]: + item = row.get("identity", {}) + pair = item.get("object"), public_version_id(item.get("versionId")) + require(pair in wanted and pair not in seen and item.get("bucket") == bucket + and item.get("bucketIncarnationId") == successor_incarnation + and type(item.get("poolIndex")) is int and item["poolIndex"] == 0 + and type(item.get("setIndex")) is int and item["setIndex"] == 0 + and row.get("disposition", {}).get("state") == ("metadata_healthy" if wanted[pair]["deleteMarker"] else "verified_healthy"), + "C11 successor diagnostic changed an ACK, kind, generation or successful disposition") + seen.add(pair) + return {"bucketIncarnationId": successor_incarnation, + "completeIndependentManifestsRequired": True, "boundedDiagnosticsAreCompleteManifest": len(wanted) <= 128} + + +def run_replay(args, topology, nodes, branch, candidate): + require(branch in {"old-checkpoint-replay", "old-mrf-replay"}, "unregistered C11 replay branch") + family = "checkpoint" if branch == "old-checkpoint-replay" else "mrf" + directory = args.evidence / topology / branch + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="incarnation-" + family, + resource_profile="cursor" if family == "checkpoint" else "standard") + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + control = bucket + "-control" + fleet.snapshot_buckets = [bucket, control] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 512, "maxInventoryBytes": 2 * 1024**3} + fleet.record("binding", binding) + tool = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + verify_quorum_contract(tool, candidate) + if family == "mrf": fleet.record("MRF-reviewed-source-contract", verify_mrf_reference(candidate)) + probe, active, pending = Probe(binding), [], None + result = {"topology": topology, "fixture": "DV", "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING", + "canonicalCasePassClaimed": False} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("actual-topology", validate_topology(fleet.snapshot("deployed-topology"), topology)) + recorder = Recorder(probe, directory / "fixture") + pending = {"operation": "seed-old-DV"} + old = seed(recorder, bucket, "DV", 80 if family == "checkpoint" else 3, [1024**2 + 123]) + old_manifest(old) + fleet.record("old-ACK-manifest", {"expected": old, **verify_seed(old, probe.inventory(bucket))}) + pending = None + frozen, bindings, obligations, gate_member = None, None, None, None + if family == "checkpoint": + token = checkpoints.admit(probe, fleet, active) + frozen, process_identity = checkpoints.freeze(fleet, token, bucket) + frozen = checkpoints.stop_for_fault(fleet, token, bucket, frozen, process_identity) + old_incarnation = frozen["bucketIncarnationId"] + fleet.record("original-old-incarnation-checkpoint", frozen) + else: + initial = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, + option_overrides={"recursive": True}) + old_incarnation = validate_version_receipts(initial, bucket, old, 2)["bucketIncarnationId"] + idle_fleet(fleet, binding) + fleet.stop() + original = fleet.snapshot("old-before-partial-write-gate", quiescent=True) + entry = next(entry for entry in old if not entry["deleteMarker"]) + target = map_target(original, bucket, entry["key"], "data", 1, entry["versionId"], latest_required=False) + shard = next(row for row in reversed(target["cohort"]) if row["node"] == nodes[-1]) + gate_member = shard["node"], shard["disk"] + fleet.drive_gates.add(gate_member[0]) + fleet.record("old-MRF-member-gate", fleet.remote(gate_member[0], "drive-gate", disk=gate_member[1], mode="permission-denied")) + fleet.start() + denial = fleet.record("old-MRF-member-denial", fleet.remote(gate_member[0], "drive-gate-probe")) + require(denial.get("readProbe", {}).get("errno") == 13 and denial.get("errno") == 13 + and denial.get("dacBypassRemoved") is True, "C11 MRF partial write has no actual read/write denial") + earliest = int(time.time() * 1000) - 5000 + pending = {"operation": "old-MRF-partial-write-ACKs"} + old, obligations, _ = mrf.acknowledge(recorder, bucket, "offline-put", old) + verify_seed(old, probe.inventory(bucket)) + pending = None + deadline = time.monotonic() + mrf.PERSIST_SECONDS + while True: + try: + bindings = mrf.pending_bindings(mrf.read_checkpoints(fleet, nodes, "C11-old-committed-MRF"), obligations, earliest) + break + except ValueError as exc: + if not str(exc).startswith("NOT_OBSERVED:"): raise + require(time.monotonic() < deadline, "C11 old ACK-bound MRF persistence window not observed") + time.sleep(.5) + require(all(owner["bucketIncarnationId"] == old_incarnation for owners in bindings.values() for owner in owners), + "C11 MRF ACKs belong to another old generation") + fleet.record("original-old-MRF-ACK-bindings", {"obligations": obligations, + "bindings": [{"identity": list(key), "owners": value} for key, value in bindings.items()]}) + fleet.stop() + bindings = mrf.pending_bindings(mrf.read_checkpoints(fleet, nodes, "C11-old-MRF-stopped", quiescent=True), obligations, earliest) + token = None + params = {"family": family, "taskId": token, "bucketIncarnationId": old_incarnation} + for node in nodes: + if family == "checkpoint": + observed = [*fleet.remote(node, "journal-census", family="checkpoint", taskId=token)["records"], + *fleet.remote(node, "journal-census", family="root", taskId=token)["records"]] + # The adapter uses disk-major order over both exact families. + observed.sort(key=lambda row: (row["disk"], [*journal.names("checkpoint", token), *journal.names("root", token)].index(row["relativePath"]))) + else: + observed = fleet.remote(node, "incarnation-census", **params)["records"] + fleet.incarnation_replays.append((node, params.copy())) + receipt = fleet.record("archive-and-detach-original-replay", {"node": node, **fleet.remote( + node, "incarnation-capture", **params, expectedRecords=record_digests(observed))}) + require(receipt.get("detached") is True and receipt.get("originalProductBytes") is True + and receipt.get("replacementBytesAccepted") is False, "C11 original durable record detachment is incomplete") + fleet.record("C11-transition-runtime", {"node": node, **fleet.remote(node, "incarnation-phase", phase="transition")}) + if gate_member is not None: + fleet.record("C11-old-MRF-member-release", fleet.remote(gate_member[0], "drive-gate", mode="clear")) + fleet.drive_gates.remove(gate_member[0]) + fleet.start() + pending = {"operation": "normal-S3-delete-and-successor-recreate"} + result["deletedOriginal"] = fleet.record("normal-S3-bucket-deletion", delete_old_bucket(recorder, bucket, old)) + successor = seed_successor(recorder, bucket, old) + controls = seed(recorder, control, "DB", 1, [SIZE]) + baseline = fleet.record("successor-ACK-bound-S3", probe.inventory(bucket)) + verify_seed(successor, baseline) + fleet.record("successor-config-ACK", config(probe, bucket)) + pending = None + successor_status = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, + option_overrides={"recursive": True}) + successor_responsibility = successor_receipts(successor_status, bucket, successor) + require(successor_responsibility["bucketIncarnationId"] != old_incarnation, + "C11 successor generation was not distinct before any original replay") + idle_fleet(fleet, binding) + fleet.stop() + authority = fleet.snapshot("complete-successor-physical-before-original-replay", quiescent=True) + successor_physical(authority, authority, bucket, successor, old, (None, None)) + for node in nodes: + receipt = fleet.record("install-exact-original-replay", {"node": node, **fleet.remote(node, "incarnation-install", **params)}) + require(receipt.get("installed") is True and receipt.get("originalArchiveRetained") is True, + "C11 original record transaction was not completely installed") + fleet.record("C11-replay-runtime", {"node": node, **fleet.remote(node, "incarnation-phase", phase="replay")}) + authority = fleet.snapshot("successor-physical-before-original-replay", quiescent=True) + system_before = system_snapshot(fleet, [bucket, control], "successor-system-before-original-replay") + window = JournalWindow(fleet, "C11-original-replay", event_targets={"rustfs::heal::manager", "rustfs::heal::task", "rustfs::heal::mrf"}) + fleet.start() + if family == "checkpoint": + deadline = time.monotonic() + 180 + while True: + current = recovery_status(probe, fleet, token) + if current.get("summary") != "running": break + require(time.monotonic() < deadline, "C11 original checkpoint has no bounded terminal") + time.sleep(.2) + result["originalCheckpointRejection"] = checkpoint_rejection(current, frozen) + for endpoint in binding["endpoints"]: + peer = status(Probe(binding, endpoint), fleet, checkpoints.PATH, token, "C11-original-rejection-peer") + require(journal.same_json(peer.get("outcome"), current.get("outcome")) + and peer.get("summary") == current.get("summary") and peer.get("detail") == current.get("detail"), + "C11 peer lost the original rejected checkpoint outcome") + active.remove((checkpoints.PATH, token)) + else: + deadline = time.monotonic() + 180 + while True: + try: + result["retainedOldMrfResponsibility"] = retained_mrf( + mrf.read_checkpoints(fleet, nodes, "C11-old-MRF-replay-retained"), obligations, bindings, + successor_responsibility["bucketIncarnationId"]) + break + except ValueError as exc: + fleet.record("C11-MRF-rejection-not-yet-observed", {"error": str(exc)}) + require(time.monotonic() < deadline, "C11 MRF durable stale-generation diagnostic not observed: " + str(exc)) + time.sleep(1) + result["originalMrfReplayEvents"] = window.capture() + fleet.stop() + replay_isolation(authority, fleet.snapshot("successor-physical-after-original-replay-before-reads", quiescent=True)) + require(system_files(system_before) == system_files(system_snapshot(fleet, [bucket, control], "successor-system-after-original-replay")), + "C11 original replay changed successor bucket configuration/incarnation bytes") + if family == "checkpoint": + result["durableOriginalRejection"] = checkpoint_terminal(checkpoints.records( + fleet, token, "C11-durable-original-terminal", quiescent=True, family="root"), frozen, current) + fleet.start() + responsibility = successor_responsibility + if family == "mrf": + deadline = time.monotonic() + 180 + while True: + try: + result["retainedOldMrfResponsibility"] = retained_mrf(mrf.read_checkpoints(fleet, nodes, "C11-retained-old-MRF"), + obligations, bindings, responsibility["bucketIncarnationId"]) + break + except ValueError as exc: + fleet.record("C11-MRF-rejection-not-yet-observed", {"error": str(exc)}) + require(time.monotonic() < deadline, "C11 MRF durable stale-generation diagnostic not observed: " + str(exc)) + time.sleep(1) + result["originalMrfReplayEvents"] = window.capture() + idle_fleet(fleet, binding) + fleet.stop() + final = fleet.snapshot("successor-physical-before-business-reads", quiescent=True) + # Persisted old work has no authority to change any surviving successor bytes. + replay_isolation(authority, final) + cohorts = successor_physical(final, final, bucket, successor, old, (None, None)) + cohorts.extend(map_target(final, control, entry["key"], "data", 1, entry["versionId"], latest_required=False) + for entry in controls if not entry["deleteMarker"]) + result["fullShardIntegrity"] = [verify_cohort(fleet, target, "successor-full-shard-integrity") for target in cohorts] + fleet.start() + for endpoint in binding["endpoints"]: + peer = Probe(binding, endpoint) + inventory = fleet.record("all-peer-successor-S3", {"endpoint": endpoint, **peer.inventory(bucket)}) + verify_seed(successor, inventory) + compare_inventories(baseline, inventory) + verify_seed(controls, fleet.record("all-peer-control-S3", {"endpoint": endpoint, **peer.inventory(control, unversioned_current_tags=True)})) + fleet.record("all-peer-successor-config", {"endpoint": endpoint, **config(peer, bucket)}) + for key, version in old_manifest(old): + missing = fleet.record("all-peer-old-incarnation-version", peer.s3("head_object", {"Bucket": bucket, "Key": key, "VersionId": version})) + require(missing.get("acknowledged") is False and missing.get("resultUnknown") is False + and missing.get("response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode") == 404, + "C11 original version/marker reappeared in the successor namespace") + fleet.stop() + replay_isolation(final, fleet.snapshot("successor-after-business-reads", quiescent=True)) + require(system_files(system_before) == system_files(system_snapshot(fleet, [bucket, control], "successor-system-after-business-reads")), + "C11 business reads changed acknowledged successor bucket records") + if family == "checkpoint": + checkpoint_terminal(checkpoints.records(fleet, token, "C11-retained-terminal-after-business-reads", quiescent=True, family="root"), frozen, current) + else: + retained_mrf(mrf.read_checkpoints(fleet, nodes, "C11-retained-MRF-after-business-reads", quiescent=True), + obligations, bindings, responsibility["bucketIncarnationId"]) + result.update(verdict="PASS", checkpointReplayClaimed=family == "checkpoint", mrfReplayClaimed=family == "mrf", + successorResponsibility=responsibility, oldBackupRestorationUsed=False, nativeCommitFenceClaimed=False) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + try: + fleet.stop() + fleet.snapshot("original-replay-failure-scene", quiescent=True) + except Exception as error: result["failureEvidenceError"] = str(error) + finally: + clean = fleet.cleanup(preserve_namespaces=pending is not None) + result["cleanup"] = "PASS" if clean and pending is None else "FAIL" + if pending is not None: + result.update(unresolvedMutation=pending, environmentCleanupSucceeded=clean) + fleet.record("incarnation-replay-result", result) + return result + + def main(): def interrupted(signum, frame): raise KeyboardInterrupt(f"C11 interrupted by signal {signum}") signal.signal(signal.SIGTERM, interrupted) @@ -335,14 +680,16 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"C11 interrupted by sig "evidenceScope": "independent exact-candidate native storage regression", "nativeProof": storage, "canonicalCasePassClaimed": False} else: - result = {"topology": topology, "fixture": "DV", "branch": branch, "verdict": "BLOCKED", "cleanup": "PASS", - "reason": BLOCKERS[branch], "fleetBranchExecuted": False, "canonicalCasePassClaimed": False} + result = run_replay(args, topology, selected, branch, candidate) results.append(result) if result["cleanup"] != "PASS": return 1 finally: + diagnostic_complete = len(results) == 8 and all(row["verdict"] == ( + "PREREQUISITE_PASS" if row["branch"] == "native-incarnation-fence" else "PASS") for row in results) save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "complete": False, + "diagnosticBranchesPassed": diagnostic_complete, "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL", - "canonicalCasePassClaimed": False, "reason": "old checkpoint/MRF replay remains blocked"}) + "canonicalCasePassClaimed": False, "reason": "C11 full case review and canonical dispatch remain gated"}) return 1 diff --git a/scripts/validation_heal/incarnation_replay.py b/scripts/validation_heal/incarnation_replay.py new file mode 100644 index 00000000..6e60ea2a --- /dev/null +++ b/scripts/validation_heal/incarnation_replay.py @@ -0,0 +1,218 @@ +"""C11: archive and replay only original, candidate-owned recovery records. + +The stopped adapter accepts identities and digests, never caller-supplied +replacement bytes. The archive survives every mutation and is exported before +the private namespace can be removed. +""" + +import base64 +import hashlib +import json +import os + + +LEGACY_MRF_PATHS = (".rustfs.sys/buckets/.heal/mrf/journal.bin", + ".rustfs.sys/buckets/.heal/mrf/journal-scoped.bin") + + +def require(condition, message): + if not condition: raise ValueError(message) + + +def profile(value): + return value in {"incarnation-checkpoint", "incarnation-mrf"} + + +def archive_name(relative): + return "C11-replay-" + hashlib.sha256(relative.encode()).hexdigest() + ".original" + + +def identity(owner, request, journal): + require(profile(owner.get("runtimeProfile")) and request.get("runId") == owner["runId"], + "C11 replay requires its exact private recovery runtime") + family = "checkpoint" if owner["runtimeProfile"] == "incarnation-checkpoint" else "mrf" + require(request.get("family") == family, "C11 replay family differs from the bound runtime") + incarnation = journal.task_id(request.get("bucketIncarnationId")) + token = journal.task_id(request.get("taskId")) if family == "checkpoint" else None + require(family == "checkpoint" or request.get("taskId") is None, "MRF cannot invent a root task binding") + return {"runId": owner["runId"], "binarySha256": owner["binarySha256"], "bucket": owner["bucket"], + "family": family, "taskId": token, "bucketIncarnationId": incarnation} + + +def paths(binding, journal): + if binding["family"] == "mrf": return [*journal.names("mrf"), *LEGACY_MRF_PATHS] + return [*journal.names("checkpoint", binding["taskId"]), *journal.names("root", binding["taskId"])] + + +def read_optional(fd, relative, fault, limit): + try: + with fault.parent(fd, ["data", *relative.split("/")[:-1]]) as directory: + return fault.read_file(directory, relative.split("/")[-1], limit) + except FileNotFoundError: + return None + + +def census(roots, binding, journal, fault): + rows = [] + for disk, root in enumerate(roots): + with fault.directory(root) as fd: + for relative in paths(binding, journal): + raw = read_optional(fd, relative, fault, journal.LIMIT * 4) + row = {"disk": disk, "relativePath": relative, "present": raw is not None} + if raw is not None: + row.update(journal.entry(raw), rawBase64=base64.b64encode(raw).decode()) + rows.append(row) + return rows + + +def capture(state, roots, owner, request, journal, fault, mrf, checked): + require(owner.get("incarnationPhase", "capture") == "capture", "C11 cannot capture a later replay phase") + binding = identity(owner, request, journal) + rows = census(roots, binding, journal, fault) + actual = [{name: row[name] for name in ("disk", "relativePath", "size", "sha256")} + for row in rows if row["present"]] + require(journal.same_json(actual, request.get("expectedRecords")), + "C11 original stopped record census differs from its independently bound digests") + if binding["family"] == "checkpoint": + allowed = {journal.names("checkpoint", binding["taskId"])[0], + journal.names("checkpoint", binding["taskId"])[1], + journal.names("root", binding["taskId"])[0]} + require(all(row["relativePath"] in allowed for row in actual), + "C11 checkpoint capture includes a terminal, blocked or digest record") + if actual: + names = [row["relativePath"] for row in actual] + require(len(names) == len(set(names)), "C11 checkpoint has duplicated physical record owners") + require((journal.names("checkpoint", binding["taskId"])[0] in names) + == (journal.names("checkpoint", binding["taskId"])[1] in names), + "C11 original checkpoint/resume pair is incomplete") + for row in rows: + if not row["present"]: continue + raw = base64.b64decode(row["rawBase64"], validate=True) + value = journal.decode_json(raw) + require(value.get("task_id") == binding["taskId"], "C11 captured a foreign original task") + if row["relativePath"] == journal.names("checkpoint", binding["taskId"])[0]: + value = journal.checkpoint_value(raw, binding["taskId"], binding["bucket"]) + require(value["admin"]["buckets"] == {binding["bucket"]: binding["bucketIncarnationId"]}, + "C11 captured checkpoint belongs to another bucket generation") + elif row["relativePath"] == journal.names("checkpoint", binding["taskId"])[1]: + require(value.get("schema_version") == 7 and value.get("task_type") == "admin_erasure_set" + and value.get("set_disk_id") == "pool_0_set_0" and value.get("completed") is False, + "C11 captured companion is not a runnable Admin resume state") + else: + require(value.get("schema") == 3 and value.get("heal_type") == { + "type": "erasure_set", "buckets": [], "set_disk_id": "pool_0_set_0", "all_buckets": True}, + "C11 captured root admission changed its original Admin erasure-set scope") + else: + committed = mrf.checkpoint(mrf.read_records(state, roots, owner, checked)) + require((not committed.get("present")) or committed.get("present") is True + and all(item["bucket"] == binding["bucket"] and item["kind"] == 3 + and item.get("lifecycle", {}).get("source_bucket_incarnation_id") == binding["bucketIncarnationId"] + and item["lifecycle"].get("state") == {"state": "active"} + and item["lifecycle"].get("last_operator_acceptance") is None for item in committed["intents"]), + "C11 MRF archive lacks committed active responsibility for the exact old incarnation") + digests = {item["intentDigest"] for item in committed["intents"]} + for row in rows: + if not row["present"] or not ("snapshot." in row["relativePath"] or row["relativePath"] in LEGACY_MRF_PATHS): + continue + raw = base64.b64decode(row["rawBase64"], validate=True) + intents = mrf.decode_journal(raw) + require(all(item["kind"] == 3 and item["intentDigest"] in digests for item in intents), + "C11 fallback or older snapshot contains an unbound original responsibility") + require(row["relativePath"] != LEGACY_MRF_PATHS[0] or not intents, + "C11 scoped durable intents cannot appear in the v1 compatibility mirror") + manifest = {**binding, "records": rows, "originalProductBytes": True, "replacementBytesAccepted": False} + require(len(fault.encode(manifest)) <= journal.LIMIT * 32, "C11 original archive exceeds the bounded export limit") + # Archive the entire local transaction before removing any runnable record. + for row in rows: + if not row["present"]: continue + with fault.directory(roots[row["disk"]]) as fd, fault.parent(fd, ["quarantine"]) as directory: + fault.new_file(directory, archive_name(row["relativePath"]), base64.b64decode(row["rawBase64"], validate=True)) + with fault.directory(state) as fd: + fault.new_file(fd, "C11-replay.json", fault.encode(manifest)) + require(journal.same_json(census(roots, binding, journal, fault), rows), + "C11 record changed after archiving; refusing partial replay detachment") + for row in rows: + if not row["present"]: continue + with fault.directory(roots[row["disk"]]) as fd, fault.parent( + fd, ["data", *row["relativePath"].split("/")[:-1]]) as directory: + require(fault.read_file(directory, row["relativePath"].split("/")[-1], journal.LIMIT * 4) + == base64.b64decode(row["rawBase64"], validate=True), "C11 record drifted immediately before detachment") + os.unlink(row["relativePath"].split("/")[-1], dir_fd=directory) + os.fsync(directory) + require(all(not row["present"] for row in census(roots, binding, journal, fault)), + "C11 runnable record remains after complete detachment") + with fault.directory(state) as fd: + fault.new_file(fd, "C11-replay-detached.json", fault.encode(binding)) + return {**manifest, "detached": True, "checkpointCursorClaimed": False, + "backupRescueUsed": False, "canonicalCasePassClaimed": False} + + +def archived(state, roots, owner, request, journal, fault): + binding = identity(owner, request, journal) + with fault.directory(state) as fd: + manifest = journal.decode_json(fault.read_file(fd, "C11-replay.json", journal.LIMIT * 32)) + detached = journal.decode_json(fault.read_file(fd, "C11-replay-detached.json", 16384)) + require(journal.same_json(detached, binding) and all(journal.same_json(manifest.get(key), value) + for key, value in binding.items()), "C11 archive changed its run/candidate/task/incarnation binding") + require(set(manifest) == {*binding, "records", "originalProductBytes", "replacementBytesAccepted"} + and manifest["originalProductBytes"] is True and manifest["replacementBytesAccepted"] is False, + "C11 archive contains unknown or unproved provenance") + expected = {(disk, relative) for disk in range(4) for relative in paths(binding, journal)} + rows = manifest.get("records") + require(isinstance(rows, list) and len(rows) == len(expected) + and {(row.get("disk"), row.get("relativePath")) for row in rows} == expected, + "C11 archive omitted or duplicated a physical record") + for row in rows: + require(type(row.get("present")) is bool and type(row.get("disk")) is int, "C11 archive record presence is unknown") + require(set(row) == {"disk", "relativePath", "present", *( + ("size", "sha256", "rawBase64") if row["present"] else ())}, "C11 archive record has unknown fields") + if not row["present"]: continue + with fault.directory(roots[row["disk"]]) as fd, fault.parent(fd, ["quarantine"]) as directory: + raw = fault.read_file(directory, archive_name(row["relativePath"]), journal.LIMIT * 4) + require(journal.entry(raw) == {key: row.get(key) for key in ("size", "sha256")} + and raw == base64.b64decode(row["rawBase64"], validate=True), "C11 immutable original archive changed") + return manifest + + +def install(state, roots, owner, request, journal, fault): + manifest = archived(state, roots, owner, request, journal, fault) + observed = census(roots, manifest, journal, fault) + require(owner.get("incarnationPhase") == "transition", "C11 replay installation requires the transition phase") + empty_mirrors = [row for row in observed if row["present"] and manifest["family"] == "mrf" + and row["relativePath"] in LEGACY_MRF_PATHS and row["size"] == 0] + require(all(not row["present"] or row in empty_mirrors for row in observed), + "C11 replay would overwrite product progress or an existing terminal/responsibility") + # Healthy runtime flushes may publish empty fallback mirrors. They carry no + # responsibility; preserve the read receipt and never discard nonempty work. + for row in empty_mirrors: + with fault.directory(roots[row["disk"]]) as fd, fault.parent( + fd, ["data", *row["relativePath"].split("/")[:-1]]) as directory: + require(fault.read_file(directory, row["relativePath"].split("/")[-1], journal.LIMIT) == b"", + "C11 transition mirror gained product responsibility before installation") + os.unlink(row["relativePath"].split("/")[-1], dir_fd=directory) + os.fsync(directory) + for row in manifest["records"]: + if not row["present"]: continue + with fault.directory(roots[row["disk"]]) as fd, fault.parent( + fd, ["data", *row["relativePath"].split("/")[:-1]]) as directory: + fault.new_file(directory, row["relativePath"].split("/")[-1], base64.b64decode(row["rawBase64"], validate=True)) + require(journal.same_json(census(roots, manifest, journal, fault), manifest["records"]), + "C11 replay installation differs from the original exact record set") + with fault.directory(state) as fd: + fault.new_file(fd, "C11-replay-installed.json", fault.encode({key: manifest[key] for key in + ("runId", "binarySha256", "bucket", "family", "taskId", "bucketIncarnationId")})) + return {**manifest, "installed": True, "originalArchiveRetained": True, + "discardedEmptyTransitionJournals": empty_mirrors, + "productRepairClaimed": False, "canonicalCasePassClaimed": False} + + +def finalize(state, roots, owner, request, journal, fault): + manifest = archived(state, roots, owner, request, journal, fault) + with fault.directory(state) as fd: + installed = journal.decode_json(fault.read_file(fd, "C11-replay-installed.json", 16384)) + require(all(journal.same_json(installed.get(key), manifest[key]) for key in installed) + and set(installed) == {"runId", "binarySha256", "bucket", "family", "taskId", "bucketIncarnationId"}, + "C11 replay installation is incomplete; preserve every private namespace") + return {"originalArchive": manifest, "currentRecords": census(roots, manifest, journal, fault), + "originalEvidenceExported": True, "namespaceFinalizationAllowed": True, + "backupRescueUsed": False, "productRepairClaimed": False, "canonicalCasePassClaimed": False} diff --git a/scripts/validation_heal/mrf_records.py b/scripts/validation_heal/mrf_records.py index 694e50aa..86e2e49a 100644 --- a/scripts/validation_heal/mrf_records.py +++ b/scripts/validation_heal/mrf_records.py @@ -44,13 +44,13 @@ def canonical_uuid(value): def verify_reference(candidate): - require(candidate.get("source_sha") == REFERENCE["candidateCommit"], "unsupported candidate MRF record/proof contract") + require(candidate.get("source_sha") in {REFERENCE["candidateCommit"], "1aa9be87d18cf941e5020f19e8622c26ba4b10ac"}, "unsupported candidate MRF record/proof contract") return REFERENCE def read_records(state, roots, owner, checked): """Exact allowlist, no directory enumeration, no write descriptor.""" - require(owner.get("runtimeProfile") in {"durable-mrf", "mrf-crash", "flapping", "queue-pressure"}, "MRF records require the dedicated run-owned runtime") + require(owner.get("runtimeProfile") in {"durable-mrf", "mrf-crash", "flapping", "queue-pressure", "incarnation-mrf"}, "MRF records require the dedicated run-owned runtime") require(hashlib.sha256((state / "rustfs").read_bytes()).hexdigest() == owner["binarySha256"], "MRF candidate changed") def collect(): diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index f3dfc248..db23a0a8 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -1209,7 +1209,7 @@ "dispatchable": false, "code": "C11", "reasonCode": "EVIDENCE_INCARNATION_LIVE_ORCHESTRATION", - "reason": "已有 3 项候选版本 incarnation 原生回归通过，但尚缺覆盖旧盘回归、旧 checkpoint/MRF 重放与同名新桶的完整真机编排和证据绑定；原生回归不能替代四个分支的 canonical 验收。", + "reason": "已实现 H1/H2 旧元数据重返、真实 checkpoint/resume 回放、ACK 绑定的 committed MRF 回放和精确 native 证据门禁；尚缺全部现场分支的独立审计，诊断执行保持 FAIL/BLOCKED，原生前置通过不能替代 canonical 验收。", "requiredCapabilities": [ "bucket_incarnation_isolation", "checkpoint_mrf_replay_guard", diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 57866f72..deb53e51 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -20,6 +20,7 @@ while [ "$#" -gt 0 ]; do done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization) ;; + heal.bucket-incarnation-recreation) driver=incarnation.py ;; heal.node-outage-rejoin) driver=node_outage.py ;; heal.status-partial-disabled-unknown) driver=status.py ;; heal.incremental-cursor-and-truncation) driver=cursor.py ;; @@ -185,6 +186,11 @@ if [ "${case_rc}" -eq 0 ] && [ "${cleanup_status}" = PASS ]; then elif [ "${case_rc}" -eq 0 ]; then verdict=INFRA_ERROR; reason="Heal environment finalization failed" fi +if [ "${case_id}" = heal.bucket-incarnation-recreation ] && [ "${cleanup_status}" = PASS ] && \ + jq -e '.diagnosticBranchesPassed == true and .complete == false and .canonicalCasePassClaimed == false' \ + "${evidence}/healthy-summary.json" >/dev/null 2>&1; then + verdict=BLOCKED; reason="C11 diagnostic execution remains outside canonical PASS until every H1/H2 branch is independently audited" +fi args=("${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool --result-file "${artifacts}/cases/${case_id}.json" --verdict "${verdict}" --cleanup-status "${cleanup_status}" --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --binary-sha256 "${binary_sha}" diff --git a/test_validation_heal_incarnation.py b/test_validation_heal_incarnation.py index 4436a98f..977ba93b 100644 --- a/test_validation_heal_incarnation.py +++ b/test_validation_heal_incarnation.py @@ -137,19 +137,42 @@ def passed(args, topology, nodes, candidate): patch.object(suite, "verify_manager", return_value={"canonicalCasePassClaimed": False}), \ patch.object(suite, "verify_storage", return_value={"exactStorageCommitBarrierObserved": True, "canonicalFleetBranchesExecuted": False, "canonicalCasePassClaimed": False}), \ - patch.object(suite, "run_metadata_rejoin", side_effect=passed): + patch.object(suite, "run_metadata_rejoin", side_effect=passed), \ + patch.object(suite, "run_replay", side_effect=lambda args, topology, nodes, branch, candidate: { + "topology": topology, "fixture": "DV", "branch": branch, "verdict": "PASS", "cleanup": "PASS"}): self.assertEqual(suite.main(), 1) result = json.loads((root / "healthy-summary.json").read_text()) self.assertFalse(result["complete"]) self.assertFalse(result["canonicalCasePassClaimed"]) + self.assertTrue(result["diagnosticBranchesPassed"]) self.assertEqual(len(result["results"]), 8) - self.assertEqual(sum(row["verdict"] == "BLOCKED" for row in result["results"]), 4) + self.assertEqual(sum(row["verdict"] == "PASS" for row in result["results"]), 6) native = [row for row in result['results'] if row['branch'] == 'native-incarnation-fence'] self.assertEqual(len(native), 2) self.assertTrue(all(row['verdict'] == 'PREREQUISITE_PASS' and row['topology'] is None and not row['fleetBranchExecuted'] for row in native)) self.assertTrue((root / 'independent-native-storage-commit.json').is_file()) + def test_clean_finalization_cannot_hide_a_failed_replay_branch(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp); candidate = root / "candidate.json" + candidate.write_text(json.dumps({"source_sha": "a" * 40})) + args = ["incarnation.py", "--case-id", suite.CASE_ID, "--candidate", str(candidate), + "--evidence", str(root), "--rc-bin", str(root / "rc"), "--tool-directory", str(root / "tools"), + "--testing-commit", "a" * 40, "--binary-sha", "b" * 64] + with patch("sys.argv", args), patch.dict(os.environ, {"GITHUB_RUN_ID": "12345"}), \ + patch.object(suite, "verify_manager", return_value={}), patch.object(suite, "verify_storage", return_value={}), \ + patch.object(suite, "run_metadata_rejoin", side_effect=lambda args, topology, nodes, candidate: { + "topology": topology, "branch": "old-metadata-rejoin", "verdict": "PASS", "cleanup": "PASS"}), \ + patch.object(suite, "run_replay", side_effect=lambda args, topology, nodes, branch, candidate: { + "topology": topology, "branch": branch, "verdict": "FAIL", "cleanup": "PASS"}): + self.assertEqual(suite.main(), 1) + result = json.loads((root / "healthy-summary.json").read_text()) + self.assertEqual(result["cleanup"], "PASS") + self.assertFalse(result["diagnosticBranchesPassed"]) + self.assertFalse(result["complete"]) + self.assertFalse(result["canonicalCasePassClaimed"]) + def test_unverified_storage_artifact_refuses_before_fleet_mutation(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp); candidate = root / 'candidate.json' diff --git a/test_validation_heal_incarnation_replay.py b/test_validation_heal_incarnation_replay.py new file mode 100644 index 00000000..d2b87e94 --- /dev/null +++ b/test_validation_heal_incarnation_replay.py @@ -0,0 +1,459 @@ +"""C11 original journal replay cannot fabricate or overwrite responsibility.""" + +import base64 +import copy +import hashlib +import json +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch +from contextlib import ExitStack + +from scripts.validation_heal import incarnation as suite, incarnation_replay as backend +from scripts.validation_heal import checkpoint, journal_fault as journal, heal_fault as fault, mrf_records, healthy, fleet_node +import test_validation_heal_checkpoint as checkpoint_fixtures +from test_validation_heal_checkpoint import BUCKET, TOKEN, INCARNATION, encoded +import test_validation_heal_mrf as mrf_fixtures + + +SUCCESSOR = "52345678-1234-4234-8234-123456789abc" + + +class OriginalReplayTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.state = Path(self.directory.name).resolve() / "state" + self.state.mkdir() + self.roots = [self.state.parent / str(disk) for disk in range(4)] + for root in self.roots: + (root / "data/.rustfs.sys/buckets").mkdir(parents=True) + (root / "quarantine").mkdir() + self.mount = patch.object(fault, "mount_id", return_value=7) + self.mount.start() + self.addCleanup(self.mount.stop) + self.owner = {"runId": "gh-12345-1-h1", "binarySha256": "a" * 64, + "bucket": BUCKET, "runtimeProfile": "incarnation-checkpoint"} + self.request = {"runId": self.owner["runId"], "family": "checkpoint", "taskId": TOKEN, + "bucketIncarnationId": INCARNATION} + self.observed, _, _ = checkpoint_fixtures.AdminCheckpointTests().fixture() + roots, _ = checkpoint_fixtures.AdminCheckpointTests().root_fixture() + self.rows = self.observed["node"]["records"] + roots["node"]["records"] + for row in self.rows: + if row["present"]: + (self.roots[row["disk"]] / "data" / row["relativePath"]).write_bytes(base64.b64decode(row["rawBase64"])) + self.bind_expected() + + def bind_expected(self): + binding = backend.identity(self.owner, self.request, journal) + self.request["expectedRecords"] = suite.record_digests(backend.census(self.roots, binding, journal, fault)) + + def capture(self): + return backend.capture(self.state, self.roots, self.owner, self.request, journal, fault, mrf_records, fleet_node.checked) + + def install(self): + return backend.install(self.state, self.roots, {**self.owner, "incarnationPhase": "transition"}, + self.request, journal, fault) + + def test_exact_original_transaction_detaches_then_replays_and_exports_current_and_archived_bytes(self): + archived = self.capture() + self.assertTrue(archived["originalProductBytes"]) + self.assertFalse(archived["checkpointCursorClaimed"]) + self.assertTrue(all(not row["present"] for row in backend.census(self.roots, archived, journal, fault))) + self.assertTrue(self.install()["originalArchiveRetained"]) + finalized = backend.finalize(self.state, self.roots, self.owner, self.request, journal, fault) + self.assertEqual(finalized["currentRecords"], archived["records"]) + self.assertFalse(finalized["productRepairClaimed"]) + self.assertFalse(finalized["canonicalCasePassClaimed"]) + + def test_archive_tampering_wrong_generation_and_existing_progress_cannot_be_installed(self): + self.capture() + for key, changed in (("bucketIncarnationId", SUCCESSOR), ("runId", "gh-99999-1-h1"), ("taskId", SUCCESSOR)): + with self.subTest(key=key), self.assertRaises(ValueError): + backend.install(self.state, self.roots, self.owner, {**self.request, key: changed}, journal, fault) + path = self.roots[1] / "data" / journal.names("root", TOKEN)[1] + path.write_bytes(b"product-terminal") + with self.assertRaisesRegex(ValueError, "overwrite"): self.install() + self.assertEqual(path.read_bytes(), b"product-terminal") + path.unlink() + archive = self.roots[1] / "quarantine" / backend.archive_name(journal.names("checkpoint", TOKEN)[0]) + archive.write_bytes(b"tampered") + with self.assertRaisesRegex(ValueError, "archive changed"): self.install() + + def test_incomplete_capture_installation_and_lost_ack_keep_namespace_dirty(self): + self.capture() + with self.assertRaises(FileNotFoundError): backend.finalize(self.state, self.roots, self.owner, self.request, journal, fault) + fleet = healthy.Fleet(["node"], self.state / "evidence", self.owner["runId"], runtime_profile="incarnation-checkpoint") + fleet.incarnation_replays = [("node", self.request)] + fleet.reserved = ["node"] + def remote(node, action, **params): + if action == "incarnation-finalize": raise FileNotFoundError("installation incomplete") + if action == "cleanup": self.fail("dirty original archive must not be deleted") + return {} + with patch.object(fleet, "stop"), patch.object(fleet, "remote", side_effect=remote): + self.assertFalse(fleet.cleanup()) + self.assertEqual(len(fleet.incarnation_replays), 1) + self.assertTrue((self.state / "C11-replay.json").exists()) + + def test_bad_digest_foreign_terminal_and_incomplete_pair_refuse_before_detachment(self): + original = copy.deepcopy(self.request) + self.request["expectedRecords"][0]["sha256"] = "f" * 64 + with self.assertRaisesRegex(ValueError, "digests"): self.capture() + self.request = original + terminal = self.roots[0] / "data" / journal.names("root", TOKEN)[1] + terminal.write_bytes(b"terminal") + self.bind_expected() + with self.assertRaisesRegex(ValueError, "terminal"): self.capture() + terminal.unlink() + resume = self.roots[1] / "data" / journal.names("checkpoint", TOKEN)[1] + resume.unlink() + self.bind_expected() + with self.assertRaisesRegex(ValueError, "incomplete"): self.capture() + self.assertFalse((self.state / "C11-replay.json").exists()) + + def test_symlinks_and_hardlinks_refuse_without_mutating_product_bytes(self): + path = self.roots[1] / "data" / journal.names("checkpoint", TOKEN)[0] + raw = path.read_bytes() + saved = self.state / "outside" + saved.write_bytes(raw) + path.unlink() + path.symlink_to(saved) + with self.assertRaises(OSError): self.capture() + path.unlink() + os.link(saved, path) + with self.assertRaises(ValueError): self.capture() + self.assertEqual(saved.read_bytes(), raw) + + def test_real_committed_mrf_archive_keeps_payload_commit_and_lifecycle_on_all_original_replicas(self): + for root in self.roots: + for path in (root / "data/.rustfs.sys/buckets").iterdir(): path.unlink() + for path in (root / "data/.rustfs.sys").glob("*.json"): path.unlink() + fixture = mrf_fixtures.DurableMrfTests() + self.owner["runtimeProfile"] = "incarnation-mrf" + (self.state / "rustfs").write_bytes(b"candidate") + self.owner["binarySha256"] = hashlib.sha256(b"candidate").hexdigest() + self.request.update(family="mrf", taskId=None, bucketIncarnationId=fixture.incarnation) + for row in fixture.observed()["records"]: + if row["present"]: + (self.roots[row["disk"]] / "data" / row["relativePath"]).write_bytes(base64.b64decode(row["rawBase64"])) + for root in self.roots: + for relative in backend.LEGACY_MRF_PATHS: + path = root / "data" / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(fixture.journal() if relative.endswith("journal-scoped.bin") else b"") + self.bind_expected() + archived = self.capture() + self.assertEqual(len([row for row in archived["records"] if row["present"]]), 14) + self.assertTrue(all(not row["present"] for row in backend.census(self.roots, archived, journal, fault))) + for root in self.roots: + for relative in backend.LEGACY_MRF_PATHS: + (root / "data" / relative).write_bytes(b"") + receipt = self.install() + self.assertEqual(len(receipt["discardedEmptyTransitionJournals"]), 8) + self.assertEqual(backend.census(self.roots, archived, journal, fault), archived["records"]) + observed = mrf_records.read_records(self.state, self.roots, self.owner, fleet_node.checked) + value = mrf_records.checkpoint(observed) + self.assertEqual(value["intents"][0]["lifecycle"]["responsibility_id"], fixture.responsibility) + self.assertEqual({row["disk"] for row in value["replicas"]}, {0, 2}) + + def test_unbound_or_torn_fallback_refuses_before_archiving_or_detaching(self): + self.owner["runtimeProfile"] = "incarnation-mrf" + self.request.update(family="mrf", taskId=None) + (self.state / "rustfs").write_bytes(b"candidate") + self.owner["binarySha256"] = hashlib.sha256(b"candidate").hexdigest() + relative = backend.LEGACY_MRF_PATHS[1] + path = self.roots[0] / "data" / relative + path.parent.mkdir(parents=True) + fixture = mrf_fixtures.DurableMrfTests() + for raw in (fixture.journal(), fixture.journal()[:-1]): + path.write_bytes(raw) + self.bind_expected() + with self.assertRaises(ValueError): self.capture() + self.assertEqual(path.read_bytes(), raw) + self.assertFalse((self.state / "C11-replay.json").exists()) + + def test_partial_install_keeps_original_archive(self): + self.capture() + original = fault.new_file + def interrupted(fd, name, raw): + if name.endswith("ahm_resume_state.json"): raise OSError("installation interrupted") + original(fd, name, raw) + with patch.object(fault, "new_file", side_effect=interrupted), self.assertRaises(OSError): self.install() + self.assertFalse((self.state / "C11-replay-installed.json").exists()) + with self.assertRaisesRegex(ValueError, "overwrite"): self.install() + self.assertTrue(backend.archived(self.state, self.roots, self.owner, self.request, journal, fault)) + + def test_unresolved_s3_mutation_keeps_namespaces_and_logs(self): + fleet = healthy.Fleet(["node"], self.state / "evidence", self.owner["runId"]) + fleet.reserved = ["node"] + with patch.object(fleet, "remote", return_value={}) as remote: + self.assertFalse(fleet.cleanup(preserve_namespaces=True)) + self.assertEqual([call.args[1] for call in remote.call_args_list], ["logs"]) + + +class ReplayOutcomeTests(unittest.TestCase): + def successor_fixture(self, size): + expected = [{"key": f"objects/{index // 2:04d}", "versionId": str(__import__('uuid').UUID(int=index + 1)), + "deleteMarker": index % 2 == 1, "isLatest": index % 2 == 1, + **({"payload": {"size": 1, "sha256": "a" * 64}} if index % 2 == 0 else {})} for index in range(size)] + objects = [{"identity": {"bucket": BUCKET, "object": row["key"], "versionId": row["versionId"], + "bucketIncarnationId": SUCCESSOR, "poolIndex": 0, "setIndex": 0}, + "disposition": {"state": "metadata_healthy" if row["deleteMarker"] else "verified_healthy"}} + for row in expected[:128]] + current = {"summary": "finished", "truncated": size > 128, "outcome": { + "execution": {"state": "completed"}, "coverage": "complete", "objectsTruncated": size > 128, + "objects": objects, "counters": {"processed": size, "healed": 0, "unchanged": size, + "skipped": 0, "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False}}} + return current, BUCKET, expected + + def test_successor_bounded_diagnostics_require_exact_independent_scope_and_typed_counters(self): + for size in (128, 129, 280): + self.assertEqual(suite.successor_receipts(*self.successor_fixture(size))[ + "boundedDiagnosticsAreCompleteManifest"], size <= 128) + for change in ("counter", "truncated", "duplicate", "foreign", "old-generation", "unknown", "repair", "kind"): + current, bucket, expected = self.successor_fixture(280) + outcome = current["outcome"] + if change == "counter": outcome["counters"]["failed"] = False + elif change == "truncated": outcome["objectsTruncated"] = False + elif change == "duplicate": outcome["objects"][-1] = copy.deepcopy(outcome["objects"][0]) + elif change == "foreign": outcome["objects"][0]["identity"]["bucket"] = "other" + elif change == "old-generation": outcome["objects"][0]["identity"]["bucketIncarnationId"] = INCARNATION + else: outcome["objects"][0]["disposition"]["state"] = { + "unknown": "unknown", "repair": "repaired", "kind": "metadata_healthy"}[change] + with self.subTest(change=change), self.assertRaises(ValueError): + suite.successor_receipts(current, bucket, expected) + + def checkpoint_fixture(self): + observed, _, _ = checkpoint_fixtures.AdminCheckpointTests().fixture() + frozen = checkpoint.owned_checkpoint(observed, TOKEN, BUCKET) + root, _ = checkpoint_fixtures.AdminCheckpointTests().root_fixture() + frozen["rootAdmission"] = checkpoint.owned_root_admission(root, TOKEN) + outcome = copy.deepcopy(frozen["value"]["admin"]["outcome"]) + outcome["execution"] = {"state": "aborted", "reason": "untraversable"} + current = {"summary": "stopped", "settings": checkpoint.OPTIONS, "truncated": False, + "detail": "stale_bucket_incarnation: old admission", "outcome": outcome} + names = journal.names("root", TOKEN) + terminal = {"schema": 1, "task_id": TOKEN, "heal_type": frozen["rootAdmission"]["value"]["heal_type"], + "options": frozen["rootAdmission"]["value"]["options"], "status": {"Failed": {"error": current["detail"]}}} + report = {"schema": 1, "terminal": terminal, "outcome": outcome} + root["node"]["records"][3] = {"disk": 1, "relativePath": names[0], "present": False} + root["node"]["records"][4].update(encoded(terminal)) + root["node"]["records"][5].update(encoded(report)) + return current, frozen, root + + def test_checkpoint_rejection_keeps_exact_original_progress_and_durable_owner(self): + current, frozen, roots = self.checkpoint_fixture() + self.assertTrue(suite.checkpoint_rejection(current, frozen)["originalProgressRetained"]) + self.assertTrue(suite.checkpoint_terminal(roots, frozen, current)["durableFailureRetained"]) + for change in ("success", "generic-error", "counter", "foreign-object"): + current, frozen, _ = self.checkpoint_fixture() + if change == "success": current["summary"] = "finished" + elif change == "generic-error": current["detail"] = "InternalError" + elif change == "counter": current["outcome"]["counters"]["healed"] += 1 + else: current["outcome"]["objects"] = [{"identity": {"bucketIncarnationId": SUCCESSOR}}] + with self.subTest(change=change), self.assertRaises(ValueError): suite.checkpoint_rejection(current, frozen) + + def mrf_fixture(self): + fixture = mrf_fixtures.DurableMrfTests() + original = mrf_records.checkpoint(fixture.observed()) + intent = original["intents"][0] + owner = {"node": "node", "checkpointOwner": original["owner"], "sequence": original["sequence"], + "responsibilityId": fixture.responsibility, "bucketIncarnationId": fixture.incarnation, + "intentDigest": intent["intentDigest"]} + value = copy.deepcopy(original) + value["owner"] = SUCCESSOR + value["sequence"] += 1 + value["intents"][0]["lifecycle"]["state"] = {"state": "bucket_incarnation_changed", + "source_bucket_incarnation_id": fixture.incarnation, "observed_bucket_incarnation_id": SUCCESSOR, "detected_at_ms": 123} + obligation = fixture.obligation() + return {"node": value}, [obligation], {suite.mrf.identity(obligation): [owner]}, SUCCESSOR + + def test_old_mrf_requires_durable_held_original_responsibility_without_operator_acceptance(self): + proof = suite.retained_mrf(*self.mrf_fixture()) + self.assertFalse(proof["oldResponsibilityRetired"]) + self.assertNotEqual(proof["retainedResponsibilities"][0]["originalCheckpointOwner"], + proof["retainedResponsibilities"][0]["currentCheckpointOwner"]) + for change in ("missing", "active", "same-sequence", "same-owner", "nil-owner", "successor-source", "accepted", "wrong-id", "duplicate"): + value, obligations, bindings, incarnation = self.mrf_fixture() + row = value["node"]["intents"][0] + if change == "missing": value["node"]["intents"] = [] + elif change == "active": row["lifecycle"]["state"] = {"state": "active"} + elif change == "same-sequence": value["node"]["sequence"] -= 1 + elif change == "same-owner": value["node"]["owner"] = bindings[suite.mrf.identity(obligations[0])][0]["checkpointOwner"] + elif change == "nil-owner": value["node"]["owner"] = str(__import__('uuid').UUID(int=0)) + elif change == "successor-source": row["lifecycle"]["source_bucket_incarnation_id"] = SUCCESSOR + elif change == "accepted": row["lifecycle"]["last_operator_acceptance"] = {"accepted": True} + elif change == "wrong-id": row["lifecycle"]["responsibility_id"] = SUCCESSOR + else: value["node"]["intents"].append(copy.deepcopy(row)) + with self.subTest(change=change), self.assertRaises(ValueError): + suite.retained_mrf(value, obligations, bindings, incarnation) + + def test_old_replay_cannot_change_any_successor_physical_bytes(self): + before = {"node": {"files": [{"sha256": "a" * 64}], "formats": []}} + after = copy.deepcopy(before) + after["node"]["files"][0]["sha256"] = "b" * 64 + with self.assertRaisesRegex(ValueError, "persisted work changed"): suite.replay_isolation(before, after) + + +class RuntimePhaseTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.state = Path(self.directory.name).resolve() + self.owner = {"runId": "gh-12345-1-h1", "unit": "owned-unit", "binarySha256": "a" * 64, + "runtimeProfile": "incarnation-mrf", "bucket": BUCKET} + self.roots = [self.state / str(disk) for disk in range(4)] + for root in self.roots: (root / "data").mkdir(parents=True) + self.binding = backend.identity(self.owner, {"runId": self.owner["runId"], "family": "mrf", + "taskId": None, "bucketIncarnationId": INCARNATION}, journal) + self.manifest = {**self.binding, "records": [{"disk": disk, "relativePath": relative, "present": False} + for disk in range(4) for relative in backend.paths(self.binding, journal)], + "originalProductBytes": True, "replacementBytesAccepted": False} + (self.state / "C11-replay.json").write_bytes(fault.encode(self.manifest)) + self.env = self.state / "service.env" + self.env.write_text("\n".join(f"{key}={value}" for key, value in fleet_node.runtime_settings(self.owner).items()) + "\n") + original_stat = Path.stat + def root_stat(path, *args, **kwargs): + info = original_stat(path, *args, **kwargs) + if path != self.env: return info + values = list(info); values[4] = 0 + return os.stat_result(values) + self.patches = [patch.object(Path, "stat", root_stat), + patch.object(fleet_node, "owned", side_effect=lambda _: (self.state, self.roots, self.owner.copy())), + patch.object(fleet_node, "stopped"), patch.object(fleet_node, "digest", return_value="a" * 64), + patch.dict(fleet_node.__dict__, {"incarnation_replay_backend": backend}), + patch.dict(fleet_node.__dict__, {"journal_fault_backend": journal}), + patch.object(fault, "mount_id", return_value=7)] + for item in self.patches: + item.start(); self.addCleanup(item.stop) + + def phase(self, value): + return fleet_node.incarnation_operation({"runId": self.owner["runId"], "action": "incarnation-phase", "phase": value}) + + def test_only_complete_capture_transition_replay_changes_mrf_and_keeps_one_worker(self): + with self.assertRaisesRegex(ValueError, "complete"): self.phase("transition") + (self.state / "C11-replay-detached.json").write_bytes(fault.encode(self.binding)) + receipt = self.phase("transition") + self.assertEqual((receipt["workerLimit"], receipt["mrfEnabled"]), ("1", "false")) + self.owner["incarnationPhase"] = "transition" + with self.assertRaisesRegex(ValueError, "refund"): self.phase("transition") + with self.assertRaisesRegex(ValueError, "complete"): self.phase("replay") + (self.state / "C11-replay-installed.json").write_bytes(fault.encode(self.binding)) + self.assertEqual(self.phase("replay")["mrfEnabled"], "true") + self.owner["incarnationPhase"] = "replay" + fleet_node.incarnation_configuration(self.state, self.owner) + with self.assertRaisesRegex(ValueError, "refund"): self.phase("transition") + + def test_configuration_commit_interruption_refuses_process_start_before_systemd(self): + (self.state / "C11-replay-detached.json").write_bytes(fault.encode(self.binding)) + original = fault.new_file + def interrupted(fd, name, value): + if name == "C11-phase.new": raise OSError("phase acknowledgement interrupted") + original(fd, name, value) + with patch.object(fault, "new_file", side_effect=interrupted), self.assertRaises(OSError): self.phase("transition") + self.assertFalse((self.state / "C11-phase.json").exists()) + with patch.object(fleet_node, "command") as command, self.assertRaisesRegex(ValueError, "incomplete"): + fleet_node.start({"runId": self.owner["runId"]}) + self.assertFalse(any(call.args[0] == "systemd-run" for call in command.call_args_list)) + + def test_substituted_markers_and_archive_refuse_before_changing_runtime(self): + original = self.env.read_bytes() + for change in ("runId", "binarySha256", "bucket", "family", "taskId", "bucketIncarnationId", "extra"): + bad = {**self.binding, change: "wrong"} + (self.state / "C11-replay-detached.json").write_bytes(fault.encode(bad)) + with self.subTest(change=change), self.assertRaises(ValueError): self.phase("transition") + self.assertEqual(self.env.read_bytes(), original) + (self.state / "C11-replay-detached.json").write_bytes(fault.encode(self.binding)) + (self.state / "C11-replay.json").write_bytes(fault.encode({**self.manifest, "bucketIncarnationId": SUCCESSOR})) + with self.assertRaises(ValueError): self.phase("transition") + self.assertEqual(self.env.read_bytes(), original) + + def test_duplicate_config_and_caller_runtime_override_refuse(self): + with self.env.open("a") as stream: stream.write("RUSTFS_HEAL_MRF_ENABLE=true\n") + with self.assertRaises(ValueError): fleet_node.incarnation_configuration(self.state, self.owner) + with self.assertRaisesRegex(ValueError, "arbitrary"): + fleet_node.incarnation_operation({"runId": self.owner["runId"], "action": "incarnation-phase", + "phase": "transition", "environment": {"RUSTFS_HEAL_MRF_ENABLE": "false"}}) + + def test_remote_record_operations_embed_all_real_backends_before_execution(self): + fleet = healthy.Fleet(["node"], self.state / "evidence", self.owner["runId"], runtime_profile="incarnation-mrf") + for action in ("incarnation-census", "incarnation-capture", "incarnation-install", "incarnation-finalize", "incarnation-phase"): + with self.subTest(action=action), patch.object(healthy.subprocess, "run", + return_value=SimpleNamespace(returncode=0, stdout="{}", stderr="")) as remote: + fleet.remote("node", action) + source = remote.call_args.kwargs["input"].rsplit("\nmain(", 1)[0] + namespace = {"__name__": "fleet-test"} + exec(compile(source, "remote-fleet", "exec"), namespace) + for name in ("mrf", "fault", "journal_fault_backend", "incarnation_replay_backend"): + self.assertIn(name, namespace) + self.assertTrue(callable(namespace["incarnation_replay_backend"].capture)) + + +class ReplayOrderingTests(unittest.TestCase): + def test_capture_and_install_lost_ack_are_registered_before_mutation_and_keep_incomplete_namespaces(self): + for failing_action in ("incarnation-capture", "incarnation-install"): + with self.subTest(action=failing_action), tempfile.TemporaryDirectory() as temp, ExitStack() as stack: + directory = Path(temp) + rc = directory / "rc"; rc.write_bytes(b"client") + args = SimpleNamespace(evidence=directory, rc_bin=rc, tool_directory=directory, + testing_commit="a" * 40, binary_sha="b" * 64) + events, fleets = [], [] + old = [{"key": "old", "versionId": TOKEN, "deleteMarker": False, "payload": {"size": 1}}, + {"key": "old", "versionId": SUCCESSOR, "deleteMarker": True}] + obligation = {"bucket": BUCKET, "key": "old", "versionId": TOKEN, "kind": 3, "poolIndex": 0, "setIndex": 0} + binding = {suite.mrf.identity(obligation): [{"bucketIncarnationId": INCARNATION}]} + class FakeFleet(healthy.Fleet): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs); fleets.append(self) + def deploy(self, *args): self.reserved = list(self.nodes) + def snapshot(self, *args, **kwargs): return {} + def stop(self): events.append("stop") + def start(self): events.append("start") + def remote(self, node, action, **params): + events.append((node, action, params)) + if action == failing_action: + assert (node, {key: params[key] for key in ("family", "taskId", "bucketIncarnationId")}) in self.incarnation_replays + raise OSError("SSH acknowledgement lost after mutation") + if action == "incarnation-finalize": raise ValueError("original transaction incomplete") + if action == "incarnation-capture": return {"detached": True, "originalProductBytes": True, "replacementBytesAccepted": False} + if action == "incarnation-census": return {"records": []} + if action == "drive-gate-probe": return {"readProbe": {"errno": 13}, "errno": 13, "dacBypassRemoved": True} + return {} + class FakeProbe: + def __init__(self, *args): pass + def inventory(self, *args): return {} + class FakeRecorder: + def __init__(self, probe, *args): self.probe = probe + def deletion(*args): events.append("delete-old-and-recreate"); return {} + for name, value in {"Fleet": FakeFleet, "Probe": FakeProbe, "Recorder": FakeRecorder, + "verify_inspector": lambda *args: {}, "verify_quorum_contract": lambda *args: None, + "verify_mrf_reference": lambda *args: {}, "validate_topology": lambda *args: {}, + "seed": lambda *args: copy.deepcopy(old), "verify_seed": lambda *args: {}, + "run_task": lambda *args, **kwargs: {}, "validate_version_receipts": lambda *args: {"bucketIncarnationId": INCARNATION}, + "idle_fleet": lambda *args: None, "map_target": lambda *args, **kwargs: {"cohort": [{"node": "node2", "disk": 0}]}, + "delete_old_bucket": deletion, "seed_successor": lambda *args: copy.deepcopy(old), + "config": lambda *args: {}, "successor_receipts": lambda *args: {"bucketIncarnationId": SUCCESSOR}, + "successor_physical": lambda *args: []}.items(): + stack.enter_context(patch.object(suite, name, value)) + stack.enter_context(patch.dict(os.environ, {"GITHUB_RUN_ID": "12345"})) + stack.enter_context(patch.object(suite.mrf, "acknowledge", return_value=(old, [obligation], {}))) + stack.enter_context(patch.object(suite.mrf, "read_checkpoints", return_value={})) + stack.enter_context(patch.object(suite.mrf, "pending_bindings", return_value=binding)) + result = suite.run_replay(args, "H2", ["node1", "node2"], "old-mrf-replay", {"source_sha": "c" * 40, "package_sha256": "d" * 64}) + self.assertEqual((result["verdict"], result["cleanup"]), ("FAIL", "FAIL")) + self.assertTrue(fleets[0].incarnation_replays, result.get("failure")) + self.assertFalse(any(isinstance(event, tuple) and event[1] == "cleanup" for event in events)) + if failing_action == "incarnation-capture": self.assertNotIn("delete-old-and-recreate", events) + else: + captures = [i for i, event in enumerate(events) if isinstance(event, tuple) and event[1] == "incarnation-capture"] + released = next(i for i, event in enumerate(events) if isinstance(event, tuple) and event[1] == "drive-gate" and event[2]["mode"] == "clear") + self.assertLess(max(captures), released) + self.assertLess(released, events.index("delete-old-and-recreate")) + self.assertEqual(events[released - 1][1], "incarnation-phase") + self.assertEqual(events[released + 1], "start") + + +if __name__ == "__main__": unittest.main() From 49ddd309e175b6ed80ccf47fd27cdf601cac7619 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 20:49:17 +0800 Subject: [PATCH 180/343] fix(validation): retain blocked diagnostic results --- scripts/validation_result.py | 4 ++-- test_validation_migration.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/validation_result.py b/scripts/validation_result.py index 4d074bcf..32ea51cf 100644 --- a/scripts/validation_result.py +++ b/scripts/validation_result.py @@ -38,7 +38,7 @@ def main() -> int: parser.add_argument("--case-id", required=True) parser.add_argument("--profile", required=True) parser.add_argument("--result-file", type=Path, required=True) - parser.add_argument("--verdict", choices=("RUNNING", "PASS", "FAIL", "INFRA_ERROR"), required=True) + parser.add_argument("--verdict", choices=("RUNNING", "PASS", "FAIL", "INFRA_ERROR", "BLOCKED"), required=True) parser.add_argument("--reason", default="") parser.add_argument("--cleanup-status", default="PENDING") parser.add_argument("--evidence", action="append", default=[]) @@ -53,7 +53,7 @@ def main() -> int: "schema_version": 1, "case_id": args.case_id, "source_status": case["source_status"], - "migration_status": "unverified" if args.verdict == "RUNNING" else "verified", + "migration_status": "unverified" if args.verdict in {"RUNNING", "BLOCKED"} else "verified", "profile": args.profile, "verdict": args.verdict, "cleanup_status": args.cleanup_status, diff --git a/test_validation_migration.py b/test_validation_migration.py index e62a7262..bda5d4b3 100644 --- a/test_validation_migration.py +++ b/test_validation_migration.py @@ -187,6 +187,18 @@ def test_writer_and_report_contract(self) -> None: bootstrap = json.loads((results / "s3.basic-object-round-trip.json").read_text()) self.assertEqual(bootstrap["verdict"], "INFRA_ERROR") self.assertIsNone(bootstrap["provenance"]["binary_sha256"]) + subprocess.run(command + identity + ["--verdict", "BLOCKED", "--cleanup-status", "PASS", + "--binary-sha256", "b" * 64, "--reason", "diagnostic branches require independent canonical audit", + "--evidence", "evidence.json"], check=True, capture_output=True) + blocked = json.loads((results / "s3.basic-object-round-trip.json").read_text()) + self.assertEqual(blocked["verdict"], "BLOCKED") + self.assertEqual(blocked["migration_status"], "unverified") + self.assertNotIn("assertions", blocked) + self.assertEqual(blocked["provenance"]["binary_sha256"], "b" * 64) + catalog = json.loads((ROOT / "validation-cases.json").read_text(encoding="utf-8")) + _, blocked_errors = validate_results(catalog, results, gate_statuses=set(), + gate_cases={"s3.basic-object-round-trip"}) + self.assertEqual(blocked_errors, ["s3.basic-object-round-trip: gate verdict is 'BLOCKED'"]) invalid_pass = subprocess.run(command + identity + ["--verdict", "PASS", "--cleanup-status", "PASS", "--evidence", "evidence.json"], capture_output=True) self.assertNotEqual(invalid_pass.returncode, 0) From b5634bea2d5bd46957caf52a9f1b19e5b21ef11c Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 20:57:43 +0800 Subject: [PATCH 181/343] fix(heal): bind checkpoint phase to the process environment --- scripts/validation_heal/fleet_node.py | 3 +++ test_validation_heal_incarnation_replay.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 4df2b2f1..e0520d0e 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -36,6 +36,7 @@ def runtime_config(profile): if profile in {"incarnation-checkpoint", "incarnation-mrf"}: logging_filter = "info,rustfs::heal::manager=debug,rustfs::heal::task=debug,rustfs::heal::mrf=debug" return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true" if profile == "incarnation-mrf" else "false", + "AUTO_TESTING_C11_PHASE": "capture", "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false", "RUSTFS_HEAL_MAX_CONCURRENT_HEALS": "1", "RUSTFS_HEAL_MAX_CONCURRENT_PER_SET": "1", "RUSTFS_OBS_LOGGER_LEVEL": logging_filter, "RUST_LOG": logging_filter} @@ -99,6 +100,8 @@ def runtime_config(profile): def runtime_settings(owner): config = runtime_config(owner.get("runtimeProfile", "manual")) + if owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf"}: + config["AUTO_TESTING_C11_PHASE"] = owner.get("incarnationPhase", "capture") if owner.get("incarnationPhase") == "transition": config.update(RUSTFS_HEAL_MRF_ENABLE="false") return config diff --git a/test_validation_heal_incarnation_replay.py b/test_validation_heal_incarnation_replay.py index d2b87e94..ac20bf74 100644 --- a/test_validation_heal_incarnation_replay.py +++ b/test_validation_heal_incarnation_replay.py @@ -359,6 +359,27 @@ def interrupted(fd, name, value): fleet_node.start({"runId": self.owner["runId"]}) self.assertFalse(any(call.args[0] == "systemd-run" for call in command.call_args_list)) + def test_checkpoint_phase_commit_interruption_refuses_start_even_when_product_knobs_are_unchanged(self): + self.owner["runtimeProfile"] = "incarnation-checkpoint" + self.binding = backend.identity(self.owner, {"runId": self.owner["runId"], "family": "checkpoint", + "taskId": TOKEN, "bucketIncarnationId": INCARNATION}, journal) + manifest = {**self.binding, "records": [{"disk": disk, "relativePath": relative, "present": False} + for disk in range(4) for relative in backend.paths(self.binding, journal)], + "originalProductBytes": True, "replacementBytesAccepted": False} + (self.state / "C11-replay.json").write_bytes(fault.encode(manifest)) + (self.state / "C11-replay-detached.json").write_bytes(fault.encode(self.binding)) + self.env.write_text("\n".join(f"{key}={value}" for key, value in fleet_node.runtime_settings(self.owner).items()) + "\n") + original = fault.new_file + def interrupted(fd, name, raw): + if name == "C11-phase.new": raise OSError("checkpoint phase interrupted") + original(fd, name, raw) + with patch.object(fault, "new_file", side_effect=interrupted), self.assertRaises(OSError): self.phase("transition") + self.assertIn("RUSTFS_HEAL_MRF_ENABLE=false", self.env.read_text()) + self.assertIn("AUTO_TESTING_C11_PHASE=transition", self.env.read_text()) + with patch.object(fleet_node, "command") as command, self.assertRaisesRegex(ValueError, "incomplete"): + fleet_node.start({"runId": self.owner["runId"]}) + self.assertFalse(any(call.args[0] == "systemd-run" for call in command.call_args_list)) + def test_substituted_markers_and_archive_refuse_before_changing_runtime(self): original = self.env.read_bytes() for change in ("runId", "binarySha256", "bucket", "family", "taskId", "bucketIncarnationId", "extra"): From 891f264f39ffb608f1898fef151b273e205321b5 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 21:03:31 +0800 Subject: [PATCH 182/343] test(heal): retain independent C11 replay file witnesses --- docs/validation-migration-map.md | 4 ++-- scripts/validation_heal/fleet_node.py | 4 ++-- scripts/validation_heal/incarnation.py | 8 ++++++++ test_validation_heal_incarnation_replay.py | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 12921a19..ef7da7fc 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1942,6 +1942,6 @@ C11 now implements both remaining H1/H2 DV replay flows through the existing Fle The old MRF fault gate is released while stopped after all original replay sources are detached and before normal S3 bucket deletion/recreation. Successor and control objects are written on a healthy fleet, independently verified, and frozen before original records are installed. Replayed checkpoints must reject the old generation while retaining original progress and durable terminal/report ownership. Replayed MRF must retain the exact responsibility ID, intent digest and source incarnation under a new process checkpoint owner with greater sequence, durably held as bucket_incarnation_changed without operator acceptance. Complete successor/control physical bytes, formats, system records, full shard integrity and every peer's S3/configuration remain independent evidence. -Immutable package run 38240858236/1 and tool run 38244986993/1 succeeded for source 1aa9be87d18cf941e5020f19e8622c26ba4b10ac. Independent Git-object checks verified the exact source tree, lock and all source contracts; all seven MRF sources match the reviewed reference byte-for-byte. The complete 68,625,642-byte DEB digest is b99dd915376b51f0896fc346ca4642b7d26bf3352135bb5c6f60f8179d865761 and its contained binary digest is 0103ca46386396ed15f27b9efbe4196fe62bd9e93033671a88f3b3e944602cf4. Inspector, nine native storage regressions and fifteen manager regressions passed independent artifact consumers for harness d5a41739a7a544d93ba71f51583d45ad4e6161eb. A new immutable tool build is required for the updated replay harness; older harness provenance is not accepted. +Immutable package run 38240858236/1 and tool run 38244986993/1 succeeded for source 1aa9be87d18cf941e5020f19e8622c26ba4b10ac. Independent Git-object checks verified the exact source tree, lock and all source contracts; all seven MRF sources match the reviewed reference byte-for-byte. The complete 68,625,642-byte DEB digest is b99dd915376b51f0896fc346ca4642b7d26bf3352135bb5c6f60f8179d865761 and its contained binary digest is 0103ca46386396ed15f27b9efbe4196fe62bd9e93033671a88f3b3e944602cf4. Inspector, nine native storage regressions and fifteen manager regressions passed independent artifact consumers for harness d5a41739a7a544d93ba71f51583d45ad4e6161eb. A new immutable tool build is required for the updated replay harness; older harness provenance is not accepted. The BLOCKED writer preserves unverified migration status, original provenance and no PASS assertions. A phase identity in the process environment detects interrupted checkpoint transitions even when product configuration knobs are identical. Independent stopped censuses are retained before capture, after detachment and after installation to compare each original byte and absence witness. Read-only environment run 38251226817 confirms 16 distinct eligible endpoints on the four current Heal hosts for H1/H2; H4 remains unavailable. -The full Heal contract suite passed 591 tests with eight platform skips before the additional finalization-verdict regression; all 29 focused incarnation tests, the unchanged 84/301/532 matrix, AST, bash syntax, actionlint and whitespace passed on the final code. Two sequential final reviews covered correctness, simplicity, test coverage, path/artifact security and persistence/cancellation. Explicit diagnostic execution cannot claim canonical PASS; clean finalization does not hide a failed branch. C11 remains outside automatic dispatch and complete canonical PASS remains 101/158 pending live execution and independent audit. +The final full Heal contract suite passed 593 tests with eight platform skips; all 30 focused incarnation tests, the unchanged 84/301/532 matrix, AST, bash syntax, actionlint and whitespace passed on the final code. Two sequential final reviews covered correctness, simplicity, test coverage, path/artifact security and persistence/cancellation. Explicit diagnostic execution cannot claim canonical PASS; clean finalization does not hide a failed branch. C11 remains outside automatic dispatch and complete canonical PASS remains 101/158 pending live execution and independent audit. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index e0520d0e..3a3d29ba 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -172,8 +172,8 @@ def incarnation_operation(request): if request["action"] == "incarnation-census": require(set(request) == {"runId", "action", "family", "taskId", "bucketIncarnationId"}, "C11 stopped census accepts only the original binding") - return {"readOnly": True, "stable": True, "records": backend.census( - roots, backend.identity(owner, request, journal), journal, fault)} + binding = backend.identity(owner, request, journal) + return {**binding, "readOnly": True, "stable": True, "records": backend.census(roots, binding, journal, fault)} methods = {"incarnation-capture": backend.capture, "incarnation-install": backend.install, "incarnation-finalize": backend.finalize} require(request["action"] in methods, "unknown C11 record transaction") diff --git a/scripts/validation_heal/incarnation.py b/scripts/validation_heal/incarnation.py index 6e3dc5af..708b2e7b 100644 --- a/scripts/validation_heal/incarnation.py +++ b/scripts/validation_heal/incarnation.py @@ -516,11 +516,15 @@ def run_replay(args, topology, nodes, branch, candidate): observed.sort(key=lambda row: (row["disk"], [*journal.names("checkpoint", token), *journal.names("root", token)].index(row["relativePath"]))) else: observed = fleet.remote(node, "incarnation-census", **params)["records"] + fleet.record("C11-independent-original-record-census", {"node": node, **params, "records": observed}) fleet.incarnation_replays.append((node, params.copy())) receipt = fleet.record("archive-and-detach-original-replay", {"node": node, **fleet.remote( node, "incarnation-capture", **params, expectedRecords=record_digests(observed))}) require(receipt.get("detached") is True and receipt.get("originalProductBytes") is True and receipt.get("replacementBytesAccepted") is False, "C11 original durable record detachment is incomplete") + detached = fleet.record("C11-independent-detached-record-census", {"node": node, **fleet.remote( + node, "incarnation-census", **params)}) + require(all(not row["present"] for row in detached["records"]), "C11 independent observer still sees original replay sources") fleet.record("C11-transition-runtime", {"node": node, **fleet.remote(node, "incarnation-phase", phase="transition")}) if gate_member is not None: fleet.record("C11-old-MRF-member-release", fleet.remote(gate_member[0], "drive-gate", mode="clear")) @@ -547,6 +551,10 @@ def run_replay(args, topology, nodes, branch, candidate): receipt = fleet.record("install-exact-original-replay", {"node": node, **fleet.remote(node, "incarnation-install", **params)}) require(receipt.get("installed") is True and receipt.get("originalArchiveRetained") is True, "C11 original record transaction was not completely installed") + installed = fleet.record("C11-independent-installed-record-census", {"node": node, **fleet.remote( + node, "incarnation-census", **params)}) + require(journal.same_json(installed["records"], receipt["records"]), + "C11 independent observer differs from the exact installed original bytes") fleet.record("C11-replay-runtime", {"node": node, **fleet.remote(node, "incarnation-phase", phase="replay")}) authority = fleet.snapshot("successor-physical-before-original-replay", quiescent=True) system_before = system_snapshot(fleet, [bucket, control], "successor-system-before-original-replay") diff --git a/test_validation_heal_incarnation_replay.py b/test_validation_heal_incarnation_replay.py index ac20bf74..2313ec07 100644 --- a/test_validation_heal_incarnation_replay.py +++ b/test_validation_heal_incarnation_replay.py @@ -439,7 +439,7 @@ def remote(self, node, action, **params): assert (node, {key: params[key] for key in ("family", "taskId", "bucketIncarnationId")}) in self.incarnation_replays raise OSError("SSH acknowledgement lost after mutation") if action == "incarnation-finalize": raise ValueError("original transaction incomplete") - if action == "incarnation-capture": return {"detached": True, "originalProductBytes": True, "replacementBytesAccepted": False} + if action == "incarnation-capture": return {"detached": True, "originalProductBytes": True, "replacementBytesAccepted": False, "records": []} if action == "incarnation-census": return {"records": []} if action == "drive-gate-probe": return {"readProbe": {"errno": 13}, "errno": 13, "dacBypassRemoved": True} return {} From c4fde9a4c5f28bf84ec87fd0aa2aa766df450326 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 21:12:56 +0800 Subject: [PATCH 183/343] test(heal): reuse scoped F07 native persistence evidence --- docs/validation-migration-map.md | 2 ++ scripts/validation_heal/persistence.py | 22 +++++++++++----- test_validation_heal_persistence.py | 35 ++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index ef7da7fc..7338d8ff 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1945,3 +1945,5 @@ The old MRF fault gate is released while stopped after all original replay sourc Immutable package run 38240858236/1 and tool run 38244986993/1 succeeded for source 1aa9be87d18cf941e5020f19e8622c26ba4b10ac. Independent Git-object checks verified the exact source tree, lock and all source contracts; all seven MRF sources match the reviewed reference byte-for-byte. The complete 68,625,642-byte DEB digest is b99dd915376b51f0896fc346ca4642b7d26bf3352135bb5c6f60f8179d865761 and its contained binary digest is 0103ca46386396ed15f27b9efbe4196fe62bd9e93033671a88f3b3e944602cf4. Inspector, nine native storage regressions and fifteen manager regressions passed independent artifact consumers for harness d5a41739a7a544d93ba71f51583d45ad4e6161eb. A new immutable tool build is required for the updated replay harness; older harness provenance is not accepted. The BLOCKED writer preserves unverified migration status, original provenance and no PASS assertions. A phase identity in the process environment detects interrupted checkpoint transitions even when product configuration knobs are identical. Independent stopped censuses are retained before capture, after detachment and after installation to compare each original byte and absence witness. Read-only environment run 38251226817 confirms 16 distinct eligible endpoints on the four current Heal hosts for H1/H2; H4 remains unavailable. The final full Heal contract suite passed 593 tests with eight platform skips; all 30 focused incarnation tests, the unchanged 84/301/532 matrix, AST, bash syntax, actionlint and whitespace passed on the final code. Two sequential final reviews covered correctness, simplicity, test coverage, path/artifact security and persistence/cancellation. Explicit diagnostic execution cannot claim canonical PASS; clean finalization does not hide a failed branch. C11 remains outside automatic dispatch and complete canonical PASS remains 101/158 pending live execution and independent audit. + +F07 now reuses its independently verified ten-test candidate-bound native scope as PREREQUISITE_PASS without a fleet topology or DB/DV claim. The remaining three live flows and full review of the required fsync/commit cuts remain blocked; existing publication interruptions are not relabelled as individual fsync or power-loss proof. The exact 1aa9be87 candidate is accepted only after all six checkpoint route/source contracts were independently proven byte-identical to the reviewed reference. Thirteen focused persistence/artifact tests passed. Canonical PASS remains 101/158. diff --git a/scripts/validation_heal/persistence.py b/scripts/validation_heal/persistence.py index 281d1deb..b062b362 100644 --- a/scripts/validation_heal/persistence.py +++ b/scripts/validation_heal/persistence.py @@ -48,7 +48,6 @@ "mrf-checksum": "H1/H2 ACK-bound committed MRF corruption, retention and replay orchestration is incomplete", "stale-generation": "an actual earlier checkpoint capture exists; complete rollback and durable responsibility reconciliation is still required", "journal-write-failure": "the private bounded volume exists; actual checkpoint write refusal, retained ownership and release/rescan orchestration is still required", - "native-persist-fence": "publication interruption prerequisites do not prove every required checkpoint/MRF fsync and commit interruption boundary", } SIZE = 5 * 1024**2 + 123 @@ -322,7 +321,8 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"F07 interrupted by sig args = parser.parse_args() require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "F07 recipe branches changed") candidate = json.loads(args.candidate.read_text()) - require(candidate["source_sha"] == checkpoints.REFERENCE["candidateCommit"], "F07 checkpoint failure contract requires the exact reviewed candidate") + require(candidate["source_sha"] in {checkpoints.REFERENCE["candidateCommit"], "1aa9be87d18cf941e5020f19e8622c26ba4b10ac"}, + "F07 checkpoint failure contract requires an exact reviewed candidate with identical checkpoint sources") native = verify_manager(os.environ.get("VALIDATION_HEAL_MANAGER_DIRECTORY", ""), candidate, CASE_ID) save(args.evidence / "independent-native-persistence-prerequisites.json", native) require(re.fullmatch(r"[0-9a-f]{40}", args.testing_commit) and re.fullmatch(r"[0-9a-f]{64}", args.binary_sha) @@ -335,15 +335,25 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"F07 interrupted by sig for topology, selected in (("H1", nodes[:1]), ("H2", nodes)): for fixture in ("DB", "DV"): for branch in BRANCHES: - result = run_checkpoint_fault(args, topology, selected, fixture, branch, candidate) if branch in IMPLEMENTED else { - "topology": topology, "fixture": fixture, "branch": branch, "verdict": "BLOCKED", "cleanup": "PASS", - "reason": BLOCKERS[branch], "fleetBranchExecuted": False, "canonicalCasePassClaimed": False} + if branch in IMPLEMENTED: + result = run_checkpoint_fault(args, topology, selected, fixture, branch, candidate) + elif branch == "native-persist-fence": + result = {"requestedTopology": topology, "requestedFixture": fixture, "topology": None, + "fixture": None, "branch": branch, "verdict": "PREREQUISITE_PASS", "cleanup": "PASS", + "nativeProof": native, "fleetBranchExecuted": False, "canonicalCasePassClaimed": False, + "evidenceScope": "exact-candidate checkpoint/MRF publication native prerequisite", + "individualFsyncHooksClaimed": False, "physicalPowerLossClaimed": False, + "requiredNativeCoverageComplete": False, + "remainingNativeCoverage": "review every required checkpoint/MRF fsync and commit cut against the source assertions"} + else: + result = {"topology": topology, "fixture": fixture, "branch": branch, "verdict": "BLOCKED", "cleanup": "PASS", + "reason": BLOCKERS[branch], "fleetBranchExecuted": False, "canonicalCasePassClaimed": False} results.append(result) if result["cleanup"] != "PASS": return 1 finally: save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "complete": False, "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL", - "canonicalCasePassClaimed": False, "reason": "four complete branch orchestrations remain blocked"}) + "canonicalCasePassClaimed": False, "reason": "three live branches and full native cut coverage remain blocked; existing native tests are separate prerequisites"}) return 1 diff --git a/test_validation_heal_persistence.py b/test_validation_heal_persistence.py index de3774fe..6ebff2d1 100644 --- a/test_validation_heal_persistence.py +++ b/test_validation_heal_persistence.py @@ -2,6 +2,8 @@ import base64 import copy +import json +import os from pathlib import Path import tempfile from types import SimpleNamespace @@ -124,6 +126,39 @@ def test_incomplete_branches_never_enter_fleet_mutation(self): with self.subTest(branch=branch), self.assertRaisesRegex(ValueError, "cannot mutate"): persistence.run_checkpoint_fault(None, "H1", ["node"], "DB", branch, {}) + def test_native_branch_reuses_exact_evidence_without_claiming_fleet_or_full_case_pass(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp); candidate = root / "candidate.json" + candidate.write_text(json.dumps({"source_sha": "1aa9be87d18cf941e5020f19e8622c26ba4b10ac"})) + args = ["persistence.py", "--case-id", persistence.CASE_ID, "--candidate", str(candidate), + "--evidence", str(root), "--rc-bin", str(root / "rc"), "--tool-directory", str(root / "tools"), + "--testing-commit", "a" * 40, "--binary-sha", "b" * 64] + proof = {"caseId": persistence.CASE_ID, "casePrerequisiteTests": list(range(10)), + "canonicalFleetBranchesExecuted": False, "canonicalCasePassClaimed": False} + def passed(args, topology, nodes, fixture, branch, candidate): + return {"topology": topology, "fixture": fixture, "branch": branch, "verdict": "PASS", "cleanup": "PASS"} + with patch("sys.argv", args), patch.dict(os.environ, {"GITHUB_RUN_ID": "12345"}), \ + patch.object(persistence, "verify_manager", return_value=proof) as verify, \ + patch.object(persistence, "run_checkpoint_fault", side_effect=passed) as mutate: + self.assertEqual(persistence.main(), 1) + verify.assert_called_once() + self.assertEqual(mutate.call_count, 8) + summary = json.loads((root / "healthy-summary.json").read_text()) + self.assertEqual(len(summary["results"]), 24) + self.assertFalse(summary["complete"]) + self.assertFalse(summary["canonicalCasePassClaimed"]) + prerequisites = [row for row in summary["results"] if row["branch"] == "native-persist-fence"] + self.assertEqual(len(prerequisites), 4) + for row in prerequisites: + self.assertEqual(row["verdict"], "PREREQUISITE_PASS") + self.assertIsNone(row["topology"]); self.assertIsNone(row["fixture"]) + self.assertFalse(row["fleetBranchExecuted"]) + self.assertFalse(row["individualFsyncHooksClaimed"]) + self.assertFalse(row["physicalPowerLossClaimed"]) + self.assertFalse(row["requiredNativeCoverageComplete"]) + self.assertEqual(row["nativeProof"], proof) + self.assertEqual(sum(row["verdict"] == "BLOCKED" for row in summary["results"]), 12) + def interrupted_injection(self, cleanup): _, _, frozen, _, _ = self.fixture() snapshots = {"node": {"files": [], "formats": []}} From d1b6492b17f862d6c0c98c6a8b7d60a009fb7d74 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 21:34:15 +0800 Subject: [PATCH 184/343] test(heal): retain original F07 MRF corruption responsibility --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-heal-cases.md | 2 +- docs/validation-heal-environment-gates.md | 2 +- docs/validation-migration-map.md | 2 + scripts/validation_heal/execution_policy.py | 2 +- scripts/validation_heal/persistence.py | 7 +- scripts/validation_heal/persistence_mrf.py | 234 ++++++++++++++++++ scripts/validation_heal/recipes.json | 2 +- test_validation_heal_persistence.py | 5 +- test_validation_heal_persistence_mrf.py | 124 ++++++++++ 10 files changed, 376 insertions(+), 8 deletions(-) create mode 100644 scripts/validation_heal/persistence_mrf.py create mode 100644 test_validation_heal_persistence_mrf.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index e9b10735..85b143e4 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -48,6 +48,7 @@ on: - test_validation_heal_environment_gate.py - test_validation_heal_outcome.py - test_validation_heal_persistence.py + - test_validation_heal_persistence_mrf.py - test_validation_heal_incarnation.py - test_validation_heal_incarnation_replay.py - scripts/validation_heal/** @@ -111,6 +112,7 @@ on: - test_validation_heal_environment_gate.py - test_validation_heal_outcome.py - test_validation_heal_persistence.py + - test_validation_heal_persistence_mrf.py - test_validation_heal_incarnation.py - test_validation_heal_incarnation_replay.py - scripts/validation_heal/** @@ -153,7 +155,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py - - run: python3 -B -m unittest -v test_validation_heal_persistence.py + - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py - run: python3 -B -m unittest -v test_validation_heal_incarnation.py test_validation_heal_incarnation_replay.py - run: python3 -B -m unittest -v test_validation_heal_incarnation_native.py - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index 0d6c47ce..6b50820b 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1202,7 +1202,7 @@ Topologies: H1,H2. Branches: checkpoint-truncate, mrf-checksum, future-schema, s Execution policy: `evidence-blocked`; disabled by default: `false`. -Blocked reason (`EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION`): 已有 journal 故障工具和 10 项候选版本持久化原生回归，但尚缺 H1/H2 全六分支编排、写失败诊断、责任保留和安全重扫；现有 publication interruption 证据也不等于每个 fsync 临界点或断电证据。 +Blocked reason (`EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION`): 已实现 H1/H2 checkpoint 截断、未来 schema、ACK 绑定的 committed MRF CRC 故障和独立安全重扫；10 项原生回归仅为前置证据。仍缺陈旧代次、journal 写失败的完整现场流程及所需 fsync/commit 临界点审核，已实现分支尚无完整 live 审计。 1. 先制造并证明真实 durable root/MRF 责任，CHECKPOINT 定位精确本 run task/record，其他系统记录禁止触碰。 2. 停止测试服务，按 FAULT_FORMAT 对这些已审核记录分别截断、坏校验、未来 schema、旧代次；写失败实例用独占限额故障层，不塞满真实共享盘。 diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index 654a6391..e7c26d82 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -33,7 +33,7 @@ | Code | 用例 | Block 原因 | 解锁所需证据 | | --- | --- | --- | --- | | C11 | `heal.bucket-incarnation-recreation` | H1/H2 旧元数据盘重返、原始 checkpoint/resume 与 committed MRF 重放编排已实现；精确候选的 9 项存储原生回归和 3 项 manager incarnation 前置证据已独立核验。完整现场分支尚无独立审计结果 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | -| F07 | `heal.checkpoint-and-mrf-corruption` | 已有 journal 工具和 10 项原生回归；缺 H1/H2 全六分支、写失败诊断、责任保留和安全重扫编排 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | +| F07 | `heal.checkpoint-and-mrf-corruption` | 已实现 H1/H2 checkpoint 截断、未来 schema、committed MRF CRC 故障及独立安全重扫；10 项原生回归作为前置证据。仍缺陈旧代次、写失败与完整 native cut 审核，完整 live 尚未审计 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | | I05 | `heal.lock-timeout-and-caller-abort` | 请求耗时/TCP 断开不能证明服务端锁 owner 与提交顺序 | lock owner、timeout、caller abort 和 stale read-repair 的服务端屏障 | | J02 | `heal.outcome-counters-and-error-truth` | 已有 2 项 outcome 原生回归；缺混合 healthy/repaired/failed/deferred/unknown 真机夹具、dry-run 独立任务与完整 manifest 对账 | 每个 disposition、计数和 attemptFailures 对账；不是已证实产品缺少 API | diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 7338d8ff..34680ae4 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1947,3 +1947,5 @@ Immutable package run 38240858236/1 and tool run 38244986993/1 succeeded for sou The final full Heal contract suite passed 593 tests with eight platform skips; all 30 focused incarnation tests, the unchanged 84/301/532 matrix, AST, bash syntax, actionlint and whitespace passed on the final code. Two sequential final reviews covered correctness, simplicity, test coverage, path/artifact security and persistence/cancellation. Explicit diagnostic execution cannot claim canonical PASS; clean finalization does not hide a failed branch. C11 remains outside automatic dispatch and complete canonical PASS remains 101/158 pending live execution and independent audit. F07 now reuses its independently verified ten-test candidate-bound native scope as PREREQUISITE_PASS without a fleet topology or DB/DV claim. The remaining three live flows and full review of the required fsync/commit cuts remain blocked; existing publication interruptions are not relabelled as individual fsync or power-loss proof. The exact 1aa9be87 candidate is accepted only after all six checkpoint route/source contracts were independently proven byte-identical to the reviewed reference. Thirteen focused persistence/artifact tests passed. Canonical PASS remains 101/158. + +F07 committed MRF corruption now has a non-dispatched H1/H2 DB/DV executor in persistence_mrf.py. It creates real member permission denial and ACK-bound original durable obligations, freezes complete committed payload/manifest/lifecycle bytes, corrupts only each actual payload CRC through the existing stopped journal adapter, and requires a source-matched corruption diagnostic on every mutated process. All unknown bytes and original commitment/lifecycle responsibility must survive startup, independent Admin rescan, complete shard checks and all-peer business verification. Rescan repair never retires or relabels the original corrupted MRF responsibility. Lost injection acknowledgements register recovery before mutation and retain dirty namespaces. Sixty-three focused contracts and the final 599-test Heal suite passed with eight platform skips; AST, actionlint, whitespace and 84/301/532 passed. Two sequential reviews found no unresolved correctness, simplicity, security or persistence/cancellation issues after the focused failure probes. This flow has not run live; F07 still lacks stale-generation, journal-write-failure and complete native cut review. C11 live execution uses the separately frozen 891f264f39ffb608f1898fef151b273e205321b5 harness and its own rebuilt tools. Canonical PASS remains 101/158. diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py index b67a525c..3f4016da 100644 --- a/scripts/validation_heal/execution_policy.py +++ b/scripts/validation_heal/execution_policy.py @@ -134,7 +134,7 @@ def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, . ), "heal.checkpoint-and-mrf-corruption": _native( "F07", "EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION", - "已有 journal 故障工具和 10 项候选版本持久化原生回归，但尚缺 H1/H2 全六分支编排、写失败诊断、责任保留和安全重扫；现有 publication interruption 证据也不等于每个 fsync 临界点或断电证据。", + "已实现 H1/H2 checkpoint 截断、未来 schema、ACK 绑定的 committed MRF CRC 故障和独立安全重扫；10 项原生回归仅为前置证据。仍缺陈旧代次、journal 写失败的完整现场流程及所需 fsync/commit 临界点审核，已实现分支尚无完整 live 审计。", ("durable_checkpoint_mrf_fault", "write_failure_retention", "publication_boundary_evidence_binding", "complete_live_orchestrator"), ), "heal.lock-timeout-and-caller-abort": _native( diff --git a/scripts/validation_heal/persistence.py b/scripts/validation_heal/persistence.py index b062b362..c2c272bc 100644 --- a/scripts/validation_heal/persistence.py +++ b/scripts/validation_heal/persistence.py @@ -25,6 +25,7 @@ from .shards import files, map_target, run_task, validate_fault, validate_repaired_physical from .stop import idle_fleet, status from .root_restart import recovery_status + from .persistence_mrf import run as run_mrf_checksum from .versions import public_version_id, verify_proof, version_identity except ImportError: import checkpoint as checkpoints, journal_fault as journal @@ -37,6 +38,7 @@ from shards import files, map_target, run_task, validate_fault, validate_repaired_physical from stop import idle_fleet, status from root_restart import recovery_status + from persistence_mrf import run as run_mrf_checksum from versions import public_version_id, verify_proof, version_identity @@ -45,7 +47,6 @@ "journal-write-failure", "native-persist-fence") IMPLEMENTED = {"checkpoint-truncate": "truncate", "future-schema": "future-schema"} BLOCKERS = { - "mrf-checksum": "H1/H2 ACK-bound committed MRF corruption, retention and replay orchestration is incomplete", "stale-generation": "an actual earlier checkpoint capture exists; complete rollback and durable responsibility reconciliation is still required", "journal-write-failure": "the private bounded volume exists; actual checkpoint write refusal, retained ownership and release/rescan orchestration is still required", } @@ -337,6 +338,8 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"F07 interrupted by sig for branch in BRANCHES: if branch in IMPLEMENTED: result = run_checkpoint_fault(args, topology, selected, fixture, branch, candidate) + elif branch == "mrf-checksum": + result = run_mrf_checksum(args, topology, selected, fixture, candidate) elif branch == "native-persist-fence": result = {"requestedTopology": topology, "requestedFixture": fixture, "topology": None, "fixture": None, "branch": branch, "verdict": "PREREQUISITE_PASS", "cleanup": "PASS", @@ -353,7 +356,7 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"F07 interrupted by sig finally: save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "complete": False, "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL", - "canonicalCasePassClaimed": False, "reason": "three live branches and full native cut coverage remain blocked; existing native tests are separate prerequisites"}) + "canonicalCasePassClaimed": False, "reason": "stale-generation, journal-write-failure and full native cut coverage remain blocked"}) return 1 diff --git a/scripts/validation_heal/persistence_mrf.py b/scripts/validation_heal/persistence_mrf.py new file mode 100644 index 00000000..14ff521d --- /dev/null +++ b/scripts/validation_heal/persistence_mrf.py @@ -0,0 +1,234 @@ +"""F07: corrupt actual committed MRF CRCs and retain original responsibility.""" + +import base64 +from datetime import datetime, timedelta, timezone +import hashlib +import os +import secrets +import time + +try: + from . import journal_fault as journal, mrf_records as records, mrf_replay as mrf + from .heal_probe import Probe, compare_inventories, require + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .scanner_repair import JournalWindow + from .shards import map_target, run_task, verify_cohort + from .stop import idle_fleet + from .versions import validate_version_receipts, verify_quorum_contract +except ImportError: + import journal_fault as journal, mrf_records as records, mrf_replay as mrf + from heal_probe import Probe, compare_inventories, require + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from scanner_repair import JournalWindow + from shards import map_target, run_task, verify_cohort + from stop import idle_fleet + from versions import validate_version_receipts, verify_quorum_contract + + +CASE_ID = "heal.checkpoint-and-mrf-corruption" +SIZE = 5 * 1024**2 + 123 + + +def indexed(observed): + require(observed.get("readOnly") is True and observed.get("stable") is True, + "F07 MRF evidence is not an independent stable read") + rows = observed.get("records") + expected = {(disk, path) for disk in range(4) for path in records.FILES} + require(isinstance(rows, list) and len(rows) == len(expected), "F07 MRF census is incomplete") + result = {} + for row in rows: + pair = row.get("disk"), row.get("relativePath") + require(type(pair[0]) is int and pair in expected and pair not in result and type(row.get("present")) is bool, + "F07 MRF census duplicated or escaped its exact physical scope") + if row["present"]: + raw = base64.b64decode(row["rawBase64"], validate=True) + require(journal.entry(raw) == {key: row.get(key) for key in ("size", "sha256")} + and len(raw) <= records.FILES[pair[1]], "F07 MRF evidence differs from original bytes") + result[pair] = raw + else: + require(not any(key in row for key in ("size", "sha256", "rawBase64")), "F07 absent MRF file has invented bytes") + result[pair] = None + return result + + +def retained_corruption(original, current, receipts): + require(set(original) == set(current) and len(original) in {1, 4}, "F07 MRF evidence lost a bound physical node") + expected, changed = {node: indexed(value) for node, value in original.items()}, set() + require(receipts, "F07 MRF corruption has no actual original payload mutation") + for receipt in receipts: + node, disk, path = receipt.get("node"), receipt.get("disk"), receipt.get("relativePath") + identity = node, disk, path + require(identity not in changed and node in expected and type(disk) is int and (disk, path) in expected[node] + and path in {f".rustfs.sys/.heal-mrf-snapshot.{slot}.bin" for slot in (0, 1)}, + "F07 MRF mutation changed scope or duplicated a payload owner") + raw = expected[node][disk, path] + require(raw is not None and records.decode_journal(raw) and receipt.get("family") == "mrf" + and receipt.get("taskId") is None and receipt.get("kind") == "checksum" and receipt.get("applied") is True + and receipt.get("originalBackupRetained") is True and receipt.get("productRepairClaimed") is False + and journal.same_json(receipt.get("before"), journal.entry(raw)), "F07 MRF mutation lost its original payload binding") + corrupt = raw[:-1] + bytes([raw[-1] ^ 1]) + require(journal.same_json(receipt.get("injected"), journal.entry(corrupt)), "F07 MRF corruption changed bytes outside the actual CRC") + expected[node][disk, path] = corrupt + changed.add(identity) + require(all(expected[node] == indexed(value) for node, value in current.items()), + "F07 product deleted, rewrote or rebound original corrupt MRF records, manifests or lifecycle responsibility") + return {"originalCorruptRecordsRetained": True, "commitsAndLifecycleUnchanged": True, + "corruptedPayloads": len(changed), "originalResponsibilityRetired": False, + "safeRescanIsOriginalMrfCompletion": False, "backupRescueUsed": False} + + +def diagnostics(events, mutated_nodes): + found = {} + for node in mutated_nodes: + matching = [event for event in events.get(node, []) + if event.get("target") == "rustfs::heal::mrf" + and event.get("message") == "MRF committed replay checkpoint could not be inspected" + and isinstance(event.get("error"), str) + and any(word in event["error"].lower() for word in ("corrupt", "invalid", "integrity"))] + require(matching, "NOT_OBSERVED: original MRF corruption has no candidate diagnostic on " + node) + found[node] = matching + require(found, "F07 MRF diagnostic has no bound mutated process") + return found + + +def observe(fleet, label): + return fleet.record(label, {node: fleet.remote(node, "mrf-records", quiescent=True) for node in fleet.nodes}) + + +def run(args, topology, nodes, fixture, candidate): + require(topology in {"H1", "H2"} and fixture in {"DB", "DV"}, "F07 MRF source fixture changed") + directory = args.evidence / fixture / topology / "mrf-checksum" + directory.mkdir(parents=True) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="mrf-crash") + bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + fleet.snapshot_buckets = [bucket] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, + "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat(), "bucketPrefix": bucket, + "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, + "defaultEndpoint": "node1", "candidateCommit": candidate["source_sha"], "harnessCommit": args.testing_commit, + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, + "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + "maxInventoryObjects": 128, "maxInventoryBytes": 256 * 1024**2} + fleet.record("binding", binding) + tool = fleet.record("candidate-bound-inspector", verify_inspector(args.tool_directory, candidate)) + verify_quorum_contract(tool, candidate) + fleet.record("candidate-MRF-source-contract", records.verify_reference(candidate)) + probe, active, pending = Probe(binding), [], None + result = {"topology": topology, "fixture": fixture, "branch": "mrf-checksum", "verdict": "RUNNING", + "cleanup": "PENDING", "canonicalCasePassClaimed": False} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record("actual-topology", validate_topology(fleet.snapshot("deployed-topology"), topology)) + recorder = Recorder(probe, directory / "fixture") + pending = {"operation": "seed-original-MRF-fixture"} + expected = seed(recorder, bucket, fixture, 3, [SIZE]) + verify_seed(expected, probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + pending = None + initial = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, option_overrides={"recursive": True}) + incarnation = validate_version_receipts(initial, bucket, expected, 2)["bucketIncarnationId"] + idle_fleet(fleet, binding) + fleet.stop() + before = fleet.snapshot("healthy-before-partial-write-gate", quiescent=True) + entry = next(item for item in expected if not item["deleteMarker"]) + target = map_target(before, bucket, entry["key"], "data", 1, entry["versionId"], latest_required=False) + shard = next(row for row in reversed(target["cohort"]) if row["node"] == nodes[-1]) + fleet.drive_gates.add(shard["node"]) + fleet.record("actual-partial-write-gate", fleet.remote(shard["node"], "drive-gate", disk=shard["disk"], mode="permission-denied")) + fleet.start() + denial = fleet.record("actual-member-denial", fleet.remote(shard["node"], "drive-gate-probe")) + require(denial.get("readProbe", {}).get("errno") == 13 and denial.get("errno") == 13 + and denial.get("dacBypassRemoved") is True, "F07 MRF fixture has no actual member read/write denial") + earliest = int(time.time() * 1000) - 5000 + pending = {"operation": "partial-write-durable-ACKs"} + expected, obligations, _ = mrf.acknowledge(recorder, bucket, "offline-put", expected, fixture=fixture) + baseline = fleet.record("complete-degraded-ACK-S3", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + verify_seed(expected, baseline) + pending = None + deadline = time.monotonic() + mrf.PERSIST_SECONDS + while True: + try: + bindings = mrf.pending_bindings(mrf.read_checkpoints(fleet, nodes, "original-committed-MRF"), obligations, earliest) + break + except ValueError as exc: + if not str(exc).startswith("NOT_OBSERVED:"): raise + require(time.monotonic() < deadline, "F07 ACK-bound durable MRF window was not observed") + time.sleep(.5) + require(all(owner["bucketIncarnationId"] == incarnation for owners in bindings.values() for owner in owners), + "F07 MRF ACK responsibility changed the original incarnation") + fleet.stop() + original = observe(fleet, "original-stopped-MRF-records") + mrf.pending_bindings({node: records.checkpoint(value) for node, value in original.items()}, obligations, earliest) + fleet.record("original-durable-responsibilities", {"obligations": obligations, + "bindings": [{"identity": list(key), "owners": value} for key, value in bindings.items()]}) + authorities = fleet.snapshot("original-partial-write-physical", quiescent=True) + receipts = [] + for node, observed in original.items(): + for (disk, path), raw in indexed(observed).items(): + if raw is None or ".heal-mrf-snapshot." not in path: continue + require(raw and records.decode_journal(raw), "F07 committed payload has no original responsibility") + params = {"family": "mrf", "taskId": None, "disk": disk, "relativePath": path, + "faultId": f"F07-mrf-checksum-{nodes.index(node)}-{disk}-{path[-5]}", "kind": "checksum", + "expectedBefore": journal.entry(raw)} + fleet.journal_faults.append((node, params.copy())) + receipts.append(fleet.record("actual-original-MRF-CRC-injection", {"node": node, + **fleet.remote(node, "journal-apply", **params)})) + retained_corruption(original, observe(fleet, "injected-stopped-MRF-records"), receipts) + window = JournalWindow(fleet, "F07-corrupt-MRF-replay", event_targets={"rustfs::heal::mrf"}) + fleet.start() + deadline = time.monotonic() + 60 + while True: + try: + result["originalCorruptionDiagnostics"] = diagnostics(window.capture(), {row["node"] for row in receipts}) + break + except ValueError as exc: + if not str(exc).startswith("NOT_OBSERVED:"): raise + require(time.monotonic() < deadline, str(exc)) + time.sleep(.5) + fleet.stop() + result["originalResponsibilityBeforeRescan"] = retained_corruption( + original, observe(fleet, "retained-corrupt-MRF-before-rescan"), receipts) + require(physical_content(authorities) == physical_content(fleet.snapshot("before-separate-rescan", quiescent=True)), + "F07 corrupt MRF replay silently changed original object responsibility") + fleet.record("release-original-member-gate", fleet.remote(shard["node"], "drive-gate", mode="clear")) + fleet.drive_gates.remove(shard["node"]) + fleet.start() + rescanned = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, option_overrides={"recursive": True}) + result["separateRescan"] = validate_version_receipts(rescanned, bucket, expected, 2, + required_repairs={(item["key"], item["versionId"]) for item in obligations}, + metadata_repairs={item["key"] for item in obligations}) + require(result["separateRescan"]["bucketIncarnationId"] == incarnation, "F07 rescan rebound the original incarnation") + idle_fleet(fleet, binding) + fleet.stop() + repaired = fleet.snapshot("complete-rescan-physical-before-business-reads", quiescent=True) + cohorts = mrf.replica_census(repaired, bucket, expected, authorities) + result["fullShardIntegrity"] = [verify_cohort(fleet, cohort, "F07-full-shard-proof") for cohort in cohorts] + retained_corruption(original, observe(fleet, "retained-corrupt-MRF-after-rescan"), receipts) + fleet.start() + for endpoint in binding["endpoints"]: + inventory = fleet.record("all-peer-S3", {"endpoint": endpoint, + **Probe(binding, endpoint).inventory(bucket, unversioned_current_tags=fixture == "DB")}) + verify_seed(expected, inventory); compare_inventories(baseline, inventory) + fleet.stop() + require(physical_content(repaired) == physical_content(fleet.snapshot("after-business-reads", quiescent=True)), + "F07 business reads repaired work not proved by the separate rescan") + result["originalResponsibilityAfterReads"] = retained_corruption(original, observe(fleet, "retained-corrupt-MRF-after-reads"), receipts) + result.update(verdict="PASS", originalMrfRepairClaimed=False, completePhysicalAndS3ManifestsVerified=True) + except (Exception, KeyboardInterrupt) as exc: + result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) + try: + fleet.stop(); observe(fleet, "original-MRF-failure-scene") + fleet.snapshot("original-physical-failure-scene", quiescent=True) + except Exception as error: result["failureEvidenceError"] = str(error) + finally: + clean = fleet.cleanup(preserve_namespaces=pending is not None) + result["cleanup"] = "PASS" if clean and pending is None else "FAIL" + if pending is not None: result["unresolvedMutation"] = pending + fleet.record("MRF-corruption-branch-result", result) + return result diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index db23a0a8..504761c2 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -2155,7 +2155,7 @@ "dispatchable": false, "code": "F07", "reasonCode": "EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION", - "reason": "已有 journal 故障工具和 10 项候选版本持久化原生回归，但尚缺 H1/H2 全六分支编排、写失败诊断、责任保留和安全重扫；现有 publication interruption 证据也不等于每个 fsync 临界点或断电证据。", + "reason": "已实现 H1/H2 checkpoint 截断、未来 schema、ACK 绑定的 committed MRF CRC 故障和独立安全重扫；10 项原生回归仅为前置证据。仍缺陈旧代次、journal 写失败的完整现场流程及所需 fsync/commit 临界点审核，已实现分支尚无完整 live 审计。", "requiredCapabilities": [ "durable_checkpoint_mrf_fault", "write_failure_retention", diff --git a/test_validation_heal_persistence.py b/test_validation_heal_persistence.py index 6ebff2d1..7f25bda1 100644 --- a/test_validation_heal_persistence.py +++ b/test_validation_heal_persistence.py @@ -139,10 +139,13 @@ def passed(args, topology, nodes, fixture, branch, candidate): return {"topology": topology, "fixture": fixture, "branch": branch, "verdict": "PASS", "cleanup": "PASS"} with patch("sys.argv", args), patch.dict(os.environ, {"GITHUB_RUN_ID": "12345"}), \ patch.object(persistence, "verify_manager", return_value=proof) as verify, \ + patch.object(persistence, "run_mrf_checksum", side_effect=lambda args, topology, nodes, fixture, candidate: { + "topology": topology, "fixture": fixture, "branch": "mrf-checksum", "verdict": "PASS", "cleanup": "PASS"}) as mrf_mutate, \ patch.object(persistence, "run_checkpoint_fault", side_effect=passed) as mutate: self.assertEqual(persistence.main(), 1) verify.assert_called_once() self.assertEqual(mutate.call_count, 8) + self.assertEqual(mrf_mutate.call_count, 4) summary = json.loads((root / "healthy-summary.json").read_text()) self.assertEqual(len(summary["results"]), 24) self.assertFalse(summary["complete"]) @@ -157,7 +160,7 @@ def passed(args, topology, nodes, fixture, branch, candidate): self.assertFalse(row["physicalPowerLossClaimed"]) self.assertFalse(row["requiredNativeCoverageComplete"]) self.assertEqual(row["nativeProof"], proof) - self.assertEqual(sum(row["verdict"] == "BLOCKED" for row in summary["results"]), 12) + self.assertEqual(sum(row["verdict"] == "BLOCKED" for row in summary["results"]), 8) def interrupted_injection(self, cleanup): _, _, frozen, _, _ = self.fixture() diff --git a/test_validation_heal_persistence_mrf.py b/test_validation_heal_persistence_mrf.py new file mode 100644 index 00000000..9b38d7e8 --- /dev/null +++ b/test_validation_heal_persistence_mrf.py @@ -0,0 +1,124 @@ +"""Original committed MRF corruption must remain distinct from safe rescan.""" + +import base64 +import copy +from contextlib import ExitStack +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from scripts.validation_heal import journal_fault as journal, persistence_mrf as suite +import test_validation_heal_mrf as fixtures + + +class MrfCorruptionTests(unittest.TestCase): + def fixture(self): + original = {"node": fixtures.DurableMrfTests().observed()} + current, receipts = copy.deepcopy(original), [] + for row in current["node"]["records"]: + if not row["present"] or ".heal-mrf-snapshot." not in row["relativePath"]: continue + raw = base64.b64decode(row["rawBase64"]) + corrupted = raw[:-1] + bytes([raw[-1] ^ 1]) + receipts.append({"node": "node", "disk": row["disk"], "relativePath": row["relativePath"], + "family": "mrf", "taskId": None, "kind": "checksum", "applied": True, + "originalBackupRetained": True, "productRepairClaimed": False, + "before": journal.entry(raw), "injected": journal.entry(corrupted)}) + row.update(journal.entry(corrupted), rawBase64=base64.b64encode(corrupted).decode()) + return original, current, receipts + + def test_corrupt_crc_keeps_every_manifest_lifecycle_and_unknown_byte(self): + proof = suite.retained_corruption(*self.fixture()) + self.assertEqual(proof["corruptedPayloads"], 2) + self.assertTrue(proof["commitsAndLifecycleUnchanged"]) + self.assertFalse(proof["originalResponsibilityRetired"]) + self.assertFalse(proof["safeRescanIsOriginalMrfCompletion"]) + self.assertFalse(proof["backupRescueUsed"]) + + def test_missing_rewritten_restored_rebound_or_extra_record_is_not_retention(self): + for change in ("missing-payload", "manifest", "lifecycle", "restore", "foreign-path", "partial", "duplicate", "unstable", "absent-bytes"): + original, current, receipts = self.fixture() + rows = current["node"]["records"] + payload = next(row for row in rows if row["present"] and "snapshot" in row["relativePath"]) + if change in {"missing-payload", "manifest", "lifecycle"}: + kind = "commit" if change == "manifest" else change + target = payload if change == "missing-payload" else next(row for row in rows if row["present"] and kind in row["relativePath"]) + identity = {key: target[key] for key in ("disk", "relativePath")} + target.clear(); target.update(identity, present=False) + elif change == "restore": payload.update(copy.deepcopy(next(row for row in original["node"]["records"] if row["disk"]==payload["disk"] and row["relativePath"]==payload["relativePath"]))) + elif change == "foreign-path": payload["relativePath"] = ".rustfs.sys/config/iam/user.json" + elif change == "partial": rows.pop() + elif change == "duplicate": rows[-1] = copy.deepcopy(rows[0]) + elif change == "unstable": current["node"]["stable"] = False + else: next(row for row in rows if not row["present"])["rawBase64"] = "" + with self.subTest(change=change), self.assertRaises(ValueError): suite.retained_corruption(original, current, receipts) + + def test_mutation_receipt_requires_original_bytes_exact_crc_scope_and_backup(self): + for change in ("no-mutation", "duplicate", "owner", "before", "injected", "kind", "task", "repair", "backup"): + original, current, receipts = self.fixture() + if change == "no-mutation": receipts = [] + elif change == "duplicate": receipts.append(copy.deepcopy(receipts[0])) + elif change == "owner": receipts[0]["node"] = "other" + elif change in {"before", "injected"}: receipts[0][change]["sha256"] = "f" * 64 + elif change == "kind": receipts[0]["kind"] = "truncate" + elif change == "task": receipts[0]["taskId"] = fixtures.DurableMrfTests.owner + elif change == "repair": receipts[0]["productRepairClaimed"] = True + else: receipts[0]["originalBackupRetained"] = False + with self.subTest(change=change), self.assertRaises(ValueError): suite.retained_corruption(original, current, receipts) + + def test_original_corruption_diagnostic_is_bound_to_each_mutated_node_and_typed_error(self): + event = {"target": "rustfs::heal::mrf", "message": "MRF committed replay checkpoint could not be inspected", + "error": "MRF checkpoint has an invalid or incomplete commit record"} + self.assertEqual(suite.diagnostics({"node": [event]}, {"node"}), {"node": [event]}) + for change in ("target", "message", "error", "missing-node", "no-mutated-node"): + value = copy.deepcopy(event) + if change in value: value[change] = "permission denied" if change == "error" else "unrelated" + nodes = {"node", "other"} if change == "missing-node" else set() if change == "no-mutated-node" else {"node"} + with self.subTest(change=change), self.assertRaises(ValueError): suite.diagnostics({"node": [value]}, nodes) + + def test_lost_injection_ack_registers_recovery_before_mutation_and_keeps_namespace(self): + with tempfile.TemporaryDirectory() as temp, ExitStack() as stack: + root = Path(temp); rc = root / "rc"; rc.write_bytes(b"client") + args = SimpleNamespace(evidence=root, rc_bin=rc, tool_directory=root, testing_commit="a" * 40, binary_sha="b" * 64) + original = fixtures.DurableMrfTests() + expected = [{"key": "object", "versionId": original.version, "deleteMarker": False}] + captured = [] + class Fleet(suite.Fleet): + def deploy(self, *args): self.reserved = list(self.nodes); captured.append(self) + def snapshot(self, *args, **kwargs): return {} + def start(self): pass + def stop(self): pass + def remote(self, node, action, **params): + if action == "mrf-records": return original.observed() + if action == "drive-gate-probe": return {"errno": 13, "readProbe": {"errno": 13}, "dacBypassRemoved": True} + if action == "journal-apply": + assert (node, params) in self.journal_faults + raise OSError("SSH acknowledgement lost after original CRC mutation") + if action == "journal-recover": raise ValueError("original transaction state must be audited") + if action == "cleanup": raise AssertionError("dirty namespace cannot be deleted") + return {} + class Probe: + def __init__(self, *args): pass + def inventory(self, *args, **kwargs): return {} + for name, replacement in {"Fleet": Fleet, "Probe": Probe, "Recorder": lambda *args: None, + "verify_inspector": lambda *args: {}, "verify_quorum_contract": lambda *args: {}, + "validate_topology": lambda *args: {}, "seed": lambda *args: expected, + "verify_seed": lambda *args: {}, "run_task": lambda *args, **kwargs: {}, + "validate_version_receipts": lambda *args: {"bucketIncarnationId": original.incarnation}, + "idle_fleet": lambda *args: None, "map_target": lambda *args, **kwargs: {"cohort": [{"node": "node", "disk": 0}]}}.items(): + stack.enter_context(patch.object(suite, name, replacement)) + stack.enter_context(patch.dict(os.environ, {"GITHUB_RUN_ID": "12345"})) + stack.enter_context(patch.object(suite.records, "verify_reference", return_value={})) + stack.enter_context(patch.object(suite.mrf, "acknowledge", return_value=(expected, [original.obligation()], {}))) + stack.enter_context(patch.object(suite.mrf, "read_checkpoints", return_value={})) + stack.enter_context(patch.object(suite.mrf, "pending_bindings", return_value={(): [{"bucketIncarnationId": original.incarnation}]})) + result = suite.run(args, "H1", ["node"], "DV", {"source_sha": "c" * 40, "package_sha256": "d" * 64}) + self.assertEqual((result["verdict"], result["cleanup"]), ("FAIL", "FAIL")) + self.assertEqual(result["failure"]["type"], "OSError") + self.assertEqual(len(captured[0].journal_faults), 1) + self.assertFalse(result["canonicalCasePassClaimed"]) + + +if __name__ == "__main__": unittest.main() From b3edd31a74eeba25b98c3a326b62bf24ad43b5bc Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 21:53:55 +0800 Subject: [PATCH 185/343] test(heal): audit retained C11 DeleteBucket refusal --- docs/validation-migration-map.md | 4 + .../validation_heal/recover_environment.py | 115 ++++++++++++++++- test_validation_heal_recovery.py | 118 ++++++++++++++++++ 3 files changed, 236 insertions(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 34680ae4..42e28701 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1949,3 +1949,7 @@ The final full Heal contract suite passed 593 tests with eight platform skips; a F07 now reuses its independently verified ten-test candidate-bound native scope as PREREQUISITE_PASS without a fleet topology or DB/DV claim. The remaining three live flows and full review of the required fsync/commit cuts remain blocked; existing publication interruptions are not relabelled as individual fsync or power-loss proof. The exact 1aa9be87 candidate is accepted only after all six checkpoint route/source contracts were independently proven byte-identical to the reviewed reference. Thirteen focused persistence/artifact tests passed. Canonical PASS remains 101/158. F07 committed MRF corruption now has a non-dispatched H1/H2 DB/DV executor in persistence_mrf.py. It creates real member permission denial and ACK-bound original durable obligations, freezes complete committed payload/manifest/lifecycle bytes, corrupts only each actual payload CRC through the existing stopped journal adapter, and requires a source-matched corruption diagnostic on every mutated process. All unknown bytes and original commitment/lifecycle responsibility must survive startup, independent Admin rescan, complete shard checks and all-peer business verification. Rescan repair never retires or relabels the original corrupted MRF responsibility. Lost injection acknowledgements register recovery before mutation and retain dirty namespaces. Sixty-three focused contracts and the final 599-test Heal suite passed with eight platform skips; AST, actionlint, whitespace and 84/301/532 passed. Two sequential reviews found no unresolved correctness, simplicity, security or persistence/cancellation issues after the focused failure probes. This flow has not run live; F07 still lacks stale-generation, journal-write-failure and complete native cut review. C11 live execution uses the separately frozen 891f264f39ffb608f1898fef151b273e205321b5 harness and its own rebuilt tools. Canonical PASS remains 101/158. + +C11 first live diagnostic and retained-scene recovery (2026-10-12) + +Tools 38253155969/1 passed inspector, nine storage regressions and fifteen manager regressions; exact source/tree/lock, collector bytes, binary digests and original results were independently verified against harness 891f264f39ffb608f1898fef151b273e205321b5. Diagnostic 38257435494/1 stopped in H1 old-metadata-rejoin: all eleven original versions/markers had explicit deletion ACKs, but DeleteBucket returned 500/InternalError with File access denied while one member was isolated. Its FAIL and retained namespace are preserved; no later branch ran and canonical PASS stays 101/158. Recovery accepts only this exact known refusal, a complete digest-bound request/ACK ledger, a restored member gate and the unchanged stopped candidate scene. It exports the retained system records and rechecks the physical manifest immediately before finalization. Unknown calls, missing ACKs, changed provenance, other bound namespaces, replay transactions and scene drift are rejected. Recovery never converts the product failure into PASS. diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index bb203352..00f75236 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -138,6 +138,8 @@ def network_recovery_plan(artifact, failed_run, candidate): def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") + if list(artifact.glob("*/run/heal.bucket-incarnation-recreation/cases/*.json")): + return recreation_recovery_plan(artifact, failed_run, candidate) if list(artifact.glob("*/run/heal.enospc-inodes-and-quota/cases/heal.enospc-inodes-and-quota.json")): require(not list(artifact.glob("*/run/heal.object-size-boundaries/cases/*.json")), "mixed failed case recovery artifact") return quota_recovery_plan(artifact, failed_run, candidate) @@ -189,6 +191,104 @@ def recovery_plan(artifact, failed_run, candidate): return plans +def recreation_recovery_plan(artifact, failed_run, candidate): + """Archive a stopped C11 fixture after an exact known DeleteBucket refusal.""" + case = "heal.bucket-incarnation-recreation" + paths = list(artifact.glob("*/run/*/cases/*.json")) + require(len(paths) == 1 and paths[0].name == case + ".json", + "C11 recovery requires one unmixed failed result") + result = json.loads(paths[0].read_text()) + provenance = result.get("provenance") or {} + require(result.get("case_id") == case and result.get("verdict") == "FAIL" + and result.get("cleanup_status") == "FAIL" + and provenance.get("source_commit") == candidate["source_sha"] + and provenance.get("candidate_sha256") == candidate["package_sha256"], + "C11 failed-result candidate or cleanup differs") + evidence = paths[0].parents[1] / f"evidence/{case}" + summary = json.loads((evidence / "healthy-summary.json").read_text()) + rows = summary.get("results", []) + require(summary.get("caseId") == case and summary.get("cleanup") == "FAIL" + and summary.get("complete") is False and summary.get("canonicalCasePassClaimed") is False + and len(rows) == 1 and rows[0].get("topology") == "H1" + and rows[0].get("fixture") == "DV" and rows[0].get("branch") == "old-metadata-rejoin" + and rows[0].get("verdict") == "FAIL" and rows[0].get("cleanup") == "FAIL" + and rows[0].get("failure", {}).get("type") == "RecordedCallError" + and rows[0].get("unresolvedMutation") == {"operation": "normal-S3-delete-and-recreate"}, + "C11 recovery supports only its first known DeleteBucket refusal") + folder = evidence / "H1/old-metadata-rejoin" + require(len(list(evidence.rglob("*-binding.json"))) == 1, + "C11 recovery cannot omit another bound namespace") + def one(label): + found = list(folder.glob("*-" + label + ".json")) + require(len(found) == 1, "C11 recovery lacks one exact " + label) + return json.loads(found[0].read_text()) + binding, scene, old = one("binding"), one("original-recreation-failure-scene"), one("old-ACK-manifest") + require(re.fullmatch(rf"gh-{failed_run}-[1-9][0-9]*-h1", binding.get("runId", "")) + and binding.get("candidateCommit") == candidate["source_sha"] + and binding.get("candidatePackageSha256") == candidate["package_sha256"] + and binding.get("candidateBinarySha256") == provenance.get("binary_sha256") + and binding.get("harnessCommit") == provenance.get("testing_commit") + and len(scene) == 1 and all(value.get("quiescent") is True and value.get("pid") == 0 + and value.get("binarySha256") == provenance.get("binary_sha256") for value in scene.values()), + "C11 archive lost its stopped run/candidate identity") + node, bucket = next(iter(scene)), binding.get("bucketPrefix", "") + require(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) + and re.fullmatch(r"rv-heal-[0-9a-f]{16}", bucket), "C11 archive has invalid owned host/bucket") + recovered = one("gate-recovery") + require(recovered.get("node") == node and recovered.get("state") == "CLEAR" + and recovered.get("originalPermissionsRestored") is True, + "C11 physical gate was not already recovered") + requests = sorted((folder / "fixture").glob("*-request.json")) + acknowledgements = sorted((folder / "fixture").glob("*-ack.json")) + require(len(requests) == len(acknowledgements) and len(requests) > 1, + "C11 archive has an unacknowledged or missing S3 call") + observed = [] + for sequence, (request_path, ack_path) in enumerate(zip(requests, acknowledgements), 1): + request, ack = json.loads(request_path.read_text()), json.loads(ack_path.read_text()) + observation = ack.get("observation") or {} + require(type(request.get("sequence")) is int and type(ack.get("sequence")) is int + and request["sequence"] == ack["sequence"] == sequence + and request.get("runId") == ack.get("runId") == binding["runId"] + and ack.get("requestFile") == request_path.name + and ack.get("requestSha256") == hashlib.sha256(request_path.read_bytes()).hexdigest() + and observation.get("operation") == request.get("operation") + and observation.get("params") == request.get("params") + and observation.get("clientCallInvoked") is True + and observation.get("resultUnknown") is False, + "C11 S3 ledger changed identity/digest or contains an unknown mutation") + require(request.get("params", {}).get("Bucket") in {bucket, bucket + "-control"}, + "C11 S3 call escaped its archived owned buckets") + if sequence < len(requests): + require(ack.get("operationCheck") == "PASS" and observation.get("acknowledged") is True, + "C11 archive has an earlier rejected S3 operation") + observed.append(observation) + refusal = observed[-1] + response = refusal.get("response") or {} + require(rows[0]["failure"].get("message") == + f"unexpected delete_bucket result; inspect ACK {len(requests):06d}, do not silently retry" + and refusal.get("operation") == "delete_bucket" and refusal.get("params") == {"Bucket": bucket} + and refusal.get("acknowledged") is False + and json.loads(acknowledgements[-1].read_text()).get("operationCheck") == "FAIL" + and response.get("Error") == {"Code": "InternalError", "Message": "File access denied"} + and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 500, + "C11 refusal is uncertain or differs from the audited product error") + wanted = {(row["key"], row["versionId"]) for row in old["expected"]} + deletes = [(row["params"].get("Key"), row["params"].get("VersionId")) for row in observed[:-1] + if row["operation"] == "delete_object" and row["params"]["Bucket"] == bucket] + require(wanted and len(wanted) == len(old["expected"]) and len(deletes) >= len(wanted) + and len(set(deletes[-len(wanted):])) == len(wanted) and set(deletes[-len(wanted):]) == wanted, + "C11 archive lacks every acknowledged original version/marker deletion") + require(not list(folder.glob("*-archive-and-detach-original-replay.json")) + and not list(folder.glob("*-successor-ACK-bound-S3.json")) + and all(row.get("path", "").split("/", 1)[0] in {bucket, bucket + "-control"} + for value in scene.values() for row in value["files"]), + "C11 retained scene includes another replay transaction or successor") + return [{"kind": "rejected-incarnation-delete", "runId": binding["runId"], "nodes": [node], + "node": node, "buckets": [bucket, bucket + "-control"], "before": scene, + "binarySha256": provenance["binary_sha256"], "originalProductError": response["Error"], + "acknowledgedVersionDeletions": len(wanted), "unknownMutationClaimed": False}] + + def performance_recovery_plan(artifact, failed_run, candidate): """Finalize a performance run that expired before its next fault was applied. @@ -342,7 +442,20 @@ def main(): require(recovery_scene_matches(current, plan), "retained fleet drifted from its exact archived scene; no recovery performed") fleet.record("retained-service-logs", fleet.remote(plan["node"], "logs")) - if plan.get("kind") == "performance-no-fault-applied": + if plan.get("kind") == "rejected-incarnation-delete": + require(all(value.get("binarySha256") == plan["binarySha256"] for value in current.values()), + "retained candidate binary differs from the archived failed scene") + gate = fleet.record("audited-gate-state", fleet.remote(plan["node"], "gate-state")) + require(gate.get("state") == "CLEAR", "C11 gate drifted after archived recovery") + system = fleet.record("retained-system-bucket-records", fleet.remote(plan["node"], + "bucket-record-snapshot", buckets=plan["buckets"])) + require(system.get("credentialsRead") is False, "C11 recovery crossed its credential boundary") + require(recovery_scene_matches(fleet.snapshot("physical-immediately-before-finalization", quiescent=True), plan), + "C11 stopped scene changed during its evidence export; retain namespace") + proof = fleet.record("audited-finalization", fleet.remote(plan["node"], "cleanup")) + require(proof.get("unitStopped") is True and proof.get("rootsRemoved") is True, + "C11 owned namespace finalization was not complete") + elif plan.get("kind") == "performance-no-fault-applied": for node in plan["nodes"]: proof = fleet.record("audited-finalization", {"node": node, **fleet.remote(node, "cleanup")}) diff --git a/test_validation_heal_recovery.py b/test_validation_heal_recovery.py index 5371186b..bb4a6799 100644 --- a/test_validation_heal_recovery.py +++ b/test_validation_heal_recovery.py @@ -1,6 +1,7 @@ """A rejected fault ID cannot bypass ownership, intent or physical-scene checks.""" import hashlib +import copy import json from pathlib import Path import tempfile @@ -55,6 +56,123 @@ def test_finalization_needs_no_intent_and_exact_archived_scene(self): cleanup.assert_not_called() +class RecreationRecoveryTests(unittest.TestCase): + def archive(self, root): + case, bucket = "heal.bucket-incarnation-recreation", "rv-heal-0123456789abcdef" + base = root / "rustfs-validation-12345-1" / "run" / case + folder = base / "evidence" / case / "H1/old-metadata-rejoin" + (base / "cases").mkdir(parents=True) + (folder / "fixture").mkdir(parents=True) + candidate = {"source_sha": "a" * 40, "package_sha256": "b" * 64} + provenance = {"source_commit": "a" * 40, "candidate_sha256": "b" * 64, + "binary_sha256": "c" * 64, "testing_commit": "d" * 40} + write = lambda path, value: path.write_text(json.dumps(value)) + write(base / "cases" / (case + ".json"), {"case_id": case, "verdict": "FAIL", + "cleanup_status": "FAIL", "provenance": provenance}) + write(folder.parent.parent / "healthy-summary.json", {"caseId": case, "cleanup": "FAIL", + "complete": False, "canonicalCasePassClaimed": False, "results": [{"topology": "H1", + "fixture": "DV", "branch": "old-metadata-rejoin", "verdict": "FAIL", "cleanup": "FAIL", + "failure": {"type": "RecordedCallError", "message": "unexpected delete_bucket result; inspect ACK 000005, do not silently retry"}, + "unresolvedMutation": {"operation": "normal-S3-delete-and-recreate"}}]}) + write(folder / "00001-binding.json", {"runId": "gh-12345-1-h1", "bucketPrefix": bucket, + "candidateCommit": "a" * 40, "candidatePackageSha256": "b" * 64, + "candidateBinarySha256": "c" * 64, "harnessCommit": "d" * 40}) + write(folder / "00002-original-recreation-failure-scene.json", {"vm000": {"quiescent": True, + "pid": 0, "binarySha256": "c" * 64, "formats": [], "files": [{"disk": 0, + "path": bucket + "/objects/key/xl.meta", "size": 1, "sha256": "e" * 64}]}}) + write(folder / "00003-old-ACK-manifest.json", {"expected": [{"key": "objects/key", "versionId": version} + for version in ("old-data", "old-marker")]}) + write(folder / "00004-gate-recovery.json", {"node": "vm000", "state": "CLEAR", "originalPermissionsRestored": True}) + calls = [("create_bucket", {"Bucket": bucket}), + ("delete_object", {"Bucket": bucket, "Key": "objects/key", "VersionId": "old-data"}), + ("delete_object", {"Bucket": bucket, "Key": "objects/key", "VersionId": "old-marker"}), + ("list_multipart_uploads", {"Bucket": bucket}), ("delete_bucket", {"Bucket": bucket})] + for sequence, (operation, params) in enumerate(calls, 1): + request_path = folder / "fixture" / f"{sequence:06d}-request.json" + write(request_path, {"sequence": sequence, "runId": "gh-12345-1-h1", "operation": operation, "params": params}) + write(folder / "fixture" / f"{sequence:06d}-ack.json", {"sequence": sequence, "runId": "gh-12345-1-h1", + "requestFile": request_path.name, "requestSha256": hashlib.sha256(request_path.read_bytes()).hexdigest(), + "operationCheck": "FAIL" if sequence == 5 else "PASS", "observation": {"operation": operation, + "params": params, "clientCallInvoked": True, "resultUnknown": False, "acknowledged": sequence != 5, + "response": {"Error": {"Code": "InternalError", "Message": "File access denied"}, + "ResponseMetadata": {"HTTPStatusCode": 500}} if sequence == 5 else {}}}) + return candidate, folder + + def test_known_refusal_retains_original_failure_without_claiming_product_repair(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp); candidate, _ = self.archive(root) + plan = recovery_plan(root, 12345, candidate)[0] + self.assertEqual(plan["kind"], "rejected-incarnation-delete") + self.assertEqual(plan["acknowledgedVersionDeletions"], 2) + self.assertFalse(plan["unknownMutationClaimed"]) + self.assertEqual(plan["originalProductError"]["Code"], "InternalError") + + def test_unknown_calls_missing_ack_drifted_digest_and_different_refusal_are_rejected(self): + for mutation in ("unknown-final", "unknown-prior", "digest", "missing", "different-error", "foreign-request"): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temp: + root = Path(temp); candidate, folder = self.archive(root) + path = folder / "fixture" / ("000002-ack.json" if mutation == "unknown-prior" else "000005-ack.json") + value = json.loads(path.read_text()) + if mutation.startswith("unknown-"): value["observation"]["resultUnknown"] = True + elif mutation == "digest": value["requestSha256"] = "f" * 64 + elif mutation == "different-error": value["observation"]["response"]["Error"]["Code"] = "BucketNotEmpty" + elif mutation == "foreign-request": value["observation"]["params"]["Bucket"] = "unowned" + if mutation == "missing": path.unlink() + else: path.write_text(json.dumps(value)) + with self.assertRaises(ValueError): recovery_plan(root, 12345, candidate) + + def test_running_scene_additional_case_or_replay_transaction_cannot_be_finalized(self): + for mutation in ("running", "source", "another-case", "replay", "successor", "unrecovered-gate"): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temp: + root = Path(temp); candidate, folder = self.archive(root) + if mutation == "running": + path = folder / "00002-original-recreation-failure-scene.json" + value = json.loads(path.read_text()); value["vm000"]["pid"] = 99; path.write_text(json.dumps(value)) + elif mutation == "source": candidate["source_sha"] = "f" * 40 + elif mutation == "another-case": + path = folder.parents[4] / "another/cases/another.json" + path.parent.mkdir(parents=True); path.write_text("{}") + elif mutation == "unrecovered-gate": + path = folder / "00004-gate-recovery.json" + value = json.loads(path.read_text()); value["state"] = "ACTIVE"; path.write_text(json.dumps(value)) + else: (folder / ("00005-" + ("archive-and-detach-original-replay" if mutation == "replay" else "successor-ACK-bound-S3") + ".json")).write_text("{}") + with self.assertRaises(ValueError): recovery_plan(root, 12345, candidate) + + def test_finalization_requires_stopped_unchanged_candidate_gate_and_credential_boundary(self): + for mutation in (None, "scene-drift", "binary", "gate", "credential"): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temp: + root = Path(temp); candidate, _ = self.archive(root) + plan = recovery_plan(root, 12345, candidate)[0] + events = [] + def snapshot(self, label, **kwargs): + events.append(label) + scene = copy.deepcopy(plan["before"]) + if mutation == "scene-drift" and label == "physical-immediately-before-finalization": scene["vm000"]["files"] = [] + if mutation == "binary": scene["vm000"]["binarySha256"] = "f" * 64 + return scene + def remote(self, node, action, **params): + events.append(action) + if action == "namespace-state": return {"state": {"exists": True}, "roots": [{"exists": True}]} + if action == "gate-state": return {"state": "ACTIVE" if mutation == "gate" else "CLEAR"} + if action == "bucket-record-snapshot": return {"credentialsRead": mutation == "credential"} + if action == "cleanup": return {"cleanup": "PASS", "unitStopped": True, "rootsRemoved": True} + return {} + candidate_path = root / "candidate.json"; candidate_path.write_text(json.dumps(candidate)) + argv = ["recover_environment.py", "--failed-run-id", "12345", "--artifact", str(root), + "--candidate", str(candidate_path), "--evidence", str(root / "recovery")] + with patch("sys.argv", argv), patch.object(Fleet, "remote", remote), patch.object(Fleet, "snapshot", snapshot), \ + patch.dict("os.environ", {"RUSTFS_NODES": "vm000 vm001 vm002 vm003"}): + if mutation is None: recovery.main() + else: + with self.assertRaises(ValueError): recovery.main() + if mutation is None: + self.assertLess(events.index("bucket-record-snapshot"), events.index("cleanup")) + summary = json.loads((root / "recovery/recovery-summary.json").read_text()) + self.assertFalse(summary["canonicalPassClaimed"]) + self.assertFalse(summary["environments"][0]["originalVerdictRewritten"]) + else: self.assertNotIn("cleanup", events) + + class QuotaRecoveryTests(unittest.TestCase): def test_standalone_remote_recovery_loads_all_cleanup_backends(self): import types From d52876c984adb9c42dd4f51d3f2510de7a5eb4ab Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 22:11:02 +0800 Subject: [PATCH 186/343] test(heal): reconcile original F07 stale checkpoint replay --- docs/validation-heal-cases.md | 2 +- docs/validation-heal-environment-gates.md | 2 +- docs/validation-migration-map.md | 6 + scripts/run_validation_case.sh | 2 +- scripts/validation_heal/execution_policy.py | 2 +- scripts/validation_heal/persistence.py | 141 ++++++++++++++++-- scripts/validation_heal/recipes.json | 2 +- scripts/validation_heal/run_case.sh | 6 + test_validation_heal_persistence.py | 156 +++++++++++++++++++- 9 files changed, 298 insertions(+), 21 deletions(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index 6b50820b..ca58a2af 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1202,7 +1202,7 @@ Topologies: H1,H2. Branches: checkpoint-truncate, mrf-checksum, future-schema, s Execution policy: `evidence-blocked`; disabled by default: `false`. -Blocked reason (`EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION`): 已实现 H1/H2 checkpoint 截断、未来 schema、ACK 绑定的 committed MRF CRC 故障和独立安全重扫；10 项原生回归仅为前置证据。仍缺陈旧代次、journal 写失败的完整现场流程及所需 fsync/commit 临界点审核，已实现分支尚无完整 live 审计。 +Blocked reason (`EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION`): 已实现 H1/H2 checkpoint 截断、未来 schema、真实较早代次回放、ACK 绑定的 committed MRF CRC 故障和独立安全重扫；10 项原生回归仅为前置证据。仍缺 journal 写失败的完整现场流程及所需 fsync/commit 临界点审核，已实现分支尚无完整 live 审计。 1. 先制造并证明真实 durable root/MRF 责任，CHECKPOINT 定位精确本 run task/record，其他系统记录禁止触碰。 2. 停止测试服务，按 FAULT_FORMAT 对这些已审核记录分别截断、坏校验、未来 schema、旧代次；写失败实例用独占限额故障层，不塞满真实共享盘。 diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index e7c26d82..ffc5eb89 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -33,7 +33,7 @@ | Code | 用例 | Block 原因 | 解锁所需证据 | | --- | --- | --- | --- | | C11 | `heal.bucket-incarnation-recreation` | H1/H2 旧元数据盘重返、原始 checkpoint/resume 与 committed MRF 重放编排已实现；精确候选的 9 项存储原生回归和 3 项 manager incarnation 前置证据已独立核验。完整现场分支尚无独立审计结果 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | -| F07 | `heal.checkpoint-and-mrf-corruption` | 已实现 H1/H2 checkpoint 截断、未来 schema、committed MRF CRC 故障及独立安全重扫；10 项原生回归作为前置证据。仍缺陈旧代次、写失败与完整 native cut 审核，完整 live 尚未审计 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | +| F07 | `heal.checkpoint-and-mrf-corruption` | 已实现 H1/H2 checkpoint 截断、未来 schema、committed MRF CRC 故障及独立安全重扫；10 项原生回归作为前置证据。真实较早代次回放已实现；仍缺写失败与完整 native cut 审核，完整 live 尚未审计 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | | I05 | `heal.lock-timeout-and-caller-abort` | 请求耗时/TCP 断开不能证明服务端锁 owner 与提交顺序 | lock owner、timeout、caller abort 和 stale read-repair 的服务端屏障 | | J02 | `heal.outcome-counters-and-error-truth` | 已有 2 项 outcome 原生回归；缺混合 healthy/repaired/failed/deferred/unknown 真机夹具、dry-run 独立任务与完整 manifest 对账 | 每个 disposition、计数和 attemptFailures 对账；不是已证实产品缺少 API | diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 42e28701..70e1161a 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1953,3 +1953,9 @@ F07 committed MRF corruption now has a non-dispatched H1/H2 DB/DV executor in pe C11 first live diagnostic and retained-scene recovery (2026-10-12) Tools 38253155969/1 passed inspector, nine storage regressions and fifteen manager regressions; exact source/tree/lock, collector bytes, binary digests and original results were independently verified against harness 891f264f39ffb608f1898fef151b273e205321b5. Diagnostic 38257435494/1 stopped in H1 old-metadata-rejoin: all eleven original versions/markers had explicit deletion ACKs, but DeleteBucket returned 500/InternalError with File access denied while one member was isolated. Its FAIL and retained namespace are preserved; no later branch ran and canonical PASS stays 101/158. Recovery accepts only this exact known refusal, a complete digest-bound request/ACK ledger, a restored member gate and the unchanged stopped candidate scene. It exports the retained system records and rechecks the physical manifest immediately before finalization. Unknown calls, missing ACKs, changed provenance, other bound namespaces, replay transactions and scene drift are rejected. Recovery never converts the product failure into PASS. + +C11 recovery 38259016213/1 completed with clean finalization. Independent audit compared both stopped physical censuses with the original failed artifact, checked the candidate binary, restored gate, retained system-record integrity and credential boundary, and verified the cleanup receipt. The driver's new merge commit preserves the Functional Chain and validation-tool workflow bytes exactly. The original DeleteBucket FAIL remains unchanged. + +F07 original stale-generation replay now captures an actual earlier product checkpoint before repair, proves later durable progress in the same task/incarnation/physical owner, crashes the exact owner and installs only the saved earlier bytes through the existing journal adapter. An independently stopped census verifies the injected checkpoint and unchanged resume companion. A valid record may replay safely; the oracle requires complete ACK accounting, original durable terminal/report, full physical repair and shard integrity before business reads, an independent unchanged rescan and every peer's original task/S3 view. Explicit rejection instead follows the retained-record failure/rescan path. Neither path claims the stale in-memory manager CAS or individual fsync barriers. Unknown fixture/admission acknowledgements retain the namespace. Cleanup never resurrects a completed checkpoint; original-byte evidence and complete repair proof precede dropping rescue registrations. F07 has an explicit diagnostic route, while its canonical matrix remains evidence-blocked because journal write-failure and complete native cut coverage are incomplete. A clean recovery cannot hide an implemented branch's FAIL. + +The final stale-generation change passed 608 Heal contracts with eight platform skips, 91 focused persistence/checkpoint/journal/manager/environment/migration checks, the unchanged 84/301/532 matrix, Python AST, shell syntax and whitespace. Two sequential final reviews retained exact bytes, task and owner bindings, rejected unknown calls and delayed rescue removal until complete evidence export. No unresolved finding remains for this executor diff; no live F07 PASS is claimed. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 54376a89..cbd367ff 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.bucket-incarnation-recreation|heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-markers-and-batch-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update|heal.mixed-version-upgrade-rollback|heal.dedicated-host-forced-restart|heal.race-ilm-expiry-and-transition|heal.race-replication-resync|heal.with-expansion-rebalance|heal.with-rebalance-stop-resume|heal.with-decommission|heal.with-decommission-cancel-clear|heal.after-retired-pool-removal|heal.ec84-multi-set-restart|heal.slow-io-and-eio|heal.kms-and-tier-outage|heal.long-soak|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) + heal.checkpoint-and-mrf-corruption|heal.bucket-incarnation-recreation|heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-markers-and-batch-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update|heal.mixed-version-upgrade-rollback|heal.dedicated-host-forced-restart|heal.race-ilm-expiry-and-transition|heal.race-replication-resync|heal.with-expansion-rebalance|heal.with-rebalance-stop-resume|heal.with-decommission|heal.with-decommission-cancel-clear|heal.after-retired-pool-removal|heal.ec84-multi-set-restart|heal.slow-io-and-eio|heal.kms-and-tier-outage|heal.long-soak|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py index 3f4016da..7c47513d 100644 --- a/scripts/validation_heal/execution_policy.py +++ b/scripts/validation_heal/execution_policy.py @@ -134,7 +134,7 @@ def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, . ), "heal.checkpoint-and-mrf-corruption": _native( "F07", "EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION", - "已实现 H1/H2 checkpoint 截断、未来 schema、ACK 绑定的 committed MRF CRC 故障和独立安全重扫；10 项原生回归仅为前置证据。仍缺陈旧代次、journal 写失败的完整现场流程及所需 fsync/commit 临界点审核，已实现分支尚无完整 live 审计。", + "已实现 H1/H2 checkpoint 截断、未来 schema、真实较早代次回放、ACK 绑定的 committed MRF CRC 故障和独立安全重扫；10 项原生回归仅为前置证据。仍缺 journal 写失败的完整现场流程及所需 fsync/commit 临界点审核，已实现分支尚无完整 live 审计。", ("durable_checkpoint_mrf_fault", "write_failure_retention", "publication_boundary_evidence_binding", "complete_live_orchestrator"), ), "heal.lock-timeout-and-caller-abort": _native( diff --git a/scripts/validation_heal/persistence.py b/scripts/validation_heal/persistence.py index c2c272bc..eb86a053 100644 --- a/scripts/validation_heal/persistence.py +++ b/scripts/validation_heal/persistence.py @@ -45,9 +45,9 @@ CASE_ID = "heal.checkpoint-and-mrf-corruption" BRANCHES = ("checkpoint-truncate", "mrf-checksum", "future-schema", "stale-generation", "journal-write-failure", "native-persist-fence") -IMPLEMENTED = {"checkpoint-truncate": "truncate", "future-schema": "future-schema"} +IMPLEMENTED = {"checkpoint-truncate": "truncate", "future-schema": "future-schema", + "stale-generation": "stale-generation"} BLOCKERS = { - "stale-generation": "an actual earlier checkpoint capture exists; complete rollback and durable responsibility reconciliation is still required", "journal-write-failure": "the private bounded volume exists; actual checkpoint write refusal, retained ownership and release/rescan orchestration is still required", } SIZE = 5 * 1024**2 + 123 @@ -95,7 +95,7 @@ def failed_task(current): "checkpoint load failure invented object processing or repair") -def retained_corruption(observed, roots, frozen, receipt, current): +def retained_corruption(observed, roots, frozen, receipt, current, *, earlier=None): token = frozen["taskId"] failed_task(current) require(receipt.get("applied") is True and receipt.get("originalBackupRetained") is True @@ -105,9 +105,11 @@ def retained_corruption(observed, roots, frozen, receipt, current): and receipt.get("relativePath") == frozen["checkpoint"]["relativePath"] and journal.same_json(receipt.get("before"), journal.entry(checkpoints.bytes_of(frozen["checkpoint"]))), "journal injection receipt changed the frozen original task or bytes") + if receipt.get("kind") == "stale-generation": stale_receipt(earlier, frozen, receipt) require(receipt.get("kind") in IMPLEMENTED.values() and receipt.get("injected") == journal.entry(journal.transform( - checkpoints.bytes_of(frozen["checkpoint"]), receipt, {"bucket": frozen["bucket"]}, None)), + checkpoints.bytes_of(frozen["checkpoint"]), receipt, {"bucket": frozen["bucket"]}, None, + earlier=checkpoints.bytes_of(earlier["checkpoint"]) if earlier is not None else None)), "journal corruption differs from its reviewed transformation") records = exact_records(observed, token, "checkpoint") names = journal.names("checkpoint", token) @@ -180,6 +182,101 @@ def rescan_outcome(current, bucket, expected, repairs, incarnation): "boundedDiagnosticsAreCompleteManifest": len(wanted) <= 128, "originalFailedTaskReclassified": False} +def stale_receipt(earlier, frozen, receipt): + require(isinstance(earlier, dict), "stale rollback has no original earlier generation") + checkpoints.after_generation(frozen, earlier) + require(earlier["value"]["admin"]["outcome"]["counters"]["healed"] == 0, + "NOT_OBSERVED: earlier capture is after the fixture's first actual repair") + require(receipt.get("kind") == "stale-generation" and receipt.get("family") == "checkpoint" + and receipt.get("taskId") == frozen["taskId"] and type(receipt.get("disk")) is int + and receipt["disk"] == frozen["disk"] + and receipt.get("relativePath") == frozen["checkpoint"]["relativePath"] + and receipt.get("captureId") == earlier["captureId"] + and journal.same_json(receipt.get("before"), journal.entry(checkpoints.bytes_of(frozen["checkpoint"]))) + and journal.same_json(receipt.get("injected"), journal.entry(checkpoints.bytes_of(earlier["checkpoint"]))) + and journal.same_json(receipt.get("capturedGeneration"), receipt["injected"]) + and receipt.get("applied") is True and receipt.get("originalBackupRetained") is True + and receipt.get("productRepairClaimed") is False, + "stale injection changed the original task, physical owner or earlier product bytes") + return {"originalTaskId": frozen["taskId"], "exactEarlierProductBytesInstalled": True, + "oldCheckpointManagerCasClaimed": False, "staleRecordRejectionClaimed": False} + + +def completed_original(roots, frozen, current): + token, owner = frozen["taskId"], frozen["rootAdmission"] + records = exact_records(roots, token, "root") + names = journal.names("root", token) + require(set(records) == set(names[1:]) and all((row["node"], row["disk"]) == + (owner["node"], owner["disk"]) for row in records.values()), + "original replay lost its terminal/report owner or retained runnable responsibility") + terminal = journal.decode_json(checkpoints.bytes_of(records[names[1]])) + report = journal.decode_json(checkpoints.bytes_of(records[names[2]])) + require(type(terminal.get("schema")) is int and terminal["schema"] == 1 + and terminal.get("task_id") == token and terminal.get("status") == "Completed" + and journal.same_json(terminal.get("heal_type"), owner["value"]["heal_type"]) + and journal.same_json(terminal.get("options"), owner["value"]["options"]) + and type(report.get("schema")) is int and report["schema"] == 1 + and journal.same_json(report.get("terminal"), terminal) + and journal.same_json(report.get("outcome"), current.get("outcome")), + "original replay silently retired work or changed its admitted scope/options/outcome") + return {"sameOriginalTaskAndPhysicalOwner": True, "durableCompletionMatchesFullManifest": True, + "originalResponsibilityRecreated": False, "backupRescueUsed": False} + + +def successful_stale_replay(fleet, probe, binding, expected, baseline, original, targets, + frozen, earlier, receipt, fault_scene, current, active): + token, bucket = frozen["taskId"], frozen["bucket"] + stale_receipt(earlier, frozen, receipt) + indexed = files(fault_scene) + missing = [target for target in targets if (target["node"], target["disk"], target["relativePath"]) not in indexed] + require(missing, "NOT_OBSERVED: rollback had no remaining actual object repair responsibility") + proof = rescan_outcome(current, bucket, expected, {version_identity(target) for target in missing}, frozen["bucketIncarnationId"]) + proof.pop("originalFailedTaskReclassified") + active.remove((checkpoints.PATH, token)) + idle_fleet(fleet, binding) + fleet.stop() + roots = checkpoints.records(fleet, token, "completed-original-stale-replay", quiescent=True, family="root") + durable = completed_original(roots, frozen, current) + healed = fleet.snapshot("original-replay-physical-before-any-rescan-or-GET", quiescent=True) + fleet.record("complete-original-replay-physical-proof", validate_repaired_physical(original, healed, targets[0], repair_targets=targets)) + for target in targets: verify_proof(fleet, target, "original-replay-full-cohort-integrity") + fleet.start() + admissions = [] + rescanned = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, + option_overrides={"recursive": True}, admissions=admissions) + require(len(admissions) == 1 and admissions[0]["clientToken"] != token, + "independent stale-record rescan reused the original responsibility") + rescan = rescan_outcome(rescanned, bucket, expected, set(), frozen["bucketIncarnationId"]) + repeated = recovery_status(probe, fleet, token) + require(all(canonical(repeated.get(name)) == canonical(current.get(name)) for name in ("summary", "settings", "outcome", "truncated")), + "independent rescan rewrote the original replay result") + for endpoint in binding["endpoints"]: + peer = Probe(binding, endpoint) + same = status(peer, fleet, checkpoints.PATH, token, "same-original-stale-replay-peer") + require(all(canonical(same.get(name)) == canonical(current.get(name)) for name in ("summary", "settings", "outcome", "truncated")), + "original stale replay token did not converge on every peer") + inventory = fleet.record("complete-peer-S3", {"endpoint": endpoint, **peer.inventory(bucket, + unversioned_current_tags=expected[0]["versionId"] == "null")}) + verify_seed(expected, inventory); compare_inventories(baseline, inventory) + fleet.stop() + require(physical_content(healed) == physical_content(fleet.snapshot("physical-after-rescan-and-business-reads", quiescent=True)), + "rescan or business reads repaired work missing from the original replay proof") + retained = checkpoints.records(fleet, token, "retained-original-stale-terminal-after-rescan", quiescent=True, family="root") + completed_original(retained, frozen, repeated) + require(canonical(exact_records(retained, token, "root")) == canonical(exact_records(roots, token, "root")), + "rescan or business reads changed original durable completion bytes") + fleet.record("complete-original-generation-reconciliation", {"earlier": earlier, "later": frozen, + "injection": receipt, "originalCompletion": proof, "durableCompletion": durable, "separateRescan": rescan, + "oldRecordResurrectionUsed": False, "canonicalCasePassClaimed": False}) + # A completed product task may remove its checkpoint. Exact original-byte + # evidence and full repair proof precede discarding rescue registrations. + fleet.journal_faults.clear() + fleet.faults.clear() + return {"verdict": "PASS", "originalGenerationReconciliation": proof, "durableOriginalCompletion": durable, + "separateRescan": rescan, "staleRecordRejectionClaimed": False, + "completePhysicalAndS3ManifestsVerified": True, "canonicalCasePassClaimed": False} + + def run_checkpoint_fault(args, topology, nodes, fixture, branch, candidate): require(branch in IMPLEMENTED, "unimplemented journal branch cannot mutate the fleet") directory = args.evidence / fixture / topology / branch @@ -200,14 +297,16 @@ def run_checkpoint_fault(args, topology, nodes, fixture, branch, candidate): "maxInventoryObjects": 512, "maxInventoryBytes": 2 * 1024**3} fleet.record("binding", binding) fleet.record("candidate-bound-inspector", verify_inspector(args.tool_directory, candidate)) - probe, active = Probe(binding), [] + probe, active, pending = Probe(binding), [], None result = {"topology": topology, "fixture": fixture, "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} try: fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) + pending = {"operation": "seed-original-checkpoint-fixture"} expected = seed(Recorder(probe, directory / "fixture"), bucket, fixture, count, [SIZE]) baseline = fleet.record("ACK-bound-complete-S3", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) fleet.record("complete-ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + pending = None idle_fleet(fleet, binding) fleet.stop() original = fleet.snapshot("healthy-physical-baseline", quiescent=True) @@ -227,14 +326,27 @@ def run_checkpoint_fault(args, topology, nodes, fixture, branch, candidate): fleet.start() require(physical_content(prior) == physical_content(fleet.snapshot("before-original-admission")), "ATTRIBUTION_NOT_OBSERVED: background activity changed the original shard faults") + pending = {"operation": "admit-original-checkpoint-task"} token = checkpoints.admit(probe, fleet, active) - frozen, identity = checkpoints.freeze(fleet, token, bucket) + pending = None + earlier = checkpoints.capture_earlier(fleet, token, bucket, "F07-earlier-generation") if branch == "stale-generation" else None + if earlier is not None: + require(earlier["value"]["admin"]["outcome"]["counters"]["healed"] == 0, + "NOT_OBSERVED: earlier generation missed the pre-repair capture window") + frozen, identity = checkpoints.freeze(fleet, token, bucket, earlier=earlier) frozen = checkpoints.stop_for_fault(fleet, token, bucket, frozen, identity) params = {"family": "checkpoint", "taskId": token, "disk": frozen["disk"], "relativePath": frozen["checkpoint"]["relativePath"], "faultId": "F07-" + branch, "kind": IMPLEMENTED[branch], "expectedBefore": journal.entry(checkpoints.bytes_of(frozen["checkpoint"]))} + if earlier is not None: params["captureId"] = earlier["captureId"] fleet.journal_faults.append((frozen["node"], params)) receipt = fleet.record("stopped-original-checkpoint-injection", fleet.remote(frozen["node"], "journal-apply", **params)) + if earlier is not None: + fleet.record("original-earlier-generation-injection-proof", stale_receipt(earlier, frozen, receipt)) + injected = checkpoints.owned_checkpoint(checkpoints.records(fleet, token, "independent-injected-original-generation", quiescent=True), token, bucket) + require(checkpoints.bytes_of(injected["checkpoint"]) == checkpoints.bytes_of(earlier["checkpoint"]) + and injected["resume"] == frozen["resume"], "stale rollback changed its original companion or actual bytes") + fault_scene = fleet.snapshot("physical-at-original-generation-replay-boundary", quiescent=True) fleet.start() deadline = time.monotonic() + 180 while True: @@ -242,13 +354,17 @@ def run_checkpoint_fault(args, topology, nodes, fixture, branch, candidate): if failed.get("summary") != "running": break require(time.monotonic() < deadline, "original corrupt checkpoint task did not produce a bounded diagnostic") time.sleep(.2) + if earlier is not None and failed.get("summary") == "finished": + result.update(successful_stale_replay(fleet, probe, binding, expected, baseline, original, targets, + frozen, earlier, receipt, fault_scene, failed, active)) + return result failed_task(failed) active.remove((checkpoints.PATH, token)) idle_fleet(fleet, binding) fleet.stop() corrupt_scene = checkpoints.records(fleet, token, "original-corrupt-checkpoint", quiescent=True) failure_roots = checkpoints.records(fleet, token, "original-durable-failure", quiescent=True, family="root") - result["originalFailure"] = retained_corruption(corrupt_scene, failure_roots, frozen, receipt, failed) + result["originalFailure"] = retained_corruption(corrupt_scene, failure_roots, frozen, receipt, failed, earlier=earlier) before_rescan = fleet.snapshot("failure-scene-before-any-rescan", quiescent=True) indexed = files(before_rescan) remaining = [target for target in targets if (target["node"], target["disk"], target["relativePath"]) not in indexed] @@ -293,7 +409,7 @@ def run_checkpoint_fault(args, topology, nodes, fixture, branch, candidate): "business reads repaired data that the independent rescan had not proved") retained_checkpoint = checkpoints.records(fleet, token, "retained-corrupt-checkpoint-after-rescan", quiescent=True) retained_roots = checkpoints.records(fleet, token, "retained-original-failure-after-rescan", quiescent=True, family="root") - retained_corruption(retained_checkpoint, retained_roots, frozen, receipt, repeated) + retained_corruption(retained_checkpoint, retained_roots, frozen, receipt, repeated, earlier=earlier) require(canonical(exact_records(retained_checkpoint, token, "checkpoint")) == canonical(exact_records(corrupt_scene, token, "checkpoint")) and canonical(exact_records(retained_roots, token, "root")) == canonical(exact_records(failure_roots, token, "root")), "separate rescan or business reads changed the original corrupt record or durable failure bytes") @@ -306,7 +422,9 @@ def run_checkpoint_fault(args, topology, nodes, fixture, branch, candidate): fleet.snapshot("original-failure-scene", quiescent=True) except Exception as error: result["failureEvidenceError"] = str(error) finally: - result["cleanup"] = "PASS" if fleet.cleanup() else "FAIL" + clean = fleet.cleanup(preserve_namespaces=pending is not None) + result["cleanup"] = "PASS" if clean and pending is None else "FAIL" + if pending is not None: result["unresolvedMutation"] = pending fleet.record("checkpoint-branch-result", result) return result @@ -356,7 +474,10 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"F07 interrupted by sig finally: save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "complete": False, "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL", - "canonicalCasePassClaimed": False, "reason": "stale-generation, journal-write-failure and full native cut coverage remain blocked"}) + "diagnosticBranchesPassed": len(results) == 24 and all(row["cleanup"] == "PASS" and + row["verdict"] == ("BLOCKED" if row["branch"] in BLOCKERS else + "PREREQUISITE_PASS" if row["branch"] == "native-persist-fence" else "PASS") for row in results), + "canonicalCasePassClaimed": False, "reason": "journal-write-failure and full native cut coverage remain blocked"}) return 1 diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index 504761c2..b4c6883e 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -2155,7 +2155,7 @@ "dispatchable": false, "code": "F07", "reasonCode": "EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION", - "reason": "已实现 H1/H2 checkpoint 截断、未来 schema、ACK 绑定的 committed MRF CRC 故障和独立安全重扫；10 项原生回归仅为前置证据。仍缺陈旧代次、journal 写失败的完整现场流程及所需 fsync/commit 临界点审核，已实现分支尚无完整 live 审计。", + "reason": "已实现 H1/H2 checkpoint 截断、未来 schema、真实较早代次回放、ACK 绑定的 committed MRF CRC 故障和独立安全重扫；10 项原生回归仅为前置证据。仍缺 journal 写失败的完整现场流程及所需 fsync/commit 临界点审核，已实现分支尚无完整 live 审计。", "requiredCapabilities": [ "durable_checkpoint_mrf_fault", "write_failure_retention", diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index deb53e51..c2f6b533 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -21,6 +21,7 @@ done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization) ;; heal.bucket-incarnation-recreation) driver=incarnation.py ;; + heal.checkpoint-and-mrf-corruption) driver=persistence.py ;; heal.node-outage-rejoin) driver=node_outage.py ;; heal.status-partial-disabled-unknown) driver=status.py ;; heal.incremental-cursor-and-truncation) driver=cursor.py ;; @@ -191,6 +192,11 @@ if [ "${case_id}" = heal.bucket-incarnation-recreation ] && [ "${cleanup_status} "${evidence}/healthy-summary.json" >/dev/null 2>&1; then verdict=BLOCKED; reason="C11 diagnostic execution remains outside canonical PASS until every H1/H2 branch is independently audited" fi +if [ "${case_id}" = heal.checkpoint-and-mrf-corruption ] && [ "${cleanup_status}" = PASS ] && \ + jq -e '.diagnosticBranchesPassed == true and .complete == false and .canonicalCasePassClaimed == false' \ + "${evidence}/healthy-summary.json" >/dev/null 2>&1; then + verdict=BLOCKED; reason="F07 diagnostic execution lacks journal write-failure and full native cut coverage; partial success cannot certify the canonical case" +fi args=("${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool --result-file "${artifacts}/cases/${case_id}.json" --verdict "${verdict}" --cleanup-status "${cleanup_status}" --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --binary-sha256 "${binary_sha}" diff --git a/test_validation_heal_persistence.py b/test_validation_heal_persistence.py index 7f25bda1..f91ed47d 100644 --- a/test_validation_heal_persistence.py +++ b/test_validation_heal_persistence.py @@ -144,12 +144,13 @@ def passed(args, topology, nodes, fixture, branch, candidate): patch.object(persistence, "run_checkpoint_fault", side_effect=passed) as mutate: self.assertEqual(persistence.main(), 1) verify.assert_called_once() - self.assertEqual(mutate.call_count, 8) + self.assertEqual(mutate.call_count, 12) self.assertEqual(mrf_mutate.call_count, 4) summary = json.loads((root / "healthy-summary.json").read_text()) self.assertEqual(len(summary["results"]), 24) self.assertFalse(summary["complete"]) self.assertFalse(summary["canonicalCasePassClaimed"]) + self.assertTrue(summary["diagnosticBranchesPassed"]) prerequisites = [row for row in summary["results"] if row["branch"] == "native-persist-fence"] self.assertEqual(len(prerequisites), 4) for row in prerequisites: @@ -160,9 +161,140 @@ def passed(args, topology, nodes, fixture, branch, candidate): self.assertFalse(row["physicalPowerLossClaimed"]) self.assertFalse(row["requiredNativeCoverageComplete"]) self.assertEqual(row["nativeProof"], proof) - self.assertEqual(sum(row["verdict"] == "BLOCKED" for row in summary["results"]), 8) + self.assertEqual(sum(row["verdict"] == "BLOCKED" for row in summary["results"]), 4) + def failed(*values): + row = passed(*values) + if row["branch"] == "stale-generation": row["verdict"] = "FAIL" + return row + failed_directory = root / "failed-diagnostic"; failed_directory.mkdir() + failed_args = args.copy(); failed_args[failed_args.index("--evidence") + 1] = str(failed_directory) + with patch("sys.argv", failed_args), patch.dict(os.environ, {"GITHUB_RUN_ID": "12345"}), \ + patch.object(persistence, "verify_manager", return_value=proof), \ + patch.object(persistence, "run_mrf_checksum", side_effect=lambda args, topology, nodes, fixture, candidate: { + "topology": topology, "fixture": fixture, "branch": "mrf-checksum", "verdict": "PASS", "cleanup": "PASS"}), \ + patch.object(persistence, "run_checkpoint_fault", side_effect=failed): + self.assertEqual(persistence.main(), 1) + failed_summary = json.loads((failed_directory / "healthy-summary.json").read_text()) + self.assertEqual(failed_summary["cleanup"], "PASS") + self.assertFalse(failed_summary["diagnosticBranchesPassed"]) + self.assertFalse(failed_summary["canonicalCasePassClaimed"]) + + def stale(self): + observed, value, _ = fixture_source.AdminCheckpointTests().fixture() + value["admin"]["outcome"]["counters"].update(healed=0, unchanged=2) + observed["node"]["records"][4].update(encoded(value, signed=True)) + early = checkpoint.owned_checkpoint(observed, TOKEN, BUCKET) + early["captureId"] = "F07-earlier-generation" + value["checkpoint_time"] += 1; value["current_object_index"] += 1 + value["admin"]["outcome"]["counters"]["processed"] += 1 + value["admin"]["outcome"]["counters"]["unchanged"] += 1 + observed["node"]["records"][4].update(encoded(value, signed=True)) + later = checkpoint.owned_checkpoint(observed, TOKEN, BUCKET) + later["rootAdmission"] = self.fixture()[2]["rootAdmission"] + receipt = {"family": "checkpoint", "taskId": TOKEN, "disk": 1, + "relativePath": later["checkpoint"]["relativePath"], "kind": "stale-generation", + "captureId": early["captureId"], "before": journal.entry(checkpoint.bytes_of(later["checkpoint"])), + "injected": journal.entry(checkpoint.bytes_of(early["checkpoint"])), + "capturedGeneration": journal.entry(checkpoint.bytes_of(early["checkpoint"])), + "applied": True, "originalBackupRetained": True, "productRepairClaimed": False} + return early, later, receipt + + def completion_roots(self, frozen, current): + roots = self.fixture()[1] + terminal = {"schema": 1, "task_id": TOKEN, "heal_type": frozen["rootAdmission"]["value"]["heal_type"], + "options": frozen["rootAdmission"]["value"]["options"], "status": "Completed"} + roots["node"]["records"][4].update(encoded(terminal)) + roots["node"]["records"][5].update(encoded({"schema": 1, "terminal": terminal, "outcome": current["outcome"]})) + return roots - def interrupted_injection(self, cleanup): + def test_stale_valid_generation_is_exact_original_progress_without_cas_or_rejection_claim(self): + proof = persistence.stale_receipt(*self.stale()) + self.assertTrue(proof["exactEarlierProductBytesInstalled"]) + self.assertFalse(proof["oldCheckpointManagerCasClaimed"]) + self.assertFalse(proof["staleRecordRejectionClaimed"]) + for change in ("same-progress", "owner", "capture", "injected", "backup", "already-healed"): + early, later, receipt = self.stale() + if change == "same-progress": later = copy.deepcopy(early) + elif change == "owner": later["disk"] = 2 + elif change == "capture": receipt["captureId"] = "F07-other" + elif change == "injected": receipt["injected"] = receipt["before"] + elif change == "backup": receipt["originalBackupRetained"] = False + else: early["value"]["admin"]["outcome"]["counters"].update(healed=1, unchanged=1) + with self.subTest(change=change), self.assertRaises(ValueError): persistence.stale_receipt(early, later, receipt) + + def test_original_completion_needs_exact_admission_owner_scope_and_report(self): + _, frozen, _ = self.stale(); current = self.rescan()[0] + proof = persistence.completed_original(self.completion_roots(frozen, current), frozen, current) + self.assertTrue(proof["sameOriginalTaskAndPhysicalOwner"]) + for change in ("runnable", "missing", "moved", "scope", "different-task", "different-report"): + roots = self.completion_roots(frozen, current); rows = roots["node"]["records"] + if change == "runnable": rows[3].update(encoded(frozen["rootAdmission"]["value"])) + elif change == "missing": rows[4] = {key: rows[4][key] for key in ("disk", "relativePath")}; rows[4]["present"] = False + elif change == "moved": rows[1] = {**rows[4], "disk": 0}; rows[4] = {key: rows[4][key] for key in ("disk", "relativePath")}; rows[4]["present"] = False + elif change == "different-report": rows[5].update(encoded({"schema": 1, "terminal": {}, "outcome": current["outcome"]})) + else: + value = journal.decode_json(checkpoint.bytes_of(rows[4])) + if change == "scope": value["heal_type"] = {"type": "cluster"} + else: value["task_id"] = INCARNATION + rows[4].update(encoded(value)) + with self.subTest(change=change), self.assertRaises(ValueError): persistence.completed_original(roots, frozen, current) + + def test_rejected_stale_generation_retains_original_bytes_and_failed_responsibility(self): + early, frozen, receipt = self.stale() + observed, roots, _, _, current = self.fixture() + observed["node"]["records"][4] = copy.deepcopy(early["checkpoint"]) + proof = persistence.retained_corruption(observed, roots, frozen, receipt, current, earlier=early) + self.assertTrue(proof["durableOriginalFailureRetained"]) + self.assertFalse(proof["originalTaskRepairClaimed"]) + self.assertFalse(proof["backupRescueUsed"]) + with self.assertRaises(ValueError): persistence.retained_corruption(observed, roots, frozen, receipt, current) + + def test_successful_replay_cannot_discard_rescue_before_manifest_terminal_and_evidence_agree(self): + for change in (None, "read-repair", "late-archive-failure", "rescan-relabels-original"): + early, frozen, receipt = self.stale(); current, _, expected, _, _ = self.rescan() + roots = self.completion_roots(frozen, current) + repaired = {"node": {"files": [{"disk": 1, "path": "bound/part.1", "sha256": "a" * 64}], "formats": []}} + events, active = [], [(checkpoint.PATH, TOKEN)] + class ReplayFleet: + journal_faults, faults = [("node", {"faultId": "F07-stale-generation"})], [("node", {"disk": 1})] + def start(self): pass + def stop(self): pass + def record(self, label, value): + events.append(label) + if change == "late-archive-failure" and label == "complete-original-generation-reconciliation": raise OSError("evidence write failed") + return value + def snapshot(self, label, **kwargs): + scene = copy.deepcopy(repaired) + if change == "read-repair" and label == "physical-after-rescan-and-business-reads": scene["node"]["files"] = [] + return scene + fleet = ReplayFleet() + rescan = copy.deepcopy(current); rescan["outcome"]["counters"].update(healed=0, unchanged=257) + def run_task(*args, **kwargs): + kwargs["admissions"].append({"clientToken": TOKEN if change == "rescan-relabels-original" else INCARNATION}) + return rescan + target = {"node": "node", "disk": 1, "relativePath": "bound/part.1", + "key": expected[-1]["key"], "versionId": "null"} + binding = {"endpoints": {"node1": "http://owned"}} + with patch.object(persistence, "rescan_outcome", wraps=persistence.rescan_outcome), \ + patch.object(persistence.checkpoints, "records", return_value=roots), patch.object(persistence, "idle_fleet"), \ + patch.object(persistence, "validate_repaired_physical"), patch.object(persistence, "verify_proof"), \ + patch.object(persistence, "run_task", side_effect=run_task), \ + patch.object(persistence, "recovery_status", return_value=current), patch.object(persistence, "status", return_value=current), \ + patch.object(persistence, "Probe", return_value=SimpleNamespace(inventory=lambda *a, **kw: {})), \ + patch.object(persistence, "verify_seed"), patch.object(persistence, "compare_inventories"): + args = (fleet, None, binding, expected, {}, repaired, [target], frozen, early, receipt, + {"node": {"files": [], "formats": []}}, current, active) + if change is None: + result = persistence.successful_stale_replay(*args) + self.assertFalse(result["canonicalCasePassClaimed"]) + self.assertFalse(result["staleRecordRejectionClaimed"]) + self.assertIn("complete-original-generation-reconciliation", events) + self.assertFalse(fleet.journal_faults); self.assertFalse(fleet.faults) + else: + with self.assertRaises((ValueError, OSError)): persistence.successful_stale_replay(*args) + self.assertTrue(fleet.journal_faults); self.assertTrue(fleet.faults) + + def interrupted_injection(self, cleanup, mutation=None): _, _, frozen, _, _ = self.fixture() snapshots = {"node": {"files": [], "formats": []}} captured = {} @@ -184,9 +316,10 @@ def remote(self, node, action, **params): raise OSError("journal injection acknowledgement lost") return {} - def cleanup(self): + def cleanup(self, *, preserve_namespaces=False): captured["cleanupFaults"] = copy.deepcopy(self.faults) captured["cleanupJournals"] = copy.deepcopy(self.journal_faults) + captured["preserveNamespaces"] = preserve_namespaces return cleanup class FixtureProbe: @@ -205,11 +338,13 @@ def inventory(self, *args, **kwargs): return {} patch.object(persistence, "Fleet", InterruptedFleet), patch.object(persistence, "Probe", FixtureProbe), \ patch.object(persistence, "verify_inspector", return_value={}), \ patch.object(persistence, "validate_topology", return_value={}), \ - patch.object(persistence, "seed", return_value=expected), \ + patch.object(persistence, "seed", return_value=expected, + side_effect=OSError("seed acknowledgement lost") if mutation == "seed" else None), \ patch.object(persistence, "verify_seed", return_value={}), \ patch.object(persistence, "idle_fleet"), patch.object(persistence, "map_target", return_value=target), \ patch.object(persistence, "verify_proof"), patch.object(persistence, "validate_fault"), \ - patch.object(persistence.checkpoints, "admit", return_value=TOKEN), \ + patch.object(persistence.checkpoints, "admit", return_value=TOKEN, + side_effect=OSError("admission acknowledgement lost") if mutation == "admission" else None), \ patch.object(persistence.checkpoints, "freeze", return_value=(frozen, {})), \ patch.object(persistence.checkpoints, "stop_for_fault", return_value=frozen): result = persistence.run_checkpoint_fault(args, "H1", ["node"], "DB", "checkpoint-truncate", @@ -232,5 +367,14 @@ def test_failed_cleanup_preserves_dirty_result_after_lost_injection_ack(self): self.assertEqual(len(captured["cleanupJournals"]), 1) self.assertNotIn("completePhysicalAndS3ManifestsVerified", result) + def test_unknown_fixture_or_original_admission_preserves_namespace_even_if_cleanup_reports_success(self): + for mutation in ("seed", "admission"): + with self.subTest(mutation=mutation): + result, captured = self.interrupted_injection(True, mutation) + self.assertEqual((result["verdict"], result["cleanup"]), ("FAIL", "FAIL")) + self.assertTrue(captured["preserveNamespaces"]) + self.assertIn("unresolvedMutation", result) + self.assertNotIn("separateRescan", result) + if __name__ == "__main__": unittest.main() From 15f9000d6ed25083c471dac0c87cf7e5c84afcec Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 22:46:28 +0800 Subject: [PATCH 187/343] test(validation): exercise retained checkpoint write failures --- docs/validation-heal-cases.md | 2 +- docs/validation-heal-environment-gates.md | 2 +- docs/validation-migration-map.md | 2 + scripts/validation_heal/execution_policy.py | 2 +- scripts/validation_heal/fleet_node.py | 14 +- scripts/validation_heal/persistence.py | 210 ++++++++++++++++++-- scripts/validation_heal/recipes.json | 2 +- scripts/validation_heal/run_case.sh | 2 +- test_validation_heal_capacity.py | 26 +++ test_validation_heal_persistence.py | 171 +++++++++++++++- 10 files changed, 407 insertions(+), 26 deletions(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index ca58a2af..6ed174b4 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1202,7 +1202,7 @@ Topologies: H1,H2. Branches: checkpoint-truncate, mrf-checksum, future-schema, s Execution policy: `evidence-blocked`; disabled by default: `false`. -Blocked reason (`EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION`): 已实现 H1/H2 checkpoint 截断、未来 schema、真实较早代次回放、ACK 绑定的 committed MRF CRC 故障和独立安全重扫；10 项原生回归仅为前置证据。仍缺 journal 写失败的完整现场流程及所需 fsync/commit 临界点审核，已实现分支尚无完整 live 审计。 +Blocked reason (`EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION`): 已实现 H1/H2 checkpoint 截断、未来 schema、真实较早代次回放、committed MRF CRC 故障、私有限额卷的真实 journal ENOSPC 和独立安全重扫；10 项原生回归仅为前置证据。仍缺全部所需 fsync/commit 临界点审核和完整现场分支的独立 live 审计，诊断执行不能提升 canonical 验收。 1. 先制造并证明真实 durable root/MRF 责任，CHECKPOINT 定位精确本 run task/record，其他系统记录禁止触碰。 2. 停止测试服务，按 FAULT_FORMAT 对这些已审核记录分别截断、坏校验、未来 schema、旧代次；写失败实例用独占限额故障层，不塞满真实共享盘。 diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index ffc5eb89..c43bfcaf 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -33,7 +33,7 @@ | Code | 用例 | Block 原因 | 解锁所需证据 | | --- | --- | --- | --- | | C11 | `heal.bucket-incarnation-recreation` | H1/H2 旧元数据盘重返、原始 checkpoint/resume 与 committed MRF 重放编排已实现；精确候选的 9 项存储原生回归和 3 项 manager incarnation 前置证据已独立核验。完整现场分支尚无独立审计结果 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | -| F07 | `heal.checkpoint-and-mrf-corruption` | 已实现 H1/H2 checkpoint 截断、未来 schema、committed MRF CRC 故障及独立安全重扫；10 项原生回归作为前置证据。真实较早代次回放已实现；仍缺写失败与完整 native cut 审核，完整 live 尚未审计 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | +| F07 | `heal.checkpoint-and-mrf-corruption` | 已实现 H1/H2 checkpoint 截断、未来 schema、committed MRF CRC 故障及独立安全重扫；10 项原生回归作为前置证据。真实较早代次回放与私有限额卷的 journal ENOSPC 已实现；仍缺完整 native cut 审核，完整 live 尚未审计 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | | I05 | `heal.lock-timeout-and-caller-abort` | 请求耗时/TCP 断开不能证明服务端锁 owner 与提交顺序 | lock owner、timeout、caller abort 和 stale read-repair 的服务端屏障 | | J02 | `heal.outcome-counters-and-error-truth` | 已有 2 项 outcome 原生回归；缺混合 healthy/repaired/failed/deferred/unknown 真机夹具、dry-run 独立任务与完整 manifest 对账 | 每个 disposition、计数和 attemptFailures 对账；不是已证实产品缺少 API | diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 70e1161a..e4625edd 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1959,3 +1959,5 @@ C11 recovery 38259016213/1 completed with clean finalization. Independent audit F07 original stale-generation replay now captures an actual earlier product checkpoint before repair, proves later durable progress in the same task/incarnation/physical owner, crashes the exact owner and installs only the saved earlier bytes through the existing journal adapter. An independently stopped census verifies the injected checkpoint and unchanged resume companion. A valid record may replay safely; the oracle requires complete ACK accounting, original durable terminal/report, full physical repair and shard integrity before business reads, an independent unchanged rescan and every peer's original task/S3 view. Explicit rejection instead follows the retained-record failure/rescan path. Neither path claims the stale in-memory manager CAS or individual fsync barriers. Unknown fixture/admission acknowledgements retain the namespace. Cleanup never resurrects a completed checkpoint; original-byte evidence and complete repair proof precede dropping rescue registrations. F07 has an explicit diagnostic route, while its canonical matrix remains evidence-blocked because journal write-failure and complete native cut coverage are incomplete. A clean recovery cannot hide an implemented branch's FAIL. The final stale-generation change passed 608 Heal contracts with eight platform skips, 91 focused persistence/checkpoint/journal/manager/environment/migration checks, the unchanged 84/301/532 matrix, Python AST, shell syntax and whitespace. Two sequential final reviews retained exact bytes, task and owner bindings, rejected unknown calls and delayed rescue removal until complete evidence export. No unresolved finding remains for this executor diff; no live F07 PASS is claimed. + +F07 original checkpoint write failure now has a diagnostic executor using the existing four-node `journal-bounded` profile. It fills only the four fixed private journal volumes while the candidate is stopped, then proves a candidate UID and mount namespace witness for the real `ENOSPC` write. The original checkpoint, resume bytes, root owner, exact failed task outcome and source-matched `Heal task failed` event remain durable. The executor pauses and records every owner before releasing only its private filler, performs a distinct Admin rescan with a new token, verifies complete physical, shard and peer S3 evidence, and preserves the first failed attempt separately from any later same-task retry. Cleanup keeps rescue state when a fill ACK, release, evidence export or reconciliation step is incomplete. The branch is implemented but remains outside canonical dispatch until a live run and the remaining reviewed native persistence cuts complete. The final full Heal suite passes 615 tests with eight platform skips; 21 focused persistence contracts, the catalog, 84/301/532 matrix, Python AST, shell syntax and whitespace pass. Two fresh sequential reviews cover correctness, simplicity, test coverage, path security and durability. Frozen process-bound censuses and decreasing root timeout budgets retain their original identity checks. Canonical PASS remains 101/158. diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py index 7c47513d..56dd9471 100644 --- a/scripts/validation_heal/execution_policy.py +++ b/scripts/validation_heal/execution_policy.py @@ -134,7 +134,7 @@ def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, . ), "heal.checkpoint-and-mrf-corruption": _native( "F07", "EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION", - "已实现 H1/H2 checkpoint 截断、未来 schema、真实较早代次回放、ACK 绑定的 committed MRF CRC 故障和独立安全重扫；10 项原生回归仅为前置证据。仍缺 journal 写失败的完整现场流程及所需 fsync/commit 临界点审核，已实现分支尚无完整 live 审计。", + "已实现 H1/H2 checkpoint 截断、未来 schema、真实较早代次回放、committed MRF CRC 故障、私有限额卷的真实 journal ENOSPC 和独立安全重扫；10 项原生回归仅为前置证据。仍缺全部所需 fsync/commit 临界点审核和完整现场分支的独立 live 审计，诊断执行不能提升 canonical 验收。", ("durable_checkpoint_mrf_fault", "write_failure_retention", "publication_boundary_evidence_binding", "complete_live_orchestrator"), ), "heal.lock-timeout-and-caller-abort": _native( diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 3a3d29ba..e62ca516 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -57,8 +57,10 @@ def runtime_config(profile): if profile == "multi-pool": return {**BACKGROUND_CONFIG, "RUSTFS_ERASURE_SET_DRIVE_COUNT": "4", "RUSTFS_STORAGE_CLASS_STANDARD": "EC:2"} if profile in {"bounded-capacity", "journal-bounded"}: + logging_filter = "info,rustfs::heal::manager=debug,rustfs::heal::task=debug,rustfs::heal::resume=debug" return {**BACKGROUND_CONFIG, "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true", - **({"RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false", "RUSTFS_OBS_LOGGER_LEVEL": "debug"} + **({"RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false", "RUSTFS_OBS_LOGGER_LEVEL": logging_filter, + "RUST_LOG": logging_filter} if profile == "journal-bounded" else {})} if profile == "lock-quorum": return {**BACKGROUND_CONFIG, "RUSTFS_STORAGE_CLASS_STANDARD": "EC:8"} @@ -632,6 +634,16 @@ def volume_operation(request): changed = [module.release_filler(view) for view in module.views(state, roots, owner)] return {"changes": changed, "ownedFillerReleased": True, "productRepairClaimed": False, "quiescent": True, "privateMountRemovedByStoppedService": True} + if request["action"] == "bounded-volumes-fill" and request.get("quiescent") is True: + require(set(request) == {"runId", "action", "quiescent"} + and owner.get("runtimeProfile") == "journal-bounded", + "stopped exhaustion requires the dedicated journal runtime") + stopped(owner) + require(module.read(state, roots, owner)["branch"] == "bounded-volume-enospc", + "journal exhaustion cannot use another resource fault") + changed = [module.fill(view, "bounded-volume-enospc") for view in module.views(state, roots, owner)] + return {"changes": changed, "quiescent": True, "hostSharedMountRemounted": False, + "candidateProcessWitnessRequired": True, "productRepairClaimed": False} require(set(request) == {"runId", "action", "expected"}, "bounded operation lacks exact process binding") before = process(request) require(before["pid"] > 1 and isinstance(request["expected"], dict) diff --git a/scripts/validation_heal/persistence.py b/scripts/validation_heal/persistence.py index eb86a053..1b3867b2 100644 --- a/scripts/validation_heal/persistence.py +++ b/scripts/validation_heal/persistence.py @@ -24,7 +24,9 @@ from .matrix import select from .shards import files, map_target, run_task, validate_fault, validate_repaired_physical from .stop import idle_fleet, status - from .root_restart import recovery_status + from .root_restart import recovery_status, same_owner + from .scanner_repair import JournalWindow + from .fleet_node import PROCESS_FIELDS from .persistence_mrf import run as run_mrf_checksum from .versions import public_version_id, verify_proof, version_identity except ImportError: @@ -37,7 +39,9 @@ from matrix import select from shards import files, map_target, run_task, validate_fault, validate_repaired_physical from stop import idle_fleet, status - from root_restart import recovery_status + from root_restart import recovery_status, same_owner + from scanner_repair import JournalWindow + from fleet_node import PROCESS_FIELDS from persistence_mrf import run as run_mrf_checksum from versions import public_version_id, verify_proof, version_identity @@ -47,19 +51,19 @@ "journal-write-failure", "native-persist-fence") IMPLEMENTED = {"checkpoint-truncate": "truncate", "future-schema": "future-schema", "stale-generation": "stale-generation"} -BLOCKERS = { - "journal-write-failure": "the private bounded volume exists; actual checkpoint write refusal, retained ownership and release/rescan orchestration is still required", -} +BLOCKERS = {} SIZE = 5 * 1024**2 + 123 -def exact_records(observed, token, family): +def exact_records(observed, token, family, *, allow_frozen=False): names = journal.names(family, token) require(isinstance(observed, dict) and len(observed) in {1, 4}, "journal omitted bound physical nodes") present = {} for node, census in observed.items(): rows = census.get("records") - require(census.get("readOnly") is True and census.get("quiescent") is True + require(census.get("readOnly") is True and (census.get("quiescent") is True + or allow_frozen and census.get("stable") is True and census.get("serviceFrozen") is True + and census.get("processBound") is True and census.get("independentReads") == 2) and census.get("family") == family and census.get("taskId") == token and census.get("directoryEnumerationUsed") is False and isinstance(rows, list) and len(rows) == 4 * len(names), @@ -277,17 +281,186 @@ def successful_stale_replay(fleet, probe, binding, expected, baseline, original, "completePhysicalAndS3ManifestsVerified": True, "canonicalCasePassClaimed": False} +def retained_write_failure(observed, roots, frozen, current, events): + token = frozen["taskId"] + outcome = current.get("outcome") or {} + detail = current.get("detail", "") + require(current.get("summary") == "stopped" and current.get("settings") == checkpoints.OPTIONS + and outcome.get("execution") == {"state": "aborted", "reason": "untraversable"} + and outcome.get("coverage") == "partial" and isinstance(detail, str) + and "failed to save checkpoint" in detail.lower() + and any(word in detail.lower() for word in ("no space", "enospc", "os error 28")), + "journal write refusal was not an actual checkpoint ENOSPC diagnostic") + expected = frozen["value"]["admin"]["outcome"] + require(all(canonical(outcome.get(name)) == canonical(expected.get(name)) + for name in ("counters", "objects", "objectsTruncated")), + "failed checkpoint write lost or invented acknowledged durable progress") + owner = checkpoints.owned_checkpoint(observed, token, frozen["bucket"]) + require(owner["checkpoint"] == frozen["checkpoint"] and owner["resume"] == frozen["resume"], + "journal failure removed or rewrote the last original durable checkpoint/resume") + root = exact_records(roots, token, "root", allow_frozen=True) + names, original = journal.names("root", token), frozen["rootAdmission"] + require(set(root) in ({names[0]}, set(names[1:]), set(names)) + and all((row["node"], row["disk"]) == (original["node"], original["disk"]) for row in root.values()), + "journal failure silently retired or changed its original durable owner") + if names[0] in root: + raw = checkpoints.bytes_of(root[names[0]]) + if raw != checkpoints.bytes_of(original): + # checkpoint_failed_execution may consume the original timeout; + # every other admitted field and the physical owner stay fixed. + same_owner(original, {**root[names[0]], "value": journal.decode_json(raw)}) + if names[1] in root: + terminal = journal.decode_json(checkpoints.bytes_of(root[names[1]])) + report = journal.decode_json(checkpoints.bytes_of(root[names[2]])) + require(type(terminal.get("schema")) is int and terminal["schema"] == 1 and terminal.get("task_id") == token + and journal.same_json(terminal.get("heal_type"), original["value"]["heal_type"]) + and journal.same_json(terminal.get("options"), original["value"]["options"]) + and terminal.get("status") == {"Failed": {"error": detail}} + and type(report.get("schema")) is int and report["schema"] == 1 + and journal.same_json(report.get("terminal"), terminal) + and journal.same_json(report.get("outcome"), outcome), + "journal failure lost its exact durable original failed terminal/report") + diagnostics = [event for values in events.values() for event in values + if event.get("target") == "rustfs::heal::task" and event.get("message") == "Heal task failed" + and event.get("task_id") == token and isinstance(event.get("error"), str) + and "failed to save checkpoint" in event["error"].lower() + and any(word in event["error"].lower() for word in ("no space", "enospc", "os error 28"))] + require(diagnostics, "NOT_OBSERVED: candidate has no original-task checkpoint write error") + return {"originalTaskId": token, "lastDurableProgressRetained": True, "originalResponsibilityRetained": True, + "originalAdmissionRetained": names[0] in root, + "actualCheckpointWriteFailure": True, "diagnostics": diagnostics, + "failedWriteAcknowledged": False, "originalFailureReclassified": False, "backupRescueUsed": False} + + +def journal_write_failure(fleet, probe, binding, expected, baseline, original, targets, frozen, active): + token, bucket = frozen["taskId"], frozen["bucket"] + require(frozen["value"]["admin"]["outcome"]["counters"]["healed"] == 0, + "NOT_OBSERVED: journal failure missed the checkpoint before any actual repair") + window = JournalWindow(fleet, "F07-original-checkpoint-write-failure", event_targets={"rustfs::heal::task", "rustfs::heal::manager"}) + for node in fleet.nodes: + filled = fleet.record("stopped-private-journal-exhaustion", {"node": node, + **fleet.remote(node, "bounded-volumes-fill", quiescent=True)}) + require(filled.get("quiescent") is True and filled.get("hostSharedMountRemounted") is False + and filled.get("candidateProcessWitnessRequired") is True + and len(filled.get("changes", [])) == 4 + and all(row.get("actualErrno") == 28 and row.get("freeBlocks") == 0 for row in filled["changes"]), + "journal failure did not exhaust only its four fixed owned volumes") + fleet.start() + for node in fleet.nodes: + identity = fleet.remote(node, "process") + witness = fleet.record("candidate-private-journal-ENOSPC", {"node": node, + **fleet.remote(node, "bounded-volumes-witness", expected=identity)}) + require(witness.get("sameOriginalProcess") is True and witness.get("hostSharedMountRemounted") is False + and len(witness.get("disks", [])) == 4 and all(row.get("writeErrno") == 28 + and row.get("sameServiceUidAndMountNamespace") is True for row in witness["disks"]), + "journal write fault lacks a real candidate UID/mount/process ENOSPC witness") + deadline = time.monotonic() + 180 + while True: + current = recovery_status(probe, fleet, token) + if current.get("summary") != "running": break + require(time.monotonic() < deadline, "original checkpoint write failure exceeded its bounded observation window") + time.sleep(.2) + # Pause every process before exporting the failed durable scene or freeing + # space. Partial pause/fill ACKs remain recoverable by the Fleet registry. + for node in fleet.nodes: + identity = fleet.record("write-failure-before-freeze", {"node": node, **fleet.remote(node, "process")}) + fleet.frozen_processes[node] = {name: identity[name] for name in identity if name != "node"} + fleet.record("write-failure-freeze", {"node": node, **fleet.remote(node, "process-signal", + signal="SIGSTOP", expected=fleet.frozen_processes[node])}) + after = fleet.remote(node, "process") + require(after.get("state") == "T" and all(after.get(name) == identity.get(name) for name in PROCESS_FIELDS), + "journal failure freeze missed its original candidate") + proof = retained_write_failure(checkpoints.records(fleet, token, "original-write-failed-checkpoint"), + checkpoints.records(fleet, token, "original-write-failed-admission", family="root"), frozen, current, window.capture()) + scene = fleet.snapshot("original-journal-write-failure-physical") + indexed = files(scene) + remaining = [target for target in targets if (target["node"], target["disk"], target["relativePath"]) not in indexed] + require(remaining, "NOT_OBSERVED: failed journal task left no actual repair for its separate rescan") + partial = copy.deepcopy(original) + missing = {(target["node"], target["disk"], target["relativePath"]) for target in remaining} + for node, value in partial.items(): + value["files"] = [row for row in value["files"] if (node, row["disk"], row["path"]) not in missing] + validate_repaired_physical(partial, scene, targets[0], repair_targets=targets) + for node in fleet.nodes: + released = fleet.record("release-only-owned-journal-filler", {"node": node, + **fleet.remote(node, "bounded-volumes-release", expected=fleet.frozen_processes[node])}) + require(released.get("sameOriginalProcess") is True and released.get("hostSharedMountRemounted") is False + and len(released.get("disks", [])) == 4 and all(row.get("writeErrno") == 0 + and row.get("freeBlocks", 0) > 0 for row in released["disks"]), + "journal release did not restore its candidate's exact owned resource") + for node, identity in tuple(fleet.frozen_processes.items()): + fleet.record("write-failure-resume-after-release", {"node": node, + **fleet.remote(node, "process-signal", signal="SIGCONT", expected=identity)}) + del fleet.frozen_processes[node] + active.remove((checkpoints.PATH, token)) + idle_fleet(fleet, binding) + admissions = [] + rescanned = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, + option_overrides={"recursive": True}, admissions=admissions) + require(len(admissions) == 1 and admissions[0]["clientToken"] != token, + "journal release rescan reused the failed original responsibility") + rescan = rescan_outcome(rescanned, bucket, expected, {version_identity(target) for target in remaining}, frozen["bucketIncarnationId"]) + repeated = recovery_status(probe, fleet, token) + require(all(canonical(repeated.get(name)) == canonical(current.get(name)) for name in ("summary", "settings", "outcome", "detail")), + "safe rescan rewrote the original journal write failure") + fleet.stop() + repaired = fleet.snapshot("journal-rescan-complete-physical-before-any-GET", quiescent=True) + validate_repaired_physical(original, repaired, targets[0], repair_targets=targets) + for target in targets: verify_proof(fleet, target, "journal-rescan-full-shard-integrity") + retained_write_failure(checkpoints.records(fleet, token, "retained-write-failed-checkpoint", quiescent=True), + checkpoints.records(fleet, token, "retained-write-failed-admission", quiescent=True, family="root"), frozen, repeated, window.events) + fleet.start() + deadline = time.monotonic() + 180 + while True: + final = recovery_status(probe, fleet, token) + if final.get("summary") != "running": break + require(time.monotonic() < deadline, "released original responsibility has no bounded restart outcome") + time.sleep(.2) + if final.get("summary") == "finished": + replay = rescan_outcome(final, bucket, expected, set(), frozen["bucketIncarnationId"]) + replay.pop("originalFailedTaskReclassified") + replay.update(realSameTaskReplayAfterSeparateRescan=True, failedFirstAttemptRetained=True, + separateRescanIsOriginalTaskCompletion=False) + else: + require(all(canonical(final.get(name)) == canonical(current.get(name)) for name in ("summary", "settings", "outcome", "detail")), + "restart neither retained the original failure nor completed a verified same-task replay") + replay = {"originalFailureStillRetained": True, "sameTaskCompletionClaimed": False} + for endpoint in binding["endpoints"]: + peer = Probe(binding, endpoint) + same = status(peer, fleet, checkpoints.PATH, token, "same-original-write-failure-peer") + require(all(canonical(same.get(name)) == canonical(final.get(name)) for name in ("summary", "settings", "outcome", "detail")), + "peer lost the failed original journal responsibility") + inventory = fleet.record("complete-peer-S3", {"endpoint": endpoint, **peer.inventory(bucket, + unversioned_current_tags=expected[0]["versionId"] == "null")}) + verify_seed(expected, inventory); compare_inventories(baseline, inventory) + fleet.stop() + require(physical_content(repaired) == physical_content(fleet.snapshot("journal-after-business-reads", quiescent=True)), + "business reads repaired objects missing from the journal rescan proof") + final_roots = checkpoints.records(fleet, token, "retained-write-admission-after-reads", quiescent=True, family="root") + if final.get("summary") == "finished": replay["durableOriginalCompletion"] = completed_original(final_roots, frozen, final) + else: + retained_write_failure(checkpoints.records(fleet, token, "retained-write-failure-after-reads", quiescent=True), + final_roots, frozen, final, window.events) + fleet.record("complete-original-journal-write-reconciliation", {"firstFailedAttempt": current, + "retainedFailureProof": proof, "separateRescan": rescan, "originalAfterRestart": final, "restartProof": replay}) + fleet.faults.clear() + return {"verdict": "PASS", "originalWriteFailure": proof, "separateRescan": rescan, "originalAfterRestart": replay, + "completePhysicalAndS3ManifestsVerified": True, "canonicalCasePassClaimed": False} + + def run_checkpoint_fault(args, topology, nodes, fixture, branch, candidate): - require(branch in IMPLEMENTED, "unimplemented journal branch cannot mutate the fleet") + require(branch in IMPLEMENTED or branch == "journal-write-failure", "unimplemented journal branch cannot mutate the fleet") directory = args.evidence / fixture / topology / branch directory.mkdir(parents=True) run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" - fleet = Fleet(nodes, directory, run_id, args.tool_directory, resource_profile="cursor", runtime_profile="root-restart") + bounded = branch == "journal-write-failure" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, resource_profile="cursor", + runtime_profile="journal-bounded" if bounded else "root-restart") bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] fleet.snapshot_buckets = [bucket] access, secret = secrets.token_hex(16), secrets.token_hex(24) os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) - count = 257 if fixture == "DB" else 80 + count = (129 if fixture == "DB" else 24) if bounded else (257 if fixture == "DB" else 80) binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat(), "bucketPrefix": bucket, "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, @@ -299,11 +472,17 @@ def run_checkpoint_fault(args, topology, nodes, fixture, branch, candidate): fleet.record("candidate-bound-inspector", verify_inspector(args.tool_directory, candidate)) probe, active, pending = Probe(binding), [], None result = {"topology": topology, "fixture": fixture, "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} + def allocate(): + for node in nodes: + fleet.record("fixed-private-journal-volume-allocation", {"node": node, + **fleet.remote(node, "bounded-volumes-create", branch="bounded-volume-enospc")}) try: - fleet.deploy(args.binary_sha, bucket, access, secret) + if bounded: fleet.deploy(args.binary_sha, bucket, access, secret, before_start=allocate) + else: fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) pending = {"operation": "seed-original-checkpoint-fixture"} - expected = seed(Recorder(probe, directory / "fixture"), bucket, fixture, count, [SIZE]) + expected = seed(Recorder(probe, directory / "fixture"), bucket, fixture, count, + [256 * 1024 + 123] if bounded else [SIZE]) baseline = fleet.record("ACK-bound-complete-S3", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) fleet.record("complete-ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) pending = None @@ -335,6 +514,9 @@ def run_checkpoint_fault(args, topology, nodes, fixture, branch, candidate): "NOT_OBSERVED: earlier generation missed the pre-repair capture window") frozen, identity = checkpoints.freeze(fleet, token, bucket, earlier=earlier) frozen = checkpoints.stop_for_fault(fleet, token, bucket, frozen, identity) + if bounded: + result.update(journal_write_failure(fleet, probe, binding, expected, baseline, original, targets, frozen, active)) + return result params = {"family": "checkpoint", "taskId": token, "disk": frozen["disk"], "relativePath": frozen["checkpoint"]["relativePath"], "faultId": "F07-" + branch, "kind": IMPLEMENTED[branch], "expectedBefore": journal.entry(checkpoints.bytes_of(frozen["checkpoint"]))} @@ -454,7 +636,7 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"F07 interrupted by sig for topology, selected in (("H1", nodes[:1]), ("H2", nodes)): for fixture in ("DB", "DV"): for branch in BRANCHES: - if branch in IMPLEMENTED: + if branch in IMPLEMENTED or branch == "journal-write-failure": result = run_checkpoint_fault(args, topology, selected, fixture, branch, candidate) elif branch == "mrf-checksum": result = run_mrf_checksum(args, topology, selected, fixture, candidate) @@ -477,7 +659,7 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"F07 interrupted by sig "diagnosticBranchesPassed": len(results) == 24 and all(row["cleanup"] == "PASS" and row["verdict"] == ("BLOCKED" if row["branch"] in BLOCKERS else "PREREQUISITE_PASS" if row["branch"] == "native-persist-fence" else "PASS") for row in results), - "canonicalCasePassClaimed": False, "reason": "journal-write-failure and full native cut coverage remain blocked"}) + "canonicalCasePassClaimed": False, "reason": "full native cut coverage and independent live audit remain blocked"}) return 1 diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index b4c6883e..53272cdf 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -2155,7 +2155,7 @@ "dispatchable": false, "code": "F07", "reasonCode": "EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION", - "reason": "已实现 H1/H2 checkpoint 截断、未来 schema、真实较早代次回放、ACK 绑定的 committed MRF CRC 故障和独立安全重扫；10 项原生回归仅为前置证据。仍缺 journal 写失败的完整现场流程及所需 fsync/commit 临界点审核，已实现分支尚无完整 live 审计。", + "reason": "已实现 H1/H2 checkpoint 截断、未来 schema、真实较早代次回放、committed MRF CRC 故障、私有限额卷的真实 journal ENOSPC 和独立安全重扫；10 项原生回归仅为前置证据。仍缺全部所需 fsync/commit 临界点审核和完整现场分支的独立 live 审计，诊断执行不能提升 canonical 验收。", "requiredCapabilities": [ "durable_checkpoint_mrf_fault", "write_failure_retention", diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index c2f6b533..77c219ea 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -195,7 +195,7 @@ fi if [ "${case_id}" = heal.checkpoint-and-mrf-corruption ] && [ "${cleanup_status}" = PASS ] && \ jq -e '.diagnosticBranchesPassed == true and .complete == false and .canonicalCasePassClaimed == false' \ "${evidence}/healthy-summary.json" >/dev/null 2>&1; then - verdict=BLOCKED; reason="F07 diagnostic execution lacks journal write-failure and full native cut coverage; partial success cannot certify the canonical case" + verdict=BLOCKED; reason="F07 diagnostic execution lacks full native cut coverage and independent live audit; partial success cannot certify the canonical case" fi args=("${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool --result-file "${artifacts}/cases/${case_id}.json" --verdict "${verdict}" --cleanup-status "${cleanup_status}" diff --git a/test_validation_heal_capacity.py b/test_validation_heal_capacity.py index 49ef78d0..7ad4ab8c 100644 --- a/test_validation_heal_capacity.py +++ b/test_validation_heal_capacity.py @@ -178,6 +178,32 @@ def test_bounded_journal_capacity_refuses_arbitrary_limits_paths_or_process_birt with patch.object(fleet_node, 'process', return_value={**identity, 'startTicks': 999}), self.assertRaises(ValueError): fleet_node.volume_operation(request) + def test_stopped_fill_is_restricted_to_fixed_journal_volumes_and_stopped_candidate(self): + owner = {"runtimeProfile": "journal-bounded"} + events = [] + backend = SimpleNamespace(read=lambda *args: {"branch": "bounded-volume-enospc"}, + views=lambda *args: tuple(range(4)), fill=lambda view, branch: events.append((view, branch)) or {"actualErrno": 28}) + request = {"runId": "gh-1-1-h1", "action": "bounded-volumes-fill", "quiescent": True} + with patch.object(fleet_node, "owned", return_value=(None, (), owner)), \ + patch.object(fleet_node, "stopped") as stopped, patch.object(fleet_node, "volume_module", return_value=backend): + result = fleet_node.volume_operation(request) + stopped.assert_called_once_with(owner) + self.assertEqual(events, [(index, "bounded-volume-enospc") for index in range(4)]) + self.assertTrue(result["candidateProcessWitnessRequired"]) + self.assertFalse(result["productRepairClaimed"]) + for changes in ({"path": "/shared"}, {"expected": {}}, {"limitBytes": 1}): + events.clear() + with self.assertRaises(ValueError): fleet_node.volume_operation({**request, **changes}) + self.assertFalse(events) + for profile in ("manual", "bounded-capacity"): + owner["runtimeProfile"] = profile; events.clear() + with self.assertRaises(ValueError): fleet_node.volume_operation(request) + self.assertFalse(events) + owner["runtimeProfile"] = "journal-bounded"; events.clear() + with patch.object(fleet_node, "stopped", side_effect=ValueError("candidate still running")): + with self.assertRaises(ValueError): fleet_node.volume_operation(request) + self.assertFalse(events) + def test_allocation_registers_exact_undo_before_formatting_only_bounded_files(self): with tempfile.TemporaryDirectory() as directory: state = Path(directory) / "state" diff --git a/test_validation_heal_persistence.py b/test_validation_heal_persistence.py index f91ed47d..9a0252f4 100644 --- a/test_validation_heal_persistence.py +++ b/test_validation_heal_persistence.py @@ -122,7 +122,7 @@ def test_bounded_diagnostics_need_complete_independent_manifests_and_exact_count with self.subTest(change=change), self.assertRaises(ValueError): persistence.rescan_outcome(*data) def test_incomplete_branches_never_enter_fleet_mutation(self): - for branch in persistence.BLOCKERS: + for branch in ("unknown", "native-persist-fence"): with self.subTest(branch=branch), self.assertRaisesRegex(ValueError, "cannot mutate"): persistence.run_checkpoint_fault(None, "H1", ["node"], "DB", branch, {}) @@ -144,7 +144,7 @@ def passed(args, topology, nodes, fixture, branch, candidate): patch.object(persistence, "run_checkpoint_fault", side_effect=passed) as mutate: self.assertEqual(persistence.main(), 1) verify.assert_called_once() - self.assertEqual(mutate.call_count, 12) + self.assertEqual(mutate.call_count, 16) self.assertEqual(mrf_mutate.call_count, 4) summary = json.loads((root / "healthy-summary.json").read_text()) self.assertEqual(len(summary["results"]), 24) @@ -161,7 +161,7 @@ def passed(args, topology, nodes, fixture, branch, candidate): self.assertFalse(row["physicalPowerLossClaimed"]) self.assertFalse(row["requiredNativeCoverageComplete"]) self.assertEqual(row["nativeProof"], proof) - self.assertEqual(sum(row["verdict"] == "BLOCKED" for row in summary["results"]), 4) + self.assertEqual(sum(row["verdict"] == "BLOCKED" for row in summary["results"]), 0) def failed(*values): row = passed(*values) if row["branch"] == "stale-generation": row["verdict"] = "FAIL" @@ -249,6 +249,153 @@ def test_rejected_stale_generation_retains_original_bytes_and_failed_responsibil self.assertFalse(proof["backupRescueUsed"]) with self.assertRaises(ValueError): persistence.retained_corruption(observed, roots, frozen, receipt, current) + def write_failure(self): + observed, _, frozen, _, _ = self.fixture() + value = frozen["value"] + value["admin"]["outcome"]["counters"].update(healed=0, unchanged=2) + frozen["checkpoint"].update(encoded(value, signed=True)) + observed["node"]["records"][4] = copy.deepcopy(frozen["checkpoint"]) + observed["node"]["records"][6] = {"disk": 1, "relativePath": journal.names("checkpoint", TOKEN)[2], "present": False} + detail = "Task execution failed: Failed to save checkpoint: No space left on device (os error 28)" + current = {"summary": "stopped", "settings": checkpoint.OPTIONS, "detail": detail, + "outcome": {**copy.deepcopy(value["admin"]["outcome"]), "execution": {"state": "aborted", "reason": "untraversable"}}} + root_observed, root_value = fixture_source.AdminCheckpointTests().root_fixture() + frozen["rootAdmission"] = checkpoint.owned_root_admission(root_observed, TOKEN) + events = {"node": [{"target": "rustfs::heal::task", "message": "Heal task failed", "task_id": TOKEN, "error": detail}]} + return observed, root_observed, frozen, current, events + + def test_actual_write_error_keeps_original_healthy_generation_and_admitted_owner(self): + proof = persistence.retained_write_failure(*self.write_failure()) + self.assertTrue(proof["lastDurableProgressRetained"]) + self.assertTrue(proof["originalAdmissionRetained"]) + self.assertFalse(proof["failedWriteAcknowledged"]) + self.assertFalse(proof["originalFailureReclassified"]) + for change in ("success", "wrong-error", "invented-progress", "missing-checkpoint", "rewrite-resume", "missing-owner", "wrong-task-event"): + observed, roots, frozen, current, events = self.write_failure() + if change == "success": current["summary"] = "finished" + elif change == "wrong-error": current["detail"] = "Failed to save checkpoint: Permission denied" + elif change == "invented-progress": current["outcome"]["counters"]["processed"] += 1 + elif change == "missing-checkpoint": observed["node"]["records"][4]["present"] = False + elif change == "rewrite-resume": observed["node"]["records"][5].update(encoded({"completed": True})) + elif change == "missing-owner": roots["node"]["records"][3]["present"] = False + else: events["node"][0]["task_id"] = INCARNATION + with self.subTest(change=change), self.assertRaises(ValueError): persistence.retained_write_failure(observed, roots, frozen, current, events) + + def test_known_write_failure_can_have_explicit_failed_terminal_without_silent_success(self): + observed, roots, frozen, current, events = self.write_failure() + roots["node"]["records"][3] = {"disk": 1, "relativePath": journal.names("root", TOKEN)[0], "present": False} + terminal = {"schema": 1, "task_id": TOKEN, "heal_type": frozen["rootAdmission"]["value"]["heal_type"], + "options": frozen["rootAdmission"]["value"]["options"], "status": {"Failed": {"error": current["detail"]}}} + roots["node"]["records"][4].update(encoded(terminal)) + roots["node"]["records"][5].update(encoded({"schema": 1, "terminal": terminal, "outcome": current["outcome"]})) + proof = persistence.retained_write_failure(observed, roots, frozen, current, events) + self.assertTrue(proof["originalResponsibilityRetained"]) + self.assertFalse(proof["originalAdmissionRetained"]) + terminal["status"] = "Completed"; roots["node"]["records"][4].update(encoded(terminal)) + with self.assertRaises(ValueError): persistence.retained_write_failure(observed, roots, frozen, current, events) + + def test_write_failure_can_consume_root_timeout_without_refunding_or_changing_intent(self): + for change in (None, "refunded", "removed", "scope", "retry", "owner"): + observed, roots, frozen, current, events = self.write_failure() + before = copy.deepcopy(frozen["rootAdmission"]["value"]) + before["options"]["timeout"] = {"secs": 120, "nanos": 0} + frozen["rootAdmission"].update(value=before, **encoded(before)) + after = copy.deepcopy(before) + after["options"]["timeout"] = {"secs": 90, "nanos": 0} + if change == "refunded": after["options"]["timeout"]["secs"] = 121 + elif change == "removed": after["options"].pop("timeout") + elif change == "scope": after["options"]["pool_index"] = 1 + elif change == "retry": after["retry_attempts"] = 1 + roots["node"]["records"][3].update(encoded(after)) + if change == "owner": + roots["node"]["records"][0].update(encoded(after)) + roots["node"]["records"][3] = {"disk": 1, "relativePath": journal.names("root", TOKEN)[0], "present": False} + if change is None: + self.assertTrue(persistence.retained_write_failure(observed, roots, frozen, current, events)["originalResponsibilityRetained"]) + else: + with self.subTest(change=change), self.assertRaises(ValueError): + persistence.retained_write_failure(observed, roots, frozen, current, events) + + def test_write_failure_live_census_requires_proven_frozen_processes(self): + for change in (None, "serviceFrozen", "stable", "processBound", "independentReads"): + observed, roots, frozen, current, events = self.write_failure() + for census in (observed["node"], roots["node"]): + census.update(quiescent=False, stable=True, serviceFrozen=True, processBound=True, independentReads=2) + if change is None: + persistence.retained_write_failure(observed, roots, frozen, current, events) + else: + roots["node"][change] = 1 if change == "independentReads" else False + with self.subTest(change=change), self.assertRaises(ValueError): + persistence.retained_write_failure(observed, roots, frozen, current, events) + + def test_write_refusal_release_rescan_and_real_original_retry_have_distinct_evidence(self): + for change in (None, "wrong-mount-witness", "release-failure", "late-export-failure"): + observed, roots, frozen, failed, events = self.write_failure() + expected = [{"key": f"objects/{i:04d}.bin", "versionId": "null", "deleteMarker": False} for i in range(3)] + def finished(heal): + return {"summary": "finished", "settings": checkpoint.OPTIONS, "truncated": False, + "outcome": {"execution": {"state": "completed"}, "coverage": "complete", "objectsTruncated": False, + "counters": dict(processed=3, healed=heal, unchanged=3-heal, skipped=0, failed=0, unknown=0, attemptFailures=0, overflowed=False), + "objects": [{"identity": {"kind": "object", "bucket": BUCKET, "object": entry["key"], "versionId": None, + "bucketIncarnationId": INCARNATION, "poolIndex": 0, "setIndex": 0}, + "disposition": {"state": "repaired" if heal and i == 2 else "verified_healthy"}} for i, entry in enumerate(expected)]}} + rescanned, final = finished(1), finished(0) + final_roots = self.completion_roots(frozen, final) + identity = {field: field for field in persistence.PROCESS_FIELDS}; identity["pid"] = 50 + target = {"node": "node", "disk": 1, "relativePath": "bound/part.1", "key": expected[-1]["key"], "versionId": "null"} + repaired = {"node": {"files": [{"disk": 1, "path": "bound/part.1", "sha256": "a" * 64}], "formats": []}} + issued = [] + class WriteFleet: + nodes, frozen_processes, faults = ["node"], {}, [("node", {"faultId": "F07-shard-0"})] + paused = False + def start(self): issued.append("start") + def stop(self): issued.append("stop") + def record(self, label, value): + issued.append(label) + if change == "late-export-failure" and label == "complete-original-journal-write-reconciliation": raise OSError("export failed") + return value + def snapshot(self, label, **kwargs): + if label == "original-journal-write-failure-physical": return {"node": {"files": [], "formats": []}} + return repaired + def remote(self, node, action, **params): + issued.append(action) + if action == "bounded-volumes-fill": return {"quiescent": True, "hostSharedMountRemounted": False, + "candidateProcessWitnessRequired": True, "changes": [{"actualErrno": 28, "freeBlocks": 0}] * 4} + if action == "bounded-volumes-witness": return {"sameOriginalProcess": True, "hostSharedMountRemounted": False, + "disks": [{"writeErrno": 28, "sameServiceUidAndMountNamespace": change != "wrong-mount-witness"}] * 4} + if action == "bounded-volumes-release": return {"sameOriginalProcess": True, "hostSharedMountRemounted": False, + "disks": [{"writeErrno": 28 if change == "release-failure" else 0, "freeBlocks": 1}] * 4} + if action == "process": return {**identity, "state": "T" if self.paused else "S"} + if action == "process-signal": self.paused = params["signal"] == "SIGSTOP"; return {} + raise AssertionError(action) + fleet = WriteFleet() + active = [(checkpoint.PATH, TOKEN)] + def records(fleet, token, label, **kwargs): + if label == "retained-write-admission-after-reads": return final_roots + return roots if kwargs.get("family") == "root" else observed + def run_task(*args, **kwargs): kwargs["admissions"].append({"clientToken": INCARNATION}); return rescanned + window = SimpleNamespace(events=events, capture=lambda: events) + with patch.object(persistence, "JournalWindow", return_value=window), \ + patch.object(persistence, "recovery_status", side_effect=[failed, failed, final]), \ + patch.object(persistence.checkpoints, "records", side_effect=records), \ + patch.object(persistence, "validate_repaired_physical"), patch.object(persistence, "verify_proof"), \ + patch.object(persistence, "idle_fleet"), patch.object(persistence, "run_task", side_effect=run_task), \ + patch.object(persistence, "status", return_value=final), \ + patch.object(persistence, "Probe", return_value=SimpleNamespace(inventory=lambda *a, **kw: {})), \ + patch.object(persistence, "verify_seed"), patch.object(persistence, "compare_inventories"): + args = (fleet, None, {"endpoints": {"node1": "http://owned"}}, expected, {}, repaired, [target], frozen, active) + if change is None: + result = persistence.journal_write_failure(*args) + self.assertTrue(result["originalWriteFailure"]["lastDurableProgressRetained"]) + self.assertTrue(result["originalAfterRestart"]["realSameTaskReplayAfterSeparateRescan"]) + self.assertFalse(result["originalAfterRestart"]["separateRescanIsOriginalTaskCompletion"]) + self.assertFalse(result["canonicalCasePassClaimed"]) + self.assertFalse(fleet.faults) + else: + with self.assertRaises((ValueError, OSError)): persistence.journal_write_failure(*args) + self.assertTrue(fleet.faults) + if change == "wrong-mount-witness": self.assertNotIn("bounded-volumes-release", issued) + def test_successful_replay_cannot_discard_rescue_before_manifest_terminal_and_evidence_agree(self): for change in (None, "read-repair", "late-archive-failure", "rescan-relabels-original"): early, frozen, receipt = self.stale(); current, _, expected, _, _ = self.rescan() @@ -295,17 +442,18 @@ def run_task(*args, **kwargs): self.assertTrue(fleet.journal_faults); self.assertTrue(fleet.faults) def interrupted_injection(self, cleanup, mutation=None): - _, _, frozen, _, _ = self.fixture() + frozen = self.write_failure()[2] if mutation == "fill" else self.fixture()[2] snapshots = {"node": {"files": [], "formats": []}} captured = {} class InterruptedFleet: def __init__(self, *args, **kwargs): self.faults, self.journal_faults = [], [] + self.nodes = ["node"] captured["fleet"] = self def record(self, label, value): return value - def deploy(self, *args): pass + def deploy(self, *args, **kwargs): pass def start(self): pass def stop(self): pass def snapshot(self, *args, **kwargs): return snapshots @@ -314,6 +462,7 @@ def remote(self, node, action, **params): if action == "journal-apply": # The node may have committed the fault before the ACK was lost. raise OSError("journal injection acknowledgement lost") + if action == "bounded-volumes-fill": raise OSError("private journal fill acknowledgement lost") return {} def cleanup(self, *, preserve_namespaces=False): @@ -346,8 +495,10 @@ def inventory(self, *args, **kwargs): return {} patch.object(persistence.checkpoints, "admit", return_value=TOKEN, side_effect=OSError("admission acknowledgement lost") if mutation == "admission" else None), \ patch.object(persistence.checkpoints, "freeze", return_value=(frozen, {})), \ + patch.object(persistence, "JournalWindow"), \ patch.object(persistence.checkpoints, "stop_for_fault", return_value=frozen): - result = persistence.run_checkpoint_fault(args, "H1", ["node"], "DB", "checkpoint-truncate", + result = persistence.run_checkpoint_fault(args, "H1", ["node"], "DB", + "journal-write-failure" if mutation == "fill" else "checkpoint-truncate", {"source_sha": "0" * 40, "package_sha256": "0" * 64}) return result, captured @@ -376,5 +527,13 @@ def test_unknown_fixture_or_original_admission_preserves_namespace_even_if_clean self.assertIn("unresolvedMutation", result) self.assertNotIn("separateRescan", result) + def test_lost_private_fill_ack_keeps_shard_rescue_and_failure_before_any_positive_rescan(self): + result, captured = self.interrupted_injection(False, "fill") + self.assertEqual((result["verdict"], result["cleanup"]), ("FAIL", "FAIL")) + self.assertEqual(result["failure"]["message"], "private journal fill acknowledgement lost") + self.assertEqual(len(captured["cleanupFaults"]), 4) + self.assertFalse(captured["cleanupJournals"]) + self.assertNotIn("separateRescan", result) + if __name__ == "__main__": unittest.main() From 87337104428796b9a4e89741715dd652f84ba3de Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 22:56:09 +0800 Subject: [PATCH 188/343] fix(validation): audit rejected MRF injections before cleanup --- docs/validation-migration-map.md | 2 + scripts/validation_heal/fleet_node.py | 32 ++++++ scripts/validation_heal/healthy.py | 8 +- .../validation_heal/recover_environment.py | 92 +++++++++++++++- test_validation_heal_recovery.py | 104 ++++++++++++++++++ 5 files changed, 232 insertions(+), 6 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index e4625edd..6ad8fcfc 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1961,3 +1961,5 @@ F07 original stale-generation replay now captures an actual earlier product chec The final stale-generation change passed 608 Heal contracts with eight platform skips, 91 focused persistence/checkpoint/journal/manager/environment/migration checks, the unchanged 84/301/532 matrix, Python AST, shell syntax and whitespace. Two sequential final reviews retained exact bytes, task and owner bindings, rejected unknown calls and delayed rescue removal until complete evidence export. No unresolved finding remains for this executor diff; no live F07 PASS is claimed. F07 original checkpoint write failure now has a diagnostic executor using the existing four-node `journal-bounded` profile. It fills only the four fixed private journal volumes while the candidate is stopped, then proves a candidate UID and mount namespace witness for the real `ENOSPC` write. The original checkpoint, resume bytes, root owner, exact failed task outcome and source-matched `Heal task failed` event remain durable. The executor pauses and records every owner before releasing only its private filler, performs a distinct Admin rescan with a new token, verifies complete physical, shard and peer S3 evidence, and preserves the first failed attempt separately from any later same-task retry. Cleanup keeps rescue state when a fill ACK, release, evidence export or reconciliation step is incomplete. The branch is implemented but remains outside canonical dispatch until a live run and the remaining reviewed native persistence cuts complete. The final full Heal suite passes 615 tests with eight platform skips; 21 focused persistence contracts, the catalog, 84/301/532 matrix, Python AST, shell syntax and whitespace pass. Two fresh sequential reviews cover correctness, simplicity, test coverage, path security and durability. Frozen process-bound censuses and decreasing root timeout budgets retain their original identity checks. Canonical PASS remains 101/158. + +F07 diagnostic 38261190390/1 retained FAIL with cleanup FAIL. Checkpoint observation rejected the candidate's actual ordered JSON digest; the MRF windows failed on H1, and H2's first CRC injection was rejected because its actual committed payload also contains system metadata responsibility. No live branch PASS is claimed. The rejected injection left all original MRF bytes and object files unchanged and created no intent or backup. Recovery now accepts only that exact first H2 DB refusal, seven continuous digest-bound create/PUT ACKs, unchanged original MRF and physical censuses, and a cleared member gate. Every node is audited before finalization; each node rechecks its stopped candidate, empty private quarantine, exact MRF digest and full physical scene immediately before removal. Unknown calls, missing ACKs, changed scene, any injection evidence or backup retain the namespace. The original FAIL is never rewritten. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index e62ca516..bf20be26 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1273,6 +1273,36 @@ def quota_audit_finalize(request): "backupRestorationUsed": True, "productRepairClaimed": False} +def journal_rejection_audit_finalize(request): + """Finalize an unchanged MRF namespace only when no journal mutation began.""" + require(set(request) == {"runId", "action", "bucket", "binarySha256", "rejectedFaultId", "expectedSceneSha256", "expectedMrfSha256"}, + "journal rejection finalization accepts only an exact archived identity") + require(request["action"] in {"journal-rejection-audit", "journal-rejection-audit-finalize"}, + "unsupported journal rejection audit action") + state, roots, owner = owned(request["runId"]) + stopped(owner) + require(request["runId"].endswith("-h2") and owner.get("runtimeProfile") == "mrf-crash" + and owner.get("bucket") == request["bucket"] and request["binarySha256"] == owner["binarySha256"] + and digest(state / "rustfs") == owner["binarySha256"] + and request["rejectedFaultId"] == "F07-mrf-checksum-0-0-0", + "journal rejection candidate, bucket, runtime or first fault identity differs") + for root in roots: + with fault.directory(root) as fd, fault.parent(fd, ["quarantine"]) as backup: + require(not os.listdir(backup), "journal mutation backup or intent exists; retain namespace") + records = mrf_records({"runId": request["runId"], "action": "mrf-records", "quiescent": True}) + mrf_sha = hashlib.sha256(json.dumps(records, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + require(re.fullmatch(r"[0-9a-f]{64}", request.get("expectedMrfSha256", "")) is not None + and mrf_sha == request["expectedMrfSha256"], "journal rejection MRF bytes drifted; retain namespace") + observed = snapshot({"runId": request["runId"], "action": "snapshot", "quiescent": True}) + content = {key: observed[key] for key in ("files", "formats")} + scene_sha = hashlib.sha256(json.dumps(content, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + require(re.fullmatch(r"[0-9a-f]{64}", request.get("expectedSceneSha256", "")) is not None + and scene_sha == request["expectedSceneSha256"], "journal rejection scene drifted; retain namespace") + result = cleanup(request) if request["action"] == "journal-rejection-audit-finalize" else {"readyToFinalize": True} + return {**result, "faultNeverApplied": True, "archivedPreFaultSceneEqual": True, + "sceneSha256": scene_sha, "backupRestorationUsed": False, "productRepairClaimed": False} + + def task_records(request): """Read three exact Admin journal names; never mutate system metadata.""" state, roots, owner = owned(request["runId"]) @@ -1404,6 +1434,8 @@ def main(request): actions["task-records"] = task_records actions["environment-capabilities"] = environment_capabilities actions["quota-audit-finalize"] = quota_audit_finalize + actions["journal-rejection-audit-finalize"] = journal_rejection_audit_finalize + actions["journal-rejection-audit"] = journal_rejection_audit_finalize for action in ("journal-census", "journal-observe", "journal-capture", "journal-apply", "journal-recover"): actions[action] = journal_operation for action in ('generation-capture', 'generation-install', 'generation-recover'): diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index f4852b1f..c70f0c2d 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -185,7 +185,7 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() - if action.startswith("network-") or action in {"cleanup", "quota-audit-finalize"}: + if action.startswith("network-") or action in {"cleanup", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize"}: network_source = NODE_SOURCE.with_name("network_gate.py").read_text() source += "\nimport types\nnetwork_gate_backend = types.ModuleType('network_gate')\nexec(" + repr( network_source) + ", network_gate_backend.__dict__)\nNETWORK_GATE_SOURCE = " + repr(network_source) + "\n" @@ -220,11 +220,11 @@ def remote(self, node, action, **params): if action == "format-schema-fault": source += "\nimport types\nschema_faults = types.ModuleType('schema_faults')\nexec(" + repr( NODE_SOURCE.with_name("schema_faults.py").read_text()) + ", schema_faults.__dict__)\n" - if action in {"encryption-configure", "snapshot", "audit-finalize", "quota-audit-finalize"}: + if action in {"encryption-configure", "snapshot", "audit-finalize", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize"}: source += "\nimport types\nencryption_runtime = types.ModuleType('encryption_runtime')\nexec(" + repr( NODE_SOURCE.with_name("encryption_runtime.py").read_text()) + ", encryption_runtime.__dict__)\n" - replacement_actions = {"start", "cleanup", "audit-finalize", "quota-audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot", "format-schema-fault"} - if action in {"start", "cleanup", "audit-finalize", "quota-audit-finalize", "drive-gate", "drive-gate-arm-live", "replace-drive", "quorum-gate", "quorum-gate-probe"}: + replacement_actions = {"start", "cleanup", "audit-finalize", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot", "format-schema-fault"} + if action in {"start", "cleanup", "audit-finalize", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize", "drive-gate", "drive-gate-arm-live", "replace-drive", "quorum-gate", "quorum-gate-probe"}: source += "\nimport types\nquorum_gate_backend = types.ModuleType('quorum_gate')\nexec(" + repr( NODE_SOURCE.with_name("quorum_gate.py").read_text()) + ", quorum_gate_backend.__dict__)\n" if action in {"drive-gate", "drive-gate-arm-live", "drive-gate-probe", "drive-gate-activate", "gate-state", "drive-gate-release-live"} | replacement_actions: diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index 00f75236..cbd68d63 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -138,6 +138,8 @@ def network_recovery_plan(artifact, failed_run, candidate): def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") + if list(artifact.glob("*/run/heal.checkpoint-and-mrf-corruption/cases/*.json")): + return rejected_mrf_recovery_plan(artifact, failed_run, candidate) if list(artifact.glob("*/run/heal.bucket-incarnation-recreation/cases/*.json")): return recreation_recovery_plan(artifact, failed_run, candidate) if list(artifact.glob("*/run/heal.enospc-inodes-and-quota/cases/heal.enospc-inodes-and-quota.json")): @@ -191,6 +193,72 @@ def recovery_plan(artifact, failed_run, candidate): return plans +def rejected_mrf_recovery_plan(artifact, failed_run, candidate): + """Finalize only a refused first MRF injection with unchanged original bytes.""" + case = "heal.checkpoint-and-mrf-corruption" + results = list(artifact.glob("*/run/*/cases/*.json")) + require(len(results) == 1 and results[0].name == case + ".json", "F07 recovery cannot omit another failed case") + result = json.loads(results[0].read_text()) + provenance = result.get("provenance") or {} + require(result.get("case_id") == case and result.get("verdict") == "FAIL" and result.get("cleanup_status") == "FAIL" + and provenance.get("source_commit") == candidate["source_sha"] + and provenance.get("candidate_sha256") == candidate["package_sha256"], "F07 recovery candidate or result differs") + evidence = results[0].parents[1] / "evidence" / case + summary = json.loads((evidence / "healthy-summary.json").read_text()) + rows = summary.get("results") or [] + require(summary.get("caseId") == case and summary.get("cleanup") == "FAIL" + and summary.get("complete") is False and summary.get("canonicalCasePassClaimed") is False + and rows and all(row.get("cleanup") == "PASS" for row in rows[:-1]) + and rows[-1].get("topology") == "H2" and rows[-1].get("fixture") == "DB" + and rows[-1].get("branch") == "mrf-checksum" and rows[-1].get("verdict") == "FAIL" + and rows[-1].get("cleanup") == "FAIL" and "unresolvedMutation" not in rows[-1] + and "MRF fault lacks an actual committed active owned responsibility on this exact slot" in + rows[-1].get("failure", {}).get("message", ""), "F07 recovery supports only its first rejected MRF injection") + folder = evidence / "DB/H2/mrf-checksum" + def one(label): + found = list(folder.glob("*-" + label + ".json")) + require(len(found) == 1, "F07 recovery lacks one exact " + label) + return json.loads(found[0].read_text()) + binding, before, failure = one("binding"), one("original-partial-write-physical"), one("original-physical-failure-scene") + original, retained = one("original-stopped-MRF-records"), one("original-MRF-failure-scene") + require(re.fullmatch(rf"gh-{failed_run}-[1-9][0-9]*-h2", binding.get("runId", "")) + and binding.get("candidateCommit") == candidate["source_sha"] + and binding.get("candidatePackageSha256") == candidate["package_sha256"] + and binding.get("candidateBinarySha256") == provenance.get("binary_sha256") + and binding.get("harnessCommit") == provenance.get("testing_commit") + and len(before) == 4 and set(before) == set(failure) == set(original) == set(retained) + and physical_content(before) == physical_content(failure) and original == retained + and all(value.get("quiescent") is True and value.get("pid") == 0 + and value.get("binarySha256") == provenance.get("binary_sha256") for value in failure.values()), + "F07 rejected injection changed its candidate, physical scene or original MRF records") + nodes, bucket = list(failure), binding.get("bucketPrefix", "") + require(all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes) + and re.fullmatch(r"rv-heal-[0-9a-f]{16}", bucket), "F07 recovery has invalid owned nodes or bucket") + gate = one("gate-recovery") + require(gate.get("node") in nodes and gate.get("state") == "CLEAR" + and gate.get("originalPermissionsRestored") is True + and not list(folder.glob("*-actual-original-MRF-CRC-injection.json")) + and not list(folder.glob("*-injected-stopped-MRF-records.json")), "F07 injection or gate recovery is ambiguous") + requests, acks = sorted((folder / "fixture").glob("*-request.json")), sorted((folder / "fixture").glob("*-ack.json")) + require(len(requests) == len(acks) == 7, "F07 recovery lacks its complete create and six PUT acknowledgements") + for seq, (request_path, ack_path) in enumerate(zip(requests, acks), 1): + request, ack = json.loads(request_path.read_text()), json.loads(ack_path.read_text()) + observed = ack.get("observation") or {} + require(type(request.get("sequence")) is int and type(ack.get("sequence")) is int + and request["sequence"] == ack["sequence"] == seq + and request.get("runId") == ack.get("runId") == binding["runId"] + and ack.get("requestFile") == request_path.name + and ack.get("requestSha256") == hashlib.sha256(request_path.read_bytes()).hexdigest() + and observed.get("operation") == request.get("operation") == ("create_bucket" if seq == 1 else "put_object") + and observed.get("params") == request.get("params") and request.get("params", {}).get("Bucket") == bucket + and observed.get("clientCallInvoked") is True and observed.get("resultUnknown") is False + and observed.get("acknowledged") is True and ack.get("operationCheck") == "PASS", + "F07 recovery ledger changed identity or contains an unknown mutation") + return [{"kind": "rejected-mrf-injection", "runId": binding["runId"], "nodes": nodes, "node": nodes[0], + "bucket": bucket, "before": failure, "originalMrf": original, + "binarySha256": provenance["binary_sha256"], "rejectedFaultId": "F07-mrf-checksum-0-0-0"}] + + def recreation_recovery_plan(artifact, failed_run, candidate): """Archive a stopped C11 fixture after an exact known DeleteBucket refusal.""" case = "heal.bucket-incarnation-recreation" @@ -442,7 +510,27 @@ def main(): require(recovery_scene_matches(current, plan), "retained fleet drifted from its exact archived scene; no recovery performed") fleet.record("retained-service-logs", fleet.remote(plan["node"], "logs")) - if plan.get("kind") == "rejected-incarnation-delete": + if plan.get("kind") == "rejected-mrf-injection": + archived = fleet.record("retained-original-MRF-records", {node: fleet.remote(node, + "mrf-records", quiescent=True) for node in plan["nodes"]}) + require(archived == plan["originalMrf"], "F07 original MRF bytes drifted; retain namespace") + for node in plan["nodes"]: + gate = fleet.record("audited-gate-state", {"node": node, **fleet.remote(node, "gate-state")}) + require(gate.get("state") == "CLEAR", "F07 member gate is not clear") + require(recovery_scene_matches(fleet.snapshot("physical-immediately-before-finalization", quiescent=True), plan), + "F07 stopped scene changed during evidence export; retain namespace") + for action in ("journal-rejection-audit", "journal-rejection-audit-finalize"): + for node in plan["nodes"]: + scene = physical_content(plan["before"])[node] + proof = fleet.record("audited-finalization" if action.endswith("finalize") else "audited-no-injection", {"node": node, + **fleet.remote(node, action, bucket=plan["bucket"], + binarySha256=plan["binarySha256"], rejectedFaultId=plan["rejectedFaultId"], + expectedMrfSha256=hashlib.sha256(json.dumps(plan["originalMrf"][node], sort_keys=True, separators=(",", ":")).encode()).hexdigest(), + expectedSceneSha256=hashlib.sha256(json.dumps(scene, sort_keys=True, separators=(",", ":")).encode()).hexdigest())}) + require(proof.get("faultNeverApplied") is True and (proof.get("readyToFinalize") is True if action.endswith("audit") else + proof.get("cleanup") == "PASS" and proof.get("unitStopped") is True and proof.get("rootsRemoved") is True), + "F07 rejected journal namespace audit or finalization failed") + elif plan.get("kind") == "rejected-incarnation-delete": require(all(value.get("binarySha256") == plan["binarySha256"] for value in current.values()), "retained candidate binary differs from the archived failed scene") gate = fleet.record("audited-gate-state", fleet.remote(plan["node"], "gate-state")) @@ -472,7 +560,7 @@ def main(): else: proof = fleet.record("audited-finalization", fleet.remote(plan["node"], "audit-finalize", **{ key: plan[key] for key in ("disk", "rejectedFaultId", "binarySha256", "expectedSceneSha256")})) - if plan.get("kind") != "performance-no-fault-applied": + if plan.get("kind") not in {"performance-no-fault-applied", "rejected-mrf-injection"}: for node in plan["nodes"]: if node != plan["node"]: fleet.record("unchanged-peer-finalization", fleet.remote(node, "cleanup")) diff --git a/test_validation_heal_recovery.py b/test_validation_heal_recovery.py index bb4a6799..f676a8bb 100644 --- a/test_validation_heal_recovery.py +++ b/test_validation_heal_recovery.py @@ -3,6 +3,7 @@ import hashlib import copy import json +import sys from pathlib import Path import tempfile import types @@ -15,6 +16,7 @@ from scripts.validation_heal.capacity import release_quota from scripts.validation_heal.healthy import Fleet from scripts.validation_heal import recover_environment as recovery +from scripts.validation_heal import fleet_node, heal_fault class RejectedFaultRecoveryTest(unittest.TestCase): @@ -56,6 +58,108 @@ def test_finalization_needs_no_intent_and_exact_archived_scene(self): cleanup.assert_not_called() +class RejectedMrfRecoveryTests(unittest.TestCase): + def test_remote_rejection_audit_loads_every_real_snapshot_and_cleanup_backend(self): + with tempfile.TemporaryDirectory() as directory: + fleet = Fleet(["vm000"], Path(directory), "gh-12345-1-h2") + def execute(*args, **kwargs): + source = kwargs["input"].rsplit("\nmain(", 1)[0] + namespace = {"__name__": "__validation_node_contract__"} + with patch.dict(sys.modules): + exec(compile(source, "remote-node.py", "exec"), namespace) + self.assertTrue({"fault", "mrf", "encryption_runtime", "gate", "quorum_gate_backend", "replacement"} <= set(namespace)) + return types.SimpleNamespace(returncode=0, stdout="{}", stderr="") + with patch("scripts.validation_heal.healthy.subprocess.run", side_effect=execute): + for action in ("journal-rejection-audit", "journal-rejection-audit-finalize"): + fleet.remote("vm000", action) + + def archive(self, root): + case, bucket = "heal.checkpoint-and-mrf-corruption", "rv-heal-0123456789abcdef" + base = root / "artifact" / "run" / case + folder = base / "evidence" / case / "DB/H2/mrf-checksum" + (base / "cases").mkdir(parents=True); (folder / "fixture").mkdir(parents=True) + write = lambda p, v: p.write_text(json.dumps(v)) + candidate = {"source_sha": "a" * 40, "package_sha256": "b" * 64} + provenance = {"source_commit": "a" * 40, "candidate_sha256": "b" * 64, + "binary_sha256": "c" * 64, "testing_commit": "d" * 40} + write(base / "cases" / (case + ".json"), {"case_id": case, "verdict": "FAIL", "cleanup_status": "FAIL", "provenance": provenance}) + row = {"topology": "H2", "fixture": "DB", "branch": "mrf-checksum", "verdict": "FAIL", "cleanup": "FAIL", + "failure": {"message": "MRF fault lacks an actual committed active owned responsibility on this exact slot"}} + write(folder.parents[2] / "healthy-summary.json", {"caseId": case, "cleanup": "FAIL", "complete": False, + "canonicalCasePassClaimed": False, "results": [row]}) + write(folder / "00001-binding.json", {"runId": "gh-12345-1-h2", "bucketPrefix": bucket, + "candidateCommit": "a" * 40, "candidatePackageSha256": "b" * 64, + "candidateBinarySha256": "c" * 64, "harnessCommit": "d" * 40}) + scene = {f"vm00{i}": {"quiescent": True, "pid": 0, "binarySha256": "c" * 64, "files": [], "formats": []} for i in range(4)} + records = {node: {"readOnly": True, "stable": True, "records": []} for node in scene} + for label, value in (("original-partial-write-physical", scene), ("original-physical-failure-scene", scene), + ("original-stopped-MRF-records", records), ("original-MRF-failure-scene", records)): + write(folder / ("00002-" + label + ".json"), value) + write(folder / "00003-gate-recovery.json", {"node": "vm003", "state": "CLEAR", "originalPermissionsRestored": True}) + for seq in range(1, 8): + request = {"sequence": seq, "runId": "gh-12345-1-h2", "operation": "create_bucket" if seq == 1 else "put_object", "params": {"Bucket": bucket}} + path = folder / "fixture" / f"{seq:06d}-request.json"; write(path, request) + write(folder / "fixture" / f"{seq:06d}-ack.json", {"sequence": seq, "runId": request["runId"], + "requestFile": path.name, "requestSha256": hashlib.sha256(path.read_bytes()).hexdigest(), "operationCheck": "PASS", + "observation": {"operation": request["operation"], "params": request["params"], "clientCallInvoked": True, + "resultUnknown": False, "acknowledged": True}}) + return candidate, folder + + def test_unknown_mutation_drift_or_another_injection_keeps_namespace(self): + for change in (None, "unknown", "missing-ack", "digest", "physical", "records", "injection", "earlier-dirty"): + with tempfile.TemporaryDirectory() as directory, self.subTest(change=change): + root = Path(directory); candidate, folder = self.archive(root) + ack_path = folder / "fixture/000007-ack.json" + ack = json.loads(ack_path.read_text()) + if change == "unknown": ack["observation"]["resultUnknown"] = True; ack_path.write_text(json.dumps(ack)) + elif change == "missing-ack": ack_path.unlink() + elif change == "digest": ack["requestSha256"] = "e" * 64; ack_path.write_text(json.dumps(ack)) + elif change in {"physical", "records"}: + label = "original-physical-failure-scene" if change == "physical" else "original-MRF-failure-scene" + path = next(folder.glob("*-" + label + ".json")); value = json.loads(path.read_text()); value["vm000"]["extra"] = True + if change == "physical": value["vm000"]["files"] = [{"disk": 0, "path": "changed"}] + path.write_text(json.dumps(value)) + elif change == "injection": (folder / "00004-actual-original-MRF-CRC-injection.json").write_text("{}") + elif change == "earlier-dirty": + path = folder.parents[2] / "healthy-summary.json"; value = json.loads(path.read_text()) + value["results"].insert(0, {"cleanup": "FAIL"}); path.write_text(json.dumps(value)) + if change is None: + plan = recovery_plan(root, 12345, candidate)[0] + self.assertEqual(plan["kind"], "rejected-mrf-injection") + else: + with self.assertRaises(ValueError): recovery_plan(root, 12345, candidate) + + def test_node_requires_empty_quarantine_and_exact_stopped_scene(self): + scene = {"files": [], "formats": []} + digest = hashlib.sha256(json.dumps(scene, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + with tempfile.TemporaryDirectory() as directory: + state = Path(directory).resolve(); roots = tuple(state / f"disk{i}" for i in range(4)) + for root in roots: (root / "quarantine").mkdir(parents=True) + owner = {"runtimeProfile": "mrf-crash", "bucket": "rv-heal-0123456789abcdef", "binarySha256": "c" * 64} + request = {"runId": "gh-12345-1-h2", "action": "journal-rejection-audit-finalize", "bucket": owner["bucket"], + "binarySha256": "c" * 64, "expectedSceneSha256": digest, "rejectedFaultId": "F07-mrf-checksum-0-0-0", + "expectedMrfSha256": hashlib.sha256(b'{}').hexdigest()} + with patch.object(fleet_node, "owned", return_value=(state, roots, owner)), patch.object(fleet_node, "stopped"), \ + patch.object(fleet_node, "digest", return_value="c" * 64), patch.object(fleet_node, "fault", heal_fault, create=True), \ + patch.object(heal_fault, "mount_id", return_value=1), patch.object(fleet_node, "snapshot", return_value=scene), \ + patch.object(fleet_node, "mrf_records", return_value={}), \ + patch.object(fleet_node, "cleanup", return_value={"cleanup": "PASS"}) as cleanup: + self.assertTrue(fleet_node.journal_rejection_audit_finalize(request)["faultNeverApplied"]) + cleanup.reset_mock() + self.assertTrue(fleet_node.journal_rejection_audit_finalize({**request, "action": "journal-rejection-audit"})["readyToFinalize"]) + cleanup.assert_not_called() + for change in ({"expectedSceneSha256": "e" * 64}, {"expectedMrfSha256": "e" * 64}, {"binarySha256": "e" * 64}, + {"rejectedFaultId": "F07-mrf-checksum-0-0-1"}, {"path": "/shared"}): + cleanup.reset_mock() + with self.assertRaises(ValueError): fleet_node.journal_rejection_audit_finalize({**request, **change}) + cleanup.assert_not_called() + (roots[2] / "quarantine/interrupted.original").write_bytes(b"evidence") + cleanup.reset_mock() + with self.assertRaisesRegex(ValueError, "backup or intent"): + fleet_node.journal_rejection_audit_finalize(request) + cleanup.assert_not_called() + + class RecreationRecoveryTests(unittest.TestCase): def archive(self, root): case, bucket = "heal.bucket-incarnation-recreation", "rv-heal-0123456789abcdef" From b11c9dc587c929334272db585fe7aa8fe9a795bb Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 23:07:45 +0800 Subject: [PATCH 189/343] fix(validation): bind actual checkpoint and MRF representations --- docs/validation-migration-map.md | 2 + scripts/validation_heal/fleet_node.py | 8 ++- scripts/validation_heal/journal_fault.py | 28 ++++++-- scripts/validation_heal/persistence_mrf.py | 14 +++- test_validation_heal_journal_fault.py | 66 ++++++++++++++++++ test_validation_heal_persistence_mrf.py | 12 +++- .../validation/checkpoint-preserve-order.json | 1 + testdata/validation/mrf-mixed-payload.bin | Bin 0 -> 536 bytes 8 files changed, 117 insertions(+), 14 deletions(-) create mode 100644 testdata/validation/checkpoint-preserve-order.json create mode 100644 testdata/validation/mrf-mixed-payload.bin diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 6ad8fcfc..0fff5fe1 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1963,3 +1963,5 @@ The final stale-generation change passed 608 Heal contracts with eight platform F07 original checkpoint write failure now has a diagnostic executor using the existing four-node `journal-bounded` profile. It fills only the four fixed private journal volumes while the candidate is stopped, then proves a candidate UID and mount namespace witness for the real `ENOSPC` write. The original checkpoint, resume bytes, root owner, exact failed task outcome and source-matched `Heal task failed` event remain durable. The executor pauses and records every owner before releasing only its private filler, performs a distinct Admin rescan with a new token, verifies complete physical, shard and peer S3 evidence, and preserves the first failed attempt separately from any later same-task retry. Cleanup keeps rescue state when a fill ACK, release, evidence export or reconciliation step is incomplete. The branch is implemented but remains outside canonical dispatch until a live run and the remaining reviewed native persistence cuts complete. The final full Heal suite passes 615 tests with eight platform skips; 21 focused persistence contracts, the catalog, 84/301/532 matrix, Python AST, shell syntax and whitespace pass. Two fresh sequential reviews cover correctness, simplicity, test coverage, path security and durability. Frozen process-bound censuses and decreasing root timeout budgets retain their original identity checks. Canonical PASS remains 101/158. F07 diagnostic 38261190390/1 retained FAIL with cleanup FAIL. Checkpoint observation rejected the candidate's actual ordered JSON digest; the MRF windows failed on H1, and H2's first CRC injection was rejected because its actual committed payload also contains system metadata responsibility. No live branch PASS is claimed. The rejected injection left all original MRF bytes and object files unchanged and created no intent or backup. Recovery now accepts only that exact first H2 DB refusal, seven continuous digest-bound create/PUT ACKs, unchanged original MRF and physical censuses, and a cleared member gate. Every node is audited before finalization; each node rechecks its stopped candidate, empty private quarantine, exact MRF digest and full physical scene immediately before removal. Unknown calls, missing ACKs, changed scene, any injection evidence or backup retain the namespace. The original FAIL is never rewritten. + +F07 recovery 38266595027/1 passed independent audit: all four stopped physical scenes and original MRF bytes matched, every node passed its no-injection audit before finalization, and every unit/root cleanup receipt passed. The original failed result is byte-identical and remains FAIL. The checkpoint observer now verifies both source-defined serde_json map representations, including the full server's unified preserve_order feature, while sorting the source-defined ledger arrays before SHA-256. A pinned unmodified candidate fixture rejects counter, cursor, incarnation and digest tampering. The MRF adapter now accepts the exact known internal metadata companions alongside active original business work, rejects foreign business or IAM responsibility, and corrupts one latest committed replica per process with every untouched replica retained as a control. Stopped original bindings replace earlier live sequence observations. The final 621-test Heal suite passes with eight platform skips; 55 focused persistence/MRF contracts, AST, catalog, 84/301/532 matrix and whitespace pass. Two sequential final reviews found no unresolved issue in these ownership, digest, injection and retention changes. H1's unobserved durable MRF window and all original live failures remain retained pending a fresh run; canonical PASS remains 101/158. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index bf20be26..5a0d9add 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1142,9 +1142,11 @@ def journal_operation(request): if request["family"] == "mrf": observed = globals()["mrf"].read_records(state, roots, owner, checked) committed = globals()["mrf"].checkpoint(observed) - require(committed.get("present") is True and committed["intents"] - and all(item["bucket"] == owner["bucket"] and item.get("lifecycle", {}).get("state") == {"state": "active"} - for item in committed["intents"]) + business = module.owned_mrf_intents(committed.get("intents", []), owner["bucket"]) + require(committed.get("present") is True + and all(item.get("lifecycle", {}).get("state") == {"state": "active"} + and item["lifecycle"].get("source_bucket_incarnation_id") + and item["lifecycle"].get("last_operator_acceptance") is None for item in business) and any(row["disk"] == request["disk"] and request["relativePath"] == f".rustfs.sys/.heal-mrf-snapshot.{row['slot']}.bin" for row in committed["replicas"]), "MRF fault lacks an actual committed active owned responsibility on this exact slot") diff --git a/scripts/validation_heal/journal_fault.py b/scripts/validation_heal/journal_fault.py index 822ebdfb..b5c0fed3 100644 --- a/scripts/validation_heal/journal_fault.py +++ b/scripts/validation_heal/journal_fault.py @@ -107,13 +107,30 @@ def checkpoint_value(raw, token, bucket): and len(set(unsigned[field])) == len(unsigned[field]), "checkpoint ledger has an unknown canonical representation") unsigned[field].sort() - # Candidate serde_json::Value uses sorted object keys and UTF-8 strings. - canonical = json.dumps(unsigned, sort_keys=True, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode() - expected = base64.b64encode(hashlib.sha256(canonical).digest()).decode() - require(value.get("integrity_digest") == expected, "original checkpoint has no valid candidate integrity digest") + # serde_json::Value sorts keys by default; the full server's unified + # preserve_order feature instead retains the serialized struct/map order. + # Both builds sort the three ledger arrays before hashing UTF-8 JSON. + digests = {base64.b64encode(hashlib.sha256(json.dumps(unsigned, sort_keys=sorted_keys, + ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode()).digest()).decode() + for sorted_keys in (False, True)} + require(value.get("integrity_digest") in digests, "original checkpoint has no valid candidate integrity digest") return value +def owned_mrf_intents(intents, bucket): + """Bind owned business work while preserving its known internal companions.""" + owned = [item for item in intents if item.get("bucket") == bucket] + system_keys = {"buckets/.rustfs.sys/.metadata.bin", "buckets/.rustfs.sys/.bucket-incarnation"} + require(owned and all(item.get("kind") in {3, 4} and item.get("poolIndex") == 0 + and item.get("setIndex") == 0 for item in owned) + and all(item in owned or item.get("bucket") == ".rustfs.sys" and item.get("kind") == 3 + and item.get("versionId") == "null" and item.get("poolIndex") == 0 and item.get("setIndex") == 0 + and (item.get("key") in system_keys or re.fullmatch(r"scanner/durable-dirty-producer-replay/[0-9a-f]{64}\.json", + item.get("key", "")) is not None) for item in intents), + "MRF payload includes a foreign or unregistered responsibility") + return owned + + def transform(raw, request, owner, mrf, *, earlier=None): family, kind = request["family"], request.get("kind") require(kind in {"truncate", "future-schema", "checksum", "stale-generation"}, "unknown durable-record byte fault") @@ -165,8 +182,7 @@ def transform(raw, request, owner, mrf, *, earlier=None): require(family == "mrf" and kind == "checksum" and re.fullmatch(r"\.heal-mrf-snapshot\.[01]\.bin", path[-1]), "MRF mutation is restricted to the exact actual payload checksum") intents = mrf.decode_journal(raw) - require(intents and all(item["bucket"] == owner["bucket"] and item["kind"] in {3, 4} for item in intents), - "MRF payload includes a foreign or unregistered responsibility") + owned_mrf_intents(intents, owner["bucket"]) # Corrupt a real record CRC without inventing, dropping or retagging ACK identities. return raw[:-1] + bytes([raw[-1] ^ 1]) if kind == "truncate": diff --git a/scripts/validation_heal/persistence_mrf.py b/scripts/validation_heal/persistence_mrf.py index 14ff521d..31d579d6 100644 --- a/scripts/validation_heal/persistence_mrf.py +++ b/scripts/validation_heal/persistence_mrf.py @@ -164,14 +164,22 @@ def run(args, topology, nodes, fixture, candidate): "F07 MRF ACK responsibility changed the original incarnation") fleet.stop() original = observe(fleet, "original-stopped-MRF-records") - mrf.pending_bindings({node: records.checkpoint(value) for node, value in original.items()}, obligations, earliest) + stopped = {node: records.checkpoint(value) for node, value in original.items()} + bindings = mrf.pending_bindings(stopped, obligations, earliest) fleet.record("original-durable-responsibilities", {"obligations": obligations, "bindings": [{"identity": list(key), "owners": value} for key, value in bindings.items()]}) authorities = fleet.snapshot("original-partial-write-physical", quiescent=True) receipts = [] for node, observed in original.items(): - for (disk, path), raw in indexed(observed).items(): - if raw is None or ".heal-mrf-snapshot." not in path: continue + if not any(owner["node"] == node for owners in bindings.values() for owner in owners): continue + selected = stopped[node] + journal.owned_mrf_intents(selected["intents"], bucket) + entries = indexed(observed) + # One committed replica per process is the corruption target; + # every untouched companion/replica remains a retention control. + for replica in selected["replicas"][:1]: + disk, path = replica["disk"], f".rustfs.sys/.heal-mrf-snapshot.{replica['slot']}.bin" + raw = entries[disk, path] require(raw and records.decode_journal(raw), "F07 committed payload has no original responsibility") params = {"family": "mrf", "taskId": None, "disk": disk, "relativePath": path, "faultId": f"F07-mrf-checksum-{nodes.index(node)}-{disk}-{path[-5]}", "kind": "checksum", diff --git a/test_validation_heal_journal_fault.py b/test_validation_heal_journal_fault.py index d408e067..aa922e94 100644 --- a/test_validation_heal_journal_fault.py +++ b/test_validation_heal_journal_fault.py @@ -25,6 +25,72 @@ class DurableRecordFaultTests(unittest.TestCase): incarnation = "22345678-1234-4234-8234-123456789abc" bucket = "rv-heal-0123456789abcdef" + def test_full_server_checkpoint_fixture_preserves_feature_unified_json_order(self): + # Original bytes from candidate 1aa9be87, diagnostic 38261190390/1. + raw = (Path(__file__).parent / "testdata/validation/checkpoint-preserve-order.json").read_bytes() + self.assertEqual(hashlib.sha256(raw).hexdigest(), "af9f1217284fdf9516d111ae8f8cc28b264bec200ba640c5f06c696b2303982b") + value = json.loads(raw) + bucket = next(iter(value["admin"]["buckets"])) + observed = journal.checkpoint_value(raw, value["task_id"], bucket) + self.assertEqual(observed["admin"]["outcome"]["counters"]["processed"], 8) + for mutate in ("counter", "cursor", "incarnation", "digest"): + changed = copy.deepcopy(value) + if mutate == "counter": changed["admin"]["outcome"]["counters"]["processed"] = 9 + elif mutate == "cursor": changed["current_object_index"] += 1 + elif mutate == "incarnation": changed["admin"]["buckets"][bucket] = self.incarnation + else: changed["integrity_digest"] = "A" * 44 + with self.subTest(mutate=mutate), self.assertRaisesRegex(ValueError, "integrity digest"): + journal.checkpoint_value(json.dumps(changed,separators=(",", ":")).encode(), value["task_id"], bucket) + + def test_original_mixed_mrf_payload_retains_only_registered_internal_companions(self): + raw = (Path(__file__).parent / "testdata/validation/mrf-mixed-payload.bin").read_bytes() + self.assertEqual(hashlib.sha256(raw).hexdigest(), "3a6db05b46bb7e3a66a971bb4cc4759b3cd4700d2f31d55f0a19fd023635eb80") + intents = mrf.decode_journal(raw) + bucket = "rv-heal-084db3ab418d55ba" + owned = journal.owned_mrf_intents(intents, bucket) + self.assertEqual(len(owned), 3) + self.assertEqual(len(intents), 6) + request = {"family": "mrf", "kind": "checksum", "relativePath": ".rustfs.sys/.heal-mrf-snapshot.0.bin"} + self.assertEqual(journal.transform(raw, request, {"bucket": bucket}, mrf), raw[:-1] + bytes([raw[-1] ^ 1])) + for change in ("foreign-business", "iam-key", "system-version", "no-business"): + values = copy.deepcopy(intents) + business = next(item for item in values if item["bucket"] == bucket) + system = next(item for item in values if item["bucket"] == ".rustfs.sys") + if change == "foreign-business": business["bucket"] = "foreign" + elif change == "iam-key": system["key"] = "config/iam/users/credentials.json" + elif change == "system-version": system["versionId"] = self.token + else: values = [item for item in values if item["bucket"] != bucket] + with self.subTest(change=change), self.assertRaises(ValueError): journal.owned_mrf_intents(values, bucket) + + def test_mrf_adapter_requires_active_original_business_work_on_the_selected_latest_replica(self): + raw = (Path(__file__).parent / "testdata/validation/mrf-mixed-payload.bin").read_bytes() + bucket = "rv-heal-084db3ab418d55ba" + for change in (None, "held", "unbound", "accepted", "other-replica"): + with tempfile.TemporaryDirectory() as directory, self.subTest(change=change): + root = Path(directory).resolve() + intents = mrf.decode_journal(raw) + for item in intents: + item["lifecycle"] = {"state": {"state": "active"}, "source_bucket_incarnation_id": self.incarnation, + "last_operator_acceptance": None} + business = next(item for item in intents if item["bucket"] == bucket) + if change == "held": business["lifecycle"]["state"] = {"state": "bucket_incarnation_changed"} + elif change == "unbound": business["lifecycle"]["source_bucket_incarnation_id"] = None + elif change == "accepted": business["lifecycle"]["last_operator_acceptance"] = {"operator": "test"} + checkpoint = {"present": True, "intents": intents, "replicas": [{"disk": 0, "slot": 0}]} + backend = types.SimpleNamespace(read_records=lambda *a: {}, checkpoint=lambda *a: checkpoint) + owner = {"runtimeProfile": "mrf-crash", "bucket": bucket, "binarySha256": "b" * 64} + request = {"runId": "gh-12345-1-h2", "action": "journal-apply", "family": "mrf", "taskId": None, + "disk": 0, "relativePath": ".rustfs.sys/.heal-mrf-snapshot.1.bin" if change == "other-replica" else ".rustfs.sys/.heal-mrf-snapshot.0.bin", + "kind": "checksum", "faultId": "F07-mrf-checksum-0-0-0", "expectedBefore": journal.entry(raw)} + with patch.object(fleet_node, "owned", return_value=(root, [root], owner)), patch.object(fleet_node, "stopped"), \ + patch.object(fleet_node, "digest", return_value="b" * 64), patch.object(fleet_node, "fault_module", return_value=fault), \ + patch.dict(fleet_node.__dict__, journal_fault_backend=journal, mrf=backend), \ + patch.object(journal, "install", return_value={"applied": True}) as install: + if change is None: self.assertTrue(fleet_node.journal_operation(request)["applied"]) + else: + with self.assertRaises(ValueError): fleet_node.journal_operation(request) + install.assert_not_called() + def signed(self, value): value = copy.deepcopy(value) value["integrity_digest"] = None diff --git a/test_validation_heal_persistence_mrf.py b/test_validation_heal_persistence_mrf.py index 9b38d7e8..68062b2d 100644 --- a/test_validation_heal_persistence_mrf.py +++ b/test_validation_heal_persistence_mrf.py @@ -3,11 +3,14 @@ import base64 import copy from contextlib import ExitStack +import hashlib import os from pathlib import Path import tempfile from types import SimpleNamespace import unittest +import struct +import zlib from unittest.mock import patch from scripts.validation_heal import journal_fault as journal, persistence_mrf as suite @@ -82,7 +85,12 @@ def test_lost_injection_ack_registers_recovery_before_mutation_and_keeps_namespa with tempfile.TemporaryDirectory() as temp, ExitStack() as stack: root = Path(temp); rc = root / "rc"; rc.write_bytes(b"client") args = SimpleNamespace(evidence=root, rc_bin=rc, tool_directory=root, testing_commit="a" * 40, binary_sha="b" * 64) - original = fixtures.DurableMrfTests() + bucket = "rv-heal-" + hashlib.sha256(b"gh-12345-1-h1").hexdigest()[:16] + class ScopedFixture(fixtures.DurableMrfTests): + def journal(self, **kwargs): + raw = super().journal(**kwargs)[:-4].replace(b"rv-heal-0123456789abcdef", bucket.encode()) + return raw + struct.pack(" 8, "cannot truncate an empty/unknown record") return raw[:len(raw) // 2] diff --git a/scripts/validation_heal/persistence_mrf.py b/scripts/validation_heal/persistence_mrf.py index 31d579d6..ded1826d 100644 --- a/scripts/validation_heal/persistence_mrf.py +++ b/scripts/validation_heal/persistence_mrf.py @@ -55,7 +55,7 @@ def indexed(observed): return result -def retained_corruption(original, current, receipts): +def retained_corruption(original, current, receipts, bucket): require(set(original) == set(current) and len(original) in {1, 4}, "F07 MRF evidence lost a bound physical node") expected, changed = {node: indexed(value) for node, value in original.items()}, set() require(receipts, "F07 MRF corruption has no actual original payload mutation") @@ -70,7 +70,7 @@ def retained_corruption(original, current, receipts): and receipt.get("taskId") is None and receipt.get("kind") == "checksum" and receipt.get("applied") is True and receipt.get("originalBackupRetained") is True and receipt.get("productRepairClaimed") is False and journal.same_json(receipt.get("before"), journal.entry(raw)), "F07 MRF mutation lost its original payload binding") - corrupt = raw[:-1] + bytes([raw[-1] ^ 1]) + corrupt = journal.transform(raw, receipt, {"bucket": bucket}, records) require(journal.same_json(receipt.get("injected"), journal.entry(corrupt)), "F07 MRF corruption changed bytes outside the actual CRC") expected[node][disk, path] = corrupt changed.add(identity) @@ -187,7 +187,7 @@ def run(args, topology, nodes, fixture, candidate): fleet.journal_faults.append((node, params.copy())) receipts.append(fleet.record("actual-original-MRF-CRC-injection", {"node": node, **fleet.remote(node, "journal-apply", **params)})) - retained_corruption(original, observe(fleet, "injected-stopped-MRF-records"), receipts) + retained_corruption(original, observe(fleet, "injected-stopped-MRF-records"), receipts, bucket) window = JournalWindow(fleet, "F07-corrupt-MRF-replay", event_targets={"rustfs::heal::mrf"}) fleet.start() deadline = time.monotonic() + 60 @@ -201,7 +201,7 @@ def run(args, topology, nodes, fixture, candidate): time.sleep(.5) fleet.stop() result["originalResponsibilityBeforeRescan"] = retained_corruption( - original, observe(fleet, "retained-corrupt-MRF-before-rescan"), receipts) + original, observe(fleet, "retained-corrupt-MRF-before-rescan"), receipts, bucket) require(physical_content(authorities) == physical_content(fleet.snapshot("before-separate-rescan", quiescent=True)), "F07 corrupt MRF replay silently changed original object responsibility") fleet.record("release-original-member-gate", fleet.remote(shard["node"], "drive-gate", mode="clear")) @@ -217,7 +217,7 @@ def run(args, topology, nodes, fixture, candidate): repaired = fleet.snapshot("complete-rescan-physical-before-business-reads", quiescent=True) cohorts = mrf.replica_census(repaired, bucket, expected, authorities) result["fullShardIntegrity"] = [verify_cohort(fleet, cohort, "F07-full-shard-proof") for cohort in cohorts] - retained_corruption(original, observe(fleet, "retained-corrupt-MRF-after-rescan"), receipts) + retained_corruption(original, observe(fleet, "retained-corrupt-MRF-after-rescan"), receipts, bucket) fleet.start() for endpoint in binding["endpoints"]: inventory = fleet.record("all-peer-S3", {"endpoint": endpoint, @@ -226,7 +226,7 @@ def run(args, topology, nodes, fixture, candidate): fleet.stop() require(physical_content(repaired) == physical_content(fleet.snapshot("after-business-reads", quiescent=True)), "F07 business reads repaired work not proved by the separate rescan") - result["originalResponsibilityAfterReads"] = retained_corruption(original, observe(fleet, "retained-corrupt-MRF-after-reads"), receipts) + result["originalResponsibilityAfterReads"] = retained_corruption(original, observe(fleet, "retained-corrupt-MRF-after-reads"), receipts, bucket) result.update(verdict="PASS", originalMrfRepairClaimed=False, completePhysicalAndS3ManifestsVerified=True) except (Exception, KeyboardInterrupt) as exc: result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) diff --git a/test_validation_heal_journal_fault.py b/test_validation_heal_journal_fault.py index aa922e94..9d21fa7c 100644 --- a/test_validation_heal_journal_fault.py +++ b/test_validation_heal_journal_fault.py @@ -52,6 +52,17 @@ def test_original_mixed_mrf_payload_retains_only_registered_internal_companions( self.assertEqual(len(intents), 6) request = {"family": "mrf", "kind": "checksum", "relativePath": ".rustfs.sys/.heal-mrf-snapshot.0.bin"} self.assertEqual(journal.transform(raw, request, {"bucket": bucket}, mrf), raw[:-1] + bytes([raw[-1] ^ 1])) + system_bucket, system_key = b".rustfs.sys", b"buckets/.rustfs.sys/.metadata.bin" + body = struct.pack(" Date: Mon, 12 Oct 2026 23:30:45 +0800 Subject: [PATCH 191/343] fix(validation): sign future checkpoint schema with valid digest --- docs/validation-migration-map.md | 2 ++ scripts/validation_heal/journal_fault.py | 35 ++++++++++++++---------- test_validation_heal_journal_fault.py | 25 +++++++++++++++++ 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 554f8493..38956527 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1966,4 +1966,6 @@ F07 diagnostic 38261190390/1 retained FAIL with cleanup FAIL. Checkpoint observa F07 recovery 38266595027/1 passed independent audit: all four stopped physical scenes and original MRF bytes matched, every node passed its no-injection audit before finalization, and every unit/root cleanup receipt passed. The original failed result is byte-identical and remains FAIL. The checkpoint observer now verifies both source-defined serde_json map representations, including the full server's unified preserve_order feature, while sorting the source-defined ledger arrays before SHA-256. A pinned unmodified candidate fixture rejects counter, cursor, incarnation and digest tampering. The MRF adapter now accepts the exact known internal metadata companions alongside active original business work, rejects foreign business or IAM responsibility, and corrupts one latest committed replica per process with every untouched replica retained as a control. Stopped original bindings replace earlier live sequence observations. The final 621-test Heal suite passes with eight platform skips; 55 focused persistence/MRF contracts, AST, catalog, 84/301/532 matrix and whitespace pass. Two sequential final reviews found no unresolved issue in these ownership, digest, injection and retention changes. H1's unobserved durable MRF window and all original live failures remain retained pending a fresh run; canonical PASS remains 101/158. +The F07 future-schema fault now recomputes the embedded integrity digest after changing schema 8 to 9, using the exact map representation verified in the original candidate checkpoint and the source-defined sorted ledger arrays. Original task, incarnation, options, cursor, counters and ledger order remain unchanged; an invalid original cannot be re-signed. Both the unmodified full-server fixture and sorted-map fixture independently verify the future record's digest. Diagnostic 38268715777/1 pins the preceding harness 6db679f and cannot validate this correction. Its future-schema result must not be credited as valid unknown-schema coverage even if its retained-record checks pass. Canonical PASS remains 101/158. + Mixed MRF payload faults select only an owned business record's CRC byte. Record boundaries are independently matched to each decoded original intent digest; internal metadata companions remain byte-identical even when they appear at the end of the payload. UTF-8 keys and variable-length marker purge records have explicit framing coverage. Retention verification recomputes the same owned target from the frozen bucket binding. The final 621 Heal tests pass with eight platform skips after this scope restriction; both final reviews retain exact original bytes, incarnation and recovery boundaries. diff --git a/scripts/validation_heal/journal_fault.py b/scripts/validation_heal/journal_fault.py index 2a59a92b..17845a55 100644 --- a/scripts/validation_heal/journal_fault.py +++ b/scripts/validation_heal/journal_fault.py @@ -92,28 +92,31 @@ def invalid_constant(value): return value -def checkpoint_value(raw, token, bucket): - value = decode_json(raw) - require(value.get("task_id") == task_id(token) and type(value.get("schema_version")) is int and value["schema_version"] == 8, - "checkpoint does not belong to the exact original Admin task") - admin = value.get("admin") or {} - buckets = admin.get("buckets") - require(admin.get("completed") is False and isinstance(buckets, dict) and set(buckets) == {bucket} - and isinstance(buckets[bucket], str) and str(uuid.UUID(buckets[bucket])) == buckets[bucket] - and uuid.UUID(buckets[bucket]).int != 0, "checkpoint has no original owned bucket incarnation") - unsigned = decode_json(raw) +def checkpoint_digest(value, *, sort_keys): + unsigned = dict(value) unsigned["integrity_digest"] = None for field in ("processed_objects", "failed_objects", "skipped_objects"): require(isinstance(unsigned.get(field), list) and all(isinstance(item, str) for item in unsigned[field]) and len(set(unsigned[field])) == len(unsigned[field]), "checkpoint ledger has an unknown canonical representation") - unsigned[field].sort() + unsigned[field] = sorted(unsigned[field]) # serde_json::Value sorts keys by default; the full server's unified # preserve_order feature instead retains the serialized struct/map order. # Both builds sort the three ledger arrays before hashing UTF-8 JSON. - digests = {base64.b64encode(hashlib.sha256(json.dumps(unsigned, sort_keys=sorted_keys, + return base64.b64encode(hashlib.sha256(json.dumps(unsigned, sort_keys=sort_keys, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode()).digest()).decode() - for sorted_keys in (False, True)} + + +def checkpoint_value(raw, token, bucket): + value = decode_json(raw) + require(value.get("task_id") == task_id(token) and type(value.get("schema_version")) is int and value["schema_version"] == 8, + "checkpoint does not belong to the exact original Admin task") + admin = value.get("admin") or {} + buckets = admin.get("buckets") + require(admin.get("completed") is False and isinstance(buckets, dict) and set(buckets) == {bucket} + and isinstance(buckets[bucket], str) and str(uuid.UUID(buckets[bucket])) == buckets[bucket] + and uuid.UUID(buckets[bucket]).int != 0, "checkpoint has no original owned bucket incarnation") + digests = {checkpoint_digest(value, sort_keys=sorted_keys) for sorted_keys in (False, True)} require(value.get("integrity_digest") in digests, "original checkpoint has no valid candidate integrity digest") return value @@ -209,8 +212,12 @@ def transform(raw, request, owner, mrf, *, earlier=None): if kind == "truncate": require(len(raw) > 8, "cannot truncate an empty/unknown record") return raw[:len(raw) // 2] + # Preserve the candidate's digest representation so an unknown schema is + # independently testable without also introducing checksum corruption. + sort_keys = value["integrity_digest"] != checkpoint_digest(value, sort_keys=False) value["schema_version"] = 9 - return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode() + value["integrity_digest"] = checkpoint_digest(value, sort_keys=sort_keys) + return json.dumps(value, sort_keys=sort_keys, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode() def binding(root_fd, request, owner, fault): diff --git a/test_validation_heal_journal_fault.py b/test_validation_heal_journal_fault.py index 9d21fa7c..e7dd2859 100644 --- a/test_validation_heal_journal_fault.py +++ b/test_validation_heal_journal_fault.py @@ -73,6 +73,31 @@ def test_original_mixed_mrf_payload_retains_only_registered_internal_companions( else: values = [item for item in values if item["bucket"] != bucket] with self.subTest(change=change), self.assertRaises(ValueError): journal.owned_mrf_intents(values, bucket) + def test_future_checkpoint_schema_has_valid_candidate_digest_and_original_scope(self): + live = (Path(__file__).parent / "testdata/validation/checkpoint-preserve-order.json").read_bytes() + for raw, sort_keys in ((live, False), (self.checkpoint(), True)): + original = json.loads(raw) + bucket = next(iter(original["admin"]["buckets"])) + request = {"family": "checkpoint", "kind": "future-schema", "taskId": original["task_id"], + "relativePath": journal.names("checkpoint", original["task_id"])[0]} + with self.subTest(sort_keys=sort_keys): + changed = journal.transform(raw, request, {"bucket": bucket}, mrf) + future = json.loads(changed) + self.assertEqual(future["schema_version"], 9) + unsigned = copy.deepcopy(future) + unsigned["integrity_digest"] = None + for field in ("processed_objects", "failed_objects", "skipped_objects"): unsigned[field].sort() + digest = base64.b64encode(hashlib.sha256(json.dumps(unsigned, sort_keys=sort_keys, + ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode()).digest()).decode() + self.assertEqual(future["integrity_digest"], digest) + self.assertNotEqual(future["integrity_digest"], original["integrity_digest"]) + future.update(schema_version=8, integrity_digest=original["integrity_digest"]) + self.assertEqual(future, original) + # Invalid originals must fail before a fault can re-sign them. + original["current_object_index"] += 1 + with self.assertRaisesRegex(ValueError, "integrity digest"): + journal.transform(json.dumps(original).encode(), request, {"bucket": bucket}, mrf) + def test_mrf_adapter_requires_active_original_business_work_on_the_selected_latest_replica(self): raw = (Path(__file__).parent / "testdata/validation/mrf-mixed-payload.bin").read_bytes() bucket = "rv-heal-084db3ab418d55ba" From e62c3cc003f89b7f8e222b9a65cae6534b2135df Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 12 Oct 2026 23:47:15 +0800 Subject: [PATCH 192/343] test(validation): collect existing directory fsync regressions --- docs/validation-migration-map.md | 2 + scripts/validation_heal/persistence_native.py | 153 ++++++++++++++++++ test_validation_heal_native.py | 94 +++++++++++ 3 files changed, 249 insertions(+) create mode 100644 scripts/validation_heal/persistence_native.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 38956527..2d38445b 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1968,4 +1968,6 @@ F07 recovery 38266595027/1 passed independent audit: all four stopped physical s The F07 future-schema fault now recomputes the embedded integrity digest after changing schema 8 to 9, using the exact map representation verified in the original candidate checkpoint and the source-defined sorted ledger arrays. Original task, incarnation, options, cursor, counters and ledger order remain unchanged; an invalid original cannot be re-signed. Both the unmodified full-server fixture and sorted-map fixture independently verify the future record's digest. Diagnostic 38268715777/1 pins the preceding harness 6db679f and cannot validate this correction. Its future-schema result must not be credited as valid unknown-schema coverage even if its retained-record checks pass. Canonical PASS remains 101/158. +The fixed `1aa9` candidate already contains six deterministic directory-fsync failure regressions in `disk::local::test`, and the verified original ecstore binary inventory includes all six exact names. `persistence_native.py` now executes them as an additive F07 prerequisite using that verified full candidate binary and its original nine-test suite evidence. The collector retains original one-test logs, exact commands, candidate/tree/lock/binary/run/attempt/helper identity, its own digest and strict scope limitations; source, command, artifact, inventory and log substitutions are refused. The hosted tools lane archives the supplemental evidence separately. These tests cover actual post-rename or post-delete directory-fsync errors, manifest `StorageFull`, recovery-anchor preservation and the relaxed-mode system metadata durability rule. They do not cover individual file-fsync interruptions, physical power loss, all checkpoint/commit cuts or canonical fleet branches. Hosted execution and independent original artifact audit remain pending; `requiredNativeCoverageComplete` stays false. + Mixed MRF payload faults select only an owned business record's CRC byte. Record boundaries are independently matched to each decoded original intent digest; internal metadata companions remain byte-identical even when they appear at the end of the payload. UTF-8 keys and variable-length marker purge records have explicit framing coverage. Retention verification recomputes the same owned target from the frozen bucket binding. The final 621 Heal tests pass with eight platform skips after this scope restriction; both final reviews retain exact original bytes, incarnation and recovery boundaries. diff --git a/scripts/validation_heal/persistence_native.py b/scripts/validation_heal/persistence_native.py new file mode 100644 index 00000000..9095da55 --- /dev/null +++ b/scripts/validation_heal/persistence_native.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Run existing F07 directory-fsync regressions from the verified candidate binary.""" + +import argparse +import json +import os +import re +from pathlib import Path +import subprocess +import time + +try: + from . import native_regressions as native +except ImportError: + import native_regressions as native + + +PREFIX = "disk::local::test::" +REGRESSIONS = { + "cas-update-directory-sync": ( + "conditional_file_update_dir_fsync_failure_restores_previous_bytes", + "post-rename directory fsync failure is commit-ambiguous and restores the previous owner bytes"), + "cas-create-directory-sync": ( + "conditional_file_update_dir_fsync_failure_removes_new_file_without_anchor", + "post-rename directory fsync failure removes an uncommitted first owner"), + "cas-delete-directory-sync": ( + "conditional_file_delete_dir_fsync_failure_restores_previous_bytes", + "post-delete directory fsync failure restores the previous owner bytes"), + "mrf-manifest-directory-sync": ( + "conditional_mrf_manifest_dir_fsync_failure_keeps_recovery_anchors", + "actual system-metadata directory fsync is required even in relaxed mode; failed replacement and first creation retain recovery anchors"), + "mrf-manifest-storage-full": ( + "conditional_mrf_manifest_storage_full_keeps_recovery_anchors", + "StorageFull at the actual manifest directory fsync preserves the previous manifest and legacy journal"), + "mrf-delete-storage-full": ( + "conditional_mrf_manifest_storage_full_delete_keeps_recovery_anchors", + "StorageFull at cleanup directory fsync restores the committed manifest and preserves the legacy journal"), +} +LIMITS = { + "directoryFsyncFailureTestScope": True, + "individualFileFsyncCovered": False, + "physicalPowerLossClaimed": False, + "requiredNativeCoverageComplete": False, + "canonicalCasePassClaimed": False, + "fleetPassClaimed": False, + "emittedBarrierLogsClaimed": False, +} + + +def binding(directory, evidence): + manifest, binary, _ = native.verify_manifest(directory) + native.require(manifest["candidateCommit"] == native.INCARNATION_CANDIDATE, + "F07 directory fsync tests are reviewed only for the exact immutable candidate") + native.verify_evidence(directory, evidence, {"source_sha": manifest["candidateCommit"]}, + manifest["runId"], manifest["runAttempt"], manifest["testingCommit"]) + listing = (directory / "test-inventory.log").read_text() + for suffix, _ in REGRESSIONS.values(): native.exact_listing(listing, PREFIX + suffix) + return manifest, binary + + +def identity(manifest, testing_commit): + native.require(isinstance(testing_commit, str) and re.fullmatch(r"[0-9a-f]{40}", testing_commit), + "F07 collector requires an exact independent harness commit") + return {"candidateCommit": manifest["candidateCommit"], "candidateSourceTree": manifest["candidateSourceTree"], + "candidateCargoLockSha256": manifest["candidateCargoLockSha256"], + "binarySha256": manifest["binarySha256"], "runId": manifest["runId"], + "runAttempt": manifest["runAttempt"], "nativeTestingCommit": manifest["testingCommit"], + "collectorCommit": testing_commit, + "collectorSha256": native.sha(Path(__file__)), **LIMITS} + + +def run(directory, native_evidence, out, testing_commit): + manifest, binary = binding(directory, native_evidence) + proof_identity = identity(manifest, testing_commit) + native.require(not out.exists(), "F07 native evidence must be fresh") + out.mkdir(parents=True) + listing = subprocess.run([str(binary), "--list"], capture_output=True, text=True, timeout=60) + (out / "runtime-inventory.log").write_text(listing.stdout + listing.stderr) + native.require(listing.returncode == 0, "F07 native runtime inventory failed") + rows = [] + for key, (suffix, assertion) in REGRESSIONS.items(): + name = PREFIX + suffix + native.exact_listing(listing.stdout, name) + native.require(not binary.is_symlink() and native.sha(binary) == manifest["binarySha256"], + "F07 native executable changed before exact execution") + command = native.exact_command(binary, name, manifest["candidateCommit"]) + row = {**proof_identity, "regression": key, "test": name, "sourceAssertion": assertion, + "command": command, "runtimeEnvironment": manifest["runtimeEnvironment"]} + started = time.monotonic_ns() + try: + child = subprocess.run(command, capture_output=True, text=True, timeout=120, + env={**os.environ, **manifest["runtimeEnvironment"]}) + output = child.stdout + child.stderr + (out / (key + ".log")).write_text(output) + row["exitCode"] = child.returncode + native.test_result(output, name, child.returncode) + row["verdict"] = "PASS" + except subprocess.TimeoutExpired as error: + parts = [part.decode(errors="replace") if isinstance(part, bytes) else part or "" + for part in (error.stdout, error.stderr)] + (out / (key + ".log")).write_text("".join(parts)) + row.update(verdict="FAIL", reason="fixed 120-second F07 native deadline exceeded") + except (OSError, ValueError) as error: + if not (out / (key + ".log")).exists(): (out / (key + ".log")).write_text(str(error)) + row.update(verdict="FAIL", reason=str(error)) + row["elapsedNanoseconds"] = time.monotonic_ns() - started + native.save(out / (key + ".json"), row) + rows.append(row) + complete = all(row["verdict"] == "PASS" for row in rows) + native.save(out / "persistence-native-summary.json", {**proof_identity, "complete": complete, "results": rows}) + native.require(complete, "F07 native fsync failures remain in original evidence") + + +def verify(directory, native_evidence, evidence, testing_commit): + manifest, binary = binding(directory, native_evidence) + summary = json.loads((evidence / "persistence-native-summary.json").read_text()) + rows = summary.get("results", []) + expected_identity = identity(manifest, testing_commit) + native.require(all(type(summary.get(key)) is type(value) and summary[key] == value + for key, value in expected_identity.items()) and summary.get("complete") is True + and len(rows) == len(REGRESSIONS) and {row.get("regression") for row in rows} == set(REGRESSIONS), + "F07 native summary changed source, scope, limitations or collector identity") + listings = [(directory / "test-inventory.log").read_text(), (evidence / "runtime-inventory.log").read_text()] + for row in rows: + key = row["regression"] + suffix, assertion = REGRESSIONS[key] + name = PREFIX + suffix + command = row.get("command", []) + native.require(row == json.loads((evidence / (key + ".json")).read_text()) and row.get("verdict") == "PASS" + and all(type(row.get(k)) is type(v) and row[k] == v for k, v in expected_identity.items()) + and row.get("test") == name and row.get("sourceAssertion") == assertion + and len(command) == 5 and Path(command[0]).name == binary.name + and command[1:] == native.exact_command(binary, name, manifest["candidateCommit"])[1:] + and row.get("runtimeEnvironment") == manifest["runtimeEnvironment"], + "F07 native result changed exact test, command, source assertion or binary") + for listing in listings: native.exact_listing(listing, name) + native.test_result((evidence / (key + ".log")).read_text(), name, row.get("exitCode")) + return {**expected_identity, "exactTests": len(rows), "complete": True} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("run", "verify")) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--native-evidence", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + if args.command == "run": run(args.directory, args.native_evidence, args.out, args.testing_commit) + else: print(json.dumps(verify(args.directory, args.native_evidence, args.out, args.testing_commit), indent=2)) + + +if __name__ == "__main__": main() diff --git a/test_validation_heal_native.py b/test_validation_heal_native.py index a30f9186..6dbe91b2 100644 --- a/test_validation_heal_native.py +++ b/test_validation_heal_native.py @@ -7,6 +7,7 @@ import subprocess from scripts.validation_heal import native_regressions as native +from scripts.validation_heal import persistence_native as persistence def evidence_fixture(root, commit=native.INCARNATION_CANDIDATE): @@ -101,6 +102,99 @@ def test_wrong_source_modified_contract_or_collector_fail_before_execution(self) self.assertFalse((directory / 'evidence').exists()) +class PersistenceNativeTests(unittest.TestCase): + def fixture(self, root): + tools, original, _, _ = evidence_fixture(root) + inventory = tools / 'test-inventory.log' + inventory.write_text(inventory.read_text() + '\n' + '\n'.join( + persistence.PREFIX + suffix + ': test' for suffix, _ in persistence.REGRESSIONS.values())) + return tools, original, root / 'persistence' + + def execute(self, tools, fail=None): + def child(command, **kwargs): + if command[-1] == '--list': + return subprocess.CompletedProcess(command, 0, (tools / 'test-inventory.log').read_text(), '') + self.assertEqual(command[2:], ['--exact', '--show-output', '--test-threads=1']) + self.assertEqual(kwargs['env']['RUST_MIN_STACK'], '33554432') + if fail == 'timeout': raise subprocess.TimeoutExpired(command, 120, output=b'original timeout output\n') + name = command[1] + output = f'running 1 test\ntest {name} ... ok\ntest result: ok. 1 passed; 0 failed; 0 ignored;\n' + if fail == 'zero-test': output = 'running 0 tests\ntest result: ok. 0 passed; 0 failed; 0 ignored;\n' + return subprocess.CompletedProcess(command, 0, output, '') + return child + + def run_fixture(self, root): + tools, original, out = self.fixture(root) + with patch.object(persistence.subprocess, 'run', side_effect=self.execute(tools)): + persistence.run(tools, original, out, 'b' * 40) + return tools, original, out + + def test_executes_existing_exact_tests_without_claiming_complete_cuts(self): + with tempfile.TemporaryDirectory() as temp: + tools, original, out = self.run_fixture(Path(temp)) + proof = persistence.verify(tools, original, out, 'b' * 40) + self.assertEqual(proof['exactTests'], 6) + self.assertEqual(proof['nativeTestingCommit'], 'a' * 40) + self.assertEqual(proof['collectorCommit'], 'b' * 40) + for flag in ('requiredNativeCoverageComplete', 'individualFileFsyncCovered', 'physicalPowerLossClaimed', + 'canonicalCasePassClaimed', 'fleetPassClaimed', 'emittedBarrierLogsClaimed'): + self.assertIs(proof[flag], False) + + def test_unverified_original_suite_cannot_launch_supplement(self): + for change in ('source', 'binary', 'original-log', 'missing-test'): + with tempfile.TemporaryDirectory() as temp, self.subTest(change=change): + tools, original, out = self.fixture(Path(temp)) + if change == 'source': + path = tools / 'native-regressions.json' + value = json.loads(path.read_text()); value['candidateCommit'] = native.CANDIDATE + path.write_text(json.dumps(value)) + elif change == 'binary': (tools / 'ecstore-native-tests').write_bytes(b'changed executable') + elif change == 'original-log': (original / 'incarnation-final-rename.log').write_text('running 0 tests') + else: + path = tools / 'test-inventory.log' + path.write_text(path.read_text().replace(persistence.PREFIX + next(iter(persistence.REGRESSIONS.values()))[0] + ': test', '')) + with patch.object(persistence.subprocess, 'run') as execute, self.assertRaises(ValueError): + persistence.run(tools, original, out, 'b' * 40) + execute.assert_not_called() + self.assertFalse(out.exists()) + + def test_zero_test_and_timeout_keep_original_failed_diagnostics(self): + for failure in ('zero-test', 'timeout'): + with tempfile.TemporaryDirectory() as temp, self.subTest(failure=failure): + tools, original, out = self.fixture(Path(temp)) + with patch.object(persistence.subprocess, 'run', side_effect=self.execute(tools, failure)), self.assertRaises(ValueError): + persistence.run(tools, original, out, 'b' * 40) + summary = json.loads((out / 'persistence-native-summary.json').read_text()) + self.assertIs(summary['complete'], False) + self.assertEqual(len(summary['results']), 6) + self.assertTrue(all(row['verdict'] == 'FAIL' for row in summary['results'])) + self.assertIn('original timeout output' if failure == 'timeout' else 'running 0 tests', + (out / 'mrf-manifest-directory-sync.log').read_text()) + with self.assertRaises(ValueError): persistence.verify(tools, original, out, 'b' * 40) + + def test_tampered_source_commands_logs_or_scope_cannot_certify_fsync(self): + for change in ('candidate', 'attempt', 'collector', 'powerloss', 'complete-cuts', 'duplicate', 'command', 'assertion', 'log', 'inventory'): + with tempfile.TemporaryDirectory() as temp, self.subTest(change=change): + tools, original, out = self.run_fixture(Path(temp)) + path = out / 'persistence-native-summary.json' + summary = json.loads(path.read_text()) + if change == 'candidate': summary['candidateCommit'] = 'f' * 40 + elif change == 'attempt': summary['runAttempt'] = '2' + elif change == 'collector': summary['collectorSha256'] = '0' * 64 + elif change == 'powerloss': summary['physicalPowerLossClaimed'] = True + elif change == 'complete-cuts': summary['requiredNativeCoverageComplete'] = True + elif change == 'duplicate': summary['results'][0] = copy.deepcopy(summary['results'][1]) + elif change in ('command', 'assertion'): + row = summary['results'][0] + if change == 'command': row['command'][2] = '--ignored' + else: row['sourceAssertion'] = 'all fsync and powerloss cuts' + (out / (row['regression'] + '.json')).write_text(json.dumps(row)) + elif change == 'log': (out / 'cas-update-directory-sync.log').write_text('running 0 tests') + else: (out / 'runtime-inventory.log').write_text('0 tests') + path.write_text(json.dumps(summary)) + with self.assertRaises(ValueError): persistence.verify(tools, original, out, 'b' * 40) + + class NativeEvidenceTests(unittest.TestCase): def verify(self, fixture, **changes): tools, evidence, candidate, helper = fixture From c0d69a3090afed46503a5be4d798d33a1ed8adae Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 13 Oct 2026 00:24:54 +0800 Subject: [PATCH 193/343] fix(validation): preserve durable journal observation windows --- docs/validation-migration-map.md | 7 ++++ scripts/validation_heal/checkpoint.py | 5 ++- scripts/validation_heal/fleet_node.py | 8 ++++- scripts/validation_heal/healthy.py | 8 ++++- scripts/validation_heal/journal_fault.py | 8 +++-- scripts/validation_heal/persistence_mrf.py | 14 ++++++++ test_validation_heal_checkpoint.py | 26 +++++++++++++-- test_validation_heal_journal_fault.py | 38 ++++++++++++++++++++++ test_validation_heal_persistence_mrf.py | 37 ++++++++++++++++++--- 9 files changed, 136 insertions(+), 15 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 2d38445b..c00c8ead 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1971,3 +1971,10 @@ The F07 future-schema fault now recomputes the embedded integrity digest after c The fixed `1aa9` candidate already contains six deterministic directory-fsync failure regressions in `disk::local::test`, and the verified original ecstore binary inventory includes all six exact names. `persistence_native.py` now executes them as an additive F07 prerequisite using that verified full candidate binary and its original nine-test suite evidence. The collector retains original one-test logs, exact commands, candidate/tree/lock/binary/run/attempt/helper identity, its own digest and strict scope limitations; source, command, artifact, inventory and log substitutions are refused. The hosted tools lane archives the supplemental evidence separately. These tests cover actual post-rename or post-delete directory-fsync errors, manifest `StorageFull`, recovery-anchor preservation and the relaxed-mode system metadata durability rule. They do not cover individual file-fsync interruptions, physical power loss, all checkpoint/commit cuts or canonical fleet branches. Hosted execution and independent original artifact audit remain pending; `requiredNativeCoverageComplete` stays false. Mixed MRF payload faults select only an owned business record's CRC byte. Record boundaries are independently matched to each decoded original intent digest; internal metadata companions remain byte-identical even when they appear at the end of the payload. UTF-8 keys and variable-length marker purge records have explicit framing coverage. Retention verification recomputes the same owned target from the frozen bucket binding. The final 621 Heal tests pass with eight platform skips after this scope restriction; both final reviews retain exact original bytes, incarnation and recovery boundaries. + + +F07 diagnostic `38268715777/1` completed with FAIL and clean finalization. Independent audit verified all twenty live branch instances, fifty node cleanup receipts and 240 embedded original-byte witnesses; no diagnostic branch or canonical PASS is credited. The checkpoint task can finish in roughly three seconds: an extra live root census missed its actual checkpoint window. The observer now freezes the exact checkpoint process before reading and validating the runnable root, and rechecks both original records before crash/mutation. Live read/capture races retain a structured `NOT_OBSERVED` refusal across SSH within the existing deadline; stopped operations, changed process identity and path/ownership failures remain fatal. MRF corruption restart now re-arms its same member permission gate while stopped and reactivates it after readiness, avoiding disk initialization with an already active chmod denial. Original corruption, physical scene and responsibility checks still gate all later steps. Seventy-four focused contracts and 631 Heal contracts pass with eight platform skips; two fresh sequential reviews found no unresolved findings. A new immutable diagnostic run is required; H1's missing committed MRF window remains unverified. + +Hosted tools `38272891095/1` passed the original nine exact storage regressions and all six supplemental directory-fsync tests. Independent consumers verified candidate `1aa9`, source tree/lock and source-file Git objects, executable/collector identity, exact one-test commands and original logs. These are scoped native prerequisites; individual file fsync, physical power loss, complete required native cuts and canonical PASS remain unclaimed. Isolated package publication `38271092352/1` also passed: fresh hosted publisher bytes equal the opaque build artifact, the complete public DEB digest matches schema 3, and the existing candidate resolver accepts its source/run/attempt binding. Current fleet candidate `38240858236/1` remains fixed. + +B07 fixed-candidate replay `38275626918/1` remains FAIL with clean finalization. Independent audit binds all four H1/H2 DB/DV instances to `1aa9` source, package/binary and helper `e62c3cc`. H1 GET still returns `500/InternalError`: reader setup reports one available shard against two required shards and reduces missing local files to a generic I/O error. H2 DB GET returns `503/SlowDownRead`, but Admin completion records `deferred/dangling_delete_grace`, zero failed objects and completed execution instead of the frozen failed responsibility. H2 DV startup misses readiness with system metadata read quorum unavailable. Original failures remain retained. Canonical PASS stays 101/158. diff --git a/scripts/validation_heal/checkpoint.py b/scripts/validation_heal/checkpoint.py index 162e5781..0037a67c 100644 --- a/scripts/validation_heal/checkpoint.py +++ b/scripts/validation_heal/checkpoint.py @@ -201,7 +201,6 @@ def freeze(fleet, token, bucket, *, timeout=60, earlier=None): try: owner = owned_checkpoint(records(fleet, token, "Admin-checkpoint-live"), token, bucket) if earlier is not None: after_generation(owner, earlier) - root = owned_root_admission(records(fleet, token, "Admin-root-live", family="root"), token) break except ValueError as exc: if not str(exc).startswith("NOT_OBSERVED:"): raise @@ -219,9 +218,9 @@ def freeze(fleet, token, bucket, *, timeout=60, earlier=None): require((current["node"], current["disk"], current["bucketIncarnationId"]) == (owner["node"], owner["disk"], owner["bucketIncarnationId"]), "checkpoint owner changed during freeze") if earlier is not None: after_generation(current, earlier) + # Reading another live census can outlast the actual checkpoint window. + # Validate the runnable root only after freezing its original checkpoint owner. frozen_root = owned_root_admission(records(fleet, token, "Admin-root-frozen", family="root"), token) - require(all(frozen_root[name] == root[name] for name in ("node", "disk", "relativePath", "size", "sha256")), - "original durable root admission changed during its checkpoint freeze") return {**current, "rootAdmission": frozen_root}, identity diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 5a0d9add..d9d12caf 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1460,4 +1460,10 @@ def main(request): actions.update({"format-export": format_export, "foreign-format": foreign_format, "restore-replacement": restore_replacement}) actions.update({"process": process, "process-metrics": process_metrics, "process-signal": process_signal, "process-graceful": process_graceful, "process-crash": process_crash}) require(request.get("action") in actions, "unsupported node operation") - print(json.dumps(actions[request["action"]](request))) + try: + result = actions[request["action"]](request) + except ValueError as exc: + if request["action"] not in {"journal-observe", "journal-capture"} or not str(exc).startswith("NOT_OBSERVED:"): + raise + result = {"observationUnavailable": str(exc)} + print(json.dumps(result)) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index c70f0c2d..772698f7 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -240,7 +240,13 @@ def remote(self, node, action, **params): result = subprocess.run([*self.ssh(node), "sudo -n python3 -"], input=source, capture_output=True, text=True, timeout=timeout) require(result.returncode == 0, f"Heal node {node} {action} failed: {result.stderr[-1200:]}") - return json.loads(result.stdout) + response = json.loads(result.stdout) + if action in {"journal-observe", "journal-capture"} and isinstance(response, dict) and "observationUnavailable" in response: + message = response["observationUnavailable"] + require(set(response) == {"observationUnavailable"} and isinstance(message, str) + and message.startswith("NOT_OBSERVED:"), "invalid durable-record observation refusal") + raise ValueError(message) + return response def snapshot(self, label, *, quiescent=False, metadata_faults=None): return self.record(label, {node: self.remote(node, "snapshot", quiescent=quiescent, diff --git a/scripts/validation_heal/journal_fault.py b/scripts/validation_heal/journal_fault.py index 17845a55..cfe40dab 100644 --- a/scripts/validation_heal/journal_fault.py +++ b/scripts/validation_heal/journal_fault.py @@ -72,8 +72,12 @@ def census(roots, request, fault, *, quiescent=True): def stable_census(roots, request, fault): - first = census(roots, request, fault, quiescent=False) - second = census(roots, request, fault, quiescent=False) + try: + first = census(roots, request, fault, quiescent=False) + second = census(roots, request, fault, quiescent=False) + except ValueError as exc: + if str(exc) != "file changed while reading": raise + raise ValueError("NOT_OBSERVED: exact durable record changed during a bounded read") from exc require(first == second, "NOT_OBSERVED: exact durable records changed between the two bounded reads") return {**second, "stable": True, "independentReads": 2} diff --git a/scripts/validation_heal/persistence_mrf.py b/scripts/validation_heal/persistence_mrf.py index ded1826d..e770d231 100644 --- a/scripts/validation_heal/persistence_mrf.py +++ b/scripts/validation_heal/persistence_mrf.py @@ -189,7 +189,21 @@ def run(args, topology, nodes, fixture, candidate): **fleet.remote(node, "journal-apply", **params)})) retained_corruption(original, observe(fleet, "injected-stopped-MRF-records"), receipts, bucket) window = JournalWindow(fleet, "F07-corrupt-MRF-replay", event_targets={"rustfs::heal::mrf"}) + # An active chmod gate prevents the candidate's disk initialization. + # All services are stopped here; re-arm it before startup and activate + # it again only after readiness, preserving the original denied member. + released = fleet.record("stopped-member-permission-reset", fleet.remote(shard["node"], "drive-gate", mode="clear")) + require(released.get("state") == "CLEAR" and released.get("originalPermissionsRestored") is True, + "F07 denied member was not restored while stopped") + rearmed = fleet.record("stopped-member-gate-rearmed", fleet.remote(shard["node"], "drive-gate", + disk=shard["disk"], mode="permission-denied")) + require(rearmed.get("state") == "ARMED" and rearmed.get("phase") == "armed" + and type(rearmed.get("disk")) is int and rearmed["disk"] == shard["disk"], + "F07 restart gate changed its original denied member") fleet.start() + denied = fleet.record("restarted-original-member-denial", fleet.remote(shard["node"], "drive-gate-probe")) + require(denied.get("readProbe", {}).get("errno") == 13 and denied.get("errno") == 13 + and denied.get("dacBypassRemoved") is True, "F07 restart lost its actual member read/write denial") deadline = time.monotonic() + 60 while True: try: diff --git a/test_validation_heal_checkpoint.py b/test_validation_heal_checkpoint.py index 15b03712..8f53278a 100644 --- a/test_validation_heal_checkpoint.py +++ b/test_validation_heal_checkpoint.py @@ -219,17 +219,37 @@ def records(fleet, token, label, **params): return roots if params.get('family') self.assertTrue(stopped['rootContinuity']['budgetNotRefunded']) self.assertEqual(fleet.frozen_processes, {}) - def test_root_drift_after_freeze_retains_the_original_paused_process_for_cleanup(self): + def test_freeze_precedes_root_observation_to_keep_the_actual_checkpoint_window(self): + from scripts.validation_heal.fleet_node import PROCESS_FIELDS + observed, _, _ = self.fixture(); roots, _ = self.root_fixture() + identity = {name: name for name in PROCESS_FIELDS}; identity.update(pid=50, state='S') + actions = [] + fleet = SimpleNamespace(frozen_processes={}, record=lambda label, value: value) + def remote(node, action, **params): + actions.append(action) + return {**identity, 'state': 'T' if 'process-signal' in actions else 'S'} + fleet.remote = remote + def records(fleet, token, label, **params): + if params.get('family') == 'root': + self.assertIn('process-signal', actions) + self.assertEqual(label, 'Admin-root-frozen') + return roots + return observed + with patch.object(reader, 'records', side_effect=records): + frozen, _ = reader.freeze(fleet, TOKEN, BUCKET) + self.assertTrue(frozen['rootAdmission']['originalDurableResponsibility']) + + def test_foreign_root_after_freeze_retains_the_original_paused_process_for_cleanup(self): from scripts.validation_heal.fleet_node import PROCESS_FIELDS observed, _, _ = self.fixture(); roots, value = self.root_fixture() identity = {name: name for name in PROCESS_FIELDS}; identity.update(pid=50, state='T') fleet = SimpleNamespace(frozen_processes={}, record=lambda label, value: value, remote=lambda *args, **kwargs: identity) def records(fleet, token, label, **params): if label == 'Admin-root-frozen': - changed = copy.deepcopy(roots); altered = copy.deepcopy(value); altered['created_at']['secs_since_epoch'] += 1 + changed = copy.deepcopy(roots); altered = copy.deepcopy(value); altered['task_id'] = INCARNATION changed['node']['records'][3].update(encoded(altered)); return changed return roots if params.get('family') == 'root' else observed - with patch.object(reader, 'records', side_effect=records), self.assertRaisesRegex(ValueError, 'root admission changed'): + with patch.object(reader, 'records', side_effect=records), self.assertRaisesRegex(ValueError, 'original explicit Admin'): reader.freeze(fleet, TOKEN, BUCKET) self.assertEqual(fleet.frozen_processes, {'node': identity}) diff --git a/test_validation_heal_journal_fault.py b/test_validation_heal_journal_fault.py index e7dd2859..11111c22 100644 --- a/test_validation_heal_journal_fault.py +++ b/test_validation_heal_journal_fault.py @@ -3,6 +3,8 @@ import base64 import copy import hashlib +import contextlib +import io import json import os from pathlib import Path @@ -17,6 +19,7 @@ from scripts.validation_heal import journal_fault as journal, heal_fault as fault, mrf_records as mrf from scripts.validation_heal.healthy import Fleet from scripts.validation_heal import fleet_node +from scripts.validation_heal import healthy from scripts.validation_heal.checkpoint import OPTIONS @@ -25,6 +28,41 @@ class DurableRecordFaultTests(unittest.TestCase): incarnation = "22345678-1234-4234-8234-123456789abc" bucket = "rv-heal-0123456789abcdef" + def test_live_read_race_is_retryable_but_stopped_read_and_path_failures_are_not(self): + for message in ('file changed while reading', 'file bind-mount escape', 'expected bounded regular non-hardlinked file'): + with self.subTest(message=message), patch.object(journal, 'census', side_effect=ValueError(message)): + expected = 'NOT_OBSERVED:' if message == 'file changed while reading' else message + with self.assertRaisesRegex(ValueError, expected): journal.stable_census([], {}, fault) + with self.assertRaisesRegex(ValueError, message): journal.census([], {}, fault) + + def test_only_live_observation_refusals_survive_the_remote_boundary_as_retryable(self): + refusal = 'NOT_OBSERVED: exact durable records changed between the two bounded reads' + for action in ('journal-observe', 'journal-capture', 'journal-apply', 'journal-census'): + output = io.StringIO() + with self.subTest(action=action), patch.object(fleet_node.os, 'geteuid', return_value=0), \ + patch.object(fleet_node, 'journal_operation', side_effect=ValueError(refusal)), contextlib.redirect_stdout(output): + if action in ('journal-observe', 'journal-capture'): + fleet_node.main({'action': action}) + self.assertEqual(json.loads(output.getvalue()), {'observationUnavailable': refusal}) + else: + with self.assertRaisesRegex(ValueError, 'NOT_OBSERVED:'): fleet_node.main({'action': action}) + with patch.object(fleet_node.os, 'geteuid', return_value=0), \ + patch.object(fleet_node, 'journal_operation', side_effect=ValueError('service restarted while observing exact durable records')): + with self.assertRaisesRegex(ValueError, 'service restarted'): fleet_node.main({'action': 'journal-observe'}) + with tempfile.TemporaryDirectory() as temp: + fleet = Fleet(['node'], Path(temp), 'gh-12345-1-h1') + for action in ('journal-observe', 'journal-capture'): + response = types.SimpleNamespace(returncode=0, stdout=json.dumps({'observationUnavailable': refusal}), stderr='') + with self.subTest(action=action), patch.object(healthy.subprocess, 'run', return_value=response): + with self.assertRaisesRegex(ValueError, '^NOT_OBSERVED:'): fleet.remote('node', action) + for body in ({'observationUnavailable': 'other'}, {'observationUnavailable': refusal, 'records': []}, {'observationUnavailable': None}): + response = types.SimpleNamespace(returncode=0, stdout=json.dumps(body), stderr='') + with self.subTest(body=body), patch.object(healthy.subprocess, 'run', return_value=response): + with self.assertRaisesRegex(ValueError, 'invalid durable-record'): fleet.remote('node', 'journal-observe') + response = types.SimpleNamespace(returncode=1, stdout=json.dumps({'observationUnavailable': refusal}), stderr='failed') + with patch.object(healthy.subprocess, 'run', return_value=response): + with self.assertRaisesRegex(ValueError, '^Heal node'): fleet.remote('node', 'journal-observe') + def test_full_server_checkpoint_fixture_preserves_feature_unified_json_order(self): # Original bytes from candidate 1aa9be87, diagnostic 38261190390/1. raw = (Path(__file__).parent / "testdata/validation/checkpoint-preserve-order.json").read_bytes() diff --git a/test_validation_heal_persistence_mrf.py b/test_validation_heal_persistence_mrf.py index f6d0d2c1..b6425fad 100644 --- a/test_validation_heal_persistence_mrf.py +++ b/test_validation_heal_persistence_mrf.py @@ -82,6 +82,19 @@ def test_original_corruption_diagnostic_is_bound_to_each_mutated_node_and_typed_ with self.subTest(change=change), self.assertRaises(ValueError): suite.diagnostics({"node": [value]}, nodes) def test_lost_injection_ack_registers_recovery_before_mutation_and_keeps_namespace(self): + self.run_until_corruption_failure('lost-ack') + + def test_corrupt_replay_rearms_the_same_stopped_gate_before_startup(self): + actions = self.run_until_corruption_failure('replay') + replay = actions[actions.index('journal-apply') + 1:] + self.assertEqual(replay[:4], ['clear', 'arm', 'start', 'drive-gate-probe']) + + def test_failed_stopped_permission_reset_keeps_rescue_and_does_not_restart(self): + actions = self.run_until_corruption_failure('bad-reset') + replay = actions[actions.index('journal-apply') + 1:] + self.assertNotIn('start', replay) + + def run_until_corruption_failure(self, phase): with tempfile.TemporaryDirectory() as temp, ExitStack() as stack: root = Path(temp); rc = root / "rc"; rc.write_bytes(b"client") args = SimpleNamespace(evidence=root, rc_bin=rc, tool_directory=root, testing_commit="a" * 40, binary_sha="b" * 64) @@ -92,18 +105,27 @@ def journal(self, **kwargs): return raw + struct.pack(" Date: Tue, 13 Oct 2026 00:59:51 +0800 Subject: [PATCH 194/343] fix(validation): audit advanced MRF fixtures before cleanup --- docs/validation-migration-map.md | 5 + scripts/validation_heal/fleet_node.py | 65 ++++++++++ scripts/validation_heal/healthy.py | 8 +- .../validation_heal/recover_environment.py | 115 ++++++++++++++++- test_validation_heal_recovery.py | 117 ++++++++++++++++++ 5 files changed, 304 insertions(+), 6 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index c00c8ead..3c6289a4 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1978,3 +1978,8 @@ F07 diagnostic `38268715777/1` completed with FAIL and clean finalization. Indep Hosted tools `38272891095/1` passed the original nine exact storage regressions and all six supplemental directory-fsync tests. Independent consumers verified candidate `1aa9`, source tree/lock and source-file Git objects, executable/collector identity, exact one-test commands and original logs. These are scoped native prerequisites; individual file fsync, physical power loss, complete required native cuts and canonical PASS remain unclaimed. Isolated package publication `38271092352/1` also passed: fresh hosted publisher bytes equal the opaque build artifact, the complete public DEB digest matches schema 3, and the existing candidate resolver accepts its source/run/attempt binding. Current fleet candidate `38240858236/1` remains fixed. B07 fixed-candidate replay `38275626918/1` remains FAIL with clean finalization. Independent audit binds all four H1/H2 DB/DV instances to `1aa9` source, package/binary and helper `e62c3cc`. H1 GET still returns `500/InternalError`: reader setup reports one available shard against two required shards and reduces missing local files to a generic I/O error. H2 DB GET returns `503/SlowDownRead`, but Admin completion records `deferred/dangling_delete_grace`, zero failed objects and completed execution instead of the frozen failed responsibility. H2 DV startup misses readiness with system metadata read quorum unavailable. Original failures remain retained. Canonical PASS stays 101/158. + + +F07 diagnostic `38277312274/1` pins helper `c0d69a3` and the unchanged `1aa9` candidate. It retains FAIL/cleanup FAIL after twelve live branch instances and two native prerequisites; later H2 instances were not executed. Independent audit verified 156 embedded original-byte witnesses. H1 checkpoint truncation and future-schema injection now reach actual stopped original records and durable failed terminals, but report reconciliation still fails; this is not canonical PASS. H2 DB MRF restart advances its committed records and repairs nine missing target files through its untouched companion replica; its exact corruption diagnostic is not observed. Cleanup correctly refuses old-record resurrection. Recovery now audits the complete acknowledged fixture, exact original corruption, current stopped scene and all private backup/intent/applied bytes on all four nodes before finalization, and repeats those checks before removing each dedicated namespace. It exports evidence without restoring advanced product records, preserves the original FAIL and rejects unknown writes or other mutations. The 634 Heal contracts pass with eight platform skips; two fresh sequential final reviews found no unresolved correctness, simplicity, coverage, security, durability or compatibility finding. Live retained-scene recovery remains pending. + +B07 read-quorum error propagation now has a real external-part regression in the companion RustFS PR. Reverting the dominant-I/O return reproduces `Io(NotFound)` at R-1; the fix preserves object-bound `InsufficientReadQuorum` in both legacy and streaming setup. Exactly R and healthy reads return every original payload byte. All 123 reader tests pass on Rust 1.99. This is local product regression evidence; fixed-candidate live run `38275626918/1`, H2 deferred responsibility and startup failures remain retained, and canonical PASS stays 101/158. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index d9d12caf..941a9d40 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1305,6 +1305,69 @@ def journal_rejection_audit_finalize(request): "sceneSha256": scene_sha, "backupRestorationUsed": False, "productRepairClaimed": False} +def journal_retained_audit_finalize(request): + """Export an advanced MRF fault fixture; never restore its earlier product record.""" + require(set(request) == {"runId", "action", "bucket", "binarySha256", "faults", "originalBackup", + "expectedSceneSha256", "expectedMrfSha256"}, "retained journal audit accepts only its archived identity") + require(request["action"] in {"journal-retained-audit", "journal-retained-audit-finalize"}, "invalid retained journal audit") + state, roots, owner = owned(request["runId"]) + stopped(owner) + require(request["runId"].endswith("-h2") and owner.get("runtimeProfile") == "mrf-crash" + and owner.get("bucket") == request["bucket"] and request["binarySha256"] == owner["binarySha256"] + and digest(state / "rustfs") == owner["binarySha256"] + and gate_module().checked_gate(state, roots) is None, + "retained journal candidate, bucket, runtime or member gate differs") + module, fault = globals()["journal_fault_backend"], fault_module() + faults, originals = request["faults"], request["originalBackup"] + require(isinstance(faults, list) and len(faults) <= 1 and isinstance(originals, dict) + and set(originals) == {item.get("faultId") for item in faults}, "retained journal backup bindings differ") + by_disk = {} + exported = [] + for item in faults: + require(isinstance(item, dict) and set(item) == {"schemaVersion", "runId", "disk", "family", "taskId", "relativePath", + "faultId", "binarySha256", "device", "mountId", "productRepairClaimed", "kind", "before", "injected"} + and item.get("disk") == 0 and type(item["disk"]) is int and item.get("family") == "mrf" + and item.get("taskId") is None and item.get("kind") == "checksum" + and item.get("faultId") == "F07-mrf-checksum-0-0-0" + and item.get("relativePath") == ".rustfs.sys/.heal-mrf-snapshot.0.bin", "retained journal fault escaped its exact identity") + by_disk[item["disk"]] = item + for disk, root in enumerate(roots): + with fault.directory(root) as fd, fault.parent(fd, ["quarantine"]) as backup: + item = by_disk.get(disk) + names = set() if item is None else {item["faultId"] + suffix for suffix in (".original", ".intent.json", ".applied.json")} + require(set(os.listdir(backup)) == names, "retained journal has an unarchived mutation or missing backup") + if item is None: continue + identity = module.binding(fd, item, owner, fault) + require(all(module.same_json(item.get(key), value) for key, value in identity.items()), "retained journal physical fault binding differs") + original = fault.read_file(backup, item["faultId"] + ".original", module.LIMIT) + require(isinstance(originals[item["faultId"]], str) + and len(originals[item["faultId"]]) <= (module.LIMIT + 2) // 3 * 4, + "retained journal original bytes exceed their bound") + require(base64.b64decode(originals[item["faultId"]], validate=True) == original + and module.entry(original) == item["before"], "retained journal original backup drifted") + raw_intent = fault.read_file(backup, item["faultId"] + ".intent.json", 16384) + raw_applied = fault.read_file(backup, item["faultId"] + ".applied.json", 16384) + require(module.same_json(module.decode_json(raw_intent), item) + and module.same_json(module.decode_json(raw_applied), item) + and module.entry(module.transform(original, item, owner, globals()["mrf"])) == item["injected"], + "retained journal intent, applied receipt or exact corruption differs") + exported.append({"disk": disk, "faultId": item["faultId"], + "original": {**module.entry(original), "rawBase64": base64.b64encode(original).decode()}, + "intent": {**module.entry(raw_intent), "rawBase64": base64.b64encode(raw_intent).decode()}, + "applied": {**module.entry(raw_applied), "rawBase64": base64.b64encode(raw_applied).decode()}}) + records = mrf_records({"runId": request["runId"], "action": "mrf-records", "quiescent": True}) + mrf_sha = hashlib.sha256(json.dumps(records, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + observed = snapshot({"runId": request["runId"], "action": "snapshot", "quiescent": True}) + content = {key: observed[key] for key in ("files", "formats")} + scene_sha = hashlib.sha256(json.dumps(content, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + require(all(re.fullmatch(r"[0-9a-f]{64}", request.get(name, "")) for name in ("expectedMrfSha256", "expectedSceneSha256")) + and mrf_sha == request["expectedMrfSha256"] and scene_sha == request["expectedSceneSha256"], + "retained journal or physical scene drifted; retain namespace") + result = cleanup(request) if request["action"] == "journal-retained-audit-finalize" else {"readyToFinalize": True} + return {**result, "originalEvidenceExported": True, "originalJournalBackups": exported, "retainedMrf": records, + "sceneSha256": scene_sha, "backupRestorationUsed": False, "productRepairClaimed": False} + + def task_records(request): """Read three exact Admin journal names; never mutate system metadata.""" state, roots, owner = owned(request["runId"]) @@ -1438,6 +1501,8 @@ def main(request): actions["quota-audit-finalize"] = quota_audit_finalize actions["journal-rejection-audit-finalize"] = journal_rejection_audit_finalize actions["journal-rejection-audit"] = journal_rejection_audit_finalize + actions["journal-retained-audit"] = journal_retained_audit_finalize + actions["journal-retained-audit-finalize"] = journal_retained_audit_finalize for action in ("journal-census", "journal-observe", "journal-capture", "journal-apply", "journal-recover"): actions[action] = journal_operation for action in ('generation-capture', 'generation-install', 'generation-recover'): diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 772698f7..1e71f4bd 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -185,7 +185,7 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() - if action.startswith("network-") or action in {"cleanup", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize"}: + if action.startswith("network-") or action in {"cleanup", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize", "journal-retained-audit", "journal-retained-audit-finalize"}: network_source = NODE_SOURCE.with_name("network_gate.py").read_text() source += "\nimport types\nnetwork_gate_backend = types.ModuleType('network_gate')\nexec(" + repr( network_source) + ", network_gate_backend.__dict__)\nNETWORK_GATE_SOURCE = " + repr(network_source) + "\n" @@ -220,11 +220,11 @@ def remote(self, node, action, **params): if action == "format-schema-fault": source += "\nimport types\nschema_faults = types.ModuleType('schema_faults')\nexec(" + repr( NODE_SOURCE.with_name("schema_faults.py").read_text()) + ", schema_faults.__dict__)\n" - if action in {"encryption-configure", "snapshot", "audit-finalize", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize"}: + if action in {"encryption-configure", "snapshot", "audit-finalize", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize", "journal-retained-audit", "journal-retained-audit-finalize"}: source += "\nimport types\nencryption_runtime = types.ModuleType('encryption_runtime')\nexec(" + repr( NODE_SOURCE.with_name("encryption_runtime.py").read_text()) + ", encryption_runtime.__dict__)\n" - replacement_actions = {"start", "cleanup", "audit-finalize", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot", "format-schema-fault"} - if action in {"start", "cleanup", "audit-finalize", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize", "drive-gate", "drive-gate-arm-live", "replace-drive", "quorum-gate", "quorum-gate-probe"}: + replacement_actions = {"start", "cleanup", "audit-finalize", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize", "journal-retained-audit", "journal-retained-audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot", "format-schema-fault"} + if action in {"start", "cleanup", "audit-finalize", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize", "journal-retained-audit", "journal-retained-audit-finalize", "drive-gate", "drive-gate-arm-live", "replace-drive", "quorum-gate", "quorum-gate-probe"}: source += "\nimport types\nquorum_gate_backend = types.ModuleType('quorum_gate')\nexec(" + repr( NODE_SOURCE.with_name("quorum_gate.py").read_text()) + ", quorum_gate_backend.__dict__)\n" if action in {"drive-gate", "drive-gate-arm-live", "drive-gate-probe", "drive-gate-activate", "gate-state", "drive-gate-release-live"} | replacement_actions: diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index cbd68d63..80e54b07 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -2,6 +2,7 @@ """Audit and finalize only explicitly supported, exactly archived failures.""" import argparse +import base64 import hashlib import json import os @@ -139,6 +140,9 @@ def network_recovery_plan(artifact, failed_run, candidate): def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") if list(artifact.glob("*/run/heal.checkpoint-and-mrf-corruption/cases/*.json")): + folder = next(artifact.glob("*/run/heal.checkpoint-and-mrf-corruption/evidence/heal.checkpoint-and-mrf-corruption")) + if list((folder / "DB/H2/mrf-checksum").glob("*-actual-original-MRF-CRC-injection.json")): + return retained_mrf_recovery_plan(artifact, failed_run, candidate) return rejected_mrf_recovery_plan(artifact, failed_run, candidate) if list(artifact.glob("*/run/heal.bucket-incarnation-recreation/cases/*.json")): return recreation_recovery_plan(artifact, failed_run, candidate) @@ -259,6 +263,96 @@ def one(label): "binarySha256": provenance["binary_sha256"], "rejectedFaultId": "F07-mrf-checksum-0-0-0"}] +def retained_mrf_recovery_plan(artifact, failed_run, candidate): + """Export and discard an advanced stopped fixture without resurrecting MRF bytes.""" + try: + from . import persistence_mrf, journal_fault + except ImportError: + import persistence_mrf, journal_fault + case = "heal.checkpoint-and-mrf-corruption" + paths = list(artifact.glob("*/run/*/cases/*.json")) + require(len(paths) == 1 and paths[0].name == case + ".json", "retained MRF recovery requires one unmixed result") + result = json.loads(paths[0].read_text()) + provenance = result.get("provenance") or {} + require(result.get("case_id") == case and result.get("verdict") == "FAIL" and result.get("cleanup_status") == "FAIL" + and provenance.get("source_commit") == candidate["source_sha"] + and provenance.get("candidate_sha256") == candidate["package_sha256"], "retained MRF candidate or result differs") + evidence = paths[0].parents[1] / "evidence" / case + summary = json.loads((evidence / "healthy-summary.json").read_text()) + rows = summary.get("results") or [] + require(summary.get("caseId") == case and summary.get("cleanup") == "FAIL" and summary.get("complete") is False + and summary.get("canonicalCasePassClaimed") is False and rows + and all(row.get("cleanup") == "PASS" for row in rows[:-1]) + and rows[-1].get("topology") == "H2" and rows[-1].get("fixture") == "DB" + and rows[-1].get("branch") == "mrf-checksum" and rows[-1].get("verdict") == "FAIL" + and rows[-1].get("cleanup") == "FAIL" and "unresolvedMutation" not in rows[-1] + and rows[-1].get("failure", {}).get("message") == + "NOT_OBSERVED: original MRF corruption has no candidate diagnostic on vm000", + "retained MRF recovery supports only its archived post-restart diagnostic refusal") + folder = evidence / "DB/H2/mrf-checksum" + def one(label): + found = [path for path in folder.glob("*-" + label + ".json") + if re.fullmatch(r"[0-9]{5}-" + re.escape(label) + r"\.json", path.name)] + require(len(found) == 1, "retained MRF recovery lacks one exact " + label) + return json.loads(found[0].read_text()) + binding, failure = one("binding"), one("original-physical-failure-scene") + original, injected, retained = (one(label) for label in + ("original-stopped-MRF-records", "injected-stopped-MRF-records", "original-MRF-failure-scene")) + nodes = [f"vm00{index}" for index in range(4)] + bucket = binding.get("bucketPrefix", "") + require(re.fullmatch(rf"gh-{failed_run}-[1-9][0-9]*-h2", binding.get("runId", "")) + and re.fullmatch(r"rv-heal-[0-9a-f]{16}", bucket) + and binding.get("candidateCommit") == candidate["source_sha"] + and binding.get("candidatePackageSha256") == candidate["package_sha256"] + and binding.get("candidateBinarySha256") == provenance.get("binary_sha256") + and binding.get("harnessCommit") == provenance.get("testing_commit") + and all(set(scene) == set(nodes) for scene in (failure, original, injected, retained)) + and all(value.get("quiescent") is True and value.get("pid") == 0 + and value.get("binarySha256") == provenance.get("binary_sha256") for value in failure.values()), + "retained MRF run, candidate, stopped scene or node identity differs") + receipts = [json.loads(path.read_text()) for path in folder.glob("*-actual-original-MRF-CRC-injection.json")] + require(len(receipts) == 1 and receipts[0].get("node") == "vm000" + and receipts[0].get("faultId") == "F07-mrf-checksum-0-0-0" and receipts[0].get("disk") == 0 + and receipts[0].get("runId") == binding["runId"] + and receipts[0].get("binarySha256") == provenance.get("binary_sha256"), + "retained MRF recovery cannot omit another injection") + persistence_mrf.retained_corruption(original, injected, receipts, bucket) + for observed in retained.values(): + persistence_mrf.indexed(observed) + require(observed.get("binarySha256") == provenance.get("binary_sha256"), "retained MRF candidate bytes differ") + gate = one("gate-recovery") + require(gate.get("node") == "vm003" and gate.get("state") == "CLEAR" + and gate.get("originalPermissionsRestored") is True, "retained MRF gate was not restored") + recovery = one("recovery") + require(recovery.get("cleanup") == "FAIL" and len(recovery.get("errors", [])) == 1 + and recovery["errors"][0].get("state") == "DIRTY" + and recovery["errors"][0].get("namespacesRetained") is True + and "product changed or advanced the durable responsibility; refusing old-record resurrection" in + recovery["errors"][0].get("error", ""), "retained MRF rescue refusal differs") + requests, acks = sorted((folder / "fixture").glob("*-request.json")), sorted((folder / "fixture").glob("*-ack.json")) + require(len(requests) == len(acks) == 7, "retained MRF recovery lacks complete fixture acknowledgements") + for seq, (request_path, ack_path) in enumerate(zip(requests, acks), 1): + request, ack = json.loads(request_path.read_text()), json.loads(ack_path.read_text()) + observed = ack.get("observation") or {} + require(type(request.get("sequence")) is int and type(ack.get("sequence")) is int + and request["sequence"] == ack["sequence"] == seq + and request.get("runId") == ack.get("runId") == binding["runId"] + and ack.get("requestFile") == request_path.name + and ack.get("requestSha256") == hashlib.sha256(request_path.read_bytes()).hexdigest() + and observed.get("operation") == request.get("operation") == ("create_bucket" if seq == 1 else "put_object") + and observed.get("params") == request.get("params") and request.get("params", {}).get("Bucket") == bucket + and observed.get("clientCallInvoked") is True and observed.get("resultUnknown") is False + and observed.get("acknowledged") is True and ack.get("operationCheck") == "PASS", + "retained MRF ledger changed identity or contains an unknown mutation") + fault = {key: value for key, value in receipts[0].items() if key not in {"node", "applied", "originalBackupRetained"}} + raw = persistence_mrf.indexed(original["vm000"])[0, fault["relativePath"]] + require(journal_fault.entry(raw) == fault["before"], "retained MRF original backup identity differs") + return [{"kind": "advanced-mrf-fixture", "runId": binding["runId"], "nodes": nodes, "node": nodes[0], + "bucket": bucket, "before": failure, "originalMrf": retained, + "binarySha256": provenance["binary_sha256"], "faults": {"vm000": [fault]}, + "originalBackup": {"vm000": {fault["faultId"]: base64.b64encode(raw).decode()}}}] + + def recreation_recovery_plan(artifact, failed_run, candidate): """Archive a stopped C11 fixture after an exact known DeleteBucket refusal.""" case = "heal.bucket-incarnation-recreation" @@ -510,7 +604,24 @@ def main(): require(recovery_scene_matches(current, plan), "retained fleet drifted from its exact archived scene; no recovery performed") fleet.record("retained-service-logs", fleet.remote(plan["node"], "logs")) - if plan.get("kind") == "rejected-mrf-injection": + if plan.get("kind") == "advanced-mrf-fixture": + archived = fleet.record("retained-advanced-MRF-records", {node: fleet.remote(node, + "mrf-records", quiescent=True) for node in plan["nodes"]}) + require(archived == plan["originalMrf"], "advanced MRF scene drifted; retain namespace") + for action in ("journal-retained-audit", "journal-retained-audit-finalize"): + for node in plan["nodes"]: + scene = physical_content(plan["before"])[node] + proof = fleet.record("audited-finalization" if action.endswith("finalize") else "exported-original-journal-backups", {"node": node, + **fleet.remote(node, action, bucket=plan["bucket"], binarySha256=plan["binarySha256"], + faults=plan["faults"].get(node, []), originalBackup=plan["originalBackup"].get(node, {}), + expectedMrfSha256=hashlib.sha256(json.dumps(plan["originalMrf"][node], sort_keys=True, separators=(",", ":")).encode()).hexdigest(), + expectedSceneSha256=hashlib.sha256(json.dumps(scene, sort_keys=True, separators=(",", ":")).encode()).hexdigest())}) + require(proof.get("originalEvidenceExported") is True and proof.get("backupRestorationUsed") is False + and proof.get("productRepairClaimed") is False + and (proof.get("readyToFinalize") is True if action.endswith("audit") else + proof.get("cleanup") == "PASS" and proof.get("unitStopped") is True and proof.get("rootsRemoved") is True), + "advanced MRF evidence export or finalization failed") + elif plan.get("kind") == "rejected-mrf-injection": archived = fleet.record("retained-original-MRF-records", {node: fleet.remote(node, "mrf-records", quiescent=True) for node in plan["nodes"]}) require(archived == plan["originalMrf"], "F07 original MRF bytes drifted; retain namespace") @@ -560,7 +671,7 @@ def main(): else: proof = fleet.record("audited-finalization", fleet.remote(plan["node"], "audit-finalize", **{ key: plan[key] for key in ("disk", "rejectedFaultId", "binarySha256", "expectedSceneSha256")})) - if plan.get("kind") not in {"performance-no-fault-applied", "rejected-mrf-injection"}: + if plan.get("kind") not in {"performance-no-fault-applied", "rejected-mrf-injection", "advanced-mrf-fixture"}: for node in plan["nodes"]: if node != plan["node"]: fleet.record("unchanged-peer-finalization", fleet.remote(node, "cleanup")) diff --git a/test_validation_heal_recovery.py b/test_validation_heal_recovery.py index f676a8bb..bfdcd4bb 100644 --- a/test_validation_heal_recovery.py +++ b/test_validation_heal_recovery.py @@ -1,6 +1,8 @@ """A rejected fault ID cannot bypass ownership, intent or physical-scene checks.""" import hashlib +import base64 +import binascii import copy import json import sys @@ -160,6 +162,121 @@ def test_node_requires_empty_quarantine_and_exact_stopped_scene(self): cleanup.assert_not_called() +class AdvancedMrfRecoveryTests(unittest.TestCase): + def archive(self, root): + from test_validation_heal_mrf import DurableMrfTests + from scripts.validation_heal import journal_fault as journal + candidate, folder = RejectedMrfRecoveryTests().archive(root) + def one(label): return next(folder.glob("*-" + label + ".json")) + summary = json.loads((folder.parents[2] / "healthy-summary.json").read_text()) + summary["results"][0]["failure"]["message"] = "NOT_OBSERVED: original MRF corruption has no candidate diagnostic on vm000" + (folder.parents[2] / "healthy-summary.json").write_text(json.dumps(summary)) + helper = DurableMrfTests(); observed = helper.observed() + observed["binarySha256"] = "c" * 64 + original = {node: copy.deepcopy(observed) for node in (f"vm00{i}" for i in range(4))} + raw = helper.journal() + receipt = {"schemaVersion": 1, "runId": "gh-12345-1-h2", "disk": 0, "family": "mrf", "taskId": None, + "relativePath": ".rustfs.sys/.heal-mrf-snapshot.0.bin", "faultId": "F07-mrf-checksum-0-0-0", + "binarySha256": "c" * 64, "device": 1, "mountId": 2, "productRepairClaimed": False, "kind": "checksum", + "before": journal.entry(raw), "injected": journal.entry(raw[:-1] + bytes([raw[-1] ^ 1])), + "node": "vm000", "applied": True, "originalBackupRetained": True} + injected = copy.deepcopy(original) + row = next(row for row in injected["vm000"]["records"] if row["disk"] == 0 and row["relativePath"] == receipt["relativePath"]) + row.update(receipt["injected"], rawBase64=base64.b64encode(raw[:-1] + bytes([raw[-1] ^ 1])).decode()) + for label, value in (("original-stopped-MRF-records", original), ("injected-stopped-MRF-records", injected), + ("original-MRF-failure-scene", original)): + one(label).write_text(json.dumps(value)) if list(folder.glob("*-" + label + ".json")) else (folder / ("00004-" + label + ".json")).write_text(json.dumps(value)) + (folder / "00005-actual-original-MRF-CRC-injection.json").write_text(json.dumps(receipt)) + (folder / "00006-recovery.json").write_text(json.dumps({"cleanup": "FAIL", "errors": [{"state": "DIRTY", + "namespacesRetained": True, "error": "product changed or advanced the durable responsibility; refusing old-record resurrection"}]})) + return candidate, folder + + def test_plan_requires_complete_acks_exact_injection_and_stopped_failure(self): + for change in (None, "unknown", "receipt", "records", "candidate", "extra-injection", "earlier-dirty"): + with tempfile.TemporaryDirectory() as temp, self.subTest(change=change): + root = Path(temp); candidate, folder = self.archive(root) + if change == "unknown": + path = folder / "fixture/000007-ack.json"; value = json.loads(path.read_text()) + value["observation"]["resultUnknown"] = True; path.write_text(json.dumps(value)) + elif change == "receipt": + path = next(folder.glob("*-actual-original-MRF-CRC-injection.json")); value = json.loads(path.read_text()) + value["injected"]["sha256"] = "f" * 64; path.write_text(json.dumps(value)) + elif change == "records": + path = next(folder.glob("*-original-MRF-failure-scene.json")); value = json.loads(path.read_text()) + value["vm000"]["records"][0]["rawBase64"] = "bad"; path.write_text(json.dumps(value)) + elif change == "candidate": candidate["package_sha256"] = "f" * 64 + elif change == "extra-injection": (folder / "00007-actual-original-MRF-CRC-injection.json").write_text("{}") + elif change == "earlier-dirty": + path = folder.parents[2] / "healthy-summary.json"; value = json.loads(path.read_text()) + value["results"].insert(0, {"cleanup": "FAIL"}); path.write_text(json.dumps(value)) + if change is None: + plan = recovery_plan(root, 12345, candidate)[0] + self.assertEqual(plan["kind"], "advanced-mrf-fixture") + self.assertEqual(set(plan["faults"]), {"vm000"}) + else: + with self.assertRaises((ValueError, binascii.Error)): recovery_plan(root, 12345, candidate) + + def test_remote_retained_audit_loads_real_guard_backends(self): + with tempfile.TemporaryDirectory() as temp: + fleet = Fleet(["vm000"], Path(temp), "gh-12345-1-h2") + def execute(*args, **kwargs): + namespace = {"__name__": "__validation_node_contract__"} + with patch.dict(sys.modules): exec(compile(kwargs["input"].rsplit("\nmain(", 1)[0], "remote-node.py", "exec"), namespace) + self.assertTrue({"fault", "mrf", "journal_fault_backend", "encryption_runtime", "gate", "quorum_gate_backend", "replacement"} <= set(namespace)) + return types.SimpleNamespace(returncode=0, stdout="{}", stderr="") + with patch("scripts.validation_heal.healthy.subprocess.run", side_effect=execute): + for action in ("journal-retained-audit", "journal-retained-audit-finalize"): fleet.remote("vm000", action) + + def test_node_exports_original_backup_and_rejects_drift_before_deletion(self): + import base64 + from test_validation_heal_mrf import DurableMrfTests + from scripts.validation_heal import journal_fault as journal, mrf_records as mrf + for change in (None, "foreign-backup", "extra", "missing", "intent", "scene", "records", "gate", "path"): + with tempfile.TemporaryDirectory() as temp, self.subTest(change=change): + state = Path(temp).resolve(); roots = tuple(state / f"disk{i}" for i in range(4)) + for root in roots: (root / "quarantine").mkdir(parents=True) + owner = {"runId": "gh-12345-1-h2", "runtimeProfile": "mrf-crash", "bucket": "rv-heal-0123456789abcdef", "binarySha256": "c" * 64} + raw = DurableMrfTests().journal() + params = {"runId": owner["runId"], "disk": 0, "family": "mrf", "taskId": None, + "relativePath": ".rustfs.sys/.heal-mrf-snapshot.0.bin", "faultId": "F07-mrf-checksum-0-0-0", "kind": "checksum"} + with patch.object(heal_fault, "mount_id", return_value=2), heal_fault.directory(roots[0]) as fd: + intent = {**journal.binding(fd, params, owner, heal_fault), "kind": "checksum", + "before": journal.entry(raw), "injected": journal.entry(journal.transform(raw, params, owner, mrf))} + backup = roots[0] / "quarantine" + for suffix, data in ((".original", raw), (".intent.json", heal_fault.encode(intent)), (".applied.json", heal_fault.encode(intent))): + (backup / (params["faultId"] + suffix)).write_bytes(data) + scene = {"files": [], "formats": []}; records = {"readOnly": True, "stable": True, "records": []} + sha = lambda value: hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + request = {"runId": owner["runId"], "action": "journal-retained-audit-finalize", "bucket": owner["bucket"], + "binarySha256": owner["binarySha256"], "faults": [intent], "originalBackup": {params["faultId"]: base64.b64encode(raw).decode()}, + "expectedSceneSha256": sha(scene), "expectedMrfSha256": sha(records)} + if change == "foreign-backup": (backup / (params["faultId"] + ".original")).write_bytes(b"foreign") + elif change == "extra": (roots[1] / "quarantine/unregistered.original").write_bytes(b"evidence") + elif change == "missing": (backup / (params["faultId"] + ".applied.json")).unlink() + elif change == "intent": (backup / (params["faultId"] + ".intent.json")).write_text("{}") + elif change == "scene": request["expectedSceneSha256"] = "f" * 64 + elif change == "records": request["expectedMrfSha256"] = "f" * 64 + elif change == "path": intent["relativePath"] = ".rustfs.sys/foreign" + with patch.object(fleet_node, "owned", return_value=(state, roots, owner)), patch.object(fleet_node, "stopped"), \ + patch.object(fleet_node, "digest", return_value="c" * 64), patch.object(fleet_node, "fault", heal_fault, create=True), \ + patch.object(heal_fault, "mount_id", return_value=2), patch.object(fleet_node, "journal_fault_backend", journal, create=True), \ + patch.object(fleet_node, "mrf", mrf, create=True), patch.object(fleet_node, "mrf_records", return_value=records), \ + patch.object(fleet_node, "snapshot", return_value=scene), patch.object(fleet_node, "gate_module") as gate, \ + patch.object(fleet_node, "cleanup", return_value={"cleanup": "PASS", "unitStopped": True, "rootsRemoved": True}) as cleanup: + gate.return_value.checked_gate.return_value = {} if change == "gate" else None + if change is None: + audit = fleet_node.journal_retained_audit_finalize({**request, "action": "journal-retained-audit"}) + cleanup.assert_not_called() + self.assertEqual(base64.b64decode(audit["originalJournalBackups"][0]["original"]["rawBase64"]), raw) + result = fleet_node.journal_retained_audit_finalize(request) + self.assertFalse(result["backupRestorationUsed"]) + self.assertFalse(result["productRepairClaimed"]) + cleanup.assert_called_once() + else: + with self.assertRaises(ValueError): fleet_node.journal_retained_audit_finalize(request) + cleanup.assert_not_called() + + class RecreationRecoveryTests(unittest.TestCase): def archive(self, root): case, bucket = "heal.bucket-incarnation-recreation", "rv-heal-0123456789abcdef" From ab37b0c4854d65e91ec2f377b1a18fc6bc494360 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 13 Oct 2026 01:17:21 +0800 Subject: [PATCH 195/343] fix(validation): reconcile source-defined heal task timeout --- docs/validation-migration-map.md | 2 + scripts/validation_heal/persistence.py | 40 ++++++-- test_validation_heal_persistence.py | 98 ++++++++++++++++++- .../validation/root-terminal-options.json | 34 +++++++ 4 files changed, 166 insertions(+), 8 deletions(-) create mode 100644 testdata/validation/root-terminal-options.json diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 3c6289a4..d0ff2b98 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1983,3 +1983,5 @@ B07 fixed-candidate replay `38275626918/1` remains FAIL with clean finalization. F07 diagnostic `38277312274/1` pins helper `c0d69a3` and the unchanged `1aa9` candidate. It retains FAIL/cleanup FAIL after twelve live branch instances and two native prerequisites; later H2 instances were not executed. Independent audit verified 156 embedded original-byte witnesses. H1 checkpoint truncation and future-schema injection now reach actual stopped original records and durable failed terminals, but report reconciliation still fails; this is not canonical PASS. H2 DB MRF restart advances its committed records and repairs nine missing target files through its untouched companion replica; its exact corruption diagnostic is not observed. Cleanup correctly refuses old-record resurrection. Recovery now audits the complete acknowledged fixture, exact original corruption, current stopped scene and all private backup/intent/applied bytes on all four nodes before finalization, and repeats those checks before removing each dedicated namespace. It exports evidence without restoring advanced product records, preserves the original FAIL and rejects unknown writes or other mutations. The 634 Heal contracts pass with eight platform skips; two fresh sequential final reviews found no unresolved correctness, simplicity, coverage, security, durability or compatibility finding. Live retained-scene recovery remains pending. B07 read-quorum error propagation now has a real external-part regression in the companion RustFS PR. Reverting the dominant-I/O return reproduces `Io(NotFound)` at R-1; the fix preserves object-bound `InsufficientReadQuorum` in both legacy and streaming setup. Exactly R and healthy reads return every original payload byte. All 123 reader tests pass on Rust 1.99. This is local product regression evidence; fixed-candidate live run `38275626918/1`, H2 deferred responsibility and startup failures remain retained, and canonical PASS stays 101/158. + +F07 terminal reconciliation now accepts only the source-defined scheduler transition from an explicit null admission timeout to exactly 300 seconds with zero nanoseconds. The two reviewed candidate commits share the exact scheduler/config sources, and the dedicated fleet verifies `RUSTFS_HEAL_TASK_TIMEOUT_SECS=300` in the actual process environment, including a fresh witness after fault restart. Every other option remains exact; explicit or checkpoint-consumed budgets cannot be refunded, and terminal/report JSON types must agree. The original H1 DB/DV truncation and future-schema records from `38277312274/1` reproduce all four prior reconciliation refusals and pass this derived retained-record check after the fix. Their original FAIL results remain unchanged; complete rescan, physical/S3 reconciliation and new live execution are still required. The 639 Heal contracts pass with eight platform skips. Two fresh sequential final reviews cover the exact normalization, source/runtime boundary, budget preservation and negative regression inputs without unresolved findings. Retained-scene recovery `38281391802/1` remains queued behind the existing shared fleet chain; no later fleet replay is started before clean recovery. Canonical PASS stays 101/158. diff --git a/scripts/validation_heal/persistence.py b/scripts/validation_heal/persistence.py index 1b3867b2..5b6d01df 100644 --- a/scripts/validation_heal/persistence.py +++ b/scripts/validation_heal/persistence.py @@ -24,7 +24,7 @@ from .matrix import select from .shards import files, map_target, run_task, validate_fault, validate_repaired_physical from .stop import idle_fleet, status - from .root_restart import recovery_status, same_owner + from .root_restart import recovery_status, same_owner, TIMEOUT_SECONDS from .scanner_repair import JournalWindow from .fleet_node import PROCESS_FIELDS from .persistence_mrf import run as run_mrf_checksum @@ -39,7 +39,7 @@ from matrix import select from shards import files, map_target, run_task, validate_fault, validate_repaired_physical from stop import idle_fleet, status - from root_restart import recovery_status, same_owner + from root_restart import recovery_status, same_owner, TIMEOUT_SECONDS from scanner_repair import JournalWindow from fleet_node import PROCESS_FIELDS from persistence_mrf import run as run_mrf_checksum @@ -53,6 +53,26 @@ "stale-generation": "stale-generation"} BLOCKERS = {} SIZE = 5 * 1024**2 + 123 +TERMINAL_OPTIONS_REFERENCE = {"candidateCommits": ( + "072b57b37559fbe626aeda8abf61617fdcbdd926", "1aa9be87d18cf941e5020f19e8622c26ba4b10ac"), + "sourceFiles": { + "crates/heal/src/heal/manager/scheduler.rs": "79ce1f9c278c664231b6a1eca7c521f8e98742fe6d17d8f852368da4e77cc33b", + "crates/heal/src/heal/manager.rs": "ba9289942fbbe8ce2632686ae86b9d1be0ae9923420e3b091d67148b4ee02094", + "crates/config/src/constants/heal.rs": "82b8614dfa7cc111e694138035e4a84ee23035c719f0286fc0565d7181ca014f"}, + "runtimeSetting": {"RUSTFS_HEAL_TASK_TIMEOUT_SECS": str(TIMEOUT_SECONDS)}, + "normalization": "scheduler get_or_insert fills only an explicit null timeout; all other fields stay exact"} + + +def same_terminal_options(admitted, terminal): + if not isinstance(admitted, dict) or not isinstance(terminal, dict): + return False + if journal.same_json(admitted, terminal): + return True + # The source-bound scheduler fills None from the process-verified fleet + # setting. It preserves explicit budgets, including a checkpointed one. + if "timeout" not in admitted or admitted["timeout"] is not None: + return False + return journal.same_json(terminal, {**admitted, "timeout": {"secs": TIMEOUT_SECONDS, "nanos": 0}}) def exact_records(observed, token, family, *, allow_frozen=False): @@ -138,13 +158,13 @@ def retained_corruption(observed, roots, frozen, receipt, current, *, earlier=No report = journal.decode_json(checkpoints.bytes_of(root[root_names[2]])) require(type(terminal.get("schema")) is int and terminal["schema"] == 1 and terminal.get("task_id") == token and journal.same_json(terminal.get("heal_type"), owner["value"]["heal_type"]) - and journal.same_json(terminal.get("options"), owner["value"]["options"]) + and same_terminal_options(owner["value"]["options"], terminal.get("options")) and isinstance(terminal.get("status"), dict) and set(terminal["status"]) == {"Failed"} and isinstance(terminal["status"]["Failed"], dict) and isinstance(terminal["status"]["Failed"].get("error"), str) and "checkpoint" in terminal["status"]["Failed"]["error"].lower() and type(report.get("schema")) is int and report["schema"] == 1 - and report.get("terminal") == terminal + and journal.same_json(report.get("terminal"), terminal) and canonical(report.get("outcome")) == canonical(current["outcome"]), "original responsibility lost its durable checkpoint failure diagnostic") if root_names[0] in root: @@ -218,7 +238,7 @@ def completed_original(roots, frozen, current): require(type(terminal.get("schema")) is int and terminal["schema"] == 1 and terminal.get("task_id") == token and terminal.get("status") == "Completed" and journal.same_json(terminal.get("heal_type"), owner["value"]["heal_type"]) - and journal.same_json(terminal.get("options"), owner["value"]["options"]) + and same_terminal_options(owner["value"]["options"], terminal.get("options")) and type(report.get("schema")) is int and report["schema"] == 1 and journal.same_json(report.get("terminal"), terminal) and journal.same_json(report.get("outcome"), current.get("outcome")), @@ -314,7 +334,7 @@ def retained_write_failure(observed, roots, frozen, current, events): report = journal.decode_json(checkpoints.bytes_of(root[names[2]])) require(type(terminal.get("schema")) is int and terminal["schema"] == 1 and terminal.get("task_id") == token and journal.same_json(terminal.get("heal_type"), original["value"]["heal_type"]) - and journal.same_json(terminal.get("options"), original["value"]["options"]) + and same_terminal_options(original["value"]["options"], terminal.get("options")) and terminal.get("status") == {"Failed": {"error": detail}} and type(report.get("schema")) is int and report["schema"] == 1 and journal.same_json(report.get("terminal"), terminal) @@ -450,6 +470,8 @@ def journal_write_failure(fleet, probe, binding, expected, baseline, original, t def run_checkpoint_fault(args, topology, nodes, fixture, branch, candidate): require(branch in IMPLEMENTED or branch == "journal-write-failure", "unimplemented journal branch cannot mutate the fleet") + require(candidate.get("source_sha") in TERMINAL_OPTIONS_REFERENCE["candidateCommits"], + "unsupported candidate terminal-options contract") directory = args.evidence / fixture / topology / branch directory.mkdir(parents=True) run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" @@ -470,6 +492,8 @@ def run_checkpoint_fault(args, topology, nodes, fixture, branch, candidate): "maxInventoryObjects": 512, "maxInventoryBytes": 2 * 1024**3} fleet.record("binding", binding) fleet.record("candidate-bound-inspector", verify_inspector(args.tool_directory, candidate)) + fleet.record("candidate-terminal-options-contract", { + **TERMINAL_OPTIONS_REFERENCE, "candidateCommit": candidate["source_sha"]}) probe, active, pending = Probe(binding), [], None result = {"topology": topology, "fixture": fixture, "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} def allocate(): @@ -530,6 +554,10 @@ def allocate(): and injected["resume"] == frozen["resume"], "stale rollback changed its original companion or actual bytes") fault_scene = fleet.snapshot("physical-at-original-generation-replay-boundary", quiescent=True) fleet.start() + for node in fleet.nodes: + identity = fleet.record("post-injection-candidate-process", {"node": node, **fleet.remote(node, "process")}) + require(identity.get("settings", {}).get("RUSTFS_HEAL_TASK_TIMEOUT_SECS") == str(TIMEOUT_SECONDS), + "terminal normalization has no original candidate runtime timeout witness") deadline = time.monotonic() + 180 while True: failed = recovery_status(probe, fleet, token) diff --git a/test_validation_heal_persistence.py b/test_validation_heal_persistence.py index 9a0252f4..02d738e6 100644 --- a/test_validation_heal_persistence.py +++ b/test_validation_heal_persistence.py @@ -16,12 +16,13 @@ class PersistenceContractTests(unittest.TestCase): - def fixture(self, kind="truncate"): + def fixture(self, kind="truncate", *, admission_options=None, terminal_options=None): observed, _, _ = fixture_source.AdminCheckpointTests().fixture() frozen = checkpoint.owned_checkpoint(observed, TOKEN, BUCKET) root_value = {"schema": 3, "task_id": TOKEN, "heal_type": { "type": "erasure_set", "buckets": [], "set_disk_id": "pool_0_set_0", "all_buckets": True}, "options": {"scan_mode": 2, "recursive": True}} + if admission_options is not None: root_value["options"] = copy.deepcopy(admission_options) owner = {"node": "node", "disk": 1, "relativePath": journal.names("root", TOKEN)[0], "value": root_value, **encoded(root_value)} frozen["rootAdmission"] = owner @@ -39,6 +40,7 @@ def fixture(self, kind="truncate"): skipped=0, failed=0, unknown=0, attemptFailures=0, overflowed=False)}} terminal = {"schema": 1, "task_id": TOKEN, "heal_type": root_value["heal_type"], "options": root_value["options"], "status": {"Failed": {"error": current["detail"]}}} + if terminal_options is not None: terminal["options"] = copy.deepcopy(terminal_options) report = {"schema": 1, "terminal": terminal, "outcome": current["outcome"]} names = journal.names("root", TOKEN) roots = {"node": {"readOnly": True, "quiescent": True, "family": "root", "taskId": TOKEN, @@ -56,6 +58,65 @@ def test_corrupt_bytes_blocked_checkpoint_and_durable_failure_remain_separate_fr self.assertFalse(proof["originalTaskRepairClaimed"]) self.assertFalse(proof["backupRescueUsed"]) + def recorded_options(self): + return json.loads((Path(__file__).parent / "testdata/validation/root-terminal-options.json").read_text()) + + def test_candidate_default_timeout_reconciles_original_corruption_and_completion(self): + recorded = self.recorded_options() + for kind in ("truncate", "future-schema"): + data = self.fixture(kind, admission_options=recorded["admissionOptions"], + terminal_options=recorded["terminalOptions"]) + proof = persistence.retained_corruption(*data) + self.assertTrue(proof["durableOriginalFailureRetained"]) + self.assertFalse(proof["originalTaskRepairClaimed"]) + roots = self.completion_roots(data[2], data[4]) + terminal = journal.decode_json(checkpoint.bytes_of(roots["node"]["records"][4])) + terminal["options"] = recorded["terminalOptions"] + roots["node"]["records"][4].update(encoded(terminal)) + roots["node"]["records"][5].update(encoded({"schema": 1, "terminal": terminal, "outcome": data[4]["outcome"]})) + self.assertTrue(persistence.completed_original(roots, data[2], data[4])["sameOriginalTaskAndPhysicalOwner"]) + + def test_default_timeout_never_normalizes_other_options_or_refunds_explicit_budgets(self): + recorded = self.recorded_options() + for change in ("shorter", "longer", "fractional", "float", "bool", "missing", "unknown-duration-field", + "missing-admission-timeout", "explicit-budget", "consumed-budget", "scope", "nolock", "extra-option"): + admitted, terminal = copy.deepcopy(recorded["admissionOptions"]), copy.deepcopy(recorded["terminalOptions"]) + if change == "shorter": terminal["timeout"]["secs"] = 299 + elif change == "longer": terminal["timeout"]["secs"] = 301 + elif change == "fractional": terminal["timeout"]["nanos"] = 1 + elif change == "float": terminal["timeout"]["secs"] = 300.0 + elif change == "bool": terminal["timeout"]["nanos"] = False + elif change == "missing": terminal.pop("timeout") + elif change == "unknown-duration-field": terminal["timeout"]["extra"] = 0 + elif change == "missing-admission-timeout": admitted.pop("timeout") + elif change == "explicit-budget": admitted["timeout"] = {"secs": 120, "nanos": 0} + elif change == "consumed-budget": admitted["timeout"] = {"secs": 299, "nanos": 0} + elif change == "scope": terminal["pool_index"] = 1 + elif change == "nolock": terminal["no_lock"] = True + else: terminal["extra"] = "foreign" + with self.subTest(change=change), self.assertRaises(ValueError): + persistence.retained_corruption(*self.fixture(admission_options=admitted, terminal_options=terminal)) + for timeout in ({"secs": 120, "nanos": 0}, {"secs": 299, "nanos": 17}): + options = {**recorded["admissionOptions"], "timeout": timeout} + self.assertTrue(persistence.retained_corruption(*self.fixture( + admission_options=options, terminal_options=options))["durableOriginalFailureRetained"]) + for change in ("float", "bool", "task", "scope", "options"): + data = self.fixture(admission_options=recorded["admissionOptions"], terminal_options=recorded["terminalOptions"]) + row = data[1]["node"]["records"][5] + report = journal.decode_json(checkpoint.bytes_of(row)) + if change == "float": report["terminal"]["options"]["timeout"]["secs"] = 300.0 + elif change == "bool": report["terminal"]["options"]["timeout"]["nanos"] = False + elif change == "task": report["terminal"]["task_id"] = INCARNATION + elif change == "scope": report["terminal"]["heal_type"]["set_disk_id"] = "pool_0_set_1" + else: report["terminal"]["options"]["recursive"] = False + row.update(encoded(report)) + with self.subTest(reportChange=change), self.assertRaises(ValueError): persistence.retained_corruption(*data) + + def test_unreviewed_candidate_options_contract_fails_before_fleet_allocation(self): + with patch.object(persistence, "Fleet") as fleet, self.assertRaisesRegex(ValueError, "terminal-options contract"): + persistence.run_checkpoint_fault(None, "H1", ["node"], "DB", "checkpoint-truncate", {"source_sha": "f" * 40}) + fleet.assert_not_called() + def test_missing_rewritten_moved_or_duplicated_corruption_never_passes(self): for change in ("missing", "restored", "blocked", "resume", "moved", "duplicate", "partial-census"): data = self.fixture(); observed, _, frozen, _, _ = data @@ -294,6 +355,28 @@ def test_known_write_failure_can_have_explicit_failed_terminal_without_silent_su terminal["status"] = "Completed"; roots["node"]["records"][4].update(encoded(terminal)) with self.assertRaises(ValueError): persistence.retained_write_failure(observed, roots, frozen, current, events) + def test_checkpoint_write_failure_terminal_keeps_the_source_default_and_exact_scope(self): + recorded = self.recorded_options() + for change in (None, "timeout", "scope", "report"): + observed, roots, frozen, current, events = self.write_failure() + admission = frozen["rootAdmission"]["value"] + admission["options"] = copy.deepcopy(recorded["admissionOptions"]) + frozen["rootAdmission"].update(**encoded(admission)) + roots["node"]["records"][3] = {"disk": 1, "relativePath": journal.names("root", TOKEN)[0], "present": False} + terminal = {"schema": 1, "task_id": TOKEN, "heal_type": admission["heal_type"], + "options": copy.deepcopy(recorded["terminalOptions"]), "status": {"Failed": {"error": current["detail"]}}} + if change == "timeout": terminal["options"]["timeout"]["secs"] = 301 + elif change == "scope": terminal["options"]["set_index"] = 1 + report = {"schema": 1, "terminal": copy.deepcopy(terminal), "outcome": current["outcome"]} + if change == "report": report["terminal"]["options"]["timeout"]["secs"] = 299 + roots["node"]["records"][4].update(encoded(terminal)) + roots["node"]["records"][5].update(encoded(report)) + if change is None: + self.assertTrue(persistence.retained_write_failure(observed, roots, frozen, current, events)["originalResponsibilityRetained"]) + else: + with self.subTest(change=change), self.assertRaises(ValueError): + persistence.retained_write_failure(observed, roots, frozen, current, events) + def test_write_failure_can_consume_root_timeout_without_refunding_or_changing_intent(self): for change in (None, "refunded", "removed", "scope", "retry", "owner"): observed, roots, frozen, current, events = self.write_failure() @@ -460,8 +543,10 @@ def snapshot(self, *args, **kwargs): return snapshots def remote(self, node, action, **params): if action == "journal-apply": + if mutation == "runtime": return {} # The node may have committed the fault before the ACK was lost. raise OSError("journal injection acknowledgement lost") + if action == "process": return {"settings": {"RUSTFS_HEAL_TASK_TIMEOUT_SECS": "299"}} if action == "bounded-volumes-fill": raise OSError("private journal fill acknowledgement lost") return {} @@ -499,7 +584,7 @@ def inventory(self, *args, **kwargs): return {} patch.object(persistence.checkpoints, "stop_for_fault", return_value=frozen): result = persistence.run_checkpoint_fault(args, "H1", ["node"], "DB", "journal-write-failure" if mutation == "fill" else "checkpoint-truncate", - {"source_sha": "0" * 40, "package_sha256": "0" * 64}) + {"source_sha": "1aa9be87d18cf941e5020f19e8622c26ba4b10ac", "package_sha256": "0" * 64}) return result, captured def test_lost_injection_ack_keeps_original_journal_and_shard_rescue_registered(self): @@ -512,6 +597,15 @@ def test_lost_injection_ack_keeps_original_journal_and_shard_rescue_registered(s self.assertEqual(captured["cleanupJournals"][0][1]["taskId"], TOKEN) self.assertNotIn("separateRescan", result) + def test_restart_runtime_change_preserves_rescue_and_refuses_terminal_normalization(self): + with patch.object(persistence, "recovery_status") as queried: + result, captured = self.interrupted_injection(True, "runtime") + self.assertEqual((result["verdict"], result["cleanup"]), ("FAIL", "PASS")) + self.assertIn("runtime timeout witness", result["failure"]["message"]) + self.assertEqual(len(captured["cleanupJournals"]), 1) + self.assertEqual(len(captured["cleanupFaults"]), 4) + queried.assert_not_called() + def test_failed_cleanup_preserves_dirty_result_after_lost_injection_ack(self): result, captured = self.interrupted_injection(False) self.assertEqual((result["verdict"], result["cleanup"]), ("FAIL", "FAIL")) diff --git a/testdata/validation/root-terminal-options.json b/testdata/validation/root-terminal-options.json new file mode 100644 index 00000000..1a9e3689 --- /dev/null +++ b/testdata/validation/root-terminal-options.json @@ -0,0 +1,34 @@ +{ + "candidateCommit": "1aa9be87d18cf941e5020f19e8622c26ba4b10ac", + "runId": 38277312274, + "runAttempt": 1, + "admissionRecordSha256": "ff199cf1a3913ce407aaf46fd8bb3c93492975a329e779f26ffb499352a9a291", + "terminalRecordSha256": "a9841d9339e3784ea12572569c32e45b58a996325c62dc57a5925ad76271b9c9", + "admissionOptions": { + "scan_mode": 2, + "remove_corrupted": false, + "recreate_missing": false, + "update_parity": false, + "recursive": true, + "dry_run": false, + "no_lock": false, + "timeout": null, + "pool_index": 0, + "set_index": 0 + }, + "terminalOptions": { + "scan_mode": 2, + "remove_corrupted": false, + "recreate_missing": false, + "update_parity": false, + "recursive": true, + "dry_run": false, + "no_lock": false, + "timeout": { + "secs": 300, + "nanos": 0 + }, + "pool_index": 0, + "set_index": 0 + } +} From f3cdcd1c19bea4e751067e163fd5ae381b9bf62c Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 13 Oct 2026 01:37:32 +0800 Subject: [PATCH 196/343] fix(validation): keep MRF storage accessible during partial writes --- docs/validation-migration-map.md | 2 + scripts/validation_heal/drive_gate.py | 77 ++++++++++-- scripts/validation_heal/fleet_node.py | 2 + scripts/validation_heal/persistence_mrf.py | 36 +++--- test_validation_heal_persistence_mrf.py | 136 +++++++++++++++++++-- 5 files changed, 217 insertions(+), 36 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index d0ff2b98..c2677507 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1985,3 +1985,5 @@ F07 diagnostic `38277312274/1` pins helper `c0d69a3` and the unchanged `1aa9` ca B07 read-quorum error propagation now has a real external-part regression in the companion RustFS PR. Reverting the dominant-I/O return reproduces `Io(NotFound)` at R-1; the fix preserves object-bound `InsufficientReadQuorum` in both legacy and streaming setup. Exactly R and healthy reads return every original payload byte. All 123 reader tests pass on Rust 1.99. This is local product regression evidence; fixed-candidate live run `38275626918/1`, H2 deferred responsibility and startup failures remain retained, and canonical PASS stays 101/158. F07 terminal reconciliation now accepts only the source-defined scheduler transition from an explicit null admission timeout to exactly 300 seconds with zero nanoseconds. The two reviewed candidate commits share the exact scheduler/config sources, and the dedicated fleet verifies `RUSTFS_HEAL_TASK_TIMEOUT_SECS=300` in the actual process environment, including a fresh witness after fault restart. Every other option remains exact; explicit or checkpoint-consumed budgets cannot be refunded, and terminal/report JSON types must agree. The original H1 DB/DV truncation and future-schema records from `38277312274/1` reproduce all four prior reconciliation refusals and pass this derived retained-record check after the fix. Their original FAIL results remain unchanged; complete rescan, physical/S3 reconciliation and new live execution are still required. The 639 Heal contracts pass with eight platform skips. Two fresh sequential final reviews cover the exact normalization, source/runtime boundary, budget preservation and negative regression inputs without unresolved findings. Retained-scene recovery `38281391802/1` remains queued behind the existing shared fleet chain; no later fleet replay is started before clean recovery. Canonical PASS stays 101/158. + +The original H1 DB/DV MRF logs show repeated committed-checkpoint publication failures and durable-admission fallback under whole-member DAC denial. The denial also hid system metadata before any readable committed anchor existed; extending the observation deadline cannot manufacture an original durable MRF responsibility. F07 now prepares its partial-write fixture with a permission gate on only the exact run-owned business bucket in the dedicated `mrf-crash` runtime. Actual candidate-namespace probes must show business read/write EACCES and successful system-format read plus exclusive system probe creation/removal under the same UID/GID and removed DAC capabilities, both before ACKs and after restart. Scope/path/owner/link checks and original device/inode/permissions fence activation and recovery; the existing whole-drive gate remains available to its other cases. Real committed ACK-bound responsibility, CRC injection, retained original records, diagnostics and complete physical/S3 reconciliation remain required. The 644 Heal contracts pass with eight platform skips; two fresh sequential final reviews have no unresolved correctness, simplicity, coverage, security, durability or compatibility finding. No new live result or canonical PASS is claimed before clean recovery and serialized replay. diff --git a/scripts/validation_heal/drive_gate.py b/scripts/validation_heal/drive_gate.py index a7507207..11be8206 100644 --- a/scripts/validation_heal/drive_gate.py +++ b/scripts/validation_heal/drive_gate.py @@ -4,6 +4,7 @@ import json import os from pathlib import Path +import re import stat import subprocess @@ -39,6 +40,24 @@ def require(value, message): if not value: raise ValueError(message) +def gate_folder(state, roots, disk, scope): + folder = roots[disk] / "data" + if scope == "disk": return folder + require(scope == "bucket", "invalid drive gate scope") + marker = state / "owner.json" + info = marker.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_uid == 0 and info.st_nlink == 1 + and info.st_size < 16384 and info.st_mode & 0o022 == 0, "invalid bucket gate owner marker") + owner = json.loads(marker.read_text()) + require(owner.get("runtimeProfile") == "mrf-crash" + and isinstance(owner.get("bucket"), str) and re.fullmatch(r"rv-heal-[0-9a-f]{16}", owner["bucket"]), + "bucket gate requires the dedicated owned MRF fixture") + info = folder.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and stat.S_IMODE(info.st_mode) == 0o700, + "bucket gate private data root changed") + return folder / owner["bucket"] + + def checked_gate(state, roots): path = state / "drive-gate.json" if not path.exists(): return None @@ -49,22 +68,38 @@ def checked_gate(state, roots): require(value.get("schemaVersion") == 1 and value.get("mode") in {"read-only", "permission-denied"} and value.get("phase", "active") in {"armed", "active"} and type(value.get("disk")) is int and 0 <= value["disk"] < len(roots) - and value.get("dataPath") == str(roots[value["disk"]] / "data") - and type(value.get("originalMode")) is int and value["originalMode"] == 0o700, + and type(value.get("originalMode")) is int, "drive gate no longer matches the run-owned data root") + scope = value.get("scope", "disk") + require(value.get("dataPath") == str(gate_folder(state, roots, value["disk"], scope)) + and (value["originalMode"] == 0o700 if scope == "disk" else + value["mode"] == "permission-denied" and value["originalMode"] in {0o700, 0o755, 0o777}), + "drive gate no longer matches its exact owned scope or original permissions") return value -def apply_gate(state, roots, disk, mode): +def apply_gate(state, roots, disk, mode, *, scope="disk"): require(type(disk) is int and 0 <= disk < len(roots) and mode in {"read-only", "permission-denied"}, "invalid drive gate") require(checked_gate(state, roots) is None, "a drive gate is already active") - folder = roots[disk] / "data" + folder = gate_folder(state, roots, disk, scope) info = folder.lstat() - require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and stat.S_IMODE(info.st_mode) == 0o700, - "expected private run-owned root with original mode 0700") + original_mode = stat.S_IMODE(info.st_mode) + require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 + and (original_mode == 0o700 if scope == "disk" else + mode == "permission-denied" and original_mode in {0o700, 0o755, 0o777}), + "expected owned gate directory with supported original permissions") + if scope == "bucket": + objects = (folder / "objects").lstat() + probe = (folder / "objects/0000.bin").lstat() + read_target = (folder / "objects/0000.bin/xl.meta").lstat() + require(stat.S_ISDIR(objects.st_mode) and objects.st_uid == 0 + and stat.S_ISDIR(probe.st_mode) and probe.st_uid == 0 + and stat.S_ISREG(read_target.st_mode) and read_target.st_uid == 0 and read_target.st_nlink == 1, + "bucket gate needs the original non-linked fixture metadata") value = {"schemaVersion": 1, "disk": disk, "mode": mode, "dataPath": str(folder), - "originalMode": 0o700, "device": info.st_dev, "inode": info.st_ino, + "originalMode": original_mode, "device": info.st_dev, "inode": info.st_ino, "phase": "armed" if mode == "permission-denied" else "active"} + if scope == "bucket": value["scope"] = scope # Intent precedes chmod/start. Cleanup can restore a partially applied gate. fd = os.open(state / "drive-gate.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) with os.fdopen(fd, "w") as stream: @@ -81,7 +116,7 @@ def activate_gate(state, roots): folder = Path(value["dataPath"]) info = folder.lstat() require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and info.st_dev == value["device"] - and info.st_ino == value["inode"] and stat.S_IMODE(info.st_mode) == 0o700, + and info.st_ino == value["inode"] and stat.S_IMODE(info.st_mode) == value["originalMode"], "armed permission root changed before live injection") # Keep the recovery intent before chmod. A partially activated gate remains # recoverable from the armed marker and original inode/mode. @@ -167,16 +202,34 @@ def probe_gate(state, roots, pid): observed = json.loads(result.stdout) read = None if value["mode"] == "permission-denied": - # A denied write alone cannot prove a stale member cannot read. Use - # the same namespace/identity/capability boundary to open its format. + # A denied write alone cannot prove a stale member cannot read. + relative = "objects/0000.bin/xl.meta" if value.get("scope") == "bucket" else ".rustfs.sys/format.json" read_args = [*args[:-4], "python3", "-c", READ_PROBE_SOURCE, - value["dataPath"] + "/.rustfs.sys/format.json"] + value["dataPath"] + "/" + relative] result = subprocess.run(read_args, capture_output=True, text=True, timeout=15) require(result.returncode == 0, "service-namespace read probe did not complete") read = json.loads(result.stdout) validate_probe(read, value["mode"], identity) - return {**validate_probe(observed, value["mode"], identity), "disk": value["disk"], + control = None + if value.get("scope") == "bucket": + system = roots[value["disk"]] / "data/.rustfs.sys" + info = system.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0, "MRF control root is not an owned non-linked directory") + control = {} + for label, source, path in (("write", PROBE_SOURCE, system / ".validation-mrf-write-probe"), + ("read", READ_PROBE_SOURCE, system / "format.json")): + result = subprocess.run([*args[:-4], "python3", "-c", source, str(path)], + capture_output=True, text=True, timeout=15) + require(result.returncode == 0, "service-namespace MRF control probe did not complete") + witness = json.loads(result.stdout) + require(type(witness.get("errno")) is int and witness["errno"] == 0 + and witness.get("uid") == identity["uid"] == 0 and witness.get("gid") == identity["gid"] == 0, + "bucket fault also blocked system metadata or changed the MRF probe identity") + control[label] = witness + control["sameServiceIdentity"] = True + return {**validate_probe(observed, value["mode"], identity), "disk": value["disk"], "scope": value.get("scope", "disk"), "dataPath": value["dataPath"], "readProbe": read, + "systemMetadataControl": control, "systemMetadataAccessible": control is not None, "privateMountNamespace": True, "hostMountRemounted": False, "observed": observed} diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 941a9d40..fd880cc7 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -760,6 +760,8 @@ def drive_gate(request): stopped(owner) if request.get("mode") == "clear": return gate_module().release_gate(state, roots) require(quorum_gate_module().checked(state, roots) is None, "single drive gate cannot share a quorum gate") + if request.get("scope", "disk") != "disk": + return gate_module().apply_gate(state, roots, request.get("disk"), request.get("mode"), scope=request["scope"]) return gate_module().apply_gate(state, roots, request.get("disk"), request.get("mode")) diff --git a/scripts/validation_heal/persistence_mrf.py b/scripts/validation_heal/persistence_mrf.py index e770d231..1d1aca24 100644 --- a/scripts/validation_heal/persistence_mrf.py +++ b/scripts/validation_heal/persistence_mrf.py @@ -99,6 +99,17 @@ def observe(fleet, label): return fleet.record(label, {node: fleet.remote(node, "mrf-records", quiescent=True) for node in fleet.nodes}) +def validate_member_denial(value, disk): + control = value.get("systemMetadataControl") or {} + require(value.get("scope") == "bucket" and type(value.get("disk")) is int and value["disk"] == disk + and value.get("readProbe", {}).get("errno") == 13 and value.get("errno") == 13 + and value.get("dacBypassRemoved") is True and value.get("systemMetadataAccessible") is True + and control.get("sameServiceIdentity") is True + and all(type(control.get(kind, {}).get("errno")) is int and control[kind]["errno"] == 0 + for kind in ("read", "write")), + "F07 needs exact business read/write denial with accessible candidate system metadata") + + def run(args, topology, nodes, fixture, candidate): require(topology in {"H1", "H2"} and fixture in {"DB", "DV"}, "F07 MRF source fixture changed") directory = args.evidence / fixture / topology / "mrf-checksum" @@ -140,11 +151,11 @@ def run(args, topology, nodes, fixture, candidate): target = map_target(before, bucket, entry["key"], "data", 1, entry["versionId"], latest_required=False) shard = next(row for row in reversed(target["cohort"]) if row["node"] == nodes[-1]) fleet.drive_gates.add(shard["node"]) - fleet.record("actual-partial-write-gate", fleet.remote(shard["node"], "drive-gate", disk=shard["disk"], mode="permission-denied")) + fleet.record("actual-partial-write-gate", fleet.remote(shard["node"], "drive-gate", disk=shard["disk"], + mode="permission-denied", scope="bucket")) fleet.start() denial = fleet.record("actual-member-denial", fleet.remote(shard["node"], "drive-gate-probe")) - require(denial.get("readProbe", {}).get("errno") == 13 and denial.get("errno") == 13 - and denial.get("dacBypassRemoved") is True, "F07 MRF fixture has no actual member read/write denial") + validate_member_denial(denial, shard["disk"]) earliest = int(time.time() * 1000) - 5000 pending = {"operation": "partial-write-durable-ACKs"} expected, obligations, _ = mrf.acknowledge(recorder, bucket, "offline-put", expected, fixture=fixture) @@ -189,21 +200,16 @@ def run(args, topology, nodes, fixture, candidate): **fleet.remote(node, "journal-apply", **params)})) retained_corruption(original, observe(fleet, "injected-stopped-MRF-records"), receipts, bucket) window = JournalWindow(fleet, "F07-corrupt-MRF-replay", event_targets={"rustfs::heal::mrf"}) - # An active chmod gate prevents the candidate's disk initialization. - # All services are stopped here; re-arm it before startup and activate - # it again only after readiness, preserving the original denied member. - released = fleet.record("stopped-member-permission-reset", fleet.remote(shard["node"], "drive-gate", mode="clear")) - require(released.get("state") == "CLEAR" and released.get("originalPermissionsRestored") is True, - "F07 denied member was not restored while stopped") - rearmed = fleet.record("stopped-member-gate-rearmed", fleet.remote(shard["node"], "drive-gate", - disk=shard["disk"], mode="permission-denied")) - require(rearmed.get("state") == "ARMED" and rearmed.get("phase") == "armed" - and type(rearmed.get("disk")) is int and rearmed["disk"] == shard["disk"], + # Only business data is denied; disk initialization can still read + # system metadata. Keep the same gate active through startup so MRF + # replay cannot repair the target before the readiness observation. + gate = fleet.record("stopped-original-business-gate", fleet.remote(shard["node"], "gate-state")) + require(gate.get("scope") == "bucket" and gate.get("mode") == "permission-denied" + and gate.get("phase") == "active" and type(gate.get("disk")) is int and gate["disk"] == shard["disk"], "F07 restart gate changed its original denied member") fleet.start() denied = fleet.record("restarted-original-member-denial", fleet.remote(shard["node"], "drive-gate-probe")) - require(denied.get("readProbe", {}).get("errno") == 13 and denied.get("errno") == 13 - and denied.get("dacBypassRemoved") is True, "F07 restart lost its actual member read/write denial") + validate_member_denial(denied, shard["disk"]) deadline = time.monotonic() + 60 while True: try: diff --git a/test_validation_heal_persistence_mrf.py b/test_validation_heal_persistence_mrf.py index b6425fad..2c8e665c 100644 --- a/test_validation_heal_persistence_mrf.py +++ b/test_validation_heal_persistence_mrf.py @@ -4,8 +4,10 @@ import copy from contextlib import ExitStack import hashlib +import json import os from pathlib import Path +import stat import tempfile from types import SimpleNamespace import unittest @@ -13,11 +15,118 @@ import zlib from unittest.mock import patch -from scripts.validation_heal import journal_fault as journal, persistence_mrf as suite +from scripts.validation_heal import journal_fault as journal, persistence_mrf as suite, drive_gate import test_validation_heal_mrf as fixtures class MrfCorruptionTests(unittest.TestCase): + def bucket_gate_fixture(self, root): + state, disk = root / "state", root / "disk" + state.mkdir() + data = disk / "data"; data.mkdir(parents=True, mode=0o700) + system = data / ".rustfs.sys"; system.mkdir(mode=0o700) + (system / "format.json").write_bytes(b"format-control") + bucket = data / "rv-heal-0123456789abcdef" + bucket.mkdir(mode=0o755) + obj = bucket / "objects/0000.bin"; obj.mkdir(parents=True) + (obj / "xl.meta").write_bytes(b"original-fixture-metadata") + (state / "owner.json").write_text(json.dumps({"runtimeProfile": "mrf-crash", "bucket": bucket.name})) + return state, disk, bucket, system + + def rooted_lstat(self): + original = Path.lstat + def observe(path): + value = original(path) + return SimpleNamespace(st_mode=value.st_mode, st_uid=0, st_nlink=value.st_nlink, + st_size=value.st_size, st_dev=value.st_dev, st_ino=value.st_ino) + return patch.object(Path, "lstat", observe) + + def test_bucket_gate_denies_only_owned_business_root_and_restores_original_permissions(self): + with tempfile.TemporaryDirectory() as temporary: + state, disk, bucket, system = self.bucket_gate_fixture(Path(temporary)) + original = {path: (path.stat().st_mode, path.stat().st_ino) for path in (disk / "data", bucket, system)} + with self.rooted_lstat(): + armed = drive_gate.apply_gate(state, [disk], 0, "permission-denied", scope="bucket") + self.assertEqual((armed["scope"], armed["dataPath"]), ("bucket", str(bucket))) + drive_gate.activate_gate(state, [disk]) + self.assertEqual(stat.S_IMODE(bucket.stat().st_mode), 0) + self.assertEqual((system.stat().st_mode, system.stat().st_ino), original[system]) + self.assertEqual((disk / "data").stat().st_mode, original[disk / "data"][0]) + self.assertEqual((system / "format.json").read_bytes(), b"format-control") + self.assertTrue(drive_gate.release_gate(state, [disk])["originalPermissionsRestored"]) + self.assertEqual((bucket.stat().st_mode, bucket.stat().st_ino), original[bucket]) + self.assertEqual((bucket / "objects/0000.bin/xl.meta").read_bytes(), b"original-fixture-metadata") + + def test_bucket_gate_refuses_foreign_runtime_path_links_and_changed_scope(self): + for change in ("runtime", "bucket", "scope", "mode", "linked-bucket", "linked-objects", "linked-object", "hardlinked-meta"): + with self.subTest(change=change), tempfile.TemporaryDirectory() as temporary: + state, disk, bucket, _ = self.bucket_gate_fixture(Path(temporary)) + owner = json.loads((state / "owner.json").read_text()) + if change == "runtime": owner["runtimeProfile"] = "incarnation-mrf" + elif change == "bucket": owner["bucket"] = "../foreign" + elif change == "hardlinked-meta": os.link(bucket / "objects/0000.bin/xl.meta", bucket / "linked-meta") + elif change.startswith("linked-"): + folder = {"linked-bucket": bucket, "linked-objects": bucket / "objects", "linked-object": bucket / "objects/0000.bin"}[change] + held = folder.with_name(folder.name + "-held"); folder.rename(held); folder.symlink_to(held, target_is_directory=True) + (state / "owner.json").write_text(json.dumps(owner)) + with self.rooted_lstat(), self.assertRaises(ValueError): + drive_gate.apply_gate(state, [disk], 0, "read-only" if change == "mode" else "permission-denied", + scope="foreign" if change == "scope" else "bucket") + self.assertFalse((state / "drive-gate.json").exists()) + + def test_bucket_gate_marker_cannot_redirect_activation_or_cleanup(self): + with tempfile.TemporaryDirectory() as temporary: + state, disk, bucket, system = self.bucket_gate_fixture(Path(temporary)) + with self.rooted_lstat(): + drive_gate.apply_gate(state, [disk], 0, "permission-denied", scope="bucket") + marker = state / "drive-gate.json"; value = json.loads(marker.read_text()) + value["dataPath"] = str(system); marker.write_text(json.dumps(value)); marker.chmod(0o600) + with self.assertRaises(ValueError): drive_gate.activate_gate(state, [disk]) + with self.assertRaises(ValueError): drive_gate.release_gate(state, [disk]) + self.assertEqual(stat.S_IMODE(bucket.stat().st_mode), 0o755) + self.assertEqual(stat.S_IMODE(system.stat().st_mode), 0o700) + + def test_bucket_probe_keeps_same_candidate_namespace_and_checks_system_read_and_write(self): + with tempfile.TemporaryDirectory() as temporary: + state, disk, bucket, _ = self.bucket_gate_fixture(Path(temporary)) + for failure in (None, "system-read", "system-write", "system-identity"): + denied = SimpleNamespace(returncode=0, stdout=json.dumps({"uid": 0, "gid": 0, "errno": 13})) + control = {"uid": 0, "gid": 0, "errno": 0} + write, read = copy.deepcopy(control), copy.deepcopy(control) + if failure == "system-read": read["errno"] = 13 + elif failure == "system-write": write["errno"] = 13 + elif failure == "system-identity": write["uid"] = 1000 + outputs = [denied, denied, *(SimpleNamespace(returncode=0, stdout=json.dumps(x)) for x in (write, read))] + value = {"mode": "permission-denied", "scope": "bucket", "phase": "active", "dataPath": str(bucket), "disk": 0} + with self.rooted_lstat(), patch.object(drive_gate, "checked_gate", return_value=value), \ + patch.object(drive_gate, "process_identity", return_value={"uid": 0, "gid": 0, "capEff": 0}), \ + patch.object(Path, "readlink", side_effect=[Path("mnt:[candidate]"), Path("mnt:[host]")]), \ + patch.object(drive_gate.subprocess, "run", side_effect=outputs) as invoke: + if failure is None: + proof = drive_gate.probe_gate(state, [disk], 123) + self.assertTrue(proof["systemMetadataAccessible"]) + self.assertTrue(proof["systemMetadataControl"]["sameServiceIdentity"]) + else: + with self.subTest(failure=failure), self.assertRaises(ValueError): drive_gate.probe_gate(state, [disk], 123) + for args in invoke.call_args_list: + self.assertIn("--mount=/proc/123/ns/mnt", args.args[0]) + self.assertIn("--bounding-set=-dac_override,-dac_read_search", args.args[0]) + + def test_mrf_fixture_refuses_disk_wide_gate_or_failed_system_control(self): + valid = {"scope": "bucket", "disk": 0, "errno": 13, "readProbe": {"errno": 13}, "dacBypassRemoved": True, + "systemMetadataAccessible": True, "systemMetadataControl": {"sameServiceIdentity": True, + "read": {"errno": 0}, "write": {"errno": 0}}} + suite.validate_member_denial(valid, 0) + for change in ("scope", "disk", "read", "write", "control", "identity", "accessible"): + value = copy.deepcopy(valid) + if change == "scope": value["scope"] = "disk" + elif change == "disk": value["disk"] = 1 + elif change in {"read", "write"}: value["systemMetadataControl"][change]["errno"] = 13 + elif change == "control": value.pop("systemMetadataControl") + elif change == "identity": value["systemMetadataControl"]["sameServiceIdentity"] = False + else: value["systemMetadataAccessible"] = False + with self.subTest(change=change), self.assertRaises(ValueError): suite.validate_member_denial(value, 0) + def fixture(self): original = {"node": fixtures.DurableMrfTests().observed()} current, receipts = copy.deepcopy(original), [] @@ -84,13 +193,15 @@ def test_original_corruption_diagnostic_is_bound_to_each_mutated_node_and_typed_ def test_lost_injection_ack_registers_recovery_before_mutation_and_keeps_namespace(self): self.run_until_corruption_failure('lost-ack') - def test_corrupt_replay_rearms_the_same_stopped_gate_before_startup(self): + def test_corrupt_replay_preserves_the_same_active_business_gate_through_startup(self): actions = self.run_until_corruption_failure('replay') replay = actions[actions.index('journal-apply') + 1:] - self.assertEqual(replay[:4], ['clear', 'arm', 'start', 'drive-gate-probe']) + self.assertEqual(replay[:3], ['gate-state', 'start', 'drive-gate-probe']) + self.assertNotIn('clear', replay[:replay.index('drive-gate-probe') + 1]) + self.assertNotIn('arm', replay) - def test_failed_stopped_permission_reset_keeps_rescue_and_does_not_restart(self): - actions = self.run_until_corruption_failure('bad-reset') + def test_changed_stopped_business_gate_keeps_rescue_and_does_not_restart(self): + actions = self.run_until_corruption_failure('bad-state') replay = actions[actions.index('journal-apply') + 1:] self.assertNotIn('start', replay) @@ -115,12 +226,19 @@ def remote(self, node, action, **params): if action == 'drive-gate': actions.append('clear' if params['mode'] == 'clear' else 'arm') if params['mode'] == 'clear': - return {'state': 'CLEAR', 'originalPermissionsRestored': phase != 'bad-reset'} - return {'state': 'ARMED', 'phase': 'armed', 'disk': 0} + return {'state': 'CLEAR', 'originalPermissionsRestored': True} + assert params['scope'] == 'bucket' + return {'state': 'ARMED', 'phase': 'armed', 'disk': 0, 'scope': 'bucket'} + if action == 'gate-state': + actions.append(action) + return {'mode': 'permission-denied', 'phase': 'active', 'disk': 0, + 'scope': 'disk' if phase == 'bad-state' else 'bucket'} if action == "mrf-records": return original.observed() if action == "drive-gate-probe": actions.append(action) - return {"errno": 13, "readProbe": {"errno": 13}, "dacBypassRemoved": True} + return {"errno": 13, "readProbe": {"errno": 13}, "dacBypassRemoved": True, "scope": "bucket", "disk": 0, + "systemMetadataAccessible": True, "systemMetadataControl": {"sameServiceIdentity": True, + "read": {"errno": 0}, "write": {"errno": 0}}} if action == "journal-apply": actions.append(action) assert (node, params) in self.journal_faults @@ -150,7 +268,7 @@ def inventory(self, *args, **kwargs): return {} stack.enter_context(patch.object(suite, 'diagnostics', side_effect=OSError('stop after replay readiness proof'))) result = suite.run(args, "H1", ["node"], "DV", {"source_sha": "c" * 40, "package_sha256": "d" * 64}) self.assertEqual((result["verdict"], result["cleanup"]), ("FAIL", "FAIL")) - self.assertEqual(result["failure"]["type"], "ValueError" if phase == 'bad-reset' else "OSError") + self.assertEqual(result["failure"]["type"], "ValueError" if phase == 'bad-state' else "OSError") self.assertEqual(len(captured[0].journal_faults), 1) self.assertFalse(result["canonicalCasePassClaimed"]) return actions From 4aea05db421bc1c1d436e0b7590cddb99b3040fb Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 13 Oct 2026 02:43:02 +0800 Subject: [PATCH 197/343] fix(validation): corrupt every owned MRF replay source --- docs/validation-migration-map.md | 2 + scripts/validation_heal/persistence_mrf.py | 45 +++++++++--- test_validation_heal_persistence_mrf.py | 85 ++++++++++++++++++++-- 3 files changed, 113 insertions(+), 19 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index c2677507..17b4ba72 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1987,3 +1987,5 @@ B07 read-quorum error propagation now has a real external-part regression in the F07 terminal reconciliation now accepts only the source-defined scheduler transition from an explicit null admission timeout to exactly 300 seconds with zero nanoseconds. The two reviewed candidate commits share the exact scheduler/config sources, and the dedicated fleet verifies `RUSTFS_HEAL_TASK_TIMEOUT_SECS=300` in the actual process environment, including a fresh witness after fault restart. Every other option remains exact; explicit or checkpoint-consumed budgets cannot be refunded, and terminal/report JSON types must agree. The original H1 DB/DV truncation and future-schema records from `38277312274/1` reproduce all four prior reconciliation refusals and pass this derived retained-record check after the fix. Their original FAIL results remain unchanged; complete rescan, physical/S3 reconciliation and new live execution are still required. The 639 Heal contracts pass with eight platform skips. Two fresh sequential final reviews cover the exact normalization, source/runtime boundary, budget preservation and negative regression inputs without unresolved findings. Retained-scene recovery `38281391802/1` remains queued behind the existing shared fleet chain; no later fleet replay is started before clean recovery. Canonical PASS stays 101/158. The original H1 DB/DV MRF logs show repeated committed-checkpoint publication failures and durable-admission fallback under whole-member DAC denial. The denial also hid system metadata before any readable committed anchor existed; extending the observation deadline cannot manufacture an original durable MRF responsibility. F07 now prepares its partial-write fixture with a permission gate on only the exact run-owned business bucket in the dedicated `mrf-crash` runtime. Actual candidate-namespace probes must show business read/write EACCES and successful system-format read plus exclusive system probe creation/removal under the same UID/GID and removed DAC capabilities, both before ACKs and after restart. Scope/path/owner/link checks and original device/inode/permissions fence activation and recovery; the existing whole-drive gate remains available to its other cases. Real committed ACK-bound responsibility, CRC injection, retained original records, diagnostics and complete physical/S3 reconciliation remain required. The 644 Heal contracts pass with eight platform skips; two fresh sequential final reviews have no unresolved correctness, simplicity, coverage, security, durability or compatibility finding. No new live result or canonical PASS is claimed before clean recovery and serialized replay. + +Independent examination of the original F07 H2 DB stopped records in `38277312274/1` establishes eight valid committed replay sources on vm000, across four disks and both sequence 10 and 11 slots. The old single-replica fault corrupts only disk 0 / slot 0; seven sources remain complete with the same six responsibilities. The fixed candidate intentionally selects a healthy committed replica even when another is damaged, explaining the missing inspection-failure diagnostic. F07 now validates every original replay source on every bound responsibility owner before applying any CRC fault, including readable older slots. Foreign or empty responsibilities refuse construction; every apply is registered for rescue before invocation, and a missing or substituted receipt prevents restart. Only owned business-record CRC bytes change, while manifests, lifecycle companions and other records remain exact retention controls. The original run remains FAIL/cleanup FAIL. The orchestration regression fails with the old single-replica loop; 108 focused contracts and the 648-test Heal suite pass with eight platform skips. Complete live execution, all native barriers and canonical PASS remain unclaimed. diff --git a/scripts/validation_heal/persistence_mrf.py b/scripts/validation_heal/persistence_mrf.py index 1d1aca24..87c78d23 100644 --- a/scripts/validation_heal/persistence_mrf.py +++ b/scripts/validation_heal/persistence_mrf.py @@ -55,6 +55,26 @@ def indexed(observed): return result +def corruption_targets(observed, bucket): + """Cover every committed replay source, including readable older slots.""" + selected = records.checkpoint(observed) + require(selected.get("present") is True, "F07 MRF owner has no committed replay source") + entries, targets = indexed(observed), [] + for disk in range(4): + for slot in (0, 1): + manifest = entries[disk, f".rustfs.sys/.heal-mrf-commit.{slot}.bin"] + if manifest is None: + continue + path = f".rustfs.sys/.heal-mrf-snapshot.{slot}.bin" + raw = entries[disk, path] + binding = records.commit(manifest, raw) + journal.owned_mrf_intents(records.decode_journal(raw), bucket) + require(binding["owner"] == selected["owner"], "F07 MRF replay sources mixed process owners") + targets.append({"disk": disk, "relativePath": path, **binding, "before": journal.entry(raw)}) + require(targets, "F07 MRF corruption has no bound replay source") + return targets + + def retained_corruption(original, current, receipts, bucket): require(set(original) == set(current) and len(original) in {1, 4}, "F07 MRF evidence lost a bound physical node") expected, changed = {node: indexed(value) for node, value in original.items()}, set() @@ -180,25 +200,26 @@ def run(args, topology, nodes, fixture, candidate): fleet.record("original-durable-responsibilities", {"obligations": obligations, "bindings": [{"identity": list(key), "owners": value} for key, value in bindings.items()]}) authorities = fleet.snapshot("original-partial-write-physical", quiescent=True) + mutation_plan = {node: corruption_targets(observed, bucket) for node, observed in original.items() + if any(owner["node"] == node for owners in bindings.values() for owner in owners)} + require(mutation_plan, "F07 MRF fault plan has no original responsibility owner") + fleet.record("all-original-committed-MRF-corruption-targets", mutation_plan) receipts = [] - for node, observed in original.items(): - if not any(owner["node"] == node for owners in bindings.values() for owner in owners): continue - selected = stopped[node] - journal.owned_mrf_intents(selected["intents"], bucket) - entries = indexed(observed) - # One committed replica per process is the corruption target; - # every untouched companion/replica remains a retention control. - for replica in selected["replicas"][:1]: - disk, path = replica["disk"], f".rustfs.sys/.heal-mrf-snapshot.{replica['slot']}.bin" - raw = entries[disk, path] - require(raw and records.decode_journal(raw), "F07 committed payload has no original responsibility") + # A healthy replica or older committed slot can still replay. Corrupt + # every owned source while retaining all manifests and lifecycle bytes. + for node, targets in mutation_plan.items(): + for target in targets: + disk, path = target["disk"], target["relativePath"] params = {"family": "mrf", "taskId": None, "disk": disk, "relativePath": path, "faultId": f"F07-mrf-checksum-{nodes.index(node)}-{disk}-{path[-5]}", "kind": "checksum", - "expectedBefore": journal.entry(raw)} + "expectedBefore": target["before"]} fleet.journal_faults.append((node, params.copy())) receipts.append(fleet.record("actual-original-MRF-CRC-injection", {"node": node, **fleet.remote(node, "journal-apply", **params)})) retained_corruption(original, observe(fleet, "injected-stopped-MRF-records"), receipts, bucket) + require({(row.get("node"), row.get("disk"), row.get("relativePath")) for row in receipts} == + {(node, row["disk"], row["relativePath"]) for node, targets in mutation_plan.items() for row in targets}, + "F07 MRF corruption omitted an original readable replay source") window = JournalWindow(fleet, "F07-corrupt-MRF-replay", event_targets={"rustfs::heal::mrf"}) # Only business data is denied; disk initialization can still read # system metadata. Keep the same gate active through startup so MRF diff --git a/test_validation_heal_persistence_mrf.py b/test_validation_heal_persistence_mrf.py index 2c8e665c..88655d02 100644 --- a/test_validation_heal_persistence_mrf.py +++ b/test_validation_heal_persistence_mrf.py @@ -141,6 +141,66 @@ def fixture(self): row.update(journal.entry(corrupted), rawBase64=base64.b64encode(corrupted).decode()) return original, current, receipts, "rv-heal-0123456789abcdef" + def add_older_slot(self, observed, *, foreign=False, empty=False): + fixture = fixtures.DurableMrfTests() + raw = fixture.journal() + if foreign: + body = raw[:-4].replace(b"rv-heal-0123456789abcdef", b"rv-heal-fedcba9876543210") + raw = body + struct.pack(" Date: Tue, 13 Oct 2026 09:40:02 +0800 Subject: [PATCH 198/343] fix(validation): identify mismatched recovery owner markers --- scripts/validation_heal/fleet_node.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index fd880cc7..d1ad81b4 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -389,7 +389,16 @@ def owned(run_id): "disk identity changed") for root in roots: marker = checked(root / "owner.json", root) - require(marker.stat().st_uid == 0 and json.loads(marker.read_text()) == owner, "drive owner differs") + require(marker.stat().st_uid == 0, f"drive owner marker is not root-owned: {root}") + observed = json.loads(marker.read_text()) + if observed != owner: + expected_bytes = json.dumps(owner, sort_keys=True, separators=(",", ":")).encode() + observed_bytes = json.dumps(observed, sort_keys=True, separators=(",", ":")).encode() + raise ValueError( + f"drive owner differs at {root}: expected_sha256={hashlib.sha256(expected_bytes).hexdigest()} " + f"observed_sha256={hashlib.sha256(observed_bytes).hexdigest()} " + f"observed_run_id={observed.get('runId')} observed_bucket={observed.get('bucket')}" + ) if owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf"}: phase_file = checked(state / "C11-phase.json", state) if phase_file.exists(): From 4bd20d036b0f478f7da8c26605f6621cec1c1e91 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 16:32:42 +0800 Subject: [PATCH 199/343] fix(validation): bind each MRF injection to its original replica --- docs/validation-migration-map.md | 2 + scripts/validation_heal/fleet_node.py | 18 ++-- scripts/validation_heal/journal_fault.py | 16 ++++ scripts/validation_heal/mrf_records.py | 12 ++- scripts/validation_heal/persistence_mrf.py | 13 ++- test_validation_heal_journal_fault.py | 14 +++- test_validation_heal_mrf.py | 21 +++++ test_validation_heal_persistence_mrf.py | 95 +++++++++++++++++++++- 8 files changed, 170 insertions(+), 21 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 17b4ba72..bd1356a5 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1989,3 +1989,5 @@ F07 terminal reconciliation now accepts only the source-defined scheduler transi The original H1 DB/DV MRF logs show repeated committed-checkpoint publication failures and durable-admission fallback under whole-member DAC denial. The denial also hid system metadata before any readable committed anchor existed; extending the observation deadline cannot manufacture an original durable MRF responsibility. F07 now prepares its partial-write fixture with a permission gate on only the exact run-owned business bucket in the dedicated `mrf-crash` runtime. Actual candidate-namespace probes must show business read/write EACCES and successful system-format read plus exclusive system probe creation/removal under the same UID/GID and removed DAC capabilities, both before ACKs and after restart. Scope/path/owner/link checks and original device/inode/permissions fence activation and recovery; the existing whole-drive gate remains available to its other cases. Real committed ACK-bound responsibility, CRC injection, retained original records, diagnostics and complete physical/S3 reconciliation remain required. The 644 Heal contracts pass with eight platform skips; two fresh sequential final reviews have no unresolved correctness, simplicity, coverage, security, durability or compatibility finding. No new live result or canonical PASS is claimed before clean recovery and serialized replay. Independent examination of the original F07 H2 DB stopped records in `38277312274/1` establishes eight valid committed replay sources on vm000, across four disks and both sequence 10 and 11 slots. The old single-replica fault corrupts only disk 0 / slot 0; seven sources remain complete with the same six responsibilities. The fixed candidate intentionally selects a healthy committed replica even when another is damaged, explaining the missing inspection-failure diagnostic. F07 now validates every original replay source on every bound responsibility owner before applying any CRC fault, including readable older slots. Foreign or empty responsibilities refuse construction; every apply is registered for rescue before invocation, and a missing or substituted receipt prevents restart. Only owned business-record CRC bytes change, while manifests, lifecycle companions and other records remain exact retention controls. The original run remains FAIL/cleanup FAIL. The orchestration regression fails with the old single-replica loop; 108 focused contracts and the 648-test Heal suite pass with eight platform skips. Complete live execution, all native barriers and canonical PASS remain unclaimed. + +The F07 node adapter now validates each exact disk/slot against its frozen committed owner, sequence, payload digest and active bucket incarnation. The all-source plan is validated before the first apply; the selected target still needs its original valid manifest/lifecycle, unchanged before-bytes and stopped candidate. Previously, the adapter decoded the whole cohort on every apply, so the first intended CRC fault prevented the second injection. A real filesystem regression fails at that second call with global revalidation restored, then passes all four disk/slot mutations and byte-exact original rescue after the fix. Global MRF observation remains strict and rejects the same damaged cohort; replica-scoped observations are explicitly marked and cannot certify complete responsibility. Held/accepted/unbound/foreign-incarnation older sources refuse preflight. All 112 focused contracts, 652 Heal tests (eight platform skips), ten migration tests, AST and whitespace checks pass. No new live or canonical PASS is claimed. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index d1ad81b4..ec8a6d14 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1107,7 +1107,7 @@ def journal_operation(request): state, roots, owner = owned(request["runId"]) require(digest(state / "rustfs") == owner["binarySha256"], "durable-record candidate changed") require(owner.get("runtimeProfile") in {"root-restart", "mrf-crash", "journal-bounded", "incarnation-checkpoint", "incarnation-mrf"}, "durable records require the owned recovery runtime") - require(set(request) <= {"runId", "action", "family", "taskId", "disk", "relativePath", "faultId", "kind", "expectedBefore", "expected", "captureId"}, + require(set(request) <= {"runId", "action", "family", "taskId", "disk", "relativePath", "faultId", "kind", "expectedBefore", "expected", "captureId", "mrfIdentity"}, "durable-record adapter accepts no arbitrary system paths or commands") if owner.get("runtimeProfile") == "journal-bounded": require(request.get("family") == "checkpoint" or request.get("family") == "root" @@ -1152,15 +1152,13 @@ def journal_operation(request): if request["action"] == "journal-apply": if request["family"] == "mrf": observed = globals()["mrf"].read_records(state, roots, owner, checked) - committed = globals()["mrf"].checkpoint(observed) - business = module.owned_mrf_intents(committed.get("intents", []), owner["bucket"]) - require(committed.get("present") is True - and all(item.get("lifecycle", {}).get("state") == {"state": "active"} - and item["lifecycle"].get("source_bucket_incarnation_id") - and item["lifecycle"].get("last_operator_acceptance") is None for item in business) - and any(row["disk"] == request["disk"] and request["relativePath"] == - f".rustfs.sys/.heal-mrf-snapshot.{row['slot']}.bin" for row in committed["replicas"]), - "MRF fault lacks an actual committed active owned responsibility on this exact slot") + path = request.get("relativePath", "") + match = re.fullmatch(r"\.rustfs.sys/\.heal-mrf-snapshot\.([01])\.bin", path) + require(match is not None, "MRF fault escaped its exact committed payload path") + committed = globals()["mrf"].checkpoint(observed, replica=(request["disk"], int(match[1]))) + identity = module.active_mrf_identity(committed, owner["bucket"]) + require(module.same_json(request.get("mrfIdentity"), identity), + "MRF fault changed its frozen owner, sequence, payload or bucket incarnation") return module.install(fd, request, owner, fault, globals()["mrf"]) return module.recover(fd, request, owner, fault) diff --git a/scripts/validation_heal/journal_fault.py b/scripts/validation_heal/journal_fault.py index cfe40dab..46bf2a18 100644 --- a/scripts/validation_heal/journal_fault.py +++ b/scripts/validation_heal/journal_fault.py @@ -139,6 +139,22 @@ def owned_mrf_intents(intents, bucket): return owned +def active_mrf_identity(committed, bucket): + """Bind one original committed payload to its active bucket generation.""" + business = owned_mrf_intents(committed.get("intents", []), bucket) + require(committed.get("present") is True + and all(item.get("lifecycle", {}).get("state") == {"state": "active"} + and item["lifecycle"].get("source_bucket_incarnation_id") + and item["lifecycle"].get("last_operator_acceptance") is None for item in business), + "MRF fault lacks an actual committed active owned responsibility on this exact slot") + incarnations = {item["lifecycle"]["source_bucket_incarnation_id"] for item in business} + require(len(incarnations) == 1, "MRF fault mixed bucket incarnations") + incarnation = next(iter(incarnations)) + task_id(incarnation) + return {"owner": committed["owner"], "sequence": committed["sequence"], + "payloadSha256": committed["payloadSha256"], "bucketIncarnationId": incarnation} + + def corrupt_owned_mrf_crc(raw, intents, bucket): owned_mrf_intents(intents, bucket) cursor, selected = 0, None diff --git a/scripts/validation_heal/mrf_records.py b/scripts/validation_heal/mrf_records.py index 86e2e49a..9eb8b6cb 100644 --- a/scripts/validation_heal/mrf_records.py +++ b/scripts/validation_heal/mrf_records.py @@ -181,7 +181,11 @@ def unique_pairs(pairs): return records -def checkpoint(observed): +def checkpoint(observed, *, replica=None): + if replica is not None: + require(isinstance(replica, tuple) and len(replica) == 2 + and all(type(item) is int for item in replica) + and 0 <= replica[0] < 4 and replica[1] in (0, 1), "invalid exact MRF replica") require(observed.get("readOnly") is True and observed.get("stable") is True, "NOT_OBSERVED: MRF publication changed throughout its bounded observation") rows = observed.get("records") @@ -202,6 +206,8 @@ def checkpoint(observed): committed = [] for disk in range(4): for slot in (0, 1): + if replica is not None and (disk, slot) != replica: + continue prefix = ".rustfs.sys/.heal-mrf-" manifest = indexed.get((disk, prefix + f"commit.{slot}.bin")) if manifest is None: continue @@ -219,6 +225,7 @@ def checkpoint(observed): intent["lifecycle"] = records[intent["intentDigest"]] committed.append({**binding, "disk": disk, "slot": slot, "intents": intents}) if not committed: + require(replica is None, "exact MRF committed replica is absent") require(not indexed, "MRF has uncommitted or orphan record files; absence is not established") return {"present": False, "intents": [], "readOnly": True, "allAllowlistedFilesAbsent": True} owners = {row["owner"] for row in committed} @@ -227,4 +234,5 @@ def checkpoint(observed): latest = [row for row in committed if row["sequence"] == sequence] require(len({row["payloadSha256"] for row in latest}) == 1 and all(row["intents"] == latest[0]["intents"] for row in latest), "MRF same-sequence committed replicas disagree") - return {**latest[0], "present": True, "replicas": [{"disk": row["disk"], "slot": row["slot"]} for row in latest]} + return {**latest[0], "present": True, "replicas": [{"disk": row["disk"], "slot": row["slot"]} for row in latest], + **({"replicaScoped": True} if replica is not None else {})} diff --git a/scripts/validation_heal/persistence_mrf.py b/scripts/validation_heal/persistence_mrf.py index 87c78d23..b6f3f8eb 100644 --- a/scripts/validation_heal/persistence_mrf.py +++ b/scripts/validation_heal/persistence_mrf.py @@ -59,6 +59,7 @@ def corruption_targets(observed, bucket): """Cover every committed replay source, including readable older slots.""" selected = records.checkpoint(observed) require(selected.get("present") is True, "F07 MRF owner has no committed replay source") + latest_identity = journal.active_mrf_identity(selected, bucket) entries, targets = indexed(observed), [] for disk in range(4): for slot in (0, 1): @@ -68,9 +69,13 @@ def corruption_targets(observed, bucket): path = f".rustfs.sys/.heal-mrf-snapshot.{slot}.bin" raw = entries[disk, path] binding = records.commit(manifest, raw) - journal.owned_mrf_intents(records.decode_journal(raw), bucket) - require(binding["owner"] == selected["owner"], "F07 MRF replay sources mixed process owners") - targets.append({"disk": disk, "relativePath": path, **binding, "before": journal.entry(raw)}) + exact = records.checkpoint(observed, replica=(disk, slot)) + identity = journal.active_mrf_identity(exact, bucket) + require(identity["owner"] == latest_identity["owner"] + and identity["bucketIncarnationId"] == latest_identity["bucketIncarnationId"], + "F07 MRF replay sources mixed process owners or bucket incarnations") + targets.append({"disk": disk, "relativePath": path, **binding, + "mrfIdentity": identity, "before": journal.entry(raw)}) require(targets, "F07 MRF corruption has no bound replay source") return targets @@ -212,7 +217,7 @@ def run(args, topology, nodes, fixture, candidate): disk, path = target["disk"], target["relativePath"] params = {"family": "mrf", "taskId": None, "disk": disk, "relativePath": path, "faultId": f"F07-mrf-checksum-{nodes.index(node)}-{disk}-{path[-5]}", "kind": "checksum", - "expectedBefore": target["before"]} + "expectedBefore": target["before"], "mrfIdentity": target["mrfIdentity"]} fleet.journal_faults.append((node, params.copy())) receipts.append(fleet.record("actual-original-MRF-CRC-injection", {"node": node, **fleet.remote(node, "journal-apply", **params)})) diff --git a/test_validation_heal_journal_fault.py b/test_validation_heal_journal_fault.py index 11111c22..95c4365b 100644 --- a/test_validation_heal_journal_fault.py +++ b/test_validation_heal_journal_fault.py @@ -136,7 +136,7 @@ def test_future_checkpoint_schema_has_valid_candidate_digest_and_original_scope( with self.assertRaisesRegex(ValueError, "integrity digest"): journal.transform(json.dumps(original).encode(), request, {"bucket": bucket}, mrf) - def test_mrf_adapter_requires_active_original_business_work_on_the_selected_latest_replica(self): + def test_mrf_adapter_requires_active_original_business_work_on_the_exact_frozen_replica(self): raw = (Path(__file__).parent / "testdata/validation/mrf-mixed-payload.bin").read_bytes() bucket = "rv-heal-084db3ab418d55ba" for change in (None, "held", "unbound", "accepted", "other-replica"): @@ -150,12 +150,18 @@ def test_mrf_adapter_requires_active_original_business_work_on_the_selected_late if change == "held": business["lifecycle"]["state"] = {"state": "bucket_incarnation_changed"} elif change == "unbound": business["lifecycle"]["source_bucket_incarnation_id"] = None elif change == "accepted": business["lifecycle"]["last_operator_acceptance"] = {"operator": "test"} - checkpoint = {"present": True, "intents": intents, "replicas": [{"disk": 0, "slot": 0}]} - backend = types.SimpleNamespace(read_records=lambda *a: {}, checkpoint=lambda *a: checkpoint) + checkpoint = {"present": True, "intents": intents, "replicas": [{"disk": 0, "slot": 0}], + "owner": self.token, "sequence": 7, "payloadSha256": hashlib.sha256(raw).hexdigest()} + def exact_checkpoint(observed, *, replica): + if replica != (0, 0): raise ValueError("exact MRF committed replica is absent") + return checkpoint + backend = types.SimpleNamespace(read_records=lambda *a: {}, checkpoint=exact_checkpoint) owner = {"runtimeProfile": "mrf-crash", "bucket": bucket, "binarySha256": "b" * 64} request = {"runId": "gh-12345-1-h2", "action": "journal-apply", "family": "mrf", "taskId": None, "disk": 0, "relativePath": ".rustfs.sys/.heal-mrf-snapshot.1.bin" if change == "other-replica" else ".rustfs.sys/.heal-mrf-snapshot.0.bin", - "kind": "checksum", "faultId": "F07-mrf-checksum-0-0-0", "expectedBefore": journal.entry(raw)} + "kind": "checksum", "faultId": "F07-mrf-checksum-0-0-0", "expectedBefore": journal.entry(raw), + "mrfIdentity": {"owner": self.token, "sequence": 7, "payloadSha256": hashlib.sha256(raw).hexdigest(), + "bucketIncarnationId": self.incarnation}} with patch.object(fleet_node, "owned", return_value=(root, [root], owner)), patch.object(fleet_node, "stopped"), \ patch.object(fleet_node, "digest", return_value="b" * 64), patch.object(fleet_node, "fault_module", return_value=fault), \ patch.dict(fleet_node.__dict__, journal_fault_backend=journal, mrf=backend), \ diff --git a/test_validation_heal_mrf.py b/test_validation_heal_mrf.py index 1d43afd6..d017413c 100644 --- a/test_validation_heal_mrf.py +++ b/test_validation_heal_mrf.py @@ -97,6 +97,27 @@ def test_committed_snapshot_binds_payload_and_lifecycle_on_exact_replicas(self): item.clear(); item.update(disk=0, relativePath=".rustfs.sys/.heal-mrf-commit.0.bin", present=False) with self.assertRaises(ValueError): records.checkpoint(bad) + def test_exact_replica_scope_never_changes_the_global_corruption_refusal(self): + original = self.observed() + changed = copy.deepcopy(original) + row = next(row for row in changed["records"] if row["disk"] == 0 and row["present"] and "snapshot" in row["relativePath"]) + raw = base64.b64decode(row["rawBase64"]) + damaged = raw[:-1] + bytes([raw[-1] ^ 1]) + row.update(size=len(damaged), sha256=hashlib.sha256(damaged).hexdigest(), rawBase64=base64.b64encode(damaged).decode()) + with self.assertRaises(ValueError): records.checkpoint(changed) + with self.assertRaises(ValueError): records.checkpoint(changed, replica=(0, 0)) + exact = records.checkpoint(changed, replica=(2, 0)) + self.assertTrue(exact["replicaScoped"]) + self.assertEqual(exact["replicas"], [{"disk": 2, "slot": 0}]) + self.assertEqual(exact["payloadSha256"], hashlib.sha256(raw).hexdigest()) + with self.assertRaises(ValueError): records.checkpoint(changed, replica=(2, 1)) + for selector in ((True, 0), (0, False), (4, 0), (0, 2), [2, 0], (), (2, 0, 1)): + with self.subTest(selector=selector), self.assertRaises(ValueError): + records.checkpoint(original, replica=selector) + bad_census = copy.deepcopy(changed) + bad_census["records"].pop() + with self.assertRaises(ValueError): records.checkpoint(bad_census, replica=(2, 0)) + def test_manifest_checksum_length_and_nonzero_epoch_are_required(self): raw = self.journal() self.assertEqual(records.commit(self.manifest(raw), raw)["owner"], self.owner) diff --git a/test_validation_heal_persistence_mrf.py b/test_validation_heal_persistence_mrf.py index 88655d02..f84c32d0 100644 --- a/test_validation_heal_persistence_mrf.py +++ b/test_validation_heal_persistence_mrf.py @@ -15,7 +15,7 @@ import zlib from unittest.mock import patch -from scripts.validation_heal import journal_fault as journal, persistence_mrf as suite, drive_gate +from scripts.validation_heal import journal_fault as journal, persistence_mrf as suite, drive_gate, fleet_node, heal_fault import test_validation_heal_mrf as fixtures @@ -201,6 +201,99 @@ def test_corruption_plan_refuses_foreign_or_empty_older_replay_source(self): suite.corruption_targets(original, "rv-heal-0123456789abcdef") self.assertEqual(original, before) + def test_preflight_refuses_an_inactive_or_foreign_incarnation_in_any_older_slot(self): + bucket = "rv-heal-0123456789abcdef" + for change in ("held", "accepted", "unbound", "foreign-incarnation"): + original = fixtures.DurableMrfTests().observed() + self.add_older_slot(original) + row = next(row for row in original["records"] if row["disk"] == 2 + and row["relativePath"] == ".rustfs.sys/.heal-mrf-lifecycle.1.bin") + raw = base64.b64decode(row["rawBase64"]) + value = json.loads(raw[49:]) + record = value["records"][0] + if change == "held": record["state"] = {"state": "bucket_incarnation_changed"} + elif change == "accepted": record["last_operator_acceptance"] = {"operator": "test"} + elif change == "unbound": record["source_bucket_incarnation_id"] = None + else: record["source_bucket_incarnation_id"] = "92345678-1234-4234-8234-123456789abc" + body = json.dumps(value).encode() + raw = b"RFMFLC01\x01" + struct.pack(" Date: Thu, 15 Oct 2026 16:36:21 +0800 Subject: [PATCH 200/343] fix(validation): preserve retained namespace ownership evidence --- docs/validation-migration-map.md | 2 ++ rustfs-table-test.sh | 2 +- scripts/validation_heal/fleet_node.py | 30 ++++++++++++++++++--- test_validation_heal_recovery.py | 39 +++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 4 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index bd1356a5..fe585cf8 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1991,3 +1991,5 @@ The original H1 DB/DV MRF logs show repeated committed-checkpoint publication fa Independent examination of the original F07 H2 DB stopped records in `38277312274/1` establishes eight valid committed replay sources on vm000, across four disks and both sequence 10 and 11 slots. The old single-replica fault corrupts only disk 0 / slot 0; seven sources remain complete with the same six responsibilities. The fixed candidate intentionally selects a healthy committed replica even when another is damaged, explaining the missing inspection-failure diagnostic. F07 now validates every original replay source on every bound responsibility owner before applying any CRC fault, including readable older slots. Foreign or empty responsibilities refuse construction; every apply is registered for rescue before invocation, and a missing or substituted receipt prevents restart. Only owned business-record CRC bytes change, while manifests, lifecycle companions and other records remain exact retention controls. The original run remains FAIL/cleanup FAIL. The orchestration regression fails with the old single-replica loop; 108 focused contracts and the 648-test Heal suite pass with eight platform skips. Complete live execution, all native barriers and canonical PASS remain unclaimed. The F07 node adapter now validates each exact disk/slot against its frozen committed owner, sequence, payload digest and active bucket incarnation. The all-source plan is validated before the first apply; the selected target still needs its original valid manifest/lifecycle, unchanged before-bytes and stopped candidate. Previously, the adapter decoded the whole cohort on every apply, so the first intended CRC fault prevented the second injection. A real filesystem regression fails at that second call with global revalidation restored, then passes all four disk/slot mutations and byte-exact original rescue after the fix. Global MRF observation remains strict and rejects the same damaged cohort; replica-scoped observations are explicitly marked and cannot certify complete responsibility. Held/accepted/unbound/foreign-incarnation older sources refuse preflight. All 112 focused contracts, 652 Heal tests (eight platform skips), ten migration tests, AST and whitespace checks pass. No new live or canonical PASS is claimed. + +Recovery diagnostic `38704914473/1` ran on helper `9c01cb8` after the independent Fault-Tolerance workflow released the shared fleet lock. It refused before the first physical snapshot: vm000 disk 0 no longer contained its owner marker. Its original namespace records show nine absent run roots (disks 0-2 on vm000-vm002), seven surviving roots and all four state directories; every service is stopped. No MRF export, restoration or finalization occurred. This is externally drifted/incomplete evidence, not clean recovery or a case PASS. Namespace inspection now emits bounded regular-file owner fingerprints (raw/canonical digests and UID/GID/mode/inode/device), without marker contents or credentials, so surviving state can be compared with the original reserved receipts. Tables installation previously recursively changed ownership of the entire mount; its chown is now restricted to its own mnmd subtree. That known interference is not asserted to explain the nine missing roots. Original failures, surviving roots and native/canonical gates remain retained pending forensic inspection. diff --git a/rustfs-table-test.sh b/rustfs-table-test.sh index 55259385..1e15747b 100755 --- a/rustfs-table-test.sh +++ b/rustfs-table-test.sh @@ -150,7 +150,7 @@ remote() { ssh -o BatchMode=yes -o ConnectTimeout=10 -p "${SSH_PORT}" "${SSH_USE setup_environment() { local v for v in "${NODES[@]}"; do - remote "${v}" "curl -fsSL --retry 3 -o /tmp/rustfs-table.deb '${PACKAGE_URL}' && ${SUDO[*]} dpkg -i --force-confold /tmp/rustfs-table.deb >/dev/null 2>&1 && ${SUDO[*]} mkdir -p /data/rustfs{1..4}/mnmd && ${SUDO[*]} chown -R rustfs:rustfs /data/rustfs{1..4}" || die "install failed on ${v}" + remote "${v}" "curl -fsSL --retry 3 -o /tmp/rustfs-table.deb '${PACKAGE_URL}' && ${SUDO[*]} dpkg -i --force-confold /tmp/rustfs-table.deb >/dev/null 2>&1 && ${SUDO[*]} mkdir -p /data/rustfs{1..4}/mnmd && ${SUDO[*]} chown -R rustfs:rustfs /data/rustfs{1..4}/mnmd" || die "install failed on ${v}" done write_node_config() { remote "$1" "${SUDO[*]} tee /etc/default/rustfs >/dev/null <<'CFG' diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index ec8a6d14..63b3a53d 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1231,12 +1231,36 @@ def namespace_state(request): for field in ("LoadState", "ActiveState", "SubState", "MainPID"): service[field] = command("systemctl", "show", "-p", field, "--value", unit(request["runId"]), check=False).stdout.strip() + def marker_evidence(folder): + path = checked(folder / "owner.json", folder) + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + except FileNotFoundError: + return {"present": False} + try: + info = os.fstat(descriptor) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size <= 16384, + "namespace owner is not a bounded non-linked regular file") + raw = os.read(descriptor, 16385) + after = os.fstat(descriptor) + require(len(raw) == info.st_size and (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + == (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns), + "namespace owner changed during inspection") + value = json.loads(raw) + require(isinstance(value, dict), "namespace owner is not an object") + return {"present": True, "size": len(raw), "uid": info.st_uid, "gid": info.st_gid, + "mode": stat.S_IMODE(info.st_mode), "device": info.st_dev, "inode": info.st_ino, + "sha256": hashlib.sha256(raw).hexdigest(), + "canonicalSha256": hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()} + finally: + os.close(descriptor) return {"runId": request["runId"], "service": service, "state": {"path": str(state), "exists": state.exists(), - "ownerMarker": (state / "owner.json").is_file()}, + "ownerMarker": (state / "owner.json").is_file(), "ownerEvidence": marker_evidence(state)}, "roots": [{"path": str(root), "exists": root.exists(), - "ownerMarker": (root / "owner.json").is_file(), - "data": (root / "data").is_dir()} for root in roots]} + "ownerMarker": (root / "owner.json").is_file(), "ownerEvidence": marker_evidence(root), + "data": (root / "data").is_dir()} for root in roots], + "inspectionOnly": True, "mutated": False} def audit_finalize(request): diff --git a/test_validation_heal_recovery.py b/test_validation_heal_recovery.py index bfdcd4bb..e6b2cc97 100644 --- a/test_validation_heal_recovery.py +++ b/test_validation_heal_recovery.py @@ -21,6 +21,45 @@ from scripts.validation_heal import fleet_node, heal_fault +class NamespaceOwnerEvidenceTests(unittest.TestCase): + def test_read_only_owner_fingerprints_keep_missing_roots_and_hide_owner_values(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory).resolve() + state = base / "state"; state.mkdir() + roots = [base / str(index) for index in range(4)] + value = {"runId": "gh-12345-1-h2", "bucket": "rv-heal-0123456789abcdef", "privateValue": "must-not-be-emitted"} + raw = json.dumps(value).encode() + (state / "owner.json").write_bytes(raw) + (roots[3] / "data").mkdir(parents=True) + (roots[3] / "owner.json").write_bytes(json.dumps(value, sort_keys=True).encode()) + before = {(state / "owner.json"): raw, (roots[3] / "owner.json"): (roots[3] / "owner.json").read_bytes()} + def command(*args, **kwargs): + return types.SimpleNamespace(stdout="0" if "MainPID" in args else "inactive") + with patch.object(fleet_node, "paths", return_value=(state, roots)), patch.object(fleet_node, "command", side_effect=command): + observed = fleet_node.namespace_state({"runId": value["runId"], "action": "namespace-state"}) + self.assertTrue(observed["inspectionOnly"]) + self.assertFalse(observed["mutated"]) + self.assertEqual([row["ownerEvidence"]["present"] for row in observed["roots"]], [False, False, False, True]) + expected = hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + self.assertEqual(observed["state"]["ownerEvidence"]["canonicalSha256"], expected) + self.assertEqual(observed["roots"][3]["ownerEvidence"]["canonicalSha256"], expected) + self.assertNotIn(value["privateValue"], json.dumps(observed)) + for path, raw in before.items(): self.assertEqual(path.read_bytes(), raw) + self.assertFalse(any(root.exists() for root in roots[:3])) + + def test_owner_fingerprint_refuses_links_and_oversized_or_non_object_markers(self): + for change in ("symlink", "hardlink", "oversized", "non-object"): + with tempfile.TemporaryDirectory() as directory, self.subTest(change=change): + base = Path(directory).resolve(); state = base / "state"; state.mkdir() + marker = state / "owner.json" + marker.write_text("[]" if change == "non-object" else "x" * 16385 if change == "oversized" else "{}") + if change == "symlink": marker.rename(state / "original"); marker.symlink_to(state / "original") + elif change == "hardlink": __import__("os").link(marker, state / "linked") + with patch.object(fleet_node, "paths", return_value=(state, [])), \ + patch.object(fleet_node, "command", return_value=types.SimpleNamespace(stdout="0")), self.assertRaises(ValueError): + fleet_node.namespace_state({"runId": "gh-12345-1-h2", "action": "namespace-state"}) + + class RejectedFaultRecoveryTest(unittest.TestCase): def test_expanded_branches_have_bounded_distinct_fault_ids(self): branch = "stripe-minus-exact-plus-s1048575-parity-missing" From f3f37bde8f60d68fd0112b22aaec1515bc3134d2 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 17:01:10 +0800 Subject: [PATCH 201/343] fix(validation): retain incomplete fixtures before isolating them --- scripts/validation_heal/fleet_node.py | 9 + scripts/validation_heal/healthy.py | 3 + .../validation_heal/recover_environment.py | 56 ++++- scripts/validation_heal/retained_scene.py | 235 ++++++++++++++++++ test_validation_heal_retained_scene.py | 235 ++++++++++++++++++ 5 files changed, 537 insertions(+), 1 deletion(-) create mode 100644 scripts/validation_heal/retained_scene.py create mode 100644 test_validation_heal_retained_scene.py diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 63b3a53d..777f085c 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1260,9 +1260,16 @@ def marker_evidence(folder): "roots": [{"path": str(root), "exists": root.exists(), "ownerMarker": (root / "owner.json").is_file(), "ownerEvidence": marker_evidence(root), "data": (root / "data").is_dir()} for root in roots], + "retained": {"state": checked(state.parent / "retained" / state.name, state.parent).exists(), + "roots": [checked(root.parent.parent / "heal-validation-retained" / root.name, + root.parent.parent).exists() for root in roots]}, "inspectionOnly": True, "mutated": False} +def retained_scene_operation(request): + return globals()["retained_scene_backend"].run(request, globals()) + + def audit_finalize(request): """Finalize only an invalid-ID rejection proved identical to its archived pre-fault scene.""" _, _, owner = owned(request["runId"]) @@ -1531,6 +1538,8 @@ def main(request): actions[action] = incarnation_operation actions["task-records"] = task_records actions["environment-capabilities"] = environment_capabilities + for action in ("retained-scene-audit", "retained-scene-isolate"): + actions[action] = retained_scene_operation actions["quota-audit-finalize"] = quota_audit_finalize actions["journal-rejection-audit-finalize"] = journal_rejection_audit_finalize actions["journal-rejection-audit"] = journal_rejection_audit_finalize diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 1e71f4bd..a5ab51c2 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -185,6 +185,9 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() + if action.startswith("retained-scene-"): + source += "\nimport types\nretained_scene_backend = types.ModuleType('retained_scene')\nexec(" + repr( + NODE_SOURCE.with_name("retained_scene.py").read_text()) + ", retained_scene_backend.__dict__)\n" if action.startswith("network-") or action in {"cleanup", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize", "journal-retained-audit", "journal-retained-audit-finalize"}: network_source = NODE_SOURCE.with_name("network_gate.py").read_text() source += "\nimport types\nnetwork_gate_backend = types.ModuleType('network_gate')\nexec(" + repr( diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index 80e54b07..930310aa 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -552,6 +552,51 @@ def recovery_scene_matches(current, plan): return True +def isolate_incomplete_scene(fleet, plan, artifact): + """Preserve a drifted fixture; this never restores or finalizes its evidence.""" + case = "heal.checkpoint-and-mrf-corruption" + folders = list(artifact.glob(f"*/run/{case}/evidence/{case}/DB/H2/mrf-checksum")) + require(len(folders) == 1, "incomplete fixture lacks its original archive") + receipts = [json.loads(path.read_text()) for path in folders[0].glob("*-reserved.json")] + require(len(receipts) == len(plan["nodes"]) and {item.get("node") for item in receipts} == set(plan["nodes"]), + "incomplete fixture lacks one original reservation per node") + owners = {} + for item in receipts: + owner = item.get("owner") or {} + require(item.get("reserved") is True and owner.get("runId") == plan["runId"] + and owner.get("bucket") == plan["bucket"] and owner.get("binarySha256") == plan["binarySha256"] + and owner.get("runtimeProfile") == "mrf-crash", "incomplete fixture original reservation differs") + owners[item["node"]] = owner + audits = {} + for node in plan["nodes"]: + audits[node] = fleet.record("incomplete-scene-preflight", {"node": node, + **fleet.remote(node, "retained-scene-audit", expectedOwner=owners[node])}) + require(audits[node].get("inspectionOnly") is True and audits[node].get("mutated") is False + and audits[node].get("credentialContentsRead") is False, "incomplete fixture audit was not read-only") + require(any(any(root is None for root in audit["baseline"]["roots"]) for audit in audits.values()), + "incomplete-scene isolation requires actual missing roots") + # All nodes are frozen and audited before the first rename. Each node keeps + # a durable inventory intent and repeats it before every same-device move. + for node in plan["nodes"]: + proof = fleet.record("incomplete-scene-isolation", {"node": node, + **fleet.remote(node, "retained-scene-isolate", expectedOwner=owners[node], + expectedBaseline=audits[node]["baseline"])}) + require(proof.get("activeNamespaceIsolated") is True and proof.get("originalSceneRestored") is False + and proof.get("originalEvidenceComplete") is False and proof.get("backupRestorationUsed") is False + and proof.get("canonicalPassClaimed") is False + and proof.get("baseline") == audits[node]["baseline"], "incomplete fixture isolation failed") + for node in plan["nodes"]: + proof = fleet.record("incomplete-scene-post-audit", {"node": node, + **fleet.remote(node, "retained-scene-audit", expectedOwner=owners[node])}) + require(proof.get("activeNamespaceIsolated") is True and proof.get("inspectionOnly") is True + and proof.get("mutated") is False and proof.get("baseline") == audits[node]["baseline"], + "incomplete fixture changed after isolation") + return {"runId": plan["runId"], "cleanup": "ISOLATED", "activeNamespaceIsolated": True, + "originalEvidenceComplete": False, "originalSceneRestored": False, "originalVerdictRewritten": False, + "missingDisks": {node: [index for index, row in enumerate(audits[node]["baseline"]["roots"]) if row is None] + for node in plan["nodes"]}} + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--failed-run-id", type=int, required=True) @@ -587,6 +632,13 @@ def main(): continue states = [fleet.record("namespace-state", {"node": node, **fleet.remote(node, "namespace-state")}) for node in plan["nodes"]] + has_retained = any(item.get("retained", {}).get("state") + or any(item.get("retained", {}).get("roots", [])) for item in states) + all_absent = all(not item["state"]["exists"] and not any(root["exists"] for root in item["roots"]) for item in states) + if plan.get("kind") == "advanced-mrf-fixture" and (has_retained or (not all_absent and any( + not root["exists"] for item in states for root in item["roots"]))): + complete.append({"failedRunId": args.failed_run_id, **isolate_incomplete_scene(fleet, plan, args.artifact)}) + continue if all(not item["state"]["exists"] and not any(root["exists"] for root in item["roots"]) for item in states): for item in states: @@ -677,7 +729,9 @@ def main(): fleet.record("unchanged-peer-finalization", fleet.remote(node, "cleanup")) require(proof.get("cleanup") == "PASS", "audited environment finalization failed") complete.append({"failedRunId": args.failed_run_id, "runId": plan["runId"], "cleanup": "PASS", "originalVerdictRewritten": False}) - save(args.evidence / "recovery-summary.json", {"cleanup": "PASS", "environments": complete, "canonicalPassClaimed": False}) + save(args.evidence / "recovery-summary.json", { + "cleanup": "ISOLATED" if any(item["cleanup"] == "ISOLATED" for item in complete) else "PASS", + "environments": complete, "canonicalPassClaimed": False}) if __name__ == "__main__": diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py new file mode 100644 index 00000000..14c2cb1f --- /dev/null +++ b/scripts/validation_heal/retained_scene.py @@ -0,0 +1,235 @@ +"""Retain incomplete stopped fixtures by same-filesystem, non-overwriting rename. + +Only owner/binary and fixture/MRF bytes are read. Other entries, including +credentials, are retained by inode and metadata without reading their contents. +""" + +import ctypes +import hashlib +import json +import os +import re +from pathlib import Path +import stat +import uuid + + +def require(value, message): + if not value: + raise ValueError(message) + + +def encoded(value): + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + + +def sha(value): + return hashlib.sha256(encoded(value)).hexdigest() + + +def stable(info): + return (info.st_mode, info.st_uid, info.st_gid, info.st_dev, info.st_ino, info.st_nlink, + info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +def private_json(path, *, root_owned=True): + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + before = os.fstat(fd) + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_size <= 16384 + and not before.st_mode & 0o022 and (not root_owned or before.st_uid == 0), + "retained scene marker has unsafe ownership, type or size") + raw = os.read(fd, 16385) + after = os.fstat(fd) + require(len(raw) == before.st_size and stable(before) == stable(after), "retained scene marker changed during audit") + value = json.loads(raw) + require(isinstance(value, dict), "retained scene marker is not an object") + return value + finally: + os.close(fd) + + +def destinations(state, roots): + return state.parent / "retained" / state.name, tuple( + root.parent.parent / "heal-validation-retained" / root.name for root in roots) + + +def sync_directory(path): + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def rename_new(source, target): + # Linux RENAME_NOREPLACE prevents replacing even an empty competing directory. + rename = ctypes.CDLL(None, use_errno=True).renameat2 + rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] + rename.restype = ctypes.c_int + result = rename(-100, os.fsencode(source), -100, os.fsencode(target), 1) + require(result == 0, "retained scene rename refused (errno=" + str(ctypes.get_errno()) + ")") + sync_directory(source.parent) + if source.parent != target.parent: + sync_directory(target.parent) + + +def parent_ready(parent, boundary, node, *, create=False): + node["checked"](parent, boundary) + if create and not parent.exists(): + parent.mkdir(mode=0o700) + sync_directory(parent.parent) + if parent.exists(): + info = parent.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and stat.S_IMODE(info.st_mode) == 0o700 + and info.st_dev == boundary.stat().st_dev, "retained archive parent is not private on the original filesystem") + + +def census(folder, owner, *, state, node): + """Hash the complete inode inventory; read only explicitly scoped content.""" + rows, content_count = [], 0 + boundary_device = folder.lstat().st_dev + def visit(path): + nonlocal content_count + node["checked"](path, folder) + info = path.lstat() + require(info.st_dev == boundary_device and (stat.S_ISDIR(info.st_mode) + or stat.S_ISREG(info.st_mode) and info.st_nlink == 1), + "retained scene contains a mount, link or special file") + relative = path.relative_to(folder).as_posix() + require(len(path.relative_to(folder).parts) <= 64, "retained scene tree is too deep") + # Renaming the top directory changes only its ctime. Descendant ctime + # remains part of the witness, including for unread credential files. + row = [hashlib.sha256(relative.encode()).hexdigest(), info.st_mode, info.st_uid, info.st_gid, + info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, + info.st_ctime_ns if path != folder else None] + require(len(rows) < 30000, "retained scene inventory exceeds its explicit bound") + rows.append(row) + if stat.S_ISDIR(info.st_mode): + for child in sorted(path.iterdir()): + visit(child) + after = path.lstat() + require(stable(info) == stable(after), "retained scene directory changed during audit") + else: + parts = Path(relative).parts + fixture = not state and len(parts) > 2 and parts[:2] == ("data", owner["bucket"]) + mrf = not state and relative in { + "data/.rustfs.sys/.heal-mrf-snapshot.0.bin", "data/.rustfs.sys/.heal-mrf-snapshot.1.bin", + "data/.rustfs.sys/.heal-mrf-manifest.json"} + if (state and relative == "rustfs") or fixture or mrf: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + require(stable(os.fstat(fd)) == stable(info) and info.st_size <= 2 * 1024**3, + "retained content changed or exceeds its bound") + digest = hashlib.sha256() + while chunk := os.read(fd, 1024**2): + digest.update(chunk) + require(stable(os.fstat(fd)) == stable(info), "retained content changed while hashing") + row.append(digest.hexdigest()) + content_count += 1 + finally: + os.close(fd) + require(stable(path.lstat()) == stable(info), "retained scene file changed during audit") + visit(folder) + return {"treeSha256": sha(rows), "entries": len(rows), "contentFilesHashed": content_count, + "device": boundary_device, "inode": folder.lstat().st_ino} + + +def mounted_paths(): + values = [] + for line in Path("/proc/self/mountinfo").read_text().splitlines(): + fields = line.split() + require(len(fields) >= 6, "retained scene mount inventory is malformed") + value = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match.group(1), 8)), fields[4]) + values.append(Path(value)) + return values + + +def inspect(request, node): + state, roots = node["paths"](request["runId"]) + retained_state, retained_roots = destinations(state, roots) + expected = request["expectedOwner"] + require(isinstance(expected, dict) and expected.get("runId") == request["runId"] + and expected.get("unit") == node["unit"](request["runId"]) + and expected.get("hostname") == node["socket"].gethostname() + and expected.get("runtimeProfile") == "mrf-crash", "retained scene archived owner differs") + current = node["mounts"]() + require(len(current) == len(expected.get("mounts", [])) == 4 + and all(all(actual[key] == old[key] for key in ("target", "source", "fstype", "uuid", "maj:min")) + for actual, old in zip(current, expected["mounts"], strict=True)), + "retained scene original mount identities changed") + node["stopped"](expected) + for parent, boundary in [(retained_state.parent, state.parent), + *((target.parent, mount) for target, mount in zip(retained_roots, node["MOUNTS"], strict=True))]: + parent_ready(parent, boundary, node) + locations, witnesses = [], [] + mounted = mounted_paths() + for index, (source, target) in enumerate(zip((*roots, state), (*retained_roots, retained_state), strict=True)): + boundary = node["MOUNTS"][index] if index < 4 else state.parent + node["checked"](target, boundary) + require(not (source.exists() and target.exists()), "both active and retained scene exist; refusing ambiguity") + folder = source if source.exists() else target if target.exists() else None + locations.append("active" if folder == source else "retained" if folder == target else "absent") + if folder is None: + require(index < 4, "retained scene lost its original state and candidate") + witnesses.append(None) + continue + require(not any(mount == folder or mount.is_relative_to(folder) for mount in mounted), + "retained scene contains a mounted subtree") + info = folder.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_dev == boundary.stat().st_dev, + "retained scene directory is not on its original filesystem") + owner = private_json(node["checked"](folder / "owner.json", folder), root_owned=index == 4) + require(encoded(owner) == encoded(expected), "retained scene owner content differs from original receipt") + if index == 4: + require(info.st_uid == 0 and stat.S_IMODE(info.st_mode) == 0o700, + "retained scene state is not private") + require(node["digest"](node["checked"](folder / "rustfs", folder)) == expected["binarySha256"], + "retained scene candidate binary differs") + require(not any((folder / name).exists() for name in ( + "drive-gate.json", "write-quorum-gate.json", "network-gate.json", "disk-replacement.json")), + "retained scene still has a gate or replacement intent") + witnesses.append(census(folder, expected, state=index == 4, node=node)) + baseline = {"runId": request["runId"], "ownerSha256": sha(expected), "roots": witnesses[:4], "state": witnesses[4]} + intent = retained_state.parent / (request["runId"] + ".intent.json") + node["checked"](intent, state.parent) + if intent.exists(): + require(encoded(private_json(intent)) == encoded(baseline), "retained scene drifted from its durable isolation intent") + else: + require("retained" not in locations, "retained directory lacks its durable isolation intent") + return baseline, locations, state, roots, retained_state, retained_roots, intent + + +def run(request, node): + require(set(request) == ({"runId", "action", "expectedOwner", "expectedBaseline"} + if request["action"] == "retained-scene-isolate" else {"runId", "action", "expectedOwner"}), + "retained scene accepts only its archived owner and frozen inventory") + require(request["action"] in {"retained-scene-audit", "retained-scene-isolate"}, "invalid retained scene action") + baseline, locations, state, roots, retained_state, retained_roots, intent = inspect(request, node) + mutated = False + if request["action"] == "retained-scene-isolate": + require(encoded(request["expectedBaseline"]) == encoded(baseline), "retained scene changed after fleet preflight") + for parent, boundary in [(retained_state.parent, state.parent), + *((target.parent, mount) for target, mount in zip(retained_roots, node["MOUNTS"], strict=True))]: + parent_ready(parent, boundary, node, create=True) + if not intent.exists(): + temporary = intent.with_name(intent.name + "." + uuid.uuid4().hex + ".new") + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "wb") as stream: + stream.write(encoded(baseline)); stream.flush(); os.fsync(stream.fileno()) + rename_new(temporary, intent) + for index, (source, target) in enumerate(zip((*roots, state), (*retained_roots, retained_state), strict=True)): + if locations[index] != "active": + continue + fresh = inspect(request, node) + require(encoded(fresh[0]) == encoded(baseline), "retained scene changed immediately before rename") + rename_new(source, target) + mutated = True + final = inspect(request, node) + require(encoded(final[0]) == encoded(baseline) and "active" not in final[1], + "retained scene post-rename inventory differs") + locations = final[1] + return {"baseline": baseline, "locations": locations, "originalEvidenceComplete": False, + "originalSceneRestored": False, "canonicalPassClaimed": False, "backupRestorationUsed": False, + "credentialContentsRead": False, "inspectionOnly": request["action"] == "retained-scene-audit", + "mutated": mutated, "activeNamespaceIsolated": "active" not in locations} diff --git a/test_validation_heal_retained_scene.py b/test_validation_heal_retained_scene.py new file mode 100644 index 00000000..495cc63a --- /dev/null +++ b/test_validation_heal_retained_scene.py @@ -0,0 +1,235 @@ +"""Incomplete scenes remain preserved and cannot become restored PASS evidence.""" + +import copy +import hashlib +import json +import os +from pathlib import Path +import shutil +import socket +import stat +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import fleet_node, retained_scene as retained, recover_environment as recovery +from scripts.validation_heal.healthy import Fleet + + +class RetainedSceneTests(unittest.TestCase): + def scene(self, directory): + base = Path(directory).resolve() + mounts = [base / str(index) for index in range(4)] + for mount in mounts: mount.mkdir() + run_id = "gh-12345-1-h2" + state = base / "state" / run_id; state.mkdir(parents=True, mode=0o700) + roots = tuple(mount / "heal-validation" / run_id for mount in mounts) + binary = b"original-candidate" + owner = {"runId": run_id, "unit": "rustfs-validation-heal-" + run_id + ".service", + "hostname": socket.gethostname(), "runtimeProfile": "mrf-crash", "bucket": "rv-heal-0123456789abcdef", + "binarySha256": hashlib.sha256(binary).hexdigest(), "mounts": [ + {key: str(index) for key in ("target", "source", "fstype", "uuid", "maj:min")} for index in range(4)]} + (state / "rustfs").write_bytes(binary) + (state / "owner.json").write_text(json.dumps(owner)) + (state / "service.env").write_bytes(b"credential-must-never-be-read") + for root in roots: + (root / "data" / owner["bucket"]).mkdir(parents=True) + (root / "data" / owner["bucket"] / "part.1").write_bytes(b"acknowledged-fixture") + (root / "data" / ".rustfs.sys" / "config" / "iam").mkdir(parents=True) + (root / "data" / ".rustfs.sys" / "config" / "iam" / "credential").write_bytes(b"private-iam") + (root / "quarantine").mkdir() + (root / "quarantine" / "original").write_bytes(b"original-backup") + (root / "owner.json").write_text(json.dumps(owner)) + shutil.rmtree(roots[0]) + node = {"paths": lambda run: (state, roots), "unit": lambda run: owner["unit"], "socket": socket, + "mounts": lambda: copy.deepcopy(owner["mounts"]), "stopped": lambda value: {}, "MOUNTS": mounts, + "digest": fleet_node.digest, "checked": fleet_node.checked} + request = {"runId": run_id, "action": "retained-scene-audit", "expectedOwner": owner} + return state, roots, node, request + + @staticmethod + def local_rename(source, target): + if target.exists(): raise ValueError("destination exists") + source.rename(target) + retained.sync_directory(source.parent) + retained.sync_directory(target.parent) + + def test_real_files_keep_inodes_bytes_and_retry_after_each_interrupted_move(self): + for interrupt_after in (1, 2, 3, 4): + with self.subTest(interrupt_after=interrupt_after), tempfile.TemporaryDirectory() as directory: + state, roots, node, request = self.scene(directory) + # Run the full production ownership check using root-owned + # stat proxies, without replacing filesystem identities. + with self.root_stats(), patch.object(retained, "rename_new", side_effect=self.local_rename): + before = retained.run(request, node) + self.assertFalse((state.parent / "retained").exists()) + original = {p: (p.stat().st_ino, p.read_bytes()) for root in (*roots[1:], state) + for p in root.rglob("*") if p.is_file()} + count = 0 + def interrupted(source, target): + nonlocal count + self.local_rename(source, target) + if target.is_dir(): + count += 1 + if count == interrupt_after: raise RuntimeError("lost SSH after atomic rename") + change = {**request, "action": "retained-scene-isolate", "expectedBaseline": before["baseline"]} + with patch.object(retained, "rename_new", side_effect=interrupted), self.assertRaises(RuntimeError): + retained.run(change, node) + result = retained.run(change, node) + self.assertTrue(result["activeNamespaceIsolated"]) + self.assertEqual(result["baseline"], before["baseline"]) + self.assertEqual(result["locations"], ["absent", "retained", "retained", "retained", "retained"]) + retained_state, retained_roots = retained.destinations(state, roots) + for old, (inode, raw) in original.items(): + source = next(root for root in (*roots[1:], state) if old.is_relative_to(root)) + target = retained_state if source == state else retained_roots[roots.index(source)] + moved = target / old.relative_to(source) + self.assertEqual((moved.stat().st_ino, moved.read_bytes()), (inode, raw)) + self.assertFalse(result["originalSceneRestored"]) + self.assertFalse(result["canonicalPassClaimed"]) + self.assertEqual(retained.run(request, node)["baseline"], before["baseline"]) + self.assertFalse(retained.run(change, node)["mutated"]) + + def root_stats(self): + # Root UID checks are meaningful in production. Only rewrite st_uid in + # returned host stat objects; preserve inode, mode and nanosecond times. + original_lstat, original_stat, original_fstat = Path.lstat, Path.stat, os.fstat + def mapped(value): + values = {name: getattr(value, name) for name in dir(value) if name.startswith("st_")} + values["st_uid"] = 0 + return types.SimpleNamespace(**values) + from contextlib import ExitStack + stack = ExitStack() + stack.enter_context(patch.object(retained, "mounted_paths", return_value=[])) + stack.enter_context(patch.object(Path, "lstat", lambda path: mapped(original_lstat(path)))) + stack.enter_context(patch.object(Path, "stat", lambda path, **kw: mapped(original_stat(path, **kw)))) + stack.enter_context(patch.object(os, "fstat", lambda fd: mapped(original_fstat(fd)))) + return stack + + def test_no_credentials_read_and_no_paths_or_marker_content_emitted(self): + with tempfile.TemporaryDirectory() as directory: + state, roots, node, request = self.scene(directory) + opened = [] + original_open = os.open + def record(path, *args, **kwargs): + opened.append(str(path)); return original_open(path, *args, **kwargs) + with self.root_stats(), patch.object(os, "open", side_effect=record): + result = retained.run(request, node) + self.assertFalse(any(name.endswith("service.env") or name.endswith("credential") for name in opened)) + serialized = json.dumps(result) + for private in ("credential-must-never-be-read", "private-iam", request["expectedOwner"]["bucket"], str(state)): + self.assertNotIn(private, serialized) + + def test_drift_or_unsafe_paths_refuse_before_any_move(self): + for variant in ("owner", "binary", "gate", "symlink", "hardlink", "mount", "live", "content-drift", "collision", "nested-mount"): + with self.subTest(variant=variant), tempfile.TemporaryDirectory() as directory: + state, roots, node, request = self.scene(directory) + with self.root_stats(): + baseline = retained.run(request, node)["baseline"] + if variant == "owner": (roots[1] / "owner.json").write_text("{}") + elif variant == "binary": (state / "rustfs").write_bytes(b"different") + elif variant == "gate": (state / "drive-gate.json").write_text("{}") + elif variant == "symlink": (roots[1] / "escape").symlink_to(state) + elif variant == "hardlink": os.link(roots[1] / "quarantine/original", roots[1] / "alias") + elif variant == "mount": node["mounts"] = lambda: [{**row, "uuid": "foreign"} for row in request["expectedOwner"]["mounts"]] + elif variant == "live": node["stopped"] = lambda owner: (_ for _ in ()).throw(ValueError("live")) + elif variant == "content-drift": (roots[1] / "quarantine/original").write_bytes(b"advanced-record") + elif variant == "nested-mount": retained.mounted_paths.return_value = [roots[1] / "data"] + elif variant == "collision": + target = retained.destinations(state, roots)[1][1]; target.mkdir(parents=True, mode=0o700) + with patch.object(retained, "rename_new") as move, self.assertRaises(ValueError): + retained.run({**request, "action": "retained-scene-isolate", "expectedBaseline": baseline}, node) + move.assert_not_called() + self.assertTrue(state.exists()); self.assertTrue(roots[1].exists()) + + def test_disappeared_retained_directory_cannot_be_reclassified_as_originally_missing(self): + with tempfile.TemporaryDirectory() as directory: + state, roots, node, request = self.scene(directory) + with self.root_stats(), patch.object(retained, "rename_new", side_effect=self.local_rename): + baseline = retained.run(request, node)["baseline"] + retained.run({**request, "action": "retained-scene-isolate", "expectedBaseline": baseline}, node) + shutil.rmtree(retained.destinations(state, roots)[1][1]) + with self.assertRaisesRegex(ValueError, "durable isolation intent"): + retained.run(request, node) + + def test_remote_adapter_loads_real_module(self): + def execute(*args, **kwargs): + source = kwargs["input"].rsplit("\nmain(", 1)[0] + scope = {"__name__": "__node_test__"} + exec(compile(source, "retained-node", "exec"), scope) + self.assertTrue(callable(scope["retained_scene_backend"].run)) + return types.SimpleNamespace(returncode=0, stdout="{}", stderr="") + with tempfile.TemporaryDirectory() as directory, patch("scripts.validation_heal.healthy.subprocess.run", side_effect=execute): + Fleet(["vm000"], Path(directory), "gh-12345-1-h2").remote("vm000", "retained-scene-audit", expectedOwner={}) + + def test_mountinfo_decodes_escaped_paths_without_accepting_a_missing_inventory(self): + with patch.object(Path, "read_text", return_value=r"1 0 8:1 / /data/rustfs1/a\040b rw - xfs /dev/disk rw"): + self.assertEqual(retained.mounted_paths(), [Path("/data/rustfs1/a b")]) + with patch.object(Path, "read_text", return_value="invalid"), self.assertRaises(ValueError): + retained.mounted_paths() + + def test_linux_rename_explicitly_uses_noreplace_and_syncs_both_parents(self): + call = unittest.mock.Mock(return_value=0) + with patch.object(retained.ctypes, "CDLL", return_value=types.SimpleNamespace(renameat2=call)), \ + patch.object(retained, "sync_directory") as sync: + retained.rename_new(Path("/original/run"), Path("/retained/run")) + call.assert_called_once_with(-100, b"/original/run", -100, b"/retained/run", 1) + self.assertEqual(sync.call_args_list, [unittest.mock.call(Path("/original")), unittest.mock.call(Path("/retained"))]) + + +class IncompleteFleetIsolationTests(unittest.TestCase): + def test_main_does_not_treat_an_archived_namespace_as_restored_or_already_clean(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); candidate = root / "candidate.json"; candidate.write_text("{}") + plan = {"kind": "advanced-mrf-fixture", "nodes": ["vm000"], "runId": "gh-12345-1-h2"} + states = {"state": {"exists": False}, "roots": [{"exists": False}] * 4, + "retained": {"state": True, "roots": [False, True, True, True]}} + result = {"cleanup": "ISOLATED", "originalSceneRestored": False} + argv = ["recover", "--failed-run-id", "12345", "--artifact", str(root), "--candidate", str(candidate), + "--evidence", str(root / "evidence")] + with patch("sys.argv", argv), patch.object(recovery, "recovery_plan", return_value=[plan]), \ + patch.object(Fleet, "remote", return_value=states) as remote, \ + patch.object(recovery, "isolate_incomplete_scene", return_value=result) as isolate: + recovery.main() + self.assertEqual([call.args[1] for call in remote.call_args_list], ["namespace-state"]) + isolate.assert_called_once() + summary = json.loads((root / "evidence/recovery-summary.json").read_text()) + self.assertEqual(summary["cleanup"], "ISOLATED") + self.assertFalse(summary["canonicalPassClaimed"]) + + def test_all_preflights_precede_moves_and_original_failure_is_not_restored(self): + for failure in (None, "audit-vm003", "move-vm001"): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + case = "heal.checkpoint-and-mrf-corruption" + folder = root / "artifact/run" / case / "evidence" / case / "DB/H2/mrf-checksum" + folder.mkdir(parents=True) + nodes = [f"vm00{i}" for i in range(4)] + plan = {"nodes": nodes, "runId": "gh-12345-1-h2", "bucket": "rv-heal-0123456789abcdef", "binarySha256": "b" * 64} + for index, node in enumerate(nodes): + (folder / f"{index:05d}-reserved.json").write_text(json.dumps({"node": node, "reserved": True, + "owner": {**{key: plan[key] for key in ("runId", "bucket", "binarySha256")}, "runtimeProfile": "mrf-crash"}})) + events = [] + def remote(node, action, **kwargs): + events.append((node, action)) + if (failure == "audit-" + node and action.endswith("audit")) or (failure == "move-" + node and action.endswith("isolate")): + raise ValueError("injected failure") + return {"baseline": {"roots": [None, {}, {}, {}]}, "inspectionOnly": action.endswith("audit"), + "mutated": action.endswith("isolate"), "credentialContentsRead": False, + "activeNamespaceIsolated": True, "originalSceneRestored": False, "originalEvidenceComplete": False, + "backupRestorationUsed": False, "canonicalPassClaimed": False} + fleet = types.SimpleNamespace(remote=remote, record=lambda label, value: value) + if failure: + with self.assertRaises(ValueError): recovery.isolate_incomplete_scene(fleet, plan, root) + else: + result = recovery.isolate_incomplete_scene(fleet, plan, root) + self.assertEqual(result["cleanup"], "ISOLATED") + self.assertFalse(result["originalSceneRestored"]) + self.assertFalse(result["originalVerdictRewritten"]) + self.assertEqual(events[:4], [(node, "retained-scene-audit") for node in nodes]) + if failure == "audit-vm003": self.assertFalse(any(action.endswith("isolate") for _, action in events)) + + +if __name__ == "__main__": + unittest.main() From 2e07d1dbcc8c7f3f2625714a438c53534ad0a6aa Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 17:47:26 +0800 Subject: [PATCH 202/343] test(validation): exercise live legacy unknown outcomes --- docs/validation-migration-map.md | 2 +- scripts/validation_heal/execution_policy.py | 2 +- scripts/validation_heal/fleet_node.py | 93 +++++++++- scripts/validation_heal/healthy.py | 4 +- scripts/validation_heal/outcome.py | 123 ++++++++++--- scripts/validation_heal/recipes.json | 2 +- .../tools/heal_meta_inspect.rs | 2 +- test_validation_heal_incarnation_replay.py | 4 +- test_validation_heal_outcome.py | 171 +++++++++++++++++- 9 files changed, 362 insertions(+), 41 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index fe585cf8..1d853ed3 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1908,7 +1908,7 @@ J06 retry `38195131282` completed all twelve fixed ABBA phases with 1,440 schedu A03 run `38198122041` passes every required H2 cross-node branch at helper `ccef050df3b28416c51356951e78cb46dd74d10b` and driver `03fb3d78a0ffc4213583dd3bcb44c7ac38f42ead`. Independent verification of the original artifact members, canonical result gate, complete branch outcomes and immutable candidate provenance increases the unique canonical census to 89. The remaining serialized backlog continues on the same candidate; no prior failure is overwritten. -J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite and its branch is explicitly BLOCKED pending a reviewed live unknown/lock-deferred fixture. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the source assertion requiring a mixed deferred population is still outstanding. The 84 cases, 301 branches and 532 topology templates are unchanged. +J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite. Its new live executor first verifies an ACK-bound protected control and original incarnation, switches only the owned candidate write mode through a durable one-way transaction, writes real SDK legacy objects, and requires every physical replica to show absent integrity descriptors. The original Deep task must report exact Unknown identities/counters and zero repairs for those objects, preserve the protected control and all original physical bytes, and remain unchanged after subsequent GETs. A fresh candidate-bound inspector build and complete live evidence remain required; this implementation is not a canonical PASS. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the source assertion requiring a mixed deferred population is still outstanding. The 84 cases, 301 branches and 532 topology templates are unchanged. The corrected live census now contains 99 unique complete canonical PASS cases: 85 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926`, 12 on the older immutable source and two on source `16cf19817e0c81e24d0d787a916f734a34517ec5`. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 35 retained live failures, 7 required native barriers, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py index 56dd9471..7a7e3d2a 100644 --- a/scripts/validation_heal/execution_policy.py +++ b/scripts/validation_heal/execution_policy.py @@ -144,7 +144,7 @@ def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, . ), "heal.outcome-counters-and-error-truth": _native( "J02", "EVIDENCE_MIXED_OUTCOME_ORCHESTRATION", - "已有 2 项候选版本 outcome 原生回归和部分计数校验，但尚缺真实混合 healthy/repaired/failed/deferred/unknown 夹具、dry-run 独立任务、逐对象完整 manifest 与计数/attemptFailures 对账；这是执行器与证据未闭环，并非已证实产品缺少 API。", + "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral 和真实 SDK legacy Unknown 执行分支及逐对象 manifest/attemptFailures 对账；仍需绑定固定候选的新 inspector、完整串行 live 证据和同批 deferred 源断言核验，故保持禁止 dispatch；本地检查不能作为完整 PASS。", ("mixed_outcome_fixture", "per_object_canonical_receipts", "complete_live_orchestrator", "attempt_failure_visibility"), ), } diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 777f085c..9bd77468 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -40,7 +40,7 @@ def runtime_config(profile): "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false", "RUSTFS_HEAL_MAX_CONCURRENT_HEALS": "1", "RUSTFS_HEAL_MAX_CONCURRENT_PER_SET": "1", "RUSTFS_OBS_LOGGER_LEVEL": logging_filter, "RUST_LOG": logging_filter} - require(profile in {"queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(profile in {"outcome-legacy", "queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") if profile == "queue-pressure": logging_filter = "info,rustfs::heal::manager=debug,rustfs::heal::task=debug,rustfs::heal::mrf=debug" return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true", @@ -106,22 +106,84 @@ def runtime_settings(owner): config["AUTO_TESTING_C11_PHASE"] = owner.get("incarnationPhase", "capture") if owner.get("incarnationPhase") == "transition": config.update(RUSTFS_HEAL_MRF_ENABLE="false") + if owner.get("runtimeProfile") == "outcome-legacy" and owner.get("outcomePhase") == "legacy": + config["RUSTFS_SHARD_INTEGRITY_WRITE"] = "false" return config -def incarnation_configuration(state, owner): +def runtime_configuration(state, owner): env = checked(state / "service.env", state) info = env.stat() require(info.st_uid == 0 and info.st_nlink == 1 and stat.S_ISREG(info.st_mode) and info.st_size <= 65536, - "C11 configuration lost its bounded private ownership") + "runtime configuration lost its bounded private ownership") lines = env.read_text().splitlines() - require(all("=" in line for line in lines), "C11 configuration has an invalid entry") + require(all("=" in line for line in lines), "runtime configuration has an invalid entry") values = dict(line.split("=", 1) for line in lines) require(len(lines) == len(values) and all(values.get(key) == value for key, value in runtime_settings(owner).items()), - "C11 phase configuration drifted or its durable phase commit is incomplete") + "runtime configuration drifted or its durable phase commit is incomplete") return values +def outcome_legacy_phase(request): + """One-way switch after a separately verified protected control task.""" + require(set(request) == {"runId", "action"}, "J02 phase accepts no runtime override") + state, _, owner = owned(request["runId"]) + stopped(owner) + require(owner.get("runtimeProfile") == "outcome-legacy" + and digest(checked(state / "rustfs", state)) == owner["binarySha256"], "J02 runtime or candidate differs") + fault = fault_module() + binding = {"runId": owner["runId"], "binarySha256": owner["binarySha256"], "phase": "legacy"} + with fault.directory(state) as fd: + info = os.stat("service.env", dir_fd=fd, follow_symlinks=False) + require(info.st_uid == 0 and not info.st_mode & 0o077, "J02 configuration lost its private ownership") + raw = fault.read_file(fd, "service.env", 65536) + lines = raw.decode().splitlines() + require(all("=" in line for line in lines), "J02 private configuration has an invalid entry") + values = dict(line.split("=", 1) for line in lines) + wanted = runtime_settings({**owner, "outcomePhase": "legacy"}) + require(len(lines) == len(values) and values.get("RUSTFS_SHARD_INTEGRITY_WRITE") in {"true", "false"} + and all(values.get(key) == value for key, value in wanted.items() if key != "RUSTFS_SHARD_INTEGRITY_WRITE"), + "J02 configuration changed outside its write-mode switch") + values["RUSTFS_SHARD_INTEGRITY_WRITE"] = "false" + changed = ("\n".join(f"{key}={value}" for key, value in values.items()) + "\n").encode() + changed_sha = hashlib.sha256(changed).hexdigest() + try: + info = os.stat("J02-legacy-intent.json", dir_fd=fd, follow_symlinks=False) + require(info.st_uid == 0 and not info.st_mode & 0o077, "J02 phase intent lost its private ownership") + intent = json.loads(fault.read_file(fd, "J02-legacy-intent.json", 16384)) + except FileNotFoundError: + require(owner.get("outcomePhase") is None, "J02 committed phase lacks its original intent") + runtime_configuration(state, owner) + intent = {**binding, "beforeEnvironmentSha256": hashlib.sha256(raw).hexdigest(), + "legacyEnvironmentSha256": changed_sha} + fault.new_file(fd, "J02-legacy-intent.json", fault.encode(intent)) + require(set(intent) == {*binding, "beforeEnvironmentSha256", "legacyEnvironmentSha256"} + and all(intent.get(key) == value for key, value in binding.items()) + and all(re.fullmatch(r"[0-9a-f]{64}", intent.get(key, "")) for key in + ("beforeEnvironmentSha256", "legacyEnvironmentSha256")) + and changed_sha == intent["legacyEnvironmentSha256"] + and hashlib.sha256(raw).hexdigest() in {intent["beforeEnvironmentSha256"], intent["legacyEnvironmentSha256"]}, + "J02 phase configuration or intent differs") + require(owner.get("outcomePhase") is None or hashlib.sha256(raw).hexdigest() == intent["legacyEnvironmentSha256"], + "J02 committed write mode drifted") + if owner.get("outcomePhase") != "legacy": + try: + fault.new_file(fd, "J02-service.env.new", changed) + except FileExistsError: + require(fault.read_file(fd, "J02-service.env.new", 65536) == changed, "J02 staged configuration differs") + os.rename("J02-service.env.new", "service.env", src_dir_fd=fd, dst_dir_fd=fd) + os.fsync(fd) + phase = fault.encode({"runId": owner["runId"], "phase": "legacy"}) + try: + fault.new_file(fd, "J02-phase.new", phase) + except FileExistsError: + require(fault.read_file(fd, "J02-phase.new", 16384) == phase, "J02 staged phase differs") + os.rename("J02-phase.new", "J02-phase.json", src_dir_fd=fd, dst_dir_fd=fd) + os.fsync(fd) + return {**binding, "oneWay": True, "candidateUnchanged": True, "credentialsDisclosed": False, + "otherRuntimeSettingsUnchanged": True} + + def incarnation_operation(request): require(set(request) <= {"runId", "action", "family", "taskId", "bucketIncarnationId", "expectedRecords", "phase"}, "C11 adapter accepts no arbitrary paths, replacement bytes or commands") @@ -153,7 +215,7 @@ def incarnation_operation(request): "C11 phase runnable records differ from the completed original transaction") except FileNotFoundError as exc: raise ValueError("C11 phase requires a complete original record transaction") from exc - values = incarnation_configuration(state, owner) + values = runtime_configuration(state, owner) owner["incarnationPhase"] = request["phase"] replacement = runtime_settings(owner) env = state / "service.env" @@ -408,6 +470,15 @@ def owned(run_id): require(set(phase) == {"runId", "phase"} and phase["runId"] == run_id and phase["phase"] in {"transition", "replay"}, "C11 phase marker differs") owner = {**owner, "incarnationPhase": phase["phase"]} + if owner.get("runtimeProfile") == "outcome-legacy": + phase_file = checked(state / "J02-phase.json", state) + if phase_file.exists(): + info = phase_file.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_uid == 0 and info.st_nlink == 1 + and info.st_size <= 16384 and not info.st_mode & 0o022, "J02 phase marker lost private ownership") + phase = json.loads(phase_file.read_text()) + require(phase == {"runId": run_id, "phase": "legacy"}, "J02 phase marker differs") + owner = {**owner, "outcomePhase": "legacy"} return state, roots, owner @@ -416,8 +487,13 @@ def start(request): stopped(owner) command("systemctl", "reset-failed", owner["unit"], check=False) require(digest(state / "rustfs") == owner["binarySha256"], "candidate changed before start") - if owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf"}: - incarnation_configuration(state, owner) + if owner.get("runtimeProfile") == "outcome-legacy": + require(not (state / "J02-legacy-intent.json").exists() or owner.get("outcomePhase") == "legacy", + "J02 phase commit is incomplete; service cannot start") + if owner.get("outcomePhase") == "legacy": + outcome_legacy_phase({"runId": owner["runId"], "action": "outcome-legacy-phase"}) + if owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf", "outcome-legacy"}: + runtime_configuration(state, owner) command("systemd-run", "--unit=" + owner["unit"], "--property=Type=exec", "--property=KillMode=control-group", "--property=TimeoutStopSec=30", "--property=RuntimeMaxSec=" + str(resource_limits(owner.get("resourceProfile", "standard"))["runtimeMaxSeconds"]), @@ -1540,6 +1616,7 @@ def main(request): actions["environment-capabilities"] = environment_capabilities for action in ("retained-scene-audit", "retained-scene-isolate"): actions[action] = retained_scene_operation + actions["outcome-legacy-phase"] = outcome_legacy_phase actions["quota-audit-finalize"] = quota_audit_finalize actions["journal-rejection-audit-finalize"] = journal_rejection_audit_finalize actions["journal-rejection-audit"] = journal_rejection_audit_finalize diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index a5ab51c2..b790ae01 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation", "incarnation-checkpoint", "incarnation-mrf"}, "unsupported Heal runtime profile") + require(runtime_profile in {"outcome-legacy", "queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation", "incarnation-checkpoint", "incarnation-mrf"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -205,7 +205,7 @@ def remote(self, node, action, **params): source += "\nimport types\ngeneration_backend = types.ModuleType('generation_fault')\nexec(" + repr( NODE_SOURCE.with_name("generation_fault.py").read_text()) + ", generation_backend.__dict__)\n" if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline", - "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure"} or action.startswith("generation-") or action.startswith("orphan-") or action.startswith(("journal-", "incarnation-")): + "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure", "outcome-legacy-phase"} or action.startswith("generation-") or action.startswith("orphan-") or action.startswith(("journal-", "incarnation-")): source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" if action.startswith(("journal-", "incarnation-")): diff --git a/scripts/validation_heal/outcome.py b/scripts/validation_heal/outcome.py index 88b29988..6d9cc067 100644 --- a/scripts/validation_heal/outcome.py +++ b/scripts/validation_heal/outcome.py @@ -14,7 +14,7 @@ try: from .heal_probe import Probe, canonical, compare_inventories, require, save - from .heal_workload import Recorder, seed, verify_seed + from .heal_workload import Recorder, payload, seed, verify_seed from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology from .heal_fault import bounded_fault_id from .inspector import verify as verify_inspector @@ -27,7 +27,7 @@ validate_version_receipts, verify_proof, verify_quorum_contract, version_identity) except ImportError: from heal_probe import Probe, canonical, compare_inventories, require, save - from heal_workload import Recorder, seed, verify_seed + from heal_workload import Recorder, payload, seed, verify_seed from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology from heal_fault import bounded_fault_id from inspector import verify as verify_inspector @@ -47,11 +47,16 @@ FAILED_ATTEMPTS = 4 -def reconcile(status, bucket, expected, options, incarnation, *, attempt_failures): +def reconcile(status, bucket, expected, options, incarnation, *, attempt_failures, legacy_unknown=()): """The manifest describes responsibility; legacy objectsHealed is not evidence.""" wanted = {version_identity(entry): entry for entry in expected} require(wanted and len(wanted) == len(expected), "J02 manifest is empty or duplicates ACK identities") require(type(attempt_failures) is int and attempt_failures >= 0, "J02 invalid attempt-failure oracle") + legacy_unknown = set(legacy_unknown) + require(legacy_unknown <= set(wanted) and legacy_unknown == {pair for pair, entry in wanted.items() if entry["state"] == "unknown"}, + "J02 unknown responsibility lacks its independently frozen legacy manifest") + require(not legacy_unknown or any(entry["state"] in {"verified_healthy", "metadata_healthy"} for entry in expected), + "J02 legacy negative requires a separate protected control") failed = sum(entry["state"] == "failed" for entry in expected) require(status.get("summary") == ("stopped" if failed else "finished") and status.get("settings") == options and status.get("truncated") is False, @@ -74,7 +79,8 @@ def reconcile(status, bucket, expected, options, incarnation, *, attempt_failure "J02 receipt has a foreign/duplicate object, version or erasure set") expected_entry = wanted[pair] state = expected_entry["state"] - require(state in {"repaired", "verified_healthy", "metadata_healthy", "failed", "dry_run_observed", "deferred"}, + require(state in {"repaired", "verified_healthy", "metadata_healthy", "failed", "dry_run_observed", "deferred"} + or state == "unknown" and pair in legacy_unknown, "J02 unknown, skipped or absent work cannot become a successful fixture result") disposition = receipt.get("disposition", {}) require(disposition.get("state") == state, "J02 disposition differs from the frozen physical fault plan") @@ -90,10 +96,13 @@ def reconcile(status, bucket, expected, options, incarnation, *, attempt_failure require(isinstance(incarnation, str) and incarnation and identity.get("bucketIncarnationId") == incarnation, "J02 receipt escaped the original healthy-task bucket incarnation") + if state == "unknown": + require(disposition == {"state": "unknown"} and receipt.get("detail") is None, + "J02 legacy unknown changed its frozen proofless disposition") if state == "failed": require(disposition.get("details") == "retry_exhausted", "J02 below-R failure lost bounded retry exhaustion") category = {"repaired": "healed", "verified_healthy": "unchanged", "metadata_healthy": "unchanged", - "failed": "failed", "dry_run_observed": "skipped", "deferred": "skipped"}[state] + "failed": "failed", "dry_run_observed": "skipped", "deferred": "skipped", "unknown": "unknown"}[state] counters[category] += 1 observed[pair] = state actual_counters = outcome.get("counters", {}) @@ -101,7 +110,68 @@ def reconcile(status, bucket, expected, options, incarnation, *, attempt_failure for name, value in counters.items()), "J02 canonical counters hide retries or differ from complete dispositions") return {"originalIncarnation": incarnation, "completeManifestReconciled": True, "canonicalCounters": counters, "receipts": len(observed), "legacyObjectsHealedUsedAsRepairEvidence": False, - "deferredIsRepair": False, "failedIsRepair": False} + "deferredIsRepair": False, "failedIsRepair": False, "unknownIsRepair": False} + + +def append_legacy_fixture(probe, fleet, directory, binding, expected, fixture): + """Keep an ACK-bound protected control while writing new real legacy objects.""" + for node in fleet.nodes: + proof = fleet.record("one-way-legacy-write-phase", {"node": node, **fleet.remote(node, "outcome-legacy-phase")}) + require(proof.get("runId") == binding["runId"] and proof.get("binarySha256") == binding["candidateBinarySha256"] + and proof.get("phase") == "legacy" and proof.get("oneWay") is True + and proof.get("candidateUnchanged") is True and proof.get("otherRuntimeSettingsUnchanged") is True + and proof.get("credentialsDisclosed") is False, "J02 phase acknowledgement differs from its original candidate") + fleet.start() + running = fleet.snapshot("actual-legacy-writing-processes") + require(all(scene.get("backgroundConfig", {}).get("RUSTFS_SHARD_INTEGRITY_WRITE") == "false" + and scene["backgroundConfig"].get("RUSTFS_HEAL_MRF_ENABLE") == "false" + and scene["backgroundConfig"].get("RUSTFS_SCANNER_ENABLED") == "false" for scene in running.values()), + "J02 legacy bytes require actual write-disabled, isolated candidate processes") + recorder = Recorder(probe, directory / "legacy-fixture") + appended = [] + for index in range(2): + key = f"legacy/{index:04d}.bin" + versions = 3 if fixture == "DV" else 1 + for version in range(versions): + body = payload(f"{binding['bucketPrefix']}/{key}/{version}", SIZE) + response = recorder.call("put_object", {"Bucket": binding["bucketPrefix"], "Key": key, + "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": fixture, "nonce": f"{index}-{version}"}, + "Tagging": "suite=heal&fixture=" + fixture}, body) + require(fixture != "DV" or response.get("VersionId") not in {None, "null"}, "J02 legacy DV PUT lacks an ACK version") + appended.append({"key": key, "versionId": response.get("VersionId", "null"), "isLatest": version == versions - 1, + "deleteMarker": False, "payload": {"sha256": hashlib.sha256(body).hexdigest(), "size": len(body)}}) + if fixture == "DV" and index == 0: + response = recorder.call("delete_object", {"Bucket": binding["bucketPrefix"], "Key": key}) + require(response.get("DeleteMarker") is True and response.get("VersionId"), "J02 legacy marker ACK is incomplete") + appended[-1]["isLatest"] = False + appended.append({"key": key, "versionId": response["VersionId"], "isLatest": True, "deleteMarker": True}) + return expected + appended + + +def legacy_manifest(original, bucket, expected): + """Require a complete physical cohort for every ACK version before admission.""" + manifest, unknown, evidence = [], [], [] + for entry in expected: + legacy = entry["key"].startswith("legacy/") + state = "metadata_healthy" if entry["deleteMarker"] else "unknown" if legacy else "verified_healthy" + manifest.append({**entry, "state": state}) + if entry["deleteMarker"]: + continue + target = map_target(original, bucket, entry["key"], "data", 1, entry["versionId"], latest_required=False) + parts = target["logical"]["parts"] + member_parts = [version["parts"] for member in target["cohort"] for version in member["metadata"]["versions"] + if public_version_id(version.get("versionId")) == public_version_id(entry["versionId"])] + require(len(member_parts) == len(target["cohort"]) and all(parts and all(type(part.get("integrityDescriptorPresent")) is bool + and part["integrityDescriptorPresent"] is (not legacy) for part in parts) for parts in member_parts), + "J02 metadata lacks exact protected/legacy part descriptor evidence") + require(target["logical"]["size"] == entry["payload"]["size"], "J02 metadata size differs from the ACK") + evidence.append({"key": entry["key"], "versionId": entry["versionId"], "legacy": legacy, + "completeCohort": target["cohort"], "parts": parts}) + if legacy: + unknown.append(version_identity(entry)) + require(unknown and any(not item["legacy"] for item in evidence), "J02 legacy scene lacks an independent protected control") + return {"manifest": manifest, "unknownIdentities": unknown, "physicalLegacyEvidence": evidence, + "syntheticMetadataWritten": False, "legacyPayloadIntegrityCertified": False} def fault_plan(original, bucket, expected, branch): @@ -157,7 +227,8 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): directory = args.evidence / fixture / topology / branch directory.mkdir(parents=True) run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" - fleet = Fleet(nodes, directory, run_id, args.tool_directory) + fleet = Fleet(nodes, directory, run_id, args.tool_directory, + runtime_profile="outcome-legacy" if branch == "unknown-negative" else "manual") bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] fleet.snapshot_buckets = [bucket] access, secret = secrets.token_hex(16), secrets.token_hex(24) @@ -177,7 +248,7 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) pending = {"operation": "seed", "bucket": bucket} - expected = seed(Recorder(probe, directory / "fixture"), bucket, fixture, 3, [SIZE]) + expected = seed(Recorder(probe, directory / "fixture"), bucket, fixture, 1 if branch == "unknown-negative" else 3, [SIZE]) baseline = fleet.record("ACK-bound-full-S3", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) fleet.record("complete-ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) pending = None @@ -187,13 +258,23 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): incarnation = validate_version_receipts(anchor, bucket, expected, 2)["bucketIncarnationId"] idle_fleet(fleet, binding) fleet.stop() + if branch == "unknown-negative": + fleet.snapshot("protected-control-before-legacy-writing", quiescent=True) + pending = {"operation": "append-real-legacy-fixture", "bucket": bucket} + expected = append_legacy_fixture(probe, fleet, directory, binding, expected, fixture) + baseline = fleet.record("legacy-ACK-bound-full-S3", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + fleet.record("complete-protected-and-legacy-ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + pending = None + idle_fleet(fleet, binding) + fleet.stop() original = fleet.snapshot("healthy-physical-baseline", quiescent=True) - plan = fault_plan(original, bucket, expected, branch) + plan = legacy_manifest(original, bucket, expected) if branch == "unknown-negative" else fault_plan(original, bucket, expected, branch) fleet.record("independent-manifest-and-frozen-fault-plan", plan) - for target in (plan["repair"], plan["failed"]): + for target in (() if branch == "unknown-negative" else (plan["repair"], plan["failed"])): verify_proof(fleet, target, "original-full-cohort-bitrot") prior = original - for index, target in enumerate(plan["faults"]): + faulted = original + for index, target in enumerate(plan.get("faults", [])): params = {"disk": target["disk"], "faultId": bounded_fault_id("J02", branch, index)} fleet.faults.append((target["node"], params)) receipt = fleet.record("stopped-original-file-fault", {"node": target["node"], **fleet.remote( @@ -218,11 +299,17 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): require(len(admissions) == 1, "J02 original fault task lacks an exact independent admission") fleet.record("original-terminal-before-any-rescue", status) result["responsibility"] = reconcile(status, bucket, plan["manifest"], options, incarnation, - attempt_failures=FAILED_ATTEMPTS if branch == "mixed-healthy-repaired-failed" else 1 if branch == "deferred" else 0) + attempt_failures=FAILED_ATTEMPTS if branch == "mixed-healthy-repaired-failed" else 1 if branch == "deferred" else 0, + legacy_unknown=tuple(tuple(pair) for pair in plan.get("unknownIdentities", []))) idle_fleet(fleet, binding) fleet.stop() after = fleet.snapshot("original-result-physical-scene", quiescent=True) - result["physical"] = retained_failure_scene(original, faulted, after, plan, branch) + if branch == "unknown-negative": + require(physical_content(original) == physical_content(after), "J02 legacy negative mutated its original physical scene") + result["physical"] = {"physicalUnchanged": True, "productRepairClaimed": False, + "legacyPayloadIntegrityCertified": False} + else: + result["physical"] = retained_failure_scene(original, faulted, after, plan, branch) if branch == "mixed-healthy-repaired-failed": verify_proof(fleet, plan["repair"], "actual-repaired-shard-before-rescue-or-GET") rescue(fleet) @@ -231,7 +318,7 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): validate_repaired_physical(original, restored, plan["repair"]) else: require(physical_content(original) == physical_content(restored), "J02 rescue changed the original ACK generation") - for target in (plan["repair"], plan["failed"]): + for target in (() if branch == "unknown-negative" else (plan["repair"], plan["failed"])): verify_proof(fleet, target, "all-original-shards-after-fixture-rescue") fleet.start() for endpoint in binding["endpoints"]: @@ -295,13 +382,7 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"J02 interrupted by sig for topology, selected in (("H1", nodes[:1]), ("H2", nodes)): for fixture in ("DB", "DV"): for branch in BRANCHES: - if branch == "unknown-negative": - result = {"topology": topology, "fixture": fixture, "branch": branch, "verdict": "BLOCKED", "cleanup": "PASS", - "evidenceKind": "separate-exact-candidate-native-outcome-negative", "native": native, - "reason": "exact candidate native unknown-identity guard is a prerequisite; a reviewed live unknown/lock-deferred fixture is still required", - "fleetUnknownWindowClaimed": False, "productRepairClaimed": False} - else: - result = run_branch(args, topology, selected, fixture, branch, candidate) + result = run_branch(args, topology, selected, fixture, branch, candidate) results.append(result) if result["cleanup"] != "PASS": return 1 finally: diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index 53272cdf..18f7fc56 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -3408,7 +3408,7 @@ "dispatchable": false, "code": "J02", "reasonCode": "EVIDENCE_MIXED_OUTCOME_ORCHESTRATION", - "reason": "已有 2 项候选版本 outcome 原生回归和部分计数校验，但尚缺真实混合 healthy/repaired/failed/deferred/unknown 夹具、dry-run 独立任务、逐对象完整 manifest 与计数/attemptFailures 对账；这是执行器与证据未闭环，并非已证实产品缺少 API。", + "reason": "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral 和真实 SDK legacy Unknown 执行分支及逐对象 manifest/attemptFailures 对账；仍需绑定固定候选的新 inspector、完整串行 live 证据和同批 deferred 源断言核验，故保持禁止 dispatch；本地检查不能作为完整 PASS。", "requiredCapabilities": [ "mixed_outcome_fixture", "per_object_canonical_receipts", diff --git a/scripts/validation_heal/tools/heal_meta_inspect.rs b/scripts/validation_heal/tools/heal_meta_inspect.rs index 5fada4c4..fe03b16e 100644 --- a/scripts/validation_heal/tools/heal_meta_inspect.rs +++ b/scripts/validation_heal/tools/heal_meta_inspect.rs @@ -268,7 +268,7 @@ fn main() -> Result<(), Box> { "erasure": info.erasure, "role": role, "parts": info.parts.iter().map(|part| json!({ "number": part.number, "size": part.size, "actualSize": part.actual_size, - "etag": part.etag + "etag": part.etag, "integrityDescriptorPresent": part.integrity.is_some() })).collect::>() })) }) diff --git a/test_validation_heal_incarnation_replay.py b/test_validation_heal_incarnation_replay.py index 2313ec07..f8ff0fbd 100644 --- a/test_validation_heal_incarnation_replay.py +++ b/test_validation_heal_incarnation_replay.py @@ -344,7 +344,7 @@ def test_only_complete_capture_transition_replay_changes_mrf_and_keeps_one_worke (self.state / "C11-replay-installed.json").write_bytes(fault.encode(self.binding)) self.assertEqual(self.phase("replay")["mrfEnabled"], "true") self.owner["incarnationPhase"] = "replay" - fleet_node.incarnation_configuration(self.state, self.owner) + fleet_node.runtime_configuration(self.state, self.owner) with self.assertRaisesRegex(ValueError, "refund"): self.phase("transition") def test_configuration_commit_interruption_refuses_process_start_before_systemd(self): @@ -394,7 +394,7 @@ def test_substituted_markers_and_archive_refuse_before_changing_runtime(self): def test_duplicate_config_and_caller_runtime_override_refuse(self): with self.env.open("a") as stream: stream.write("RUSTFS_HEAL_MRF_ENABLE=true\n") - with self.assertRaises(ValueError): fleet_node.incarnation_configuration(self.state, self.owner) + with self.assertRaises(ValueError): fleet_node.runtime_configuration(self.state, self.owner) with self.assertRaisesRegex(ValueError, "arbitrary"): fleet_node.incarnation_operation({"runId": self.owner["runId"], "action": "incarnation-phase", "phase": "transition", "environment": {"RUSTFS_HEAL_MRF_ENABLE": "false"}}) diff --git a/test_validation_heal_outcome.py b/test_validation_heal_outcome.py index e75e54e6..b23f2ee7 100644 --- a/test_validation_heal_outcome.py +++ b/test_validation_heal_outcome.py @@ -178,25 +178,26 @@ def test_native_prerequisite_rejection_happens_before_fleet_creation(self): with self.assertRaisesRegex(ValueError, "immutable evidence"): outcome.main() fleet.assert_not_called() - def test_native_only_unknown_branch_keeps_full_case_incomplete(self): + def test_unknown_branch_requires_real_executor_for_each_topology_and_fixture(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp); candidate = root / "candidate.json" candidate.write_text(json.dumps({"source_sha": outcome.CANDIDATE})) args = types.SimpleNamespace(candidate=candidate, evidence=root / "evidence", binary_sha="a" * 64) def branch(args, topology, nodes, fixture, name, candidate): - return {"topology": topology, "fixture": fixture, "branch": name, "verdict": "PASS", "cleanup": "PASS"} + return {"topology": topology, "fixture": fixture, "branch": name, "verdict": "BLOCKED" if name == "unknown-negative" else "PASS", "cleanup": "PASS"} with patch.object(outcome.argparse.ArgumentParser, "parse_args", return_value=args), \ patch.object(outcome, "verify_manager", return_value={"canonicalFleetBranchesExecuted": False}), \ - patch.object(outcome, "run_branch", side_effect=branch), patch.object(outcome.signal, "signal"), \ + patch.object(outcome, "run_branch", side_effect=branch) as executed, patch.object(outcome.signal, "signal"), \ patch.dict(os.environ, {"GITHUB_RUN_ID": "1234", "RUSTFS_NODES": "vm000 vm001 vm002 vm003"}): self.assertEqual(outcome.main(), 1) summary = json.loads((args.evidence / "healthy-summary.json").read_text()) + self.assertEqual(executed.call_count, 16) self.assertEqual(len(summary["results"]), 16) self.assertFalse(summary["complete"]) self.assertEqual(summary["cleanup"], "PASS") negative = [row for row in summary["results"] if row["branch"] == "unknown-negative"] self.assertEqual(len(negative), 4) - self.assertTrue(all(row["verdict"] == "BLOCKED" and not row["fleetUnknownWindowClaimed"] for row in negative)) + self.assertTrue(all(row["verdict"] == "BLOCKED" for row in negative)) def test_dirty_cleanup_stops_before_next_fixture_or_topology(self): with tempfile.TemporaryDirectory() as tmp: @@ -215,4 +216,166 @@ def test_dirty_cleanup_stops_before_next_fixture_or_topology(self): self.assertEqual(summary["cleanup"], "FAIL") + +class LegacyUnknownTests(unittest.TestCase): + def physical(self): + original, bucket, expected = versioned_scene(["objects/0000.bin", "legacy/0000.bin"]) + for scene in original.values(): + for row in scene["files"]: + for version in row.get("metadata", {}).get("versions", []): + for part in version["parts"]: + part["integrityDescriptorPresent"] = not row["metadata"]["key"].startswith("legacy/") + return original, bucket, expected + + def test_unknown_oracle_requires_independent_complete_physical_legacy_and_protected_control(self): + original, bucket, expected = self.physical() + plan = outcome.legacy_manifest(original, bucket, expected) + incarnation = MixedOutcomeTest.incarnation + options = {**BASE_OPTIONS, "recursive": True} + receipts = [{"identity": {"kind": "object", "bucket": bucket, "object": row["key"], "versionId": row["versionId"], + "bucketIncarnationId": incarnation, "poolIndex": 0, "setIndex": 0}, + "disposition": {"state": row["state"]}} for row in plan["manifest"]] + counts = {"processed": 6, "healed": 0, "unchanged": 3, "skipped": 0, "failed": 0, + "unknown": 3, "attemptFailures": 0, "overflowed": False} + status = {"summary": "finished", "truncated": False, "settings": options, "objectsHealed": 99, + "outcome": {"execution": {"state": "completed"}, "coverage": "complete", "objectsTruncated": False, + "objects": receipts, "counters": counts}} + unknown = [tuple(pair) for pair in plan["unknownIdentities"]] + proof = outcome.reconcile(status, bucket, plan["manifest"], options, incarnation, attempt_failures=0, legacy_unknown=unknown) + self.assertFalse(proof["unknownIsRepair"]) + self.assertEqual(proof["canonicalCounters"]["healed"], 0) + with self.assertRaises(ValueError): outcome.reconcile(status, bucket, plan["manifest"], options, incarnation, attempt_failures=0) + for mutation in ("repaired", "identity", "failed", "incomplete", "unknown-count", "attempt-failure"): + changed = copy.deepcopy(status) + if mutation == "repaired": changed["outcome"]["objects"][-1]["disposition"]["state"] = "repaired" + elif mutation == "identity": changed["outcome"]["objects"][-1]["identity"]["bucketIncarnationId"] = "foreign" + elif mutation == "failed": changed["outcome"]["execution"]["state"] = "completed_with_errors" + elif mutation == "incomplete": changed["outcome"]["coverage"] = "partial" + elif mutation == "unknown-count": changed["outcome"]["counters"]["unknown"] = 0 + else: changed["outcome"]["counters"]["attemptFailures"] = 1 + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + outcome.reconcile(changed, bucket, plan["manifest"], options, incarnation, attempt_failures=0, legacy_unknown=unknown) + + def test_descriptor_missing_true_integer_or_a_missing_cohort_never_creates_unknown_fixture(self): + for mutation in ("missing", "protected", "coerced", "missing-member", "no-control"): + original, bucket, expected = self.physical() + row = next(x for x in original["node1"]["files"] if x.get("metadata", {}).get("key", "").startswith("legacy/")) + if mutation == "missing": row["metadata"]["versions"][0]["parts"][0].pop("integrityDescriptorPresent") + elif mutation == "protected": row["metadata"]["versions"][0]["parts"][0]["integrityDescriptorPresent"] = True + elif mutation == "coerced": row["metadata"]["versions"][0]["parts"][0]["integrityDescriptorPresent"] = 0 + elif mutation == "missing-member": original["node1"]["files"].remove(row) + else: expected = [x for x in expected if x["key"].startswith("legacy/")] + with self.subTest(mutation=mutation), self.assertRaises(ValueError): outcome.legacy_manifest(original, bucket, expected) + + def test_legacy_append_uses_actual_sdk_calls_after_all_node_write_mode_switches(self): + events = [] + def call(operation, params, body=None): + events.append((operation, params)) + if operation == "put_object": return {"VersionId": str(len(events))} + return {"VersionId": str(len(events)), "DeleteMarker": True} + probe = types.SimpleNamespace() + fleet = types.SimpleNamespace(nodes=["a", "b"], remote=lambda node, action: events.append((action, node)) or { + "runId": "gh-12345-1-h1", "binarySha256": "a" * 64, "phase": "legacy", "oneWay": True, + "candidateUnchanged": True, "otherRuntimeSettingsUnchanged": True, "credentialsDisclosed": False}, + record=lambda label, value: value, start=lambda: events.append(("start", None)), + snapshot=lambda label: {node: {"backgroundConfig": {"RUSTFS_SHARD_INTEGRITY_WRITE": "false", + "RUSTFS_HEAL_MRF_ENABLE": "false", "RUSTFS_SCANNER_ENABLED": "false"}} for node in ("a", "b")}) + with patch.object(outcome, "Recorder", return_value=types.SimpleNamespace(call=call)): + value = outcome.append_legacy_fixture(probe, fleet, Path("/unused"), + {"bucketPrefix": MixedOutcomeTest.bucket, "runId": "gh-12345-1-h1", "candidateBinarySha256": "a" * 64}, + [{"key": "protected-control"}], "DV") + self.assertEqual([name for name,_ in events[:3]], ["outcome-legacy-phase", "outcome-legacy-phase", "start"]) + self.assertEqual(len(value), 8) + self.assertEqual(sum(name == "put_object" for name,_ in events), 6) + self.assertEqual(sum(name == "delete_object" for name,_ in events), 1) + self.assertEqual(sum(row.get("deleteMarker", False) for row in value), 1) + + +class LegacyPhaseTests(unittest.TestCase): + def test_one_way_private_configuration_commit_survives_each_completed_write_boundary(self): + from scripts.validation_heal import fleet_node as node, heal_fault as fault + from test_validation_heal_retained_scene import RetainedSceneTests + for cut in (None, "J02-legacy-intent.json", "J02-service.env.new", "J02-phase.new", "rename-service.env", "rename-J02-phase.json"): + with self.subTest(cut=cut), tempfile.TemporaryDirectory() as temp: + state = Path(temp).resolve(); binary = b"candidate"; (state / "rustfs").write_bytes(binary) + owner = {"runId": "gh-12345-1-h1", "unit": "owned.service", "runtimeProfile": "outcome-legacy", + "binarySha256": __import__('hashlib').sha256(binary).hexdigest()} + config = {**node.runtime_settings(owner), "RUSTFS_ADDRESS": ":9000", "RUSTFS_SECRET_KEY": "private-must-not-escape"} + env = state / "service.env"; env.write_text("".join(f"{key}={value}\n" for key,value in config.items())); env.chmod(0o600) + request = {"runId": owner["runId"], "action": "outcome-legacy-phase"} + def owned(run): + value = {**owner, "outcomePhase": "legacy"} if (state / "J02-phase.json").exists() else owner + return state, [], value + original_stat = os.stat + def root_stat(path, *args, **kwargs): + info = original_stat(path, *args, **kwargs) + values = {name: getattr(info,name) for name in dir(info) if name.startswith("st_")} + values["st_uid"] = 0 + return types.SimpleNamespace(**values) + original = fault.new_file + original_rename = os.rename + fired = False + def write(fd, name, value): + nonlocal fired + original(fd, name, value) + if cut == name and not fired: + fired = True; raise RuntimeError("lost acknowledgement") + def rename(source, target, **kwargs): + nonlocal fired + original_rename(source, target, **kwargs) + if cut == "rename-" + target and not fired: + fired = True; raise RuntimeError("lost acknowledgement after rename") + with patch.object(node, "owned", side_effect=owned), patch.object(node, "stopped"), \ + patch.object(node, "fault_module", return_value=fault), patch.object(fault, "mount_id", return_value=7), \ + RetainedSceneTests().root_stats(), patch.object(os, "stat", side_effect=root_stat), \ + patch.object(fault, "new_file", side_effect=write), patch.object(os, "rename", side_effect=rename): + if cut: + with self.assertRaises(RuntimeError): node.outcome_legacy_phase(request) + if not (state / "J02-phase.json").exists(): + with patch.object(node, "command") as command, self.assertRaisesRegex(ValueError, "incomplete"): + node.start({"runId": owner["runId"], "action": "start"}) + self.assertNotIn("systemd-run", [call.args[0] for call in command.call_args_list]) + result = node.outcome_legacy_phase(request) + self.assertTrue(result["oneWay"]) + self.assertNotIn(config["RUSTFS_SECRET_KEY"], json.dumps(result)) + actual = dict(line.split("=",1) for line in env.read_text().splitlines()) + self.assertEqual(actual, {**config, "RUSTFS_SHARD_INTEGRITY_WRITE": "false"}) + self.assertEqual(node.runtime_settings(owned(owner["runId"])[2])["RUSTFS_SHARD_INTEGRITY_WRITE"], "false") + self.assertTrue(node.outcome_legacy_phase(request)["otherRuntimeSettingsUnchanged"]) + env.write_text(env.read_text().replace("private-must-not-escape", "foreign")) + with self.assertRaises(ValueError): node.outcome_legacy_phase(request) + + def test_actual_owned_reader_rejects_uncommitted_or_foreign_phase_markers(self): + from scripts.validation_heal import fleet_node as node + from test_validation_heal_retained_scene import RetainedSceneTests + with tempfile.TemporaryDirectory() as directory: + state, roots, scope, request = RetainedSceneTests().scene(directory) + owner = {**request["expectedOwner"], "runtimeProfile": "outcome-legacy"} + for folder in (*roots, state): + folder.mkdir(parents=True, exist_ok=True) + (folder / "owner.json").write_text(json.dumps(owner)) + phase = state / "J02-phase.json" + with RetainedSceneTests().root_stats(), patch.object(node, "paths", return_value=(state, roots)), \ + patch.object(node, "mounts", return_value=owner["mounts"]): + self.assertNotIn("outcomePhase", node.owned(owner["runId"])[2]) + phase.write_text(json.dumps({"runId": owner["runId"], "phase": "legacy"})) + self.assertEqual(node.owned(owner["runId"])[2]["outcomePhase"], "legacy") + for value in ({"runId": owner["runId"], "phase": "protected"}, + {"runId": "foreign", "phase": "legacy"}, + {"runId": owner["runId"], "phase": "legacy", "extra": True}): + phase.write_text(json.dumps(value)) + with self.subTest(value=value), self.assertRaises(ValueError): node.owned(owner["runId"]) + + def test_phase_adapter_loads_real_fault_backend(self): + from scripts.validation_heal.healthy import Fleet + def execute(*args, **kwargs): + scope = {"__name__": "__J02_node__"} + exec(compile(kwargs["input"].rsplit("\nmain(",1)[0], "J02-remote", "exec"), scope) + self.assertTrue(callable(scope["fault"].new_file)) + self.assertTrue(callable(scope["outcome_legacy_phase"])) + return types.SimpleNamespace(returncode=0, stdout="{}", stderr="") + with tempfile.TemporaryDirectory() as directory, patch("scripts.validation_heal.healthy.subprocess.run", side_effect=execute): + Fleet(["node"], Path(directory), "gh-12345-1-h1", runtime_profile="outcome-legacy").remote("node", "outcome-legacy-phase") + + if __name__ == "__main__": unittest.main() From a58c10c98fe0816d87d130c151846c62d22bdb30 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 17:56:29 +0800 Subject: [PATCH 203/343] fix(validation): retain source coverage and unknown counter truth --- docs/validation-migration-map.md | 2 +- scripts/validation_heal/execution_policy.py | 2 +- scripts/validation_heal/outcome.py | 13 ++++++++--- scripts/validation_heal/recipes.json | 2 +- test_validation_heal_outcome.py | 25 ++++++++++++++++++++- 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 1d853ed3..8e1d4211 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1908,7 +1908,7 @@ J06 retry `38195131282` completed all twelve fixed ABBA phases with 1,440 schedu A03 run `38198122041` passes every required H2 cross-node branch at helper `ccef050df3b28416c51356951e78cb46dd74d10b` and driver `03fb3d78a0ffc4213583dd3bcb44c7ac38f42ead`. Independent verification of the original artifact members, canonical result gate, complete branch outcomes and immutable candidate provenance increases the unique canonical census to 89. The remaining serialized backlog continues on the same candidate; no prior failure is overwritten. -J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite. Its new live executor first verifies an ACK-bound protected control and original incarnation, switches only the owned candidate write mode through a durable one-way transaction, writes real SDK legacy objects, and requires every physical replica to show absent integrity descriptors. The original Deep task must report exact Unknown identities/counters and zero repairs for those objects, preserve the protected control and all original physical bytes, and remain unchanged after subsequent GETs. A fresh candidate-bound inspector build and complete live evidence remain required; this implementation is not a canonical PASS. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the source assertion requiring a mixed deferred population is still outstanding. The 84 cases, 301 branches and 532 topology templates are unchanged. +J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite. Its new live executor first verifies an ACK-bound protected control and original incarnation, switches only the owned candidate write mode through a durable one-way transaction, writes real SDK legacy objects, and requires every physical replica to show absent integrity descriptors. The original Deep task must report exact Unknown identities/counters and zero repairs for those objects, preserve the protected control and all original physical bytes, and remain unchanged after subsequent GETs. A fresh candidate-bound inspector build and complete live evidence remain required; this implementation is not a canonical PASS. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the original corrupt-metadata Unknown probe, version-specific grace Deferred and readonly failure/new-token compensation source procedures remain outstanding. Unknown is counted within skipped, not as repair; even if every current diagnostic branch passes, this executor emits complete=false and canonicalCasePassClaimed=false until those source procedures are covered. The 84 cases, 301 branches and 532 topology templates are unchanged. The corrected live census now contains 99 unique complete canonical PASS cases: 85 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926`, 12 on the older immutable source and two on source `16cf19817e0c81e24d0d787a916f734a34517ec5`. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 35 retained live failures, 7 required native barriers, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py index 7a7e3d2a..56246775 100644 --- a/scripts/validation_heal/execution_policy.py +++ b/scripts/validation_heal/execution_policy.py @@ -144,7 +144,7 @@ def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, . ), "heal.outcome-counters-and-error-truth": _native( "J02", "EVIDENCE_MIXED_OUTCOME_ORCHESTRATION", - "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral 和真实 SDK legacy Unknown 执行分支及逐对象 manifest/attemptFailures 对账；仍需绑定固定候选的新 inspector、完整串行 live 证据和同批 deferred 源断言核验，故保持禁止 dispatch；本地检查不能作为完整 PASS。", + "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral 和真实 SDK legacy Unknown 执行分支及逐对象 manifest/attemptFailures 对账；仍需绑定固定候选的新 inspector、完整串行 live 证据及原始损坏元数据 Unknown、grace Deferred、只读失败/新 token 补偿源流程，故保持禁止 dispatch；本地检查不能作为完整 PASS。", ("mixed_outcome_fixture", "per_object_canonical_receipts", "complete_live_orchestrator", "attempt_failure_visibility"), ), } diff --git a/scripts/validation_heal/outcome.py b/scripts/validation_heal/outcome.py index 6d9cc067..6ce75e10 100644 --- a/scripts/validation_heal/outcome.py +++ b/scripts/validation_heal/outcome.py @@ -45,6 +45,11 @@ SIZE = 5 * 1024**2 + 123 # The reviewed immutable candidate makes one attempt and three bucket-object retries. FAILED_ATTEMPTS = 4 +SOURCE_ASSERTION_GAPS = ( + "original-corrupt-metadata-unknown-negative", + "version-specific-grace-deferred-and-new-token-compensation", + "readonly-terminal-failure-and-new-token-compensation", +) def reconcile(status, bucket, expected, options, incarnation, *, attempt_failures, legacy_unknown=()): @@ -97,12 +102,13 @@ def reconcile(status, bucket, expected, options, incarnation, *, attempt_failure and identity.get("bucketIncarnationId") == incarnation, "J02 receipt escaped the original healthy-task bucket incarnation") if state == "unknown": + counters["unknown"] += 1 require(disposition == {"state": "unknown"} and receipt.get("detail") is None, "J02 legacy unknown changed its frozen proofless disposition") if state == "failed": require(disposition.get("details") == "retry_exhausted", "J02 below-R failure lost bounded retry exhaustion") category = {"repaired": "healed", "verified_healthy": "unchanged", "metadata_healthy": "unchanged", - "failed": "failed", "dry_run_observed": "skipped", "deferred": "skipped", "unknown": "unknown"}[state] + "failed": "failed", "dry_run_observed": "skipped", "deferred": "skipped", "unknown": "skipped"}[state] counters[category] += 1 observed[pair] = state actual_counters = outcome.get("counters", {}) @@ -390,8 +396,9 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"J02 interrupted by sig complete = {(row["topology"], row["fixture"], row["branch"]) for row in results} == wanted and len(results) == len(wanted) save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL", - "complete": complete and all(row["verdict"] == row["cleanup"] == "PASS" for row in results)}) - return 0 if complete and all(row["verdict"] == row["cleanup"] == "PASS" for row in results) else 1 + "diagnosticBranchesPassed": complete and all(row["verdict"] == row["cleanup"] == "PASS" for row in results), + "complete": False, "canonicalCasePassClaimed": False, "sourceAssertionGaps": list(SOURCE_ASSERTION_GAPS)}) + return 1 if __name__ == "__main__": sys.exit(main()) diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index 18f7fc56..30cb9098 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -3408,7 +3408,7 @@ "dispatchable": false, "code": "J02", "reasonCode": "EVIDENCE_MIXED_OUTCOME_ORCHESTRATION", - "reason": "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral 和真实 SDK legacy Unknown 执行分支及逐对象 manifest/attemptFailures 对账；仍需绑定固定候选的新 inspector、完整串行 live 证据和同批 deferred 源断言核验，故保持禁止 dispatch；本地检查不能作为完整 PASS。", + "reason": "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral 和真实 SDK legacy Unknown 执行分支及逐对象 manifest/attemptFailures 对账；仍需绑定固定候选的新 inspector、完整串行 live 证据及原始损坏元数据 Unknown、grace Deferred、只读失败/新 token 补偿源流程，故保持禁止 dispatch；本地检查不能作为完整 PASS。", "requiredCapabilities": [ "mixed_outcome_fixture", "per_object_canonical_receipts", diff --git a/test_validation_heal_outcome.py b/test_validation_heal_outcome.py index b23f2ee7..8914b8ab 100644 --- a/test_validation_heal_outcome.py +++ b/test_validation_heal_outcome.py @@ -199,6 +199,25 @@ def branch(args, topology, nodes, fixture, name, candidate): self.assertEqual(len(negative), 4) self.assertTrue(all(row["verdict"] == "BLOCKED" for row in negative)) + def test_all_diagnostic_branches_passing_cannot_omit_original_source_procedures(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); candidate = root / "candidate.json" + candidate.write_text(json.dumps({"source_sha": outcome.CANDIDATE})) + args = types.SimpleNamespace(candidate=candidate, evidence=root / "evidence", binary_sha="a" * 64) + def branch(args, topology, nodes, fixture, name, candidate): + return {"topology": topology, "fixture": fixture, "branch": name, "verdict": "PASS", "cleanup": "PASS"} + with patch.object(outcome.argparse.ArgumentParser, "parse_args", return_value=args), \ + patch.object(outcome, "verify_manager", return_value={}), patch.object(outcome, "run_branch", side_effect=branch), \ + patch.object(outcome.signal, "signal"), patch.dict(os.environ, {"GITHUB_RUN_ID": "1234"}): + self.assertEqual(outcome.main(), 1) + summary = json.loads((args.evidence / "healthy-summary.json").read_text()) + self.assertTrue(summary["diagnosticBranchesPassed"]) + self.assertFalse(summary["complete"]) + self.assertFalse(summary["canonicalCasePassClaimed"]) + self.assertEqual(set(summary["sourceAssertionGaps"]), { + "original-corrupt-metadata-unknown-negative", "version-specific-grace-deferred-and-new-token-compensation", + "readonly-terminal-failure-and-new-token-compensation"}) + def test_dirty_cleanup_stops_before_next_fixture_or_topology(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp); candidate = root / "candidate.json" @@ -235,7 +254,7 @@ def test_unknown_oracle_requires_independent_complete_physical_legacy_and_protec receipts = [{"identity": {"kind": "object", "bucket": bucket, "object": row["key"], "versionId": row["versionId"], "bucketIncarnationId": incarnation, "poolIndex": 0, "setIndex": 0}, "disposition": {"state": row["state"]}} for row in plan["manifest"]] - counts = {"processed": 6, "healed": 0, "unchanged": 3, "skipped": 0, "failed": 0, + counts = {"processed": 6, "healed": 0, "unchanged": 3, "skipped": 3, "failed": 0, "unknown": 3, "attemptFailures": 0, "overflowed": False} status = {"summary": "finished", "truncated": False, "settings": options, "objectsHealed": 99, "outcome": {"execution": {"state": "completed"}, "coverage": "complete", "objectsTruncated": False, @@ -244,6 +263,10 @@ def test_unknown_oracle_requires_independent_complete_physical_legacy_and_protec proof = outcome.reconcile(status, bucket, plan["manifest"], options, incarnation, attempt_failures=0, legacy_unknown=unknown) self.assertFalse(proof["unknownIsRepair"]) self.assertEqual(proof["canonicalCounters"]["healed"], 0) + counters = proof["canonicalCounters"] + self.assertEqual(counters["unknown"], 3) + self.assertEqual(counters["skipped"], 3) + self.assertEqual(counters["processed"], sum(counters[key] for key in ("healed", "unchanged", "skipped", "failed"))) with self.assertRaises(ValueError): outcome.reconcile(status, bucket, plan["manifest"], options, incarnation, attempt_failures=0) for mutation in ("repaired", "identity", "failed", "incomplete", "unknown-count", "attempt-failure"): changed = copy.deepcopy(status) From 492bbc19b575272c8219cee6be1d1a30e93660cf Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 18:46:00 +0800 Subject: [PATCH 204/343] test(validation): preserve readonly failure through compensation --- docs/validation-heal-cases.md | 4 +- docs/validation-heal-contract.md | 2 +- docs/validation-heal-operations.md | 4 +- docs/validation-migration-map.md | 5 +- scripts/validation_heal/execution_policy.py | 2 +- scripts/validation_heal/outcome.py | 137 ++++++++++++++++-- scripts/validation_heal/recipes.json | 3 +- test_validation_heal.py | 4 +- test_validation_heal_outcome.py | 151 +++++++++++++++++++- 9 files changed, 288 insertions(+), 24 deletions(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index 6ed174b4..b0819e9e 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1804,11 +1804,11 @@ Source assertions: ## J02: heal.outcome-counters-and-error-truth -Topologies: H1,H2. Branches: mixed-healthy-repaired-failed, dry-run, deferred, unknown-negative. +Topologies: H1,H2. Branches: mixed-healthy-repaired-failed, dry-run, deferred, readonly-terminal-failure, unknown-negative. Execution policy: `evidence-blocked`; disabled by default: `false`. -Blocked reason (`EVIDENCE_MIXED_OUTCOME_ORCHESTRATION`): 已有 2 项候选版本 outcome 原生回归和部分计数校验，但尚缺真实混合 healthy/repaired/failed/deferred/unknown 夹具、dry-run 独立任务、逐对象完整 manifest 与计数/attemptFailures 对账；这是执行器与证据未闭环，并非已证实产品缺少 API。 +Blocked reason (`EVIDENCE_MIXED_OUTCOME_ORCHESTRATION`): 已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral、真实 SDK legacy Unknown 和只读永久失败/新 token 补偿执行分支及逐对象 manifest/attemptFailures 对账；仍需完整串行 live 证据及原始损坏元数据 Unknown、grace Deferred/新 token 补偿源流程，故保持禁止 dispatch；本地检查不能作为完整 PASS。 1. 同批独立key建立健康、可恢复缺片、低于R的不可恢复对象；deferred由冻结的锁/依赖故障产生，dry-run另建任务。 2. 保存逐对象物理基线和期望disposition，对结果的execution/coverage/counters/object identities做逐项对账。 diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index 4e52be3a..5ebc447e 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -2,7 +2,7 @@ ## 状态与范围 -本规范保留源仓库的 84 个 Heal canonical 用例、301 个操作分支及 532 个适用拓扑分支模板。资源要求、公共通过判据和逐项附加断言必须一起满足。 +本规范保留源仓库的 84 个 Heal canonical 用例、302 个操作分支及 534 个适用拓扑分支模板。资源要求、公共通过判据和逐项附加断言必须一起满足。 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 4cdb5c31..cb86dc5c 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -1,6 +1,6 @@ # Heal 真机执行手册：84 项操作卡的公共命令 -此手册和 [84 项逐例操作卡](validation-heal-cases.md) 一起构成可执行的**人工操作方案 + 脚本辅助**。不是 84 个无人值守自动测试，也不是通过报告。操作卡列出 301 个分支模板；真实盘角色、阈值、损坏数量和拓扑仍需在每批 `parameterBindings` 展开，不能把模板数量当实测次数。 +此手册和 [84 项逐例操作卡](validation-heal-cases.md) 一起构成可执行的**人工操作方案 + 脚本辅助**。不是 84 个无人值守自动测试，也不是通过报告。操作卡列出 302 个分支模板；真实盘角色、阈值、损坏数量和拓扑仍需在每批 `parameterBindings` 展开，不能把模板数量当实测次数。 公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 已接入 60 个 Heal 用例，完整清单通过 `matrix.py list` 查看，以 `automaticExecutor` 为准。入口使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 @@ -22,7 +22,7 @@ python scripts/validation_heal/matrix.py plan --selection A01,A04,C01,C02,C04,J0 | 工具 | 已实现的职责 | 不代表什么 | | --- | --- | --- | -| `matrix.py` | 84 项/301 分支索引、逐例操作卡、冻结计划、证据完整性审计 | 不自动分配拓扑，不根据文件存在就判 PASS | +| `matrix.py` | 84 项/302 分支索引、逐例操作卡、冻结计划、证据完整性审计 | 不自动分配拓扑，不根据文件存在就判 PASS | | `heal_probe.py` | 带签名/匿名/坏签名的原始 Admin；受限 S3 操作；全版本/标签/内容摘要清单及比较 | HTTP 检查通过不代表 heal 成功；inventory 的 GET 会触发读修复 | | `heal_workload.py` | DB/DV/DM/DS/DX 真实夹具；有界并发、同key前台修改串行、请求前及ACK后持久落盘 | 不替用户推导竞态后的最终版本；超时写必须单独解析 | | `heal_fault.py` | Linux 描述符相对寻址、mount ID 保护、单文件 missing/bitflip/truncate/zero/header、快照和恢复 | 文件故障不是物理掉盘；不修改系统 metadata；不盲目恢复新代次 | diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 8e1d4211..4699a26d 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1908,7 +1908,7 @@ J06 retry `38195131282` completed all twelve fixed ABBA phases with 1,440 schedu A03 run `38198122041` passes every required H2 cross-node branch at helper `ccef050df3b28416c51356951e78cb46dd74d10b` and driver `03fb3d78a0ffc4213583dd3bcb44c7ac38f42ead`. Independent verification of the original artifact members, canonical result gate, complete branch outcomes and immutable candidate provenance increases the unique canonical census to 89. The remaining serialized backlog continues on the same candidate; no prior failure is overwritten. -J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite. Its new live executor first verifies an ACK-bound protected control and original incarnation, switches only the owned candidate write mode through a durable one-way transaction, writes real SDK legacy objects, and requires every physical replica to show absent integrity descriptors. The original Deep task must report exact Unknown identities/counters and zero repairs for those objects, preserve the protected control and all original physical bytes, and remain unchanged after subsequent GETs. A fresh candidate-bound inspector build and complete live evidence remain required; this implementation is not a canonical PASS. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the original corrupt-metadata Unknown probe, version-specific grace Deferred and readonly failure/new-token compensation source procedures remain outstanding. Unknown is counted within skipped, not as repair; even if every current diagnostic branch passes, this executor emits complete=false and canonicalCasePassClaimed=false until those source procedures are covered. The 84 cases, 301 branches and 532 topology templates are unchanged. +J02 now has a non-dispatched orchestration prerequisite in `outcome.py`. It reuses the existing Fleet, Probe, Recorder and physical fault backends for H1/H2 and DB/DV: one mixed healthy/repaired/below-R task, a separately admitted dry-run task, and an exact metadata-dependency refusal. Complete ACK identities, the actual R-1 geometry, original incarnation, typed dispositions and exact canonical counters are reconciled before fixture rescue; the original token/outcome is checked again after rescue. The reviewed candidate's four failed bucket-object attempts remain visible, and legacy `objectsHealed` is never used as proof of repair. Unknown native evidence remains a prerequisite. Its new live executor first verifies an ACK-bound protected control and original incarnation, switches only the owned candidate write mode through a durable one-way transaction, writes real SDK legacy objects, and requires every physical replica to show absent integrity descriptors. The original Deep task must report exact Unknown identities/counters and zero repairs for those objects, preserve the protected control and all original physical bytes, and remain unchanged after subsequent GETs. A fresh candidate-bound inspector build and complete live evidence remain required; this implementation is not a canonical PASS. These control flows have local negative-contract coverage but no live execution claim; J02 remains outside dispatch and the original corrupt-metadata Unknown probe, version-specific grace Deferred source procedures remain outstanding. The source readonly failure/new-token compensation branch is now implemented and requires live verification. Unknown is counted within skipped, not as repair; even if every current diagnostic branch passes, this executor emits complete=false and canonicalCasePassClaimed=false until those source procedures are covered. The 84 cases, 301 branches and 532 topology templates are unchanged. The corrected live census now contains 99 unique complete canonical PASS cases: 85 on candidate source `072b57b37559fbe626aeda8abf61617fdcbdd926`, 12 on the older immutable source and two on source `16cf19817e0c81e24d0d787a916f734a34517ec5`. C04 `heal.truncated-and-zeroed-shard` run `38203454300` and `heal.completed-multipart-object` run `38204339623` were independently audited with clean finalization. The derived ledger records 35 retained live failures, 7 required native barriers, 4 orchestration/evidence blockers, and 13 environment scenarios disabled by default. @@ -1993,3 +1993,6 @@ Independent examination of the original F07 H2 DB stopped records in `3827731227 The F07 node adapter now validates each exact disk/slot against its frozen committed owner, sequence, payload digest and active bucket incarnation. The all-source plan is validated before the first apply; the selected target still needs its original valid manifest/lifecycle, unchanged before-bytes and stopped candidate. Previously, the adapter decoded the whole cohort on every apply, so the first intended CRC fault prevented the second injection. A real filesystem regression fails at that second call with global revalidation restored, then passes all four disk/slot mutations and byte-exact original rescue after the fix. Global MRF observation remains strict and rejects the same damaged cohort; replica-scoped observations are explicitly marked and cannot certify complete responsibility. Held/accepted/unbound/foreign-incarnation older sources refuse preflight. All 112 focused contracts, 652 Heal tests (eight platform skips), ten migration tests, AST and whitespace checks pass. No new live or canonical PASS is claimed. Recovery diagnostic `38704914473/1` ran on helper `9c01cb8` after the independent Fault-Tolerance workflow released the shared fleet lock. It refused before the first physical snapshot: vm000 disk 0 no longer contained its owner marker. Its original namespace records show nine absent run roots (disks 0-2 on vm000-vm002), seven surviving roots and all four state directories; every service is stopped. No MRF export, restoration or finalization occurred. This is externally drifted/incomplete evidence, not clean recovery or a case PASS. Namespace inspection now emits bounded regular-file owner fingerprints (raw/canonical digests and UID/GID/mode/inode/device), without marker contents or credentials, so surviving state can be compared with the original reserved receipts. Tables installation previously recursively changed ownership of the entire mount; its chown is now restricted to its own mnmd subtree. That known interference is not asserted to explain the nine missing roots. Original failures, surviving roots and native/canonical gates remain retained pending forensic inspection. + + +The 2026-10-15 source reconciliation restores the omitted J02 `readonly-terminal-failure` branch from frozen source script `524eb5b32c3236497fdd8a9271df0494fa3d9974`. Heal retains 84 canonical cases; its operation inventory is now 302 branches and 534 applicable topology templates. The current-token missing-shard fixture uses an actual private service-namespace read-only gate and proves EROFS before admission. Its original task must retain one permanent, unbound key-level failure and exactly one attempt; release must leave the shard missing. A distinct compensating token must repair the original current ACK version at the protected-control incarnation, with physical reconstruction and bitrot evidence captured before GETs. Full public versions/tags and the original failed token are checked again after compensation. Lost gate acknowledgements retain the gate registration for cleanup. This is local implementation evidence, not a live PASS; original corrupt-metadata Unknown and grace Deferred/new-token source procedures remain incomplete, so all current diagnostics passing still cannot claim full J02 completion. diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py index 56246775..b8855a1a 100644 --- a/scripts/validation_heal/execution_policy.py +++ b/scripts/validation_heal/execution_policy.py @@ -144,7 +144,7 @@ def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, . ), "heal.outcome-counters-and-error-truth": _native( "J02", "EVIDENCE_MIXED_OUTCOME_ORCHESTRATION", - "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral 和真实 SDK legacy Unknown 执行分支及逐对象 manifest/attemptFailures 对账；仍需绑定固定候选的新 inspector、完整串行 live 证据及原始损坏元数据 Unknown、grace Deferred、只读失败/新 token 补偿源流程，故保持禁止 dispatch；本地检查不能作为完整 PASS。", + "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral、真实 SDK legacy Unknown 和只读永久失败/新 token 补偿执行分支及逐对象 manifest/attemptFailures 对账；仍需完整串行 live 证据及原始损坏元数据 Unknown、grace Deferred/新 token 补偿源流程，故保持禁止 dispatch；本地检查不能作为完整 PASS。", ("mixed_outcome_fixture", "per_object_canonical_receipts", "complete_live_orchestrator", "attempt_failure_visibility"), ), } diff --git a/scripts/validation_heal/outcome.py b/scripts/validation_heal/outcome.py index 6ce75e10..666cf78c 100644 --- a/scripts/validation_heal/outcome.py +++ b/scripts/validation_heal/outcome.py @@ -2,6 +2,7 @@ """J02: reconcile original mixed-task receipts with ACKs and physical faults.""" import argparse +import errno from datetime import datetime, timedelta, timezone import hashlib import json @@ -21,7 +22,7 @@ from .manager_fixture import verify as verify_manager from .matrix import select from .native_regressions import CANDIDATE - from .shards import files, map_target, run_task, validate_fault, validate_repaired_physical + from .shards import files, map_target, run_task, validate_fault, validate_repaired_physical, validate_repaired_status from .stop import idle_fleet from .versions import (metadata_target, public_version_id, validate_unrecoverable_get, validate_version_receipts, verify_proof, verify_quorum_contract, version_identity) @@ -34,21 +35,20 @@ from manager_fixture import verify as verify_manager from matrix import select from native_regressions import CANDIDATE - from shards import files, map_target, run_task, validate_fault, validate_repaired_physical + from shards import files, map_target, run_task, validate_fault, validate_repaired_physical, validate_repaired_status from stop import idle_fleet from versions import (metadata_target, public_version_id, validate_unrecoverable_get, validate_version_receipts, verify_proof, verify_quorum_contract, version_identity) CASE_ID = "heal.outcome-counters-and-error-truth" -BRANCHES = ("mixed-healthy-repaired-failed", "dry-run", "deferred", "unknown-negative") +BRANCHES = ("mixed-healthy-repaired-failed", "dry-run", "deferred", "readonly-terminal-failure", "unknown-negative") SIZE = 5 * 1024**2 + 123 # The reviewed immutable candidate makes one attempt and three bucket-object retries. FAILED_ATTEMPTS = 4 SOURCE_ASSERTION_GAPS = ( "original-corrupt-metadata-unknown-negative", "version-specific-grace-deferred-and-new-token-compensation", - "readonly-terminal-failure-and-new-token-compensation", ) @@ -181,11 +181,15 @@ def legacy_manifest(original, bucket, expected): def fault_plan(original, bucket, expected, branch): - require(branch in BRANCHES[:3], "J02 native negative branch cannot mutate fleet files") - repair_entry = next(entry for entry in expected if entry["key"] == "objects/0001.bin" and not entry["deleteMarker"]) + require(branch in BRANCHES[:4], "J02 unknown diagnostic cannot mutate fleet files") + repair_entry = next(entry for entry in expected if entry["key"] == "objects/0001.bin" and not entry["deleteMarker"] + and (branch != "readonly-terminal-failure" or entry["isLatest"])) failed_entry = next(entry for entry in expected if entry["key"] == "objects/0002.bin" and not entry["deleteMarker"]) repair = map_target(original, bucket, repair_entry["key"], "data", 1, repair_entry["versionId"], latest_required=False) failed = map_target(original, bucket, failed_entry["key"], "data", 1, failed_entry["versionId"], latest_required=False) + if branch == "readonly-terminal-failure": + require(repair["logical"]["isLatest"] is True, "J02 readonly target must be the original current ACK version") + return {"repair": repair, "faults": [repair], "originalReadonlySourceProcedure": True} ordered = sorted(failed["cohort"], key=lambda member: member["index"]) read_quorum = failed["logical"]["erasure"]["data_blocks"] faults = [{**failed, **member} for member in ordered[read_quorum - 1:]] @@ -229,6 +233,109 @@ def retained_failure_scene(original, faulted, after, plan, branch): "oracleViewOnly": True, "fixtureRescueIsProductRepair": False} +def readonly_failure(status, bucket, key, options): + """Freeze the source operation card's unbound permanent key-level failure.""" + outcome = status.get("outcome") or {} + require(status.get("summary") == "stopped" and status.get("settings") == options + and status.get("truncated") is False and outcome.get("execution", {}).get("state") == "completed_with_errors" + and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False, + "J02 readonly task lost its complete permanent terminal failure") + objects = outcome.get("objects", []) + require(len(objects) == 1, "J02 readonly task lost its single key-level responsibility") + identity = objects[0].get("identity", {}) + require(set(identity) == {"kind", "bucket", "object", "versionId", "bucketIncarnationId", "poolIndex", "setIndex"} + and identity.get("kind") == "object" and identity.get("bucket") == bucket and identity.get("object") == key + and identity.get("versionId") is None and identity.get("bucketIncarnationId") is None + and all(type(identity.get(name)) is int and identity[name] == 0 for name in ("poolIndex", "setIndex")) + and objects[0].get("disposition") == {"state": "failed", "details": "permanent"}, + "J02 readonly failure invented a version/incarnation or changed its original failure class") + expected = {"processed": 1, "healed": 0, "unchanged": 0, "skipped": 0, "failed": 1, + "unknown": 0, "attemptFailures": 1, "overflowed": False} + counters = outcome.get("counters", {}) + require(all(type(counters.get(name)) is type(value) and counters[name] == value for name, value in expected.items()), + "J02 readonly failure hid attempts or reported a repair") + return {"canonicalCounters": expected, "originalKeyLevelFailurePreserved": True, "repairClaimed": False} + + +def unchanged_token(probe, fleet, admission, status, label): + response = fleet.record(label, probe.admin(admission["path"], [("clientToken", admission["clientToken"])], b"")) + require(response.get("httpStatus") == 200 and response.get("response", {}).get("summary") == status["summary"] + and response["response"].get("settings") == status["settings"] + and response["response"].get("truncated") is False + and canonical(response["response"].get("outcome")) == canonical(status["outcome"]), + "J02 later work replaced the original readonly failure responsibility") + + +def readonly_compensation(probe, fleet, binding, original, faulted, plan, expected, baseline, incarnation, active): + target = plan["repair"] + gate = fleet.record("readonly-actual-process-probe", {"node": target["node"], + **fleet.remote(target["node"], "drive-gate-probe")}) + require(type(gate.get("disk")) is int and gate["disk"] == target["disk"] and type(gate.get("errno")) is int and gate["errno"] == errno.EROFS + and gate.get("name") == "EROFS" and gate.get("sameServiceIdentity") is True + and gate.get("privateMountNamespace") is True and gate.get("hostMountRemounted") is False, + "J02 readonly refusal was not observed in the original candidate namespace") + options = {**BASE_OPTIONS, "recursive": False} + scope = {"bucket": binding["bucketPrefix"], "key": target["key"]} + controls = [entry for entry in expected if entry["key"] != target["key"] and entry["isLatest"] and not entry["deleteMarker"]] + require(controls, "J02 readonly task lacks an independently healthy current control") + control = controls[0] + control_status = run_task(probe, fleet, {"bucket": scope["bucket"], "key": control["key"]}, 2, active, + option_overrides=options, status_endpoints=tuple(binding["endpoints"])[1:]) + fleet.record("readonly-healthy-control-under-gate", control_status) + control_proof = validate_version_receipts(control_status, scope["bucket"], [control], 2) + require(control_status.get("settings") == options and control_proof["bucketIncarnationId"] == incarnation + and control_status["outcome"]["objects"][0]["identity"].get("kind") == "object", + "J02 readonly control lost the original incarnation or actual Deep verification") + require(physical_content(fleet.snapshot("readonly-after-gated-control-before-admission")) == physical_content(faulted), + "J02 healthy control changed a fault or its own original bytes") + admissions = [] + status = run_task(probe, fleet, scope, 2, active, option_overrides=options, + status_endpoints=tuple(binding["endpoints"])[1:], admissions=admissions) + fleet.record("readonly-original-terminal-before-release", status) + negative = readonly_failure(status, scope["bucket"], scope["key"], options) + require(len(admissions) == 1, "J02 readonly task lacks one independent admission") + idle_fleet(fleet, binding) + fleet.stop() + require(physical_content(fleet.snapshot("readonly-original-failure-physical", quiescent=True)) == physical_content(faulted), + "J02 readonly task changed its still-unrepaired scene") + cleared = fleet.record("readonly-stopped-release", {"node": target["node"], + **fleet.remote(target["node"], "drive-gate", mode="clear")}) + require(cleared.get("state") == "CLEAR", "J02 readonly gate was not released") + fleet.drive_gates.remove(target["node"]) + fleet.start() + require(physical_content(fleet.snapshot("readonly-released-before-new-token")) == physical_content(faulted), + "J02 release repaired the object before compensating admission") + unchanged_token(probe, fleet, admissions[0], status, "readonly-original-token-after-release") + next_admissions = [] + repaired = run_task(probe, fleet, scope, 2, active, option_overrides=options, + status_endpoints=tuple(binding["endpoints"])[1:], admissions=next_admissions) + require(len(next_admissions) == 1 and next_admissions[0]["clientToken"] != admissions[0]["clientToken"], + "J02 compensation reused the original failed token") + fleet.record("readonly-new-token-original-result", repaired) + validate_repaired_status(repaired, target) + require(repaired.get("settings") == options and repaired["outcome"]["objects"][0]["identity"].get("kind") == "object" + and repaired["outcome"]["objects"][0]["identity"].get("bucketIncarnationId") == incarnation, + "J02 compensation changed options or the original protected-control incarnation") + idle_fleet(fleet, binding) + fleet.stop() + after = fleet.snapshot("readonly-new-token-physical-before-get", quiescent=True) + physical = validate_repaired_physical(original, after, target) + verify_proof(fleet, target, "readonly-new-token-full-shard-bitrot-before-get") + rescue(fleet) + require(physical_content(fleet.snapshot("readonly-after-fixture-rescue", quiescent=True)) == physical_content(after), + "J02 fixture rescue changed the product's compensating repair") + fleet.start() + for endpoint in binding["endpoints"]: + inventory = fleet.record("readonly-compensated-all-peer-S3", Probe(binding, endpoint).inventory( + binding["bucketPrefix"], unversioned_current_tags=all(entry["versionId"] == "null" for entry in expected))) + verify_seed(expected, inventory) + compare_inventories(baseline, inventory) + unchanged_token(probe, fleet, admissions[0], status, "readonly-original-token-after-compensation") + return {"negative": negative, "physical": physical, "originalToken": admissions[0]["clientToken"], + "compensatingToken": next_admissions[0]["clientToken"], "sameTokenRecoveryClaimed": False, + "originalFailureUnchanged": True, "productRepairByNewToken": True} + + def run_branch(args, topology, nodes, fixture, branch, candidate): directory = args.evidence / fixture / topology / branch directory.mkdir(parents=True) @@ -254,7 +361,7 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) pending = {"operation": "seed", "bucket": bucket} - expected = seed(Recorder(probe, directory / "fixture"), bucket, fixture, 1 if branch == "unknown-negative" else 3, [SIZE]) + expected = seed(Recorder(probe, directory / "fixture"), bucket, fixture, 1 if branch == "unknown-negative" else 4 if branch == "readonly-terminal-failure" else 3, [SIZE]) baseline = fleet.record("ACK-bound-full-S3", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) fleet.record("complete-ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) pending = None @@ -276,7 +383,7 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): original = fleet.snapshot("healthy-physical-baseline", quiescent=True) plan = legacy_manifest(original, bucket, expected) if branch == "unknown-negative" else fault_plan(original, bucket, expected, branch) fleet.record("independent-manifest-and-frozen-fault-plan", plan) - for target in (() if branch == "unknown-negative" else (plan["repair"], plan["failed"])): + for target in [plan[key] for key in ("repair", "failed") if key in plan]: verify_proof(fleet, target, "original-full-cohort-bitrot") prior = original faulted = original @@ -286,12 +393,22 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): receipt = fleet.record("stopped-original-file-fault", {"node": target["node"], **fleet.remote( target["node"], "apply-fault", **params, relativePath=target["relativePath"], expectedSha256=target["sha256"], kind="missing")}) - faulted = fleet.snapshot("actual-R-minus-one-fault", quiescent=True) + faulted = fleet.snapshot("readonly-single-missing-part" if branch == "readonly-terminal-failure" else "actual-R-minus-one-fault", quiescent=True) validate_fault(prior, faulted, target, receipt) prior = faulted + if branch == "readonly-terminal-failure": + target = plan["repair"] + fleet.drive_gates.add(target["node"]) + fleet.record("readonly-stopped-gate-arm", {"node": target["node"], **fleet.remote( + target["node"], "drive-gate", disk=target["disk"], mode="read-only")}) fleet.start() require(physical_content(faulted) == physical_content(fleet.snapshot("before-original-task")), "J02 attribution window lost: background activity changed a fault before admission") + if branch == "readonly-terminal-failure": + result["readonly"] = readonly_compensation(probe, fleet, binding, original, faulted, plan, + expected, baseline, incarnation, active) + result.update(verdict="PASS", fixtureRescueIsProductRepair=False, sameTokenRecoveryClaimed=False) + return result if branch == "mixed-healthy-repaired-failed": for endpoint in binding["endpoints"]: observed = fleet.record("pre-task-failed-version-GET", Probe(binding, endpoint).s3("get_object", { @@ -324,7 +441,7 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): validate_repaired_physical(original, restored, plan["repair"]) else: require(physical_content(original) == physical_content(restored), "J02 rescue changed the original ACK generation") - for target in (() if branch == "unknown-negative" else (plan["repair"], plan["failed"])): + for target in [plan[key] for key in ("repair", "failed") if key in plan]: verify_proof(fleet, target, "all-original-shards-after-fixture-rescue") fleet.start() for endpoint in binding["endpoints"]: diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index 30cb9098..e49dbc39 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -3369,6 +3369,7 @@ "mixed-healthy-repaired-failed", "dry-run", "deferred", + "readonly-terminal-failure", "unknown-negative" ], "steps": [ @@ -3408,7 +3409,7 @@ "dispatchable": false, "code": "J02", "reasonCode": "EVIDENCE_MIXED_OUTCOME_ORCHESTRATION", - "reason": "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral 和真实 SDK legacy Unknown 执行分支及逐对象 manifest/attemptFailures 对账；仍需绑定固定候选的新 inspector、完整串行 live 证据及原始损坏元数据 Unknown、grace Deferred、只读失败/新 token 补偿源流程，故保持禁止 dispatch；本地检查不能作为完整 PASS。", + "reason": "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral、真实 SDK legacy Unknown 和只读永久失败/新 token 补偿执行分支及逐对象 manifest/attemptFailures 对账；仍需完整串行 live 证据及原始损坏元数据 Unknown、grace Deferred/新 token 补偿源流程，故保持禁止 dispatch；本地检查不能作为完整 PASS。", "requiredCapabilities": [ "mixed_outcome_fixture", "per_object_canonical_receipts", diff --git a/test_validation_heal.py b/test_validation_heal.py index a522ce15..3eb79436 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -98,8 +98,8 @@ def test_preserves_all_source_cases_and_applicable_branches(self): inventory = cases() plan = compile_plan(inventory, TOPOLOGIES, "manual-heal-unit-test") self.assertEqual(len(inventory), 84) - self.assertEqual(sum(len(item["variants"]) for item in inventory), 301) - self.assertEqual(plan["branchTemplateCount"], 532) + self.assertEqual(sum(len(item["variants"]) for item in inventory), 302) + self.assertEqual(plan["branchTemplateCount"], 534) self.assertEqual(len(plan["nonApplicableCombinations"]), 4) self.assertFalse(plan["automaticExecution"]) self.assertFalse(plan["livePassClaimed"]) diff --git a/test_validation_heal_outcome.py b/test_validation_heal_outcome.py index 8914b8ab..8a52cbe5 100644 --- a/test_validation_heal_outcome.py +++ b/test_validation_heal_outcome.py @@ -191,8 +191,8 @@ def branch(args, topology, nodes, fixture, name, candidate): patch.dict(os.environ, {"GITHUB_RUN_ID": "1234", "RUSTFS_NODES": "vm000 vm001 vm002 vm003"}): self.assertEqual(outcome.main(), 1) summary = json.loads((args.evidence / "healthy-summary.json").read_text()) - self.assertEqual(executed.call_count, 16) - self.assertEqual(len(summary["results"]), 16) + self.assertEqual(executed.call_count, 20) + self.assertEqual(len(summary["results"]), 20) self.assertFalse(summary["complete"]) self.assertEqual(summary["cleanup"], "PASS") negative = [row for row in summary["results"] if row["branch"] == "unknown-negative"] @@ -215,8 +215,7 @@ def branch(args, topology, nodes, fixture, name, candidate): self.assertFalse(summary["complete"]) self.assertFalse(summary["canonicalCasePassClaimed"]) self.assertEqual(set(summary["sourceAssertionGaps"]), { - "original-corrupt-metadata-unknown-negative", "version-specific-grace-deferred-and-new-token-compensation", - "readonly-terminal-failure-and-new-token-compensation"}) + "original-corrupt-metadata-unknown-negative", "version-specific-grace-deferred-and-new-token-compensation"}) def test_dirty_cleanup_stops_before_next_fixture_or_topology(self): with tempfile.TemporaryDirectory() as tmp: @@ -401,4 +400,148 @@ def execute(*args, **kwargs): Fleet(["node"], Path(directory), "gh-12345-1-h1", runtime_profile="outcome-legacy").remote("node", "outcome-legacy-phase") + +class ReadonlySourceFlowTests(unittest.TestCase): + bucket = MixedOutcomeTest.bucket + key = "objects/0001.bin" + options = {**BASE_OPTIONS, "recursive": False} + + def failure(self): + return {"summary": "stopped", "settings": self.options, "truncated": False, "outcome": { + "execution": {"state": "completed_with_errors"}, "coverage": "complete", "objectsTruncated": False, + "counters": {"processed": 1, "healed": 0, "unchanged": 0, "skipped": 0, + "failed": 1, "unknown": 0, "attemptFailures": 1, "overflowed": False}, + "objects": [{"identity": {"kind": "object", "bucket": self.bucket, "object": self.key, + "versionId": None, "bucketIncarnationId": None, "poolIndex": 0, "setIndex": 0}, + "disposition": {"state": "failed", "details": "permanent"}}]}} + + def test_permanent_terminal_failure_must_not_invent_a_verified_version_or_incarnation(self): + status = self.failure() + self.assertFalse(outcome.readonly_failure(status, self.bucket, self.key, self.options)["repairClaimed"]) + for mutation in ("version", "incarnation", "kind", "pool-bool", "attempts", "healed", "unknown", "deferred", "recoverable", "settings"): + changed = copy.deepcopy(status); receipt = changed["outcome"]["objects"][0] + if mutation == "version": receipt["identity"]["versionId"] = "null" + elif mutation == "incarnation": receipt["identity"]["bucketIncarnationId"] = MixedOutcomeTest.incarnation + elif mutation == "kind": receipt["identity"]["kind"] = "bucket" + elif mutation == "pool-bool": receipt["identity"]["poolIndex"] = False + elif mutation == "attempts": changed["outcome"]["counters"]["attemptFailures"] = 0 + elif mutation == "healed": changed["outcome"]["counters"]["healed"] = 1 + elif mutation == "unknown": receipt["disposition"] = {"state": "unknown"} + elif mutation == "deferred": receipt["disposition"] = {"state": "deferred", "details": {"reason": "deadline"}} + elif mutation == "recoverable": receipt["disposition"]["details"] = "retry_exhausted" + else: changed["settings"] = {**self.options, "remove": True} + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + outcome.readonly_failure(changed, self.bucket, self.key, self.options) + + def test_lost_readonly_arm_ack_keeps_gate_registered_for_final_cleanup(self): + original, bucket, expected = versioned_scene([f"objects/{i:04d}.bin" for i in range(3)]) + actual_bucket = "rv-heal-" + __import__("hashlib").sha256(b"gh-12345-1-h1").hexdigest()[:16] + for scene in original.values(): + for row in scene["files"]: + row["path"] = row["path"].replace(bucket,actual_bucket) + if "metadata" in row: row["metadata"]["bucket"] = actual_bucket + bucket = actual_bucket + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); rc = root / "rc"; rc.write_bytes(b"test-cli") + args = types.SimpleNamespace(evidence=root / "evidence", rc_bin=rc, binary_sha="b" * 64, + testing_commit="a" * 40, tool_directory=root / "tools") + gates, current, cleanups = set(), copy.deepcopy(original), [] + def remote(node, action, **params): + if action == "apply-fault": + current[node]["files"] = [row for row in current[node]["files"] if row["path"] != params["relativePath"] or row["disk"] != params["disk"]] + return {} + if action == "drive-gate": + self.assertIn(node, gates) + raise RuntimeError("lost readonly gate ACK") + return {} + def cleanup(): + cleanups.append(set(gates)); return True + fleet = types.SimpleNamespace(nodes=["node1"], drive_gates=gates, reserved=["node1"], faults=[], remote=remote, + deploy=lambda *args: None, snapshot=lambda *args,**kwargs: copy.deepcopy(current), + record=lambda label,value: value, stop=lambda: None, start=lambda: None, cleanup=cleanup) + probe = types.SimpleNamespace(inventory=lambda *args,**kwargs: {"bucket":bucket,"entries":expected}) + with patch.object(outcome,"Fleet",return_value=fleet), patch.object(outcome,"Probe",return_value=probe), \ + patch.object(outcome,"Recorder"), patch.object(outcome,"seed",return_value=expected), \ + patch.object(outcome,"verify_inspector",return_value={}), patch.object(outcome,"verify_quorum_contract",return_value={}), \ + patch.object(outcome,"run_task",return_value={}), patch.object(outcome,"validate_version_receipts",return_value={"bucketIncarnationId":MixedOutcomeTest.incarnation}), \ + patch.object(outcome,"idle_fleet"), patch.object(outcome,"validate_fault"), patch.object(outcome,"verify_proof"), \ + patch.dict(os.environ,{"GITHUB_RUN_ID":"12345","GITHUB_RUN_ATTEMPT":"1"}): + result = outcome.run_branch(args,"H1",["node1"],"DV","readonly-terminal-failure", + {"source_sha":outcome.CANDIDATE,"package_sha256":"c" * 64}) + self.assertEqual(result["verdict"],"FAIL") + self.assertIn("lost readonly gate ACK",result["failure"]["message"]) + self.assertEqual(result["cleanup"],"PASS") + self.assertEqual(cleanups,[{"node1"}]) + + def test_readonly_plan_selects_exact_current_version_without_a_below_R_fault(self): + original, bucket, expected = versioned_scene([f"objects/{i:04d}.bin" for i in range(3)]) + plan = outcome.fault_plan(original, bucket, expected, "readonly-terminal-failure") + self.assertEqual(len(plan["faults"]), 1) + self.assertEqual(plan["repair"]["versionId"], "v2") + self.assertTrue(plan["repair"]["logical"]["isLatest"]) + self.assertEqual(plan["faults"][0], plan["repair"]) + + def test_compensation_observes_readonly_failure_then_distinct_new_token_without_replacing_original(self): + from scripts.validation_heal.shards import files + for variant in (None, "fake-gate", "same-token", "changed-original", "changed-control", "bad-control"): + with self.subTest(variant=variant): + original, bucket, expected = versioned_scene([f"objects/{i:04d}.bin" for i in range(3)]) + plan = outcome.fault_plan(original, bucket, expected, "readonly-terminal-failure") + target = plan["repair"]; faulted = copy.deepcopy(original) + faulted[target["node"]]["files"] = [row for row in faulted[target["node"]]["files"] + if (target["node"], row["disk"], row["path"]) != (target["node"], target["disk"], target["relativePath"])] + repaired_scene = copy.deepcopy(original) + if variant == "changed-control": repaired_scene["node1"]["files"][0]["sha256"] = "changed" + negative = self.failure() + positive = {"summary": "finished", "settings": self.options, "truncated": False, "outcome": { + "execution": {"state": "completed"}, "coverage": "complete", "objectsTruncated": False, + "counters": {"processed": 1, "healed": 1, "unchanged": 0, "skipped": 0, + "failed": 0, "unknown": 0, "attemptFailures": 0, "overflowed": False}, + "objects": [{"identity": {"kind": "object", "bucket": bucket, "object": self.key, + "versionId": target["versionId"], "bucketIncarnationId": MixedOutcomeTest.incarnation, + "poolIndex": 0, "setIndex": 0}, "disposition": {"state": "repaired"}}]}} + baseline = {"bucket": bucket, "entries": expected}; events = []; queries = [] + def remote(node, action, **params): + events.append(action) + if action == "drive-gate-probe": return {"disk": target["disk"], "errno": 13 if variant == "fake-gate" else 30, + "name": "EROFS", "sameServiceIdentity": True, "privateMountNamespace": True, "hostMountRemounted": False} + if action == "drive-gate": return {"state": "CLEAR"} + return {"restored": False} + def snapshot(label, **kwargs): + events.append(label) + return copy.deepcopy(faulted if label in {"readonly-original-failure-physical", "readonly-released-before-new-token", "readonly-after-gated-control-before-admission"} else repaired_scene) + fleet = types.SimpleNamespace(nodes=["node1"], record=lambda label,value: value, remote=remote, + drive_gates={target["node"]}, faults=[], stop=lambda: events.append("stop"), start=lambda: events.append("start"), snapshot=snapshot) + def admin(path, query, body): + queries.append(query); result = copy.deepcopy(negative) + if variant == "changed-original" and len(queries) == 2: result["outcome"]["counters"]["failed"] = 0 + return {"httpStatus": 200, "response": result} + probe = types.SimpleNamespace(admin=admin) + def task(probe,fleet,scope,mode,active,**kwargs): + number = len([item for item in events if item == "admit"]); events.append("admit") + if number == 0: + control = copy.deepcopy(positive) + control["outcome"]["counters"].update(healed=0,unchanged=1) + control["outcome"]["objects"][0]["identity"]["object"] = "objects/0000.bin" + control["outcome"]["objects"][0]["disposition"] = {"state":"unknown" if variant == "bad-control" else "verified_healthy"} + return control + kwargs["admissions"].append({"path": "/heal", "clientToken": "old" if number == 1 or variant == "same-token" else "new"}) + return negative if number == 1 else positive + binding = {"bucketPrefix":bucket,"endpoints":{"node1":"http://unused"}} + with patch.object(outcome,"run_task",side_effect=task), patch.object(outcome,"idle_fleet"), \ + patch.object(outcome,"verify_proof",side_effect=lambda *args: events.append("bitrot-proof")), \ + patch.object(outcome,"Probe",return_value=types.SimpleNamespace(inventory=lambda *args,**kwargs: baseline)): + if variant: + with self.assertRaises(ValueError): outcome.readonly_compensation(probe,fleet,binding,original,faulted,plan,expected,baseline,MixedOutcomeTest.incarnation,[]) + else: + value=outcome.readonly_compensation(probe,fleet,binding,original,faulted,plan,expected,baseline,MixedOutcomeTest.incarnation,[]) + self.assertFalse(value["sameTokenRecoveryClaimed"]) + self.assertTrue(value["originalFailureUnchanged"]) + self.assertNotEqual(value["originalToken"],value["compensatingToken"]) + self.assertEqual(sum(item == "admit" for item in events),3) + self.assertLess(events.index("readonly-original-failure-physical"),events.index("drive-gate")) + self.assertLess(events.index("readonly-released-before-new-token"),len(events)-1) + self.assertEqual(len(queries),2) + + if __name__ == "__main__": unittest.main() From d74a9fc2f175199d2aacdf060f4df2e0bba6244e Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 19:19:16 +0800 Subject: [PATCH 205/343] fix(validation): retain cancelled pre-admission heal fixtures --- docs/validation-heal-operations.md | 2 + scripts/validation_heal/fleet_node.py | 3 +- scripts/validation_heal/heal_fault.py | 6 +- scripts/validation_heal/healthy.py | 2 +- .../validation_heal/interrupted_recovery.py | 226 ++++++++++++++ .../validation_heal/recover_environment.py | 11 +- scripts/validation_heal/retained_scene.py | 28 +- test_validation_heal_interrupted_recovery.py | 275 ++++++++++++++++++ 8 files changed, 545 insertions(+), 8 deletions(-) create mode 100644 scripts/validation_heal/interrupted_recovery.py create mode 100644 test_validation_heal_interrupted_recovery.py diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index cb86dc5c..56b5efa3 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -610,6 +610,8 @@ hf recover --fault-id C02-data-middle --service-quiescent 只恢复同run的备份，原件摘要不符则拒绝；目标如果已经是原始正确字节不重写；如果变成既不是原始也不是注入后字节（可能有新ACK或合法metadata重写），拒绝盲目覆盖，交人工保留现场。恢复工具输出 `physicalRepairPassAllowed=false`；任何 `restoredFromBackup=true` 都不能作为产品补片证据。 +For an externally cancelled Functional Chain, `validation_cleanup_run_id` accepts only a completed run on the same workflow branch. F07 recovery supports the bounded H2/DV stale-generation boundary after four missing-part injections and before Admin admission: all 18 earlier branches must have complete cleanup receipts, and all 282 fixture requests must have matching successful SDK acknowledgements. Any additional action, unknown write, lost root or physical drift fails closed. Recovery stops the owned services, restores only the four original part backups, and verifies the complete original business files and disk formats. It then retains every data root and private state directory by the existing non-overwriting, same-filesystem archive protocol, with a durable inventory bound to each node's original business-scene digest. Partial restoration and partial archive moves are retryable. `ISOLATED` releases the active namespace while preserving backups; it never rewrites the original cancelled/RUNNING result, restores old system records, or claims canonical PASS. + 强制故障设置第二条恢复通道/超时撤销；本地 finally 不是网络或进程被杀时可靠的唯一恢复办法。退出前枚举本例网络规则、bind层、dm表、测试unit、临时IAM和未完成MPU，逐条撤销并复核。保留失败数据/backup，retention未到期则保留专用资源；记录剩余路径/到期安排。未恢复为 DIRTY/QUARANTINED，不继续下一用例。 每个**实际参数实例**都填写人工结论和证据索引，不只填一个基础case总绿： diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 9bd77468..92b6120f 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -935,7 +935,8 @@ def recover_fault(request): root, owner = fault_root(request) module = fault_module() with module.directory(root) as fd: - return module.recover_fault(fd, module.checked_owner(fd, owner["runId"], recovery=True), request["faultId"]) + return module.recover_fault(fd, module.checked_owner(fd, owner["runId"], recovery=True), request["faultId"], + expected_receipt_sha256=request.get("expectedReceiptSha256")) def verify_part(request): diff --git a/scripts/validation_heal/heal_fault.py b/scripts/validation_heal/heal_fault.py index 42b7efc6..fd90258b 100644 --- a/scripts/validation_heal/heal_fault.py +++ b/scripts/validation_heal/heal_fault.py @@ -207,10 +207,14 @@ def apply_fault(root_fd: int, owner: dict, relative: str, fault_id: str, kind: s return receipt -def recover_fault(root_fd: int, owner: dict, fault_id: str) -> dict: +def recover_fault(root_fd: int, owner: dict, fault_id: str, *, expected_receipt_sha256: str | None = None) -> dict: require(bool(re.fullmatch(r"[A-J][0-9]{2}-[a-z0-9-]{1,48}", fault_id)), "invalid fault ID") with parent(root_fd, ["quarantine"]) as backup_fd: receipt = json.loads(read_file(backup_fd, fault_id + ".intent.json", 16384)) + if expected_receipt_sha256 is not None: + require(isinstance(expected_receipt_sha256, str) and re.fullmatch(r"[0-9a-f]{64}", expected_receipt_sha256) + and digest(json.dumps(receipt, sort_keys=True, separators=(",", ":")).encode()) == expected_receipt_sha256, + "recovery intent differs from archived fault receipt") require(receipt["runId"] == owner["runId"] and receipt["faultId"] == fault_id and receipt["backup"] == fault_id + ".original", "recovery receipt mismatch") parts = target_parts(receipt["relativePath"], owner["bucketPrefix"]) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index b790ae01..eb20fabe 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -223,7 +223,7 @@ def remote(self, node, action, **params): if action == "format-schema-fault": source += "\nimport types\nschema_faults = types.ModuleType('schema_faults')\nexec(" + repr( NODE_SOURCE.with_name("schema_faults.py").read_text()) + ", schema_faults.__dict__)\n" - if action in {"encryption-configure", "snapshot", "audit-finalize", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize", "journal-retained-audit", "journal-retained-audit-finalize"}: + if action.startswith("retained-scene-") or action in {"encryption-configure", "snapshot", "audit-finalize", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize", "journal-retained-audit", "journal-retained-audit-finalize"}: source += "\nimport types\nencryption_runtime = types.ModuleType('encryption_runtime')\nexec(" + repr( NODE_SOURCE.with_name("encryption_runtime.py").read_text()) + ", encryption_runtime.__dict__)\n" replacement_actions = {"start", "cleanup", "audit-finalize", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize", "journal-retained-audit", "journal-retained-audit-finalize", "replace-drive", "replacement-state", "replacement-probe", "foreign-format", "restore-replacement", "snapshot", "format-schema-fault"} diff --git a/scripts/validation_heal/interrupted_recovery.py b/scripts/validation_heal/interrupted_recovery.py new file mode 100644 index 00000000..6ebc7754 --- /dev/null +++ b/scripts/validation_heal/interrupted_recovery.py @@ -0,0 +1,226 @@ +"""Restore and retain only the bounded F07 cancellation before Admin admission.""" + +from collections import Counter +import hashlib +import json +import re + +try: + from .healthy import physical_content + from .heal_probe import require +except ImportError: + from healthy import physical_content + from heal_probe import require + + +CASE = "heal.checkpoint-and-mrf-corruption" +BRANCHES = ("checkpoint-truncate", "future-schema", "mrf-checksum", "stale-generation", "journal-write-failure") + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def checkpoint_plan(artifact, failed_run, candidate): + results = list(artifact.glob("*/run/*/cases/*.json")) + require(len(results) == 1 and results[0].name == CASE + ".json", "interrupted recovery requires one unmixed case") + raw = results[0].read_bytes() + result = json.loads(raw) + require(result.get("case_id") == CASE and result.get("verdict") == "RUNNING" + and result.get("cleanup_status") == "PENDING" and not result.get("provenance"), + "interrupted recovery requires the unchanged unfinished result") + root = results[0].parents[3] + require(json.loads((root / "candidate.json").read_text()) == candidate, "interrupted candidate differs") + evidence = results[0].parents[1] / "evidence" / CASE + require(not (evidence / "healthy-summary.json").exists(), "interrupted case already has an aggregate result") + finished = {(fixture, topology, branch) for fixture in ("DB", "DV") for topology in ("H1", "H2") + for branch in BRANCHES} - {("DV", "H2", "stale-generation"), ("DV", "H2", "journal-write-failure")} + observed = list(evidence.glob("*/*/*/*branch-result.json")) + require(len(observed) == len(finished) == 18 + and {tuple(path.parent.relative_to(evidence).parts) for path in observed} == finished, + "interrupted recovery cannot omit another unfinished branch") + for path in observed: + row = json.loads(path.read_text()) + fixture, topology, branch = path.parent.relative_to(evidence).parts + require(row.get("fixture") == fixture and row.get("topology") == topology and row.get("branch") == branch + and row.get("verdict") in {"PASS", "FAIL"} and row.get("cleanup") == "PASS", + "prior interrupted branch was not finalized") + cleanup = [json.loads(p.read_text()) for p in path.parent.glob("*-cleanup.json")] + nodes = {"vm000"} if topology == "H1" else {f"vm00{i}" for i in range(4)} + require(len(cleanup) == len(nodes) and {r.get("node") for r in cleanup} == nodes + and all(r.get("cleanup") == "PASS" and r.get("unitStopped") is True and r.get("rootsRemoved") is True + for r in cleanup), "prior branch lacks its full stopped and removed fleet") + recovery = [p for p in path.parent.glob("*-recovery.json") if re.fullmatch(r"\d{5}-recovery.json", p.name)] + require(len(recovery) == 1 and json.loads(recovery[0].read_text()) == {"cleanup": "PASS", "errors": []}, + "prior branch has an unresolved recovery") + expected_folders = finished | {("DV", "H2", "stale-generation")} + require({tuple(path.relative_to(evidence).parts) for path in evidence.glob("*/*/*") if path.is_dir()} == expected_folders, + "interrupted evidence contains another unfinished branch") + folder = evidence / "DV/H2/stale-generation" + require(not (evidence / "DV/H2/journal-write-failure").exists(), "later interrupted branch already started") + require(all(path.name == "fixture" or re.fullmatch(r"[0-9]{5}-.+\.json", path.name) + for path in folder.iterdir()), "interrupted folder contains an unknown operation") + require(all(re.fullmatch(r"[0-9]{6}-(request|ack)\.json", path.name) + for path in (folder / "fixture").iterdir()), "interrupted fixture contains an unknown operation") + files = sorted(folder.glob("*.json")) + require(len(files) == 97 and all(path.name.startswith(f"{i:05d}-") for i, path in enumerate(files, 1)), + "interrupted recovery requires its exact pre-admission boundary") + labels = Counter(path.name[6:-5] for path in files) + require(labels == Counter({"binding": 1, "candidate-bound-inspector": 1, "candidate-terminal-options-contract": 1, + "preflight": 1, "integrity-fleet-attestation": 1, "reserved": 4, "inspector-upload": 4, "started": 2, + "deployed-topology": 1, "topology-proof": 1, "ACK-bound-complete-S3": 1, "complete-ACK-manifest": 1, + "task-exit-background": 4, "quiescent": 1, "healthy-physical-baseline": 1, + "baseline-full-cohort-bitrot": 64, "missing-original-shard": 4, "actual-missing-shard": 4}) + and files[-1].name == "00097-started.json", "interrupted scene has a task, durable fault or unknown action") + def one(label): + return json.loads(next(p for p in files if p.name[6:-5] == label).read_text()) + binding = one("binding") + run_id, bucket = binding.get("runId", ""), binding.get("bucketPrefix", "") + require(re.fullmatch(rf"gh-{failed_run}-[1-9][0-9]*-h2", run_id) + and re.fullmatch(r"rv-heal-[0-9a-f]{16}", bucket) + and binding.get("candidateCommit") == candidate["source_sha"] + and binding.get("candidatePackageSha256") == candidate["package_sha256"] + and re.fullmatch(r"[0-9a-f]{64}", binding.get("candidateBinarySha256", "")) + and re.fullmatch(r"[0-9a-f]{40}", binding.get("harnessCommit", "")), "interrupted binding differs") + nodes = [f"vm00{i}" for i in range(4)] + owners = {} + for path in files: + if path.name[6:-5] != "reserved": continue + receipt = json.loads(path.read_text()); owner = receipt.get("owner") or {} + node = receipt.get("node") + require(receipt.get("reserved") is True and node in nodes and node not in owners + and owner.get("runId") == run_id and owner.get("bucket") == bucket + and owner.get("binarySha256") == binding["candidateBinarySha256"] + and owner.get("unit") == "rustfs-validation-heal-" + run_id + ".service" + and owner.get("runtimeProfile") == "root-restart" and len(owner.get("mounts", [])) == 4, + "interrupted owner receipt differs") + owners[node] = owner + require(set(owners) == set(nodes), "interrupted fleet owner is incomplete") + requests, acks = sorted((folder / "fixture").glob("*-request.json")), sorted((folder / "fixture").glob("*-ack.json")) + require(len(requests) == len(acks) == 282, "interrupted fixture has unresolved SDK operations") + operations = Counter() + for seq, (request_path, ack_path) in enumerate(zip(requests, acks, strict=True), 1): + request, ack = json.loads(request_path.read_text()), json.loads(ack_path.read_text()) + observation = ack.get("observation") or {} + operation, params = request.get("operation"), request.get("params") or {} + require(type(request.get("sequence")) is int and type(ack.get("sequence")) is int + and request["sequence"] == ack["sequence"] == seq + and request.get("runId") == ack.get("runId") == run_id + and ack.get("requestFile") == request_path.name + and ack.get("requestSha256") == hashlib.sha256(request_path.read_bytes()).hexdigest() + and ack.get("operationCheck") == "PASS" and observation.get("operation") == operation + and observation.get("params") == params and params.get("Bucket") == bucket + and observation.get("acknowledged") is True and observation.get("clientCallInvoked") is True + and observation.get("resultUnknown") is False + and operation in {"create_bucket", "put_bucket_versioning", "put_object", "delete_object"}, + "interrupted SDK operation is unacknowledged, changed or outside its fixture") + if operation in {"put_object", "delete_object"}: + require(re.fullmatch(r"objects/[0-9]{4}\.bin", params.get("Key", "")) and "VersionId" not in params, + "interrupted SDK object operation differs") + operations[operation] += 1 + require(operations == Counter(create_bucket=1, put_bucket_versioning=1, put_object=240, delete_object=40), + "interrupted fixture operation census differs") + before = one("healthy-physical-baseline") + def indexed(scene): + require(set(scene) == set(nodes), "interrupted physical fleet differs") + out = {} + for node, value in scene.items(): + require(value.get("quiescent") is True and type(value.get("pid")) is int and value["pid"] == 0 + and value.get("binarySha256") == binding["candidateBinarySha256"] + and len(value.get("formats", [])) == 4, "interrupted scene is not the stopped exact candidate") + rows = value["files"] + out[node] = {(entry["disk"], entry["path"]): entry for entry in rows} + require(len(out[node]) == len(rows), "interrupted physical scene has duplicate files") + return out + previous = before; left = indexed(previous); faults = [] + for index, receipt_path in enumerate(sorted(folder.glob("*-missing-original-shard.json"))): + receipt = json.loads(receipt_path.read_text()) + number = int(receipt_path.name[:5]) + after_path = folder / f"{number+1:05d}-actual-missing-shard.json" + require(after_path in files, "interrupted injection has no immediate physical snapshot") + after = json.loads(after_path.read_text()); right = indexed(after) + removed = [(node, *key) for node in nodes for key in set(left[node]) - set(right[node])] + require(len(removed) == 1, "interrupted injection is not one exact missing part") + node, disk, relative = removed[0]; entry = left[node][(disk, relative)] + require(type(disk) is int and 0 <= disk < 4 + and re.fullmatch(re.escape(bucket) + r"/objects/[0-9]{4}\.bin/[0-9a-f-]{36}/part\.1", relative) + and receipt.get("schemaVersion") == 1 and receipt.get("runId") == run_id + and receipt.get("faultId") == f"F07-shard-{index}" and receipt.get("relativePath") == relative + and receipt.get("kind") == "missing" and receipt.get("fault") is None and receipt.get("offset") is None + and receipt.get("before") == {key: entry[key] for key in ("size", "sha256")} + and receipt.get("backup") == f"F07-shard-{index}.original" + and receipt.get("backend") == "logical-file-fault" and receipt.get("requiresQuiescentService") is True, + "interrupted missing-part receipt differs") + del left[node][(disk, relative)] + require(left == right and all(previous[n]["formats"] == after[n]["formats"] for n in nodes), + "interrupted injection changed other files or disk formats") + faults.append({"node": node, "disk": disk, "faultId": receipt["faultId"], "receipt": receipt}) + previous, left = after, right + require(len(faults) == 4, "interrupted fault census differs") + started = json.loads(files[-1].read_text()) + require(set(started) == set(nodes) and all(value == {"unit": owners[n]["unit"], "started": True} + for n, value in started.items()), "interrupted final service start differs") + return [{"kind": "interrupted-checkpoint-before-admission", "runId": run_id, "nodes": nodes, + "bucket": bucket, "binarySha256": binding["candidateBinarySha256"], "owners": owners, + "before": before, "failure": previous, "faults": faults, + "originalCaseSha256": hashlib.sha256(raw).hexdigest(), "originalHarnessCommit": binding["harnessCommit"]}] + + +def restore_and_isolate(fleet, plan, states, scene_matches): + """Preserve all backups and bytes, including when recovery is interrupted.""" + require(len(states) == 4 and {item["node"] for item in states} == set(plan["nodes"]), "interrupted current fleet differs") + active = [] + for item in states: + retained = item.get("retained") or {} + if retained.get("state") or any(retained.get("roots", [])): + continue + require(item["state"]["exists"] and len(item["roots"]) == 4 and all(root["exists"] for root in item["roots"]), + "interrupted fixture lost an original root; retain remaining scene") + active.append(item["node"]) + for node in active: + fleet.record("interrupted-recovery-stop", {"node": node, **fleet.remote(node, "stop")}) + current = {node: fleet.remote(node, "snapshot", quiescent=True) for node in active} + fleet.record("interrupted-current-physical", current) + if active: + subset = {**plan, "before": {n: plan["before"][n] for n in active}, + "failure": {n: plan["failure"][n] for n in active}} + require(all(value.get("quiescent") is True and type(value.get("pid")) is int and value["pid"] == 0 + and value.get("binarySha256") == plan["binarySha256"] for value in current.values()) + and scene_matches(current, subset), "interrupted current scene drifted; no backup restoration performed") + for fault in plan["faults"]: + if fault["node"] in active: + proof = fleet.record("interrupted-original-shard-rescue", {"node": fault["node"], + **fleet.remote(fault["node"], "recover-fault", disk=fault["disk"], faultId=fault["faultId"], + expectedReceiptSha256=digest({key: value for key, value in fault["receipt"].items() + if key != "witnessedAt"}))}) + require(proof.get("runId") == plan["runId"] and proof.get("faultId") == fault["faultId"] + and proof.get("physicalRepairPassAllowed") is False, "interrupted rescue receipt differs") + restored = {node: fleet.remote(node, "snapshot", quiescent=True) for node in active} + fleet.record("interrupted-original-business-restored", restored) + require(physical_content(restored) == {n: physical_content(plan["before"])[n] for n in active}, + "interrupted fixture did not restore its full original business scene") + audits = {} + def params(node): + return {"expectedOwner": plan["owners"][node], + "expectedOriginalSceneSha256": digest(physical_content(plan["before"])[node])} + for node in plan["nodes"]: + audits[node] = fleet.record("interrupted-archive-preflight", {"node": node, + **fleet.remote(node, "retained-scene-audit", **params(node))}) + require(audits[node].get("inspectionOnly") is True and audits[node].get("mutated") is False + and audits[node].get("baseline", {}).get("originalSceneSha256") == params(node)["expectedOriginalSceneSha256"], + "interrupted restored archive preflight differs") + for node in plan["nodes"]: + proof = fleet.record("interrupted-archive-isolation", {"node": node, + **fleet.remote(node, "retained-scene-isolate", **params(node), expectedBaseline=audits[node]["baseline"])}) + require(proof.get("activeNamespaceIsolated") is True and proof.get("canonicalPassClaimed") is False + and proof.get("baseline") == audits[node]["baseline"], "interrupted archive isolation differs") + for node in plan["nodes"]: + proof = fleet.record("interrupted-archive-post-audit", {"node": node, + **fleet.remote(node, "retained-scene-audit", **params(node))}) + require(proof.get("activeNamespaceIsolated") is True and proof.get("inspectionOnly") is True + and proof.get("mutated") is False and proof.get("baseline") == audits[node]["baseline"], + "interrupted archive changed after isolation") + return {"runId": plan["runId"], "cleanup": "ISOLATED", "activeNamespaceIsolated": True, + "originalBusinessFixtureRestored": True, "originalSceneRestored": False, + "canonicalPassClaimed": False, "originalVerdictRewritten": False, + "originalCaseSha256": plan["originalCaseSha256"]} diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index 930310aa..d74bedf0 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -12,9 +12,11 @@ try: from .healthy import Fleet, physical_content from .heal_probe import require, save + from .interrupted_recovery import checkpoint_plan, restore_and_isolate except ImportError: from healthy import Fleet, physical_content from heal_probe import require, save + from interrupted_recovery import checkpoint_plan, restore_and_isolate def quota_recovery_plan(artifact, failed_run, candidate): @@ -140,6 +142,9 @@ def network_recovery_plan(artifact, failed_run, candidate): def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") if list(artifact.glob("*/run/heal.checkpoint-and-mrf-corruption/cases/*.json")): + results = list(artifact.glob("*/run/heal.checkpoint-and-mrf-corruption/cases/*.json")) + if any(json.loads(path.read_text()).get("verdict") == "RUNNING" for path in results): + return checkpoint_plan(artifact, failed_run, candidate) folder = next(artifact.glob("*/run/heal.checkpoint-and-mrf-corruption/evidence/heal.checkpoint-and-mrf-corruption")) if list((folder / "DB/H2/mrf-checksum").glob("*-actual-original-MRF-CRC-injection.json")): return retained_mrf_recovery_plan(artifact, failed_run, candidate) @@ -532,7 +537,7 @@ def recovery_scene_matches(current, plan): """A retry may contain only the exact original shards already rescued.""" observed = physical_content(current) expected = physical_content(plan.get("failure", plan["before"])) - if plan.get("kind") != "owned-quota": + if plan.get("kind") not in {"owned-quota", "interrupted-checkpoint-before-admission"}: return observed == expected original = physical_content(plan["before"]) if set(observed) != set(expected): @@ -635,6 +640,10 @@ def main(): has_retained = any(item.get("retained", {}).get("state") or any(item.get("retained", {}).get("roots", [])) for item in states) all_absent = all(not item["state"]["exists"] and not any(root["exists"] for root in item["roots"]) for item in states) + if plan.get("kind") == "interrupted-checkpoint-before-admission": + complete.append({"failedRunId": args.failed_run_id, + **restore_and_isolate(fleet, plan, states, recovery_scene_matches)}) + continue if plan.get("kind") == "advanced-mrf-fixture" and (has_retained or (not all_absent and any( not root["exists"] for item in states for root in item["roots"]))): complete.append({"failedRunId": args.failed_run_id, **isolate_incomplete_scene(fleet, plan, args.artifact)}) diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py index 14c2cb1f..a1b01206 100644 --- a/scripts/validation_heal/retained_scene.py +++ b/scripts/validation_heal/retained_scene.py @@ -116,7 +116,10 @@ def visit(path): mrf = not state and relative in { "data/.rustfs.sys/.heal-mrf-snapshot.0.bin", "data/.rustfs.sys/.heal-mrf-snapshot.1.bin", "data/.rustfs.sys/.heal-mrf-manifest.json"} - if (state and relative == "rustfs") or fixture or mrf: + backup = (not state and owner.get("runtimeProfile") == "root-restart" + and len(parts) == 2 and parts[0] == "quarantine" + and re.fullmatch(r"F07-shard-[0-3]\.(original|intent\.json|applied\.json)", parts[1])) + if (state and relative == "rustfs") or fixture or mrf or backup: fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) try: require(stable(os.fstat(fd)) == stable(info) and info.st_size <= 2 * 1024**3, @@ -152,7 +155,7 @@ def inspect(request, node): require(isinstance(expected, dict) and expected.get("runId") == request["runId"] and expected.get("unit") == node["unit"](request["runId"]) and expected.get("hostname") == node["socket"].gethostname() - and expected.get("runtimeProfile") == "mrf-crash", "retained scene archived owner differs") + and expected.get("runtimeProfile") in {"mrf-crash", "root-restart"}, "retained scene archived owner differs") current = node["mounts"]() require(len(current) == len(expected.get("mounts", [])) == 4 and all(all(actual[key] == old[key] for key in ("target", "source", "fstype", "uuid", "maj:min")) @@ -193,6 +196,19 @@ def inspect(request, node): baseline = {"runId": request["runId"], "ownerSha256": sha(expected), "roots": witnesses[:4], "state": witnesses[4]} intent = retained_state.parent / (request["runId"] + ".intent.json") node["checked"](intent, state.parent) + if expected["runtimeProfile"] == "root-restart": + original_sha = request.get("expectedOriginalSceneSha256", "") + require(re.fullmatch(r"[0-9a-f]{64}", original_sha) is not None, + "restored fixture requires its original physical scene digest") + baseline["originalSceneSha256"] = original_sha + require("absent" not in locations, "restored fixture cannot omit an original root") + if not intent.exists(): + require(all(location == "active" for location in locations), "unbound retained fixture") + scene = node["snapshot"]({"runId": request["runId"], "action": "snapshot", "quiescent": True}) + require(scene.get("quiescent") is True and type(scene.get("pid")) is int and scene["pid"] == 0 + and scene.get("binarySha256") == expected["binarySha256"] + and sha({key: scene[key] for key in ("files", "formats")}) == original_sha, + "restored fixture differs from the original business scene") if intent.exists(): require(encoded(private_json(intent)) == encoded(baseline), "retained scene drifted from its durable isolation intent") else: @@ -201,8 +217,12 @@ def inspect(request, node): def run(request, node): - require(set(request) == ({"runId", "action", "expectedOwner", "expectedBaseline"} - if request["action"] == "retained-scene-isolate" else {"runId", "action", "expectedOwner"}), + fields = {"runId", "action", "expectedOwner"} + if request.get("expectedOwner", {}).get("runtimeProfile") == "root-restart": + fields.add("expectedOriginalSceneSha256") + if request["action"] == "retained-scene-isolate": + fields.add("expectedBaseline") + require(set(request) == fields, "retained scene accepts only its archived owner and frozen inventory") require(request["action"] in {"retained-scene-audit", "retained-scene-isolate"}, "invalid retained scene action") baseline, locations, state, roots, retained_state, retained_roots, intent = inspect(request, node) diff --git a/test_validation_heal_interrupted_recovery.py b/test_validation_heal_interrupted_recovery.py new file mode 100644 index 00000000..3b6c1f82 --- /dev/null +++ b/test_validation_heal_interrupted_recovery.py @@ -0,0 +1,275 @@ +"""Cancellation recovery rejects drift and retains every restored fixture byte.""" +import copy +import hashlib +import json +from pathlib import Path +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import interrupted_recovery as interrupted, retained_scene as retained +from scripts.validation_heal import recover_environment as recovery +from scripts.validation_heal.healthy import physical_content +import test_validation_heal_retained_scene as scene_tests + + +class InterruptedPlanTests(unittest.TestCase): + def fixture(self, root): + candidate = {"source_sha": "a" * 40, "package_sha256": "b" * 64} + artifact = root / "artifact"; case = artifact / "run" / interrupted.CASE + evidence = case / "evidence" / interrupted.CASE + result = case / "cases" / (interrupted.CASE + ".json") + result.parent.mkdir(parents=True) + result.write_text(json.dumps({"case_id": interrupted.CASE, "verdict": "RUNNING", "cleanup_status": "PENDING"})) + (artifact / "candidate.json").write_text(json.dumps(candidate)) + for fixture in ("DB", "DV"): + for topology in ("H1", "H2"): + for branch in interrupted.BRANCHES: + if fixture == "DV" and topology == "H2" and branch in {"stale-generation", "journal-write-failure"}: continue + folder = evidence / fixture / topology / branch; folder.mkdir(parents=True) + (folder / "00001-checkpoint-branch-result.json").write_text(json.dumps({ + "fixture": fixture, "topology": topology, "branch": branch, "verdict": "FAIL", "cleanup": "PASS"})) + nodes = ["vm000"] if topology == "H1" else [f"vm00{i}" for i in range(4)] + for i, node in enumerate(nodes): + (folder / f"{i+2:05d}-cleanup.json").write_text(json.dumps({"node": node, "cleanup": "PASS", + "unitStopped": True, "rootsRemoved": True})) + (folder / "00009-recovery.json").write_text(json.dumps({"cleanup": "PASS", "errors": []})) + folder = evidence / "DV/H2/stale-generation"; folder.mkdir(parents=True) + run_id, bucket = "gh-12345-1-h2", "rv-heal-0123456789abcdef" + binding = {"runId": run_id, "bucketPrefix": bucket, "candidateCommit": candidate["source_sha"], + "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": "c" * 64, + "harnessCommit": "d" * 40} + count = 0 + def emit(label, value): + nonlocal count + count += 1; (folder / f"{count:05d}-{label}.json").write_text(json.dumps(value)) + emit("binding", binding) + for label in ("candidate-bound-inspector", "candidate-terminal-options-contract", "preflight", "integrity-fleet-attestation"): + emit(label, {}) + nodes = [f"vm00{i}" for i in range(4)] + owners = {n: {"runId": run_id, "bucket": bucket, "binarySha256": "c" * 64, "runtimeProfile": "root-restart", + "unit": "rustfs-validation-heal-" + run_id + ".service", "mounts": [{}] * 4} for n in nodes} + for node in nodes: + emit("reserved", {"node": node, "reserved": True, "owner": owners[node]}); emit("inspector-upload", {}) + started = {n: {"unit": owners[n]["unit"], "started": True} for n in nodes} + emit("started", started) + for label in ("deployed-topology", "topology-proof", "ACK-bound-complete-S3", "complete-ACK-manifest"): + emit(label, {}) + for node in nodes: emit("task-exit-background", {}) + emit("quiescent", {}) + before = {n: {"quiescent": True, "pid": 0, "binarySha256": "c" * 64, "formats": [{}] * 4, "files": []} for n in nodes} + for i in range(4): + before["vm000"]["files"].append({"disk": 0, + "path": f"{bucket}/objects/{i:04d}.bin/00000000-0000-0000-0000-00000000000{i}/part.1", + "size": 17, "sha256": str(i) * 64}) + emit("healthy-physical-baseline", before) + current = copy.deepcopy(before) + for i in range(4): + for n in range(16): emit("baseline-full-cohort-bitrot", {}) + entry = current["vm000"]["files"].pop(0) + emit("missing-original-shard", {"schemaVersion": 1, "runId": run_id, "faultId": f"F07-shard-{i}", + "relativePath": entry["path"], "kind": "missing", "fault": None, "offset": None, + "before": {k: entry[k] for k in ("size", "sha256")}, "backup": f"F07-shard-{i}.original", + "backend": "logical-file-fault", "requiresQuiescentService": True}) + emit("actual-missing-shard", current) + emit("started", started) + fixture = folder / "fixture"; fixture.mkdir() + for seq in range(1, 283): + operation = "create_bucket" if seq == 1 else "put_bucket_versioning" if seq == 2 else "put_object" if seq <= 242 else "delete_object" + params = {"Bucket": bucket} + if seq > 2: params["Key"] = "objects/0001.bin" + request = {"runId": run_id, "sequence": seq, "operation": operation, "params": params} + request_path = fixture / f"{seq:06d}-request.json"; request_path.write_text(json.dumps(request)) + ack = {"runId": run_id, "sequence": seq, "requestFile": request_path.name, + "requestSha256": hashlib.sha256(request_path.read_bytes()).hexdigest(), "operationCheck": "PASS", + "observation": {"operation": operation, "params": params, "acknowledged": True, + "clientCallInvoked": True, "resultUnknown": False}} + (fixture / f"{seq:06d}-ack.json").write_text(json.dumps(ack)) + return candidate, folder, before + + def test_exact_boundary_and_rescue_retry_only_allow_original_parts(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); candidate, folder, before = self.fixture(root) + plan = recovery.recovery_plan(root, 12345, candidate)[0] + self.assertEqual(len(plan["faults"]), 4) + self.assertEqual({f["node"] for f in plan["faults"]}, {"vm000"}) + partial = copy.deepcopy(plan["failure"]) + partial["vm000"]["files"] = before["vm000"]["files"][:2] + self.assertTrue(recovery.recovery_scene_matches(partial, plan)) + partial["vm000"]["files"][0]["sha256"] = "f" * 64 + self.assertFalse(recovery.recovery_scene_matches(partial, plan)) + + def test_boundary_mutations_fail_closed(self): + for mutation in ("candidate", "pending", "prior-cleanup", "task", "unknown-dir", "unknown-fixture", "unknown-ack", + "receipt", "other-content", "format", "missing-root", "later-branch", "extra-case", "wrong-harness", "unfinished-extra-branch"): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + root = Path(directory); candidate, folder, before = self.fixture(root) + def edit(path, function): + value = json.loads(path.read_text()); function(value); path.write_text(json.dumps(value)) + if mutation == "candidate": candidate["package_sha256"] = "e" * 64 + elif mutation == "pending": edit(root / "artifact/run" / interrupted.CASE / "cases" / (interrupted.CASE + ".json"), lambda x: x.update(cleanup_status="FAIL")) + elif mutation == "prior-cleanup": edit(next((folder.parents[1] / "H1/checkpoint-truncate").glob("*-cleanup.json")), lambda x: x.update(unitStopped=False)) + elif mutation == "task": (folder / "00098-trigger.json").write_text("{}") + elif mutation == "unknown-dir": (folder / "admin").mkdir() + elif mutation == "unknown-fixture": (folder / "fixture/unknown.json").write_text("{}") + elif mutation == "unknown-ack": edit(folder / "fixture/000010-ack.json", lambda x: x["observation"].update(resultUnknown=True)) + elif mutation == "receipt": edit(next(folder.glob("*-missing-original-shard.json")), lambda x: x.update(relativePath=".rustfs.sys/config/iam")) + elif mutation == "other-content": edit(folder / "00042-actual-missing-shard.json", lambda x: x["vm000"]["files"][0].update(size=19)) + elif mutation == "format": edit(folder / "00042-actual-missing-shard.json", lambda x: x["vm001"].update(formats=[{"foreign": True}] * 4)) + elif mutation == "missing-root": edit(folder / "00024-healthy-physical-baseline.json", lambda x: x.pop("vm003")) + elif mutation == "later-branch": (folder.parent / "journal-write-failure").mkdir() + elif mutation == "extra-case": + extra = root / "artifact/run/other/cases/other.json"; extra.parent.mkdir(parents=True); extra.write_text("{}") + elif mutation == "unfinished-extra-branch": (folder.parents[2] / "DX/H2/other").mkdir(parents=True) + elif mutation == "wrong-harness": edit(folder / "00001-binding.json", lambda x: x.update(harnessCommit="invalid")) + with self.assertRaises((ValueError, KeyError)): + recovery.recovery_plan(root, 12345, candidate) + + +class RestoredArchiveTests(unittest.TestCase): + def test_original_scene_bound_before_first_rename_and_each_partial_move_is_retryable(self): + helper = scene_tests.RetainedSceneTests() + for lost_after in (1, 2, 3, 4, 5): + with self.subTest(lost_after=lost_after), tempfile.TemporaryDirectory() as directory: + state, roots, node, request = helper.scene(directory) + roots[0].mkdir(parents=True) + owner = request["expectedOwner"]; owner["runtimeProfile"] = "root-restart" + for folder in (*roots, state): (folder / "owner.json").write_text(json.dumps(owner)) + snapshot = {"files": [{"path": "original", "sha256": "a" * 64}], "formats": [{}] * 4, + "pid": 0, "quiescent": True, "binarySha256": owner["binarySha256"]} + node["snapshot"] = unittest.mock.Mock(return_value=snapshot) + request["expectedOriginalSceneSha256"] = interrupted.digest({k: snapshot[k] for k in ("files", "formats")}) + with helper.root_stats(), patch.object(retained, "rename_new", side_effect=helper.local_rename): + baseline = retained.run(request, node)["baseline"] + change = {**request, "action": "retained-scene-isolate", "expectedBaseline": baseline} + count = 0 + def interrupted_move(source, target): + nonlocal count + helper.local_rename(source, target) + if target.is_dir(): + count += 1 + if count == lost_after: raise RuntimeError("lost ACK after rename") + with patch.object(retained, "rename_new", side_effect=interrupted_move), self.assertRaises(RuntimeError): + retained.run(change, node) + node["snapshot"].side_effect = AssertionError("must use durable intent after partial archive") + result = retained.run(change, node) + self.assertEqual(result["locations"], ["retained"] * 5) + self.assertEqual(result["baseline"], baseline) + self.assertFalse(result["canonicalPassClaimed"]) + with self.assertRaisesRegex(ValueError, "durable isolation intent"): + retained.run({**request, "expectedOriginalSceneSha256": "b" * 64}, node) + + def test_unrestored_or_missing_scene_cannot_be_archived(self): + helper = scene_tests.RetainedSceneTests() + for mutation in ("digest", "missing-root", "live-pid", "unknown-pid"): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + state, roots, node, request = helper.scene(directory) + if mutation != "missing-root": roots[0].mkdir(parents=True) + request["expectedOwner"]["runtimeProfile"] = "root-restart" + for folder in (*roots, state): + if folder.exists(): (folder / "owner.json").write_text(json.dumps(request["expectedOwner"])) + snapshot = {"files": [], "formats": [], "pid": 0, "quiescent": True, + "binarySha256": request["expectedOwner"]["binarySha256"]} + request["expectedOriginalSceneSha256"] = interrupted.digest({"files": [], "formats": []}) + if mutation == "digest": request["expectedOriginalSceneSha256"] = "b" * 64 + elif mutation == "live-pid": snapshot["pid"] = 7 + elif mutation == "unknown-pid": snapshot["pid"] = False + node["snapshot"] = lambda request: snapshot + with helper.root_stats(), patch.object(retained, "rename_new") as move, self.assertRaises(ValueError): + retained.run(request, node) + move.assert_not_called() + + +class InterruptedFleetTests(unittest.TestCase): + def test_all_stops_precede_restore_and_all_audits_precede_moves(self): + for variant in ("fresh", "retry-restored", "retry-partial-archive", "drift", "lost-root", "archive-drift"): + with self.subTest(variant=variant), tempfile.TemporaryDirectory() as directory: + root = Path(directory); candidate, folder, before = InterruptedPlanTests().fixture(root) + plan = recovery.recovery_plan(root, 12345, candidate)[0] + states = [{"node": n, "state": {"exists": True}, "roots": [{"exists": True}] * 4, + "retained": {"state": False, "roots": [False] * 4}} for n in plan["nodes"]] + current = copy.deepcopy(before if variant == "retry-restored" else plan["failure"]) + if variant == "retry-partial-archive": + states[0]["retained"]["roots"][0] = True + elif variant == "drift": current["vm001"]["formats"] = [{"foreign": True}] * 4 + elif variant == "lost-root": states[2]["roots"] = [{"exists": False}] * 4 + events = [] + def remote(node, action, **params): + events.append((node, action)) + if action == "stop": return {"quiescent": True} + if action == "snapshot": return copy.deepcopy(current[node]) + if action == "recover-fault": + fault = next(f for f in plan["faults"] if f["faultId"] == params["faultId"]) + original = next(entry for entry in before[node]["files"] if entry["path"] == fault["receipt"]["relativePath"]) + if original not in current[node]["files"]: current[node]["files"].append(copy.deepcopy(original)) + current[node]["files"].sort(key=lambda row: (row["disk"], row["path"])) + return {"runId": plan["runId"], "faultId": params["faultId"], "physicalRepairPassAllowed": False} + if action == "retained-scene-audit" and variant == "archive-drift" and node == "vm003": + raise ValueError("archive drift") + baseline = {"originalSceneSha256": params["expectedOriginalSceneSha256"], "node": node} + return {"baseline": baseline, "inspectionOnly": action == "retained-scene-audit", "mutated": False, + "activeNamespaceIsolated": True, "canonicalPassClaimed": False} + fleet = types.SimpleNamespace(remote=remote, record=lambda label, value: value) + if variant in {"drift", "lost-root", "archive-drift"}: + with self.assertRaises(ValueError): + interrupted.restore_and_isolate(fleet, plan, states, recovery.recovery_scene_matches) + self.assertFalse(any(action == "retained-scene-isolate" for _, action in events)) + if variant != "archive-drift": self.assertFalse(any(action == "recover-fault" for _, action in events)) + continue + result = interrupted.restore_and_isolate(fleet, plan, states, recovery.recovery_scene_matches) + self.assertEqual(result["cleanup"], "ISOLATED") + self.assertTrue(result["originalBusinessFixtureRestored"]) + self.assertFalse(result["originalSceneRestored"]) + first_move = next(i for i, (_, action) in enumerate(events) if action == "retained-scene-isolate") + self.assertEqual(sum(action == "retained-scene-audit" for _, action in events[:first_move]), 4) + if variant == "retry-partial-archive": + self.assertFalse(any(n == "vm000" and a in {"stop", "snapshot", "recover-fault"} for n, a in events)) + else: + first_rescue = next(i for i, (_, action) in enumerate(events) if action == "recover-fault") + self.assertEqual(sum(action == "stop" for _, action in events[:first_rescue]), 4) + self.assertEqual(physical_content(current), physical_content(before)) + self.assertFalse(any(a in {"start", "cleanup", "journal-recover"} for _, a in events)) + + +class InterruptedAdapterTests(unittest.TestCase): + def test_retained_remote_has_the_real_snapshot_encryption_dependency(self): + from scripts.validation_heal.healthy import Fleet + def execute(*args, **kwargs): + source = kwargs["input"].rsplit("\nmain(", 1)[0] + scope = {"__name__": "__node_test__"} + exec(compile(source, "restored-archive-node", "exec"), scope) + module = scope["encryption_module"]() + with tempfile.TemporaryDirectory() as state: + self.assertIsNone(module.attest(Path(state).resolve(), {"runtimeProfile": "root-restart"}, scope["checked"], None)) + self.assertTrue(callable(scope["snapshot"])) + return types.SimpleNamespace(returncode=0, stdout="{}", stderr="") + with tempfile.TemporaryDirectory() as directory, patch("scripts.validation_heal.healthy.subprocess.run", side_effect=execute): + Fleet(["vm000"], Path(directory), "gh-12345-1-h2").remote("vm000", "retained-scene-audit", expectedOwner={}) + + +class ArchivedFaultBindingTests(unittest.TestCase): + def test_mismatched_intent_cannot_write_backup_and_exact_intent_is_retryable(self): + from scripts.validation_heal import heal_fault + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve(); (root / "quarantine").mkdir() + bucket = "rv-heal-0123456789abcdef" + relative = bucket + "/objects/0001.bin/version/part.1" + target = root / "data" / relative; target.parent.mkdir(parents=True) + original = b"original-acknowledged-shard" + receipt = {"runId": "gh-12345-1-h2", "faultId": "F07-shard-0", "backup": "F07-shard-0.original", + "relativePath": relative, "kind": "missing", "fault": None, + "before": {"size": len(original), "sha256": hashlib.sha256(original).hexdigest()}} + (root / "quarantine/F07-shard-0.original").write_bytes(original) + (root / "quarantine/F07-shard-0.intent.json").write_bytes(heal_fault.encode(receipt)) + owner = {"runId": receipt["runId"], "bucketPrefix": bucket} + with patch.object(heal_fault, "mount_id", return_value=1), heal_fault.directory(root) as fd: + with self.assertRaisesRegex(ValueError, "archived fault receipt"): + heal_fault.recover_fault(fd, owner, receipt["faultId"], expected_receipt_sha256="0" * 64) + self.assertFalse(target.exists()) + proof = heal_fault.recover_fault(fd, owner, receipt["faultId"], expected_receipt_sha256=interrupted.digest(receipt)) + self.assertTrue(proof["restoredFromBackup"]) + self.assertEqual(target.read_bytes(), original) + proof = heal_fault.recover_fault(fd, owner, receipt["faultId"], expected_receipt_sha256=interrupted.digest(receipt)) + self.assertFalse(proof["restoredFromBackup"]) + self.assertFalse(proof["physicalRepairPassAllowed"]) From a00cf74015534d61389006c6f22db6a736091b07 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 19:30:50 +0800 Subject: [PATCH 206/343] fix(validation): bind F07 replica census to declared topology --- scripts/validation_heal/node_outage.py | 4 ++-- scripts/validation_heal/persistence_mrf.py | 2 +- test_validation_heal.py | 24 ++++++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/scripts/validation_heal/node_outage.py b/scripts/validation_heal/node_outage.py index 22ff0148..d026390f 100644 --- a/scripts/validation_heal/node_outage.py +++ b/scripts/validation_heal/node_outage.py @@ -125,8 +125,8 @@ def workload(recorder, bucket, expected, fixture): return manifest -def replica_census(snapshot, bucket, expected, offline_snapshot): - validate_topology(snapshot, "H2") +def replica_census(snapshot, bucket, expected, offline_snapshot, *, topology="H2"): + validate_topology(snapshot, topology) original = files(offline_snapshot) wanted = {} for entry in expected: diff --git a/scripts/validation_heal/persistence_mrf.py b/scripts/validation_heal/persistence_mrf.py index b6f3f8eb..e4d0a982 100644 --- a/scripts/validation_heal/persistence_mrf.py +++ b/scripts/validation_heal/persistence_mrf.py @@ -261,7 +261,7 @@ def run(args, topology, nodes, fixture, candidate): idle_fleet(fleet, binding) fleet.stop() repaired = fleet.snapshot("complete-rescan-physical-before-business-reads", quiescent=True) - cohorts = mrf.replica_census(repaired, bucket, expected, authorities) + cohorts = mrf.replica_census(repaired, bucket, expected, authorities, topology=topology) result["fullShardIntegrity"] = [verify_cohort(fleet, cohort, "F07-full-shard-proof") for cohort in cohorts] retained_corruption(original, observe(fleet, "retained-corrupt-MRF-after-rescan"), receipts, bucket) fleet.start() diff --git a/test_validation_heal.py b/test_validation_heal.py index 3eb79436..3e9017ab 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -1014,6 +1014,30 @@ def test_physical_census_requires_each_drive_and_preserves_surviving_authority(s else: changed["node4"]["files"][0]["metadata"]["versions"][0]["role"] = "data" with self.subTest(poison=poison), self.assertRaises(ValueError): replica_census(changed, bucket, expected, offline) + def test_explicit_h1_census_keeps_each_drive_version_and_surviving_shard_checks(self): + bucket, expected, h2 = self.snapshot() + h1 = {"node1": copy.deepcopy(h2["node1"])} + identities = [row["value"]["xl"]["this"] for row in h1["node1"]["formats"]] + for row in h1["node1"]["formats"]: row["value"]["xl"]["sets"] = [identities] + for row in h1["node1"]["files"]: + for version in (row.get("metadata") or {}).get("versions", []): + if version["deleted"]: continue + version["role"] = "data" if row["disk"] < 2 else "parity" + version["erasure"].update(data_blocks=2, parity_blocks=2, distribution=[1, 2, 3, 4]) + self.assertEqual(len(replica_census(h1, bucket, expected, h1, topology="H1")), 1) + with self.assertRaisesRegex(ValueError, "wrong node count"): + replica_census(h1, bucket, expected, h1) + with self.assertRaisesRegex(ValueError, "wrong node count"): + replica_census(h2, bucket, expected, h2, topology="H1") + for poison in ("missing-drive", "missing-marker", "changed-survivor", "role-mismatch"): + changed = copy.deepcopy(h1) + if poison == "missing-drive": changed["node1"]["files"] = [row for row in changed["node1"]["files"] if row["disk"] != 3] + elif poison == "missing-marker": changed["node1"]["files"][0]["metadata"]["versions"].pop() + elif poison == "changed-survivor": changed["node1"]["files"][1]["sha256"] = "different" + else: changed["node1"]["files"][4]["metadata"]["versions"][0]["role"] = "data" + with self.subTest(poison=poison), self.assertRaises(ValueError): + replica_census(changed, bucket, expected, h1, topology="H1") + def test_candidate_write_quorum_recognition_is_independent_of_read_contract(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) From dba9f5899bbcef387d1ad8db1ae4aeeea88afc0c Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 19:54:29 +0800 Subject: [PATCH 207/343] test(validation): preserve original opaque metadata probe --- docs/validation-heal-operations.md | 6 ++ scripts/validation_heal/outcome.py | 129 ++++++++++++++++++++++++++++- test_validation_heal_outcome.py | 126 +++++++++++++++++++++++++++- 3 files changed, 258 insertions(+), 3 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 56b5efa3..09a871dc 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -645,3 +645,9 @@ J06 的自动入口为 `heal.performance-abba`。执行器在部署前冻结三 A02 的自动入口为 `heal.admin-scope-isolation`。H2 的专用 `scope-isolation` runtime 使用当前四节点/十六盘，按盘序交织节点并设置 `RUSTFS_ERASURE_SET_DRIVE_COUNT=8`，要求真实两个 set 各跨全部四个节点。普通执行器仍只接受原单 set。两个桶的完整 ACK/实际物理 set 清单冻结后，root/bucket/prefix/object/pool-set 各自新部署、注入选中与控制故障。完整物理和 streaming 证明必须先于 GET 和控制备份恢复。H1 保留原四盘单 set，明确记录不存在未选 set 的适用边界；不能拿同盘目录冒充第二个 set。未限制选择器的原始 receipt 在该固定候选中允许 null，单独保存实际物理映射，显式 pool/set 则必须严格匹配原始字段。 + +### J02 original opaque metadata probe + +The `unknown-negative` diagnostic includes the source procedure before the supplemental legacy-object diagnostic. A separate acknowledged object first receives an independently healthy Deep receipt. With the owned services stopped, the existing descriptor-relative fault backend flips byte zero of every original `xl.meta` copy: four copies in H1 and sixteen in H2. Each mutation has its own original backup, receipt and complete physical delta; the candidate-bound decoder must refuse every exact damaged copy. No other metadata, part or format may change. + +A separate protected prefix is checked during the fault. The original key-level task must retain complete truthful counters and an unresolved disposition, with no invented healthy, repaired, absent, version or incarnation claim. Unknown must be counted within skipped and carry the legacy warning. If the candidate reports another truthful unresolved disposition, record `NOT_OBSERVED`; the legacy diagnostic cannot promote it to PASS. Archive the original terminal result and unchanged complete fault scene before restoring original metadata bytes. Verify the full original physical and SDK inventories and the unchanged original token after restoration. Fixture rescue is never product repair. J02 remains outside canonical dispatch until the separate version-specific grace Deferred/new-token procedure and complete live evidence are available. diff --git a/scripts/validation_heal/outcome.py b/scripts/validation_heal/outcome.py index 666cf78c..3f51888e 100644 --- a/scripts/validation_heal/outcome.py +++ b/scripts/validation_heal/outcome.py @@ -47,7 +47,6 @@ # The reviewed immutable candidate makes one attempt and three bucket-object retries. FAILED_ATTEMPTS = 4 SOURCE_ASSERTION_GAPS = ( - "original-corrupt-metadata-unknown-negative", "version-specific-grace-deferred-and-new-token-compensation", ) @@ -336,6 +335,127 @@ def readonly_compensation(probe, fleet, binding, original, faulted, plan, expect "originalFailureUnchanged": True, "productRepairByNewToken": True} + +def opaque_metadata_outcome(status, bucket, key, options, incarnation): + """An unobserved Unknown must remain unobserved, never successful repair.""" + require(isinstance(incarnation, str) and bool(incarnation) + and status.get("summary") in {"finished", "stopped"} and status.get("settings") == options + and status.get("truncated") is False, "J02 opaque metadata task did not terminate with its exact options") + outcome = status.get("outcome") or {} + counters = outcome.get("counters") or {} + names = ("processed", "healed", "unchanged", "skipped", "failed", "unknown", "attemptFailures") + require(all(type(counters.get(name)) is int and counters[name] >= 0 for name in names) + and counters.get("overflowed") is False and counters["processed"] == 1 + and counters["processed"] == sum(counters[name] for name in ("healed", "unchanged", "skipped", "failed")) + and counters["unknown"] <= counters["skipped"] + and counters["healed"] == counters["unchanged"] == 0, + "J02 opaque metadata counters falsely resolve or omit original responsibility") + objects = outcome.get("objects") + require(outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False + and isinstance(objects, list) and len(objects) == 1, + "J02 opaque metadata responsibility is incomplete") + identity = objects[0].get("identity") or {} + state = (objects[0].get("disposition") or {}).get("state") + require(identity.get("kind") == "object" and identity.get("bucket") == bucket and identity.get("object") == key + and identity.get("versionId") is None and identity.get("bucketIncarnationId") in {None, incarnation} + and type(identity.get("poolIndex")) is int and identity["poolIndex"] == 0 + and type(identity.get("setIndex")) is int and identity["setIndex"] == 0 + and state in {"unknown", "deferred", "failed"}, + "J02 opaque metadata invented a healthy, absent or resolved identity") + require(counters["unknown"] == (1 if state == "unknown" else 0) + and counters["failed"] == (1 if state == "failed" else 0) + and counters["skipped"] == (0 if state == "failed" else 1) + and outcome.get("execution", {}).get("state") == ("completed_with_errors" if state == "failed" else "completed") + and status["summary"] == ("stopped" if state == "failed" else "finished"), + "J02 opaque metadata counters or execution disagree with the original disposition") + require(state != "unknown" or isinstance(status.get("detail"), str) and bool(status["detail"].strip()), + "J02 opaque Unknown lacks the original legacy warning") + return {"verdict": "PASS" if state == "unknown" else "NOT_OBSERVED", "unknownObserved": state == "unknown", + "originalCorruptMetadataSourceProcedure": True, "originalDisposition": state, + "canonicalCounters": counters, "productRepairClaimed": False, "canonicalPassClaimed": False} + + +def original_opaque_probe(probe, fleet, binding, directory, expected, fixture, incarnation, active): + """Corrupt every original metadata copy through the existing file-fault path.""" + bucket, key = binding["bucketPrefix"], "opaque/source.bin" + fleet.start() + body = payload(bucket + "/" + key, SIZE) + response = Recorder(probe, directory / "original-opaque-fixture").call("put_object", { + "Bucket": bucket, "Key": key, "ContentType": "application/octet-stream"}, body) + version = public_version_id(response.get("VersionId")) + require(fixture != "DV" or version != "null", "J02 opaque DV fixture lacks an exact ACK version") + entry = {"key": key, "versionId": version, "isLatest": True, "deleteMarker": False, + "payload": {"sha256": hashlib.sha256(body).hexdigest(), "size": len(body)}} + expected = [*expected, entry] + baseline = fleet.record("original-opaque-ACK-full-S3", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) + fleet.record("original-opaque-complete-ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) + options = {**BASE_OPTIONS, "recursive": False} + anchor = run_task(probe, fleet, {"bucket": bucket, "key": key}, 2, active, option_overrides=options, + status_endpoints=tuple(binding["endpoints"])[1:]) + require(validate_version_receipts(anchor, bucket, [entry], 2)["bucketIncarnationId"] == incarnation, + "J02 opaque anchor changed the original incarnation") + idle_fleet(fleet, binding); fleet.stop() + original = fleet.snapshot("original-opaque-complete-physical", quiescent=True) + target = map_target(original, bucket, key, "data", 1, version, latest_required=False) + verify_proof(fleet, target, "original-opaque-full-cohort-bitrot") + targets = [metadata_target(original, {**target, **member}) for member in target["cohort"]] + require(len(targets) == len(fleet.nodes) * 4 + and len({(row["node"], row["disk"]) for row in targets}) == len(targets), + "J02 opaque corruption must cover every original physical metadata copy") + fleet.record("original-opaque-metadata-fault-plan", targets) + metadata_faults, prior = {}, original + for index, row in enumerate(targets): + params = {"disk": row["disk"], "faultId": bounded_fault_id("J02", "original-opaque", index)} + fleet.faults.append((row["node"], params)) + receipt = fleet.record("original-opaque-byte-zero-bitflip", {"node": row["node"], **fleet.remote( + row["node"], "apply-fault", **params, relativePath=row["relativePath"], expectedSha256=row["sha256"], + kind="bitflip", offset=0)}) + require(receipt.get("kind") == "bitflip" and type(receipt.get("offset")) is int and receipt["offset"] == 0 + and isinstance(receipt.get("fault"), dict), "J02 opaque injection changed its original byte-zero fault") + metadata_faults.setdefault(row["node"], {})[row["relativePath"]] = receipt["fault"] + damaged = fleet.snapshot("original-opaque-physical-injection", quiescent=True, metadata_faults=metadata_faults) + validate_fault(prior, damaged, row, receipt); prior = damaged + damaged_files = files(damaged) + require(all(damaged_files[(row["node"], row["disk"], row["relativePath"])].get("metadataError", {}) + .get("exactWitnessedFault") is True + and damaged_files[(row["node"], row["disk"], row["relativePath"])]["metadataError"].get("validMetadataClaimed") is False + and "metadata" not in damaged_files[(row["node"], row["disk"], row["relativePath"])] for row in targets), + "J02 original opaque fixture retained a readable metadata copy") + fleet.start() + # A separate protected prefix remains observable during the opaque fault. + control = run_task(probe, fleet, {"bucket": bucket, "key": "objects/"}, 2, active, + option_overrides={**options, "recursive": True}, status_endpoints=tuple(binding["endpoints"])[1:]) + require(validate_version_receipts(control, bucket, [item for item in expected if item["key"].startswith("objects/")], 2) + ["bucketIncarnationId"] == incarnation, "J02 opaque fault lost its independent healthy control") + require(physical_content(fleet.snapshot("original-opaque-before-admission", metadata_faults=metadata_faults)) == physical_content(damaged), + "J02 opaque physical fault changed before original admission") + admissions = [] + status = run_task(probe, fleet, {"bucket": bucket, "key": key}, 2, active, option_overrides=options, + status_endpoints=tuple(binding["endpoints"])[1:], admissions=admissions) + fleet.record("original-opaque-terminal-before-rescue", status) + result = opaque_metadata_outcome(status, bucket, key, options, incarnation) + require(len(admissions) == 1, "J02 opaque original admission is ambiguous") + idle_fleet(fleet, binding); fleet.stop() + require(physical_content(fleet.snapshot("original-opaque-after-task-before-rescue", quiescent=True, + metadata_faults=metadata_faults)) == physical_content(damaged), + "J02 opaque original task changed metadata, payload or control bytes") + rescue(fleet) + require(physical_content(fleet.snapshot("original-opaque-restored-original-bytes", quiescent=True)) == physical_content(original), + "J02 opaque rescue did not restore the exact original metadata generation") + fleet.start() + for endpoint in binding["endpoints"]: + observed = fleet.record("original-opaque-restored-full-S3", Probe(binding, endpoint).inventory( + bucket, unversioned_current_tags=fixture == "DB")) + verify_seed(expected, observed); compare_inventories(baseline, observed) + unchanged_token(probe, fleet, admissions[0], status, "original-opaque-token-after-fixture-rescue") + idle_fleet(fleet, binding); fleet.stop() + require(physical_content(fleet.snapshot("original-opaque-after-restored-business-reads", quiescent=True)) == physical_content(original), + "J02 opaque business reads changed the restored generation") + result.update(originalMetadataCopiesCorrupted=len(targets), physicalFaultRetainedBeforeRescue=True, + originalBytesRestored=True, originalTokenUnchanged=True, fixtureRescueIsProductRepair=False) + return expected, baseline, result + + def run_branch(args, topology, nodes, fixture, branch, candidate): directory = args.evidence / fixture / topology / branch directory.mkdir(parents=True) @@ -372,6 +492,10 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): idle_fleet(fleet, binding) fleet.stop() if branch == "unknown-negative": + pending = {"operation": "original-corrupt-metadata-source-probe", "bucket": bucket} + expected, baseline, result["originalCorruptMetadataProbe"] = original_opaque_probe( + probe, fleet, binding, directory, expected, fixture, incarnation, active) + pending = None fleet.snapshot("protected-control-before-legacy-writing", quiescent=True) pending = {"operation": "append-real-legacy-fixture", "bucket": bucket} expected = append_legacy_fixture(probe, fleet, directory, binding, expected, fixture) @@ -456,7 +580,8 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): and repeated["response"].get("truncated") is False and canonical(repeated["response"].get("outcome")) == canonical(status["outcome"]), "J02 rescue overwrote the original failed/deferred/dry-run responsibility") - result.update(verdict="PASS", recoveryIsProductRepair=False) + result.update(verdict="NOT_OBSERVED" if branch == "unknown-negative" + and not result["originalCorruptMetadataProbe"]["unknownObserved"] else "PASS", recoveryIsProductRepair=False) except (Exception, KeyboardInterrupt) as exc: result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) if fleet.reserved: diff --git a/test_validation_heal_outcome.py b/test_validation_heal_outcome.py index 8a52cbe5..00c89e40 100644 --- a/test_validation_heal_outcome.py +++ b/test_validation_heal_outcome.py @@ -215,7 +215,7 @@ def branch(args, topology, nodes, fixture, name, candidate): self.assertFalse(summary["complete"]) self.assertFalse(summary["canonicalCasePassClaimed"]) self.assertEqual(set(summary["sourceAssertionGaps"]), { - "original-corrupt-metadata-unknown-negative", "version-specific-grace-deferred-and-new-token-compensation"}) + "version-specific-grace-deferred-and-new-token-compensation"}) def test_dirty_cleanup_stops_before_next_fixture_or_topology(self): with tempfile.TemporaryDirectory() as tmp: @@ -544,4 +544,128 @@ def task(probe,fleet,scope,mode,active,**kwargs): self.assertEqual(len(queries),2) + +class OriginalOpaqueMetadataTests(unittest.TestCase): + bucket = 'rv-heal-0123456789abcdef' + key = 'opaque/source.bin' + options = {**outcome.BASE_OPTIONS, 'recursive': False} + + def status(self, state='unknown'): + failed = state == 'failed' + return {'summary': 'stopped' if failed else 'finished', 'settings': self.options, 'truncated': False, + 'detail': 'legacy incomplete outcome; unknown is not proof of repair' if state == 'unknown' else None, + 'outcome': {'execution': {'state': 'completed_with_errors' if failed else 'completed'}, + 'coverage': 'complete', 'objectsTruncated': False, + 'counters': {'processed': 1, 'healed': 0, 'unchanged': 0, 'skipped': 0 if failed else 1, + 'failed': int(failed), 'unknown': int(state == 'unknown'), 'attemptFailures': int(state != 'unknown'), 'overflowed': False}, + 'objects': [{'identity': {'kind': 'object', 'bucket': self.bucket, 'object': self.key, + 'versionId': None, 'bucketIncarnationId': None, 'poolIndex': 0, 'setIndex': 0}, + 'disposition': {'state': state}}]}} + + def test_original_metadata_unknown_observation_and_nonobservation_are_distinct(self): + for state in ('unknown', 'deferred', 'failed'): + result = outcome.opaque_metadata_outcome(self.status(state), self.bucket, self.key, self.options, "original-anchor") + self.assertEqual(result['verdict'], 'PASS' if state == 'unknown' else 'NOT_OBSERVED') + self.assertEqual(result['unknownObserved'], state == 'unknown') + self.assertFalse(result['canonicalPassClaimed']) + self.assertFalse(result['productRepairClaimed']) + + def test_known_original_anchor_is_allowed_without_inventing_a_new_incarnation(self): + status = self.status() + status["outcome"]["objects"][0]["identity"]["bucketIncarnationId"] = "original-anchor" + self.assertTrue(outcome.opaque_metadata_outcome(status, self.bucket, self.key, self.options, "original-anchor")["unknownObserved"]) + + def test_false_repair_unknown_counters_or_identity_are_rejected(self): + for poison in ('healthy', 'absent', 'repaired', 'unknown-partition', 'boolean-counter', 'overflow', 'omitted', + 'wrong-key', 'invented-version', 'invented-incarnation', 'boolean-pool', 'warning', 'running', 'partial', 'execution'): + with self.subTest(poison=poison): + value = self.status(); body = value['outcome']; obj = body['objects'][0] + if poison in ('healthy', 'absent', 'repaired'): + obj['disposition']['state'] = {'healthy': 'verified_healthy', 'absent': 'authoritatively_absent', 'repaired': 'repaired'}[poison] + elif poison == 'unknown-partition': body['counters']['skipped'] = 0 + elif poison == 'boolean-counter': body['counters']['processed'] = True + elif poison == 'overflow': body['counters']['overflowed'] = True + elif poison == 'omitted': body['objects'] = [] + elif poison == 'wrong-key': obj['identity']['object'] = 'other' + elif poison == 'invented-version': obj['identity']['versionId'] = 'new-version' + elif poison == 'invented-incarnation': obj['identity']['bucketIncarnationId'] = 'fabricated' + elif poison == 'boolean-pool': obj['identity']['poolIndex'] = False + elif poison == 'warning': value['detail'] = None + elif poison == 'running': value['summary'] = 'running' + elif poison == 'partial': body['coverage'] = 'partial' + elif poison == 'execution': body['execution']['state'] = 'completed_with_errors' + with self.assertRaises(ValueError): + outcome.opaque_metadata_outcome(value, self.bucket, self.key, self.options, "original-anchor") + + def test_original_probe_mutates_all_four_metadata_copies_and_restores_before_reads(self): + original, bucket, entries = versioned_scene([self.key, 'objects/control.bin']) + expected = [entry for entry in entries if entry['key'] == 'objects/control.bin'] + current = copy.deepcopy(original) + events, registrations = [], {} + active = [] + status = self.status() + fleet = types.SimpleNamespace(nodes=['node1'], faults=[]) + fleet.start = lambda: events.append('start') + fleet.stop = lambda: events.append('stop') + def record(label, value): + events.append(label); return value + fleet.record = record + def snapshot(label, **params): + events.append(label) + if any('metadataError' in row for row in current['node1']['files']): + self.assertTrue(params.get('metadata_faults')) + return copy.deepcopy(current) + fleet.snapshot = snapshot + def remote(node, action, **params): + events.append(action) + if action == 'apply-fault': + self.assertEqual(fleet.faults[-1], (node, {'disk': params['disk'], 'faultId': params['faultId']})) + self.assertEqual((params['kind'], params['offset']), ('bitflip', 0)) + self.assertTrue(params['relativePath'].endswith('/opaque/source.bin/xl.meta')) + row = next(row for row in current[node]['files'] if row['disk'] == params['disk'] and row['path'] == params['relativePath']) + registrations[params['faultId']] = copy.deepcopy(row) + before = {key: row[key] for key in ('sha256', 'size')} + row['sha256'] = 'corrupt-' + str(row['disk']) + row.pop('metadata') + row['metadataError'] = {'exactWitnessedFault': True, 'validMetadataClaimed': False} + return {'kind': 'bitflip', 'offset': 0, 'relativePath': row['path'], 'before': before, + 'fault': {key: row[key] for key in ('sha256', 'size')}} + self.assertEqual(action, 'recover-fault') + saved = registrations[params['faultId']] + row = next(row for row in current[node]['files'] if row['disk'] == saved['disk'] and row['path'] == saved['path']) + row.clear(); row.update(copy.deepcopy(saved)) + return {'physicalRepairPassAllowed': False} + fleet.remote = remote + def inventory(*args, **kwargs): + events.append('business-GET-inventory') + self.assertEqual(physical_content(current), physical_content(original)) + return {'entries': []} + probe = types.SimpleNamespace(inventory=inventory, admin=lambda *args: {'httpStatus': 200, 'response': status}) + def task(probe, fleet, scope, mode, active, **params): + events.append('task:' + scope['key']) + if 'admissions' in params: + params['admissions'].append({'path': '/heal/opaque', 'clientToken': 'original-opaque-token'}) + return status + return {} + binding = {'bucketPrefix': bucket, 'endpoints': {'node1': 'http://node1'}} + with tempfile.TemporaryDirectory() as directory, patch.object(outcome, 'Recorder') as recorder, \ + patch.object(outcome, 'Probe', return_value=probe), patch.object(outcome, 'run_task', side_effect=task), \ + patch.object(outcome, 'validate_version_receipts', return_value={'bucketIncarnationId': 'incarnation'}), \ + patch.object(outcome, 'verify_seed', return_value={}), patch.object(outcome, 'compare_inventories'), \ + patch.object(outcome, 'verify_proof'), patch.object(outcome, 'payload', return_value=b'opaque-fixture'), \ + patch.object(outcome, 'idle_fleet'): + recorder.return_value.call.return_value = {'VersionId': 'v2'} + updated, baseline, result = outcome.original_opaque_probe( + probe, fleet, binding, Path(directory), expected, 'DV', 'incarnation', active) + self.assertEqual(result['originalMetadataCopiesCorrupted'], 4) + self.assertTrue(result['unknownObserved']) + self.assertTrue(result['originalTokenUnchanged']) + self.assertEqual(len(updated), len(expected) + 1) + self.assertEqual(fleet.faults, []) + self.assertEqual(events.count('apply-fault'), 4) + self.assertEqual(events.count('recover-fault'), 4) + self.assertLess(events.index('original-opaque-terminal-before-rescue'), events.index('recover-fault')) + self.assertLess(events.index('original-opaque-restored-original-bytes'), events.index('original-opaque-restored-full-S3')) + + if __name__ == "__main__": unittest.main() From b293423acf6ed91260d11b633c93c44da9e6c6f4 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 20:03:12 +0800 Subject: [PATCH 208/343] fix(validation): check exact deferred retry wire field --- scripts/validation_heal/outcome.py | 3 +-- test_validation_heal_outcome.py | 10 +++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/validation_heal/outcome.py b/scripts/validation_heal/outcome.py index 3f51888e..02f3f5e2 100644 --- a/scripts/validation_heal/outcome.py +++ b/scripts/validation_heal/outcome.py @@ -93,8 +93,7 @@ def reconcile(status, bucket, expected, options, incarnation, *, attempt_failure # This is one key-level refusal, never a version repair receipt. require(expected_entry.get("keyLevelRefusal") is True and pair[1] == "null" and identity.get("versionId") is None and identity.get("bucketIncarnationId") is None - and disposition.get("details", {}).get("reason") == "transient_existence_check" - and disposition["details"].get("retry_not_before") is None, + and disposition.get("details") == {"reason": "transient_existence_check", "retryNotBefore": None}, "J02 deferred dependency claimed a version/incarnation or changed its typed reason") else: require(isinstance(incarnation, str) and incarnation diff --git a/test_validation_heal_outcome.py b/test_validation_heal_outcome.py index 00c89e40..3513c223 100644 --- a/test_validation_heal_outcome.py +++ b/test_validation_heal_outcome.py @@ -108,7 +108,7 @@ def test_key_level_deferral_cannot_invent_an_incarnation_or_version(self): "failed": 0, "unknown": 0, "attemptFailures": 1, "overflowed": False}, "objects": [{"identity": {"kind": "object", "bucket": self.bucket, "object": expected[0]["key"], "versionId": None, "bucketIncarnationId": None, "poolIndex": 0, "setIndex": 0}, "disposition": { - "state": "deferred", "details": {"reason": "transient_existence_check", "retry_not_before": None}}}]}} + "state": "deferred", "details": {"reason": "transient_existence_check", "retryNotBefore": None}}}]}} self.check(expected, status, options, 1) for field, value in (("bucketIncarnationId", self.incarnation), ("versionId", "null"), ("versionId", "v0")): changed = copy.deepcopy(status); changed["outcome"]["objects"][0]["identity"][field] = value @@ -116,6 +116,14 @@ def test_key_level_deferral_cannot_invent_an_incarnation_or_version(self): changed = copy.deepcopy(status) changed["outcome"]["objects"][0]["disposition"]["details"]["reason"] = "dangling_delete_grace" with self.assertRaises(ValueError): self.check(expected, changed, options, 1) + for details in ({"reason": "transient_existence_check"}, + {"reason": "transient_existence_check", "retry_not_before": None}, + {"reason": "transient_existence_check", "retryNotBefore": {"secs_since_epoch": 123, "nanos_since_epoch": 0}}, + {"reason": "transient_existence_check", "retryNotBefore": None, "retry_not_before": None}): + changed = copy.deepcopy(status) + changed["outcome"]["objects"][0]["disposition"]["details"] = details + with self.subTest(details=details), self.assertRaises(ValueError): + self.check(expected, changed, options, 1) def test_planner_uses_actual_erasure_geometry_and_preserves_unselected_versions(self): scene, bucket, expected = versioned_scene([f"objects/{index:04d}.bin" for index in range(3)]) From 8a1a00202abb90a94449e3c004fac02ba4110029 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 21:47:38 +0800 Subject: [PATCH 209/343] feat(validation): preserve original version grace outcome procedure --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-heal-cases.md | 4 +- scripts/run_validation_case.sh | 2 +- scripts/validation_heal/execution_policy.py | 2 +- scripts/validation_heal/fleet_node.py | 25 +- scripts/validation_heal/healthy.py | 2 +- scripts/validation_heal/orphan_fault.py | 2 +- scripts/validation_heal/outcome.py | 21 +- scripts/validation_heal/outcome_grace.py | 282 ++++++++++++ scripts/validation_heal/recipes.json | 6 +- scripts/validation_heal/run_case.sh | 7 + test_validation_heal_outcome.py | 5 +- test_validation_heal_outcome_grace.py | 415 ++++++++++++++++++ 13 files changed, 756 insertions(+), 21 deletions(-) create mode 100644 scripts/validation_heal/outcome_grace.py create mode 100644 test_validation_heal_outcome_grace.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 85b143e4..ae4befb8 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -47,6 +47,7 @@ on: - test_validation_heal_race.py - test_validation_heal_environment_gate.py - test_validation_heal_outcome.py + - test_validation_heal_outcome_grace.py - test_validation_heal_persistence.py - test_validation_heal_persistence_mrf.py - test_validation_heal_incarnation.py @@ -111,6 +112,7 @@ on: - test_validation_heal_race.py - test_validation_heal_environment_gate.py - test_validation_heal_outcome.py + - test_validation_heal_outcome_grace.py - test_validation_heal_persistence.py - test_validation_heal_persistence_mrf.py - test_validation_heal_incarnation.py @@ -154,7 +156,7 @@ jobs: fi - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - run: python3 -B scripts/validation_heal/matrix.py check - - run: python3 -B -m unittest -v test_validation_heal_outcome.py + - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py - run: python3 -B -m unittest -v test_validation_heal_incarnation.py test_validation_heal_incarnation_replay.py - run: python3 -B -m unittest -v test_validation_heal_incarnation_native.py diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index b0819e9e..63f6808d 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1808,12 +1808,14 @@ Topologies: H1,H2. Branches: mixed-healthy-repaired-failed, dry-run, deferred, r Execution policy: `evidence-blocked`; disabled by default: `false`. -Blocked reason (`EVIDENCE_MIXED_OUTCOME_ORCHESTRATION`): 已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral、真实 SDK legacy Unknown 和只读永久失败/新 token 补偿执行分支及逐对象 manifest/attemptFailures 对账；仍需完整串行 live 证据及原始损坏元数据 Unknown、grace Deferred/新 token 补偿源流程，故保持禁止 dispatch；本地检查不能作为完整 PASS。 +Blocked reason (`EVIDENCE_MIXED_OUTCOME_ORCHESTRATION`): 已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral、真实 SDK legacy Unknown、原始全副本损坏元数据 Unknown、只读永久失败/新 token 补偿及版本级 grace Deferred/新 token 清理执行流程，均按逐对象 manifest/attemptFailures 对账；仍需完整串行 live 证据及独立验收，故保持禁止默认 dispatch 和 canonical admission；本地检查不能作为完整 PASS。 1. 同批独立key建立健康、可恢复缺片、低于R的不可恢复对象；deferred由冻结的锁/依赖故障产生，dry-run另建任务。 2. 保存逐对象物理基线和期望disposition，对结果的execution/coverage/counters/object identities做逐项对账。 3. unknown/skipped/completed_with_errors不算修复成功，legacy objectsHealed与canonical healed分开解释；attemptFailures不得被隐藏。 4. 使用独立完整manifest弥补有界outcome窗口，保持不可修对象的正确失败记录；恢复后另任务修复不覆盖原结果。 +5. grace 分支先用 SDK PUT 建立两个真实版本并验证完整分片；保存一个成员的原对象树后，SDK DELETE 精确历史版本，逐盘验证仅当前版本存在且其逻辑元数据、数据片及控制对象未变。只向原成员回放原对象树；实际候选进程固定 grace=120 秒，不修改元数据年龄或时钟。原任务须在窗口内完成，精确报告旧版 deferred、当前版 verified_healthy 及 retryNotBefore；自然等待过期后原 token 仍不变，新 token 才能清理旧版。清理后先核对全部物理字节等于 DELETE 后权威现场，再跨节点 GET/404 和复查旧 token，最后才做 fixture finalization。错过窗口保留 NOT_OBSERVED；异常保留现场并标 DIRTY；救援仅恢复 DELETE 后权威树，不复活已删除版本。 +6. Manual diagnostic entry: explicitly select heal.outcome-counters-and-error-truth through Functional Chain with the frozen candidate and candidate-bound manager/inspector tools. The default catalog policy stays evidence-blocked; even all passing diagnostic branches retain complete=false, canonicalCasePassClaimed=false and a blocking gate verdict until independent acceptance. Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [LOCK_WINDOW](validation-heal-operations.md#LOCK_WINDOW) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index cbd367ff..980c54f7 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.checkpoint-and-mrf-corruption|heal.bucket-incarnation-recreation|heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-markers-and-batch-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update|heal.mixed-version-upgrade-rollback|heal.dedicated-host-forced-restart|heal.race-ilm-expiry-and-transition|heal.race-replication-resync|heal.with-expansion-rebalance|heal.with-rebalance-stop-resume|heal.with-decommission|heal.with-decommission-cancel-clear|heal.after-retired-pool-removal|heal.ec84-multi-set-restart|heal.slow-io-and-eio|heal.kms-and-tier-outage|heal.long-soak|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) + heal.outcome-counters-and-error-truth|heal.checkpoint-and-mrf-corruption|heal.bucket-incarnation-recreation|heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-markers-and-batch-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update|heal.mixed-version-upgrade-rollback|heal.dedicated-host-forced-restart|heal.race-ilm-expiry-and-transition|heal.race-replication-resync|heal.with-expansion-rebalance|heal.with-rebalance-stop-resume|heal.with-decommission|heal.with-decommission-cancel-clear|heal.after-retired-pool-removal|heal.ec84-multi-set-restart|heal.slow-io-and-eio|heal.kms-and-tier-outage|heal.long-soak|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py index b8855a1a..0eb4e367 100644 --- a/scripts/validation_heal/execution_policy.py +++ b/scripts/validation_heal/execution_policy.py @@ -144,7 +144,7 @@ def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, . ), "heal.outcome-counters-and-error-truth": _native( "J02", "EVIDENCE_MIXED_OUTCOME_ORCHESTRATION", - "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral、真实 SDK legacy Unknown 和只读永久失败/新 token 补偿执行分支及逐对象 manifest/attemptFailures 对账；仍需完整串行 live 证据及原始损坏元数据 Unknown、grace Deferred/新 token 补偿源流程，故保持禁止 dispatch；本地检查不能作为完整 PASS。", + "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral、真实 SDK legacy Unknown、原始全副本损坏元数据 Unknown、只读永久失败/新 token 补偿及版本级 grace Deferred/新 token 清理执行流程，均按逐对象 manifest/attemptFailures 对账；仍需完整串行 live 证据及独立验收，故保持禁止默认 dispatch 和 canonical admission；本地检查不能作为完整 PASS。", ("mixed_outcome_fixture", "per_object_canonical_receipts", "complete_live_orchestrator", "attempt_failure_visibility"), ), } diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 92b6120f..56b0eecd 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -40,7 +40,7 @@ def runtime_config(profile): "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false", "RUSTFS_HEAL_MAX_CONCURRENT_HEALS": "1", "RUSTFS_HEAL_MAX_CONCURRENT_PER_SET": "1", "RUSTFS_OBS_LOGGER_LEVEL": logging_filter, "RUST_LOG": logging_filter} - require(profile in {"outcome-legacy", "queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") + require(profile in {"outcome-legacy", "outcome-grace", "queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "replacement-target", "schema-refusal", "scope-isolation"}, "unsupported Heal runtime profile") if profile == "queue-pressure": logging_filter = "info,rustfs::heal::manager=debug,rustfs::heal::task=debug,rustfs::heal::mrf=debug" return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true", @@ -49,9 +49,9 @@ def runtime_config(profile): "RUSTFS_HEAL_MAX_CONCURRENT_PER_SET": "1", "RUSTFS_HEAL_PAGE_PARALLEL_ENABLE": "false", "RUSTFS_HEAL_OVERLAP_POLICY": "merge", "RUSTFS_HEAL_LOW_PRIORITY_MERGE_ENABLE": "true", "RUSTFS_HEAL_LOW_PRIORITY_DROP_WHEN_FULL": "true", "RUSTFS_HEAL_MRF_REPLAY_BATCH": "1"} - if profile in {"orphan-preserve", "orphan-before", "orphan-after"}: + if profile in {"orphan-preserve", "orphan-before", "orphan-after", "outcome-grace"}: return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS": - {"orphan-preserve": "0", "orphan-before": "300", "orphan-after": "30"}[profile]} + {"orphan-preserve": "0", "orphan-before": "300", "orphan-after": "30", "outcome-grace": "120"}[profile]} if profile == "authority-split": return {**BACKGROUND_CONFIG, "RUSTFS_STORAGE_CLASS_STANDARD": "EC:1"} if profile == "multi-pool": @@ -617,6 +617,19 @@ def process_metrics(request): return result +def process_clock(request): + require(set(request) == {"runId", "action"}, "process clock accepts no PID, path or clock override") + require(owned(request["runId"])[2].get("runtimeProfile") == "outcome-grace", "process clock requires J02 grace runtime") + before = process(request) + require(type(before.get("pid")) is int and before["pid"] > 1 + and before["settings"].get("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS") == "120", + "J02 grace process is absent or has another policy") + epoch = time.time() + after = process(request) + require(all(after.get(name) == before[name] for name in PROCESS_FIELDS), "clock witness crossed a process identity change") + return {"identity": before, "nodeEpoch": epoch, "readOnly": True, "clockChanged": False} + + def process_signal(request): name = request.get("signal") require(name in {"SIGSTOP", "SIGCONT"}, "only owned-process pause/resume signals are allowed") @@ -1543,10 +1556,13 @@ def generation_operation(request): def orphan_operation(request): state, roots, owner = owned(request['runId']) stopped(owner) - require(owner.get('runtimeProfile') in {'orphan-preserve', 'orphan-before', 'orphan-after'} + require(owner.get('runtimeProfile') in {'orphan-preserve', 'orphan-before', 'orphan-after', 'outcome-grace'} and digest(state / 'rustfs') == owner['binarySha256'], 'C10 runtime/candidate changed') fault, generation, backend = fault_module(), globals()['generation_backend'], globals()['orphan_backend'] action = request['action'] + require(owner.get('runtimeProfile') != 'outcome-grace' or action in { + 'orphan-observe', 'orphan-capture', 'orphan-arm', 'orphan-install', 'orphan-recover'}, + 'J02 grace permits only stopped original-object archive/replay/recovery') tool = state / 'heal-shared-fixture' if action == 'orphan-install-tool': require(re.fullmatch(r'[0-9a-f]{64}', request['sha256']), 'C10 encoder digest absent') @@ -1614,6 +1630,7 @@ def main(request): for action in ("incarnation-census", "incarnation-capture", "incarnation-install", "incarnation-finalize", "incarnation-phase"): actions[action] = incarnation_operation actions["task-records"] = task_records + actions["process-clock"] = process_clock actions["environment-capabilities"] = environment_capabilities for action in ("retained-scene-audit", "retained-scene-isolate"): actions[action] = retained_scene_operation diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index eb20fabe..829feb9c 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"outcome-legacy", "queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation", "incarnation-checkpoint", "incarnation-mrf"}, "unsupported Heal runtime profile") + require(runtime_profile in {"outcome-legacy", "outcome-grace", "queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation", "incarnation-checkpoint", "incarnation-mrf"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory diff --git a/scripts/validation_heal/orphan_fault.py b/scripts/validation_heal/orphan_fault.py index bf203099..da6d02a4 100644 --- a/scripts/validation_heal/orphan_fault.py +++ b/scripts/validation_heal/orphan_fault.py @@ -14,7 +14,7 @@ def require(value, message): def scope(owner, request): - require(owner.get('runtimeProfile') in {'orphan-preserve', 'orphan-before', 'orphan-after'} + require(owner.get('runtimeProfile') in {'orphan-preserve', 'orphan-before', 'orphan-after', 'outcome-grace'} and request.get('bucket') == owner['bucket'] and request.get('key') == 'orphan.bin', 'C10 escaped its registered runtime or exact owned object') require(type(request.get('disk')) is int and 0 <= request['disk'] < 4, 'C10 original disk absent') diff --git a/scripts/validation_heal/outcome.py b/scripts/validation_heal/outcome.py index 02f3f5e2..8a370128 100644 --- a/scripts/validation_heal/outcome.py +++ b/scripts/validation_heal/outcome.py @@ -22,6 +22,7 @@ from .manager_fixture import verify as verify_manager from .matrix import select from .native_regressions import CANDIDATE + from .outcome_grace import run as original_grace_probe from .shards import files, map_target, run_task, validate_fault, validate_repaired_physical, validate_repaired_status from .stop import idle_fleet from .versions import (metadata_target, public_version_id, validate_unrecoverable_get, @@ -35,6 +36,7 @@ from manager_fixture import verify as verify_manager from matrix import select from native_regressions import CANDIDATE + from outcome_grace import run as original_grace_probe from shards import files, map_target, run_task, validate_fault, validate_repaired_physical, validate_repaired_status from stop import idle_fleet from versions import (metadata_target, public_version_id, validate_unrecoverable_get, @@ -46,9 +48,7 @@ SIZE = 5 * 1024**2 + 123 # The reviewed immutable candidate makes one attempt and three bucket-object retries. FAILED_ATTEMPTS = 4 -SOURCE_ASSERTION_GAPS = ( - "version-specific-grace-deferred-and-new-token-compensation", -) +SOURCE_ASSERTION_GAPS = () def reconcile(status, bucket, expected, options, incarnation, *, attempt_failures, legacy_unknown=()): @@ -460,7 +460,7 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): directory.mkdir(parents=True) run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" fleet = Fleet(nodes, directory, run_id, args.tool_directory, - runtime_profile="outcome-legacy" if branch == "unknown-negative" else "manual") + runtime_profile="outcome-legacy" if branch == "unknown-negative" else "outcome-grace" if branch == "deferred" else "manual") bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] fleet.snapshot_buckets = [bucket] access, secret = secrets.token_hex(16), secrets.token_hex(24) @@ -503,6 +503,11 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): pending = None idle_fleet(fleet, binding) fleet.stop() + if branch == "deferred": + pending = {"operation": "original-version-specific-grace-source-procedure", "bucket": bucket} + expected, baseline, result["originalGraceProcedure"] = original_grace_probe( + probe, fleet, binding, directory, expected, incarnation, active, unchanged_token) + pending = None original = fleet.snapshot("healthy-physical-baseline", quiescent=True) plan = legacy_manifest(original, bucket, expected) if branch == "unknown-negative" else fault_plan(original, bucket, expected, branch) fleet.record("independent-manifest-and-frozen-fault-plan", plan) @@ -579,8 +584,9 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): and repeated["response"].get("truncated") is False and canonical(repeated["response"].get("outcome")) == canonical(status["outcome"]), "J02 rescue overwrote the original failed/deferred/dry-run responsibility") - result.update(verdict="NOT_OBSERVED" if branch == "unknown-negative" - and not result["originalCorruptMetadataProbe"]["unknownObserved"] else "PASS", recoveryIsProductRepair=False) + unobserved = any(result.get(name, {}).get("verdict") == "NOT_OBSERVED" + for name in ("originalCorruptMetadataProbe", "originalGraceProcedure")) + result.update(verdict="NOT_OBSERVED" if unobserved else "PASS", recoveryIsProductRepair=False) except (Exception, KeyboardInterrupt) as exc: result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) if fleet.reserved: @@ -638,7 +644,8 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"J02 interrupted by sig save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL", "diagnosticBranchesPassed": complete and all(row["verdict"] == row["cleanup"] == "PASS" for row in results), - "complete": False, "canonicalCasePassClaimed": False, "sourceAssertionGaps": list(SOURCE_ASSERTION_GAPS)}) + "complete": False, "canonicalCasePassClaimed": False, "sourceAssertionGaps": list(SOURCE_ASSERTION_GAPS), + "canonicalAdmissionEnabled": False, "liveSourceProcedureAcceptancePending": True}) return 1 diff --git a/scripts/validation_heal/outcome_grace.py b/scripts/validation_heal/outcome_grace.py new file mode 100644 index 00000000..d105f10f --- /dev/null +++ b/scripts/validation_heal/outcome_grace.py @@ -0,0 +1,282 @@ +"""J02: preserve exact version grace responsibility through new-token cleanup.""" + +from datetime import datetime +import hashlib +import math +import time +import uuid + +try: + from . import orphan + from .authority import object_files + from .fleet_node import PROCESS_FIELDS + from .heal_probe import require + from .heal_workload import Recorder, payload, verify_seed + from .healthy import BASE_OPTIONS, physical_content + from .shards import files, map_target, run_task + from .stop import idle_fleet + from .versions import validate_version_receipts, verify_proof +except ImportError: + import orphan + from authority import object_files + from fleet_node import PROCESS_FIELDS + from heal_probe import require + from heal_workload import Recorder, payload, verify_seed + from healthy import BASE_OPTIONS, physical_content + from shards import files, map_target, run_task + from stop import idle_fleet + from versions import validate_version_receipts, verify_proof + + +GRACE_SECONDS = 120 +KEY = orphan.KEY +SIZE = 5 * 1024**2 + 123 + + +def finite_epoch(value): + return type(value) in (int, float) and math.isfinite(value) and value >= 0 + + +def grace_outcome(status, bucket, old, current, options, incarnation, observed_at, *, cleanup=False): + require(finite_epoch(observed_at) and finite_epoch(old.get('lastModifiedEpoch')) + and old['versionId'] != current['versionId'] and isinstance(incarnation, str) and incarnation, + 'J02 grace lacks distinct ACK versions, original incarnation or a real observation clock') + outcome = status.get('outcome') or {} + require(status.get('summary') == 'finished' and status.get('settings') == options and status.get('truncated') is False + and outcome.get('execution', {}).get('state') == 'completed' and outcome.get('coverage') == 'complete' + and outcome.get('objectsTruncated') is False, 'J02 grace task lacks complete original terminal truth') + wanted = dict(processed=2, healed=1 if cleanup else 0, unchanged=1, skipped=0 if cleanup else 1, + failed=0, unknown=0, attemptFailures=0 if cleanup else 1, overflowed=False) + counters = outcome.get('counters') or {} + require(set(counters) == set(wanted) and all(type(counters[name]) is type(value) and counters[name] == value + for name, value in wanted.items()), 'J02 grace counters do not match both original versions') + objects = outcome.get('objects') + require(isinstance(objects, list) and len(objects) == 2, 'J02 grace omitted a version responsibility') + seen, due = set(), None + for item in objects: + identity = item.get('identity') or {} + version = identity.get('versionId') + require(version in {old['versionId'], current['versionId']} and version not in seen + and identity == dict(kind='object', bucket=bucket, object=KEY, versionId=version, + bucketIncarnationId=incarnation, poolIndex=0, setIndex=0) + and type(identity.get('poolIndex')) is int and type(identity.get('setIndex')) is int, + 'J02 grace rebound, duplicated or omitted an original version/incarnation') + seen.add(version) + disposition = item.get('disposition') or {} + if version == current['versionId']: + require(disposition == {'state': 'verified_healthy'}, 'J02 grace current ACK lost its Deep integrity') + elif cleanup: + require(disposition == {'state': 'repaired'}, 'J02 post-grace cleanup is not a distinct product repair') + else: + details = disposition.get('details') or {} + timestamp = details.get('retryNotBefore') + require(disposition.get('state') == 'deferred' and set(details) == {'reason', 'retryNotBefore'} + and details['reason'] == 'dangling_delete_grace' and isinstance(timestamp, dict) + and set(timestamp) == {'secs_since_epoch', 'nanos_since_epoch'} + and type(timestamp.get('secs_since_epoch')) is int and 0 <= timestamp['secs_since_epoch'] < 2**64 + and type(timestamp.get('nanos_since_epoch')) is int and 0 <= timestamp['nanos_since_epoch'] < 10**9, + 'J02 grace lacks the exact typed future retry timestamp') + due = timestamp['secs_since_epoch'] + timestamp['nanos_since_epoch'] / 10**9 + require(due > observed_at and abs(due - old['lastModifiedEpoch'] - GRACE_SECONDS) <= 5, + 'J02 grace retry timestamp is past or unrelated to the original server version age') + require(not cleanup or observed_at >= old['lastModifiedEpoch'] + GRACE_SECONDS, + 'J02 cleanup was attributed before natural grace expiry') + return {'bothExactVersionResponsibilities': True, 'canonicalCounters': counters, 'retryNotBeforeEpoch': due, + 'newTaskProductCleanup': cleanup, 'sameTokenRecoveryClaimed': False} + + +def grace_fixture(initial, authority, injected, bucket, old, current, target, directory): + initial_refs = orphan.references(initial, bucket, KEY) + authority_refs = orphan.references(authority, bucket, KEY) + require(set(initial_refs) == set(authority_refs) and target in initial_refs + and all(set(value) == {old['versionId'], current['versionId']} for value in initial_refs.values()) + and all(set(value) == {current['versionId']} for value in authority_refs.values()), + 'J02 grace DELETE did not establish complete exact-version authority on every original member') + require(all(values[current['versionId']] == authority_refs[member][current['versionId']] + and values[current['versionId']].get('deleted') is False + and values[current['versionId']].get('isLatest') is True + and values[old['versionId']].get('dataDir') == directory + and values[current['versionId']].get('dataDir') != directory + for member, values in initial_refs.items()), + 'J02 grace DELETE changed the acknowledged current logical version or its original dataDir') + left, right = files(initial), files(authority) + old_prefix = bucket + '/' + KEY + '/' + directory + '/' + metadata = bucket + '/' + KEY + '/xl.meta' + require({n: v['formats'] for n, v in initial.items()} == {n: v['formats'] for n, v in authority.items()} + and not any(identity[2].startswith(old_prefix) for identity in right), + 'J02 grace DELETE changed formats or retained an unacknowledged old payload') + require({key: value for key, value in left.items() if key[2] != metadata and not key[2].startswith(old_prefix)} + == {key: value for key, value in right.items() if key[2] != metadata}, + 'J02 grace DELETE changed current/control payload bytes or sidecars') + if injected is not None: + observed = files(injected) + prefix = bucket + '/' + KEY + '/' + require({n: v['formats'] for n, v in injected.items()} == {n: v['formats'] for n, v in authority.items()} + and {key: value for key, value in observed.items() if not (key[:2] == target and key[2].startswith(prefix))} + == {key: value for key, value in right.items() if not (key[:2] == target and key[2].startswith(prefix))} + and {key: value for key, value in observed.items() if key[:2] == target and key[2].startswith(prefix)} + == {key: value for key, value in left.items() if key[:2] == target and key[2].startswith(prefix)}, + 'J02 grace replay is not exactly one original member object tree') + return {'allOriginalMembers': len(initial_refs), 'exactHistoricalDeleteAcknowledged': True, + 'originalCurrentAndControlBytesPreserved': True, 'singleOriginalMemberReplay': injected is not None} + + +def clocks(fleet, binding, label, previous=None): + samples = {} + for node in fleet.nodes: + start = time.time() + sample = fleet.record(label, {'node': node, **fleet.remote(node, 'process-clock')}) + finish = time.time() + identity = sample.get('identity') or {} + require(sample.get('readOnly') is True and sample.get('clockChanged') is False + and finite_epoch(sample.get('nodeEpoch')) and start - 3 <= sample['nodeEpoch'] <= finish + 3 + and type(identity.get('pid')) is int and identity['pid'] > 1 + and identity.get('binarySha256') == binding['candidateBinarySha256'] + and isinstance(identity.get('settings'), dict) + and identity.get('settings', {}).get('RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS') == str(GRACE_SECONDS), + 'J02 grace lacks the actual candidate process policy or synchronized node clock') + if previous is not None: + require(all(identity.get(key) == previous[node]['identity'].get(key) for key in PROCESS_FIELDS), + 'J02 original grace process changed between original and compensating tasks') + samples[node] = sample + return samples + + +def archive_member(fleet, snapshot, bucket, target, label): + node, disk = target + params = dict(disk=disk, bucket=bucket, key=KEY) + expected = object_files(snapshot, bucket, KEY)[target] + start = time.time() + observed = fleet.record('J02-grace-stopped-member', {'node': node, **fleet.remote(node, 'orphan-observe', **params)}) + require(observed.get('present') is True and observed.get('files') == expected + and finite_epoch(observed.get('nodeEpoch')) and start - 3 <= observed['nodeEpoch'] <= time.time() + 3, + 'J02 stopped grace object/clock differs from the original physical witness') + return fleet.record('J02-grace-archive-' + label, {'node': node, **fleet.remote(node, 'orphan-capture', + **params, generation=label, expectedFiles=expected, expectedPresent=True)}) + + +def run(probe, fleet, binding, directory, expected, incarnation, active, preserve_token): + """Use real SDK version history and natural wall-clock grace, never aged bytes.""" + bucket = binding['bucketPrefix'] + recorder = Recorder(probe, directory / 'original-grace-fixture') + fleet.start() + recorder.call('put_bucket_versioning', {'Bucket': bucket, 'VersioningConfiguration': {'Status': 'Enabled'}}) + versions = [] + for label in ('old', 'current'): + body = payload(bucket + '/J02-grace/' + label, SIZE) + response = recorder.call('put_object', {'Bucket': bucket, 'Key': KEY, 'ContentType': 'application/octet-stream'}, body) + version = response.get('VersionId') + require(isinstance(version, str) and str(uuid.UUID(version)) == version and uuid.UUID(version).int != 0 + and version not in {value['versionId'] for value in versions}, 'J02 grace PUT lacks a distinct real ACK version') + head = fleet.record('J02-grace-ACK-server-age', probe.s3('head_object', {'Bucket': bucket, 'Key': KEY, 'VersionId': version})) + info = head.get('response') or {} + require(head.get('acknowledged') is True and head.get('resultUnknown') is False and info.get('VersionId') == version + and info.get('ContentLength') == len(body) and isinstance(response.get('ETag'), str) and response['ETag'] + and info.get('ETag') == response['ETag'], + 'J02 grace server version age does not bind the actual PUT ACK') + modified = info.get('LastModified') + epoch = modified.timestamp() if isinstance(modified, datetime) else datetime.fromisoformat(modified).timestamp() + require(finite_epoch(epoch) and abs(epoch - time.time()) <= 5, 'J02 grace server version clock is not current') + if versions: versions[0]['isLatest'] = False + versions.append(dict(key=KEY, versionId=version, isLatest=True, deleteMarker=False, + payload={'sha256': hashlib.sha256(body).hexdigest(), 'size': len(body)}, lastModifiedEpoch=epoch)) + old, current = versions + seeded = [*expected, old, current] + fleet.record('J02-grace-initial-ACK-S3', verify_seed(seeded, probe.inventory(bucket))) + healthy = run_task(probe, fleet, {'bucket': bucket, 'key': KEY}, 2, active, + option_overrides={'recursive': True}, status_endpoints=tuple(binding['endpoints'])[1:]) + require(validate_version_receipts(healthy, bucket, versions, 2)['bucketIncarnationId'] == incarnation, + 'J02 grace initial versions escaped the independently anchored bucket incarnation') + idle_fleet(fleet, binding); fleet.stop() + initial = fleet.snapshot('J02-grace-original-two-version-physical', quiescent=True) + old_target = map_target(initial, bucket, KEY, 'data', 1, old['versionId'], latest_required=False) + current_target = map_target(initial, bucket, KEY, 'data', 1, current['versionId']) + verify_proof(fleet, old_target, 'J02-grace-original-old-full-bitrot') + verify_proof(fleet, current_target, 'J02-grace-original-current-full-bitrot') + target = (fleet.nodes[-1], 0) + archive_member(fleet, initial, bucket, target, 'old') + fleet.start() + deleted = recorder.call('delete_object', {'Bucket': bucket, 'Key': KEY, 'VersionId': old['versionId']}) + require(deleted.get('VersionId') == old['versionId'] and deleted.get('DeleteMarker') in {None, False} + and deleted.get('ResponseMetadata', {}).get('HTTPStatusCode') == 204, 'J02 grace DELETE lacks its exact historical ACK') + expected = [*expected, current] + baseline = probe.inventory(bucket) + fleet.record('J02-grace-post-delete-ACK-S3', {'inventory': baseline, **verify_seed(expected, baseline)}) + idle_fleet(fleet, binding); fleet.stop() + authority = fleet.snapshot('J02-grace-post-delete-all-member-authority', quiescent=True) + grace_fixture(initial, authority, None, bucket, old, current, target, old_target['dataDir']) + archive_member(fleet, authority, bucket, target, 'authority') + params = dict(disk=target[1], bucket=bucket, key=KEY, generation='authority') + fleet.orphan_faults.append((target[0], params)) + fleet.record('J02-grace-arm-post-delete-authority', {'node': target[0], **fleet.remote(target[0], 'orphan-arm', **params)}) + fleet.record('J02-grace-single-member-old-replay', {'node': target[0], **fleet.remote(target[0], 'orphan-install', + disk=target[1], bucket=bucket, key=KEY, dataDir=old_target['dataDir'], mode='object')}) + faulted = fleet.snapshot('J02-grace-original-replayed-fault', quiescent=True) + physical = grace_fixture(initial, authority, faulted, bucket, old, current, target, old_target['dataDir']) + fleet.start() + before_clocks = clocks(fleet, binding, 'J02-grace-actual-process-clock-before-original') + options = {**BASE_OPTIONS, 'recursive': True, 'remove': True} + scope = dict(bucket=bucket, key=KEY) + result = {'originalGraceSourceProcedure': True, 'physicalFixture': physical, 'globalClockChanged': False, + 'metadataAgeRewritten': False, 'sameTokenRecoveryClaimed': False, 'canonicalPassClaimed': False} + trigger_at = time.time() + expires = old['lastModifiedEpoch'] + GRACE_SECONDS + fleet.record('J02-grace-original-admission-window', {'observedAt': trigger_at, 'graceSeconds': GRACE_SECONDS, + 'oldServerModifiedEpoch': old['lastModifiedEpoch'], 'releaseAt': expires + 5}) + require(physical_content(fleet.snapshot('J02-grace-immediately-before-original-task')) == physical_content(faulted), + 'J02 grace fault changed before original admission') + if expires - trigger_at < 10: + result.update(verdict='NOT_OBSERVED', reason='Original pre-grace admission window elapsed') + else: + admissions = [] + status = run_task(probe, fleet, scope, 2, active, option_overrides=options, + status_endpoints=tuple(binding['endpoints'])[1:], admissions=admissions) + observed_at = time.time() + fleet.record('J02-grace-original-terminal-before-compensation', {'observedAt': observed_at, 'status': status}) + require(len(admissions) == 1, 'J02 grace original task has ambiguous admission') + idle_fleet(fleet, binding) + protected = fleet.snapshot('J02-grace-original-task-physical-before-any-GET') + if observed_at >= expires: + unchanged = physical_content(protected) == physical_content(faulted) + cleaned = physical_content(protected) == physical_content(authority) + require(unchanged or cleaned, 'J02 late grace observation lost current/control bytes') + grace_outcome(status, bucket, old, current, options, incarnation, + trigger_at if unchanged else observed_at, cleanup=not unchanged) + result.update(verdict='NOT_OBSERVED', reason='Original terminal observation missed the pre-grace window') + else: + result['original'] = grace_outcome(status, bucket, old, current, options, incarnation, observed_at) + require(physical_content(protected) == physical_content(faulted), 'J02 pre-grace task modified protected old/current bytes') + deadline = time.monotonic() + GRACE_SECONDS + 10 + while time.time() < expires + 5: + require(time.monotonic() < deadline, 'J02 natural grace clock did not converge within its bounded wait') + time.sleep(min(1, max(0, expires + 5 - time.time()))) + clocks(fleet, binding, 'J02-grace-actual-process-clock-after-expiry', before_clocks) + preserve_token(probe, fleet, admissions[0], status, 'J02-grace-original-token-after-natural-expiry') + require(physical_content(fleet.snapshot('J02-grace-original-fault-before-new-token')) == physical_content(protected), + 'J02 completed original token resumed or altered bytes after grace') + next_admissions = [] + recovered = run_task(probe, fleet, scope, 2, active, option_overrides=options, + status_endpoints=tuple(binding['endpoints'])[1:], admissions=next_admissions) + require(len(next_admissions) == 1 and next_admissions[0]['clientToken'] != admissions[0]['clientToken'], + 'J02 grace compensation reused the original completed token') + fleet.record('J02-grace-new-token-product-terminal', recovered) + result['compensation'] = grace_outcome(recovered, bucket, old, current, options, incarnation, time.time(), cleanup=True) + idle_fleet(fleet, binding) + clocks(fleet, binding, 'J02-grace-actual-process-clock-after-compensation', before_clocks) + product = fleet.snapshot('J02-grace-product-cleaned-before-any-GET-or-rescue') + require(physical_content(product) == physical_content(authority), 'J02 new token did not restore exact post-DELETE physical authority') + orphan.verify_public(fleet, binding, bucket, expected, [old['versionId']], 'J02-grace-product-S3-before-rescue') + preserve_token(probe, fleet, admissions[0], status, 'J02-grace-original-token-after-new-task-and-GET') + clocks(fleet, binding, 'J02-grace-actual-process-clock-after-public-and-token-checks', before_clocks) + result.update(verdict='PASS', originalToken=admissions[0]['clientToken'], compensatingToken=next_admissions[0]['clientToken'], + originalTokenUnchanged=True, currentPayloadMatchesIndependentlyVerifiedOriginal=True, + productCleanupBeforeFixtureRescue=True, oldVersionAbsentAfterProductCleanup=True) + idle_fleet(fleet, binding); fleet.stop() + final = fleet.snapshot('J02-grace-complete-result-physical-before-fixture-rescue', quiescent=True) + if result['verdict'] == 'PASS': + require(physical_content(final) == physical_content(authority), 'J02 business reads changed the product cleanup scene') + orphan.recover_all(fleet) + require(physical_content(fleet.snapshot('J02-grace-post-delete-authority-after-fixture-finalization', quiescent=True)) == physical_content(authority), + 'J02 grace finalization failed to preserve exact acknowledged DELETE authority') + result.update(fixtureRescueIsProductRepair=False, sourceDiagnosticComplete=True) + return expected, baseline, result diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index e49dbc39..682ec746 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -3376,7 +3376,9 @@ "同批独立key建立健康、可恢复缺片、低于R的不可恢复对象；deferred由冻结的锁/依赖故障产生，dry-run另建任务。", "保存逐对象物理基线和期望disposition，对结果的execution/coverage/counters/object identities做逐项对账。", "unknown/skipped/completed_with_errors不算修复成功，legacy objectsHealed与canonical healed分开解释；attemptFailures不得被隐藏。", - "使用独立完整manifest弥补有界outcome窗口，保持不可修对象的正确失败记录；恢复后另任务修复不覆盖原结果。" + "使用独立完整manifest弥补有界outcome窗口，保持不可修对象的正确失败记录；恢复后另任务修复不覆盖原结果。", + "grace 分支先用 SDK PUT 建立两个真实版本并验证完整分片；保存一个成员的原对象树后，SDK DELETE 精确历史版本，逐盘验证仅当前版本存在且其逻辑元数据、数据片及控制对象未变。只向原成员回放原对象树；实际候选进程固定 grace=120 秒，不修改元数据年龄或时钟。原任务须在窗口内完成，精确报告旧版 deferred、当前版 verified_healthy 及 retryNotBefore；自然等待过期后原 token 仍不变，新 token 才能清理旧版。清理后先核对全部物理字节等于 DELETE 后权威现场，再跨节点 GET/404 和复查旧 token，最后才做 fixture finalization。错过窗口保留 NOT_OBSERVED；异常保留现场并标 DIRTY；救援仅恢复 DELETE 后权威树，不复活已删除版本。", + "Manual diagnostic entry: explicitly select heal.outcome-counters-and-error-truth through Functional Chain with the frozen candidate and candidate-bound manager/inspector tools. The default catalog policy stays evidence-blocked; even all passing diagnostic branches retain complete=false, canonicalCasePassClaimed=false and a blocking gate verdict until independent acceptance." ], "commands": [ "SEED_DB", @@ -3409,7 +3411,7 @@ "dispatchable": false, "code": "J02", "reasonCode": "EVIDENCE_MIXED_OUTCOME_ORCHESTRATION", - "reason": "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral、真实 SDK legacy Unknown 和只读永久失败/新 token 补偿执行分支及逐对象 manifest/attemptFailures 对账；仍需完整串行 live 证据及原始损坏元数据 Unknown、grace Deferred/新 token 补偿源流程，故保持禁止 dispatch；本地检查不能作为完整 PASS。", + "reason": "已有 mixed healthy/repaired/failed、独立 dry-run、依赖 deferral、真实 SDK legacy Unknown、原始全副本损坏元数据 Unknown、只读永久失败/新 token 补偿及版本级 grace Deferred/新 token 清理执行流程，均按逐对象 manifest/attemptFailures 对账；仍需完整串行 live 证据及独立验收，故保持禁止默认 dispatch 和 canonical admission；本地检查不能作为完整 PASS。", "requiredCapabilities": [ "mixed_outcome_fixture", "per_object_canonical_receipts", diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 77c219ea..ae119e93 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -21,6 +21,7 @@ done case "${case_id}" in heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization) ;; heal.bucket-incarnation-recreation) driver=incarnation.py ;; + heal.outcome-counters-and-error-truth) driver=outcome.py ;; heal.checkpoint-and-mrf-corruption) driver=persistence.py ;; heal.node-outage-rejoin) driver=node_outage.py ;; heal.status-partial-disabled-unknown) driver=status.py ;; @@ -197,6 +198,12 @@ if [ "${case_id}" = heal.checkpoint-and-mrf-corruption ] && [ "${cleanup_status} "${evidence}/healthy-summary.json" >/dev/null 2>&1; then verdict=BLOCKED; reason="F07 diagnostic execution lacks full native cut coverage and independent live audit; partial success cannot certify the canonical case" fi +if [ "${case_id}" = heal.outcome-counters-and-error-truth ] && [ "${cleanup_status}" = PASS ]; then + if [ "${case_rc}" -eq 0 ] || jq -e '.diagnosticBranchesPassed == true and .complete == false and .canonicalCasePassClaimed == false' \ + "${evidence}/healthy-summary.json" >/dev/null 2>&1; then + verdict=BLOCKED; reason="J02 manual diagnostics require independent acceptance of all original source procedures; canonical admission remains disabled" + fi +fi args=("${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool --result-file "${artifacts}/cases/${case_id}.json" --verdict "${verdict}" --cleanup-status "${cleanup_status}" --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --binary-sha256 "${binary_sha}" diff --git a/test_validation_heal_outcome.py b/test_validation_heal_outcome.py index 3513c223..d2235e4f 100644 --- a/test_validation_heal_outcome.py +++ b/test_validation_heal_outcome.py @@ -222,8 +222,9 @@ def branch(args, topology, nodes, fixture, name, candidate): self.assertTrue(summary["diagnosticBranchesPassed"]) self.assertFalse(summary["complete"]) self.assertFalse(summary["canonicalCasePassClaimed"]) - self.assertEqual(set(summary["sourceAssertionGaps"]), { - "version-specific-grace-deferred-and-new-token-compensation"}) + self.assertEqual(summary["sourceAssertionGaps"], []) + self.assertFalse(summary["canonicalAdmissionEnabled"]) + self.assertTrue(summary["liveSourceProcedureAcceptancePending"]) def test_dirty_cleanup_stops_before_next_fixture_or_topology(self): with tempfile.TemporaryDirectory() as tmp: diff --git a/test_validation_heal_outcome_grace.py b/test_validation_heal_outcome_grace.py new file mode 100644 index 00000000..9952c184 --- /dev/null +++ b/test_validation_heal_outcome_grace.py @@ -0,0 +1,415 @@ +"""Exercise real grace oracles and source-procedure ordering without a live fleet.""" + +import copy +import json +import os +from datetime import datetime, timezone +from pathlib import Path +import tempfile +import subprocess +import types +import unittest +import uuid +from contextlib import ExitStack +from unittest.mock import patch + +from scripts.validation_heal import fleet_node, healthy, outcome, outcome_grace as grace +from scripts.validation_heal.healthy import BASE_OPTIONS, physical_content +from test_validation_heal import versioned_scene +import test_validation_heal_orphan as orphan_tests + + +BUCKET = 'rv-heal-0123456789abcdef' +INCARNATION = str(uuid.UUID(int=30)) +OLD, CURRENT = str(uuid.UUID(int=21)), str(uuid.UUID(int=22)) +OPTIONS = {**BASE_OPTIONS, 'recursive': True, 'remove': True} + + +def terminal(*, cleanup=False, healthy_only=False): + old_state = {'state': 'repaired' if cleanup else 'verified_healthy'} if cleanup or healthy_only else { + 'state': 'deferred', 'details': {'reason': 'dangling_delete_grace', + 'retryNotBefore': {'secs_since_epoch': 1120, 'nanos_since_epoch': 0}}} + return {'summary': 'finished', 'settings': OPTIONS, 'truncated': False, 'outcome': { + 'execution': {'state': 'completed'}, 'coverage': 'complete', 'objectsTruncated': False, + 'counters': dict(processed=2, healed=int(cleanup), unchanged=2 if healthy_only else 1, + skipped=int(not cleanup and not healthy_only), failed=0, unknown=0, + attemptFailures=int(not cleanup and not healthy_only), overflowed=False), + 'objects': [{'identity': dict(kind='object', bucket=BUCKET, object=grace.KEY, versionId=version, + bucketIncarnationId=INCARNATION, poolIndex=0, setIndex=0), 'disposition': disposition} + for version, disposition in ((OLD, old_state), (CURRENT, {'state': 'verified_healthy'}))]}} + + +def scene(nodes=1): + initial, bucket, entries = versioned_scene([grace.KEY, 'control.bin']) + for entry in initial['node1']['files']: + if not entry['path'].startswith(bucket + '/' + grace.KEY + '/'): continue + if 'metadata' in entry: + versions = entry['metadata']['versions'][:2] + for index, version in enumerate(versions): + version.update(versionId=(OLD, CURRENT)[index], isLatest=index == 1) + entry['metadata']['versions'] = versions + old_dir = next(entry['metadata']['versions'][0]['dataDir'] for entry in initial['node1']['files'] + if entry['path'] == bucket + '/' + grace.KEY + '/xl.meta') + unused = '12345678-1234-1234-1234-000000000002' + initial['node1']['files'] = [entry for entry in initial['node1']['files'] + if not entry['path'].startswith(bucket + '/' + grace.KEY + '/' + unused + '/')] + for index in range(2, nodes + 1): initial['node' + str(index)] = copy.deepcopy(initial['node1']) + authority = copy.deepcopy(initial) + for value in authority.values(): + value['files'] = [entry for entry in value['files'] + if not entry['path'].startswith(bucket + '/' + grace.KEY + '/' + old_dir + '/')] + for entry in value['files']: + if entry['path'] == bucket + '/' + grace.KEY + '/xl.meta': + entry['metadata']['versions'] = entry['metadata']['versions'][1:] + entry['sha256'] = 'e' * 64 + target = ('node' + str(nodes), 0) + injected = copy.deepcopy(authority) + prefix = bucket + '/' + grace.KEY + '/' + injected[target[0]]['files'] = [entry for entry in injected[target[0]]['files'] + if not (entry['disk'] == 0 and entry['path'].startswith(prefix))] + injected[target[0]]['files'] += [copy.deepcopy(entry) for entry in initial[target[0]]['files'] + if entry['disk'] == 0 and entry['path'].startswith(prefix)] + return initial, authority, injected, target, old_dir + + +class GraceOracleTest(unittest.TestCase): + old = {'versionId': OLD, 'lastModifiedEpoch': 1000.0} + current = {'versionId': CURRENT} + + def check(self, value, observed=1010, cleanup=False): + return grace.grace_outcome(value, BUCKET, self.old, self.current, OPTIONS, INCARNATION, observed, cleanup=cleanup) + + def test_exact_future_deferral_and_distinct_post_expiry_repair(self): + self.assertEqual(self.check(terminal())['retryNotBeforeEpoch'], 1120) + self.assertTrue(self.check(terminal(cleanup=True), 1120, True)['newTaskProductCleanup']) + with self.assertRaises(ValueError): self.check(terminal(cleanup=True), 1119.999, True) + with self.assertRaises(ValueError): self.check(terminal(), 1120) + + def test_wrong_truth_and_legacy_counts_cannot_replace_original_responsibility(self): + for path, values in ( + (('summary',), ['running', 'stopped']), (('truncated',), [True, None]), + (('outcome', 'coverage'), ['partial', None]), + (('outcome', 'execution'), [{'state': 'completed_with_errors'}]), + (('outcome', 'objectsTruncated'), [True, None]), + (('settings',), [{**OPTIONS, 'remove': False}])): + for value in values: + changed = terminal(); parent = changed + for name in path[:-1]: parent = parent[name] + parent[path[-1]] = value + with self.subTest(path=path, value=value), self.assertRaises(ValueError): self.check(changed) + for name, original in terminal()['outcome']['counters'].items(): + for value in (None, not original if type(original) is bool else original + 1, + 0 if type(original) is bool else bool(original)): + changed = terminal(); changed['outcome']['counters'][name] = value + with self.subTest(counter=name, value=value), self.assertRaises(ValueError): self.check(changed) + changed = terminal(); changed['outcome']['counters']['legacyHealed'] = 88 + with self.assertRaises(ValueError): self.check(changed) + + def test_every_identity_boundary_and_duplicate_version_is_rejected(self): + for name, values in {'kind': ['bucket'], 'bucket': ['foreign'], 'object': ['other.bin'], + 'versionId': [None, CURRENT], 'bucketIncarnationId': [None, 'new'], + 'poolIndex': [True, 1], 'setIndex': [False, 1]}.items(): + for value in values: + changed = terminal(); changed['outcome']['objects'][0]['identity'][name] = value + with self.subTest(name=name, value=value), self.assertRaises(ValueError): self.check(changed) + for items in ([], terminal()['outcome']['objects'][:1], terminal()['outcome']['objects'] * 2): + changed = terminal(); changed['outcome']['objects'] = items + with self.assertRaises(ValueError): self.check(changed) + + def test_exact_retry_wire_shape_bounds_and_original_age_are_required(self): + for value in (None, 1120, {}, {'secs_since_epoch': True, 'nanos_since_epoch': 0}, + {'secs_since_epoch': -1, 'nanos_since_epoch': 0}, + {'secs_since_epoch': 2**64, 'nanos_since_epoch': 0}, + {'secs_since_epoch': 1120, 'nanos_since_epoch': -1}, + {'secs_since_epoch': 1120, 'nanos_since_epoch': 10**9}, + {'secs_since_epoch': 1120, 'nanos_since_epoch': False}, + {'secs_since_epoch': 1120, 'nanos_since_epoch': 0, 'extra': 0}, + {'secs_since_epoch': 1126, 'nanos_since_epoch': 0}): + changed = terminal(); changed['outcome']['objects'][0]['disposition']['details']['retryNotBefore'] = value + with self.subTest(value=value), self.assertRaises(ValueError): self.check(changed) + for details in ({'reason': 'dangling_delete_grace', 'retry_not_before': None}, + {'reason': 'transient_existence_check', 'retryNotBefore': None}): + changed = terminal(); changed['outcome']['objects'][0]['disposition']['details'] = details + with self.assertRaises(ValueError): self.check(changed) + for value in (True, -1, float('inf'), float('nan')): + with self.assertRaises(ValueError): self.check(terminal(), value) + + def test_false_health_unknown_skip_or_repair_cannot_certify_deferral(self): + for index in (0, 1): + for state in ('unknown', 'skipped', 'failed', 'repaired', 'metadata_healthy'): + changed = terminal(); changed['outcome']['objects'][index]['disposition'] = {'state': state} + with self.subTest(index=index, state=state), self.assertRaises(ValueError): self.check(changed) + + def test_all_original_members_and_current_logical_metadata_remain_exact(self): + for count in (1, 4): + initial, authority, injected, target, directory = scene(count) + proof = grace.grace_fixture(initial, authority, injected, BUCKET, self.old, self.current, target, directory) + self.assertEqual(proof['allOriginalMembers'], count * 4) + for field, value in (('dataDir', str(uuid.UUID(int=99))), ('deleted', True), ('isLatest', False), ('size', 999)): + changed = copy.deepcopy(authority) + meta = next(entry for entry in changed['node1']['files'] + if entry['path'] == BUCKET + '/' + grace.KEY + '/xl.meta') + meta['metadata']['versions'][0][field] = value + with self.subTest(nodes=count, field=field), self.assertRaises(ValueError): + grace.grace_fixture(initial, changed, None, BUCKET, self.old, self.current, target, directory) + for source, label in ((authority, 'authority'), (injected, 'injected')): + for kind in ('control', 'format', 'extra-old'): + changed = copy.deepcopy(source) + if kind == 'control': + next(entry for entry in changed['node1']['files'] if '/control.bin/' in entry['path'])['sha256'] = 'b' * 64 + elif kind == 'format': changed['node1']['formats'][0]['sha256'] = 'b' * 64 + else: + extra = next(entry for entry in initial['node1']['files'] if '/' + directory + '/' in entry['path']) + # An additional replay on a different disk is forbidden, including H1. + extra = copy.deepcopy(extra); extra['disk'] = 1 + changed['node1']['files'].append(extra) + with self.subTest(nodes=count, source=label, poison=kind), self.assertRaises(ValueError): + grace.grace_fixture(initial, changed if label == 'authority' else authority, + changed if label == 'injected' else None, BUCKET, self.old, self.current, target, directory) + + +class GraceRuntimeTest(unittest.TestCase): + def identity(self): + return {**{name: name for name in fleet_node.PROCESS_FIELDS}, 'pid': 123, + 'binarySha256': 'a' * 64, 'settings': fleet_node.runtime_config('outcome-grace')} + + def test_owned_clock_is_read_only_fixed_policy_and_process_bound(self): + identity = self.identity() + request = {'runId': 'gh-123-1-h1', 'action': 'process-clock'} + with patch.object(fleet_node, 'owned', return_value=(None, None, {'runtimeProfile': 'outcome-grace'})), \ + patch.object(fleet_node, 'process', return_value=identity), patch.object(fleet_node.time, 'time', return_value=1000): + witness = fleet_node.process_clock(request) + self.assertEqual(witness, dict(identity=identity, nodeEpoch=1000, readOnly=True, clockChanged=False)) + for name in ('pid', 'clock', 'path'): + with self.assertRaises(ValueError): fleet_node.process_clock({**request, name: 1}) + for name in fleet_node.PROCESS_FIELDS: + changed = copy.deepcopy(identity); changed[name] = 'changed' + with patch.object(fleet_node, 'owned', return_value=(None, None, {'runtimeProfile': 'outcome-grace'})), \ + patch.object(fleet_node, 'process', side_effect=[identity, changed]), self.subTest(field=name), self.assertRaises(ValueError): + fleet_node.process_clock(request) + for profile in ('manual', 'orphan-before'): + with patch.object(fleet_node, 'owned', return_value=(None, None, {'runtimeProfile': profile})), self.assertRaises(ValueError): + fleet_node.process_clock(request) + for changes in ({'pid': 0}, {'pid': True}, {'settings': {'RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS': '0'}}): + with patch.object(fleet_node, 'owned', return_value=(None, None, {'runtimeProfile': 'outcome-grace'})), \ + patch.object(fleet_node, 'process', return_value={**identity, **changes}), self.assertRaises(ValueError): + fleet_node.process_clock(request) + + def test_grace_descriptor_reuses_last_ack_authority_without_reviving_old(self): + helper = orphan_tests.OrphanDescriptorTests() + helper.setUp(); self.addCleanup(helper.tearDown) + helper.owner['runtimeProfile'] = 'outcome-grace' + helper.authority() + helper.invoke('install', mode='object', dataDir=helper.old_dir) + self.assertTrue((helper.object / helper.old_dir).is_dir()) + self.assertTrue(helper.recover()['lastACKAuthorityRestored']) + self.assertFalse((helper.object / helper.old_dir).exists()) + self.assertEqual((helper.object / 'xl.meta').read_bytes(), b'XL2 live') + for field, value in (('key', '../escape'), ('disk', True), ('disk', 4), ('bucket', 'foreign')): + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + helper.invoke('capture', **{field: value}, generation='authority', expectedFiles=[], expectedPresent=False) + + def test_grace_runtime_cannot_access_other_orphan_mutators(self): + owner = {'runtimeProfile': 'outcome-grace', 'binarySha256': 'a' * 64} + with patch.object(fleet_node, 'owned', return_value=(Path('/owned'), (), owner)), \ + patch.object(fleet_node, 'stopped'), patch.object(fleet_node, 'digest', return_value='a' * 64), \ + patch.object(fleet_node, 'fault_module'), \ + patch.dict(fleet_node.__dict__, generation_backend=object(), orphan_backend=object()): + for action in ('orphan-convert', 'orphan-install-tool', 'orphan-disarm', 'orphan-multipart'): + with self.subTest(action=action), self.assertRaises(ValueError): + fleet_node.orphan_operation({'runId': 'gh-123-1-h1', 'action': action}) + + def test_controller_rejects_clock_skew_policy_change_and_process_replacement(self): + identity = self.identity() + original = dict(identity=identity, nodeEpoch=1000, readOnly=True, clockChanged=False) + fleet = types.SimpleNamespace(nodes=['node1'], record=lambda label, value: value, + remote=lambda *a: original) + binding = {'candidateBinarySha256': 'a' * 64} + with patch.object(grace, 'time', types.SimpleNamespace(time=lambda: 1000)): + previous = grace.clocks(fleet, binding, 'before') + grace.clocks(fleet, binding, 'after', previous) + for name, value in (('nodeEpoch', 1004), ('nodeEpoch', float('nan')), + ('nodeEpoch', True), ('readOnly', False), ('clockChanged', True)): + changed = {**original, name: value}; fleet.remote = lambda *a: changed + with self.subTest(name=name, value=value), self.assertRaises(ValueError): + grace.clocks(fleet, binding, 'poison') + for name in fleet_node.PROCESS_FIELDS: + changed = {**original, 'identity': {**identity, name: 'changed'}} + fleet.remote = lambda *a: changed + with self.subTest(field=name), self.assertRaises(ValueError): + grace.clocks(fleet, binding, 'replaced', previous) + + def test_remote_adapter_contains_registered_clock_and_descriptor_backends(self): + with tempfile.TemporaryDirectory() as temporary: + fleet = healthy.Fleet(['node1'], Path(temporary), 'gh-123-1-h1', runtime_profile='outcome-grace') + for action in ('process-clock', 'orphan-observe', 'orphan-capture', 'orphan-arm', 'orphan-install', 'orphan-recover'): + with patch.object(healthy.subprocess, 'run', return_value=types.SimpleNamespace(returncode=0, stdout='{}', stderr='')) as remote: + fleet.remote('node1', action) + source = remote.call_args.kwargs['input'] + compile(source, '', 'exec') + self.assertIn('actions["process-clock"] = process_clock', source) + if action.startswith('orphan-'): + self.assertIn('orphan_backend = types.ModuleType', source) + self.assertIn('generation_backend = types.ModuleType', source) + + +class GraceIntegrationTest(unittest.TestCase): + def test_uncertain_original_grace_procedure_retains_namespace_and_stops_next_branch(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary); rc = root / 'rc'; rc.write_bytes(b'cli') + args = types.SimpleNamespace(evidence=root / 'evidence', rc_bin=rc, binary_sha='b' * 64, + testing_commit='a' * 40, tool_directory=root / 'tools') + with ExitStack() as stack: + factory = stack.enter_context(patch.object(outcome, 'Fleet')) + fleet = factory.return_value + fleet.reserved = ['node1']; fleet.record.side_effect = lambda label, value: value + stack.enter_context(patch.dict(os.environ, GITHUB_RUN_ID='12345', GITHUB_RUN_ATTEMPT='1')) + for name in ('Probe', 'Recorder', 'verify_inspector', 'verify_quorum_contract', 'validate_topology', 'verify_seed', 'idle_fleet', 'run_task'): + stack.enter_context(patch.object(outcome, name)) + stack.enter_context(patch.object(outcome, 'seed', return_value=[])) + stack.enter_context(patch.object(outcome, 'validate_version_receipts', return_value={'bucketIncarnationId': INCARNATION})) + stack.enter_context(patch.object(outcome, 'original_grace_probe', side_effect=TimeoutError('lost replay ACK'))) + result = outcome.run_branch(args, 'H1', ['node1'], 'DB', 'deferred', + {'source_sha': outcome.CANDIDATE, 'package_sha256': 'c' * 64}) + self.assertEqual(factory.call_args.kwargs['runtime_profile'], 'outcome-grace') + self.assertEqual((result['verdict'], result['cleanup']), ('FAIL', 'FAIL')) + fleet.cleanup.assert_not_called() + retained = next(call.args[1] for call in fleet.record.call_args_list if call.args[0] == 'retain-unknown-seed-receipt') + self.assertEqual(retained['request']['operation'], 'original-version-specific-grace-source-procedure') + self.assertTrue(retained['namespacesRetained']) + self.assertGreaterEqual(fleet.stop.call_count, 2) + + def test_manual_diagnostic_entry_cannot_certify_canonical_even_if_driver_returns_zero(self): + root = Path(__file__).resolve().parent + dry = subprocess.run(['bash', str(root / 'scripts/validation_heal/run_case.sh'), '--validation-case', + outcome.CASE_ID, '--dry-run'], capture_output=True, text=True) + self.assertEqual(dry.returncode, 0, dry.stderr) + shell = (root / 'scripts/validation_heal/run_case.sh').read_text() + verdict = shell[shell.index('verdict=FAIL reason='):shell.index('args=("${ROOT}/scripts/validation_result.py"')] + with tempfile.TemporaryDirectory() as temporary: + evidence = Path(temporary) + for code, all_pass, cleanup in ((0, True, 'PASS'), (1, True, 'PASS'), (1, False, 'PASS'), (0, True, 'FAIL')): + (evidence / 'healthy-summary.json').write_text(json.dumps(dict(diagnosticBranchesPassed=all_pass, + complete=False, canonicalCasePassClaimed=False))) + script = verdict + '\nprintf "%s" "$verdict"\n' + checked = subprocess.run(['bash', '-euc', script], capture_output=True, text=True, + env={**os.environ, 'case_id': outcome.CASE_ID, 'cleanup_status': cleanup, + 'case_rc': str(code), 'evidence': str(evidence)}) + self.assertEqual(checked.returncode, 0, checked.stderr) + self.assertEqual(checked.stdout, 'BLOCKED' if cleanup == 'PASS' and (code == 0 or all_pass) + else 'INFRA_ERROR' if code == 0 else 'FAIL') + + +class GraceProtocolTest(unittest.TestCase): + def execute(self, *, nodes=1, mode='normal'): + initial, authority, injected, target, directory = scene(nodes) + clock = types.SimpleNamespace(now=1000.0, elapsed=0.0) + events, records, admissions = [], {}, [] + def sleep(seconds): + self.assertLessEqual(seconds, 1) + if mode != 'stalled-clock': clock.now += seconds + clock.elapsed += seconds; events.append(('wait', seconds)) + timer = types.SimpleNamespace(time=lambda: clock.now, monotonic=lambda: clock.elapsed, sleep=sleep) + identity = GraceRuntimeTest().identity() + class Fleet: + def __init__(self): self.nodes = list(initial); self.orphan_faults = []; self.phase = 'initial' + def start(self): events.append(('start', clock.now)) + def stop(self): events.append(('stop', clock.now)) + def record(self, label, value): records[label] = copy.deepcopy(value); return value + def remote(self, node, action, **params): + events.append((action, node)) + if action == 'orphan-observe': + snapshot = initial if self.phase == 'initial' else authority + return dict(present=True, files=grace.object_files(snapshot, BUCKET, grace.KEY)[target], nodeEpoch=clock.now) + if action == 'orphan-install': + if mode == 'miss-admission': clock.now = 1111 + self.phase = 'fault' + if action == 'process-clock': + actual = {**identity, 'pid': 124} if mode == 'replacement-after-compensation' and len(tasks) >= 3 else identity + return dict(identity=actual, nodeEpoch=clock.now, readOnly=True, clockChanged=False) + return {} + def snapshot(self, label, **kwargs): + records[label] = self.phase + return copy.deepcopy(initial if self.phase == 'initial' else authority if self.phase in ('authority', 'clean') else injected) + fleet = Fleet() + class Probe: + def inventory(self, *args, **kwargs): return {'inventory': True} + def s3(self, operation, params, body=None): + self_outer.assertEqual(operation, 'head_object') + return dict(acknowledged=True, resultUnknown=False, response=dict(VersionId=params['VersionId'], + ContentLength=grace.SIZE, ETag='etag', LastModified=datetime.fromtimestamp(1000, timezone.utc))) + self_outer = self + class Recorder: + def __init__(self, *args): self.versions = iter((OLD, CURRENT)) + def call(self, operation, params, body=None): + events.append((operation, clock.now)) + if operation == 'put_object': return dict(VersionId=next(self.versions), ETag='etag') + if operation == 'delete_object': + fleet.phase = 'authority' + return dict(VersionId=OLD, ResponseMetadata={'HTTPStatusCode': 204}) + return {} + tasks = [] + def task(*args, **kwargs): + tasks.append(clock.now); events.append(('task', clock.now)) + if len(tasks) == 1: return terminal(healthy_only=True) + token = 'same' if mode == 'reused-token' else 'token-' + str(len(tasks)) + admission = {'clientToken': token}; kwargs['admissions'].append(admission); admissions.append(admission) + if len(tasks) == 2: + if mode == 'late-original': clock.now = 1121 + if mode == 'late-cleanup': clock.now = 1121; fleet.phase = 'clean'; return terminal(cleanup=True) + if mode == 'partial-original': + fleet.phase = 'clean' + raise ValueError('partial outcome') + return terminal() + if mode != 'false-cleanup': fleet.phase = 'clean' + return terminal(cleanup=True) + def preserve(*args): events.append(('preserve', clock.now)) + def verify_public(*args): + self.assertEqual(fleet.phase, 'clean'); events.append(('public', clock.now)) + def recover(*args): + events.append(('rescue', clock.now)); fleet.phase = 'authority'; fleet.orphan_faults.clear() + with tempfile.TemporaryDirectory() as temporary, ExitStack() as stack: + for name, value in {'time': timer, 'Recorder': Recorder, 'verify_seed': lambda *a: {}, + 'run_task': task, 'idle_fleet': lambda *a: None, 'verify_proof': lambda *a: {}, + 'map_target': lambda *a, **kw: {'dataDir': directory}}.items(): + stack.enter_context(patch.object(grace, name, value)) + stack.enter_context(patch.object(grace.orphan, 'verify_public', verify_public)) + stack.enter_context(patch.object(grace.orphan, 'recover_all', recover)) + self.events, self.fleet = events, fleet + result = grace.run(Probe(), fleet, dict(bucketPrefix=BUCKET, candidateBinarySha256='a' * 64, + endpoints={node: 'http://' + node for node in fleet.nodes}), Path(temporary), [], INCARNATION, [], preserve) + return result, records, tasks, events + + def test_natural_expiry_new_token_and_product_cleanup_precede_get_and_rescue(self): + for nodes in (1, 4): + result, records, tasks, events = self.execute(nodes=nodes) + self.assertEqual(result[2]['verdict'], 'PASS') + self.assertFalse(result[2]['canonicalPassClaimed']) + self.assertEqual(result[2]['physicalFixture']['allOriginalMembers'], 4 * nodes) + self.assertGreaterEqual(tasks[2], 1125) + kinds = [event[0] for event in events] + self.assertLess(kinds.index('public'), kinds.index('rescue')) + self.assertEqual(kinds.count('preserve'), 2) + # No service restart across original, wait, compensation, public checks or token rechecks. + first, last = kinds.index('wait'), kinds.index('public') + self.assertFalse(set(kinds[first:last + 1]) & {'start', 'stop'}) + self.assertEqual(records['J02-grace-product-cleaned-before-any-GET-or-rescue'], 'clean') + self.assertEqual(self.fleet.orphan_faults, []) + + def test_missed_admission_or_terminal_window_remains_not_observed(self): + for mode in ('miss-admission', 'late-original', 'late-cleanup'): + result, _, tasks, events = self.execute(mode=mode) + self.assertEqual(result[2]['verdict'], 'NOT_OBSERVED') + self.assertEqual(len(tasks), 1 if mode == 'miss-admission' else 2) + self.assertNotIn('public', [event[0] for event in events]) + self.assertEqual(self.fleet.orphan_faults, []) + + def test_reused_token_false_cleanup_and_partial_truth_never_rescue_or_pass(self): + for mode in ('reused-token', 'false-cleanup', 'partial-original', 'stalled-clock', 'replacement-after-compensation'): + with self.subTest(mode=mode), self.assertRaises(ValueError): self.execute(mode=mode) + self.assertNotIn('rescue', [event[0] for event in self.events]) + self.assertNotIn('public', [event[0] for event in self.events]) + self.assertEqual(len(self.fleet.orphan_faults), 1) + + +if __name__ == '__main__': unittest.main() From 6a4127813c230b56c84c5dee20f4222ec13a01b0 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 22:02:36 +0800 Subject: [PATCH 210/343] fix(validation): preserve acknowledged pre-replay grace fixtures --- docs/validation-heal-cases.md | 1 + scripts/validation_heal/outcome_grace.py | 6 +- scripts/validation_heal/outcome_recovery.py | 147 ++++++++++++++++ scripts/validation_heal/recipes.json | 3 +- .../validation_heal/recover_environment.py | 7 + scripts/validation_heal/retained_scene.py | 6 +- test_validation_heal_outcome_grace.py | 157 +++++++++++++++++- 7 files changed, 318 insertions(+), 9 deletions(-) create mode 100644 scripts/validation_heal/outcome_recovery.py diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index 63f6808d..08b3d450 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1816,6 +1816,7 @@ Blocked reason (`EVIDENCE_MIXED_OUTCOME_ORCHESTRATION`): 已有 mixed healthy/re 4. 使用独立完整manifest弥补有界outcome窗口，保持不可修对象的正确失败记录；恢复后另任务修复不覆盖原结果。 5. grace 分支先用 SDK PUT 建立两个真实版本并验证完整分片；保存一个成员的原对象树后，SDK DELETE 精确历史版本，逐盘验证仅当前版本存在且其逻辑元数据、数据片及控制对象未变。只向原成员回放原对象树；实际候选进程固定 grace=120 秒，不修改元数据年龄或时钟。原任务须在窗口内完成，精确报告旧版 deferred、当前版 verified_healthy 及 retryNotBefore；自然等待过期后原 token 仍不变，新 token 才能清理旧版。清理后先核对全部物理字节等于 DELETE 后权威现场，再跨节点 GET/404 和复查旧 token，最后才做 fixture finalization。错过窗口保留 NOT_OBSERVED；异常保留现场并标 DIRTY；救援仅恢复 DELETE 后权威树，不复活已删除版本。 6. Manual diagnostic entry: explicitly select heal.outcome-counters-and-error-truth through Functional Chain with the frozen candidate and candidate-bound manager/inspector tools. The default catalog policy stays evidence-blocked; even all passing diagnostic branches retain complete=false, canonicalCasePassClaimed=false and a blocking gate verdict until independent acceptance. +7. For a pre-replay public-manifest refusal, recover only after the complete SDK request/ACK ledger proves all writes acknowledged and no DELETE or replay occurred. Bind all four original physical members and the reserved owner, stop the exact candidate, then preserve all roots and state using durable same-filesystem non-overwriting rename. Partial archive retries require the same durable inventory; missing roots or drift retain DIRTY. Isolation is not product repair or restoration and never rewrites the failed case. Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [LOCK_WINDOW](validation-heal-operations.md#LOCK_WINDOW) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) diff --git a/scripts/validation_heal/outcome_grace.py b/scripts/validation_heal/outcome_grace.py index d105f10f..f98584c9 100644 --- a/scripts/validation_heal/outcome_grace.py +++ b/scripts/validation_heal/outcome_grace.py @@ -181,7 +181,9 @@ def run(probe, fleet, binding, directory, expected, incarnation, active, preserv versions.append(dict(key=KEY, versionId=version, isLatest=True, deleteMarker=False, payload={'sha256': hashlib.sha256(body).hexdigest(), 'size': len(body)}, lastModifiedEpoch=epoch)) old, current = versions - seeded = [*expected, old, current] + # Server age belongs to the grace witness, not the public S3 manifest. + public_versions = [{key: value for key, value in version.items() if key != 'lastModifiedEpoch'} for version in versions] + seeded = [*expected, *public_versions] fleet.record('J02-grace-initial-ACK-S3', verify_seed(seeded, probe.inventory(bucket))) healthy = run_task(probe, fleet, {'bucket': bucket, 'key': KEY}, 2, active, option_overrides={'recursive': True}, status_endpoints=tuple(binding['endpoints'])[1:]) @@ -199,7 +201,7 @@ def run(probe, fleet, binding, directory, expected, incarnation, active, preserv deleted = recorder.call('delete_object', {'Bucket': bucket, 'Key': KEY, 'VersionId': old['versionId']}) require(deleted.get('VersionId') == old['versionId'] and deleted.get('DeleteMarker') in {None, False} and deleted.get('ResponseMetadata', {}).get('HTTPStatusCode') == 204, 'J02 grace DELETE lacks its exact historical ACK') - expected = [*expected, current] + expected = [*expected, public_versions[1]] baseline = probe.inventory(bucket) fleet.record('J02-grace-post-delete-ACK-S3', {'inventory': baseline, **verify_seed(expected, baseline)}) idle_fleet(fleet, binding); fleet.stop() diff --git a/scripts/validation_heal/outcome_recovery.py b/scripts/validation_heal/outcome_recovery.py new file mode 100644 index 00000000..0cc32a1c --- /dev/null +++ b/scripts/validation_heal/outcome_recovery.py @@ -0,0 +1,147 @@ +"""Preserve the fully acknowledged J02 fixture that failed before grace replay.""" + +import hashlib +import json +import re +import uuid + +try: + from .healthy import physical_content, validate_topology + from .heal_probe import require + from .native_regressions import CANDIDATE + from .orphan import KEY, references + from .shards import files, map_target + from .versions import public_version_id +except ImportError: + from healthy import physical_content, validate_topology + from heal_probe import require + from native_regressions import CANDIDATE + from orphan import KEY, references + from shards import files, map_target + from versions import public_version_id + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def acknowledged_grace_plan(artifact, failed_run, candidate): + case = 'heal.outcome-counters-and-error-truth' + paths = list(artifact.glob('*/run/*/cases/*.json')) + require(len(paths) == 1 and paths[0].name == case + '.json', 'J02 recovery requires one unmixed original result') + result = json.loads(paths[0].read_text()); provenance = result.get('provenance') or {} + require(result.get('case_id') == case and result.get('verdict') == result.get('cleanup_status') == 'FAIL' + and candidate.get('source_sha') == provenance.get('source_commit') == CANDIDATE + and candidate.get('package_sha256') == provenance.get('candidate_sha256'), 'J02 recovery candidate/result differs') + evidence = paths[0].parents[1] / 'evidence' / case + summary = json.loads((evidence / 'healthy-summary.json').read_text()); rows = summary.get('results') or [] + require(summary.get('caseId') == case and summary.get('cleanup') == 'FAIL' and summary.get('complete') is False + and summary.get('canonicalCasePassClaimed') is False and len(rows) == 3 + and [(row.get('topology'), row.get('fixture'), row.get('branch')) for row in rows] == [ + ('H1', 'DB', branch) for branch in ('mixed-healthy-repaired-failed', 'dry-run', 'deferred')] + and all(row.get('cleanup') == 'PASS' for row in rows[:-1]) and rows[-1].get('cleanup') == 'FAIL' + and rows[-1].get('verdict') == 'FAIL' and rows[-1].get('failure') == { + 'type': 'FixtureStateMismatch', 'message': 'baseline content/version/marker differs from deterministic fixture ACK'}, + 'J02 recovery supports only its pre-replay public-manifest refusal') + folder = evidence / 'DB/H1/deferred' + def one(label): + found = [p for p in folder.glob('*-' + label + '.json') + if re.fullmatch(r'[0-9]{5}-' + re.escape(label) + r'\.json', p.name)] + require(len(found) == 1, 'J02 recovery lacks exact ' + label) + return json.loads(found[0].read_text()) + binding, failure, reservation = one('binding'), one('failure-scene-before-fault-rescue'), one('reserved') + run_id, bucket = binding.get('runId', ''), binding.get('bucketPrefix', '') + require(re.fullmatch(rf'gh-{failed_run}-[1-9][0-9]*-h1', run_id) is not None + and re.fullmatch(r'rv-heal-[0-9a-f]{16}', bucket) is not None + and binding.get('harnessCommit') == provenance.get('testing_commit') + and binding.get('candidateCommit') == candidate['source_sha'] + and binding.get('candidatePackageSha256') == candidate['package_sha256'] + and binding.get('candidateBinarySha256') == provenance.get('binary_sha256') + and set(failure) == {'vm000'} and reservation.get('node') == 'vm000' and reservation.get('reserved') is True, + 'J02 archived run/helper/node binding differs') + owner = reservation.get('owner') or {} + require(owner.get('runId') == run_id and owner.get('bucket') == bucket and owner.get('runtimeProfile') == 'outcome-grace' + and owner.get('binarySha256') == provenance['binary_sha256'] and len(owner.get('mounts', [])) == 4, + 'J02 original reservation differs') + require(one('retain-unknown-seed-receipt') == dict(state='DIRTY', namespacesRetained=True, + request=dict(operation='original-version-specific-grace-source-procedure', bucket=bucket)), 'J02 pending boundary differs') + require(not any('grace-' in path.name and not path.name.endswith('-J02-grace-ACK-server-age.json') + for path in folder.glob('*.json')), + 'J02 grace advanced beyond original version PUTs') + wanted = list(one('complete-ACK-manifest')['expected']) + for name, operations in (('fixture', ['create_bucket'] + ['put_object'] * 3), + ('original-grace-fixture', ['put_bucket_versioning', 'put_object', 'put_object'])): + requests = sorted((folder / name).glob('*-request.json')); acks = sorted((folder / name).glob('*-ack.json')) + require(len(requests) == len(acks) == len(operations), 'J02 source ledger is incomplete or contains a later mutation') + for sequence, (request_path, ack_path, operation) in enumerate(zip(requests, acks, operations, strict=True), 1): + request, ack = json.loads(request_path.read_text()), json.loads(ack_path.read_text()) + observed = ack.get('observation') or {}; params = request.get('params') or {} + require(type(request.get('sequence')) is int and request['sequence'] == ack.get('sequence') == sequence + and type(ack.get('sequence')) is int and request.get('runId') == ack.get('runId') == run_id + and ack.get('requestFile') == request_path.name + and ack.get('requestSha256') == hashlib.sha256(request_path.read_bytes()).hexdigest() + and request.get('operation') == observed.get('operation') == operation + and params.get('Bucket') == bucket and params == observed.get('params') + and ack.get('operationCheck') == 'PASS' and observed.get('acknowledged') is True + and observed.get('clientCallInvoked') is True and observed.get('resultUnknown') is False + and observed.get('response', {}).get('ResponseMetadata', {}).get('HTTPStatusCode') == 200, + 'J02 source mutation has an unknown or rebound ACK') + if operation == 'put_object': + require(request.get('payload') == observed.get('payload') and re.fullmatch(r'[0-9a-f]{64}', request['payload']['sha256']), + 'J02 payload ACK differs from its durable request') + if name == 'original-grace-fixture': + version = observed.get('response', {}).get('VersionId') + require(params.get('Key') == KEY and isinstance(version, str) and str(uuid.UUID(version)) == version + and uuid.UUID(version).int != 0, 'J02 original grace PUT lacks a real version') + wanted.append(dict(key=KEY, versionId=version, payload=request['payload'])) + else: + matches = [entry for entry in wanted if entry['key'] == params.get('Key')] + require(len(matches) == 1 and matches[0]['versionId'] == 'null' + and matches[0]['payload'] == request['payload'], 'J02 control manifest differs from original ACK') + require(len(wanted) == 5 and len({(entry['key'], entry['versionId']) for entry in wanted}) == 5, + 'J02 source lacks five distinct acknowledged objects/versions') + validate_topology(failure, 'H1') + files(failure) + require({entry['metadata']['key'] for entry in failure['vm000']['files'] if 'metadata' in entry} + == {entry['key'] for entry in wanted} + and all(any(entry['path'].startswith(bucket + '/' + version['key'] + '/') for version in wanted) + for entry in failure['vm000']['files']), 'J02 physical scene has another key/bucket') + for key in {entry['key'] for entry in wanted}: + expected = {entry['versionId'] for entry in wanted if entry['key'] == key} + require(all({public_version_id(version) for version in values} == expected + for values in references(failure, bucket, key).values()), 'J02 physical version authority differs from ACKs') + for entry in wanted: + target = map_target(failure, bucket, entry['key'], 'data', 1, entry['versionId'], latest_required=False) + require(target['logical']['size'] == entry['payload']['size'] and len(target['cohort']) == 4, + 'J02 ACK lacks its complete original physical cohort') + return [dict(kind='acknowledged-j02-before-replay', runId=run_id, nodes=['vm000'], owners={'vm000': owner}, + before=failure, binarySha256=provenance['binary_sha256'], buckets=[bucket], + originalCaseSha256=hashlib.sha256(paths[0].read_bytes()).hexdigest())] + + +def isolate_acknowledged_grace(fleet, plan, states): + require(len(states) == 1 and states[0].get('node') == 'vm000', 'J02 recovery fleet differs') + state = states[0]; retained = state.get('retained') or {} + if not retained.get('state') and not any(retained.get('roots', [])): + require(state['state']['exists'] and len(state['roots']) == 4 and all(root['exists'] for root in state['roots']), + 'J02 acknowledged fixture lost a root; retain remaining scene') + fleet.record('J02-acknowledged-recovery-stop', fleet.remote('vm000', 'stop')) + current = fleet.record('J02-acknowledged-current-physical', fleet.snapshot('J02-stopped-preserved-scene', quiescent=True)) + require(physical_content(current) == physical_content(plan['before']) + and all(value.get('pid') == 0 and type(value.get('pid')) is int and value.get('quiescent') is True + and value.get('binarySha256') == plan['binarySha256'] for value in current.values()), + 'J02 acknowledged fixture drifted; no isolation or restoration performed') + params = dict(expectedOwner=plan['owners']['vm000'], + expectedOriginalSceneSha256=digest(physical_content(plan['before'])['vm000'])) + audit = fleet.record('J02-acknowledged-archive-preflight', fleet.remote('vm000', 'retained-scene-audit', **params)) + require(audit.get('inspectionOnly') is True and audit.get('mutated') is False + and audit.get('baseline', {}).get('originalSceneSha256') == params['expectedOriginalSceneSha256'], 'J02 archive preflight differs') + isolated = fleet.record('J02-acknowledged-archive-isolation', fleet.remote('vm000', 'retained-scene-isolate', + **params, expectedBaseline=audit['baseline'])) + checked = fleet.record('J02-acknowledged-archive-post-audit', fleet.remote('vm000', 'retained-scene-audit', **params)) + require(all(value.get('activeNamespaceIsolated') is True and value.get('canonicalPassClaimed') is False + and value.get('baseline') == audit['baseline'] for value in (isolated, checked)) + and checked.get('inspectionOnly') is True and checked.get('mutated') is False, 'J02 retained scene isolation drifted') + return dict(runId=plan['runId'], cleanup='ISOLATED', activeNamespaceIsolated=True, backupRestorationUsed=False, + originalSceneRestored=False, originalBusinessFixtureRestored=False, canonicalPassClaimed=False, + originalVerdictRewritten=False, originalCaseSha256=plan['originalCaseSha256']) diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index 682ec746..454be10c 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -3378,7 +3378,8 @@ "unknown/skipped/completed_with_errors不算修复成功，legacy objectsHealed与canonical healed分开解释；attemptFailures不得被隐藏。", "使用独立完整manifest弥补有界outcome窗口，保持不可修对象的正确失败记录；恢复后另任务修复不覆盖原结果。", "grace 分支先用 SDK PUT 建立两个真实版本并验证完整分片；保存一个成员的原对象树后，SDK DELETE 精确历史版本，逐盘验证仅当前版本存在且其逻辑元数据、数据片及控制对象未变。只向原成员回放原对象树；实际候选进程固定 grace=120 秒，不修改元数据年龄或时钟。原任务须在窗口内完成，精确报告旧版 deferred、当前版 verified_healthy 及 retryNotBefore；自然等待过期后原 token 仍不变，新 token 才能清理旧版。清理后先核对全部物理字节等于 DELETE 后权威现场，再跨节点 GET/404 和复查旧 token，最后才做 fixture finalization。错过窗口保留 NOT_OBSERVED；异常保留现场并标 DIRTY；救援仅恢复 DELETE 后权威树，不复活已删除版本。", - "Manual diagnostic entry: explicitly select heal.outcome-counters-and-error-truth through Functional Chain with the frozen candidate and candidate-bound manager/inspector tools. The default catalog policy stays evidence-blocked; even all passing diagnostic branches retain complete=false, canonicalCasePassClaimed=false and a blocking gate verdict until independent acceptance." + "Manual diagnostic entry: explicitly select heal.outcome-counters-and-error-truth through Functional Chain with the frozen candidate and candidate-bound manager/inspector tools. The default catalog policy stays evidence-blocked; even all passing diagnostic branches retain complete=false, canonicalCasePassClaimed=false and a blocking gate verdict until independent acceptance.", + "For a pre-replay public-manifest refusal, recover only after the complete SDK request/ACK ledger proves all writes acknowledged and no DELETE or replay occurred. Bind all four original physical members and the reserved owner, stop the exact candidate, then preserve all roots and state using durable same-filesystem non-overwriting rename. Partial archive retries require the same durable inventory; missing roots or drift retain DIRTY. Isolation is not product repair or restoration and never rewrites the failed case." ], "commands": [ "SEED_DB", diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index d74bedf0..642f2cc5 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -13,10 +13,12 @@ from .healthy import Fleet, physical_content from .heal_probe import require, save from .interrupted_recovery import checkpoint_plan, restore_and_isolate + from .outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace except ImportError: from healthy import Fleet, physical_content from heal_probe import require, save from interrupted_recovery import checkpoint_plan, restore_and_isolate + from outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace def quota_recovery_plan(artifact, failed_run, candidate): @@ -141,6 +143,8 @@ def network_recovery_plan(artifact, failed_run, candidate): def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") + if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/cases/*.json")): + return acknowledged_grace_plan(artifact, failed_run, candidate) if list(artifact.glob("*/run/heal.checkpoint-and-mrf-corruption/cases/*.json")): results = list(artifact.glob("*/run/heal.checkpoint-and-mrf-corruption/cases/*.json")) if any(json.loads(path.read_text()).get("verdict") == "RUNNING" for path in results): @@ -640,6 +644,9 @@ def main(): has_retained = any(item.get("retained", {}).get("state") or any(item.get("retained", {}).get("roots", [])) for item in states) all_absent = all(not item["state"]["exists"] and not any(root["exists"] for root in item["roots"]) for item in states) + if plan.get("kind") == "acknowledged-j02-before-replay": + complete.append({"failedRunId": args.failed_run_id, **isolate_acknowledged_grace(fleet, plan, states)}) + continue if plan.get("kind") == "interrupted-checkpoint-before-admission": complete.append({"failedRunId": args.failed_run_id, **restore_and_isolate(fleet, plan, states, recovery_scene_matches)}) diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py index a1b01206..e8f66de1 100644 --- a/scripts/validation_heal/retained_scene.py +++ b/scripts/validation_heal/retained_scene.py @@ -155,7 +155,7 @@ def inspect(request, node): require(isinstance(expected, dict) and expected.get("runId") == request["runId"] and expected.get("unit") == node["unit"](request["runId"]) and expected.get("hostname") == node["socket"].gethostname() - and expected.get("runtimeProfile") in {"mrf-crash", "root-restart"}, "retained scene archived owner differs") + and expected.get("runtimeProfile") in {"mrf-crash", "root-restart", "outcome-grace"}, "retained scene archived owner differs") current = node["mounts"]() require(len(current) == len(expected.get("mounts", [])) == 4 and all(all(actual[key] == old[key] for key in ("target", "source", "fstype", "uuid", "maj:min")) @@ -196,7 +196,7 @@ def inspect(request, node): baseline = {"runId": request["runId"], "ownerSha256": sha(expected), "roots": witnesses[:4], "state": witnesses[4]} intent = retained_state.parent / (request["runId"] + ".intent.json") node["checked"](intent, state.parent) - if expected["runtimeProfile"] == "root-restart": + if expected["runtimeProfile"] in {"root-restart", "outcome-grace"}: original_sha = request.get("expectedOriginalSceneSha256", "") require(re.fullmatch(r"[0-9a-f]{64}", original_sha) is not None, "restored fixture requires its original physical scene digest") @@ -218,7 +218,7 @@ def inspect(request, node): def run(request, node): fields = {"runId", "action", "expectedOwner"} - if request.get("expectedOwner", {}).get("runtimeProfile") == "root-restart": + if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace"}: fields.add("expectedOriginalSceneSha256") if request["action"] == "retained-scene-isolate": fields.add("expectedBaseline") diff --git a/test_validation_heal_outcome_grace.py b/test_validation_heal_outcome_grace.py index 9952c184..53515949 100644 --- a/test_validation_heal_outcome_grace.py +++ b/test_validation_heal_outcome_grace.py @@ -1,6 +1,7 @@ """Exercise real grace oracles and source-procedure ordering without a live fleet.""" import copy +import hashlib import json import os from datetime import datetime, timezone @@ -305,6 +306,7 @@ def execute(self, *, nodes=1, mode='normal'): initial, authority, injected, target, directory = scene(nodes) clock = types.SimpleNamespace(now=1000.0, elapsed=0.0) events, records, admissions = [], {}, [] + ack_entries = [] def sleep(seconds): self.assertLessEqual(seconds, 1) if mode != 'stalled-clock': clock.now += seconds @@ -333,7 +335,7 @@ def snapshot(self, label, **kwargs): return copy.deepcopy(initial if self.phase == 'initial' else authority if self.phase in ('authority', 'clean') else injected) fleet = Fleet() class Probe: - def inventory(self, *args, **kwargs): return {'inventory': True} + def inventory(self, *args, **kwargs): return {'bucket': BUCKET, 'entries': copy.deepcopy(ack_entries)} def s3(self, operation, params, body=None): self_outer.assertEqual(operation, 'head_object') return dict(acknowledged=True, resultUnknown=False, response=dict(VersionId=params['VersionId'], @@ -343,9 +345,15 @@ class Recorder: def __init__(self, *args): self.versions = iter((OLD, CURRENT)) def call(self, operation, params, body=None): events.append((operation, clock.now)) - if operation == 'put_object': return dict(VersionId=next(self.versions), ETag='etag') + if operation == 'put_object': + version = next(self.versions) + for entry in ack_entries: entry['isLatest'] = False + ack_entries.append(dict(key=grace.KEY, versionId=version, isLatest=True, deleteMarker=False, + payload=dict(size=len(body), sha256=hashlib.sha256(body).hexdigest()))) + return dict(VersionId=version, ETag='etag') if operation == 'delete_object': fleet.phase = 'authority' + ack_entries[:] = [entry for entry in ack_entries if entry['versionId'] != OLD] return dict(VersionId=OLD, ResponseMetadata={'HTTPStatusCode': 204}) return {} tasks = [] @@ -369,7 +377,7 @@ def verify_public(*args): def recover(*args): events.append(('rescue', clock.now)); fleet.phase = 'authority'; fleet.orphan_faults.clear() with tempfile.TemporaryDirectory() as temporary, ExitStack() as stack: - for name, value in {'time': timer, 'Recorder': Recorder, 'verify_seed': lambda *a: {}, + for name, value in {'time': timer, 'Recorder': Recorder, 'run_task': task, 'idle_fleet': lambda *a: None, 'verify_proof': lambda *a: {}, 'map_target': lambda *a, **kw: {'dataDir': directory}}.items(): stack.enter_context(patch.object(grace, name, value)) @@ -412,4 +420,147 @@ def test_reused_token_false_cleanup_and_partial_truth_never_rescue_or_pass(self) self.assertEqual(len(self.fleet.orphan_faults), 1) + + +class GraceRecoveryTest(unittest.TestCase): + def fixture(self, root): + from scripts.validation_heal import outcome_recovery as recovery + keys = ['objects/0000.bin', 'objects/0001.bin', 'objects/0002.bin', grace.KEY] + original, bucket, _ = versioned_scene(keys) + value = original.pop('node1'); original['vm000'] = value + expected = [] + for key in keys: + ids = (OLD, CURRENT) if key == grace.KEY else ('null',) + expected += [dict(key=key, versionId=version, isLatest=index == len(ids) - 1, + deleteMarker=False, payload=dict(size=100, sha256='d' * 64)) for index, version in enumerate(ids)] + for entry in value['files']: + if entry['path'] == bucket + '/' + key + '/xl.meta': + entry['metadata']['versions'] = entry['metadata']['versions'][:len(ids)] + for index, version in enumerate(entry['metadata']['versions']): + version.update(versionId=ids[index], isLatest=index == len(ids) - 1) + candidate = dict(source_sha=recovery.CANDIDATE, package_sha256='a' * 64) + run_id = 'gh-12345-1-h1'; case = outcome.CASE_ID + folder = root / 'artifact/run' / case + evidence = folder / 'evidence' / case; branch = evidence / 'DB/H1/deferred' + branch.mkdir(parents=True); (folder / 'cases').mkdir() + def save(path, data): path.write_text(json.dumps(data)) + save(folder / 'cases' / (case + '.json'), dict(case_id=case, verdict='FAIL', cleanup_status='FAIL', + provenance=dict(source_commit=candidate['source_sha'], candidate_sha256='a' * 64, + testing_commit='b' * 40, binary_sha256='c' * 64))) + rows = [dict(topology='H1', fixture='DB', branch=name, verdict='FAIL', cleanup='PASS') + for name in ('mixed-healthy-repaired-failed', 'dry-run')] + rows.append(dict(topology='H1', fixture='DB', branch='deferred', verdict='FAIL', cleanup='FAIL', + failure=dict(type='FixtureStateMismatch', message='baseline content/version/marker differs from deterministic fixture ACK'))) + save(evidence / 'healthy-summary.json', dict(caseId=case, results=rows, cleanup='FAIL', complete=False, canonicalCasePassClaimed=False)) + owner = dict(runId=run_id, bucket=bucket, runtimeProfile='outcome-grace', binarySha256='c' * 64, mounts=[{}] * 4) + for index, (label, data) in enumerate((('binding', dict(runId=run_id, bucketPrefix=bucket, + harnessCommit='b' * 40, candidateCommit=candidate['source_sha'], candidatePackageSha256='a' * 64, + candidateBinarySha256='c' * 64)), ('failure-scene-before-fault-rescue', original), + ('reserved', dict(node='vm000', reserved=True, owner=owner)), + ('retain-unknown-seed-receipt', dict(state='DIRTY', namespacesRetained=True, + request=dict(operation='original-version-specific-grace-source-procedure', bucket=bucket))), + ('complete-ACK-manifest', dict(expected=expected[:3]))), 1): + save(branch / (f'{index:05d}-' + label + '.json'), data) + for name, operations in (('fixture', ['create_bucket'] + ['put_object'] * 3), + ('original-grace-fixture', ['put_bucket_versioning', 'put_object', 'put_object'])): + ledger = branch / name; ledger.mkdir() + for sequence, operation in enumerate(operations, 1): + params = {'Bucket': bucket}; payload = None; response = {'ResponseMetadata': {'HTTPStatusCode': 200}} + if operation == 'put_object': + entry = expected[sequence - 2 if name == 'fixture' else sequence + 1] + params['Key'] = entry['key']; payload = entry['payload']; response['VersionId'] = entry['versionId'] + request = dict(sequence=sequence, runId=run_id, operation=operation, params=params) + if payload: request['payload'] = payload + path = ledger / f'{sequence:06d}-request.json'; save(path, request) + save(ledger / f'{sequence:06d}-ack.json', dict(sequence=sequence, runId=run_id, + requestFile=path.name, requestSha256=hashlib.sha256(path.read_bytes()).hexdigest(), operationCheck='PASS', + observation=dict(operation=operation, params=params, payload=payload, response=response, + acknowledged=True, clientCallInvoked=True, resultUnknown=False))) + return candidate, branch + + def test_acknowledged_pre_replay_plan_and_poisoned_ledger_boundaries(self): + from scripts.validation_heal import outcome_recovery as recovery + for poison in (None, 'unknown', 'request-digest', 'extra-delete', 'later-grace', 'foreign-key', 'lost-root', 'another-retained-branch'): + with self.subTest(poison=poison), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary); candidate, branch = self.fixture(root) + path = branch / 'original-grace-fixture/000003-ack.json' + ack = json.loads(path.read_text()) + if poison == 'unknown': ack['observation']['resultUnknown'] = True + elif poison == 'request-digest': ack['requestSha256'] = 'e' * 64 + elif poison == 'foreign-key': ack['observation']['params']['Key'] = 'foreign' + path.write_text(json.dumps(ack)) + if poison == 'extra-delete': (path.parent / '000004-request.json').write_text('{}') + elif poison == 'later-grace': (branch / '00100-J02-grace-single-member-old-replay.json').write_text('{}') + elif poison == 'lost-root': + path = branch / '00002-failure-scene-before-fault-rescue.json'; scene = json.loads(path.read_text()) + scene['vm000']['formats'].pop(); path.write_text(json.dumps(scene)) + elif poison == 'another-retained-branch': + path = branch.parents[2] / 'healthy-summary.json'; summary = json.loads(path.read_text()) + summary['results'][0]['cleanup'] = 'FAIL'; path.write_text(json.dumps(summary)) + if poison: + with self.assertRaises(ValueError): recovery.acknowledged_grace_plan(root, 12345, candidate) + else: + plan = recovery.acknowledged_grace_plan(root, 12345, candidate)[0] + self.assertEqual(plan['nodes'], ['vm000']) + self.assertEqual(plan['owners']['vm000']['runtimeProfile'], 'outcome-grace') + + def test_real_descriptor_archive_requires_complete_scene_and_retries_after_every_move(self): + from scripts.validation_heal import retained_scene as retained, outcome_recovery as recovery + import test_validation_heal_retained_scene as retained_tests + helper = retained_tests.RetainedSceneTests() + for lost_after in range(1, 6): + with self.subTest(lost_after=lost_after), tempfile.TemporaryDirectory() as temporary: + state, roots, node, request = helper.scene(temporary) + roots[0].mkdir(parents=True); owner = request['expectedOwner']; owner['runtimeProfile'] = 'outcome-grace' + for folder in (*roots, state): (folder / 'owner.json').write_text(json.dumps(owner)) + snapshot = dict(files=[dict(path='original', sha256='d' * 64)], formats=[{}] * 4, + pid=0, quiescent=True, binarySha256=owner['binarySha256']) + node['snapshot'] = lambda value: snapshot + request['expectedOriginalSceneSha256'] = recovery.digest({key: snapshot[key] for key in ('files', 'formats')}) + with helper.root_stats(), patch.object(retained, 'rename_new', side_effect=helper.local_rename): + baseline = retained.run(request, node)['baseline'] + change = {**request, 'action': 'retained-scene-isolate', 'expectedBaseline': baseline} + count = 0 + def move(source, target): + nonlocal count + helper.local_rename(source, target) + if target.is_dir(): + count += 1 + if count == lost_after: raise RuntimeError('lost rename ACK') + with patch.object(retained, 'rename_new', side_effect=move), self.assertRaises(RuntimeError): retained.run(change, node) + checked = retained.run(change, node) + self.assertEqual(checked['locations'], ['retained'] * 5) + self.assertEqual(checked['baseline'], baseline) + self.assertFalse(checked['canonicalPassClaimed']) + self.assertFalse(checked['backupRestorationUsed']) + self.assertFalse(checked['credentialContentsRead']) + with self.assertRaises(ValueError): retained.run({**request, 'expectedOriginalSceneSha256': 'e' * 64}, node) + def remote(host, action, **params): + return retained.run(dict(runId=request['runId'], action=action, **params), node) + fleet = types.SimpleNamespace(record=lambda label, value: value, remote=remote) + plan = dict(runId=request['runId'], before={'vm000': snapshot}, owners={'vm000': owner}, + originalCaseSha256='d' * 64) + result = recovery.isolate_acknowledged_grace(fleet, plan, [dict(node='vm000', + retained={'state': True, 'roots': [True] * 4})]) + self.assertEqual(result['cleanup'], 'ISOLATED') + self.assertFalse(result['originalBusinessFixtureRestored']) + self.assertFalse(result['originalVerdictRewritten']) + + def test_drifted_acknowledged_scene_never_starts_archive(self): + from scripts.validation_heal import outcome_recovery as recovery + for poison in ('bytes', 'missing-root', 'live-pid'): + with self.subTest(poison=poison): + before = {'vm000': dict(files=[], formats=[], pid=0, quiescent=True, binarySha256='a' * 64)} + current = copy.deepcopy(before) + state = dict(node='vm000', state={'exists': True}, roots=[{'exists': True}] * 4, retained={}) + if poison == 'bytes': current['vm000']['files'] = [{}] + elif poison == 'live-pid': current['vm000']['pid'] = 123 + else: state['roots'] = [{'exists': False}] * 4 + fleet = types.SimpleNamespace(record=lambda label, value: value, + remote=unittest.mock.Mock(), snapshot=lambda *a, **kw: current) + plan = dict(before=before, binarySha256='a' * 64) + with self.assertRaises(ValueError): recovery.isolate_acknowledged_grace(fleet, plan, [state]) + self.assertFalse(any(call.args[1].startswith('retained-scene-') for call in fleet.remote.call_args_list)) + + if __name__ == '__main__': unittest.main() From 2f5ae1e22572bc42d80448bc6515aca8543847f8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 22:08:46 +0800 Subject: [PATCH 211/343] fix(validation): honor omitted false public truncation flag --- scripts/validation_heal/outcome.py | 10 ++++---- scripts/validation_heal/outcome_grace.py | 2 +- test_validation_heal_outcome.py | 31 ++++++++++++++++++++++++ test_validation_heal_outcome_grace.py | 11 +++++++++ 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/scripts/validation_heal/outcome.py b/scripts/validation_heal/outcome.py index 8a370128..ec95ae99 100644 --- a/scripts/validation_heal/outcome.py +++ b/scripts/validation_heal/outcome.py @@ -63,7 +63,7 @@ def reconcile(status, bucket, expected, options, incarnation, *, attempt_failure "J02 legacy negative requires a separate protected control") failed = sum(entry["state"] == "failed" for entry in expected) require(status.get("summary") == ("stopped" if failed else "finished") - and status.get("settings") == options and status.get("truncated") is False, + and status.get("settings") == options and status.get("truncated", False) is False, "J02 task has a wrong terminal, options or incomplete public result") outcome = status.get("outcome") or {} require(outcome.get("execution", {}).get("state") == ("completed_with_errors" if failed else "completed") @@ -235,7 +235,7 @@ def readonly_failure(status, bucket, key, options): """Freeze the source operation card's unbound permanent key-level failure.""" outcome = status.get("outcome") or {} require(status.get("summary") == "stopped" and status.get("settings") == options - and status.get("truncated") is False and outcome.get("execution", {}).get("state") == "completed_with_errors" + and status.get("truncated", False) is False and outcome.get("execution", {}).get("state") == "completed_with_errors" and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is False, "J02 readonly task lost its complete permanent terminal failure") objects = outcome.get("objects", []) @@ -259,7 +259,7 @@ def unchanged_token(probe, fleet, admission, status, label): response = fleet.record(label, probe.admin(admission["path"], [("clientToken", admission["clientToken"])], b"")) require(response.get("httpStatus") == 200 and response.get("response", {}).get("summary") == status["summary"] and response["response"].get("settings") == status["settings"] - and response["response"].get("truncated") is False + and response["response"].get("truncated", False) is False and canonical(response["response"].get("outcome")) == canonical(status["outcome"]), "J02 later work replaced the original readonly failure responsibility") @@ -339,7 +339,7 @@ def opaque_metadata_outcome(status, bucket, key, options, incarnation): """An unobserved Unknown must remain unobserved, never successful repair.""" require(isinstance(incarnation, str) and bool(incarnation) and status.get("summary") in {"finished", "stopped"} and status.get("settings") == options - and status.get("truncated") is False, "J02 opaque metadata task did not terminate with its exact options") + and status.get("truncated", False) is False, "J02 opaque metadata task did not terminate with its exact options") outcome = status.get("outcome") or {} counters = outcome.get("counters") or {} names = ("processed", "healed", "unchanged", "skipped", "failed", "unknown", "attemptFailures") @@ -581,7 +581,7 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): [("clientToken", admissions[0]["clientToken"])], b"")) require(repeated.get("httpStatus") == 200 and repeated.get("response", {}).get("summary") == status["summary"] and repeated["response"].get("settings") == options - and repeated["response"].get("truncated") is False + and repeated["response"].get("truncated", False) is False and canonical(repeated["response"].get("outcome")) == canonical(status["outcome"]), "J02 rescue overwrote the original failed/deferred/dry-run responsibility") unobserved = any(result.get(name, {}).get("verdict") == "NOT_OBSERVED" diff --git a/scripts/validation_heal/outcome_grace.py b/scripts/validation_heal/outcome_grace.py index f98584c9..5dd1bf29 100644 --- a/scripts/validation_heal/outcome_grace.py +++ b/scripts/validation_heal/outcome_grace.py @@ -42,7 +42,7 @@ def grace_outcome(status, bucket, old, current, options, incarnation, observed_a and old['versionId'] != current['versionId'] and isinstance(incarnation, str) and incarnation, 'J02 grace lacks distinct ACK versions, original incarnation or a real observation clock') outcome = status.get('outcome') or {} - require(status.get('summary') == 'finished' and status.get('settings') == options and status.get('truncated') is False + require(status.get('summary') == 'finished' and status.get('settings') == options and status.get('truncated', False) is False and outcome.get('execution', {}).get('state') == 'completed' and outcome.get('coverage') == 'complete' and outcome.get('objectsTruncated') is False, 'J02 grace task lacks complete original terminal truth') wanted = dict(processed=2, healed=1 if cleanup else 0, unchanged=1, skipped=0 if cleanup else 1, diff --git a/test_validation_heal_outcome.py b/test_validation_heal_outcome.py index d2235e4f..bc4d3c6b 100644 --- a/test_validation_heal_outcome.py +++ b/test_validation_heal_outcome.py @@ -677,4 +677,35 @@ def task(probe, fleet, scope, mode, active, **params): self.assertLess(events.index('original-opaque-restored-original-bytes'), events.index('original-opaque-restored-full-S3')) +class OmittedFalseTruncationTests(unittest.TestCase): + def test_original_wire_omission_does_not_relax_canonical_manifest_or_counter_checks(self): + mixed = MixedOutcomeTest(); expected, status = mixed.mixed() + readonly = ReadonlySourceFlowTests(); opaque = OriginalOpaqueMetadataTests() + oracles = [(status, lambda value: mixed.check(expected, value)), + (readonly.failure(), lambda value: outcome.readonly_failure(value, readonly.bucket, readonly.key, readonly.options)), + (opaque.status(), lambda value: outcome.opaque_metadata_outcome(value, opaque.bucket, opaque.key, opaque.options, 'original-anchor'))] + for source, check in oracles: + source.pop('truncated'); check(source) + for value in (None, True, 0, 'false'): + with self.subTest(value=value), self.assertRaises(ValueError): check({**source, 'truncated': value}) + for field, value in (('objectsTruncated', True), ('coverage', 'partial')): + changed = copy.deepcopy(source); changed['outcome'][field] = value + with self.assertRaises(ValueError): check(changed) + changed = copy.deepcopy(source); changed['outcome'].pop('objectsTruncated') + with self.assertRaises(ValueError): check(changed) + changed = copy.deepcopy(source); changed['outcome']['objects'].pop() + with self.assertRaises(ValueError): check(changed) + + def test_repeated_original_token_accepts_only_omitted_false_or_explicit_false(self): + status = ReadonlySourceFlowTests().failure(); status.pop('truncated') + response = dict(httpStatus=200, response=copy.deepcopy(status)) + probe = types.SimpleNamespace(admin=lambda *args: response) + fleet = types.SimpleNamespace(record=lambda label, value: value) + admission = dict(path='/heal', clientToken='original') + outcome.unchanged_token(probe, fleet, admission, status, 'original') + for value in (None, True): + response['response']['truncated'] = value + with self.assertRaises(ValueError): outcome.unchanged_token(probe, fleet, admission, status, 'changed') + + if __name__ == "__main__": unittest.main() diff --git a/test_validation_heal_outcome_grace.py b/test_validation_heal_outcome_grace.py index 53515949..f501f9fe 100644 --- a/test_validation_heal_outcome_grace.py +++ b/test_validation_heal_outcome_grace.py @@ -86,6 +86,17 @@ def test_exact_future_deferral_and_distinct_post_expiry_repair(self): with self.assertRaises(ValueError): self.check(terminal(cleanup=True), 1119.999, True) with self.assertRaises(ValueError): self.check(terminal(), 1120) + def test_source_omits_false_public_truncation_without_losing_canonical_coverage(self): + status = terminal(); status.pop('truncated') + self.assertEqual(self.check(status)['retryNotBeforeEpoch'], 1120) + for value in (None, True, 0, 'false'): + with self.subTest(value=value), self.assertRaises(ValueError): self.check({**status, 'truncated': value}) + for value in (None, True): + changed = copy.deepcopy(status) + if value is None: changed['outcome'].pop('objectsTruncated') + else: changed['outcome']['objectsTruncated'] = value + with self.assertRaises(ValueError): self.check(changed) + def test_wrong_truth_and_legacy_counts_cannot_replace_original_responsibility(self): for path, values in ( (('summary',), ['running', 'stopped']), (('truncated',), [True, None]), From efa7f8c4a9c7a66bc2a63e87f93c7f61d6182b22 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 22:18:05 +0800 Subject: [PATCH 212/343] test(validation): bind manager prerequisites to corrected candidate --- scripts/validation_heal/manager_native.py | 14 +++++++++++--- test_validation_heal_manager_native.py | 12 +++++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/scripts/validation_heal/manager_native.py b/scripts/validation_heal/manager_native.py index d45ad1d5..6f474bc4 100644 --- a/scripts/validation_heal/manager_native.py +++ b/scripts/validation_heal/manager_native.py @@ -47,6 +47,13 @@ "crates/heal/src/heal/erasure_healer/resume_loop_tests/admin_outcome.rs": "f2251a9ab5f2df10a2299cd23bb3231615d946ce95a104002088f06235dde621", "crates/heal/src/heal/storage.rs": "52ce6edff5b64b69a0d0c2a425b980984cf927fab79139e33d5ff9c7ad98efe9", } +# Native prerequisites only; fleet executors retain their own candidate gates. +REVIEWED_CANDIDATES["1c7fc818dd858247c45b39542abd8291b1e6e31b"] = { + "sourceTree": "9713c550ce148e1e32004f166a19e3a67eb83434", + "cargoLockSha256": "2589b3f5afc54f94419a3e2bfd537c11fae7ae3ee2d2a73ba31c72ee8d8a62be", + "sourceFiles": {**SOURCE_FILES, + "crates/heal/src/heal/storage.rs": "0e27cc1fbb7b732d211e744f385223bee23844c95ba86918e1765b546c197096"}, +} RESUME = "heal::resume::tests::" SNAPSHOT = "heal::mrf_queue::snapshot::tests::" ROOT = "heal::manager::tests::root_recovery::" @@ -115,7 +122,7 @@ def verify_manifest(directory): native.require(manifest.get("schemaVersion") == 1 and contract is not None and manifest.get("candidateSourceTree") == contract["sourceTree"] and manifest.get("candidateCargoLockSha256") == contract["cargoLockSha256"] - and manifest.get("sourceFiles") == SOURCE_FILES and manifest.get("collectorSources") == collectors() + and manifest.get("sourceFiles") == contract.get("sourceFiles", SOURCE_FILES) and manifest.get("collectorSources") == collectors() and manifest.get("regressions") == REGRESSIONS and manifest.get("sourceUnmodified") is True and manifest.get("optimized") is True and manifest.get("buildEnvironment") == BUILD_ENV and manifest.get("binaryName") == "heal-manager-native-tests" and manifest.get("fleetPassClaimed") is False, @@ -132,7 +139,8 @@ def build(source, commit, out): ["git", "-C", str(source), "rev-parse", "HEAD"], text=True).strip() == commit, "manager native tests require the exact reviewed candidate") native.require(subprocess.check_output(["git", "-C", str(source), "rev-parse", "HEAD^{tree}"], text=True).strip() == contract["sourceTree"] - and native.sha(source / "Cargo.lock") == contract["cargoLockSha256"] and all(native.sha(source / path) == digest for path, digest in SOURCE_FILES.items()), + and native.sha(source / "Cargo.lock") == contract["cargoLockSha256"] + and all(native.sha(source / path) == digest for path, digest in contract.get("sourceFiles", SOURCE_FILES).items()), "manager candidate source/tree/lock differs") for args in (["diff", "--quiet"], ["diff", "--cached", "--quiet"]): native.require(subprocess.run(["git", "-C", str(source), *args]).returncode == 0, "candidate source is modified") @@ -141,7 +149,7 @@ def build(source, commit, out): native.require(not out.exists() and shutil.disk_usage(source).free >= 12 * 1024**3, "fresh output and 12 GiB build space required") out.mkdir(parents=True) request = {"schemaVersion": 1, "candidateCommit": commit, "candidateSourceTree": contract["sourceTree"], - "candidateCargoLockSha256": contract["cargoLockSha256"], "sourceFiles": SOURCE_FILES, "collectorSources": collectors(), + "candidateCargoLockSha256": contract["cargoLockSha256"], "sourceFiles": contract.get("sourceFiles", SOURCE_FILES), "collectorSources": collectors(), "regressions": REGRESSIONS, "testingCommit": os.environ.get("AUTO_TESTING_SHA"), "runId": os.environ.get("GITHUB_RUN_ID"), "runAttempt": os.environ.get("GITHUB_RUN_ATTEMPT"), "fleetPassClaimed": False, "sourceUnmodified": True, "optimized": True, "buildEnvironment": BUILD_ENV} diff --git a/test_validation_heal_manager_native.py b/test_validation_heal_manager_native.py index 96170f4c..4e0087cc 100644 --- a/test_validation_heal_manager_native.py +++ b/test_validation_heal_manager_native.py @@ -21,7 +21,7 @@ def fixture(self, root, candidate=suite.native.CANDIDATE): contract = suite.REVIEWED_CANDIDATES[candidate] manifest = {"schemaVersion": 1, "candidateCommit": candidate, "candidateSourceTree": contract["sourceTree"], "candidateCargoLockSha256": contract["cargoLockSha256"], - "sourceFiles": suite.SOURCE_FILES, "collectorSources": suite.collectors(), + "sourceFiles": contract.get("sourceFiles", suite.SOURCE_FILES), "collectorSources": suite.collectors(), "regressions": suite.REGRESSIONS, "sourceUnmodified": True, "optimized": True, "buildEnvironment": suite.BUILD_ENV, "binaryName": binary.name, "binarySha256": digest, "fleetPassClaimed": False, "runId": "12345", "runAttempt": "1", "testingCommit": helper} @@ -74,6 +74,16 @@ def test_old_tree_cannot_be_relabelled_as_new_candidate(self): with self.assertRaisesRegex(ValueError, "reviewed source"): self.verify(fixture) + def test_b07_candidate_requires_its_actual_storage_source_and_lock(self): + candidate = '1c7fc818dd858247c45b39542abd8291b1e6e31b' + for field, old in (('sourceFiles', suite.SOURCE_FILES), ('candidateCargoLockSha256', suite.LOCK_SHA)): + with self.subTest(field=field), tempfile.TemporaryDirectory() as temporary: + fixture = self.fixture(Path(temporary), candidate) + path = fixture[0] / 'manager-native.json'; manifest = json.loads(path.read_text()) + self.assertNotEqual(manifest[field], old) + manifest[field] = old; path.write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, 'reviewed source'): self.verify(fixture) + def test_old_result_rows_cannot_certify_new_candidate_manifest(self): newer = next(commit for commit in suite.REVIEWED_CANDIDATES if commit != suite.native.CANDIDATE) with tempfile.TemporaryDirectory() as temp: From 7f1caad3a48f80549fdd5c2f236f96ff1bde8611 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 15 Oct 2026 23:15:10 +0800 Subject: [PATCH 213/343] feat(validation): admit corrected J02 candidate with exact tools --- docs/validation-heal-cases.md | 1 + scripts/validation_heal/outcome.py | 3 ++- scripts/validation_heal/recipes.json | 3 ++- test_validation_heal_outcome.py | 34 ++++++++++++++++++++-------- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index 08b3d450..930f1b1d 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1817,6 +1817,7 @@ Blocked reason (`EVIDENCE_MIXED_OUTCOME_ORCHESTRATION`): 已有 mixed healthy/re 5. grace 分支先用 SDK PUT 建立两个真实版本并验证完整分片；保存一个成员的原对象树后，SDK DELETE 精确历史版本，逐盘验证仅当前版本存在且其逻辑元数据、数据片及控制对象未变。只向原成员回放原对象树；实际候选进程固定 grace=120 秒，不修改元数据年龄或时钟。原任务须在窗口内完成，精确报告旧版 deferred、当前版 verified_healthy 及 retryNotBefore；自然等待过期后原 token 仍不变，新 token 才能清理旧版。清理后先核对全部物理字节等于 DELETE 后权威现场，再跨节点 GET/404 和复查旧 token，最后才做 fixture finalization。错过窗口保留 NOT_OBSERVED；异常保留现场并标 DIRTY；救援仅恢复 DELETE 后权威树，不复活已删除版本。 6. Manual diagnostic entry: explicitly select heal.outcome-counters-and-error-truth through Functional Chain with the frozen candidate and candidate-bound manager/inspector tools. The default catalog policy stays evidence-blocked; even all passing diagnostic branches retain complete=false, canonicalCasePassClaimed=false and a blocking gate verdict until independent acceptance. 7. For a pre-replay public-manifest refusal, recover only after the complete SDK request/ACK ledger proves all writes acknowledged and no DELETE or replay occurred. Bind all four original physical members and the reserved owner, stop the exact candidate, then preserve all roots and state using durable same-filesystem non-overwriting rename. Partial archive retries require the same durable inventory; missing roots or drift retain DIRTY. Isolation is not product repair or restoration and never rewrites the failed case. +8. Manual J02 candidate admission is limited to the original 072b57b37559fbe626aeda8abf61617fdcbdd926 and corrected 1c7fc818dd858247c45b39542abd8291b1e6e31b sources. Each execution requires the matching immutable package and independently audited inspector/manager bundle. A corrected-candidate run is new evidence; it cannot replace an original failure. The full exact-version, physical, timing, retry and responsibility oracles remain unchanged, and all passing diagnostics still require independent canonical acceptance. Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [LOCK_WINDOW](validation-heal-operations.md#LOCK_WINDOW) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) diff --git a/scripts/validation_heal/outcome.py b/scripts/validation_heal/outcome.py index ec95ae99..23c21440 100644 --- a/scripts/validation_heal/outcome.py +++ b/scripts/validation_heal/outcome.py @@ -44,6 +44,7 @@ CASE_ID = "heal.outcome-counters-and-error-truth" +REVIEWED_CANDIDATES = frozenset({CANDIDATE, "1c7fc818dd858247c45b39542abd8291b1e6e31b"}) BRANCHES = ("mixed-healthy-repaired-failed", "dry-run", "deferred", "readonly-terminal-failure", "unknown-negative") SIZE = 5 * 1024**2 + 123 # The reviewed immutable candidate makes one attempt and three bucket-object retries. @@ -622,7 +623,7 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"J02 interrupted by sig args = parser.parse_args() require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "J02 executor omitted a source branch") candidate = json.loads(args.candidate.read_text()) - require(candidate["source_sha"] == CANDIDATE, "J02 retry/deferral contract is not reviewed for this candidate") + require(candidate["source_sha"] in REVIEWED_CANDIDATES, "J02 retry/deferral contract is not reviewed for this candidate") native = verify_manager(os.environ.get("VALIDATION_HEAL_MANAGER_DIRECTORY", ""), candidate, CASE_ID) save(args.evidence / "independent-native-outcome-guard.json", native) require(re.fullmatch(r"[0-9a-f]{64}", args.binary_sha) diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index 454be10c..79831e8e 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -3379,7 +3379,8 @@ "使用独立完整manifest弥补有界outcome窗口，保持不可修对象的正确失败记录；恢复后另任务修复不覆盖原结果。", "grace 分支先用 SDK PUT 建立两个真实版本并验证完整分片；保存一个成员的原对象树后，SDK DELETE 精确历史版本，逐盘验证仅当前版本存在且其逻辑元数据、数据片及控制对象未变。只向原成员回放原对象树；实际候选进程固定 grace=120 秒，不修改元数据年龄或时钟。原任务须在窗口内完成，精确报告旧版 deferred、当前版 verified_healthy 及 retryNotBefore；自然等待过期后原 token 仍不变，新 token 才能清理旧版。清理后先核对全部物理字节等于 DELETE 后权威现场，再跨节点 GET/404 和复查旧 token，最后才做 fixture finalization。错过窗口保留 NOT_OBSERVED；异常保留现场并标 DIRTY；救援仅恢复 DELETE 后权威树，不复活已删除版本。", "Manual diagnostic entry: explicitly select heal.outcome-counters-and-error-truth through Functional Chain with the frozen candidate and candidate-bound manager/inspector tools. The default catalog policy stays evidence-blocked; even all passing diagnostic branches retain complete=false, canonicalCasePassClaimed=false and a blocking gate verdict until independent acceptance.", - "For a pre-replay public-manifest refusal, recover only after the complete SDK request/ACK ledger proves all writes acknowledged and no DELETE or replay occurred. Bind all four original physical members and the reserved owner, stop the exact candidate, then preserve all roots and state using durable same-filesystem non-overwriting rename. Partial archive retries require the same durable inventory; missing roots or drift retain DIRTY. Isolation is not product repair or restoration and never rewrites the failed case." + "For a pre-replay public-manifest refusal, recover only after the complete SDK request/ACK ledger proves all writes acknowledged and no DELETE or replay occurred. Bind all four original physical members and the reserved owner, stop the exact candidate, then preserve all roots and state using durable same-filesystem non-overwriting rename. Partial archive retries require the same durable inventory; missing roots or drift retain DIRTY. Isolation is not product repair or restoration and never rewrites the failed case.", + "Manual J02 candidate admission is limited to the original 072b57b37559fbe626aeda8abf61617fdcbdd926 and corrected 1c7fc818dd858247c45b39542abd8291b1e6e31b sources. Each execution requires the matching immutable package and independently audited inspector/manager bundle. A corrected-candidate run is new evidence; it cannot replace an original failure. The full exact-version, physical, timing, retry and responsibility oracles remain unchanged, and all passing diagnostics still require independent canonical acceptance." ], "commands": [ "SEED_DB", diff --git a/test_validation_heal_outcome.py b/test_validation_heal_outcome.py index bc4d3c6b..252e35f0 100644 --- a/test_validation_heal_outcome.py +++ b/test_validation_heal_outcome.py @@ -175,16 +175,30 @@ def test_lost_rescue_ack_keeps_registered_fault_for_independent_recovery(self): with self.assertRaises(RuntimeError): outcome.rescue(fleet) self.assertEqual(len(fleet.faults), 1) - def test_native_prerequisite_rejection_happens_before_fleet_creation(self): - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp); candidate = root / "candidate.json" - candidate.write_text(json.dumps({"source_sha": outcome.CANDIDATE})) - args = types.SimpleNamespace(candidate=candidate) - with patch.object(outcome.argparse.ArgumentParser, "parse_args", return_value=args), \ - patch.object(outcome, "verify_manager", side_effect=ValueError("wrong immutable evidence")), \ - patch.object(outcome, "Fleet") as fleet, patch.object(outcome.signal, "signal"): - with self.assertRaisesRegex(ValueError, "immutable evidence"): outcome.main() - fleet.assert_not_called() + def test_each_reviewed_candidate_requires_native_evidence_before_fleet_creation(self): + self.assertEqual(outcome.REVIEWED_CANDIDATES, + {outcome.CANDIDATE, "1c7fc818dd858247c45b39542abd8291b1e6e31b"}) + for commit in outcome.REVIEWED_CANDIDATES: + with self.subTest(commit=commit), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); candidate = root / "candidate.json" + manifest = {"source_sha": commit}; candidate.write_text(json.dumps(manifest)) + args = types.SimpleNamespace(candidate=candidate) + with patch.object(outcome.argparse.ArgumentParser, "parse_args", return_value=args), \ + patch.object(outcome, "verify_manager", side_effect=ValueError("wrong immutable evidence")) as native, \ + patch.object(outcome, "Fleet") as fleet, patch.object(outcome.signal, "signal"): + with self.assertRaisesRegex(ValueError, "immutable evidence"): outcome.main() + native.assert_called_once_with(os.environ.get("VALIDATION_HEAL_MANAGER_DIRECTORY", ""), manifest, outcome.CASE_ID) + fleet.assert_not_called() + + def test_unreviewed_candidate_is_rejected_before_native_or_fleet_activity(self): + for commit in ("f" * 40, "1aa9be87d18cf941e5020f19e8622c26ba4b10ac", "main"): + with self.subTest(commit=commit), tempfile.TemporaryDirectory() as tmp: + candidate = Path(tmp) / "candidate.json"; candidate.write_text(json.dumps({"source_sha": commit})) + with patch.object(outcome.argparse.ArgumentParser, "parse_args", return_value=types.SimpleNamespace(candidate=candidate)), \ + patch.object(outcome, "verify_manager") as native, patch.object(outcome, "Fleet") as fleet, \ + patch.object(outcome.signal, "signal"): + with self.assertRaisesRegex(ValueError, "not reviewed"): outcome.main() + native.assert_not_called(); fleet.assert_not_called() def test_unknown_branch_requires_real_executor_for_each_topology_and_fixture(self): with tempfile.TemporaryDirectory() as tmp: From a67801a4f519b65d8b06381f1fbd76712e6b7f01 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 16 Oct 2026 08:43:55 +0800 Subject: [PATCH 214/343] fix(validation): bind opaque metadata faults to each physical disk --- .../validation_heal/interrupted_recovery.py | 6 +- scripts/validation_heal/outcome.py | 2 +- scripts/validation_heal/outcome_recovery.py | 130 +++++++++- .../validation_heal/recover_environment.py | 19 +- scripts/validation_heal/retained_scene.py | 11 +- test_validation_heal_outcome.py | 7 + test_validation_heal_outcome_recovery.py | 230 ++++++++++++++++++ 7 files changed, 390 insertions(+), 15 deletions(-) create mode 100644 test_validation_heal_outcome_recovery.py diff --git a/scripts/validation_heal/interrupted_recovery.py b/scripts/validation_heal/interrupted_recovery.py index 6ebc7754..59f2d730 100644 --- a/scripts/validation_heal/interrupted_recovery.py +++ b/scripts/validation_heal/interrupted_recovery.py @@ -168,7 +168,8 @@ def indexed(scene): def restore_and_isolate(fleet, plan, states, scene_matches): """Preserve all backups and bytes, including when recovery is interrupted.""" - require(len(states) == 4 and {item["node"] for item in states} == set(plan["nodes"]), "interrupted current fleet differs") + require(len(states) == len(plan["nodes"]) and len(states) in {1, 4} + and {item["node"] for item in states} == set(plan["nodes"]), "interrupted current fleet differs") active = [] for item in states: retained = item.get("retained") or {} @@ -179,7 +180,8 @@ def restore_and_isolate(fleet, plan, states, scene_matches): active.append(item["node"]) for node in active: fleet.record("interrupted-recovery-stop", {"node": node, **fleet.remote(node, "stop")}) - current = {node: fleet.remote(node, "snapshot", quiescent=True) for node in active} + current = {node: fleet.remote(node, "snapshot", quiescent=True, + **({"metadataFaults": plan["metadataFaults"][node]} if plan.get("metadataFaults") else {})) for node in active} fleet.record("interrupted-current-physical", current) if active: subset = {**plan, "before": {n: plan["before"][n] for n in active}, diff --git a/scripts/validation_heal/outcome.py b/scripts/validation_heal/outcome.py index 23c21440..e97b2f6e 100644 --- a/scripts/validation_heal/outcome.py +++ b/scripts/validation_heal/outcome.py @@ -412,7 +412,7 @@ def original_opaque_probe(probe, fleet, binding, directory, expected, fixture, i kind="bitflip", offset=0)}) require(receipt.get("kind") == "bitflip" and type(receipt.get("offset")) is int and receipt["offset"] == 0 and isinstance(receipt.get("fault"), dict), "J02 opaque injection changed its original byte-zero fault") - metadata_faults.setdefault(row["node"], {})[row["relativePath"]] = receipt["fault"] + metadata_faults.setdefault(row["node"], {})[f"{row['disk']}:{row['relativePath']}"] = receipt["fault"] damaged = fleet.snapshot("original-opaque-physical-injection", quiescent=True, metadata_faults=metadata_faults) validate_fault(prior, damaged, row, receipt); prior = damaged damaged_files = files(damaged) diff --git a/scripts/validation_heal/outcome_recovery.py b/scripts/validation_heal/outcome_recovery.py index 0cc32a1c..b81f1cdb 100644 --- a/scripts/validation_heal/outcome_recovery.py +++ b/scripts/validation_heal/outcome_recovery.py @@ -1,5 +1,6 @@ -"""Preserve the fully acknowledged J02 fixture that failed before grace replay.""" +"""Restore and retain exactly archived J02 failures before negative task admission.""" +import copy import hashlib import json import re @@ -10,15 +11,19 @@ from .heal_probe import require from .native_regressions import CANDIDATE from .orphan import KEY, references - from .shards import files, map_target - from .versions import public_version_id + from .shards import files, map_target, validate_fault + from .heal_fault import bounded_fault_id + from .heal_workload import verify_seed + from .versions import public_version_id, metadata_target except ImportError: from healthy import physical_content, validate_topology from heal_probe import require from native_regressions import CANDIDATE from orphan import KEY, references - from shards import files, map_target - from versions import public_version_id + from shards import files, map_target, validate_fault + from heal_fault import bounded_fault_id + from heal_workload import verify_seed + from versions import public_version_id, metadata_target def digest(value): @@ -145,3 +150,118 @@ def isolate_acknowledged_grace(fleet, plan, states): return dict(runId=plan['runId'], cleanup='ISOLATED', activeNamespaceIsolated=True, backupRestorationUsed=False, originalSceneRestored=False, originalBusinessFixtureRestored=False, canonicalPassClaimed=False, originalVerdictRewritten=False, originalCaseSha256=plan['originalCaseSha256']) + + +def interrupted_opaque_plan(artifact, failed_run, candidate): + case = 'heal.outcome-counters-and-error-truth' + paths = list(artifact.glob('*/run/*/cases/*.json')) + require(len(paths) == 1 and paths[0].name == case + '.json', 'opaque recovery requires one unmixed J02 result') + raw = paths[0].read_bytes(); result = json.loads(raw); provenance = result.get('provenance') or {} + require(result.get('case_id') == case and result.get('verdict') == result.get('cleanup_status') == 'FAIL' + and candidate.get('source_sha') == provenance.get('source_commit') == CANDIDATE + and candidate.get('package_sha256') == provenance.get('candidate_sha256'), 'opaque recovery candidate differs') + evidence = paths[0].parents[1] / 'evidence' / case + summary = json.loads((evidence / 'healthy-summary.json').read_text()); rows = summary.get('results') or [] + require(summary.get('caseId') == case and summary.get('cleanup') == 'FAIL' and summary.get('complete') is False + and summary.get('canonicalCasePassClaimed') is False and len(rows) == 5 + and [(r.get('topology'), r.get('fixture'), r.get('branch')) for r in rows] == [ + ('H1', 'DB', branch) for branch in ('mixed-healthy-repaired-failed', 'dry-run', 'deferred', + 'readonly-terminal-failure', 'unknown-negative')] + and all(r.get('cleanup') == 'PASS' for r in rows[:-1]) and rows[-1].get('cleanup') == 'FAIL' + and rows[-1].get('verdict') == 'FAIL' and rows[-1].get('failure', {}).get('type') == 'ValueError' + and rows[-1]['failure']['message'].startswith('Heal node vm000 snapshot failed:') + and rows[-1]['failure']['message'].endswith('ValueError: metadata decoder failed outside an exact witnessed file fault\n'), + 'opaque recovery only supports its second-file witness refusal before task admission') + folder = evidence / 'DB/H1/unknown-negative' + records = sorted(folder.glob('*.json')) + require({p.name for p in folder.iterdir() if p.is_dir()} == {'fixture', 'original-opaque-fixture'}, + 'opaque recovery contains another task or mutation directory') + labels = ['binding', 'inspector-build', 'preflight', 'integrity-fleet-attestation', 'reserved', 'inspector-upload', + 'started', 'deployed-topology', 'topology-proof', 'ACK-bound-full-S3', 'complete-ACK-manifest', 'trigger', + 'control', 'control-repeat', 'control-repeat', 'task-exit-background', 'quiescent', 'started', + 'original-opaque-ACK-full-S3', 'original-opaque-complete-ACK-manifest', 'trigger', 'control', + 'control-repeat', 'control-repeat', 'task-exit-background', 'quiescent', 'original-opaque-complete-physical', + *(['original-opaque-full-cohort-bitrot'] * 4), 'original-opaque-metadata-fault-plan', + 'original-opaque-byte-zero-bitflip', 'original-opaque-physical-injection', 'original-opaque-byte-zero-bitflip', + 'failure-scene-error', 'quiescent', 'retain-unknown-seed-receipt', 'branch-result'] + require([p.name for p in records] == [f'{index:05d}-{label}.json' for index, label in enumerate(labels, 1)] + and set(folder.iterdir()) == set(records) | {folder / name for name in ('fixture', 'original-opaque-fixture')}, + 'opaque recovery has a later task, mutation or unknown boundary') + def one(label): + found = [p for p in records if p.name[6:-5] == label] + require(len(found) == 1, 'opaque recovery lacks exact ' + label) + return json.loads(found[0].read_text()) + binding, original, reservation = one('binding'), one('original-opaque-complete-physical'), one('reserved') + run_id, bucket = binding.get('runId', ''), binding.get('bucketPrefix', '') + require(re.fullmatch(rf'gh-{failed_run}-[1-9][0-9]*-h1', run_id) is not None + and re.fullmatch(r'rv-heal-[0-9a-f]{16}', bucket) is not None and set(original) == {'vm000'} + and binding.get('candidateCommit') == candidate['source_sha'] + and binding.get('candidatePackageSha256') == candidate['package_sha256'] + and binding.get('candidateBinarySha256') == provenance.get('binary_sha256') + and binding.get('harnessCommit') == provenance.get('testing_commit'), 'opaque archived ownership differs') + owner = reservation.get('owner') or {} + require(reservation.get('node') == 'vm000' and reservation.get('reserved') is True + and owner.get('runId') == run_id and owner.get('bucket') == bucket + and owner.get('runtimeProfile') == 'outcome-legacy' and len(owner.get('mounts', [])) == 4 + and owner.get('binarySha256') == binding['candidateBinarySha256'] + and original['vm000'].get('quiescent') is True and type(original['vm000'].get('pid')) is int + and original['vm000']['pid'] == 0 and original['vm000'].get('binarySha256') == binding['candidateBinarySha256'], + 'opaque original reservation or stopped candidate differs') + require(one('retain-unknown-seed-receipt') == dict(state='DIRTY', namespacesRetained=True, + request=dict(operation='original-corrupt-metadata-source-probe', bucket=bucket)), 'opaque pending source boundary differs') + wanted = one('original-opaque-complete-ACK-manifest')['expected'] + require(len(wanted) == 2 and {e['key'] for e in wanted} == {'objects/0000.bin', 'opaque/source.bin'} + and all(e['versionId'] == 'null' for e in wanted), 'opaque source ACK manifest differs') + verify_seed(wanted, one('original-opaque-ACK-full-S3')) + for name, operations in (('fixture', ('create_bucket', 'put_object')), ('original-opaque-fixture', ('put_object',))): + requests = sorted((folder / name).glob('*-request.json')); acks = sorted((folder / name).glob('*-ack.json')) + require(len(requests) == len(acks) == len(operations) + and set((folder / name).iterdir()) == set(requests + acks), 'opaque original SDK ledger is incomplete') + for sequence, (request_path, ack_path, operation) in enumerate(zip(requests, acks, operations, strict=True), 1): + request, ack = json.loads(request_path.read_text()), json.loads(ack_path.read_text()); observed = ack.get('observation') or {} + require(type(request.get('sequence')) is int and type(ack.get('sequence')) is int + and request['sequence'] == ack['sequence'] == sequence and request.get('runId') == ack.get('runId') == run_id + and ack.get('requestFile') == request_path.name + and ack.get('requestSha256') == hashlib.sha256(request_path.read_bytes()).hexdigest() + and observed.get('operation') == request.get('operation') == operation + and observed.get('params') == request.get('params') and request['params'].get('Bucket') == bucket + and ack.get('operationCheck') == 'PASS' and observed.get('acknowledged') is True + and observed.get('clientCallInvoked') is True and observed.get('resultUnknown') is False + and observed.get('response', {}).get('ResponseMetadata', {}).get('HTTPStatusCode') == 200, + 'opaque original mutation is unknown or rebound') + if operation == 'put_object': + key = 'opaque/source.bin' if name == 'original-opaque-fixture' else 'objects/0000.bin' + expected = next(e for e in wanted if e['key'] == key) + require(request['params'].get('Key') == key and request.get('payload') == observed.get('payload') == expected['payload'] + and public_version_id(observed['response'].get('VersionId')) == expected['versionId'], 'opaque original PUT differs from ACK') + validate_topology(original, 'H1') + target = map_target(original, bucket, 'opaque/source.bin', 'data', 1, 'null', latest_required=False) + targets = [metadata_target(original, {**target, **member}) for member in target['cohort']] + require(one('original-opaque-metadata-fault-plan') == targets and len(targets) == 4, 'opaque original target plan differs') + receipts = [json.loads(p.read_text()) for p in records if p.name[6:-5] == 'original-opaque-byte-zero-bitflip'] + first = one('original-opaque-physical-injection'); validate_fault(original, first, targets[0], receipts[0]) + first_error = files(first)[('vm000', targets[0]['disk'], targets[0]['relativePath'])].get('metadataError') or {} + require(first_error.get('exactWitnessedFault') is True and first_error.get('validMetadataClaimed') is False, + 'opaque first metadata corruption lacks its exact decoder witness') + failure, faults, witnesses = copy.deepcopy(original), [], {'vm000': {}} + for index, (receipt, selected) in enumerate(zip(receipts, targets[:2], strict=True)): + fault_id = bounded_fault_id('J02', 'original-opaque', index) + require(receipt.get('node') == selected['node'] == 'vm000' and receipt.get('runId') == run_id + and receipt.get('faultId') == fault_id and receipt.get('kind') == 'bitflip' + and type(receipt.get('offset')) is int and receipt['offset'] == 0 + and receipt.get('relativePath') == selected['relativePath'] and receipt.get('backup') == fault_id + '.original' + and receipt.get('before') == {'sha256': selected['sha256'], 'size': selected['size']} + and receipt.get('requiresQuiescentService') is True and receipt.get('backend') == 'logical-file-fault' + and receipt.get('fault', {}).get('size') == selected['size'] + and re.fullmatch(r'[0-9a-f]{64}', receipt.get('fault', {}).get('sha256', '')) is not None, + 'opaque injected fault differs from the original planned file') + changed = copy.deepcopy(failure) + entry = next(e for e in changed['vm000']['files'] if e['disk'] == selected['disk'] and e['path'] == selected['relativePath']) + entry.pop('metadata'); entry.update(receipt['fault'], metadataError=first_error) + validate_fault(failure, changed, selected, receipt); failure = changed + faults.append(dict(node='vm000', disk=selected['disk'], faultId=fault_id, + receipt={k:v for k,v in receipt.items() if k != 'node'})) + witnesses['vm000'][f"{selected['disk']}:{selected['relativePath']}"] = receipt['fault'] + return [dict(kind='interrupted-j02-opaque', runId=run_id, nodes=['vm000'], owners={'vm000': owner}, before=original, + failure=failure, faults=faults, metadataFaults=witnesses, buckets=[bucket], binarySha256=binding['candidateBinarySha256'], + originalCaseSha256=hashlib.sha256(raw).hexdigest())] diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index 642f2cc5..bf7e219f 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -13,12 +13,12 @@ from .healthy import Fleet, physical_content from .heal_probe import require, save from .interrupted_recovery import checkpoint_plan, restore_and_isolate - from .outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace + from .outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace, interrupted_opaque_plan except ImportError: from healthy import Fleet, physical_content from heal_probe import require, save from interrupted_recovery import checkpoint_plan, restore_and_isolate - from outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace + from outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace, interrupted_opaque_plan def quota_recovery_plan(artifact, failed_run, candidate): @@ -144,6 +144,8 @@ def network_recovery_plan(artifact, failed_run, candidate): def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/cases/*.json")): + if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/evidence/heal.outcome-counters-and-error-truth/DB/H1/unknown-negative/*-original-opaque-byte-zero-bitflip.json")): + return interrupted_opaque_plan(artifact, failed_run, candidate) return acknowledged_grace_plan(artifact, failed_run, candidate) if list(artifact.glob("*/run/heal.checkpoint-and-mrf-corruption/cases/*.json")): results = list(artifact.glob("*/run/heal.checkpoint-and-mrf-corruption/cases/*.json")) @@ -541,6 +543,17 @@ def recovery_scene_matches(current, plan): """A retry may contain only the exact original shards already rescued.""" observed = physical_content(current) expected = physical_content(plan.get("failure", plan["before"])) + if plan.get("kind") == "interrupted-j02-opaque": + if set(observed) != set(expected): return False + original = physical_content(plan["before"]) + for node, scene in observed.items(): + if scene["formats"] != expected[node]["formats"]: return False + current = {(entry["disk"], entry["path"]): entry for entry in scene["files"]} + failed = {(entry["disk"], entry["path"]): entry for entry in expected[node]["files"]} + before = {(entry["disk"], entry["path"]): entry for entry in original[node]["files"]} + if len(current) != len(scene["files"]) or set(current) != set(failed) or set(before) != set(failed): return False + if any(value != failed[key] and value != before[key] for key, value in current.items()): return False + return True if plan.get("kind") not in {"owned-quota", "interrupted-checkpoint-before-admission"}: return observed == expected original = physical_content(plan["before"]) @@ -647,7 +660,7 @@ def main(): if plan.get("kind") == "acknowledged-j02-before-replay": complete.append({"failedRunId": args.failed_run_id, **isolate_acknowledged_grace(fleet, plan, states)}) continue - if plan.get("kind") == "interrupted-checkpoint-before-admission": + if plan.get("kind") in {"interrupted-checkpoint-before-admission", "interrupted-j02-opaque"}: complete.append({"failedRunId": args.failed_run_id, **restore_and_isolate(fleet, plan, states, recovery_scene_matches)}) continue diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py index e8f66de1..15287e12 100644 --- a/scripts/validation_heal/retained_scene.py +++ b/scripts/validation_heal/retained_scene.py @@ -119,7 +119,10 @@ def visit(path): backup = (not state and owner.get("runtimeProfile") == "root-restart" and len(parts) == 2 and parts[0] == "quarantine" and re.fullmatch(r"F07-shard-[0-3]\.(original|intent\.json|applied\.json)", parts[1])) - if (state and relative == "rustfs") or fixture or mrf or backup: + opaque_backup = (not state and owner.get("runtimeProfile") == "outcome-legacy" + and len(parts) == 2 and parts[0] == "quarantine" + and re.fullmatch(r"J02-[0-9a-f]{32}\.(original|intent\.json|applied\.json|failed-[0-9a-f]{64})", parts[1])) + if (state and relative == "rustfs") or fixture or mrf or backup or opaque_backup: fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) try: require(stable(os.fstat(fd)) == stable(info) and info.st_size <= 2 * 1024**3, @@ -155,7 +158,7 @@ def inspect(request, node): require(isinstance(expected, dict) and expected.get("runId") == request["runId"] and expected.get("unit") == node["unit"](request["runId"]) and expected.get("hostname") == node["socket"].gethostname() - and expected.get("runtimeProfile") in {"mrf-crash", "root-restart", "outcome-grace"}, "retained scene archived owner differs") + and expected.get("runtimeProfile") in {"mrf-crash", "root-restart", "outcome-grace", "outcome-legacy"}, "retained scene archived owner differs") current = node["mounts"]() require(len(current) == len(expected.get("mounts", [])) == 4 and all(all(actual[key] == old[key] for key in ("target", "source", "fstype", "uuid", "maj:min")) @@ -196,7 +199,7 @@ def inspect(request, node): baseline = {"runId": request["runId"], "ownerSha256": sha(expected), "roots": witnesses[:4], "state": witnesses[4]} intent = retained_state.parent / (request["runId"] + ".intent.json") node["checked"](intent, state.parent) - if expected["runtimeProfile"] in {"root-restart", "outcome-grace"}: + if expected["runtimeProfile"] in {"root-restart", "outcome-grace", "outcome-legacy"}: original_sha = request.get("expectedOriginalSceneSha256", "") require(re.fullmatch(r"[0-9a-f]{64}", original_sha) is not None, "restored fixture requires its original physical scene digest") @@ -218,7 +221,7 @@ def inspect(request, node): def run(request, node): fields = {"runId", "action", "expectedOwner"} - if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace"}: + if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace", "outcome-legacy"}: fields.add("expectedOriginalSceneSha256") if request["action"] == "retained-scene-isolate": fields.add("expectedBaseline") diff --git a/test_validation_heal_outcome.py b/test_validation_heal_outcome.py index 252e35f0..bda417a3 100644 --- a/test_validation_heal_outcome.py +++ b/test_validation_heal_outcome.py @@ -637,6 +637,13 @@ def snapshot(label, **params): events.append(label) if any('metadataError' in row for row in current['node1']['files']): self.assertTrue(params.get('metadata_faults')) + from scripts.validation_heal import fleet_node + with patch.object(fleet_node, 'command', return_value=types.SimpleNamespace(returncode=1, stderr='Error: FileCorrupt\n')): + for row in current['node1']['files']: + if 'metadataError' in row: + witness = fleet_node.inspect_metadata(Path('/inspector'), Path('/xl.meta'), bucket, self.key, + row, params['metadata_faults']['node1']) + self.assertTrue(witness['metadataError']['exactWitnessedFault']) return copy.deepcopy(current) fleet.snapshot = snapshot def remote(node, action, **params): diff --git a/test_validation_heal_outcome_recovery.py b/test_validation_heal_outcome_recovery.py new file mode 100644 index 00000000..a843294c --- /dev/null +++ b/test_validation_heal_outcome_recovery.py @@ -0,0 +1,230 @@ +"""Interrupted opaque probes restore only the two archived pre-task file faults.""" +import copy +import hashlib +import json +from pathlib import Path +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import heal_fault, interrupted_recovery, outcome_recovery, recover_environment, retained_scene +from scripts.validation_heal.healthy import physical_content +from scripts.validation_heal.shards import map_target +from scripts.validation_heal.versions import metadata_target +from test_validation_heal import versioned_scene +import test_validation_heal_retained_scene as scene_tests + + +def emit(path, value): + path.write_text(json.dumps(value)) + + +class OpaqueRecoveryTests(unittest.TestCase): + def fixture(self, root): + case = 'heal.outcome-counters-and-error-truth' + base = root / 'artifact/run' / case + folder = base / 'evidence' / case / 'DB/H1/unknown-negative' + folder.mkdir(parents=True); (base / 'cases').mkdir() + candidate = dict(source_sha=outcome_recovery.CANDIDATE, package_sha256='a' * 64) + provenance = dict(source_commit=candidate['source_sha'], candidate_sha256='a' * 64, + testing_commit='b' * 40, binary_sha256='c' * 64) + emit(base / 'cases' / (case + '.json'), dict(case_id=case, verdict='FAIL', cleanup_status='FAIL', provenance=provenance)) + failure = dict(type='ValueError', message='Heal node vm000 snapshot failed: ValueError: metadata decoder failed outside an exact witnessed file fault\n') + branches = ('mixed-healthy-repaired-failed', 'dry-run', 'deferred', 'readonly-terminal-failure', 'unknown-negative') + rows = [dict(topology='H1', fixture='DB', branch=branch, verdict='FAIL', + cleanup='FAIL' if branch == branches[-1] else 'PASS', failure=failure) for branch in branches] + emit(base / 'evidence' / case / 'healthy-summary.json', dict(caseId=case, results=rows, + cleanup='FAIL', complete=False, canonicalCasePassClaimed=False)) + original, bucket, expected = versioned_scene(['objects/0000.bin', 'opaque/source.bin'], sizes=(100,)) + original['vm000'] = original.pop('node1') + original['vm000'].update(pid=0, quiescent=True, binarySha256='c' * 64) + for entry in expected: + entry.update(versionId='null', isLatest=True); entry['payload']['sha256'] = 'd' * 64 + for entry in original['vm000']['files']: + if 'metadata' in entry: + entry['metadata']['versions'][0].update(versionId='null', isLatest=True) + entry['sha256'] = hashlib.sha256(str(entry['disk']).encode()).hexdigest() + run_id = 'gh-12345-1-h1' + owner = dict(runId=run_id, bucket=bucket, runtimeProfile='outcome-legacy', binarySha256='c' * 64, mounts=[{}] * 4) + labels = ['binding', 'inspector-build', 'preflight', 'integrity-fleet-attestation', 'reserved', 'inspector-upload', + 'started', 'deployed-topology', 'topology-proof', 'ACK-bound-full-S3', 'complete-ACK-manifest', 'trigger', + 'control', 'control-repeat', 'control-repeat', 'task-exit-background', 'quiescent', 'started', + 'original-opaque-ACK-full-S3', 'original-opaque-complete-ACK-manifest', 'trigger', 'control', + 'control-repeat', 'control-repeat', 'task-exit-background', 'quiescent', 'original-opaque-complete-physical', + *(['original-opaque-full-cohort-bitrot'] * 4), 'original-opaque-metadata-fault-plan', + 'original-opaque-byte-zero-bitflip', 'original-opaque-physical-injection', 'original-opaque-byte-zero-bitflip', + 'failure-scene-error', 'quiescent', 'retain-unknown-seed-receipt', 'branch-result'] + data = {1: dict(runId=run_id, bucketPrefix=bucket, candidateCommit=candidate['source_sha'], + candidatePackageSha256='a' * 64, candidateBinarySha256='c' * 64, harnessCommit='b' * 40), + 5: dict(node='vm000', reserved=True, owner=owner), 19: dict(entries=expected), + 20: dict(expected=expected), 27: original, + 38: dict(state='DIRTY', namespacesRetained=True, + request=dict(operation='original-corrupt-metadata-source-probe', bucket=bucket))} + target = map_target(original, bucket, 'opaque/source.bin', 'data', 1, 'null', latest_required=False) + targets = [metadata_target(original, {**target, **member}) for member in target['cohort']] + data[32] = targets + first = copy.deepcopy(original) + for index, number in enumerate((33, 35)): + selected = targets[index]; fault_id = heal_fault.bounded_fault_id('J02', 'original-opaque', index) + receipt = dict(runId=run_id, faultId=fault_id, node='vm000', relativePath=selected['relativePath'], kind='bitflip', + offset=0, before={k: selected[k] for k in ('size', 'sha256')}, + fault=dict(size=selected['size'], sha256=str(index + 4) * 64), backup=fault_id + '.original', + requiresQuiescentService=True, backend='logical-file-fault', witnessedAt='timestamp') + data[number] = receipt + if index == 0: + entry = next(e for e in first['vm000']['files'] if e['disk'] == selected['disk'] and e['path'] == selected['relativePath']) + entry.pop('metadata'); entry.update(receipt['fault'], metadataError=dict(code=1, message='Error: FileCorrupt\n', + exactWitnessedFault=True, validMetadataClaimed=False)) + data[34] = first + for index, label in enumerate(labels, 1): emit(folder / f'{index:05d}-{label}.json', data.get(index, {})) + for name, operations in (('fixture', ('create_bucket', 'put_object')), ('original-opaque-fixture', ('put_object',))): + ledger = folder / name; ledger.mkdir() + for seq, operation in enumerate(operations, 1): + params = dict(Bucket=bucket); response = dict(ResponseMetadata=dict(HTTPStatusCode=200)) + request = dict(sequence=seq, runId=run_id, operation=operation, params=params) + if operation == 'put_object': + entry = expected[0 if name == 'fixture' else 1] + params['Key'] = entry['key']; request['payload'] = entry['payload']; response['VersionId'] = 'null' + path = ledger / f'{seq:06d}-request.json'; emit(path, request) + emit(ledger / f'{seq:06d}-ack.json', dict(sequence=seq, runId=run_id, requestFile=path.name, + requestSha256=hashlib.sha256(path.read_bytes()).hexdigest(), operationCheck='PASS', + observation=dict(operation=operation, params=params, payload=request.get('payload'), response=response, + acknowledged=True, clientCallInvoked=True, resultUnknown=False))) + return candidate, folder + + def test_plan_binds_each_disk_to_its_original_receipt_and_rejects_unknown_boundaries(self): + for poison in (None, 'candidate', 'unknown-ack', 'request-hash', 'extra-delete', 'extra-ledger', 'later-task', + 'task-directory', 'replaced-bitrot', 'fault-id', 'before-hash', 'fault-offset', 'first-witness', 'lost-format', 'prior-cleanup', 'owner'): + with self.subTest(poison=poison), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary); candidate, folder = self.fixture(root) + def change(pattern, function): + path = next(folder.glob(pattern)); value = json.loads(path.read_text()); function(value); emit(path, value) + if poison == 'candidate': candidate['source_sha'] = 'e' * 40 + elif poison == 'unknown-ack': change('original-opaque-fixture/*-ack.json', lambda x: x['observation'].update(resultUnknown=True)) + elif poison == 'request-hash': change('fixture/*-ack.json', lambda x: x.update(requestSha256='f' * 64)) + elif poison == 'extra-delete': emit(folder / 'fixture/000003-request.json', {}) + elif poison == 'extra-ledger': emit(folder / 'fixture/unknown.json', {}) + elif poison == 'later-task': emit(folder / '00040-trigger.json', {}) + elif poison == 'task-directory': (folder / 'admin').mkdir() + elif poison == 'replaced-bitrot': next(folder.glob('00031-*')).rename(folder / '00031-trigger.json') + elif poison == 'fault-id': change('00035-*', lambda x: x.update(faultId=heal_fault.bounded_fault_id('J02', 'original-opaque', 0))) + elif poison == 'before-hash': change('00035-*', lambda x: x['before'].update(sha256='e' * 64)) + elif poison == 'fault-offset': change('00035-*', lambda x: x.update(offset=1)) + elif poison == 'first-witness': change('00034-*', lambda x: next(e for e in x['vm000']['files'] if 'metadataError' in e)['metadataError'].update(exactWitnessedFault=False)) + elif poison == 'lost-format': change('00027-*', lambda x: x['vm000']['formats'].pop()) + elif poison == 'owner': change('00005-*', lambda x: x['owner'].update(runtimeProfile='manual')) + elif poison == 'prior-cleanup': + path = folder.parents[2] / 'healthy-summary.json'; value = json.loads(path.read_text()) + value['results'][0]['cleanup'] = 'FAIL'; emit(path, value) + if poison: + with self.assertRaises((ValueError, KeyError)): recover_environment.recovery_plan(root, 12345, candidate) + else: + plan = recover_environment.recovery_plan(root, 12345, candidate)[0] + self.assertEqual(plan['kind'], 'interrupted-j02-opaque') + self.assertEqual([f['disk'] for f in plan['faults']], [0, 1]) + self.assertEqual(len(plan['metadataFaults']['vm000']), 2) + self.assertNotEqual(*[f['receipt']['fault']['sha256'] for f in plan['faults']]) + for current in (plan['failure'], plan['before']): self.assertTrue(recover_environment.recovery_scene_matches(current, plan)) + partial = copy.deepcopy(plan['failure']); key = plan['faults'][0]['receipt']['relativePath'] + for entry in partial['vm000']['files']: + if entry['path'] == key and entry['disk'] == 0: + entry.clear(); entry.update(copy.deepcopy(next(e for e in plan['before']['vm000']['files'] if e['path'] == key and e['disk'] == 0))) + self.assertTrue(recover_environment.recovery_scene_matches(partial, plan)) + for drift in ('control', 'unknown-meta', 'format', 'extra-file', 'lost-file'): + current = copy.deepcopy(partial) + if drift == 'control': next(e for e in current['vm000']['files'] if 'objects/0000.bin' in e['path'])['sha256'] = 'e' * 64 + elif drift == 'unknown-meta': next(e for e in current['vm000']['files'] if 'metadataError' in e)['sha256'] = 'e' * 64 + elif drift == 'format': current['vm000']['formats'].pop() + elif drift == 'extra-file': current['vm000']['files'].append(copy.deepcopy(current['vm000']['files'][0])) + elif drift == 'lost-file': current['vm000']['files'].pop() + with self.subTest(drift=drift): self.assertFalse(recover_environment.recovery_scene_matches(current, plan)) + + def test_restore_passes_disk_witnesses_and_exact_intent_then_archives_only_restored_bytes(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary); candidate, _ = self.fixture(root) + plan = recover_environment.recovery_plan(root, 12345, candidate)[0] + current = copy.deepcopy(plan['failure']); events = [] + def remote(node, action, **params): + events.append(action) + if action == 'stop': return {} + if action == 'snapshot': + self.assertEqual(params, dict(quiescent=True, **({'metadataFaults': plan['metadataFaults'][node]} if len(events) == 2 else {}))) + return copy.deepcopy(current[node]) + if action == 'recover-fault': + fault = next(f for f in plan['faults'] if f['faultId'] == params['faultId']) + self.assertEqual(params['disk'], fault['disk']) + self.assertEqual(params['expectedReceiptSha256'], outcome_recovery.digest({k: v for k, v in fault['receipt'].items() if k != 'witnessedAt'})) + entry = next(e for e in current[node]['files'] if e['disk'] == fault['disk'] and e['path'] == fault['receipt']['relativePath']) + entry.clear(); entry.update(copy.deepcopy(next(e for e in plan['before'][node]['files'] if e['disk'] == fault['disk'] and e['path'] == fault['receipt']['relativePath']))) + return dict(runId=plan['runId'], faultId=fault['faultId'], physicalRepairPassAllowed=False) + self.assertEqual(physical_content(current), physical_content(plan['before'])) + return dict(baseline=dict(originalSceneSha256=params['expectedOriginalSceneSha256']), inspectionOnly=True, + mutated=False, activeNamespaceIsolated=True, canonicalPassClaimed=False) + fleet = types.SimpleNamespace(remote=remote, record=lambda label, value: value) + states = [dict(node='vm000', state=dict(exists=True), roots=[dict(exists=True)] * 4)] + result = interrupted_recovery.restore_and_isolate(fleet, plan, states, recover_environment.recovery_scene_matches) + self.assertEqual(result['cleanup'], 'ISOLATED'); self.assertFalse(result['canonicalPassClaimed']) + self.assertEqual(events[:4], ['stop', 'snapshot', 'recover-fault', 'recover-fault']) + self.assertFalse(set(events) & {'start', 'cleanup', 'journal-recover'}) + + def test_real_opaque_archive_retries_all_moves_and_hashes_fault_backups_without_credentials(self): + helper = scene_tests.RetainedSceneTests() + for lost_after in range(1, 6): + with self.subTest(lost_after=lost_after), tempfile.TemporaryDirectory() as temporary: + state, roots, node, request = helper.scene(temporary) + roots[0].mkdir(parents=True); owner = request['expectedOwner']; owner['runtimeProfile'] = 'outcome-legacy' + for folder in (*roots, state): emit(folder / 'owner.json', owner) + fault_id = heal_fault.bounded_fault_id('J02', 'original-opaque', 0) + for suffix in ('original', 'intent.json', 'applied.json', 'failed-' + 'a' * 64): + (roots[1] / 'quarantine' / (fault_id + '.' + suffix)).write_bytes(b'fault-evidence') + snapshot = dict(files=[], formats=[{}] * 4, pid=0, quiescent=True, binarySha256=owner['binarySha256']) + node['snapshot'] = lambda value: snapshot + request['expectedOriginalSceneSha256'] = outcome_recovery.digest({k: snapshot[k] for k in ('files', 'formats')}) + opened = []; original_open = retained_scene.os.open + def checked_open(path, *args, **kwargs): + opened.append(str(path)); return original_open(path, *args, **kwargs) + with helper.root_stats(), patch.object(retained_scene.os, 'open', side_effect=checked_open), patch.object(retained_scene, 'rename_new', side_effect=helper.local_rename): + baseline = retained_scene.run(request, node)['baseline'] + self.assertTrue(all(any(fault_id + '.' + suffix in p for p in opened) for suffix in ('original', 'intent.json', 'applied.json', 'failed-' + 'a' * 64))) + self.assertFalse(any('service.env' in p or '/iam/credential' in p for p in opened)) + count = 0 + def move(source, target): + nonlocal count + helper.local_rename(source, target) + if target.is_dir(): + count += 1 + if count == lost_after: raise RuntimeError('lost ACK') + change = dict(request, action='retained-scene-isolate', expectedBaseline=baseline) + with patch.object(retained_scene, 'rename_new', side_effect=move), self.assertRaises(RuntimeError): retained_scene.run(change, node) + node['snapshot'] = lambda value: self.fail('partial archive must use durable original digest') + result = retained_scene.run(change, node) + self.assertEqual(result['locations'], ['retained'] * 5); self.assertEqual(result['baseline'], baseline) + self.assertFalse(result['canonicalPassClaimed']); self.assertFalse(result['credentialContentsRead']) + + def test_real_file_recovery_preserves_corrupt_bytes_and_refuses_later_data(self): + for drift in (None, 'intent', 'backup', 'target'): + with self.subTest(drift=drift), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve(); (root / 'quarantine').mkdir() + bucket = 'rv-heal-0123456789abcdef'; relative = bucket + '/opaque/source.bin/xl.meta' + target = root / 'data' / relative; target.parent.mkdir(parents=True) + original = b'XL2 original acknowledged metadata'; target.write_bytes(original) + owner = dict(runId='gh-12345-1-h1', bucketPrefix=bucket) + fault_id = heal_fault.bounded_fault_id('J02', 'original-opaque', 1) + with patch.object(heal_fault, 'mount_id', return_value=1), heal_fault.directory(root) as fd: + receipt = heal_fault.apply_fault(fd, owner, relative, fault_id, 'bitflip', hashlib.sha256(original).hexdigest(), 0) + corrupt = target.read_bytes(); receipt_sha = outcome_recovery.digest({k: v for k, v in receipt.items() if k != "witnessedAt"}) + if drift == 'intent': receipt_sha = 'f' * 64 + elif drift == 'backup': (root / 'quarantine' / (fault_id + '.original')).write_bytes(b'foreign backup') + elif drift == 'target': target.write_bytes(b'later acknowledged bytes') + before = target.read_bytes() + if drift: + with self.assertRaises(ValueError): heal_fault.recover_fault(fd, owner, fault_id, expected_receipt_sha256=receipt_sha) + self.assertEqual(target.read_bytes(), before) + continue + proof = heal_fault.recover_fault(fd, owner, fault_id, expected_receipt_sha256=receipt_sha) + self.assertTrue(proof['restoredFromBackup']); self.assertFalse(proof['physicalRepairPassAllowed']) + self.assertEqual(target.read_bytes(), original) + preserved = root / 'quarantine' / (fault_id + '.failed-' + hashlib.sha256(corrupt).hexdigest()) + self.assertEqual(preserved.read_bytes(), corrupt) + self.assertFalse(heal_fault.recover_fault(fd, owner, fault_id, expected_receipt_sha256=receipt_sha)['restoredFromBackup']) From da5dccb36c80acd91141759fa18011e260e0fd6a Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 16 Oct 2026 14:35:33 +0800 Subject: [PATCH 215/343] fix(validation): recover audited J02 owner marker metadata --- scripts/validation_heal/fleet_node.py | 2 + scripts/validation_heal/healthy.py | 7 +- .../validation_heal/interrupted_recovery.py | 15 ++ scripts/validation_heal/owner_recovery.py | 177 ++++++++++++++++++ test_validation_heal_outcome_recovery.py | 12 +- test_validation_heal_owner_recovery.py | 164 ++++++++++++++++ 6 files changed, 372 insertions(+), 5 deletions(-) create mode 100644 scripts/validation_heal/owner_recovery.py create mode 100644 test_validation_heal_owner_recovery.py diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 56b0eecd..3d931ad7 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1634,6 +1634,8 @@ def main(request): actions["environment-capabilities"] = environment_capabilities for action in ("retained-scene-audit", "retained-scene-isolate"): actions[action] = retained_scene_operation + for action in ("owner-markers-audit", "owner-markers-recover"): + actions[action] = lambda value: globals()["owner_recovery_backend"].run(value, globals()) actions["outcome-legacy-phase"] = outcome_legacy_phase actions["quota-audit-finalize"] = quota_audit_finalize actions["journal-rejection-audit-finalize"] = journal_rejection_audit_finalize diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 829feb9c..88d0ab22 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -185,9 +185,12 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() - if action.startswith("retained-scene-"): + if action.startswith(("retained-scene-", "owner-markers-")): source += "\nimport types\nretained_scene_backend = types.ModuleType('retained_scene')\nexec(" + repr( NODE_SOURCE.with_name("retained_scene.py").read_text()) + ", retained_scene_backend.__dict__)\n" + if action.startswith("owner-markers-"): + source += "\nimport types\nowner_recovery_backend = types.ModuleType('owner_recovery')\nexec(" + repr( + NODE_SOURCE.with_name("owner_recovery.py").read_text()) + ", owner_recovery_backend.__dict__)\n" if action.startswith("network-") or action in {"cleanup", "quota-audit-finalize", "journal-rejection-audit", "journal-rejection-audit-finalize", "journal-retained-audit", "journal-retained-audit-finalize"}: network_source = NODE_SOURCE.with_name("network_gate.py").read_text() source += "\nimport types\nnetwork_gate_backend = types.ModuleType('network_gate')\nexec(" + repr( @@ -205,7 +208,7 @@ def remote(self, node, action, **params): source += "\nimport types\ngeneration_backend = types.ModuleType('generation_fault')\nexec(" + repr( NODE_SOURCE.with_name("generation_fault.py").read_text()) + ", generation_backend.__dict__)\n" if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline", - "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure", "outcome-legacy-phase"} or action.startswith("generation-") or action.startswith("orphan-") or action.startswith(("journal-", "incarnation-")): + "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure", "outcome-legacy-phase"} or action.startswith("generation-") or action.startswith("orphan-") or action.startswith(("journal-", "incarnation-", "owner-markers-")): source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" if action.startswith(("journal-", "incarnation-")): diff --git a/scripts/validation_heal/interrupted_recovery.py b/scripts/validation_heal/interrupted_recovery.py index 59f2d730..58b958f1 100644 --- a/scripts/validation_heal/interrupted_recovery.py +++ b/scripts/validation_heal/interrupted_recovery.py @@ -178,6 +178,21 @@ def restore_and_isolate(fleet, plan, states, scene_matches): require(item["state"]["exists"] and len(item["roots"]) == 4 and all(root["exists"] for root in item["roots"]), "interrupted fixture lost an original root; retain remaining scene") active.append(item["node"]) + if plan.get("kind") == "interrupted-j02-opaque": + for node in active: + params = dict(expectedOwner=plan["owners"][node], before=plan["before"][node], + faults=[{k: v for k, v in f.items() if k != "node"} for f in plan["faults"] if f["node"] == node]) + audit = fleet.record("interrupted-owner-markers-audit", {"node": node, + **fleet.remote(node, "owner-markers-audit", **params)}) + require(audit.get("inspectionOnly") is True and audit.get("mutated") is False + and audit.get("canonicalPassClaimed") is False and type(audit.get("markerRecoveryRequired")) is bool, + "interrupted owner-marker audit is incomplete") + if audit.get("markerRecoveryRequired") is True: + proof = fleet.record("interrupted-owner-markers-recovery", {"node": node, + **fleet.remote(node, "owner-markers-recover", **params, + expectedAudit={k: v for k, v in audit.items() if k != "node"})}) + require(proof.get("ownerMetadataRestored") is True and proof.get("businessBytesChanged") is False + and proof.get("canonicalPassClaimed") is False, "interrupted owner-marker recovery differs") for node in active: fleet.record("interrupted-recovery-stop", {"node": node, **fleet.remote(node, "stop")}) current = {node: fleet.remote(node, "snapshot", quiescent=True, diff --git a/scripts/validation_heal/owner_recovery.py b/scripts/validation_heal/owner_recovery.py new file mode 100644 index 00000000..bdd45420 --- /dev/null +++ b/scripts/validation_heal/owner_recovery.py @@ -0,0 +1,177 @@ +"""Audit exact stopped J02 bytes before restoring four owner-marker UID/GID pairs.""" + +from contextlib import ExitStack +import hashlib +import json +import os +from pathlib import Path +import pwd +import re +import stat + + +def require(value, message): + if not value: + raise ValueError(message) + + +def encoded(value): + return json.dumps(value, sort_keys=True, separators=(',', ':')).encode() + + +def sha(value): + return hashlib.sha256(encoded(value)).hexdigest() + + +def identity(info): + return {name: getattr(info, 'st_' + name) for name in + ('dev', 'ino', 'uid', 'gid', 'mode', 'nlink', 'size', 'mtime_ns', 'ctime_ns')} + + +def marker(path, checked, service): + with os.fdopen(os.open(checked(path, path.parent), os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK), 'rb') as stream: + before = os.fstat(stream.fileno()) + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_size <= 16384 + and stat.S_IMODE(before.st_mode) in {0o600, 0o644} and (before.st_uid, before.st_gid) in {(0, 0), service}, + 'owner recovery marker has unknown permissions or ownership') + raw = stream.read(16385) + require(len(raw) == before.st_size and identity(before) == identity(os.fstat(stream.fileno())), + 'owner recovery marker changed during audit') + return json.loads(raw), dict(identity(before), sha256=hashlib.sha256(raw).hexdigest()) + + +def inspect(request, node): + state, roots = node['paths'](request['runId']); checked = node['checked'] + expected = request['expectedOwner']; original = request['before']; faults = request['faults'] + service = pwd.getpwnam('rustfs'); service = (service.pw_uid, service.pw_gid) + require(service[0] > 0 and isinstance(expected, dict) and expected.get('runId') == request['runId'] + and expected.get('unit') == node['unit'](request['runId']) + and expected.get('hostname') == node['socket'].gethostname() + and expected.get('runtimeProfile') == 'outcome-legacy' + and re.fullmatch(r'rv-heal-[0-9a-f]{16}', expected.get('bucket', '')) is not None, + 'owner recovery requires the exact original J02 reservation') + state_info = checked(state, state.parent).lstat() + require(stat.S_ISDIR(state_info.st_mode) and state_info.st_uid == 0 and stat.S_IMODE(state_info.st_mode) == 0o700, + 'owner recovery lost its private root-owned state') + owner, central = marker(state / 'owner.json', checked, service) + require((central['uid'], central['gid']) == (0, 0) and owner == expected, 'owner recovery central reservation differs') + require(len(roots) == len(expected.get('mounts', [])) == 4 + and all(all(a[k] == b[k] for k in ('target', 'source', 'fstype', 'uuid', 'maj:min')) + for a, b in zip(node['mounts'](), expected['mounts'], strict=True)), 'owner recovery mounts differ') + require(not any((state / name).exists() for name in ('J02-phase.json', 'J02-legacy-intent.json', + 'drive-gate.json', 'quorum-gate.json', 'write-quorum-gate.json', 'network-gate.json', 'disk-replacement.json')), + 'owner recovery has a phase or gate beyond the archived boundary') + stopped = node['stopped'](owner) + require(node['digest'](checked(state / 'rustfs', state)) == owner['binarySha256'] + and owner.get('inspectorSha256') and node['digest'](checked(state / 'heal-meta-inspect', state)) == owner['inspectorSha256'], + 'owner recovery candidate or inspector changed') + wanted = {(e['disk'], e['path']): e for e in original['files']} + require(len(wanted) == len(original['files']) and 8 <= len(wanted) <= 1024 and len(original['formats']) == 4 + and all(e['path'].startswith(owner['bucket'] + '/') for e in wanted.values()), 'owner recovery physical baseline differs') + require(len(faults) == 2 and len({f['disk'] for f in faults}) == 2 + and all(type(f['disk']) is int and 0 <= f['disk'] < 4 and re.fullmatch(r'J02-[0-9a-f]{32}', f['faultId']) + and f['receipt']['relativePath'] == owner['bucket'] + '/opaque/source.bin/xl.meta' + and f['receipt']['runId'] == request['runId'] and f['receipt']['faultId'] == f['faultId'] + and f['receipt']['kind'] == 'bitflip' and type(f['receipt']['offset']) is int + and f['receipt']['offset'] == 0 for f in faults), 'owner recovery original faults differ') + damaged = {(f['disk'], f['receipt']['relativePath']): f['receipt']['fault'] for f in faults} + require(all(f['receipt']['before'] == {k: wanted[(f['disk'], f['receipt']['relativePath'])][k] for k in ('size', 'sha256')} + and f['receipt']['backup'] == f['faultId'] + '.original' + and f['receipt']['fault']['size'] == f['receipt']['before']['size'] + and re.fullmatch(r'[0-9a-f]{64}', f['receipt']['fault']['sha256']) for f in faults), + 'owner recovery faults are not bound to the original metadata') + raw_files, markers, backups, census = [], [], [], [] + module, retained = node['fault_module'](), node['retained_scene_backend'] + for disk, root in enumerate(roots): + checked(root, root.parent) + require(root.is_dir() and root.lstat().st_dev == node['MOUNTS'][disk].stat().st_dev + and not any(p == root or p.is_relative_to(root) for p in retained.mounted_paths()), + 'owner recovery lost a drive root or found another mount') + observed, evidence = marker(root / 'owner.json', checked, service) + require(observed == owner and evidence['sha256'] == central['sha256'] and evidence['mode'] == central['mode'], 'owner recovery drive reservation differs') + markers.append(dict(disk=disk, **evidence)) + data = checked(root / 'data', root); folder = checked(data / owner['bucket'], data) + for directory, children, names in os.walk(folder, followlinks=False): + for name in (*children, *names): checked(Path(directory) / name, data) + for name in sorted(names): + path = Path(directory) / name; info = path.lstat(); relative = str(path.relative_to(data)); key = (disk, relative) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and key in wanted + and info.st_size <= 64 * 1024**2, 'owner recovery has an extra or unsafe business file') + raw = dict(size=info.st_size, sha256=node['digest'](path)) + require(identity(info) == identity(path.lstat()) and raw in + ({k: wanted[key][k] for k in ('size', 'sha256')}, damaged.get(key)), + 'owner recovery business bytes differ from original or exact fault') + raw_files.append(dict(disk=disk, path=relative, **raw)) + format_path = checked(data / '.rustfs.sys/format.json', data) + require(format_path.stat().st_size < 65536 and original['formats'][disk]['disk'] == disk + and node['digest'](format_path) == original['formats'][disk]['sha256'] + and json.loads(format_path.read_text()) == original['formats'][disk]['value'], 'owner recovery disk format differs') + for fault in (f for f in faults if f['disk'] == disk): + receipt = {k: v for k, v in fault['receipt'].items() if k != 'witnessedAt'} + with module.directory(root) as fd, module.parent(fd, ['quarantine']) as backup_fd: + binding = module.checked_owner(fd, request['runId'], recovery=True) + require(binding['bucketPrefix'] == owner['bucket'], 'owner recovery fault owner differs') + names = {fault['faultId'] + suffix for suffix in ('.original', '.intent.json', '.applied.json')} + preserved = fault['faultId'] + '.failed-' + receipt['fault']['sha256'] + actual_names = set(os.listdir(backup_fd)) + require(names <= actual_names <= names | {preserved}, 'owner recovery quarantine differs') + for name in sorted(actual_names): + raw = module.read_file(backup_fd, name, 64 * 1024**2) + if name.endswith('.intent.json'): require(json.loads(raw) == receipt, 'owner recovery intent differs') + elif name.endswith('.applied.json'): require(json.loads(raw) == fault['receipt'], 'owner recovery applied receipt differs') + else: + wanted_bytes = receipt['before'] if name.endswith('.original') else receipt['fault'] + require(dict(size=len(raw), sha256=hashlib.sha256(raw).hexdigest()) == wanted_bytes, 'owner recovery backup bytes differ') + backups.append(dict(disk=disk, name=name, size=len(raw), sha256=hashlib.sha256(raw).hexdigest())) + if not any(f['disk'] == disk for f in faults): + require(not any(checked(root / 'quarantine', root).iterdir()), 'owner recovery has another fault backup') + census.append(retained.census(root, owner, state=False, node=node)) + require(len(raw_files) == len(wanted) and {(e['disk'], e['path']) for e in raw_files} == set(wanted), + 'owner recovery lost an original business file') + contract = dict(runId=request['runId'], ownerSha256=sha(owner), beforeSha256=sha(original), faultsSha256=sha(faults)) + intent_path = checked(state / 'owner-marker-recovery.intent.json', state) + prior = retained.private_json(intent_path) if intent_path.exists() else None + if prior is not None: + require(prior.get('contract') == contract and len(prior.get('markers', [])) == 4 + and all(all(old[k] == now[k] for k in ('disk', 'dev', 'ino', 'mode', 'nlink', 'size', 'mtime_ns', 'sha256')) + for old, now in zip(prior['markers'], markers, strict=True)), 'owner recovery durable intent differs') + return dict(contract=contract, markers=markers, backupFiles=backups, rootCensus=census, + rawPhysicalSha256=sha(sorted(raw_files, key=lambda e: (e['disk'], e['path']))), + service=stopped, priorIntent=prior, markerRecoveryRequired=any((m['uid'], m['gid']) != (0, 0) for m in markers), + inspectionOnly=True, mutated=False, canonicalPassClaimed=False), state, roots + + +def run(request, node): + fields = {'runId', 'action', 'expectedOwner', 'before', 'faults'} + if request['action'] == 'owner-markers-recover': fields.add('expectedAudit') + require(set(request) == fields and request['action'] in {'owner-markers-audit', 'owner-markers-recover'}, + 'owner recovery accepts only its archived scene and audit') + audit, state, roots = inspect(request, node) + if request['action'] == 'owner-markers-audit': return audit + require(audit == request['expectedAudit'], 'owner recovery changed after read-only preflight') + module = node['fault_module'](); prior = audit['priorIntent'] + if prior is None: + prior = {k: audit[k] for k in ('contract', 'markers', 'backupFiles', 'rootCensus', 'rawPhysicalSha256')} + with module.directory(state) as fd: + module.new_file(fd, 'owner-marker-recovery.intent.json', encoded(prior)) + changed = [] + with ExitStack() as stack: + descriptors = [stack.enter_context(os.fdopen(os.open(node['checked'](root / 'owner.json', root), + os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK), 'rb')) for root in roots] + for disk, stream in enumerate(descriptors): + fresh, _, _ = inspect(request, node) + require(fresh['rawPhysicalSha256'] == audit['rawPhysicalSha256'] and fresh['backupFiles'] == audit['backupFiles'] + and all(all(a[k] == b[k] for k in ('disk', 'dev', 'ino', 'mode', 'nlink', 'size', 'mtime_ns', 'sha256')) + for a, b in zip(audit['markers'], fresh['markers'], strict=True)), 'owner recovery bytes or identities changed') + info = identity(os.fstat(stream.fileno())) + require(all(info[k] == fresh['markers'][disk][k] for k in info), 'owner recovery descriptor changed') + if (info['uid'], info['gid']) != (0, 0): + os.fchown(stream.fileno(), 0, 0); os.fsync(stream.fileno()) + node['retained_scene_backend'].sync_directory(roots[disk]) + changed.append(disk) + final, _, _ = inspect(request, node) + require(not final['markerRecoveryRequired'] and final['rawPhysicalSha256'] == audit['rawPhysicalSha256'] + and final['backupFiles'] == audit['backupFiles'], 'owner recovery post-audit differs') + return dict(ownerMetadataRestored=True, changedDisks=changed, before=audit, after=final, + intentSha256=sha(prior), businessBytesChanged=False, backupRestorationUsed=False, + canonicalPassClaimed=False, originalVerdictRewritten=False) diff --git a/test_validation_heal_outcome_recovery.py b/test_validation_heal_outcome_recovery.py index a843294c..cdf83d61 100644 --- a/test_validation_heal_outcome_recovery.py +++ b/test_validation_heal_outcome_recovery.py @@ -144,12 +144,18 @@ def test_restore_passes_disk_witnesses_and_exact_intent_then_archives_only_resto with tempfile.TemporaryDirectory() as temporary: root = Path(temporary); candidate, _ = self.fixture(root) plan = recover_environment.recovery_plan(root, 12345, candidate)[0] - current = copy.deepcopy(plan['failure']); events = [] + current = copy.deepcopy(plan['failure']); events = []; snapshots = 0 def remote(node, action, **params): + nonlocal snapshots events.append(action) + if action == 'owner-markers-audit': return dict(markerRecoveryRequired=True, inspectionOnly=True, mutated=False, canonicalPassClaimed=False) + if action == 'owner-markers-recover': + self.assertEqual(params['expectedAudit'], dict(markerRecoveryRequired=True, inspectionOnly=True, mutated=False, canonicalPassClaimed=False)) + return dict(ownerMetadataRestored=True, businessBytesChanged=False, canonicalPassClaimed=False) if action == 'stop': return {} if action == 'snapshot': - self.assertEqual(params, dict(quiescent=True, **({'metadataFaults': plan['metadataFaults'][node]} if len(events) == 2 else {}))) + snapshots += 1 + self.assertEqual(params, dict(quiescent=True, **({'metadataFaults': plan['metadataFaults'][node]} if snapshots == 1 else {}))) return copy.deepcopy(current[node]) if action == 'recover-fault': fault = next(f for f in plan['faults'] if f['faultId'] == params['faultId']) @@ -165,7 +171,7 @@ def remote(node, action, **params): states = [dict(node='vm000', state=dict(exists=True), roots=[dict(exists=True)] * 4)] result = interrupted_recovery.restore_and_isolate(fleet, plan, states, recover_environment.recovery_scene_matches) self.assertEqual(result['cleanup'], 'ISOLATED'); self.assertFalse(result['canonicalPassClaimed']) - self.assertEqual(events[:4], ['stop', 'snapshot', 'recover-fault', 'recover-fault']) + self.assertEqual(events[:6], ['owner-markers-audit', 'owner-markers-recover', 'stop', 'snapshot', 'recover-fault', 'recover-fault']) self.assertFalse(set(events) & {'start', 'cleanup', 'journal-recover'}) def test_real_opaque_archive_retries_all_moves_and_hashes_fault_backups_without_credentials(self): diff --git a/test_validation_heal_owner_recovery.py b/test_validation_heal_owner_recovery.py new file mode 100644 index 00000000..8cc87bd7 --- /dev/null +++ b/test_validation_heal_owner_recovery.py @@ -0,0 +1,164 @@ +"""Owner drift never authorizes a business write or bypasses archived byte checks.""" +from contextlib import ExitStack, contextmanager +import copy +import hashlib +import json +import os +from pathlib import Path +import socket +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import fleet_node, heal_fault, healthy, owner_recovery as recovery, retained_scene +from test_validation_heal import versioned_scene + + +class OwnerRecoveryTests(unittest.TestCase): + @contextmanager + def fixture(self): + with tempfile.TemporaryDirectory() as directory, ExitStack() as stack: + base = Path(directory).resolve(); run = 'gh-12345-1-h1' + state = base / 'state' / run; state.mkdir(parents=True, mode=0o700) + mounts = [base / f'disk{i}' for i in range(4)] + roots = [p / 'heal-validation' / run for p in mounts] + before, bucket, _ = versioned_scene(['objects/0000.bin', 'opaque/source.bin'], sizes=(100,)) + before = before['node1']; before.update(pid=0, quiescent=True) + owner = dict(runId=run, hostname=socket.gethostname(), unit='rustfs-validation-heal-' + run + '.service', + runtimeProfile='outcome-legacy', bucket=bucket, binarySha256=hashlib.sha256(b'candidate').hexdigest(), + inspectorSha256=hashlib.sha256(b'inspector').hexdigest(), mounts=[dict( + target=str(p), source=str(i), fstype='xfs', uuid=str(i), **{'maj:min': str(i)}) for i, p in enumerate(mounts)]) + before['binarySha256'] = owner['binarySha256'] + (state / 'owner.json').write_text(json.dumps(owner)); (state / 'rustfs').write_bytes(b'candidate') + (state / 'heal-meta-inspect').write_bytes(b'inspector'); (state / 'service.env').write_bytes(b'credential-not-to-read') + markers = {} + for disk, root in enumerate(roots): + root.mkdir(parents=True, mode=0o700); (root / 'quarantine').mkdir() + (root / 'owner.json').write_text(json.dumps(owner)) + info = (root / 'owner.json').stat(); markers[(info.st_dev, info.st_ino)] = (999, 988) + for entry in (e for e in before['files'] if e['disk'] == disk): + path = root / 'data' / entry['path']; path.parent.mkdir(parents=True, exist_ok=True) + raw = json.dumps(entry.get('metadata', {'shard': disk})).encode() + path.write_bytes(raw); entry.update(size=len(raw), sha256=hashlib.sha256(raw).hexdigest()) + path = root / 'data/.rustfs.sys/format.json'; path.parent.mkdir(parents=True) + raw = json.dumps(before['formats'][disk]['value']).encode(); path.write_bytes(raw) + before['formats'][disk]['sha256'] = hashlib.sha256(raw).hexdigest() + stack.enter_context(patch.object(heal_fault, 'mount_id', return_value=1)) + faults = [] + for disk in (0, 1): + with heal_fault.directory(roots[disk]) as fd: + binding = dict(schemaVersion=1, runId=run, bucketPrefix=bucket, mountId=1, + device=os.fstat(fd).st_dev, leaseExpiresAt='2020-01-01T00:00:00+00:00') + heal_fault.new_file(fd, 'fault-owner.json', heal_fault.encode(binding)) + relative = bucket + '/opaque/source.bin/xl.meta' + original = next(e for e in before['files'] if e['disk'] == disk and e['path'] == relative) + fault_id = heal_fault.bounded_fault_id('J02', 'original-opaque', disk) + receipt = heal_fault.apply_fault(fd, binding, relative, fault_id, 'bitflip', original['sha256'], 0) + faults.append(dict(disk=disk, faultId=fault_id, receipt=receipt)) + original_fstat, original_stat, original_lstat = os.fstat, Path.stat, Path.lstat + def mapped(info): + row = {key: getattr(info, key) for key in dir(info) if key.startswith('st_')} + row['st_uid'], row['st_gid'] = markers.get((info.st_dev, info.st_ino), (0, 0)) + return types.SimpleNamespace(**row) + stack.enter_context(patch.object(os, 'fstat', side_effect=lambda fd: mapped(original_fstat(fd)))) + stack.enter_context(patch.object(Path, 'stat', lambda p, **kw: mapped(original_stat(p, **kw)))) + stack.enter_context(patch.object(Path, 'lstat', lambda p: mapped(original_lstat(p)))) + stack.enter_context(patch.object(recovery.pwd, 'getpwnam', return_value=types.SimpleNamespace(pw_uid=999, pw_gid=988))) + stack.enter_context(patch.object(retained_scene, 'mounted_paths', return_value=[])) + changed = [] + def chown(fd, uid, gid): + info = original_fstat(fd); key = (info.st_dev, info.st_ino) + self.assertIn(key, markers); self.assertEqual((uid, gid), (0, 0)) + markers[key] = (0, 0); changed.append(key) + stack.enter_context(patch.object(os, 'fchown', side_effect=chown)) + node = dict(paths=lambda run_id: (state, roots), checked=fleet_node.checked, MOUNTS=mounts, + unit=lambda run_id: owner['unit'], socket=socket, mounts=lambda: copy.deepcopy(owner['mounts']), + stopped=lambda value: dict(MainPID='0', ActiveState='inactive'), digest=fleet_node.digest, + fault_module=lambda: heal_fault, retained_scene_backend=retained_scene) + node['testMarkerOwners'] = markers + request = dict(runId=run, action='owner-markers-audit', expectedOwner=owner, before=before, faults=faults) + yield state, roots, node, request, changed, chown + + def test_audit_and_recovery_preserve_all_bytes_and_durable_prechange_evidence(self): + with self.fixture() as (state, roots, node, request, changed, chown): + original = {p: (p.stat().st_ino, p.read_bytes()) for folder in (state, *roots) for p in folder.rglob('*') if p.is_file()} + opened = []; original_open = os.open + def observed_open(path, *args, **kw): + opened.append(str(path)); return original_open(path, *args, **kw) + with patch.object(os, 'open', side_effect=observed_open): + audit = recovery.run(request, node) + self.assertTrue(audit['markerRecoveryRequired']); self.assertEqual(changed, []) + self.assertFalse((state / 'owner-marker-recovery.intent.json').exists()) + result = recovery.run(dict(request, action='owner-markers-recover', expectedAudit=audit), node) + self.assertEqual(len(changed), 4); self.assertTrue(result['ownerMetadataRestored']) + self.assertFalse(result['businessBytesChanged']); self.assertFalse(result['canonicalPassClaimed']) + self.assertFalse(any('service.env' in p for p in opened)) + intent = json.loads((state / 'owner-marker-recovery.intent.json').read_text()) + self.assertEqual(intent['markers'], audit['markers']); self.assertEqual(intent['rootCensus'], audit['rootCensus']) + self.assertEqual(result['intentSha256'], recovery.sha(intent)) + for path, saved in original.items(): self.assertEqual((path.stat().st_ino, path.read_bytes()), saved) + again = recovery.run(request, node) + self.assertFalse(again['markerRecoveryRequired']); self.assertEqual(again['priorIntent'], intent) + + def test_recovery_retries_after_each_marker_change_without_rewriting_the_original_intent(self): + for lost_after in range(1, 5): + with self.subTest(lost_after=lost_after), self.fixture() as (state, roots, node, request, changed, chown): + audit = recovery.run(request, node) + def interrupted(fd, uid, gid): + chown(fd, uid, gid) + if len(changed) == lost_after: raise RuntimeError('lost remote acknowledgement') + with patch.object(os, 'fchown', side_effect=interrupted), self.assertRaises(RuntimeError): + recovery.run(dict(request, action='owner-markers-recover', expectedAudit=audit), node) + raw_intent = (state / 'owner-marker-recovery.intent.json').read_bytes() + current = recovery.run(request, node) + result = recovery.run(dict(request, action='owner-markers-recover', expectedAudit=current), node) + self.assertTrue(result['ownerMetadataRestored']); self.assertEqual(len(changed), 4) + self.assertEqual((state / 'owner-marker-recovery.intent.json').read_bytes(), raw_intent) + + def test_drift_and_unknown_authority_refuse_before_chown(self): + for poison in ('owner', 'binary', 'inspector', 'empty-prior', 'format', 'business', 'missing', 'extra', 'backup', 'intent', 'applied', + 'foreign-backup', 'mount', 'nested-mount', 'foreign-uid', 'central-uid', 'live', 'legacy-phase', 'gate', 'symlink', 'hardlink', 'audit-race'): + with self.subTest(poison=poison), self.fixture() as (state, roots, node, request, changed, _): + audit = recovery.run(request, node) + target = roots[0] / 'data' / request['expectedOwner']['bucket'] / 'objects/0000.bin/xl.meta' + if poison == 'owner': (roots[0] / 'owner.json').write_text('{}') + elif poison == 'binary': (state / 'rustfs').write_bytes(b'foreign') + elif poison == 'inspector': (state / 'heal-meta-inspect').write_bytes(b'foreign') + elif poison == 'empty-prior': + (state / 'owner-marker-recovery.intent.json').write_text('{}') + with self.assertRaises(ValueError): recovery.run(request, node) + self.assertEqual(changed, []) + continue + elif poison == 'format': (roots[0] / 'data/.rustfs.sys/format.json').write_text('{}') + elif poison == 'business': target.write_bytes(b'later acknowledged data') + elif poison == 'missing': target.unlink() + elif poison == 'extra': (target.parent / 'other').write_bytes(b'foreign') + elif poison in ('backup', 'intent', 'applied'): + suffix = {'backup': 'original', 'intent': 'intent.json', 'applied': 'applied.json'}[poison] + next((roots[0] / 'quarantine').glob('*.' + suffix)).write_bytes(b'foreign') + elif poison == 'foreign-backup': (roots[2] / 'quarantine/foreign').write_bytes(b'unknown') + elif poison == 'mount': node['mounts'] = lambda: [] + elif poison == 'nested-mount': retained_scene.mounted_paths.return_value = [roots[0] / 'data'] + elif poison in ('foreign-uid', 'central-uid'): + path = roots[0] / 'owner.json' if poison == 'foreign-uid' else state / 'owner.json' + info = path.stat(); node['testMarkerOwners'][(info.st_dev, info.st_ino)] = (777, 777) if poison == 'foreign-uid' else (999, 988) + elif poison == 'live': node['stopped'] = lambda owner: (_ for _ in ()).throw(ValueError('live PID')) + elif poison == 'legacy-phase': (state / 'J02-phase.json').write_text('{}') + elif poison == 'gate': (state / 'drive-gate.json').write_text('{}') + elif poison == 'symlink': target.unlink(); target.symlink_to(state / 'service.env') + elif poison == 'hardlink': os.link(target, target.parent / 'alias') + elif poison == 'audit-race': audit['rawPhysicalSha256'] = 'f' * 64 + with self.assertRaises((ValueError, KeyError)): recovery.run(dict(request, action='owner-markers-recover', expectedAudit=audit), node) + self.assertEqual(changed, []) + + def test_remote_adapter_bundles_both_auditors_and_descriptor_backend(self): + def execute(*args, **kwargs): + source = kwargs['input'].rsplit('\nmain(', 1)[0]; scope = dict(__name__='__test_node__') + exec(compile(source, 'owner-recovery-node', 'exec'), scope) + self.assertTrue(callable(scope['owner_recovery_backend'].run)) + self.assertTrue(callable(scope['retained_scene_backend'].census)) + self.assertTrue(callable(scope['fault_module']().read_file)) + return types.SimpleNamespace(returncode=0, stdout='{}', stderr='') + with tempfile.TemporaryDirectory() as directory, patch.object(healthy.subprocess, 'run', side_effect=execute): + healthy.Fleet(['vm000'], Path(directory), 'gh-12345-1-h1').remote('vm000', 'owner-markers-audit') From 72f14430cecd7cea4127dca3268ba1e83a73f795 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 16 Oct 2026 14:42:58 +0800 Subject: [PATCH 216/343] fix(validation): retain opaque scenes under private run parents --- scripts/validation_heal/fleet_node.py | 10 ++++-- scripts/validation_heal/retained_scene.py | 25 ++++++++++++-- test_validation_heal_outcome_recovery.py | 42 ++++++++++++++++++++++- 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 3d931ad7..a8f9acbf 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1344,15 +1344,19 @@ def marker_evidence(folder): "canonicalSha256": hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()} finally: os.close(descriptor) + retained_states = [checked(state.parent / name / state.name, state.parent).exists() + for name in ("retained", "retained-" + state.name)] + retained_roots = [[checked(root.parent.parent / name / root.name, root.parent.parent).exists() + for name in ("heal-validation-retained", "heal-validation-retained-" + root.name)] for root in roots] + require(sum(retained_states) <= 1 and all(sum(row) <= 1 for row in retained_roots), + "namespace has both shared and per-run retained copies") return {"runId": request["runId"], "service": service, "state": {"path": str(state), "exists": state.exists(), "ownerMarker": (state / "owner.json").is_file(), "ownerEvidence": marker_evidence(state)}, "roots": [{"path": str(root), "exists": root.exists(), "ownerMarker": (root / "owner.json").is_file(), "ownerEvidence": marker_evidence(root), "data": (root / "data").is_dir()} for root in roots], - "retained": {"state": checked(state.parent / "retained" / state.name, state.parent).exists(), - "roots": [checked(root.parent.parent / "heal-validation-retained" / root.name, - root.parent.parent).exists() for root in roots]}, + "retained": {"state": any(retained_states), "roots": [any(row) for row in retained_roots]}, "inspectionOnly": True, "mutated": False} diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py index 15287e12..fbde6284 100644 --- a/scripts/validation_heal/retained_scene.py +++ b/scripts/validation_heal/retained_scene.py @@ -49,7 +49,10 @@ def private_json(path, *, root_owned=True): os.close(fd) -def destinations(state, roots): +def destinations(state, roots, *, per_run_parent=False): + if per_run_parent: + return state.parent / ("retained-" + state.name) / state.name, tuple( + root.parent.parent / ("heal-validation-retained-" + root.name) / root.name for root in roots) return state.parent / "retained" / state.name, tuple( root.parent.parent / "heal-validation-retained" / root.name for root in roots) @@ -153,8 +156,14 @@ def mounted_paths(): def inspect(request, node): state, roots = node["paths"](request["runId"]) - retained_state, retained_roots = destinations(state, roots) expected = request["expectedOwner"] + per_run = expected.get("runtimeProfile") == "outcome-legacy" + retained_state, retained_roots = destinations(state, roots, per_run_parent=per_run) + if per_run: + old_state, old_roots = destinations(state, roots) + require(not any(node["checked"](path, state.parent if index == 4 else node["MOUNTS"][index]).exists() + for index, path in enumerate((*old_roots, old_state))), + "J02 shared archive already exists; refusing another retention layout") require(isinstance(expected, dict) and expected.get("runId") == request["runId"] and expected.get("unit") == node["unit"](request["runId"]) and expected.get("hostname") == node["socket"].gethostname() @@ -197,6 +206,8 @@ def inspect(request, node): "retained scene still has a gate or replacement intent") witnesses.append(census(folder, expected, state=index == 4, node=node)) baseline = {"runId": request["runId"], "ownerSha256": sha(expected), "roots": witnesses[:4], "state": witnesses[4]} + if per_run: + baseline["archiveLayout"] = "per-run-parent" intent = retained_state.parent / (request["runId"] + ".intent.json") node["checked"](intent, state.parent) if expected["runtimeProfile"] in {"root-restart", "outcome-grace", "outcome-legacy"}: @@ -252,7 +263,15 @@ def run(request, node): require(encoded(final[0]) == encoded(baseline) and "active" not in final[1], "retained scene post-rename inventory differs") locations = final[1] - return {"baseline": baseline, "locations": locations, "originalEvidenceComplete": False, + result = {"baseline": baseline, "locations": locations, "originalEvidenceComplete": False, "originalSceneRestored": False, "canonicalPassClaimed": False, "backupRestorationUsed": False, "credentialContentsRead": False, "inspectionOnly": request["action"] == "retained-scene-audit", "mutated": mutated, "activeNamespaceIsolated": "active" not in locations} + if request["expectedOwner"].get("runtimeProfile") == "outcome-legacy": + result["privateArchiveParents"] = [] + for target in (*retained_roots, retained_state): + info = target.parent.lstat() if target.parent.exists() else None + result["privateArchiveParents"].append({"exists": False} if info is None else { + "exists": True, "uid": info.st_uid, "gid": info.st_gid, "mode": stat.S_IMODE(info.st_mode), + "device": info.st_dev, "inode": info.st_ino}) + return result diff --git a/test_validation_heal_outcome_recovery.py b/test_validation_heal_outcome_recovery.py index cdf83d61..2bb2e52e 100644 --- a/test_validation_heal_outcome_recovery.py +++ b/test_validation_heal_outcome_recovery.py @@ -8,7 +8,7 @@ import unittest from unittest.mock import patch -from scripts.validation_heal import heal_fault, interrupted_recovery, outcome_recovery, recover_environment, retained_scene +from scripts.validation_heal import fleet_node, heal_fault, interrupted_recovery, outcome_recovery, recover_environment, retained_scene from scripts.validation_heal.healthy import physical_content from scripts.validation_heal.shards import map_target from scripts.validation_heal.versions import metadata_target @@ -181,6 +181,12 @@ def test_real_opaque_archive_retries_all_moves_and_hashes_fault_backups_without_ state, roots, node, request = helper.scene(temporary) roots[0].mkdir(parents=True); owner = request['expectedOwner']; owner['runtimeProfile'] = 'outcome-legacy' for folder in (*roots, state): emit(folder / 'owner.json', owner) + old_state, old_roots = retained_scene.destinations(state, roots) + old_parents = [path.parent for path in (*old_roots, old_state)] + for parent in old_parents: + parent.mkdir(mode=0o755, parents=True) + (parent / 'unrelated-scene').write_bytes(b'preserve-shared-parent') + preserved = [(p.stat().st_ino, p.stat().st_mode, (p / 'unrelated-scene').read_bytes()) for p in old_parents] fault_id = heal_fault.bounded_fault_id('J02', 'original-opaque', 0) for suffix in ('original', 'intent.json', 'applied.json', 'failed-' + 'a' * 64): (roots[1] / 'quarantine' / (fault_id + '.' + suffix)).write_bytes(b'fault-evidence') @@ -203,10 +209,22 @@ def move(source, target): if count == lost_after: raise RuntimeError('lost ACK') change = dict(request, action='retained-scene-isolate', expectedBaseline=baseline) with patch.object(retained_scene, 'rename_new', side_effect=move), self.assertRaises(RuntimeError): retained_scene.run(change, node) + def command(*args, **kw): + return types.SimpleNamespace(stdout='0' if 'MainPID' in args else 'inactive') + with patch.object(fleet_node, 'paths', return_value=(state, roots)), patch.object(fleet_node, 'command', side_effect=command): + observed = fleet_node.namespace_state(dict(runId=request['runId'], action='namespace-state')) + self.assertEqual(observed['retained']['roots'], [i < lost_after for i in range(4)]) + self.assertEqual(observed['retained']['state'], lost_after == 5) node['snapshot'] = lambda value: self.fail('partial archive must use durable original digest') result = retained_scene.run(change, node) self.assertEqual(result['locations'], ['retained'] * 5); self.assertEqual(result['baseline'], baseline) self.assertFalse(result['canonicalPassClaimed']); self.assertFalse(result['credentialContentsRead']) + self.assertEqual(result['baseline']['archiveLayout'], 'per-run-parent') + self.assertEqual(len(result['privateArchiveParents']), 5) + self.assertTrue(all(p['exists'] and p['uid'] == 0 and p['mode'] == 0o700 for p in result['privateArchiveParents'])) + checked = retained_scene.run(request, node) + self.assertEqual(checked['privateArchiveParents'], result['privateArchiveParents']) + self.assertEqual([(p.stat().st_ino, p.stat().st_mode, (p / 'unrelated-scene').read_bytes()) for p in old_parents], preserved) def test_real_file_recovery_preserves_corrupt_bytes_and_refuses_later_data(self): for drift in (None, 'intent', 'backup', 'target'): @@ -234,3 +252,25 @@ def test_real_file_recovery_preserves_corrupt_bytes_and_refuses_later_data(self) preserved = root / 'quarantine' / (fault_id + '.failed-' + hashlib.sha256(corrupt).hexdigest()) self.assertEqual(preserved.read_bytes(), corrupt) self.assertFalse(heal_fault.recover_fault(fd, owner, fault_id, expected_receipt_sha256=receipt_sha)['restoredFromBackup']) + + def test_opaque_archive_refuses_shared_copies_and_namespace_inspection_rejects_both_layouts(self): + helper = scene_tests.RetainedSceneTests() + for collision in ('root', 'state'): + with self.subTest(collision=collision), tempfile.TemporaryDirectory() as temporary: + state, roots, node, request = helper.scene(temporary) + roots[0].mkdir(parents=True); owner = request['expectedOwner']; owner['runtimeProfile'] = 'outcome-legacy' + for folder in (*roots, state): emit(folder / 'owner.json', owner) + request['expectedOriginalSceneSha256'] = 'a' * 64 + shared_state, shared_roots = retained_scene.destinations(state, roots) + private_state, private_roots = retained_scene.destinations(state, roots, per_run_parent=True) + shared = shared_roots[0] if collision == 'root' else shared_state + private = private_roots[0] if collision == 'root' else private_state + shared.mkdir(parents=True) + with helper.root_stats(), patch.object(retained_scene, 'rename_new') as move, self.assertRaisesRegex(ValueError, 'shared archive already exists'): + retained_scene.run(request, node) + move.assert_not_called(); self.assertFalse(private.parent.exists()) + private.mkdir(parents=True) + def command(*args, **kw): return types.SimpleNamespace(stdout='0' if 'MainPID' in args else 'inactive') + with patch.object(fleet_node, 'paths', return_value=(state, roots)), patch.object(fleet_node, 'command', side_effect=command), \ + self.assertRaisesRegex(ValueError, 'both shared and per-run'): + fleet_node.namespace_state(dict(runId=request['runId'], action='namespace-state')) From 52a69497690f9fbd8beab0e9e4f7656452efbece Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 16 Oct 2026 14:51:46 +0800 Subject: [PATCH 217/343] fix(validation): finish deferred branch at its frozen source boundary --- docs/validation-heal-cases.md | 1 + scripts/validation_heal/outcome.py | 9 ++++++++ scripts/validation_heal/recipes.json | 3 ++- test_validation_heal_outcome_grace.py | 31 +++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index 930f1b1d..2e64d836 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1818,6 +1818,7 @@ Blocked reason (`EVIDENCE_MIXED_OUTCOME_ORCHESTRATION`): 已有 mixed healthy/re 6. Manual diagnostic entry: explicitly select heal.outcome-counters-and-error-truth through Functional Chain with the frozen candidate and candidate-bound manager/inspector tools. The default catalog policy stays evidence-blocked; even all passing diagnostic branches retain complete=false, canonicalCasePassClaimed=false and a blocking gate verdict until independent acceptance. 7. For a pre-replay public-manifest refusal, recover only after the complete SDK request/ACK ledger proves all writes acknowledged and no DELETE or replay occurred. Bind all four original physical members and the reserved owner, stop the exact candidate, then preserve all roots and state using durable same-filesystem non-overwriting rename. Partial archive retries require the same durable inventory; missing roots or drift retain DIRTY. Isolation is not product repair or restoration and never rewrites the failed case. 8. Manual J02 candidate admission is limited to the original 072b57b37559fbe626aeda8abf61617fdcbdd926 and corrected 1c7fc818dd858247c45b39542abd8291b1e6e31b sources. Each execution requires the matching immutable package and independently audited inspector/manager bundle. A corrected-candidate run is new evidence; it cannot replace an original failure. The full exact-version, physical, timing, retry and responsibility oracles remain unchanged, and all passing diagnostics still require independent canonical acceptance. +9. The deferred source branch follows Batch.deferred in frozen validation script 524eb5b32c3236497fdd8a9271df0494fa3d9974: complete the exact-version grace, natural-expiry and new-token cleanup procedure, then finalize this branch. An additional R-1 metadata existence-check diagnostic is not part of that source procedure and does not gate its verdict. Its historical failures and strict replay assertions remain retained; original source failures, missed-window NOT_OBSERVED and uncertain-mutation DIRTY are unchanged. Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [LOCK_WINDOW](validation-heal-operations.md#LOCK_WINDOW) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [BACKGROUND](validation-heal-operations.md#BACKGROUND) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) diff --git a/scripts/validation_heal/outcome.py b/scripts/validation_heal/outcome.py index e97b2f6e..ead8f38e 100644 --- a/scripts/validation_heal/outcome.py +++ b/scripts/validation_heal/outcome.py @@ -508,7 +508,16 @@ def run_branch(args, topology, nodes, fixture, branch, candidate): pending = {"operation": "original-version-specific-grace-source-procedure", "bucket": bucket} expected, baseline, result["originalGraceProcedure"] = original_grace_probe( probe, fleet, binding, directory, expected, incarnation, active, unchanged_token) + source = result["originalGraceProcedure"] + require(source.get("originalGraceSourceProcedure") is True and source.get("sourceDiagnosticComplete") is True + and source.get("verdict") in {"PASS", "NOT_OBSERVED"} + and all(source.get(name) is False for name in ("fixtureRescueIsProductRepair", "sameTokenRecoveryClaimed", + "globalClockChanged", "metadataAgeRewritten", "canonicalPassClaimed")), + "J02 original grace procedure has incomplete source evidence") pending = None + result.update(verdict=source["verdict"], recoveryIsProductRepair=False, + sourceProcedure="heal_status_batch.Batch.deferred") + return result original = fleet.snapshot("healthy-physical-baseline", quiescent=True) plan = legacy_manifest(original, bucket, expected) if branch == "unknown-negative" else fault_plan(original, bucket, expected, branch) fleet.record("independent-manifest-and-frozen-fault-plan", plan) diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index 79831e8e..eb805aa1 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -3380,7 +3380,8 @@ "grace 分支先用 SDK PUT 建立两个真实版本并验证完整分片；保存一个成员的原对象树后，SDK DELETE 精确历史版本，逐盘验证仅当前版本存在且其逻辑元数据、数据片及控制对象未变。只向原成员回放原对象树；实际候选进程固定 grace=120 秒，不修改元数据年龄或时钟。原任务须在窗口内完成，精确报告旧版 deferred、当前版 verified_healthy 及 retryNotBefore；自然等待过期后原 token 仍不变，新 token 才能清理旧版。清理后先核对全部物理字节等于 DELETE 后权威现场，再跨节点 GET/404 和复查旧 token，最后才做 fixture finalization。错过窗口保留 NOT_OBSERVED；异常保留现场并标 DIRTY；救援仅恢复 DELETE 后权威树，不复活已删除版本。", "Manual diagnostic entry: explicitly select heal.outcome-counters-and-error-truth through Functional Chain with the frozen candidate and candidate-bound manager/inspector tools. The default catalog policy stays evidence-blocked; even all passing diagnostic branches retain complete=false, canonicalCasePassClaimed=false and a blocking gate verdict until independent acceptance.", "For a pre-replay public-manifest refusal, recover only after the complete SDK request/ACK ledger proves all writes acknowledged and no DELETE or replay occurred. Bind all four original physical members and the reserved owner, stop the exact candidate, then preserve all roots and state using durable same-filesystem non-overwriting rename. Partial archive retries require the same durable inventory; missing roots or drift retain DIRTY. Isolation is not product repair or restoration and never rewrites the failed case.", - "Manual J02 candidate admission is limited to the original 072b57b37559fbe626aeda8abf61617fdcbdd926 and corrected 1c7fc818dd858247c45b39542abd8291b1e6e31b sources. Each execution requires the matching immutable package and independently audited inspector/manager bundle. A corrected-candidate run is new evidence; it cannot replace an original failure. The full exact-version, physical, timing, retry and responsibility oracles remain unchanged, and all passing diagnostics still require independent canonical acceptance." + "Manual J02 candidate admission is limited to the original 072b57b37559fbe626aeda8abf61617fdcbdd926 and corrected 1c7fc818dd858247c45b39542abd8291b1e6e31b sources. Each execution requires the matching immutable package and independently audited inspector/manager bundle. A corrected-candidate run is new evidence; it cannot replace an original failure. The full exact-version, physical, timing, retry and responsibility oracles remain unchanged, and all passing diagnostics still require independent canonical acceptance.", + "The deferred source branch follows Batch.deferred in frozen validation script 524eb5b32c3236497fdd8a9271df0494fa3d9974: complete the exact-version grace, natural-expiry and new-token cleanup procedure, then finalize this branch. An additional R-1 metadata existence-check diagnostic is not part of that source procedure and does not gate its verdict. Its historical failures and strict replay assertions remain retained; original source failures, missed-window NOT_OBSERVED and uncertain-mutation DIRTY are unchanged." ], "commands": [ "SEED_DB", diff --git a/test_validation_heal_outcome_grace.py b/test_validation_heal_outcome_grace.py index f501f9fe..38be2d41 100644 --- a/test_validation_heal_outcome_grace.py +++ b/test_validation_heal_outcome_grace.py @@ -291,6 +291,37 @@ def test_uncertain_original_grace_procedure_retains_namespace_and_stops_next_bra self.assertTrue(retained['namespacesRetained']) self.assertGreaterEqual(fleet.stop.call_count, 2) + def test_completed_source_grace_does_not_run_a_second_non_source_fault(self): + for verdict in ('PASS', 'NOT_OBSERVED', 'FAIL', 'incomplete'): + with self.subTest(verdict=verdict), tempfile.TemporaryDirectory() as temporary, ExitStack() as stack: + root = Path(temporary); rc = root / 'rc'; rc.write_bytes(b'cli') + args = types.SimpleNamespace(evidence=root / 'evidence', rc_bin=rc, binary_sha='b' * 64, + testing_commit='a' * 40, tool_directory=root / 'tools') + factory = stack.enter_context(patch.object(outcome, 'Fleet')); fleet = factory.return_value + fleet.reserved = ['node1']; fleet.record.side_effect = lambda label, value: value + fleet.cleanup.return_value = True + stack.enter_context(patch.dict(os.environ, GITHUB_RUN_ID='12345', GITHUB_RUN_ATTEMPT='1')) + for name in ('Probe', 'Recorder', 'verify_inspector', 'verify_quorum_contract', 'validate_topology', + 'verify_seed', 'idle_fleet', 'run_task'): + stack.enter_context(patch.object(outcome, name)) + stack.enter_context(patch.object(outcome, 'seed', return_value=[])) + stack.enter_context(patch.object(outcome, 'validate_version_receipts', return_value={'bucketIncarnationId': INCARNATION})) + source = dict(originalGraceSourceProcedure=True, verdict=verdict, sourceDiagnosticComplete=verdict != 'incomplete', fixtureRescueIsProductRepair=False, + sameTokenRecoveryClaimed=False, globalClockChanged=False, metadataAgeRewritten=False, canonicalPassClaimed=False) + stack.enter_context(patch.object(outcome, 'original_grace_probe', return_value=([], {}, source))) + supplemental = stack.enter_context(patch.object(outcome, 'fault_plan', side_effect=AssertionError('unexpected extra fault'))) + result = outcome.run_branch(args, 'H1', ['node1'], 'DB', 'deferred', + {'source_sha': outcome.CANDIDATE, 'package_sha256': 'c' * 64}) + supplemental.assert_not_called() + if verdict in ('PASS', 'NOT_OBSERVED'): + self.assertEqual((result['verdict'], result['cleanup']), (verdict, 'PASS')) + self.assertEqual(result['sourceProcedure'], 'heal_status_batch.Batch.deferred') + fleet.cleanup.assert_called_once() + else: + self.assertEqual((result['verdict'], result['cleanup']), ('FAIL', 'FAIL')) + fleet.cleanup.assert_not_called() + self.assertFalse(any(call.args[0] == 'healthy-physical-baseline' for call in fleet.snapshot.call_args_list)) + def test_manual_diagnostic_entry_cannot_certify_canonical_even_if_driver_returns_zero(self): root = Path(__file__).resolve().parent dry = subprocess.run(['bash', str(root / 'scripts/validation_heal/run_case.sh'), '--validation-case', From 9375790b194f1a7b4da374d5ee953c8e0dccc139 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 16 Oct 2026 15:04:05 +0800 Subject: [PATCH 218/343] fix(validation): retain acknowledged J02 scenes before legacy writes --- scripts/validation_heal/healthy.py | 2 +- scripts/validation_heal/outcome_recovery.py | 130 +++++++++++++++--- .../validation_heal/recover_environment.py | 8 +- test_validation_heal_legacy_recovery.py | 87 ++++++++++++ 4 files changed, 202 insertions(+), 25 deletions(-) create mode 100644 test_validation_heal_legacy_recovery.py diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 88d0ab22..17f82c2a 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -208,7 +208,7 @@ def remote(self, node, action, **params): source += "\nimport types\ngeneration_backend = types.ModuleType('generation_fault')\nexec(" + repr( NODE_SOURCE.with_name("generation_fault.py").read_text()) + ", generation_backend.__dict__)\n" if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline", - "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure", "outcome-legacy-phase"} or action.startswith("generation-") or action.startswith("orphan-") or action.startswith(("journal-", "incarnation-", "owner-markers-")): + "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure", "outcome-legacy-phase", "start"} or action.startswith("generation-") or action.startswith("orphan-") or action.startswith(("journal-", "incarnation-", "owner-markers-")): source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" if action.startswith(("journal-", "incarnation-")): diff --git a/scripts/validation_heal/outcome_recovery.py b/scripts/validation_heal/outcome_recovery.py index b81f1cdb..96313651 100644 --- a/scripts/validation_heal/outcome_recovery.py +++ b/scripts/validation_heal/outcome_recovery.py @@ -213,27 +213,7 @@ def one(label): require(len(wanted) == 2 and {e['key'] for e in wanted} == {'objects/0000.bin', 'opaque/source.bin'} and all(e['versionId'] == 'null' for e in wanted), 'opaque source ACK manifest differs') verify_seed(wanted, one('original-opaque-ACK-full-S3')) - for name, operations in (('fixture', ('create_bucket', 'put_object')), ('original-opaque-fixture', ('put_object',))): - requests = sorted((folder / name).glob('*-request.json')); acks = sorted((folder / name).glob('*-ack.json')) - require(len(requests) == len(acks) == len(operations) - and set((folder / name).iterdir()) == set(requests + acks), 'opaque original SDK ledger is incomplete') - for sequence, (request_path, ack_path, operation) in enumerate(zip(requests, acks, operations, strict=True), 1): - request, ack = json.loads(request_path.read_text()), json.loads(ack_path.read_text()); observed = ack.get('observation') or {} - require(type(request.get('sequence')) is int and type(ack.get('sequence')) is int - and request['sequence'] == ack['sequence'] == sequence and request.get('runId') == ack.get('runId') == run_id - and ack.get('requestFile') == request_path.name - and ack.get('requestSha256') == hashlib.sha256(request_path.read_bytes()).hexdigest() - and observed.get('operation') == request.get('operation') == operation - and observed.get('params') == request.get('params') and request['params'].get('Bucket') == bucket - and ack.get('operationCheck') == 'PASS' and observed.get('acknowledged') is True - and observed.get('clientCallInvoked') is True and observed.get('resultUnknown') is False - and observed.get('response', {}).get('ResponseMetadata', {}).get('HTTPStatusCode') == 200, - 'opaque original mutation is unknown or rebound') - if operation == 'put_object': - key = 'opaque/source.bin' if name == 'original-opaque-fixture' else 'objects/0000.bin' - expected = next(e for e in wanted if e['key'] == key) - require(request['params'].get('Key') == key and request.get('payload') == observed.get('payload') == expected['payload'] - and public_version_id(observed['response'].get('VersionId')) == expected['versionId'], 'opaque original PUT differs from ACK') + validate_opaque_ledger(folder, run_id, bucket, wanted) validate_topology(original, 'H1') target = map_target(original, bucket, 'opaque/source.bin', 'data', 1, 'null', latest_required=False) targets = [metadata_target(original, {**target, **member}) for member in target['cohort']] @@ -265,3 +245,111 @@ def one(label): return [dict(kind='interrupted-j02-opaque', runId=run_id, nodes=['vm000'], owners={'vm000': owner}, before=original, failure=failure, faults=faults, metadataFaults=witnesses, buckets=[bucket], binarySha256=binding['candidateBinarySha256'], originalCaseSha256=hashlib.sha256(raw).hexdigest())] + + +def validate_opaque_ledger(folder, run_id, bucket, wanted): + for name, operations in (('fixture', ('create_bucket', 'put_object')), ('original-opaque-fixture', ('put_object',))): + requests = sorted((folder / name).glob('*-request.json')); acks = sorted((folder / name).glob('*-ack.json')) + require(len(requests) == len(acks) == len(operations) + and set((folder / name).iterdir()) == set(requests + acks), 'opaque original SDK ledger is incomplete') + for sequence, (request_path, ack_path, operation) in enumerate(zip(requests, acks, operations, strict=True), 1): + request, ack = json.loads(request_path.read_text()), json.loads(ack_path.read_text()); observed = ack.get('observation') or {} + require(type(request.get('sequence')) is int and type(ack.get('sequence')) is int + and request['sequence'] == ack['sequence'] == sequence and request.get('runId') == ack.get('runId') == run_id + and ack.get('requestFile') == request_path.name + and ack.get('requestSha256') == hashlib.sha256(request_path.read_bytes()).hexdigest() + and observed.get('operation') == request.get('operation') == operation + and observed.get('params') == request.get('params') and request['params'].get('Bucket') == bucket + and ack.get('operationCheck') == 'PASS' and observed.get('acknowledged') is True + and observed.get('clientCallInvoked') is True and observed.get('resultUnknown') is False + and observed.get('response', {}).get('ResponseMetadata', {}).get('HTTPStatusCode') == 200, + 'opaque original mutation is unknown or rebound') + if operation == 'put_object': + key = 'opaque/source.bin' if name == 'original-opaque-fixture' else 'objects/0000.bin' + expected = next(e for e in wanted if e['key'] == key) + require(request['params'].get('Key') == key and request.get('payload') == observed.get('payload') == expected['payload'] + and public_version_id(observed['response'].get('VersionId')) == expected['versionId'], 'opaque original PUT differs from ACK') + + +def acknowledged_legacy_plan(artifact, failed_run, candidate): + case = 'heal.outcome-counters-and-error-truth' + paths = list(artifact.glob('*/run/*/cases/*.json')) + require(len(paths) == 1 and paths[0].name == case + '.json', 'legacy recovery requires one unmixed J02 result') + raw = paths[0].read_bytes(); result = json.loads(raw); provenance = result.get('provenance') or {} + require(result.get('case_id') == case and result.get('verdict') == result.get('cleanup_status') == 'FAIL' + and candidate.get('source_sha') == provenance.get('source_commit') == '1c7fc818dd858247c45b39542abd8291b1e6e31b' + and candidate.get('package_sha256') == provenance.get('candidate_sha256'), 'legacy recovery candidate differs') + evidence = paths[0].parents[1] / 'evidence' / case + summary = json.loads((evidence / 'healthy-summary.json').read_text()); rows = summary.get('results') or [] + require(summary.get('caseId') == case and summary.get('cleanup') == 'FAIL' and summary.get('complete') is False + and summary.get('canonicalCasePassClaimed') is False and len(rows) == 5 + and [(r.get('topology'), r.get('fixture'), r.get('branch')) for r in rows] == [ + ('H1', 'DB', branch) for branch in ('mixed-healthy-repaired-failed', 'dry-run', 'deferred', + 'readonly-terminal-failure', 'unknown-negative')] + and all(r.get('cleanup') == 'PASS' for r in rows[:-1]) and rows[-1].get('cleanup') == 'FAIL' + and rows[-1].get('verdict') == 'FAIL' and rows[-1].get('failure', {}).get('type') == 'ValueError' + and rows[-1]['failure']['message'].startswith('Heal node vm000 start failed:') + and rows[-1]['failure']['message'].endswith('ImportError: attempted relative import with no known parent package\n'), + 'legacy recovery requires the exact pre-SDK start refusal') + folder = evidence / 'DB/H1/unknown-negative'; records = sorted(folder.glob('*.json')) + require(len(records) == 71 and all(p.name.startswith(f'{i:05d}-') for i, p in enumerate(records, 1)) + and [p.name[6:-5] for p in records[-7:]] == ['original-opaque-after-restored-business-reads', + 'protected-control-before-legacy-writing', 'one-way-legacy-write-phase', 'failure-scene-error', + 'quiescent', 'retain-unknown-seed-receipt', 'branch-result'] + and set(folder.iterdir()) == set(records) | {folder / name for name in ('fixture', 'original-opaque-fixture')}, + 'legacy recovery has another task or SDK mutation boundary') + def one(label): + found = [p for p in records if p.name[6:-5] == label] + require(len(found) == 1, 'legacy recovery lacks exact ' + label) + return json.loads(found[0].read_text()) + binding, original, reserved = one('binding'), one('original-opaque-complete-physical'), one('reserved') + run_id, bucket = binding.get('runId', ''), binding.get('bucketPrefix', '') + owner = reserved.get('owner') or {} + require(re.fullmatch(rf'gh-{failed_run}-[1-9][0-9]*-h1', run_id) and set(original) == {'vm000'} + and re.fullmatch(r'rv-heal-[0-9a-f]{16}', bucket) + and binding.get('candidateCommit') == candidate['source_sha'] + and binding.get('candidatePackageSha256') == candidate['package_sha256'] + and binding.get('candidateBinarySha256') == provenance.get('binary_sha256') + and binding.get('harnessCommit') == provenance.get('testing_commit') + and reserved.get('node') == 'vm000' and reserved.get('reserved') is True + and owner.get('runId') == run_id and owner.get('bucket') == bucket and owner.get('runtimeProfile') == 'outcome-legacy' + and owner.get('binarySha256') == binding['candidateBinarySha256'] and len(owner.get('mounts', [])) == 4, + 'legacy recovery reservation differs') + require(one('retain-unknown-seed-receipt') == dict(state='DIRTY', namespacesRetained=True, + request=dict(operation='append-real-legacy-fixture', bucket=bucket)), 'legacy pending boundary differs') + require(one('one-way-legacy-write-phase') == dict(node='vm000', runId=run_id, binarySha256=owner['binarySha256'], + phase='legacy', oneWay=True, candidateUnchanged=True, credentialsDisclosed=False, otherRuntimeSettingsUnchanged=True), + 'legacy original phase acknowledgement differs') + wanted = one('original-opaque-complete-ACK-manifest')['expected'] + require(len(wanted) == 2 and {e['key'] for e in wanted} == {'objects/0000.bin', 'opaque/source.bin'} + and all(e['versionId'] == 'null' for e in wanted), 'legacy pre-PUT manifest differs') + validate_opaque_ledger(folder, run_id, bucket, wanted) + verify_seed(wanted, one('original-opaque-ACK-full-S3')); verify_seed(wanted, one('original-opaque-restored-full-S3')) + validate_topology(original, 'H1') + for entry in wanted: + target = map_target(original, bucket, entry['key'], 'data', 1, 'null', latest_required=False) + require(len(target['cohort']) == 4 and target['logical']['size'] == entry['payload']['size'], 'legacy original cohort differs') + target = map_target(original, bucket, 'opaque/source.bin', 'data', 1, 'null', latest_required=False) + targets = [metadata_target(original, {**target, **member}) for member in target['cohort']] + require(one('original-opaque-metadata-fault-plan') == targets, 'legacy original metadata target plan differs') + injected = [json.loads(p.read_text()) for p in records if p.name[6:-5] == 'original-opaque-byte-zero-bitflip'] + rescued = [json.loads(p.read_text()) for p in records if p.name[6:-5] == 'restore-fixture-without-relabeling-original-task'] + require(len(injected) == len(rescued) == 4, 'legacy original rescue receipt census differs') + for i, (receipt, proof, target) in enumerate(zip(injected, rescued, targets, strict=True)): + fault_id = bounded_fault_id('J02', 'original-opaque', i) + require(receipt.get('faultId') == proof.get('faultId') == fault_id and receipt.get('node') == proof.get('node') == 'vm000' + and receipt.get('runId') == proof.get('runId') == run_id and receipt.get('relativePath') == target['relativePath'] + and receipt.get('before') == {k:target[k] for k in ('sha256', 'size')} + and receipt.get('kind') == 'bitflip' and type(receipt.get('offset')) is int and receipt['offset'] == 0 + and proof.get('restoredFromBackup') is True and proof.get('physicalRepairPassAllowed') is False, + 'legacy original rescue is not bound to its source fault') + for label in ('original-opaque-restored-original-bytes', 'original-opaque-after-restored-business-reads', + 'protected-control-before-legacy-writing'): + value = one(label) + require(physical_content(value) == physical_content(original) + and value['vm000'].get('quiescent') is True and type(value['vm000'].get('pid')) is int + and value['vm000']['pid'] == 0 and value['vm000'].get('binarySha256') == owner['binarySha256'], + 'legacy original business authority advanced before phase') + return [dict(kind='acknowledged-j02-before-legacy-put', runId=run_id, nodes=['vm000'], owners={'vm000':owner}, + before=original, binarySha256=owner['binarySha256'], buckets=[bucket], + originalCaseSha256=hashlib.sha256(raw).hexdigest())] diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index bf7e219f..cc6ce463 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -13,12 +13,12 @@ from .healthy import Fleet, physical_content from .heal_probe import require, save from .interrupted_recovery import checkpoint_plan, restore_and_isolate - from .outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace, interrupted_opaque_plan + from .outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace, interrupted_opaque_plan, acknowledged_legacy_plan except ImportError: from healthy import Fleet, physical_content from heal_probe import require, save from interrupted_recovery import checkpoint_plan, restore_and_isolate - from outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace, interrupted_opaque_plan + from outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace, interrupted_opaque_plan, acknowledged_legacy_plan def quota_recovery_plan(artifact, failed_run, candidate): @@ -144,6 +144,8 @@ def network_recovery_plan(artifact, failed_run, candidate): def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/cases/*.json")): + if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/evidence/heal.outcome-counters-and-error-truth/DB/H1/unknown-negative/*-one-way-legacy-write-phase.json")): + return acknowledged_legacy_plan(artifact, failed_run, candidate) if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/evidence/heal.outcome-counters-and-error-truth/DB/H1/unknown-negative/*-original-opaque-byte-zero-bitflip.json")): return interrupted_opaque_plan(artifact, failed_run, candidate) return acknowledged_grace_plan(artifact, failed_run, candidate) @@ -657,7 +659,7 @@ def main(): has_retained = any(item.get("retained", {}).get("state") or any(item.get("retained", {}).get("roots", [])) for item in states) all_absent = all(not item["state"]["exists"] and not any(root["exists"] for root in item["roots"]) for item in states) - if plan.get("kind") == "acknowledged-j02-before-replay": + if plan.get("kind") in {"acknowledged-j02-before-replay", "acknowledged-j02-before-legacy-put"}: complete.append({"failedRunId": args.failed_run_id, **isolate_acknowledged_grace(fleet, plan, states)}) continue if plan.get("kind") in {"interrupted-checkpoint-before-admission", "interrupted-j02-opaque"}: diff --git a/test_validation_heal_legacy_recovery.py b/test_validation_heal_legacy_recovery.py new file mode 100644 index 00000000..e97fcc02 --- /dev/null +++ b/test_validation_heal_legacy_recovery.py @@ -0,0 +1,87 @@ +"""An acknowledged one-way phase before any legacy SDK write can only be isolated.""" +import copy +import json +from pathlib import Path +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import healthy, outcome_recovery, recover_environment, heal_fault +import test_validation_heal_outcome_recovery as opaque_tests + + +class LegacyRecoveryTests(unittest.TestCase): + def fixture(self, root): + candidate, folder = opaque_tests.OpaqueRecoveryTests().fixture(root) + def read(path): return json.loads(path.read_text()) + records = sorted(folder.glob('*.json')); original = read(records[26]); targets = read(records[31]) + data = {i: read(p) for i, p in enumerate(records[:32], 1)} + for path in records: path.unlink() + labels = [p.name[6:-5] for p in records[:32]] + [ + 'original-opaque-byte-zero-bitflip', 'original-opaque-physical-injection', + 'original-opaque-byte-zero-bitflip', 'original-opaque-physical-injection', + 'original-opaque-byte-zero-bitflip', 'original-opaque-physical-injection', + 'original-opaque-byte-zero-bitflip', 'original-opaque-physical-injection', 'started', 'trigger', 'control', + 'control-repeat', 'control-repeat', 'original-opaque-before-admission', 'trigger', 'control', 'control-repeat', + 'control-repeat', 'original-opaque-terminal-before-rescue', 'task-exit-background', 'quiescent', + 'original-opaque-after-task-before-rescue', *(['restore-fixture-without-relabeling-original-task'] * 4), + 'original-opaque-restored-original-bytes', 'started', 'original-opaque-restored-full-S3', + 'original-opaque-token-after-fixture-rescue', 'task-exit-background', 'quiescent', + 'original-opaque-after-restored-business-reads', 'protected-control-before-legacy-writing', + 'one-way-legacy-write-phase', 'failure-scene-error', 'quiescent', 'retain-unknown-seed-receipt', 'branch-result'] + candidate['source_sha'] = '1c7fc818dd858247c45b39542abd8291b1e6e31b' + data[1]['candidateCommit'] = candidate['source_sha']; binding = data[1] + for i, target in enumerate(targets): + fault_id = heal_fault.bounded_fault_id('J02', 'original-opaque', i) + data[33 + 2 * i] = dict(node='vm000', runId=binding['runId'], faultId=fault_id, relativePath=target['relativePath'], + before={k:target[k] for k in ('sha256','size')}, kind='bitflip', offset=0) + data[55 + i] = dict(node='vm000', runId=binding['runId'], faultId=fault_id, + restoredFromBackup=True, physicalRepairPassAllowed=False) + for i in (59,65,66): data[i] = copy.deepcopy(original) + data[61] = data[19] + data[67] = dict(node='vm000', runId=binding['runId'], binarySha256=binding['candidateBinarySha256'], + phase='legacy', oneWay=True, candidateUnchanged=True, credentialsDisclosed=False, otherRuntimeSettingsUnchanged=True) + data[70] = dict(state='DIRTY', namespacesRetained=True, request=dict(operation='append-real-legacy-fixture', bucket=binding['bucketPrefix'])) + for i,label in enumerate(labels,1): opaque_tests.emit(folder/f'{i:05d}-{label}.json',data.get(i,{})) + case_path = next((root/'artifact/run/heal.outcome-counters-and-error-truth/cases').glob('*.json')) + value=read(case_path); value['provenance']['source_commit']=candidate['source_sha']; opaque_tests.emit(case_path,value) + path=folder.parents[2]/'healthy-summary.json';value=read(path) + value['results'][-1]['failure']=dict(type='ValueError',message='Heal node vm000 start failed: ImportError: attempted relative import with no known parent package\n') + opaque_tests.emit(path,value) + return candidate,folder + + def test_exact_pre_sdk_plan_refuses_later_writes_or_changed_original_authority(self): + for poison in (None,'source','unknown-ack','later-sdk','phase','control','rescue','fault-id','extra-task','prior-cleanup'): + with self.subTest(poison=poison),tempfile.TemporaryDirectory() as temporary: + root=Path(temporary);candidate,folder=self.fixture(root) + def edit(pattern,change): + path=next(folder.glob(pattern));value=json.loads(path.read_text());change(value);opaque_tests.emit(path,value) + if poison=='source': candidate['source_sha']=outcome_recovery.CANDIDATE + elif poison=='unknown-ack':edit('original-opaque-fixture/*-ack.json',lambda v:v['observation'].update(resultUnknown=True)) + elif poison=='later-sdk':(folder/'legacy-fixture').mkdir() + elif poison=='phase':edit('00067-*',lambda v:v.update(oneWay=False)) + elif poison=='control':edit('00066-*',lambda v:v['vm000']['files'][0].update(sha256='f'*64)) + elif poison=='rescue':edit('00055-*',lambda v:v.update(restoredFromBackup=False)) + elif poison=='fault-id':edit('00037-*',lambda v:v.update(faultId='J02-'+'f'*32)) + elif poison=='extra-task':opaque_tests.emit(folder/'00072-trigger.json',{}) + elif poison=='prior-cleanup': + path=folder.parents[2]/'healthy-summary.json';value=json.loads(path.read_text());value['results'][0]['cleanup']='FAIL';opaque_tests.emit(path,value) + if poison: + with self.assertRaises((ValueError,KeyError)):recover_environment.recovery_plan(root,12345,candidate) + else: + plan=recover_environment.recovery_plan(root,12345,candidate)[0] + self.assertEqual(plan['kind'],'acknowledged-j02-before-legacy-put') + self.assertEqual(plan['nodes'],['vm000']);self.assertNotIn('faults',plan) + + def test_start_bundles_real_fault_dependency_for_common_or_node_specific_legacy_profile(self): + for override in (False,True,None): + with self.subTest(override=override),tempfile.TemporaryDirectory() as temporary: + def execute(*args,**kw): + source=kw['input'].rsplit('\nmain(',1)[0];scope=dict(__name__='__standalone_node__') + exec(compile(source,'legacy-start-node','exec'),scope) + self.assertTrue(callable(scope['fault_module']().new_file)) + return types.SimpleNamespace(returncode=0,stdout='{}',stderr='') + fleet=healthy.Fleet(['vm000'],Path(temporary),'gh-12345-1-h1',runtime_profile='outcome-legacy' if override is False else 'manual') + if override:fleet.node_runtime_profiles['vm000']='outcome-legacy' + with patch.object(healthy.subprocess,'run',side_effect=execute):fleet.remote('vm000','start') From 7c30249caebcb8cb5329949fb79ca88dd2f65010 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 16 Oct 2026 15:36:12 +0800 Subject: [PATCH 219/343] test(validation): pin admin batch receipt fix candidate --- scripts/validation_heal/manager_native.py | 8 ++++++++ scripts/validation_heal/outcome.py | 3 ++- test_validation_heal_manager_native.py | 14 ++++++++++++++ test_validation_heal_outcome.py | 3 ++- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/validation_heal/manager_native.py b/scripts/validation_heal/manager_native.py index 6f474bc4..8d6ded80 100644 --- a/scripts/validation_heal/manager_native.py +++ b/scripts/validation_heal/manager_native.py @@ -54,6 +54,14 @@ "sourceFiles": {**SOURCE_FILES, "crates/heal/src/heal/storage.rs": "0e27cc1fbb7b732d211e744f385223bee23844c95ba86918e1765b546c197096"}, } +REVIEWED_CANDIDATES["7951bb629821246868ee3761b321dd07c06db45c"] = { + "sourceTree": "e4d863efff2792c0ab4da0036c04c3e0902aca89", + "cargoLockSha256": "2589b3f5afc54f94419a3e2bfd537c11fae7ae3ee2d2a73ba31c72ee8d8a62be", + "sourceFiles": {**REVIEWED_CANDIDATES["1c7fc818dd858247c45b39542abd8291b1e6e31b"]["sourceFiles"], + "crates/heal/src/heal/manager.rs": "6e41b39d74aedf3b40b6c6a8f7c5de60bf2cb7927d7d76a10745033002dbc00d", + "crates/heal/src/heal/manager/tests.rs": "844b84870e85fca4de1c72faf66e4f7a43561d8d6db341db43fd089223ff9a1e", + "crates/heal/tests/shard_identity_receipt_test.rs": "51ffe71b57efceb03ef40aa2333e235c5515149c89b64df0553dbe26b331db32"}, +} RESUME = "heal::resume::tests::" SNAPSHOT = "heal::mrf_queue::snapshot::tests::" ROOT = "heal::manager::tests::root_recovery::" diff --git a/scripts/validation_heal/outcome.py b/scripts/validation_heal/outcome.py index ead8f38e..74dab961 100644 --- a/scripts/validation_heal/outcome.py +++ b/scripts/validation_heal/outcome.py @@ -44,7 +44,8 @@ CASE_ID = "heal.outcome-counters-and-error-truth" -REVIEWED_CANDIDATES = frozenset({CANDIDATE, "1c7fc818dd858247c45b39542abd8291b1e6e31b"}) +REVIEWED_CANDIDATES = frozenset({CANDIDATE, "1c7fc818dd858247c45b39542abd8291b1e6e31b", + "7951bb629821246868ee3761b321dd07c06db45c"}) BRANCHES = ("mixed-healthy-repaired-failed", "dry-run", "deferred", "readonly-terminal-failure", "unknown-negative") SIZE = 5 * 1024**2 + 123 # The reviewed immutable candidate makes one attempt and three bucket-object retries. diff --git a/test_validation_heal_manager_native.py b/test_validation_heal_manager_native.py index 4e0087cc..40145ac1 100644 --- a/test_validation_heal_manager_native.py +++ b/test_validation_heal_manager_native.py @@ -97,6 +97,20 @@ def test_old_result_rows_cannot_certify_new_candidate_manifest(self): with self.assertRaisesRegex(ValueError, "exact scope or identity"): self.verify(fixture) + def test_admin_batch_candidate_rejects_previous_manager_source_proof(self): + candidate = "7951bb629821246868ee3761b321dd07c06db45c" + previous = suite.REVIEWED_CANDIDATES["1c7fc818dd858247c45b39542abd8291b1e6e31b"] + for field, old in (("sourceFiles", previous["sourceFiles"]), + ("candidateSourceTree", previous["sourceTree"])): + with self.subTest(field=field), tempfile.TemporaryDirectory() as temp: + fixture = self.fixture(Path(temp), candidate) + path = fixture[0] / "manager-native.json" + manifest = json.loads(path.read_text()) + manifest[field] = old + path.write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, "reviewed source"): + self.verify(fixture) + def test_wrong_candidate_build_attempt_or_helper_is_rejected(self): with tempfile.TemporaryDirectory() as temp: fixture = self.fixture(Path(temp)) diff --git a/test_validation_heal_outcome.py b/test_validation_heal_outcome.py index bda417a3..7a4ec00f 100644 --- a/test_validation_heal_outcome.py +++ b/test_validation_heal_outcome.py @@ -177,7 +177,8 @@ def test_lost_rescue_ack_keeps_registered_fault_for_independent_recovery(self): def test_each_reviewed_candidate_requires_native_evidence_before_fleet_creation(self): self.assertEqual(outcome.REVIEWED_CANDIDATES, - {outcome.CANDIDATE, "1c7fc818dd858247c45b39542abd8291b1e6e31b"}) + {outcome.CANDIDATE, "1c7fc818dd858247c45b39542abd8291b1e6e31b", + "7951bb629821246868ee3761b321dd07c06db45c"}) for commit in outcome.REVIEWED_CANDIDATES: with self.subTest(commit=commit), tempfile.TemporaryDirectory() as tmp: root = Path(tmp); candidate = root / "candidate.json" From c21a20bc0d05ae6837ae2b75927bd03a10e00413 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 16 Oct 2026 16:07:55 +0800 Subject: [PATCH 220/343] docs(validation): reconcile current migration acceptance --- docs/validation-heal-environment-gates.md | 4 ++-- docs/validation-migration-map.md | 24 ++++++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index c43bfcaf..c6d5b68b 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -28,14 +28,14 @@ `matrix.py list` 同时输出 `automaticExecutor`、`dispatchable` 和 `executionPolicy`：67 个 Heal 自动执行器、13 个环境门控、4 个证据/执行器阻断。调度覆盖共 80 个 Heal，连同 74 个非 Heal 为 154/158；这不代表 154 个 live PASS。 -第二批 4 个用例未进入可执行调度。经核实，C11/F07/J02 已有 candidate-bound 原生回归；这三项主要缺完整真机编排与证据闭环。I05 仍缺精确服务端屏障，增加节点不能解决： +第二批 4 个用例仍未进入 canonical 调度。C11/F07/J02 已有 candidate-bound 原生回归和显式诊断入口；诊断分支通过仍需完整矩阵和独立审计后才能解除门控。I05 仍缺精确服务端屏障，增加节点不能解决： | Code | 用例 | Block 原因 | 解锁所需证据 | | --- | --- | --- | --- | | C11 | `heal.bucket-incarnation-recreation` | H1/H2 旧元数据盘重返、原始 checkpoint/resume 与 committed MRF 重放编排已实现；精确候选的 9 项存储原生回归和 3 项 manager incarnation 前置证据已独立核验。完整现场分支尚无独立审计结果 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | | F07 | `heal.checkpoint-and-mrf-corruption` | 已实现 H1/H2 checkpoint 截断、未来 schema、committed MRF CRC 故障及独立安全重扫；10 项原生回归作为前置证据。真实较早代次回放与私有限额卷的 journal ENOSPC 已实现；仍缺完整 native cut 审核，完整 live 尚未审计 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | | I05 | `heal.lock-timeout-and-caller-abort` | 请求耗时/TCP 断开不能证明服务端锁 owner 与提交顺序 | lock owner、timeout、caller abort 和 stale read-repair 的服务端屏障 | -| J02 | `heal.outcome-counters-and-error-truth` | 已有 2 项 outcome 原生回归；缺混合 healthy/repaired/failed/deferred/unknown 真机夹具、dry-run 独立任务与完整 manifest 对账 | 每个 disposition、计数和 attemptFailures 对账；不是已证实产品缺少 API | +| J02 | `heal.outcome-counters-and-error-truth` | 混合责任、dry-run、原始 grace、只读失败与独立补偿、全元数据损坏及真实 legacy 夹具和 manifest 对账均已实现。原实机观察到整桶重试覆盖首轮 repaired 记录；候选 `7951bb629821246868ee3761b321dd07c06db45c` 已修复并通过 210 项 manager 与 5 项独立进程集成测试，完整新候选实机矩阵仍待验收 | H1/H2 × DB/DV 全分支原始证据；Unknown 必须真实观察；每个 disposition、计数、attemptFailures 与物理故障对账，原失败保持不变 | I05 和已接入 A09、G01–G06 的精确屏障分支仍受候选包对应、经过审核的 `e2e-test-hooks` 限制。第一批 G07/G08 的 native 分支与 H04 容量锁窗口也保留 BLOCKED；不能用外部请求重试或最终对象正确来声称精确 lock/version/marker/MPU/copy 提交屏障已命中。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 4699a26d..dc8f5102 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -7,6 +7,24 @@ and SHA-256 bind the snapshot to the source catalog. `reuse_candidates` lists co it is not a coverage claim. A case's run result remains `unverified` until a live run proves each source assertion on the required topology. +## Current acceptance and remaining work + +All 158 canonical IDs are retained. Independent complete live acceptance is **102/158**: the prior 101-case census plus B07 `heal.below-read-quorum`, accepted from [run 38727279347/1](https://github.com/rustfs/rustfs/actions/runs/38727279347). Its complete H1/H2 never-versioned/historical matrix, exact candidate/package/tools and cleanup were independently audited; earlier B07 failures remain retained. + +| Acceptance state | Cases | Remaining requirement | +| --- | ---: | --- | +| Independently accepted complete live PASS | 102 | Preserve original provenance and complete results. | +| Retained live failures with existing executors | 32 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | +| Required native barriers | 7 | A09 and G01–G06 need exact candidate-bound server lock/version/commit barriers. | +| Orchestration or evidence blocked | 4 | C11, F07, I05 and J02 need their remaining native or full live evidence closure. | +| Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | + +The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 43 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. Fresh candidate-bound full J02 live acceptance is still pending. + +The sections below retain earlier implementation and run history. Historical failed or partial results are not promoted by this summary. + +## Original reuse assessment + | Source module | Cases | Existing auto-testing code to inspect | Current gap | | --- | ---: | --- | --- | | Pool lifecycle | 13 | `rustfs_pool_expand.sh` | Current steps combine operations; each validation ID needs its own assertions, object manifest, and result. | @@ -131,9 +149,9 @@ immutable package URL and SHA-256 from that build attempt, then calls executor before touching the fleet and runs the result report with `--gate-case`, so a missing, failed, or incomplete selected case fails the workflow. Each run uploads its candidate identity, canonical result, and -case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. Independent census currently confirms 101 unique complete canonical live PASS cases: 85 on source `072b57b37559fbe626aeda8abf61617fdcbdd926`, 12 on the older immutable source, two on source `16cf19817e0c81e24d0d787a916f734a34517ec5`, and two on source `e224eb48e5311cff022b04fd78c97a8572d5c270`. The latest additions are C08 metadata/payload mismatch `38235978152` and E03 read-repair attribution `38237270640`. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, D09 run `38205854024`, D10 run `38207381816`, and manual-normal attribution run `38207956101` are included. Native, read-only, recovery and duplicate runs are excluded; the other 57 cases have no complete canonical live PASS claim. +case evidence. The 154 currently dispatchable IDs include all 74 non-Heal cases, 67 automatic Heal executors, and 13 disabled-by-default environment gates. The gates preserve the operator procedures and emit explicit BLOCKED results; they are not automatic live executors or PASS claims. The preceding census confirmed 101 unique complete canonical live PASS cases: 85 on source `072b57b37559fbe626aeda8abf61617fdcbdd926`, 12 on the older immutable source, two on source `16cf19817e0c81e24d0d787a916f734a34517ec5`, and two on source `e224eb48e5311cff022b04fd78c97a8572d5c270`. Its latest additions were C08 metadata/payload mismatch `38235978152` and E03 read-repair attribution `38237270640`. The independently verified Pool tail runs `38110959836` and `38111198632`, A03 run `38198122041`, C04 run `38203454300`, the completed multipart run `38204339623`, D07 run `38205235080`, D09 run `38205854024`, D10 run `38207381816`, and manual-normal attribution run `38207956101` are included. Native, read-only, recovery and duplicate runs are excluded; B07 run `38727279347` subsequently raises complete acceptance to 102, leaving 56 cases without complete canonical live PASS. -All 84 Heal recipes, 301 operator branches and 532 applicable topology branch +All 84 Heal recipes, 302 operator branches and 534 applicable topology branch templates are preserved under `scripts/validation_heal`, with the public contract, operation book, signed probe, ACK-logged workload, guarded file-fault tool and source-bound metadata inspector source. `matrix.py plan` binds current @@ -197,7 +215,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -Four Heal cases remain native/evidence-blocked and outside dispatch: C11, F07, I05 and J02. The 13 environment-gated cases have disabled-by-default control flows and oracles in `environment_cases.py`; actual execution requires an assigned and reviewed auto-testing resource adapter, and canonical results remain review-gated. See [the environment gate reference](validation-heal-environment-gates.md) for each missing resource, capability and native barrier. +Four Heal cases remain outside canonical dispatch: C11, F07, I05 and J02. C11/F07/J02 have explicit diagnostic routes that preserve their blocking canonical verdicts; I05 still needs exact server barriers. The 13 environment-gated cases have disabled-by-default control flows and oracles in `environment_cases.py`; actual execution requires an assigned and reviewed auto-testing resource adapter, and canonical results remain review-gated. See [the environment gate reference](validation-heal-environment-gates.md) for each missing resource, capability and native barrier. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. From 17f41c5c1e75a20a933af911690596d7c3d4f1b8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 16 Oct 2026 17:54:03 +0800 Subject: [PATCH 221/343] docs(validation): record audited J02 live boundaries --- docs/validation-heal-environment-gates.md | 2 +- docs/validation-migration-map.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index c6d5b68b..5f0b26dc 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -35,7 +35,7 @@ | C11 | `heal.bucket-incarnation-recreation` | H1/H2 旧元数据盘重返、原始 checkpoint/resume 与 committed MRF 重放编排已实现；精确候选的 9 项存储原生回归和 3 项 manager incarnation 前置证据已独立核验。完整现场分支尚无独立审计结果 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | | F07 | `heal.checkpoint-and-mrf-corruption` | 已实现 H1/H2 checkpoint 截断、未来 schema、committed MRF CRC 故障及独立安全重扫；10 项原生回归作为前置证据。真实较早代次回放与私有限额卷的 journal ENOSPC 已实现；仍缺完整 native cut 审核，完整 live 尚未审计 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | | I05 | `heal.lock-timeout-and-caller-abort` | 请求耗时/TCP 断开不能证明服务端锁 owner 与提交顺序 | lock owner、timeout、caller abort 和 stale read-repair 的服务端屏障 | -| J02 | `heal.outcome-counters-and-error-truth` | 混合责任、dry-run、原始 grace、只读失败与独立补偿、全元数据损坏及真实 legacy 夹具和 manifest 对账均已实现。原实机观察到整桶重试覆盖首轮 repaired 记录；候选 `7951bb629821246868ee3761b321dd07c06db45c` 已修复并通过 210 项 manager 与 5 项独立进程集成测试，完整新候选实机矩阵仍待验收 | H1/H2 × DB/DV 全分支原始证据；Unknown 必须真实观察；每个 disposition、计数、attemptFailures 与物理故障对账，原失败保持不变 | +| J02 | `heal.outcome-counters-and-error-truth` | 全部夹具与 manifest 对账均已实现。候选 `7951bb629821246868ee3761b321dd07c06db45c` 修复整桶重试覆盖 repaired 记录，210 项 manager 与 5 项独立进程集成测试通过。新实机 `38874900846/1` 完成 20 个分支：16 个报告通过，4 个原始全元数据损坏探针返回永久 Failed，保留 NOT_OBSERVED；四组混合任务与全部 50 条节点清理记录已独立审计。真实 legacy Unknown 已观察，不能代替原始探针 | 原始元数据损坏分支的 Unknown 仍未观察；完整 source-procedure 验收尚未闭合。原始 Failed、NOT_OBSERVED 和完整用例门控保持不变 | I05 和已接入 A09、G01–G06 的精确屏障分支仍受候选包对应、经过审核的 `e2e-test-hooks` 限制。第一批 G07/G08 的 native 分支与 H04 容量锁窗口也保留 BLOCKED；不能用外部请求重试或最终对象正确来声称精确 lock/version/marker/MPU/copy 提交屏障已命中。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index dc8f5102..0e945d8d 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -19,7 +19,9 @@ All 158 canonical IDs are retained. Independent complete live acceptance is **10 | Orchestration or evidence blocked | 4 | C11, F07, I05 and J02 need their remaining native or full live evidence closure. | | Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | -The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 43 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. Fresh candidate-bound full J02 live acceptance is still pending. +The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 43 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. + +[J02 diagnostic 38874900846/1](https://github.com/rustfs/rustfs/actions/runs/38874900846) completed all 20 H1/H2 × DB/DV branch instances on that exact candidate. Sixteen branches report PASS; the four original all-metadata-corruption probes report permanent Failed rather than Unknown and retain NOT_OBSERVED. Independent archive and provenance audits passed. Independent mixed-branch audits verified all four repaired shards against original complete bytes, exact receipts and counters, surviving below-quorum failures, original terminal preservation and post-rescue S3 inventories. All 50 node namespace finalization receipts passed audit. Real legacy writes separately produce Unknown with zero repairs and unchanged physical files, but do not replace the original source procedure. Full J02 canonical acceptance remains blocked; these results do not increase 102/158. The sections below retain earlier implementation and run history. Historical failed or partial results are not promoted by this summary. From d3533662a737cda438c2733478e178523abeee9b Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 16 Oct 2026 18:04:59 +0800 Subject: [PATCH 222/343] fix(validation): target current ACK in exact object option probes --- scripts/validation_heal/options.py | 11 +++++-- test_validation_heal_options.py | 47 +++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/scripts/validation_heal/options.py b/scripts/validation_heal/options.py index 795df124..619236d8 100644 --- a/scripts/validation_heal/options.py +++ b/scripts/validation_heal/options.py @@ -180,6 +180,7 @@ def map_fixture(snapshot, bucket, expected, topology, *, multi_set=True): else: mapped[identity] = {**map_target(snapshot, bucket, entry['key'], 'data', 1, entry['versionId'], latest_required=entry['isLatest'], multi_set=topology == 'H2' and multi_set, set_index=set_index), 'deleteMarker': False} + mapped[identity]['isLatest'] = entry['isLatest'] if topology == 'H2' and multi_set: require({target['setIndex'] for target in mapped.values()} == {0, 1}, 'A10 fixture did not populate both actual sets') require(len(mapped) == len(expected), 'A10 physical mapping omitted an ACK') @@ -192,7 +193,11 @@ def selection(branch, options, mapped): if branch.startswith('selector-'): return {identity for identity, target in mapped.items() if options['pool'] is None or (target['poolIndex'], target['setIndex']) == (options['pool'], options['set'])} - return {identity for identity in mapped if identity[1] == KEYS[0]} + # An exact object request has no versionId and resolves the current ACK. + current = {identity for identity, target in mapped.items() + if identity[1] == KEYS[0] and target.get('isLatest') is True} + require(len(current) == 1, 'A10 exact object request lacks one unambiguous current ACK') + return current def request_path(branch, bucket): @@ -319,8 +324,10 @@ def scope_probe(fleet, probe, binding, recorder, bucket, topology, fixture, bran verify_parts(fleet, data, 'A10-all-original-streaming') selected = selection(branch, options, mapped) targets = [] + current_object_request = not branch.startswith(('recursive-', 'selector-')) for index, key in enumerate(KEYS[:4]): - identity = next(identity for identity in data if identity[1] == key) + identity = next(identity for identity in data if identity[1] == key + and (not current_object_request or key != KEYS[0] or identity in selected)) target = data[identity] role = 'parity' if index % 2 or branch == 'update-parity' and index == 0 else 'data' member = next(row for row in target['cohort'] if row['role'] == role) diff --git a/test_validation_heal_options.py b/test_validation_heal_options.py index 2c288312..cde7e6c7 100644 --- a/test_validation_heal_options.py +++ b/test_validation_heal_options.py @@ -18,7 +18,8 @@ def mapped(self): rows = [(options.KEYS[0], 'null', 0, False), (options.KEYS[0], 'historic', 0, False), (options.KEYS[1], 'child', 1, False), (options.KEYS[2], 'control', 1, False), (options.KEYS[-1], 'marker', 0, True)] - return {(self.bucket, key, version): {'poolIndex': 0, 'setIndex': number, 'deleteMarker': marker} + return {(self.bucket, key, version): {'poolIndex': 0, 'setIndex': number, 'deleteMarker': marker, + 'isLatest': version != 'historic'} for key, version, number, marker in rows} def status(self, branch='selector-valid'): @@ -72,6 +73,50 @@ def test_selector_uses_measured_sets_and_retains_historical_and_marker_identitie self.assertEqual(proof['repairedACKs'], 1) self.assertIn((self.bucket, options.KEYS[-1], 'marker'), selected) + def test_exact_object_request_selects_only_the_acknowledged_current_version(self): + mapped = self.mapped() + for branch in ('recreate', 'update-parity', 'read-repair'): + for request in options.paired_options(branch): + with self.subTest(branch=branch, request=request): + selected = options.selection(branch, request, mapped) + self.assertEqual(selected, {(self.bucket, options.KEYS[0], 'null')}) + self.assertIn((self.bucket, options.KEYS[0], 'historic'), + options.selection('recursive-true', {'recursive': True}, mapped)) + + def test_exact_object_scope_rejects_missing_or_ambiguous_current_ack(self): + for kind in ('missing', 'duplicate', 'untyped'): + mapped = self.mapped() + if kind == 'missing': mapped[(self.bucket, options.KEYS[0], 'null')]['isLatest'] = False + elif kind == 'duplicate': mapped[(self.bucket, options.KEYS[0], 'historic')]['isLatest'] = True + else: mapped[(self.bucket, options.KEYS[0], 'null')]['isLatest'] = 1 + with self.subTest(kind=kind), self.assertRaisesRegex(ValueError, 'current ACK'): + options.selection('recreate', options.paired_options('recreate')[0], mapped) + + def test_exact_object_fault_targets_current_ack_before_remote_mutation(self): + mapped = self.mapped() + mapped.pop((self.bucket, options.KEYS[-1], 'marker')) + mapped[(self.bucket, options.KEYS[3], 'other')] = { + 'poolIndex': 0, 'setIndex': 0, 'deleteMarker': False, 'isLatest': True} + # A historical generation appears first, as it does in the real PUT ledger. + current = (self.bucket, options.KEYS[0], 'null') + historical = (self.bucket, options.KEYS[0], 'historic') + mapped = {historical: mapped[historical], **{k: v for k, v in mapped.items() if k != historical}} + for number, (identity, target) in enumerate(mapped.items()): + target.update(bucket=identity[0], key=identity[1], versionId=identity[2], node='vm000', disk=0, + relativePath='generation-' + str(number), sha256='sha-' + identity[2], size=128, + cohort=[{'role': 'data'}, {'role': 'parity'}]) + snapshots = types.SimpleNamespace(stop=lambda: None, snapshot=lambda *a, **k: {}, faults=[], + record=lambda label, value: value) + snapshots.remote = unittest.mock.Mock(side_effect=RuntimeError('remote mutation boundary')) + with patch.object(options, 'seed_fixture', return_value=[]), patch.object(options, 'verify_seed'), \ + patch.object(options, 'map_fixture', return_value=mapped), patch.object(options, 'verify_parts'): + probe = types.SimpleNamespace(inventory=lambda *a, **k: {}) + with self.assertRaisesRegex(RuntimeError, 'remote mutation boundary'): + options.scope_probe(snapshots, probe, {}, None, self.bucket, 'H1', 'DV', 'recreate', + options.paired_options('recreate')[0], []) + self.assertEqual(snapshots.remote.call_args.kwargs['relativePath'], mapped[current]['relativePath']) + self.assertEqual(snapshots.remote.call_args.kwargs['expectedSha256'], mapped[current]['sha256']) + def test_complete_oracle_rejects_lost_versions_wrong_scopes_markers_and_false_counters(self): for kind in ('version', 'missing', 'duplicate', 'unknown', 'scope', 'null-scope', 'bool-scope', 'incarnation', 'marker', 'counter', 'overflow', 'coverage', 'settings', 'truncated'): From bb1b825900d2722e182fce8a3271599ce79df3bb Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 18 Oct 2026 10:14:12 +0800 Subject: [PATCH 223/343] fix(validation): freeze checkpoint owner before fleet observation --- scripts/validation_heal/checkpoint.py | 77 +++++++++++++--------- scripts/validation_heal/persistence.py | 4 +- test_validation_heal_checkpoint.py | 90 ++++++++++++++++++++++++-- test_validation_heal_persistence.py | 14 ++++ 4 files changed, 146 insertions(+), 39 deletions(-) diff --git a/scripts/validation_heal/checkpoint.py b/scripts/validation_heal/checkpoint.py index 0037a67c..1e84f7ce 100644 --- a/scripts/validation_heal/checkpoint.py +++ b/scripts/validation_heal/checkpoint.py @@ -136,9 +136,11 @@ def owned_root_admission(observed, token): return {**owner, "value": value, "originalDurableResponsibility": True} -def records(fleet, token, label, *, quiescent=False, family="checkpoint"): +def records(fleet, token, label, *, quiescent=False, family="checkpoint", nodes=None): result = {} - for node in fleet.nodes: + selected = fleet.nodes if nodes is None else nodes + require(selected and set(selected) <= set(fleet.nodes), "checkpoint observation escaped its bound nodes") + for node in selected: params = {"family": family, "taskId": token} if not quiescent: params["expected"] = fleet.record(label + "-process", {"node": node, **fleet.remote(node, "process")}) @@ -158,27 +160,26 @@ def admit(probe, fleet, active): def capture_earlier(fleet, token, bucket, identifier, *, timeout=60): require(type(timeout) is int and 0 < timeout <= 120, "unreviewed checkpoint capture budget") - deadline = time.monotonic() + timeout - while True: - try: - owner = owned_checkpoint(records(fleet, token, "Admin-checkpoint-before-capture"), token, bucket) - identity = fleet.record("checkpoint-capture-process", fleet.remote(owner["node"], "process")) - receipt = fleet.record("Admin-checkpoint-original-capture", fleet.remote(owner["node"], "journal-capture", - family="checkpoint", taskId=token, disk=owner["disk"], relativePath=owner["checkpoint"]["relativePath"], - faultId=identifier, expectedBefore=journal.entry(bytes_of(owner["checkpoint"])), expected=identity)) - require(receipt.get("processBound") is True and receipt.get("originalProductBytes") is True - and receipt.get("productMetadataMutated") is False - and all(journal.same_json(receipt.get(name), value) for name, value in { - "runId": fleet.run_id, "family": "checkpoint", "taskId": token, "disk": owner["disk"], - "relativePath": owner["checkpoint"]["relativePath"], "faultId": identifier, - "binarySha256": identity.get("binarySha256"), - "captured": journal.entry(bytes_of(owner["checkpoint"]))}.items()), - "checkpoint capture lost its original process, bytes or exact physical owner") - return {**owner, "captureId": identifier, "captureReceipt": receipt} - except ValueError as exc: - if not str(exc).startswith("NOT_OBSERVED:"): raise - require(time.monotonic() < deadline, "NOT_OBSERVED: actual earlier checkpoint capture exceeded its frozen budget") - time.sleep(.1) + owner, identity = freeze(fleet, token, bucket, timeout=timeout) + node = owner["node"] + receipt = fleet.record("Admin-checkpoint-original-capture", fleet.remote(node, "journal-capture", + family="checkpoint", taskId=token, disk=owner["disk"], relativePath=owner["checkpoint"]["relativePath"], + faultId=identifier, expectedBefore=journal.entry(bytes_of(owner["checkpoint"])), expected=identity)) + require(receipt.get("processBound") is True and receipt.get("originalProductBytes") is True + and receipt.get("productMetadataMutated") is False + and all(journal.same_json(receipt.get(name), value) for name, value in { + "runId": fleet.run_id, "family": "checkpoint", "taskId": token, "disk": owner["disk"], + "relativePath": owner["checkpoint"]["relativePath"], "faultId": identifier, + "binarySha256": identity.get("binarySha256"), + "captured": journal.entry(bytes_of(owner["checkpoint"]))}.items()), + "checkpoint capture lost its original process, bytes or exact physical owner") + fleet.record("checkpoint-capture-resume", fleet.remote(node, "process-signal", signal="SIGCONT", expected=identity)) + resumed = fleet.record("checkpoint-capture-resumed-process", fleet.remote(node, "process")) + require(resumed.get("state") in {"R", "S", "D", "I"} + and all(resumed.get(name) == identity.get(name) for name in PROCESS_FIELDS), + "checkpoint capture did not resume its original process") + del fleet.frozen_processes[node] + return {**owner, "captureId": identifier, "captureReceipt": receipt} def after_generation(current, earlier): @@ -198,16 +199,30 @@ def freeze(fleet, token, bucket, *, timeout=60, earlier=None): require(type(timeout) is int and 0 < timeout <= 120, "unreviewed checkpoint observation budget") deadline = time.monotonic() + timeout while True: - try: - owner = owned_checkpoint(records(fleet, token, "Admin-checkpoint-live"), token, bucket) - if earlier is not None: after_generation(owner, earlier) + owner = None + for candidate in fleet.nodes: + observed = records(fleet, token, "Admin-checkpoint-live", nodes=(candidate,)) + try: + owner = owned_checkpoint(observed, token, bucket) + if earlier is not None: after_generation(owner, earlier) + except ValueError as exc: + if not str(exc).startswith("NOT_OBSERVED:"): raise + owner = None + continue + identity = observed[candidate].get("processIdentity") + require(owner["node"] == candidate and observed[candidate].get("processBound") is True + and observed[candidate].get("stable") is True and observed[candidate].get("independentReads") == 2 + and isinstance(identity, dict) and all(name in identity for name in PROCESS_FIELDS) + and type(identity.get("pid")) is int and identity["pid"] > 1, + "checkpoint owner has no census-bound process birth identity") break - except ValueError as exc: - if not str(exc).startswith("NOT_OBSERVED:"): raise - require(time.monotonic() < deadline, "NOT_OBSERVED: actual durable Admin checkpoint window exceeded its frozen budget") - time.sleep(.1) + if owner is not None: break + require(time.monotonic() < deadline, "NOT_OBSERVED: actual durable Admin checkpoint window exceeded its frozen budget") + time.sleep(.1) node = owner["node"] - identity = fleet.record("checkpoint-owner-process", fleet.remote(node, "process")) + # Freeze the witnessed owner before another SSH read can outlast its task. + # The complete census below still rejects any second physical owner. + fleet.record("checkpoint-owner-process", identity) require(node not in fleet.frozen_processes, "checkpoint owner is already tracked as frozen") fleet.frozen_processes[node] = identity fleet.record("checkpoint-owner-freeze", fleet.remote(node, "process-signal", signal="SIGSTOP", expected=identity)) diff --git a/scripts/validation_heal/persistence.py b/scripts/validation_heal/persistence.py index 5b6d01df..bbc6df68 100644 --- a/scripts/validation_heal/persistence.py +++ b/scripts/validation_heal/persistence.py @@ -179,8 +179,10 @@ def rescan_outcome(current, bucket, expected, repairs, incarnation): wanted = {version_identity(entry): entry for entry in expected} require(wanted and len(wanted) == len(expected) and repairs <= set(wanted), "rescan manifest duplicated or escaped ACKs") outcome = current.get("outcome") or {} + # HealTaskStatusPayload omits a false legacy truncation flag. Canonical + # receipt truncation remains explicit and independently reconciled below. require(current.get("summary") == "finished" and outcome.get("execution") == {"state": "completed"} - and current.get("settings") == checkpoints.OPTIONS and type(current.get("truncated")) is bool + and current.get("settings") == checkpoints.OPTIONS and type(current.get("truncated", False)) is bool and outcome.get("coverage") == "complete" and outcome.get("objectsTruncated") is (len(wanted) > 128), "safe rescan lacks complete truthful coverage") counts = {"processed": len(wanted), "healed": len(repairs), "unchanged": len(wanted) - len(repairs), diff --git a/test_validation_heal_checkpoint.py b/test_validation_heal_checkpoint.py index 8f53278a..e96059d2 100644 --- a/test_validation_heal_checkpoint.py +++ b/test_validation_heal_checkpoint.py @@ -112,26 +112,32 @@ def test_set_admission_keeps_explicit_selectors_and_actual_canonical_token(self) def test_earlier_capture_requires_original_bytes_bound_to_process_and_physical_owner(self): observed, _, _ = self.fixture() owner = reader.owned_checkpoint(observed, TOKEN, BUCKET) - identity = {"binarySha256": "a" * 64} + from scripts.validation_heal.fleet_node import PROCESS_FIELDS + identity = {name: name for name in PROCESS_FIELDS} + identity.update(pid=50, state="S", binarySha256="a" * 64) receipt = {"processBound": True, "originalProductBytes": True, "productMetadataMutated": False, "runId": "gh-12345-1-h1", "family": "checkpoint", "taskId": TOKEN, "disk": 1, "relativePath": owner["checkpoint"]["relativePath"], "faultId": "F07-early", "binarySha256": "a" * 64, "captured": journal.entry(reader.bytes_of(owner["checkpoint"]))} requests = [] - fleet = SimpleNamespace(run_id=receipt["runId"], record=lambda label, value: value, + fleet = SimpleNamespace(run_id=receipt["runId"], frozen_processes={"node": identity}, record=lambda label, value: value, remote=lambda node, action, **params: requests.append((node, action, params)) or (identity if action == "process" else receipt)) - with patch.object(reader, "records", return_value=observed): + with patch.object(reader, "freeze", return_value=(owner, identity)): result = reader.capture_earlier(fleet, TOKEN, BUCKET, "F07-early") self.assertEqual(result["captureId"], "F07-early") - self.assertEqual(requests[-1], ("node", "journal-capture", { + self.assertEqual(requests[0], ("node", "journal-capture", { "family": "checkpoint", "taskId": TOKEN, "disk": 1, "relativePath": receipt["relativePath"], "faultId": "F07-early", "expectedBefore": receipt["captured"], "expected": identity})) + self.assertEqual([request[1] for request in requests], ['journal-capture', 'process-signal', 'process']) + self.assertEqual(fleet.frozen_processes, {}) for key, value in [("processBound", False), ("productMetadataMutated", True), ("disk", True), ("binarySha256", "b" * 64), ("captured", {"size": 3, "sha256": "a" * 64})]: + fleet.frozen_processes["node"] = identity original = receipt[key]; receipt[key] = value with self.subTest(key=key), self.assertRaises(ValueError): reader.capture_earlier(fleet, TOKEN, BUCKET, "F07-early") + self.assertEqual(fleet.frozen_processes, {"node": identity}) receipt[key] = original def test_stale_boundary_requires_actual_newer_progress_in_the_same_original_responsibility(self): @@ -201,8 +207,9 @@ def test_freeze_and_crash_preserve_checkpoint_resume_and_the_actual_root_admissi observed, _, _ = self.fixture(); roots, _ = self.root_fixture() root = reader.owned_root_admission(roots, TOKEN) identity = {name: name for name in PROCESS_FIELDS}; identity.update(pid=50, state='S') + observed['node'].update(processIdentity=identity, processBound=True, stable=True, independentReads=2) actions = [] - fleet = SimpleNamespace(frozen_processes={}, record=lambda label, value: value, stop=lambda: actions.append('stop')) + fleet = SimpleNamespace(nodes=['node'], frozen_processes={}, record=lambda label, value: value, stop=lambda: actions.append('stop')) def remote(node, action, **params): actions.append(action) if action == 'process': return {**identity, 'state': 'T' if 'process-signal' in actions else 'S'} @@ -223,8 +230,9 @@ def test_freeze_precedes_root_observation_to_keep_the_actual_checkpoint_window(s from scripts.validation_heal.fleet_node import PROCESS_FIELDS observed, _, _ = self.fixture(); roots, _ = self.root_fixture() identity = {name: name for name in PROCESS_FIELDS}; identity.update(pid=50, state='S') + observed['node'].update(processIdentity=identity, processBound=True, stable=True, independentReads=2) actions = [] - fleet = SimpleNamespace(frozen_processes={}, record=lambda label, value: value) + fleet = SimpleNamespace(nodes=['node'], frozen_processes={}, record=lambda label, value: value) def remote(node, action, **params): actions.append(action) return {**identity, 'state': 'T' if 'process-signal' in actions else 'S'} @@ -243,7 +251,8 @@ def test_foreign_root_after_freeze_retains_the_original_paused_process_for_clean from scripts.validation_heal.fleet_node import PROCESS_FIELDS observed, _, _ = self.fixture(); roots, value = self.root_fixture() identity = {name: name for name in PROCESS_FIELDS}; identity.update(pid=50, state='T') - fleet = SimpleNamespace(frozen_processes={}, record=lambda label, value: value, remote=lambda *args, **kwargs: identity) + observed['node'].update(processIdentity=identity, processBound=True, stable=True, independentReads=2) + fleet = SimpleNamespace(nodes=['node'], frozen_processes={}, record=lambda label, value: value, remote=lambda *args, **kwargs: identity) def records(fleet, token, label, **params): if label == 'Admin-root-frozen': changed = copy.deepcopy(roots); altered = copy.deepcopy(value); altered['task_id'] = INCARNATION @@ -253,5 +262,72 @@ def records(fleet, token, label, **params): reader.freeze(fleet, TOKEN, BUCKET) self.assertEqual(fleet.frozen_processes, {'node': identity}) + def test_owner_is_frozen_before_observing_other_nodes_or_refreshing_its_process(self): + from scripts.validation_heal.fleet_node import PROCESS_FIELDS + observed, _, _ = self.fixture(); roots, _ = self.root_fixture() + identity = {name: name for name in PROCESS_FIELDS}; identity.update(pid=50) + observed['node'].update(processIdentity=identity, processBound=True, stable=True, independentReads=2) + fleet = SimpleNamespace(nodes=['node', 'peer1', 'peer2', 'peer3'], frozen_processes={}, + record=lambda label, value: value) + actions = [] + def remote(node, action, **params): + if action == 'process': self.assertIn('process-signal', actions) + actions.append(action) + return {**identity, 'state': 'T'} + fleet.remote = remote + def all_nodes(value): + full = copy.deepcopy(value) + for peer in fleet.nodes[1:]: + full[peer] = copy.deepcopy(value['node']) + full[peer]['records'] = [{'disk': r['disk'], 'relativePath': r['relativePath'], 'present': False} + for r in full[peer]['records']] + return full + def records(fleet, token, label, **params): + if label == 'Admin-checkpoint-live': + self.assertEqual(params.get('nodes'), ('node',)) + return observed + self.assertIn('process-signal', actions) + self.assertNotIn('nodes', params) + return all_nodes(roots if params.get('family') == 'root' else observed) + with patch.object(reader, 'records', side_effect=records): + frozen, birth = reader.freeze(fleet, TOKEN, BUCKET) + self.assertEqual(birth, identity) + self.assertTrue(frozen['rootAdmission']['originalDurableResponsibility']) + + def test_full_census_after_early_freeze_still_rejects_a_second_physical_owner(self): + from scripts.validation_heal.fleet_node import PROCESS_FIELDS + observed, _, _ = self.fixture() + identity = {name: name for name in PROCESS_FIELDS}; identity.update(pid=50) + observed['node'].update(processIdentity=identity, processBound=True, stable=True, independentReads=2) + fleet = SimpleNamespace(nodes=['node', 'peer1', 'peer2', 'peer3'], frozen_processes={}, + record=lambda label, value: value, remote=lambda *args, **params: {**identity, 'state': 'T'}) + def records(fleet, token, label, **params): + if label == 'Admin-checkpoint-live': return observed + full = {node: copy.deepcopy(observed['node']) for node in fleet.nodes} + return full + with patch.object(reader, 'records', side_effect=records), self.assertRaises(ValueError): + reader.freeze(fleet, TOKEN, BUCKET) + self.assertEqual(fleet.frozen_processes, {'node': identity}) + + def test_unbound_census_birth_never_signals_a_process(self): + from scripts.validation_heal.fleet_node import PROCESS_FIELDS + for change in ('missing-birth', 'missing-field', 'boolean-pid', 'unstable', 'unbound'): + observed, _, _ = self.fixture() + identity = {name: name for name in PROCESS_FIELDS}; identity.update(pid=50) + observed['node'].update(processIdentity=identity, processBound=True, stable=True, independentReads=2) + if change == 'missing-birth': observed['node'].pop('processIdentity') + elif change == 'missing-field': identity.pop('startTicks') + elif change == 'boolean-pid': identity['pid'] = True + elif change == 'unstable': observed['node']['stable'] = False + else: observed['node']['processBound'] = False + actions = [] + fleet = SimpleNamespace(nodes=['node'], frozen_processes={}, record=lambda label, value: value, + remote=lambda *a, **k: actions.append(a)) + with patch.object(reader, 'records', return_value=observed), self.subTest(change=change), \ + self.assertRaises(ValueError): + reader.freeze(fleet, TOKEN, BUCKET) + self.assertEqual(actions, []) + self.assertEqual(fleet.frozen_processes, {}) + if __name__ == "__main__": unittest.main() diff --git a/test_validation_heal_persistence.py b/test_validation_heal_persistence.py index 02d738e6..28a429aa 100644 --- a/test_validation_heal_persistence.py +++ b/test_validation_heal_persistence.py @@ -182,6 +182,20 @@ def test_bounded_diagnostics_need_complete_independent_manifests_and_exact_count else: outcome["objectsTruncated"] = False with self.subTest(change=change), self.assertRaises(ValueError): persistence.rescan_outcome(*data) + def test_rescan_accepts_omitted_false_legacy_truncation_without_defaulting_invalid_values(self): + data = self.rescan() + del data[0]['truncated'] + before = copy.deepcopy(data[0]) + proof = persistence.rescan_outcome(*data) + self.assertTrue(proof['completeIndependentManifestsRequired']) + self.assertEqual(data[0], before) + self.assertTrue(data[0]['outcome']['objectsTruncated']) + for value in (None, 0, 1, 'false', []): + invalid = self.rescan() + invalid[0]['truncated'] = value + with self.subTest(value=value), self.assertRaisesRegex(ValueError, 'truthful coverage'): + persistence.rescan_outcome(*invalid) + def test_incomplete_branches_never_enter_fleet_mutation(self): for branch in ("unknown", "native-persist-fence"): with self.subTest(branch=branch), self.assertRaisesRegex(ValueError, "cannot mutate"): From 9dff4950e1014b55da84bb7ea3d1e50520cf638b Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 09:08:28 +0800 Subject: [PATCH 224/343] fix(validation): fill bounded tail blocks and extend real fixture work --- scripts/validation_heal/bounded_volume.py | 15 +++++++++++-- scripts/validation_heal/persistence.py | 18 +++++++++++++--- test_validation_heal_capacity.py | 26 +++++++++++++++++++++++ test_validation_heal_persistence.py | 18 ++++++++++++++++ 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/scripts/validation_heal/bounded_volume.py b/scripts/validation_heal/bounded_volume.py index 1a23b353..f454a7f3 100644 --- a/scripts/validation_heal/bounded_volume.py +++ b/scripts/validation_heal/bounded_volume.py @@ -194,15 +194,26 @@ def fill(folder, branch): deadline, written, count = time.monotonic() + 60, 0, 0 if branch == "bounded-volume-enospc": fd = os.open(padding / "blocks.bin", os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + chunk = 1024**2 try: while True: require(time.monotonic() < deadline and written < IMAGE_BYTES, "bounded block fill exceeded its budget") try: - written += os.write(fd, b"\0" * min(1024**2, IMAGE_BYTES - written)) + requested = min(chunk, IMAGE_BYTES - written) + actual = os.write(fd, b"\0" * requested) + require(0 < actual <= requested, "bounded block fill made no valid byte progress") + written += actual os.fsync(fd) except OSError as error: require(error.errno == errno.ENOSPC, "bounded fill reached an unrelated IO error") - break + info = os.statvfs(folder) + if info.f_bfree == 0: + break + require(type(info.f_frsize) is int and 0 < info.f_frsize < chunk, + f"bounded minimum-block fill left {info.f_bfree} free blocks") + # A large allocation can fail while smaller tail blocks + # remain. The final zero-block and candidate probes stay mandatory. + chunk = info.f_frsize finally: os.close(fd) else: diff --git a/scripts/validation_heal/persistence.py b/scripts/validation_heal/persistence.py index bbc6df68..767d241e 100644 --- a/scripts/validation_heal/persistence.py +++ b/scripts/validation_heal/persistence.py @@ -63,6 +63,19 @@ "normalization": "scheduler get_or_insert fills only an explicit null timeout; all other fields stay exact"} +def checkpoint_fixture(branch, fixture): + require(branch in {*IMPLEMENTED, "journal-write-failure"} and fixture in {"DB", "DV"}, + "unreviewed checkpoint fixture") + if branch == "journal-write-failure": + # Keep all data and inodes inside the four fixed 128 MiB filesystems. + return (169 if fixture == "DB" else 36), 1024**2 + 123 + if branch == "stale-generation": + # More real healthy responsibilities keep both durable generations + # observable without changing task deadlines or synthesizing records. + return (511 if fixture == "DB" else 144), 2 * 1024**2 + 123 + return (257 if fixture == "DB" else 80), SIZE + + def same_terminal_options(admitted, terminal): if not isinstance(admitted, dict) or not isinstance(terminal, dict): return False @@ -484,7 +497,7 @@ def run_checkpoint_fault(args, topology, nodes, fixture, branch, candidate): fleet.snapshot_buckets = [bucket] access, secret = secrets.token_hex(16), secrets.token_hex(24) os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) - count = (129 if fixture == "DB" else 24) if bounded else (257 if fixture == "DB" else 80) + count, object_size = checkpoint_fixture(branch, fixture) binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, "leaseExpiresAt": (datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat(), "bucketPrefix": bucket, "endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, len(nodes) + 1)}, @@ -507,8 +520,7 @@ def allocate(): else: fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) pending = {"operation": "seed-original-checkpoint-fixture"} - expected = seed(Recorder(probe, directory / "fixture"), bucket, fixture, count, - [256 * 1024 + 123] if bounded else [SIZE]) + expected = seed(Recorder(probe, directory / "fixture"), bucket, fixture, count, [object_size]) baseline = fleet.record("ACK-bound-complete-S3", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) fleet.record("complete-ACK-manifest", {"expected": expected, **verify_seed(expected, baseline)}) pending = None diff --git a/test_validation_heal_capacity.py b/test_validation_heal_capacity.py index 7ad4ab8c..27b24cd6 100644 --- a/test_validation_heal_capacity.py +++ b/test_validation_heal_capacity.py @@ -101,6 +101,32 @@ def test_recovery_rejects_unrecognized_filler_without_deleting_it(self): with self.assertRaises(ValueError): volume.release_filler(folder) self.assertEqual(unexpected.read_bytes(), b"preserve") + def test_block_fill_consumes_small_tail_after_large_write_enospc(self): + tail = SimpleNamespace(f_bfree=1, f_ffree=900, f_frsize=4096) + full = SimpleNamespace(f_bfree=0, f_ffree=900, f_frsize=4096) + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory); (folder / 'padding').mkdir() + with patch.object(volume.os, 'open', return_value=7), patch.object(volume.os, 'close'), \ + patch.object(volume.os, 'write', side_effect=[1024**2, OSError(errno.ENOSPC, 'large allocation'), + 4096, OSError(errno.ENOSPC, 'full')]) as write, patch.object(volume.os, 'fsync'), \ + patch.object(volume.os, 'statvfs', side_effect=[tail, full, full]): + result = volume.fill(folder, capacity.BRANCHES[0]) + self.assertEqual(result['allocatedFillerBytes'], 1024**2 + 4096) + self.assertEqual(result['freeBlocks'], 0) + self.assertEqual(result['actualErrno'], errno.ENOSPC) + self.assertEqual(len(write.call_args_list[2].args[1]), 4096) + + def test_minimum_block_refusal_with_free_blocks_never_claims_full_exhaustion(self): + tail = SimpleNamespace(f_bfree=1, f_ffree=900, f_frsize=4096) + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory); (folder / 'padding').mkdir() + with patch.object(volume.os, 'open', return_value=7), patch.object(volume.os, 'close'), \ + patch.object(volume.os, 'write', side_effect=OSError(errno.ENOSPC, 'refused')) as write, \ + patch.object(volume.os, 'statvfs', return_value=tail), \ + self.assertRaisesRegex(ValueError, 'minimum-block fill left 1 free blocks'): + volume.fill(folder, capacity.BRANCHES[0]) + self.assertEqual(write.call_count, 2) + def test_loop_detach_binding_requires_the_exact_backing_inode_and_device(self): row = {"name": "/dev/loop7", "back-file": "/owned/image", "back-ino": 31, "back-maj:min": f"{os.major(2049)}:{os.minor(2049)}"} diff --git a/test_validation_heal_persistence.py b/test_validation_heal_persistence.py index 28a429aa..a5d9748c 100644 --- a/test_validation_heal_persistence.py +++ b/test_validation_heal_persistence.py @@ -16,6 +16,24 @@ class PersistenceContractTests(unittest.TestCase): + def test_observation_fixtures_fit_existing_inventory_inode_and_volume_limits(self): + from scripts.validation_heal import bounded_volume + for fixture in ('DB', 'DV'): + for branch in (*persistence.IMPLEMENTED, 'journal-write-failure'): + count, size = persistence.checkpoint_fixture(branch, fixture) + data = count * (3 if fixture == 'DV' else 1) + entries = data + ((count + 1) // 2 if fixture == 'DV' else 0) + self.assertGreater(count, 4) + self.assertGreater(size, 1024**2) + self.assertLessEqual(entries, 512) + self.assertLess(data * size, 2 * 1024**3) + if branch == 'journal-write-failure': + # Conservatively allow per-version integrity and metadata, + # object directories, and 128 system/working inodes. + self.assertLess(3 * data + 2 * count + 128, bounded_volume.MAX_INODES) + self.assertLess(data * ((size + 1) // 2 + 16384) + 16 * 1024**2, + bounded_volume.IMAGE_BYTES) + def fixture(self, kind="truncate", *, admission_options=None, terminal_options=None): observed, _, _ = fixture_source.AdminCheckpointTests().fixture() frozen = checkpoint.owned_checkpoint(observed, TOKEN, BUCKET) From e6c7bff29478478adfaf2ecdaf4342cd07bd2894 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 10:58:18 +0800 Subject: [PATCH 225/343] fix(validation): exhaust private ext4 reserves and retry live census drift --- scripts/validation_heal/bounded_volume.py | 61 +++++++++++++-- scripts/validation_heal/checkpoint.py | 2 +- test_validation_heal_capacity.py | 93 +++++++++++++++++++++++ test_validation_heal_checkpoint.py | 31 ++++++++ 4 files changed, 179 insertions(+), 8 deletions(-) diff --git a/scripts/validation_heal/bounded_volume.py b/scripts/validation_heal/bounded_volume.py index f454a7f3..e4ff0dd1 100644 --- a/scripts/validation_heal/bounded_volume.py +++ b/scripts/validation_heal/bounded_volume.py @@ -109,6 +109,37 @@ def mounted(row): return entry +def configure_block_exhaustion(row): + # ext4's internal reserve is separate from mkfs -m 0. Only this newly + # created, UUID/backing-inode-bound loop may expose that reserve to padding. + entry = mounted(row) + require(entry is not None and re.fullmatch(r"/dev/loop[0-9]+", entry["source"]), + "block exhaustion requires its exact mounted private loop") + source = entry["source"] + knob = Path("/sys/fs/ext4") / Path(source).name / "reserved_clusters" + fd = os.open(knob, os.O_RDWR | os.O_NOFOLLOW) + try: + info = os.fstat(fd) + require(stat.S_ISREG(info.st_mode) and info.st_uid == 0, + "private ext4 reserve control is not root-owned sysfs data") + raw = os.read(fd, 64) + require(re.fullmatch(rb"[0-9]{1,20}\n", raw) is not None, + "private ext4 reserve is not a bounded cluster count") + before = int(raw) + require(before <= IMAGE_BYTES // 1024 and mounted(row) == entry, + "private ext4 mount changed before configuring its reserve") + os.lseek(fd, 0, os.SEEK_SET) + require(os.write(fd, b"0\n") == 2, "private ext4 reserve write was incomplete") + os.lseek(fd, 0, os.SEEK_SET) + require(os.read(fd, 64) == b"0\n" and mounted(row) == entry, + "private ext4 reserve or mount differs after configuration") + finally: + os.close(fd) + return {"disk": row["disk"], "uuid": row["uuid"], "source": source, + "reservedClustersBefore": before, "reservedClustersAfter": 0, + "sharedFilesystemModified": False} + + def create(state, roots, owner, branch): require(owner.get("runtimeProfile") in {"bounded-capacity", "journal-bounded"} and read(state, roots, owner) is None and branch in {"bounded-volume-enospc", "bounded-inode-exhaustion"}, "invalid bounded capacity allocation") @@ -128,6 +159,7 @@ def create(state, roots, owner, branch): stream.flush() os.fsync(stream.fileno()) (state / "bounded").mkdir(mode=0o700) + reserve_configuration = [] for row in value["volumes"]: image = Path(row["image"]) descriptor = os.open(image, os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_NOFOLLOW, 0o600) @@ -146,13 +178,16 @@ def create(state, roots, owner, branch): folder.mkdir(mode=0o700) command("mount", "-t", "ext4", "-o", "nodev,nosuid,noexec", loop, str(folder)) mounted(row) + if branch == "bounded-volume-enospc": + reserve_configuration.append(configure_block_exhaustion(row)) folder.chmod(0o700) for name in ("data", "quarantine", "padding"): (folder / name).mkdir(mode=0o700) info = os.statvfs(folder) require(info.f_blocks * info.f_frsize <= IMAGE_BYTES and 0 < info.f_files <= MAX_INODES, "actual bounded filesystem exceeds registered block/inode limits") - return {**value, "hostSharedMountRemounted": False, "allocatedBytesPerNode": len(roots) * IMAGE_BYTES, + return {**value, "ext4ReservedClusterConfiguration": reserve_configuration, + "hostSharedMountRemounted": False, "allocatedBytesPerNode": len(roots) * IMAGE_BYTES, "independentPhysicalDriveCountIncreased": False} @@ -194,7 +229,7 @@ def fill(folder, branch): deadline, written, count = time.monotonic() + 60, 0, 0 if branch == "bounded-volume-enospc": fd = os.open(padding / "blocks.bin", os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) - chunk = 1024**2 + chunk, file_written = 1024**2, 0 try: while True: require(time.monotonic() < deadline and written < IMAGE_BYTES, "bounded block fill exceeded its budget") @@ -203,19 +238,31 @@ def fill(folder, branch): actual = os.write(fd, b"\0" * requested) require(0 < actual <= requested, "bounded block fill made no valid byte progress") written += actual + file_written += actual os.fsync(fd) except OSError as error: require(error.errno == errno.ENOSPC, "bounded fill reached an unrelated IO error") info = os.statvfs(folder) if info.f_bfree == 0: break - require(type(info.f_frsize) is int and 0 < info.f_frsize < chunk, + require(type(info.f_frsize) is int and 0 < info.f_frsize <= chunk, + "bounded fill has no valid minimum allocation unit") + if info.f_frsize < chunk: + chunk = info.f_frsize + continue + # A separate private inode may allocate a final block that + # the current file cannot extend into. Never retry an empty + # inode or exceed the fixture's 128 padding-inode budget. + require(file_written > 0 and count < IMAGE_BYTES // 1024**2 - 1, f"bounded minimum-block fill left {info.f_bfree} free blocks") - # A large allocation can fail while smaller tail blocks - # remain. The final zero-block and candidate probes stay mandatory. - chunk = info.f_frsize + os.close(fd) + fd = None + fd = os.open(padding / f"inode-{count:04d}", os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + count += 1 + file_written = 0 finally: - os.close(fd) + if fd is not None: + os.close(fd) else: require(branch == "bounded-inode-exhaustion", "unreviewed bounded fill mode") while True: diff --git a/scripts/validation_heal/checkpoint.py b/scripts/validation_heal/checkpoint.py index 1e84f7ce..72ca095c 100644 --- a/scripts/validation_heal/checkpoint.py +++ b/scripts/validation_heal/checkpoint.py @@ -201,8 +201,8 @@ def freeze(fleet, token, bucket, *, timeout=60, earlier=None): while True: owner = None for candidate in fleet.nodes: - observed = records(fleet, token, "Admin-checkpoint-live", nodes=(candidate,)) try: + observed = records(fleet, token, "Admin-checkpoint-live", nodes=(candidate,)) owner = owned_checkpoint(observed, token, bucket) if earlier is not None: after_generation(owner, earlier) except ValueError as exc: diff --git a/test_validation_heal_capacity.py b/test_validation_heal_capacity.py index 27b24cd6..0f8759d2 100644 --- a/test_validation_heal_capacity.py +++ b/test_validation_heal_capacity.py @@ -116,6 +116,55 @@ def test_block_fill_consumes_small_tail_after_large_write_enospc(self): self.assertEqual(result['actualErrno'], errno.ENOSPC) self.assertEqual(len(write.call_args_list[2].args[1]), 4096) + def test_same_file_tail_refusal_uses_a_separate_owned_file(self): + tail = SimpleNamespace(f_bfree=1, f_ffree=900, f_frsize=4096) + full = SimpleNamespace(f_bfree=0, f_ffree=900, f_frsize=4096) + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory); (folder / 'padding').mkdir() + with patch.object(volume.os, 'open', side_effect=[7, 8]) as opened, \ + patch.object(volume.os, 'close') as close, patch.object(volume.os, 'fsync'), \ + patch.object(volume.os, 'write', side_effect=[1024**2, + OSError(errno.ENOSPC, 'large'), OSError(errno.ENOSPC, 'same inode allocation'), + 4096, OSError(errno.ENOSPC, 'full')]), \ + patch.object(volume.os, 'statvfs', side_effect=[tail, tail, full, full]): + result = volume.fill(folder, capacity.BRANCHES[0]) + self.assertEqual(result['freeBlocks'], 0) + self.assertEqual(result['allocatedFillerBytes'], 1024**2 + 4096) + self.assertEqual(result['allocatedFillerInodes'], 1) + self.assertEqual(opened.call_args_list[1].args, (folder / 'padding' / 'inode-0000', + os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600)) + self.assertEqual([call.args[0] for call in close.call_args_list], [7, 8]) + + def test_separate_tail_file_cannot_loop_without_progress_or_claim_remaining_blocks(self): + tail = SimpleNamespace(f_bfree=1, f_ffree=900, f_frsize=4096) + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory); (folder / 'padding').mkdir() + with patch.object(volume.os, 'open', side_effect=[7, 8]) as opened, \ + patch.object(volume.os, 'close') as close, patch.object(volume.os, 'fsync'), \ + patch.object(volume.os, 'write', side_effect=[1024**2] + + [OSError(errno.ENOSPC, 'refused')] * 3), \ + patch.object(volume.os, 'statvfs', return_value=tail), \ + self.assertRaisesRegex(ValueError, 'minimum-block fill left 1 free blocks'): + volume.fill(folder, capacity.BRANCHES[0]) + self.assertEqual(opened.call_count, 2) + self.assertEqual([call.args[0] for call in close.call_args_list], [7, 8]) + + def test_block_tail_never_exceeds_the_128_total_padding_inode_budget(self): + tail = SimpleNamespace(f_bfree=1, f_ffree=900, f_frsize=4096) + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory); (folder / 'padding').mkdir() + refused = OSError(errno.ENOSPC, 'same inode allocation') + with patch.object(volume.os, 'open', side_effect=range(7, 136)) as opened, \ + patch.object(volume.os, 'close') as close, patch.object(volume.os, 'fsync'), \ + patch.object(volume.os, 'write', side_effect=[1024**2, refused, refused] + + [v for _ in range(127) for v in (4096, refused)]), \ + patch.object(volume.os, 'statvfs', return_value=tail), \ + self.assertRaisesRegex(ValueError, 'minimum-block fill left 1 free blocks'): + volume.fill(folder, capacity.BRANCHES[0]) + self.assertEqual(opened.call_count, 128) + self.assertEqual(close.call_count, 128) + self.assertEqual(opened.call_args.args[0].name, 'inode-0126') + def test_minimum_block_refusal_with_free_blocks_never_claims_full_exhaustion(self): tail = SimpleNamespace(f_bfree=1, f_ffree=900, f_frsize=4096) with tempfile.TemporaryDirectory() as directory: @@ -253,16 +302,60 @@ def checked_command(*args): patch.object(volume, "regular", return_value=SimpleNamespace(st_size=100)), \ patch.object(volume, "loop_for", return_value="/dev/loop7"), \ patch.object(volume, "mounted", return_value={"verified": True}), \ + patch.object(volume, "configure_block_exhaustion", return_value={"reservedClustersAfter": 0}) as reserve, \ patch.object(volume.os, "posix_fallocate", create=True) as allocate, \ patch.object(volume.os, "statvfs", return_value=SimpleNamespace(f_blocks=32768, f_frsize=4096, f_files=1024)), \ patch.object(volume.shutil, "disk_usage", return_value=SimpleNamespace(free=10 * 1024**3)): result = volume.create(state, roots, owner, capacity.BRANCHES[0]) self.assertEqual(allocate.call_count, 4) + self.assertEqual(reserve.call_count, 4) + self.assertEqual(len(result["ext4ReservedClusterConfiguration"]), 4) self.assertTrue(all(call.args[1:] == (0, volume.IMAGE_BYTES) for call in allocate.call_args_list)) self.assertEqual(sum(args[0] == "mkfs.ext4" for args in calls), 4) self.assertFalse(result["hostSharedMountRemounted"]) self.assertFalse(result["independentPhysicalDriveCountIncreased"]) + def test_private_ext4_reserve_configuration_requires_exact_mount_and_readback(self): + row = {"disk": 1, "uuid": "owned-uuid"} + entry = {"source": "/dev/loop7", "uuid": row["uuid"]} + for change in ('valid', 'already-zero', 'unmounted', 'shared-device', 'changed-before', + 'changed-after', 'malformed', 'oversize', 'foreign-owner', 'symlink-type', + 'short-write', 'wrong-readback'): + mounts = [entry, entry, entry] + reads = [b'655\n', b'0\n'] + mode, uid, written = 0o100644, 0, 2 + if change == 'already-zero': reads[0] = b'0\n' + elif change == 'unmounted': mounts[0] = None + elif change == 'shared-device': mounts[0] = {**entry, 'source': '/dev/sda1'} + elif change == 'changed-before': mounts[1] = {**entry, 'uuid': 'foreign'} + elif change == 'changed-after': mounts[2] = None + elif change == 'malformed': reads[0] = b'655x\n' + elif change == 'oversize': reads[0] = b'9999999999\n' + elif change == 'foreign-owner': uid = 1000 + elif change == 'symlink-type': mode = 0o120777 + elif change == 'short-write': written = 1 + elif change == 'wrong-readback': reads[1] = b'655\n' + with self.subTest(change=change), patch.object(volume, 'mounted', side_effect=mounts), \ + patch.object(volume.os, 'open', return_value=7) as opened, \ + patch.object(volume.os, 'fstat', return_value=SimpleNamespace(st_mode=mode, st_uid=uid)), \ + patch.object(volume.os, 'read', side_effect=reads), patch.object(volume.os, 'lseek'), \ + patch.object(volume.os, 'write', return_value=written) as write, \ + patch.object(volume.os, 'close') as close: + if change in {'valid', 'already-zero'}: + result = volume.configure_block_exhaustion(row) + self.assertEqual(result['reservedClustersBefore'], int(reads[0])) + self.assertEqual(result['reservedClustersAfter'], 0) + self.assertFalse(result['sharedFilesystemModified']) + self.assertEqual(opened.call_args.args, (Path('/sys/fs/ext4/loop7/reserved_clusters'), + os.O_RDWR | os.O_NOFOLLOW)) + write.assert_called_once_with(7, b'0\n') + else: + with self.assertRaises(ValueError): volume.configure_block_exhaustion(row) + if change not in {'changed-after', 'short-write', 'wrong-readback'}: + write.assert_not_called() + if opened.called: close.assert_called_once_with(7) + else: close.assert_not_called() + def test_registered_quota_probe_signs_and_preserves_the_exact_request(self): class Response(BytesIO): code = 200 diff --git a/test_validation_heal_checkpoint.py b/test_validation_heal_checkpoint.py index e96059d2..c143afd8 100644 --- a/test_validation_heal_checkpoint.py +++ b/test_validation_heal_checkpoint.py @@ -294,6 +294,37 @@ def records(fleet, token, label, **params): self.assertEqual(birth, identity) self.assertTrue(frozen['rootAdmission']['originalDurableResponsibility']) + def test_live_record_drift_is_retried_inside_the_original_observation_budget(self): + from scripts.validation_heal.fleet_node import PROCESS_FIELDS + observed, _, _ = self.fixture(); roots, _ = self.root_fixture() + identity = {name: name for name in PROCESS_FIELDS}; identity.update(pid=50) + observed['node'].update(processIdentity=identity, processBound=True, stable=True, independentReads=2) + fleet = SimpleNamespace(nodes=['node'], frozen_processes={}, record=lambda label, value: value, + remote=lambda *a, **k: {**identity, 'state': 'T'}) + calls = [] + def records(fleet, token, label, **params): + calls.append(label) + if len(calls) == 1: + raise ValueError('NOT_OBSERVED: exact durable records changed between the two bounded reads') + return roots if params.get('family') == 'root' else observed + with patch.object(reader, 'records', side_effect=records), patch.object(reader.time, 'sleep'): + frozen, birth = reader.freeze(fleet, TOKEN, BUCKET) + self.assertEqual(calls.count('Admin-checkpoint-live'), 2) + self.assertEqual(birth, identity) + self.assertTrue(frozen['rootAdmission']['originalDurableResponsibility']) + + def test_record_drift_timeout_and_non_observation_errors_do_not_signal(self): + for message in ('NOT_OBSERVED: exact durable record changed during a bounded read', + 'foreign process birth', 'checkpoint digest is invalid'): + events = [] + fleet = SimpleNamespace(nodes=['node'], frozen_processes={}, record=lambda label, value: value, + remote=lambda *a, **k: events.append(a)) + with self.subTest(message=message), patch.object(reader, 'records', side_effect=ValueError(message)), \ + patch.object(reader.time, 'monotonic', side_effect=[0, 61]), self.assertRaises(ValueError): + reader.freeze(fleet, TOKEN, BUCKET) + self.assertEqual(events, []) + self.assertEqual(fleet.frozen_processes, {}) + def test_full_census_after_early_freeze_still_rejects_a_second_physical_owner(self): from scripts.validation_heal.fleet_node import PROCESS_FIELDS observed, _, _ = self.fixture() From b16e6072679d6a60a044b9d4520b30b4a79b5557 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 11:14:25 +0800 Subject: [PATCH 226/343] fix(validation): restore source A10 recursive orphan cleanup procedure --- docs/validation-heal-cases.md | 11 ++ docs/validation-migration-map.md | 52 ++++--- scripts/validation_heal/options.py | 120 ++++++++++++---- test_validation_heal_options.py | 220 ++++++++++++++++++++++++++++- 4 files changed, 348 insertions(+), 55 deletions(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index 2e64d836..9ff27c47 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -211,6 +211,17 @@ Topologies: H1,H2. Branches: recursive-false, recursive-true, recreate, remove-f 3. 逐一 START 并对比只改变一个字段的基线；recursive 检查子前缀，pool/set 检查实际目标身份，非法组合要求结构化拒绝且无副作用。 4. WAIT、PHYSICAL、VERIFY 三者验收；updateParity/readRepair 若候选不支持必须显示明确拒绝，不可悄悄按默认修复算覆盖。 +Source implementation detail: frozen `heal_h1_options_batch.py` and +`heal_h1_options_node.py` at `524eb5b32c3236497fdd8a9271df0494fa3d9974` +run the remove=true orphan with a recursive bucket sweep after exact-version +DELETE and one-member full-object replay (including metadata). Its false +baseline is an option reference; remove=false is a separate repairable-object +branch. The adapter keeps every original H1/H2 DB/DV branch, with 68 registered +execution instances, and rejects the four formerly invented orphan baseline +instances. Original failures remain archived. Physical closure requires the +whole orphan to disappear, every post-DELETE control/format to remain exact, +and complete version-addressed task receipts/counters before reads or rescue. + Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) Source assertions: diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 0e945d8d..a4de2e01 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -1658,29 +1658,35 @@ passes 399 tests with eight macOS skips; C10 now has 22 tests. Its queued live plan is updated to this corrected revision before fleet dispatch. -A10 now dispatches all nine source option branches on H1/H2, with fresh DB and -DV namespaces for both baseline and selected requests: 72 distinct instances. -Each pair changes only the reviewed option, or the inseparable pool/set -selector pair. The exact candidate Admin/selector source hashes and supported -fields are recorded in the contract; this is API coverage, not an rc claim. -H2 scope probes use two actual eight-drive sets across the original four -nodes. Recursive probes include a child prefix, an adjacent prefix and outside -controls; selectors bind each ACK version to its measured physical set. -Every historical version and marker must have complete, exact responsibility, -truthful counters and retained settings. `readRepair=true` and invalid selectors -must return HTTP 400 `InvalidArgument`, leave no active responsibility and -preserve both missing and bitflipped control parts. - -The remove pair reuses C10 descriptor capture, complete original-member census, -post-DELETE authority and safe recovery. Both DB and DV delete every original -target generation through separately recorded S3 ACKs, prove no live metadata -or payload reference, then replay only one owned data directory. Both phases -use the same data-only fault; only `remove` changes. Product physical and all -peer S3 checks precede recovery. An accepted parameter which fails its registered -physical behavior remains FAIL; it is never counted as default repair or as -option rejection. Unknown, wrong or duplicate mutation ACKs and incomplete -post-DELETE restoration retain DIRTY namespaces. This executor has 18 local -contract tests and still requires canonical live evidence. +A10 dispatches all nine source option branches on H1/H2 and fresh DB/DV +namespaces. The frozen source `524eb5b32c3236497fdd8a9271df0494fa3d9974` +executes its proven orphan only with `recursive=true,remove=true`; its +`remove=false` branch independently heals a repairable ACK-bound object. +The former migration added four synthetic remove=false orphan baseline +instances. Those extra instances are outside the source procedure and have +been removed; all original branches, both topologies and both fixture types +remain mandatory, for 68 execution instances. Other option probes retain +baseline/selected pairs changing one option or the pool/set selector pair. +Exact candidate Admin/selector and validation-source hashes are recorded; +this is API coverage. H2 scope probes use two actual eight-drive sets across +the original four nodes. Recursive probes include child/adjacent/outside +controls; selectors bind every ACK version to its measured physical set. +Historical versions, markers, exact responsibility, counters, retained settings +and structured rejections remain required. + +The source-aligned remove=true probe seeds one orphan ACK version and retains +all control versions/markers, streams every original shard, archives the full +original tree, performs an exact DELETE, and proves S3 and all-member physical +absence. It then replays the complete original object including xl.meta and +integrity sidecars on exactly one below-quorum member. A recursive bucket +sweep must remove the whole orphan object before GET or rescue and leave +all post-DELETE controls/formats unchanged. Every original orphan/control +version must have an exact complete canonical receipt and counter partition; +only the removed ACK may report repaired or authoritatively_absent. Rescue +restores only post-DELETE authority. Lost mutation ACKs and incomplete restore +retain DIRTY namespaces. The old data-only/exact-object construction and its +original live failures remain archived; they do not establish a source remove +product failure. No new live or complete canonical PASS is claimed here. B08 live run `38104014806` remains FAIL with cleanup PASS: both write-loss branches produced incomplete target identities; H2 lock loss also failed a diff --git a/scripts/validation_heal/options.py b/scripts/validation_heal/options.py index 619236d8..2fc09ddc 100644 --- a/scripts/validation_heal/options.py +++ b/scripts/validation_heal/options.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""A10: paired Admin option probes on fresh DB/DV and owned C10 orphans.""" +"""A10: source option probes on fresh DB/DV and owned C10 orphan archives.""" import argparse import copy @@ -60,7 +60,12 @@ 'pool/set': 'supported paired selector; exact independently measured physical set'}, 'dangerousEffects': {'remove': 'may delete unreferenced persisted bytes; never applied to a live fixture', 'recreate': 'may reconstruct missing data; original ACK identities and complete controls stay fixed'}, - 'cliCoverageClaimed': False} + 'cliCoverageClaimed': False, + 'validationSource': {'commit': '524eb5b32c3236497fdd8a9271df0494fa3d9974', 'sourceFiles': { + 'scripts/heal_h1_options_batch.py': '918d7b6d7a9bacb37095210b5a72a10b024b21f7a9cf46501eeae6272dd625d2', + 'scripts/heal_h1_options_node.py': 'b78e7a65d071e08be3ee9d615091c69f801d5a48badb85f246acf9340fd829bc'}}, + 'removeTrueProcedure': 'one ACK version, exact DELETE authority, one complete original below-quorum object tree, recursive bucket sweep, whole object removal', + 'removeTrueBaseline': 'option reference only, as in the frozen source; no synthetic remove=false orphan task'} def paired_options(branch): @@ -70,7 +75,9 @@ def paired_options(branch): if branch == 'recursive-false': baseline['recursive'] = True elif branch == 'recursive-true': selected['recursive'] = True elif branch == 'recreate': selected['recreate'] = True - elif branch == 'remove-true': selected['remove'] = True + elif branch == 'remove-true': + baseline['recursive'] = selected['recursive'] = True + selected['remove'] = True elif branch == 'update-parity': selected['updateParity'] = True elif branch == 'read-repair': selected['readRepair'] = True elif branch.startswith('selector-'): @@ -87,6 +94,13 @@ def paired_options(branch): return baseline, selected +def source_phases(branch): + require(branch in BRANCHES, 'unknown A10 source branch') + # The frozen source runs the proven orphan only with remove=true. Its + # remove=false branch independently heals a repairable acknowledged object. + return ('selected',) if branch == 'remove-true' else ('baseline', 'selected') + + class FixtureRecorder(Recorder): """Retain the namespace unless all mutation receipts and identities resolve.""" def __init__(self, *args, fixture='DB', **kwargs): @@ -128,14 +142,16 @@ def call(self, operation, params, body=None, **kwargs): return value -def seed_fixture(recorder, bucket, fixture, keys=KEYS): - require(fixture in FIXTURES and len(keys) == len(set(keys)), 'invalid A10 fixture') +def seed_fixture(recorder, bucket, fixture, keys=KEYS, *, single_version_key=None): + require(fixture in FIXTURES and len(keys) == len(set(keys)) and keys + and (single_version_key is None or single_version_key in keys[:-1]), 'invalid A10 fixture') recorder.call('create_bucket', {'Bucket': bucket}) if fixture == 'DV': recorder.call('put_bucket_versioning', {'Bucket': bucket, 'VersioningConfiguration': {'Status': 'Enabled'}}) expected = [] for key in keys: - for number in range(3 if fixture == 'DV' else 1): + generations = 3 if fixture == 'DV' and key != single_version_key else 1 + for number in range(generations): body = payload(f'A10/{fixture}/{key}/{number}', SIZE) response = recorder.call('put_object', {'Bucket': bucket, 'Key': key, 'Metadata': {'heal-fixture': 'A10', 'generation': str(number)}, 'Tagging': 'suite=heal&fixture=' + fixture}, body) @@ -145,7 +161,7 @@ def seed_fixture(recorder, bucket, fixture, keys=KEYS): 'A10 candidate ACK disagrees with DB/DV versioning') require((key, version) not in {(entry['key'], entry['versionId']) for entry in expected}, 'A10 PUT ACK reused a version') - expected.append({'key': key, 'versionId': version, 'isLatest': number == (2 if fixture == 'DV' else 0), + expected.append({'key': key, 'versionId': version, 'isLatest': number == generations - 1, 'deleteMarker': False, 'payload': {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}}) recorder.pending = None if fixture == 'DV': @@ -202,6 +218,7 @@ def selection(branch, options, mapped): def request_path(branch, bucket): if branch.startswith('selector-'): return '/rustfs/admin/v3/heal/' + if branch == 'remove-true': return '/rustfs/admin/v3/heal/' + quote(bucket, safe='') key = 'scope/' if branch.startswith('recursive-') else KEYS[0] return '/rustfs/admin/v3/heal/' + quote(bucket, safe='') + '/' + quote(key, safe='/') @@ -239,8 +256,11 @@ def run_request(probe, fleet, path, options, active, *, rejected=False): return status -def validate_outcome(status, mapped, selected, repaired, options): +def validate_outcome(status, mapped, selected, repaired, options, *, absent=frozenset()): outcome = status.get('outcome') or {} + require(absent <= selected and not (repaired & absent) + and (not absent or options.get('remove') is True and options.get('recursive') is True), + 'A10 absence receipts require the exact destructive source scope') if not selected: buckets = {identity[0] for identity in mapped} require(not repaired and len(buckets) == 1, 'A10 absent prefix changed its physical scope or claimed a repair') @@ -268,14 +288,16 @@ def validate_outcome(status, mapped, selected, repaired, options): incarnation = item.get('bucketIncarnationId') require(isinstance(incarnation, str) and incarnation and incarnations.setdefault(identity[0], incarnation) == incarnation, 'A10 lost or mixed a bucket incarnation') - state = 'repaired' if identity in repaired else 'metadata_healthy' if target['deleteMarker'] else 'verified_healthy' + state = ('authoritatively_absent' if identity in absent else 'repaired' if identity in repaired + else 'metadata_healthy' if target['deleteMarker'] else 'verified_healthy') require(receipt.get('disposition', {}).get('state') == state, 'A10 false repair, unknown or incomplete integrity') seen.add(identity) expected = {'processed': len(selected), 'healed': len(repaired), 'unchanged': len(selected) - len(repaired), 'failed': 0, 'unknown': 0, 'skipped': 0, 'attemptFailures': 0, 'overflowed': False} require(all(type(counters.get(name)) is type(value) and counters[name] == value for name, value in expected.items()), 'A10 counters disagree with the complete independent ACK/physical manifest') - return {'selectedACKs': len(selected), 'repairedACKs': len(repaired), 'completeVersionsAndMarkers': True} + return {'selectedACKs': len(selected), 'repairedACKs': len(repaired), + 'authoritativelyAbsentACKs': len(absent), 'completeVersionsAndMarkers': True} def physical_selection(original, after, targets, selected): @@ -386,11 +408,38 @@ def scope_probe(fleet, probe, binding, recorder, bucket, topology, fixture, bran return {'terminal': terminal, 'physical': physical, 'fixture': fixture, 'originalACKs': len(expected)} +def validate_orphan_replay(original, before, authority, bucket, target): + prefix = bucket + '/' + orphan.KEY + '/' + source, actual, deleted = files(original), files(before), files(authority) + selected = target['node'], target['disk'] + coding = target['logical']['erasure'] + require(type(coding.get('data_blocks')) is int and 1 < coding['data_blocks'] + and type(coding.get('parity_blocks')) is int and coding['parity_blocks'] > 0 + and coding['data_blocks'] + coding['parity_blocks'] == len(orphan.members(original)), + 'A10 orphan replay is not below the independently decoded read quorum') + expected = {k: r for k, r in source.items() if k[:2] == selected and k[2].startswith(prefix)} + require(expected and (selected[0], selected[1], prefix + 'xl.meta') in expected + and any(k[2].startswith(prefix + target['dataDir'] + '/') for k in expected), + 'A10 source orphan lacks its original metadata or payload') + require(not any(k[2].startswith(prefix) for k in deleted) + and {k: r for k, r in actual.items() if k[2].startswith(prefix)} == expected, + 'A10 replay must preserve the complete original object on exactly one member') + require({k: r for k, r in actual.items() if not k[2].startswith(prefix)} == deleted + == {k: r for k, r in source.items() if not k[2].startswith(prefix)} + and {n: d['formats'] for n, d in original.items()} + == {n: d['formats'] for n, d in before.items()} == {n: d['formats'] for n, d in authority.items()} + and orphan.members(before) == orphan.members(authority) == orphan.members(original), + 'A10 orphan replay changed post-DELETE controls or physical membership') + return {'completeOriginalObjectTree': True, 'replayedMembers': 1, + 'independentlyDecodedReadQuorum': coding['data_blocks'], 'postDeleteAuthorityRetained': True} + + def orphan_probe(fleet, probe, binding, recorder, bucket, topology, fixture, options, active): - # A10 reuses C10's complete descriptor archives and post-DELETE authority, - # while keeping the same data-only replay in both remove option phases. + require(options['recursive'] is True and options['remove'] is True and options['dryRun'] is False, + 'A10 source orphan requires a recursive destructive sweep') + # C10 archives preserve the full old tree and a distinct post-DELETE rescue. keys = (orphan.KEY, 'control.bin', 'marker-control.bin') - expected = seed_fixture(recorder, bucket, fixture, keys) + expected = seed_fixture(recorder, bucket, fixture, keys, single_version_key=orphan.KEY) doomed = next(entry for entry in expected if entry['key'] == orphan.KEY and entry['isLatest']) if fixture == 'DV': # Remove every orphan-key generation, with separate exact durable ACKs. @@ -401,6 +450,9 @@ def orphan_probe(fleet, probe, binding, recorder, bucket, topology, fixture, opt original = fleet.snapshot('A10-orphan-original-all-members', quiescent=True) mapped = map_fixture(original, bucket, expected, topology, multi_set=False) target = mapped[(bucket, orphan.KEY, doomed['versionId'])] + require(len(doomed_entries) == 1, 'A10 source orphan requires one acknowledged generation') + verify_parts(fleet, {identity: item for identity, item in mapped.items() if not item['deleteMarker']}, + 'A10-all-original-orphan-and-control-streaming') orphan.capture(fleet, original, bucket, 'old') fleet.start() for entry in doomed_entries: @@ -424,24 +476,29 @@ def orphan_probe(fleet, probe, binding, recorder, bucket, topology, fixture, opt fleet.a10_orphan_restored = False orphan.arm_all(fleet, authority, bucket, 'authority') try: - fleet.record('A10-owned-unreferenced-data-only-replay', fleet.remote(target['node'], 'orphan-install', - disk=target['disk'], bucket=bucket, key=orphan.KEY, dataDir=target['dataDir'], mode='data')) + fleet.record('A10-owned-complete-original-object-replay', fleet.remote(target['node'], 'orphan-install', + disk=target['disk'], bucket=bucket, key=orphan.KEY, dataDir=target['dataDir'], mode='object')) before = fleet.snapshot('A10-orphan-fault-before-GET', quiescent=True) - require({identity[:2] for identity in files(before) if identity[2].startswith(bucket + '/' + orphan.KEY + '/')} - == {(target['node'], target['disk'])}, 'A10 replay escaped its one original member') - require(all(not values for values in orphan.references(before, bucket, orphan.KEY).values()), - 'A10 controlled orphan still has a live metadata reference') + fleet.record('A10-source-orphan-replay-proof', validate_orphan_replay(original, before, authority, bucket, target)) fleet.start() require(physical_content(before) == physical_content(fleet.snapshot('A10-orphan-immediately-before-START')), 'NOT_OBSERVED: A10 orphan changed before admission') - status = run_request(probe, fleet, '/rustfs/admin/v3/heal/' + bucket + '/' + orphan.KEY, options, active) + status = run_request(probe, fleet, request_path('remove-true', bucket), options, active) idle_fleet(fleet, binding) fleet.stop() after = fleet.snapshot('A10-orphan-product-before-GET-or-rescue', quiescent=True) - branch = 'a10-remove-true' if options['remove'] else 'remove-false' - physical = orphan.physical_safety(before, after, authority, bucket, expected, branch, - doomed['versionId'], target['dataDir'], (target['node'], target['disk'])) - terminal = orphan.terminal_safety(status, branch, bucket, options) + require(not any(identity[2].startswith(bucket + '/' + orphan.KEY + '/') for identity in files(after)) + and physical_content(after) == physical_content(authority), + 'A10 source sweep did not remove the whole orphan while preserving every post-DELETE control') + physical = {'wholeOrphanObjectRemoved': True, 'allPostDeleteControlsAndFormatsUnchanged': True, + 'beforeAnyTargetGETOrBackupRescue': True} + removed = {(bucket, orphan.KEY, doomed['versionId'])} + absent = {(r.get('identity', {}).get('bucket'), r.get('identity', {}).get('object'), + public_version_id(r.get('identity', {}).get('versionId'))) + for r in status.get('outcome', {}).get('objects', []) + if r.get('disposition', {}).get('state') == 'authoritatively_absent'} + require(absent <= removed, 'A10 sweep claimed a live control was absent') + terminal = validate_outcome(status, mapped, set(mapped), removed - absent, options, absent=absent) fleet.start() verify_public(fleet, binding, bucket, expected, baseline, fixture, 'A10-orphan-product-before-rescue-S3') for endpoint in binding['endpoints']: @@ -467,16 +524,17 @@ def orphan_probe(fleet, probe, binding, recorder, bucket, topology, fixture, opt fleet.start() verify_public(fleet, binding, bucket, expected, baseline, fixture, 'A10-orphan-all-peer-final-S3') return {'fixture': fixture, 'terminal': terminal, 'physical': physical, 'allOriginalPostDeleteAuthoritiesRestored': True, - 'C10UnreferencedOrphanProof': True, 'deletedACKs': len(doomed_entries), 'rescueIsProductRepair': False} + 'sourceCompleteBelowQuorumOrphanProof': True, 'deletedACKs': len(doomed_entries), 'rescueIsProductRepair': False} def run_instance(args, topology, nodes, branch, fixture, phase): + require(phase in source_phases(branch), 'A10 phase is not part of its source procedure') directory = args.evidence / topology / branch / fixture / phase directory.mkdir(parents=True) candidate = json.loads(args.candidate.read_text()) require(candidate['source_sha'] == REFERENCE['candidateCommit'], 'A10 option contract is not reviewed for this candidate') run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" - is_orphan = branch.startswith('remove-') + is_orphan = branch == 'remove-true' runtime = 'orphan-preserve' if is_orphan else 'scope-isolation' if topology == 'H2' else 'manual' fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile=runtime) bucket = 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] @@ -493,7 +551,9 @@ def run_instance(args, topology, nodes, branch, fixture, phase): options = baseline_options if phase == 'baseline' else selected_options fleet.record('A10-source-bound-contract', {**REFERENCE, 'branch': branch, 'fixture': fixture, 'phase': phase, 'baselineOptions': baseline_options, 'selectedOptions': selected_options, - 'singleLogicalFieldDelta': 'pool/set selector pair' if branch.startswith('selector-') else branch}) + 'singleLogicalFieldDelta': 'pool/set selector pair' if branch.startswith('selector-') else branch, + 'registeredExecutionPhases': source_phases(branch), + 'baselineExecutionRequired': branch != 'remove-true'}) fleet.record('binding', binding) fleet.record('inspector-build', verify_inspector(args.tool_directory, candidate)) probe, active, recorder = Probe(binding), [], None @@ -535,7 +595,7 @@ def run_instance(args, topology, nodes, branch, fixture, phase): def complete_results(results): wanted = {(topology, branch, fixture, phase) for topology in ('H1', 'H2') for branch in BRANCHES - for fixture in FIXTURES for phase in ('baseline', 'selected')} + for fixture in FIXTURES for phase in source_phases(branch)} actual = {(row['topology'], row['branch'], row['fixture'], row['phase']) for row in results} require(len(actual) == len(results) and actual <= wanted, 'A10 duplicate or foreign parameter instance') return actual == wanted and all(row['verdict'] == row['cleanup'] == 'PASS' and row.get('fullProbeCompleted') is True @@ -563,7 +623,7 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f'A10 interrupted by sig for topology in ('H1', 'H2'): for branch in BRANCHES: for fixture in FIXTURES: - for phase in ('baseline', 'selected'): + for phase in source_phases(branch): row = run_instance(args, topology, nodes[:1] if topology == 'H1' else nodes[:4], branch, fixture, phase) results.append(row) if row['cleanup'] != 'PASS': return @@ -571,7 +631,7 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f'A10 interrupted by sig complete = complete_results(results) save(args.evidence / 'healthy-summary.json', {'caseId': CASE_ID, 'results': results, 'complete': complete, 'cleanup': 'PASS' if results and all(row['cleanup'] == 'PASS' for row in results) else 'FAIL'}) - require(complete, 'A10 requires every H1/H2 DB/DV baseline/selected option instance and complete cleanup') + require(complete, 'A10 requires every registered H1/H2 DB/DV source option instance and complete cleanup') if __name__ == '__main__': diff --git a/test_validation_heal_options.py b/test_validation_heal_options.py index cde7e6c7..924951a2 100644 --- a/test_validation_heal_options.py +++ b/test_validation_heal_options.py @@ -1,4 +1,6 @@ import copy +from contextlib import ExitStack +import json from pathlib import Path import tempfile import types @@ -57,6 +59,187 @@ def test_request_pairs_do_not_change_dangerous_or_unrelated_defaults(self): changed = {name for name in baseline if baseline[name] != selected[name]} self.assertEqual(len(changed), 2 if branch.startswith('selector-') else 0 if branch == 'remove-false' else 1) + def test_remove_true_preserves_the_frozen_recursive_bucket_source_procedure(self): + baseline, selected = options.paired_options('remove-true') + self.assertTrue(baseline['recursive']) + self.assertTrue(selected['recursive']) + self.assertEqual(options.request_path('remove-true', self.bucket), + '/rustfs/admin/v3/heal/' + self.bucket) + self.assertEqual(options.source_phases('remove-true'), ('selected',)) + self.assertEqual(options.source_phases('remove-false'), ('baseline', 'selected')) + with self.assertRaisesRegex(ValueError, 'source procedure'): + options.run_instance(None, 'H1', ['node'], 'remove-true', 'DV', 'baseline') + + def orphan_scene(self): + import uuid + directory = str(uuid.UUID(int=11)); version = str(uuid.UUID(int=12)) + authority = {'node': {'formats': [{'disk': disk, 'sha256': str(disk)} for disk in range(4)], + 'files': [{'disk': disk, 'path': self.bucket + '/control.bin/xl.meta', 'size': 7, 'sha256': 'control'} + for disk in range(4)]}} + original = copy.deepcopy(authority) + for disk in range(4): + for name in ('xl.meta', directory + '/part.1', directory + '/part.1.integrity'): + original['node']['files'].append({'disk': disk, 'path': self.bucket + '/' + options.orphan.KEY + '/' + name, + 'size': 100, 'sha256': str(disk) + name}) + before = copy.deepcopy(authority) + before['node']['files'] += [copy.deepcopy(r) for r in original['node']['files'] + if r['disk'] == 0 and '/' + options.orphan.KEY + '/' in r['path']] + target = {'node': 'node', 'disk': 0, 'dataDir': directory, 'versionId': version, + 'logical': {'erasure': {'data_blocks': 2, 'parity_blocks': 2}}} + return original, before, authority, target + + def test_full_original_orphan_tree_and_one_below_quorum_member_are_required(self): + original, before, authority, target = self.orphan_scene() + options.validate_orphan_replay(original, before, authority, self.bucket, target) + for kind in ('data-only', 'sidecar', 'second-member', 'control', 'format', 'original-format', 'original-control', 'not-below-quorum'): + scene, changed, source = copy.deepcopy(before), copy.deepcopy(target), copy.deepcopy(original) + if kind == 'data-only': scene['node']['files'] = [r for r in scene['node']['files'] + if not r['path'].endswith(options.orphan.KEY + '/xl.meta')] + elif kind == 'sidecar': scene['node']['files'] = [r for r in scene['node']['files'] + if not r['path'].endswith('part.1.integrity')] + elif kind == 'second-member': + row = copy.deepcopy(scene['node']['files'][-1]); row['disk'] = 1; scene['node']['files'].append(row) + elif kind == 'control': scene['node']['files'][0]['sha256'] = 'changed' + elif kind == 'format': scene['node']['formats'][0]['sha256'] = 'changed' + elif kind == 'original-format': source['node']['formats'][0]['sha256'] = 'changed' + elif kind == 'original-control': source['node']['files'][0]['sha256'] = 'changed' + else: changed['logical']['erasure']['data_blocks'] = 1 + with self.subTest(kind=kind), self.assertRaises(ValueError): + options.validate_orphan_replay(source, scene, authority, self.bucket, changed) + + def test_source_orphan_replay_uses_actual_h2_sixteen_member_read_quorum(self): + original, before, authority, target = self.orphan_scene() + source = {}; injected = {}; deleted = {} + for index in range(4): + name = 'node' + str(index) + source[name] = copy.deepcopy(original['node']) + deleted[name] = copy.deepcopy(authority['node']) + injected[name] = copy.deepcopy(before['node'] if index == 0 else authority['node']) + target['node'] = 'node0' + target['logical']['erasure'] = {'data_blocks': 8, 'parity_blocks': 8} + result = options.validate_orphan_replay(source, injected, deleted, self.bucket, target) + self.assertEqual(result['replayedMembers'], 1) + self.assertEqual(result['independentlyDecodedReadQuorum'], 8) + injected['node1'] = copy.deepcopy(before['node']) + with self.assertRaises(ValueError): options.validate_orphan_replay(source, injected, deleted, self.bucket, target) + + def test_single_deleted_orphan_generation_keeps_all_dv_controls_and_marker_acks(self): + calls = [] + def call(op, params, body=None): + calls.append((op, dict(params))) + return {'VersionId': str(__import__('uuid').UUID(int=len(calls))), + **({'DeleteMarker': True} if op == 'delete_object' else {})} + recorder = types.SimpleNamespace(call=call, pending=None) + keys = (options.orphan.KEY, 'control.bin', 'marker-control.bin') + manifest = options.seed_fixture(recorder, self.bucket, 'DV', keys, + single_version_key=options.orphan.KEY) + orphan_rows = [r for r in manifest if r['key'] == options.orphan.KEY] + self.assertEqual(len(orphan_rows), 1) + self.assertTrue(orphan_rows[0]['isLatest']) + self.assertFalse(orphan_rows[0]['deleteMarker']) + self.assertEqual(len([r for r in manifest if r['key'] == 'control.bin']), 3) + self.assertEqual(len([r for r in manifest if r['key'] == 'marker-control.bin']), 4) + self.assertEqual(sum(r['deleteMarker'] for r in manifest), 1) + + def test_source_orphan_execution_removes_whole_tree_before_reads_and_rescues_only_delete_authority(self): + for state, failure in (('repaired', None), ('authoritatively_absent', None), + ('repaired', 'leftover-metadata'), ('repaired', 'lost-install-ack'), + ('repaired', 'failed-authority-restore')): + original, before, authority, target = self.orphan_scene() + _, request = options.paired_options('remove-true') + version = target['versionId']; control_version = str(__import__('uuid').UUID(int=13)) + manifest = [{'key': options.orphan.KEY, 'versionId': version, 'isLatest': True, 'deleteMarker': False}, + {'key': 'control.bin', 'versionId': control_version, 'isLatest': True, 'deleteMarker': False}] + mapped = {(self.bucket, r['key'], r['versionId']): { + **(target if r['key'] == options.orphan.KEY else {}), 'poolIndex': 0, 'setIndex': 0, + 'deleteMarker': False, 'isLatest': True} for r in manifest} + status = {'summary': 'finished', 'settings': request, 'outcome': { + 'execution': {'state': 'completed'}, 'coverage': 'complete', 'objectsTruncated': False, + 'counters': {'processed': 2, 'healed': int(state == 'repaired'), + 'unchanged': 2 - int(state == 'repaired'), 'failed': 0, 'unknown': 0, + 'skipped': 0, 'attemptFailures': 0, 'overflowed': False}, + 'objects': [{'identity': {'kind': 'object', 'bucket': self.bucket, 'object': r['key'], + 'versionId': r['versionId'], 'poolIndex': None, 'setIndex': None, + 'bucketIncarnationId': 'one-incarnation'}, 'disposition': { + 'state': state if r['key'] == options.orphan.KEY else 'verified_healthy'}} for r in manifest]}} + events = [] + after = copy.deepcopy(authority) + if failure == 'leftover-metadata': + after['node']['files'].append(copy.deepcopy(next(r for r in before['node']['files'] + if r['path'].endswith(options.orphan.KEY + '/xl.meta')))) + def snapshot(label, **kw): + events.append(label) + if label == 'A10-orphan-original-all-members': return original + if label in ('A10-orphan-fault-before-GET', 'A10-orphan-immediately-before-START'): return before + if label == 'A10-orphan-product-before-GET-or-rescue': return after + if label == 'A10-orphan-last-DELETE-restored' and failure == 'failed-authority-restore': return before + return authority + def remote(node, action, **params): + events.append((action, params)) + self.assertEqual(action, 'orphan-install') + self.assertEqual(params['mode'], 'object') + if failure == 'lost-install-ack': raise RuntimeError('lost original install ACK') + return {'state': 'APPLIED', 'mode': 'object'} + fleet = types.SimpleNamespace(record=lambda label, value: value, snapshot=snapshot, remote=remote, + stop=lambda: None, start=lambda: None, orphan_faults=[]) + recorder = types.SimpleNamespace(pending=None, call=unittest.mock.Mock(return_value={ + 'VersionId': version, 'ResponseMetadata': {'HTTPStatusCode': 204}})) + probe = types.SimpleNamespace(inventory=lambda *a, **k: {}) + peer = types.SimpleNamespace(s3=unittest.mock.Mock(return_value={ + 'acknowledged': False, 'resultUnknown': False, 'response': { + 'ResponseMetadata': {'HTTPStatusCode': 404}, 'Error': {'Code': 'NoSuchVersion'}}})) + with self.subTest(state=state, failure=failure), ExitStack() as stack: + for name, value in [('seed_fixture', lambda *a, **k: copy.deepcopy(manifest)), + ('verify_seed', lambda *a: None), ('verify_parts', lambda *a: {}), + ('verify_public', lambda *a: None), ('idle_fleet', lambda *a: None), + ('Probe', lambda *a: peer)]: stack.enter_context(patch.object(options, name, value)) + stack.enter_context(patch.object(options, 'map_fixture', side_effect=[mapped, + {k: v for k, v in mapped.items() if k[1] != options.orphan.KEY}])) + task = stack.enter_context(patch.object(options, 'run_request', return_value=status)) + captured = stack.enter_context(patch.object(options.orphan, 'capture')) + armed = stack.enter_context(patch.object(options.orphan, 'arm_all')) + recovered = stack.enter_context(patch.object(options.orphan, 'recover_all')) + if failure: + with self.assertRaises((ValueError, RuntimeError)): + options.orphan_probe(fleet, probe, {'endpoints': ['peer']}, recorder, + self.bucket, 'H1', 'DV', request, []) + else: + result = options.orphan_probe(fleet, probe, {'endpoints': ['peer']}, recorder, + self.bucket, 'H1', 'DV', request, []) + self.assertTrue(result['physical']['wholeOrphanObjectRemoved']) + self.assertFalse(result['rescueIsProductRepair']) + captured.assert_called_once_with(fleet, original, self.bucket, 'old') + armed.assert_called_once_with(fleet, authority, self.bucket, 'authority') + recovered.assert_called_once_with(fleet) + self.assertEqual(fleet.a10_orphan_restored, failure != 'failed-authority-restore') + if failure in ('leftover-metadata', 'lost-install-ack'): peer.s3.assert_not_called() + if failure == 'lost-install-ack': task.assert_not_called() + else: self.assertEqual(task.call_args.args[2], '/rustfs/admin/v3/heal/' + self.bucket) + self.assertEqual(recorder.call.call_args.args, ('delete_object', { + 'Bucket': self.bucket, 'Key': options.orphan.KEY, 'VersionId': version})) + + def test_remove_false_uses_original_repairable_scope_and_only_source_remove_true_uses_orphan_runtime(self): + for branch, phase in (('remove-false', 'baseline'), ('remove-false', 'selected'), ('remove-true', 'selected')): + with tempfile.TemporaryDirectory() as directory, ExitStack() as stack: + root = Path(directory); candidate = root / 'candidate.json'; rc = root / 'rc'; rc.write_bytes(b'runner') + candidate.write_text(json.dumps({'source_sha': options.REFERENCE['candidateCommit'], 'package_sha256': 'a' * 64})) + args = types.SimpleNamespace(evidence=root, candidate=candidate, rc_bin=rc, binary_sha='b' * 64, + tool_directory=root, testing_commit='c' * 40) + fleet = types.SimpleNamespace(record=lambda label, value: value, deploy=lambda *a: None, cleanup=lambda: True) + factory = stack.enter_context(patch.object(options, 'Fleet', return_value=fleet)) + stack.enter_context(patch.object(options, 'verify_inspector', return_value={})) + stack.enter_context(patch.object(options, 'Probe', return_value=object())) + stack.enter_context(patch.object(options, 'FixtureRecorder', return_value=types.SimpleNamespace(pending=None))) + stack.enter_context(patch.dict(options.os.environ, {'GITHUB_RUN_ID': '12345', 'GITHUB_RUN_ATTEMPT': '1'})) + scope = stack.enter_context(patch.object(options, 'scope_probe', return_value={})) + orphan = stack.enter_context(patch.object(options, 'orphan_probe', return_value={})) + result = options.run_instance(args, 'H1', ['node'], branch, 'DB', phase) + self.assertEqual(result['verdict'], 'PASS') + self.assertEqual(factory.call_args.kwargs['runtime_profile'], + 'orphan-preserve' if branch == 'remove-true' else 'manual') + self.assertEqual(orphan.call_count, int(branch == 'remove-true')) + self.assertEqual(scope.call_count, int(branch == 'remove-false')) + def test_prefix_exact_and_recursive_scopes_do_not_include_adjacent_control(self): mapped = self.mapped() baseline, selected = options.paired_options('recursive-true') @@ -141,6 +324,32 @@ def test_complete_oracle_rejects_lost_versions_wrong_scopes_markers_and_false_co else: status['truncated'] = True with self.assertRaises(ValueError): options.validate_outcome(status, mapped, selected, repaired, request) + def test_source_cleanup_receipts_keep_exact_versions_controls_and_counter_partition(self): + for state in ('repaired', 'authoritatively_absent'): + status, mapped, selected, repaired, request = self.status() + request = {**request, 'remove': True}; status['settings'] = request + target = next(iter(repaired)) + receipt = next(r for r in status['outcome']['objects'] + if r['identity']['versionId'] == target[2] and r['identity']['object'] == target[1]) + receipt['disposition']['state'] = state + absent = {target} if state == 'authoritatively_absent' else set() + if absent: + status['outcome']['counters']['healed'] -= 1 + status['outcome']['counters']['unchanged'] += 1 + proof = options.validate_outcome(status, mapped, selected, repaired - absent, request, absent=absent) + self.assertEqual(proof['authoritativelyAbsentACKs'], len(absent)) + for kind in ('missing-target', 'wrong-version', 'unknown', 'false-counter', 'marker', 'wrong-options'): + bad = copy.deepcopy(status) + if kind == 'missing-target': bad['outcome']['objects'].remove(bad['outcome']['objects'][status['outcome']['objects'].index(receipt)]) + elif kind == 'wrong-version': bad['outcome']['objects'][status['outcome']['objects'].index(receipt)]['identity']['versionId'] = 'unacknowledged' + elif kind == 'unknown': bad['outcome']['objects'][status['outcome']['objects'].index(receipt)]['disposition']['state'] = 'unknown' + elif kind == 'false-counter': bad['outcome']['counters']['healed'] += 1 + elif kind == 'marker': + next(r for r in bad['outcome']['objects'] if r['identity']['versionId'] == 'marker')['disposition']['state'] = 'authoritatively_absent' + else: bad['settings'] = {**request, 'remove': False} + with self.subTest(state=state, kind=kind), self.assertRaises(ValueError): + options.validate_outcome(bad, mapped, selected, repaired - absent, request, absent=absent) + def test_nullable_unrestricted_scope_keeps_the_independent_physical_binding(self): status, mapped, selected, repaired, request = self.status('recursive-true') before = copy.deepcopy(status) @@ -270,8 +479,15 @@ def test_no_single_branch_checkpoint_or_duplicate_can_certify_canonical_completi rows = [{'topology': topology, 'branch': branch, 'fixture': fixture, 'phase': phase, 'verdict': 'PASS', 'cleanup': 'PASS', 'fullProbeCompleted': True} for topology in ('H1', 'H2') for branch in options.BRANCHES for fixture in options.FIXTURES - for phase in ('baseline', 'selected')] - self.assertEqual(len(rows), 72) + for phase in options.source_phases(branch)] + self.assertEqual(len(rows), 68) + self.assertEqual({r['branch'] for r in rows}, set(options.BRANCHES)) + for topology in ('H1', 'H2'): + for fixture in options.FIXTURES: + self.assertEqual({r['branch'] for r in rows if r['topology'] == topology + and r['fixture'] == fixture and r['phase'] == 'selected'}, set(options.BRANCHES)) + with self.assertRaises(ValueError): + options.complete_results([*rows, {**rows[0], 'branch': 'remove-true', 'phase': 'baseline'}]) self.assertTrue(options.complete_results(rows)) self.assertFalse(options.complete_results(rows[:-1])) for name, value in (('verdict', 'FAIL'), ('cleanup', 'FAIL'), ('fullProbeCompleted', False)): From 91d802c03f35d62309c5c63a29a1e548a69c4825 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 11:55:28 +0800 Subject: [PATCH 227/343] fix(validation): capture checkpoint advance with one held pidfd --- scripts/validation_heal/checkpoint.py | 79 ++++++---- scripts/validation_heal/fleet_node.py | 114 +++++++++++++- scripts/validation_heal/persistence.py | 2 +- test_validation_heal_checkpoint.py | 67 ++++++++ test_validation_heal_journal_fault.py | 208 ++++++++++++++++++++++++- 5 files changed, 433 insertions(+), 37 deletions(-) diff --git a/scripts/validation_heal/checkpoint.py b/scripts/validation_heal/checkpoint.py index 72ca095c..0f00e795 100644 --- a/scripts/validation_heal/checkpoint.py +++ b/scripts/validation_heal/checkpoint.py @@ -158,11 +158,13 @@ def admit(probe, fleet, active): return token -def capture_earlier(fleet, token, bucket, identifier, *, timeout=60): +def capture_earlier(fleet, token, bucket, identifier, *, timeout=60, advance=False): require(type(timeout) is int and 0 < timeout <= 120, "unreviewed checkpoint capture budget") + require(type(advance) is bool and (not advance or timeout == 60), "atomic checkpoint capture uses its frozen budget") owner, identity = freeze(fleet, token, bucket, timeout=timeout) node = owner["node"] - receipt = fleet.record("Admin-checkpoint-original-capture", fleet.remote(node, "journal-capture", + receipt = fleet.record("Admin-checkpoint-original-capture", fleet.remote(node, + "journal-capture-advance" if advance else "journal-capture", family="checkpoint", taskId=token, disk=owner["disk"], relativePath=owner["checkpoint"]["relativePath"], faultId=identifier, expectedBefore=journal.entry(bytes_of(owner["checkpoint"])), expected=identity)) require(receipt.get("processBound") is True and receipt.get("originalProductBytes") is True @@ -173,6 +175,14 @@ def capture_earlier(fleet, token, bucket, identifier, *, timeout=60): "binarySha256": identity.get("binarySha256"), "captured": journal.entry(bytes_of(owner["checkpoint"]))}.items()), "checkpoint capture lost its original process, bytes or exact physical owner") + if advance: + require(receipt.get("checkpointAdvancedAndFrozen") is True and receipt.get("pidfdBound") is True + and receipt.get("serviceFrozen") is True and receipt.get("processIdentity") == identity + and type(receipt.get("observedProcessed")) is int + and receipt["observedProcessed"] > owner["value"]["admin"]["outcome"]["counters"]["processed"], + "atomic checkpoint advance lacks its original frozen process and real progress") + return {**owner, "captureId": identifier, "captureReceipt": receipt, + "advancedAndFrozen": True, "processIdentity": identity} fleet.record("checkpoint-capture-resume", fleet.remote(node, "process-signal", signal="SIGCONT", expected=identity)) resumed = fleet.record("checkpoint-capture-resumed-process", fleet.remote(node, "process")) require(resumed.get("state") in {"R", "S", "D", "I"} @@ -197,35 +207,42 @@ def after_generation(current, earlier): def freeze(fleet, token, bucket, *, timeout=60, earlier=None): require(type(timeout) is int and 0 < timeout <= 120, "unreviewed checkpoint observation budget") - deadline = time.monotonic() + timeout - while True: - owner = None - for candidate in fleet.nodes: - try: - observed = records(fleet, token, "Admin-checkpoint-live", nodes=(candidate,)) - owner = owned_checkpoint(observed, token, bucket) - if earlier is not None: after_generation(owner, earlier) - except ValueError as exc: - if not str(exc).startswith("NOT_OBSERVED:"): raise - owner = None - continue - identity = observed[candidate].get("processIdentity") - require(owner["node"] == candidate and observed[candidate].get("processBound") is True - and observed[candidate].get("stable") is True and observed[candidate].get("independentReads") == 2 - and isinstance(identity, dict) and all(name in identity for name in PROCESS_FIELDS) - and type(identity.get("pid")) is int and identity["pid"] > 1, - "checkpoint owner has no census-bound process birth identity") - break - if owner is not None: break - require(time.monotonic() < deadline, "NOT_OBSERVED: actual durable Admin checkpoint window exceeded its frozen budget") - time.sleep(.1) - node = owner["node"] - # Freeze the witnessed owner before another SSH read can outlast its task. - # The complete census below still rejects any second physical owner. - fleet.record("checkpoint-owner-process", identity) - require(node not in fleet.frozen_processes, "checkpoint owner is already tracked as frozen") - fleet.frozen_processes[node] = identity - fleet.record("checkpoint-owner-freeze", fleet.remote(node, "process-signal", signal="SIGSTOP", expected=identity)) + if earlier is not None and earlier.get("advancedAndFrozen") is True: + owner, node = earlier, earlier["node"] + identity = fleet.frozen_processes.get(node) + require(node in fleet.nodes and isinstance(identity, dict) and identity == earlier.get("processIdentity") + and all(name in identity for name in PROCESS_FIELDS) and type(identity.get("pid")) is int + and identity["pid"] > 1, "atomic checkpoint advance lost its registered original frozen birth") + else: + deadline = time.monotonic() + timeout + while True: + owner = None + for candidate in fleet.nodes: + try: + observed = records(fleet, token, "Admin-checkpoint-live", nodes=(candidate,)) + owner = owned_checkpoint(observed, token, bucket) + if earlier is not None: after_generation(owner, earlier) + except ValueError as exc: + if not str(exc).startswith("NOT_OBSERVED:"): raise + owner = None + continue + identity = observed[candidate].get("processIdentity") + require(owner["node"] == candidate and observed[candidate].get("processBound") is True + and observed[candidate].get("stable") is True and observed[candidate].get("independentReads") == 2 + and isinstance(identity, dict) and all(name in identity for name in PROCESS_FIELDS) + and type(identity.get("pid")) is int and identity["pid"] > 1, + "checkpoint owner has no census-bound process birth identity") + break + if owner is not None: break + require(time.monotonic() < deadline, "NOT_OBSERVED: actual durable Admin checkpoint window exceeded its frozen budget") + time.sleep(.1) + node = owner["node"] + # Freeze the witnessed owner before another SSH read can outlast its task. + # The complete census below still rejects any second physical owner. + fleet.record("checkpoint-owner-process", identity) + require(node not in fleet.frozen_processes, "checkpoint owner is already tracked as frozen") + fleet.frozen_processes[node] = identity + fleet.record("checkpoint-owner-freeze", fleet.remote(node, "process-signal", signal="SIGSTOP", expected=identity)) frozen = fleet.record("checkpoint-owner-frozen-process", fleet.remote(node, "process")) require(frozen.get("state") == "T" and all(frozen.get(name) == identity.get(name) for name in PROCESS_FIELDS), "checkpoint freeze missed the original process") diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index a8f9acbf..79c4ac0c 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1193,6 +1193,104 @@ def configure_encryption(request): return encryption_module().configure(state, owner, request, fault_module()) +def advance_captured_checkpoint(request, roots, owner, module, fault, before, capture): + require(owner.get("runtimeProfile") == "root-restart" and before.get("state") == "T" + and request["family"] == "checkpoint" and len(roots) == 4, + "checkpoint advance requires its paused owned root-restart service") + require(hasattr(os, "pidfd_open") and hasattr(signal, "pidfd_send_signal"), + "checkpoint advance requires Linux pidfd support") + descriptor = os.pidfd_open(before["pid"], 0) + resumed = False + try: + bound = process(request) + require(bound.get("state") == "T" and all(bound.get(name) == before[name] for name in PROCESS_FIELDS), + "checkpoint advance lost its original paused process birth") + with fault.directory(roots[request["disk"]]) as root_fd: + require(os.fstat(root_fd).st_dev == capture.get("device") + and fault.mount_id(root_fd) == capture.get("mountId"), + "checkpoint capture root device or mount changed before advance") + path = module.relative(request) + earlier = module.read(root_fd, path, fault) + require(module.same_json(module.entry(earlier), capture["captured"]), + "captured checkpoint changed before resuming its original process") + old = module.checkpoint_value(earlier, request["taskId"], owner["bucket"]) + old_count = old["admin"]["outcome"]["counters"]["processed"] + require(type(old_count) is int and 0 < old_count < 2**64, + "earlier checkpoint has no exact positive progress") + deadline = time.monotonic() + 60 + # Register the resume intent before the syscall: interruption or a + # lost local receipt must still refreeze this exact held pidfd. + resumed = True + signal.pidfd_send_signal(descriptor, signal.SIGCONT, None, 0) + while True: + require(time.monotonic() < deadline, + "NOT_OBSERVED: actual checkpoint advance exceeded its frozen 60-second budget") + # The held pidfd binds signals. Cheap procfs birth checks avoid + # hashing the full candidate between progress and SIGSTOP. + fields = Path(f"/proc/{before['pid']}/stat").read_text().rpartition(") ")[2].split() + require(len(fields) >= 20 and fields[19].isdigit() and int(fields[19]) == before["startTicks"], + "original checkpoint process exited or changed birth during advance") + try: + raw = module.read(root_fd, path, fault) + except FileNotFoundError: + raw = None + except ValueError as exc: + if str(exc) != "file changed while reading": raise + raw = None + if raw is not None: + value = module.checkpoint_value(raw, request["taskId"], owner["bucket"]) + require(all(module.same_json(value["admin"].get(name), old["admin"].get(name)) + for name in ("buckets", "options", "set_disk_id")), + "checkpoint advance changed original incarnation, options or set") + count = value["admin"].get("outcome", {}).get("counters", {}).get("processed") + require(type(count) is int and old_count <= count < 2**64, + "checkpoint advance regressed or lost its exact progress") + if count > old_count: + require(time.monotonic() < deadline, + "NOT_OBSERVED: actual checkpoint advance exceeded its frozen 60-second budget") + signal.pidfd_send_signal(descriptor, signal.SIGSTOP, None, 0) + resumed = False + break + time.sleep(.01) + # Signal delivery can precede the actual group stop. Await T on + # the same birth within the original observation deadline. + while True: + require(time.monotonic() < deadline, + "NOT_OBSERVED: original process did not stop within its frozen advance budget") + fields = Path(f"/proc/{before['pid']}/stat").read_text().rpartition(") ")[2].split() + require(len(fields) >= 20 and fields[19].isdigit() and int(fields[19]) == before["startTicks"], + "checkpoint process changed birth before its actual group stop") + if fields[0] == "T": break + time.sleep(.001) + after = process(request) + require(after.get("state") == "T" and all(after.get(name) == before[name] for name in PROCESS_FIELDS), + "advanced checkpoint did not freeze the original candidate process") + observed = module.stable_census(roots, request, fault) + selected = [row for row in observed["records"] + if (row["disk"], row["relativePath"]) == (request["disk"], request["relativePath"])] + require(len(selected) == 1 and selected[0].get("present") is True, + "NOT_OBSERVED: advanced original checkpoint disappeared before freeze") + row = selected[0] + current = base64.b64decode(row["rawBase64"], validate=True) + require(module.same_json(module.entry(current), {name: row.get(name) for name in ("size", "sha256")}), + "advanced checkpoint census bytes differ") + # This validates the original digest, all counters, time and cursor, + # without installing or otherwise modifying any product record. + module.transform(current, {**request, "kind": "stale-generation"}, owner, None, earlier=earlier) + final_count = module.checkpoint_value(current, request["taskId"], owner["bucket"])["admin"]["outcome"]["counters"]["processed"] + require(final_count >= count, "checkpoint progress regressed between observation and actual freeze") + return {**capture, "observation": observed, "processIdentity": {name: after[name] for name in PROCESS_FIELDS}, + "processBound": True, "serviceFrozen": True, "checkpointAdvancedAndFrozen": True, + "pidfdBound": True, "observedProcessed": count, "naturalProductProgressObserved": True} + finally: + try: + if resumed: + try: signal.pidfd_send_signal(descriptor, signal.SIGSTOP, None, 0) + except ProcessLookupError: pass + finally: + os.close(descriptor) + + def journal_operation(request): state, roots, owner = owned(request["runId"]) require(digest(state / "rustfs") == owner["binarySha256"], "durable-record candidate changed") @@ -1205,9 +1303,9 @@ def journal_operation(request): "bounded journal runtime mutates only the owned Admin checkpoint family") roots = volume_views(state, roots, owner) module, fault = globals()["journal_fault_backend"], fault_module() - if request["action"] in {"journal-observe", "journal-capture"}: + if request["action"] in {"journal-observe", "journal-capture", "journal-capture-advance"}: fields = {"runId", "action", "family", "taskId", "expected"} - if request["action"] == "journal-capture": + if request["action"] in {"journal-capture", "journal-capture-advance"}: fields |= {"disk", "relativePath", "faultId", "expectedBefore"} require(set(request) == fields, "live durable-record observation/capture has unknown mutation or path inputs") before = process(request) @@ -1215,8 +1313,14 @@ def journal_operation(request): require(isinstance(expected, dict) and before.get("pid", 0) > 1 and all(name in expected and before.get(name) == expected[name] for name in PROCESS_FIELDS), "live durable-record observation lost its exact service birth or candidate") + if request["action"] == "journal-capture-advance": + require(owner.get("runtimeProfile") == "root-restart" and before.get("state") == "T" + and request["family"] == "checkpoint" and len(roots) == 4, + "checkpoint advance requires its paused owned root-restart service") + require(hasattr(os, "pidfd_open") and hasattr(signal, "pidfd_send_signal"), + "checkpoint advance requires Linux pidfd support") observed = module.stable_census(roots, request, fault) - if request["action"] == "journal-capture": + if request["action"] in {"journal-capture", "journal-capture-advance"}: require(request["family"] == "checkpoint" and type(request["disk"]) is int and 0 <= request["disk"] < 4, "earlier checkpoint capture has no exact owned Admin record/disk") selected = [row for row in observed["records"] @@ -1226,6 +1330,8 @@ def journal_operation(request): "NOT_OBSERVED: earlier checkpoint differs from its two stable process-bound reads") with fault.directory(roots[request["disk"]]) as fd: observed = {**module.capture(fd, request, owner, fault), "observation": observed} + if request["action"] == "journal-capture-advance": + return advance_captured_checkpoint(request, roots, owner, module, fault, before, observed) after = process(request) require(all(after.get(name) == before.get(name) for name in PROCESS_FIELDS), "service restarted while observing exact durable records") @@ -1646,7 +1752,7 @@ def main(request): actions["journal-rejection-audit"] = journal_rejection_audit_finalize actions["journal-retained-audit"] = journal_retained_audit_finalize actions["journal-retained-audit-finalize"] = journal_retained_audit_finalize - for action in ("journal-census", "journal-observe", "journal-capture", "journal-apply", "journal-recover"): + for action in ("journal-census", "journal-observe", "journal-capture", "journal-capture-advance", "journal-apply", "journal-recover"): actions[action] = journal_operation for action in ('generation-capture', 'generation-install', 'generation-recover'): actions[action] = generation_operation diff --git a/scripts/validation_heal/persistence.py b/scripts/validation_heal/persistence.py index 767d241e..3472e8b7 100644 --- a/scripts/validation_heal/persistence.py +++ b/scripts/validation_heal/persistence.py @@ -546,7 +546,7 @@ def allocate(): pending = {"operation": "admit-original-checkpoint-task"} token = checkpoints.admit(probe, fleet, active) pending = None - earlier = checkpoints.capture_earlier(fleet, token, bucket, "F07-earlier-generation") if branch == "stale-generation" else None + earlier = checkpoints.capture_earlier(fleet, token, bucket, "F07-earlier-generation", advance=True) if branch == "stale-generation" else None if earlier is not None: require(earlier["value"]["admin"]["outcome"]["counters"]["healed"] == 0, "NOT_OBSERVED: earlier generation missed the pre-repair capture window") diff --git a/test_validation_heal_checkpoint.py b/test_validation_heal_checkpoint.py index c143afd8..0f8d8405 100644 --- a/test_validation_heal_checkpoint.py +++ b/test_validation_heal_checkpoint.py @@ -140,6 +140,73 @@ def test_earlier_capture_requires_original_bytes_bound_to_process_and_physical_o self.assertEqual(fleet.frozen_processes, {"node": identity}) receipt[key] = original + def test_atomic_earlier_capture_keeps_the_registered_birth_frozen_without_an_ssh_resume_gap(self): + observed, _, _ = self.fixture(); owner = reader.owned_checkpoint(observed, TOKEN, BUCKET) + from scripts.validation_heal.fleet_node import PROCESS_FIELDS + identity = {name: name for name in PROCESS_FIELDS}; identity.update(pid=50) + receipt = {'processBound': True, 'originalProductBytes': True, 'productMetadataMutated': False, + 'runId': 'gh-12345-1-h1', 'family': 'checkpoint', 'taskId': TOKEN, 'disk': owner['disk'], + 'relativePath': owner['checkpoint']['relativePath'], 'faultId': 'F07-early', + 'binarySha256': identity['binarySha256'], 'captured': journal.entry(reader.bytes_of(owner['checkpoint'])), + 'checkpointAdvancedAndFrozen': True, 'pidfdBound': True, 'serviceFrozen': True, + 'observedProcessed': 3, 'processIdentity': identity} + calls = [] + fleet = SimpleNamespace(run_id=receipt['runId'], frozen_processes={'node': identity}, + record=lambda label, value: value, remote=lambda node, action, **params: calls.append(action) or receipt) + with patch.object(reader, 'freeze', return_value=(owner, identity)): + earlier = reader.capture_earlier(fleet, TOKEN, BUCKET, 'F07-early', advance=True) + self.assertEqual(calls, ['journal-capture-advance']) + self.assertTrue(earlier['advancedAndFrozen']) + self.assertEqual(earlier['processIdentity'], identity) + self.assertEqual(fleet.frozen_processes, {'node': identity}) + for key, bad in [('checkpointAdvancedAndFrozen', False), ('pidfdBound', False), ('serviceFrozen', False), + ('observedProcessed', True), ('observedProcessed', 2), ('processIdentity', {})]: + original = receipt[key]; receipt[key] = bad + with self.subTest(key=key, bad=bad), patch.object(reader, 'freeze', return_value=(owner, identity)), \ + self.assertRaises(ValueError): + reader.capture_earlier(fleet, TOKEN, BUCKET, 'F07-early', advance=True) + receipt[key] = original + self.assertEqual(fleet.frozen_processes, {'node': identity}) + + def test_atomic_frozen_generation_still_requires_full_fleet_checkpoint_and_root_censuses(self): + observed, value, _ = self.fixture(); roots, _ = self.root_fixture() + earlier = reader.owned_checkpoint(observed, TOKEN, BUCKET) + from scripts.validation_heal.fleet_node import PROCESS_FIELDS + identity = {name: name for name in PROCESS_FIELDS}; identity.update(pid=50) + earlier.update(advancedAndFrozen=True, processIdentity=identity) + value['checkpoint_time'] = 101; value['current_object_index'] = 3 + value['processed_objects'].append('objects/third'); value['successful_objects'] = 3 + value['admin']['outcome']['counters'].update(processed=3, unchanged=2) + observed['node']['records'][4].update(encoded(value, signed=True)) + for tree in (observed, roots): + for peer in ('peer1', 'peer2', 'peer3'): + tree[peer] = copy.deepcopy(tree['node']) + tree[peer]['records'] = [{'disk': r['disk'], 'relativePath': r['relativePath'], 'present': False} + for r in tree[peer]['records']] + for change in (None, 'unregistered', 'changed-birth', 'not-frozen', 'second-owner'): + fleet = SimpleNamespace(nodes=['node', 'peer1', 'peer2', 'peer3'], + frozen_processes={} if change == 'unregistered' else {'node': identity}, + record=lambda label, value: value) + if change == 'changed-birth': fleet.frozen_processes['node'] = {**identity, 'pid': 51} + calls = [] + fleet.remote = lambda node, action, **params: calls.append(action) or { + **identity, 'state': 'S' if change == 'not-frozen' else 'T'} + current = copy.deepcopy(observed) + if change == 'second-owner': current['peer1'] = copy.deepcopy(current['node']) + def census(fleet, token, label, **params): + self.assertNotIn('nodes', params) + self.assertIn(label, ('Admin-checkpoint-frozen', 'Admin-root-frozen')) + return roots if params.get('family') == 'root' else current + with self.subTest(change=change), patch.object(reader, 'records', side_effect=census) as records: + if change: + with self.assertRaises(ValueError): reader.freeze(fleet, TOKEN, BUCKET, earlier=earlier) + else: + frozen, birth = reader.freeze(fleet, TOKEN, BUCKET, earlier=earlier) + self.assertEqual(birth, identity) + self.assertEqual(frozen['value']['admin']['outcome']['counters']['processed'], 3) + self.assertEqual(records.call_count, 2) + self.assertNotIn('process-signal', calls) + def test_stale_boundary_requires_actual_newer_progress_in_the_same_original_responsibility(self): observed, value, _ = self.fixture() earlier = reader.owned_checkpoint(observed, TOKEN, BUCKET) diff --git a/test_validation_heal_journal_fault.py b/test_validation_heal_journal_fault.py index 95c4365b..a9a74c1a 100644 --- a/test_validation_heal_journal_fault.py +++ b/test_validation_heal_journal_fault.py @@ -9,6 +9,9 @@ import os from pathlib import Path import struct +import subprocess +import sys +import time import tempfile import types import unittest @@ -37,7 +40,7 @@ def test_live_read_race_is_retryable_but_stopped_read_and_path_failures_are_not( def test_only_live_observation_refusals_survive_the_remote_boundary_as_retryable(self): refusal = 'NOT_OBSERVED: exact durable records changed between the two bounded reads' - for action in ('journal-observe', 'journal-capture', 'journal-apply', 'journal-census'): + for action in ('journal-observe', 'journal-capture', 'journal-capture-advance', 'journal-apply', 'journal-census'): output = io.StringIO() with self.subTest(action=action), patch.object(fleet_node.os, 'geteuid', return_value=0), \ patch.object(fleet_node, 'journal_operation', side_effect=ValueError(refusal)), contextlib.redirect_stdout(output): @@ -63,6 +66,209 @@ def test_only_live_observation_refusals_survive_the_remote_boundary_as_retryable with patch.object(healthy.subprocess, 'run', return_value=response): with self.assertRaisesRegex(ValueError, '^Heal node'): fleet.remote('node', 'journal-observe') + def test_atomic_advance_uses_one_held_pidfd_and_refreezes_on_observation_failure(self): + for change in (None, 'timeout', 'birth', 'corrupt', 'incarnation', 'options', 'counter', 'read-error', 'unstable', 'post-birth', 'late-progress', 'continue-interrupted', 'stop-pending', 'stop-timeout', 'stop-birth'): + with self.subTest(change=change), tempfile.TemporaryDirectory() as temp, contextlib.ExitStack() as stack: + base = Path(temp).resolve(); roots = tuple(base / str(i) for i in range(4)) + for root in roots: root.mkdir() + owner, request, path, raw = self.fixture(roots[0]); request.pop('kind') + for root in roots[1:]: + (root / 'data/.rustfs.sys/buckets').mkdir(parents=True); (root / 'quarantine').mkdir() + request['action'] = 'journal-capture-advance' + identity = {name: name for name in fleet_node.PROCESS_FIELDS} + identity.update(pid=50, startTicks=20, binarySha256=owner['binarySha256']) + request['expected'] = identity + next_raw = self.checkpoint(processed=3, time=101) + value = json.loads(next_raw) + if change == 'incarnation': value['admin']['buckets'][self.bucket] = str(uuid.uuid4()) + elif change == 'options': value['admin']['options']['nolock'] = True + elif change == 'counter': value['admin']['outcome']['counters']['processed'] = True + if change in ('incarnation', 'options', 'counter'): next_raw = self.signed(value) + if change == 'corrupt': next_raw = b'invalid checkpoint' + events = []; running = [False]; raced = [False] + def signal_pidfd(fd, signum, *args): + self.assertEqual(fd, 55); events.append(signum) + running[0] = signum == fleet_node.signal.SIGCONT + if running[0] and change != 'timeout': path.write_bytes(next_raw) + if running[0] and change == 'continue-interrupted': raise KeyboardInterrupt('lost CONT receipt') + def process(request): + if len(process_calls) >= 2: self.assertEqual(events[-1], fleet_node.signal.SIGSTOP) + process_calls.append(True) + return {**identity, 'state': 'S' if change == 'stop-pending' and len(process_calls) == 3 + and not stop_ready[0] else 'T', + **({'startTicks': 21} if change == 'post-birth' and len(process_calls) == 3 else {})} + process_calls = []; stop_ready = [False]; stopped_reads = [0] + old_read_text = Path.read_text + def read_text(p, *a, **kw): + if str(p) == '/proc/50/stat': + state = 'S' if running[0] else 'T' + if not running[0] and events: + stopped_reads[0] += 1 + if change == 'stop-timeout' or change == 'stop-pending' and stopped_reads[0] == 1: state = 'S' + stop_ready[0] = state == 'T' + changed_birth = change == 'birth' or change == 'stop-birth' and not running[0] and events + return '50 (owned candidate) ' + state + ' ' + ' '.join(['0'] * 18 + ['21' if changed_birth else '20']) + return old_read_text(p, *a, **kw) + real_read = journal.read + def read(*args, **kwargs): + if running[0] and not raced[0] and change in ('read-error', 'unstable'): + raced[0] = True + if change == 'read-error': raise PermissionError('unexpected access denial') + raise ValueError('file changed while reading') + return real_read(*args, **kwargs) + for context in (patch.object(fleet_node, 'owned', return_value=(base, roots, owner)), + patch.object(fleet_node, 'digest', return_value=owner['binarySha256']), + patch.object(fleet_node, 'fault_module', return_value=fault), + patch.dict(fleet_node.__dict__, journal_fault_backend=journal), + patch.object(fleet_node, 'process', side_effect=process)): + stack.enter_context(context) + opened = stack.enter_context(patch.object(fleet_node.os, 'pidfd_open', create=True, return_value=55)) + sent = stack.enter_context(patch.object(fleet_node.signal, 'pidfd_send_signal', create=True, side_effect=signal_pidfd)) + real_close = os.close + closed = [] + def close(fd): + if fd == 55: closed.append(fd) + else: real_close(fd) + stack.enter_context(patch.object(fleet_node.os, 'close', side_effect=close)) + stack.enter_context(patch.object(fault, 'mount_id', return_value=1)) + stack.enter_context(patch.object(Path, 'read_text', read_text)) + stack.enter_context(patch.object(journal, 'read', side_effect=read)) + if change == 'timeout': stack.enter_context(patch.object(fleet_node.time, 'monotonic', side_effect=[0, 61])) + elif change == 'late-progress': stack.enter_context(patch.object(fleet_node.time, 'monotonic', side_effect=[0, .01, 61])) + elif change == 'stop-timeout': stack.enter_context(patch.object(fleet_node.time, 'monotonic', side_effect=[0, .01, .02, 61])) + if change in (None, 'unstable', 'stop-pending'): + result = fleet_node.journal_operation(request) + self.assertTrue(result['checkpointAdvancedAndFrozen']) + self.assertTrue(result['pidfdBound']) + self.assertEqual(result['observedProcessed'], 3) + self.assertEqual(path.read_bytes(), next_raw) + self.assertEqual(len(process_calls), 3) + else: + with self.assertRaises((ValueError, OSError, KeyboardInterrupt)): fleet_node.journal_operation(request) + self.assertEqual(events, [fleet_node.signal.SIGCONT, fleet_node.signal.SIGSTOP]) + self.assertEqual(closed, [55]) + opened.assert_called_once_with(50, 0) + self.assertEqual((roots[0] / ('quarantine/' + request['faultId'] + '.early.checkpoint')).read_bytes(), raw) + + def test_atomic_advance_rejects_unpaused_foreign_or_unbounded_requests_before_capture_or_signal(self): + for change in ('runtime', 'live', 'birth', 'input', 'budget', 'path', 'disk', 'family'): + with self.subTest(change=change), tempfile.TemporaryDirectory() as temp, contextlib.ExitStack() as stack: + root = Path(temp).resolve(); owner, request, path, raw = self.fixture(root); request.pop('kind') + request['action'] = 'journal-capture-advance' + identity = {name: name for name in fleet_node.PROCESS_FIELDS} + identity.update(pid=50, startTicks=20, binarySha256=owner['binarySha256']); request['expected'] = identity.copy() + if change == 'runtime': owner['runtimeProfile'] = 'mrf-crash' + elif change == 'birth': request['expected']['startTicks'] = 21 + elif change == 'input': request['command'] = 'arbitrary' + elif change == 'budget': request['timeout'] = 600 + elif change == 'path': request['relativePath'] = '.rustfs.sys/foreign.json' + elif change == 'disk': request['disk'] = True + elif change == 'family': request['family'] = 'mrf' + for context in (patch.object(fleet_node, 'owned', return_value=(root, [root] * 4, owner)), + patch.object(fleet_node, 'digest', return_value=owner['binarySha256']), + patch.object(fleet_node, 'fault_module', return_value=fault), + patch.dict(fleet_node.__dict__, journal_fault_backend=journal), + patch.object(fleet_node, 'process', return_value={**identity, 'state': 'S' if change == 'live' else 'T'}), + patch.object(fleet_node.os, 'pidfd_open', create=True, return_value=55), + patch.object(fleet_node.signal, 'pidfd_send_signal', create=True)): + stack.enter_context(context) + captured = stack.enter_context(patch.object(journal, 'capture')) + with self.assertRaises(ValueError): fleet_node.journal_operation(request) + captured.assert_not_called() + self.assertEqual(path.read_bytes(), raw) + self.assertEqual(list((root / 'quarantine').iterdir()), []) + + def test_advance_rejects_reopened_capture_device_or_mount_before_resuming(self): + for key in ('device', 'mountId'): + with self.subTest(key=key), tempfile.TemporaryDirectory() as temp, contextlib.ExitStack() as stack: + root = Path(temp).resolve(); owner, request, path, raw = self.fixture(root); request.pop('kind') + request['action'] = 'journal-capture-advance' + identity = {name: name for name in fleet_node.PROCESS_FIELDS} + identity.update(pid=50, startTicks=20, binarySha256=owner['binarySha256']); request['expected'] = identity + real_capture = journal.capture + def capture(*args): return {**real_capture(*args), key: -1} + for context in (patch.object(fleet_node, 'owned', return_value=(root, [root] * 4, owner)), + patch.object(fleet_node, 'digest', return_value=owner['binarySha256']), + patch.object(fleet_node, 'fault_module', return_value=fault), + patch.dict(fleet_node.__dict__, journal_fault_backend=journal), + patch.object(fleet_node, 'process', return_value={**identity, 'state': 'T'}), + patch.object(fleet_node.os, 'pidfd_open', create=True, return_value=55), + patch.object(fault, 'mount_id', return_value=1), patch.object(journal, 'capture', side_effect=capture)): + stack.enter_context(context) + signals = stack.enter_context(patch.object(fleet_node.signal, 'pidfd_send_signal', create=True)) + real_close = os.close; closed = [] + def close(fd): + if fd == 55: closed.append(fd) + else: real_close(fd) + stack.enter_context(patch.object(fleet_node.os, 'close', side_effect=close)) + with self.assertRaisesRegex(ValueError, 'root device or mount changed'): fleet_node.journal_operation(request) + signals.assert_not_called(); self.assertEqual(closed, [55]); self.assertEqual(path.read_bytes(), raw) + + @unittest.skipUnless(sys.platform == 'linux' and hasattr(os, 'pidfd_open') + and hasattr(fleet_node.signal, 'pidfd_send_signal'), 'actual Linux pidfd required') + def test_actual_linux_pidfd_stops_the_same_child_after_real_checkpoint_file_progress(self): + with tempfile.TemporaryDirectory() as temp: + base = Path(temp).resolve(); roots = tuple(base / str(i) for i in range(4)) + for root in roots: root.mkdir() + owner, request, path, early = self.fixture(roots[0]); request.pop('kind') + request['action'] = 'journal-capture-advance' + for root in roots[1:]: (root / 'data/.rustfs.sys/buckets').mkdir(parents=True) + resume_path = roots[0] / 'data' / journal.names('checkpoint', self.token)[1] + resume = json.dumps({'schema_version': 7, 'task_id': self.token, 'task_type': 'admin_erasure_set', + 'set_disk_id': 'pool_0_set_0', 'completed': False, 'replacement_targets': [], 'replacement_generation': None}).encode() + resume_path.write_bytes(resume) + later = self.checkpoint(processed=3, time=101); next_bytes = base / 'next'; next_bytes.write_bytes(later) + child_source = ("import os,signal,sys,time; os.kill(os.getpid(),signal.SIGSTOP); " + "raw=open(sys.argv[2],'rb').read(); f=open(sys.argv[1]+'.next','xb'); " + "f.write(raw); f.flush(); os.fsync(f.fileno()); f.close(); os.replace(sys.argv[1]+'.next',sys.argv[1]); " + "time.sleep(120)") + child = subprocess.Popen([sys.executable, '-c', child_source, str(path), str(next_bytes)]) + cleanup_fd = None + real_send = fleet_node.signal.pidfd_send_signal + def stat_fields(): return Path(f'/proc/{child.pid}/stat').read_text().rpartition(') ')[2].split() + try: + cleanup_fd = os.pidfd_open(child.pid, 0) + deadline = time.monotonic() + 5 + while stat_fields()[0] != 'T': + self.assertIsNone(child.poll()); self.assertLess(time.monotonic(), deadline); time.sleep(.01) + birth = int(stat_fields()[19]) + identity = {name: name for name in fleet_node.PROCESS_FIELDS} + identity.update(pid=child.pid, startTicks=birth, binarySha256=owner['binarySha256']) + request['expected'] = identity + snapshots, signals = [], [] + def process(request): + fields = stat_fields(); self.assertEqual(int(fields[19]), birth) + if len(snapshots) >= 2: self.assertEqual(signals[-1], fleet_node.signal.SIGSTOP) + snapshots.append(fields[0]); return {**identity, 'state': fields[0]} + def send(fd, signum, *args): + signals.append(signum); return real_send(fd, signum, *args) + with patch.object(fleet_node, 'owned', return_value=(base, roots, owner)), \ + patch.object(fleet_node, 'digest', return_value=owner['binarySha256']), \ + patch.object(fleet_node, 'fault_module', return_value=fault), \ + patch.dict(fleet_node.__dict__, journal_fault_backend=journal), \ + patch.object(fleet_node, 'process', side_effect=process), \ + patch.object(fleet_node.signal, 'pidfd_send_signal', side_effect=send): + result = fleet_node.journal_operation(request) + self.assertEqual(signals, [fleet_node.signal.SIGCONT, fleet_node.signal.SIGSTOP]) + self.assertEqual(snapshots, ['T', 'T', 'T']) + self.assertEqual(result['observedProcessed'], 3) + self.assertEqual(stat_fields()[0], 'T') + self.assertEqual(path.read_bytes(), later) + self.assertEqual(resume_path.read_bytes(), resume) + self.assertEqual((roots[0] / ('quarantine/' + request['faultId'] + '.early.checkpoint')).read_bytes(), early) + print('ATOMIC_PIDFD_COMPONENT ' + json.dumps({'simulatedCheckpoint': True, 'actualLinuxPidfd': True, + 'sameProcessBirth': True, 'checkpointCounts': [2, 3], 'earlierBackupByteIdentical': True, + 'laterFileByteIdentical': True, 'companionUnchanged': True, 'childStoppedBeforeReturn': True, + 'rustfsCandidateExecuted': False, 'canonicalPassClaimed': False})) + finally: + try: + if cleanup_fd is not None: real_send(cleanup_fd, fleet_node.signal.SIGKILL, None, 0) + elif child.poll() is None: child.kill() + except ProcessLookupError: pass + finally: + if cleanup_fd is not None: os.close(cleanup_fd) + child.wait(timeout=5) + def test_full_server_checkpoint_fixture_preserves_feature_unified_json_order(self): # Original bytes from candidate 1aa9be87, diagnostic 38261190390/1. raw = (Path(__file__).parent / "testdata/validation/checkpoint-preserve-order.json").read_bytes() From 2e7f2a37fbbb892811060f279f08e63caadb96f3 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 14:08:18 +0800 Subject: [PATCH 228/343] docs(validation): record complete A10 live acceptance --- docs/validation-heal-cases.md | 7 +++++++ docs/validation-migration-map.md | 12 +++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index 9ff27c47..dd48e947 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -222,6 +222,13 @@ instances. Original failures remain archived. Physical closure requires the whole orphan to disappear, every post-DELETE control/format to remain exact, and complete version-addressed task receipts/counters before reads or rescue. +Complete live acceptance: [run 39197088183/1](https://github.com/rustfs/rustfs/actions/runs/39197088183) +on helper `b16e6072679d6a60a044b9d4520b30b4a79b5557` passed all 68 instances +and independent ACK/task/physical/public/cleanup audit, including 170 node +finalizations. Candidate `072b57b37559fbe626aeda8abf61617fdcbdd926`, package +`38043283428/1` and inspector `38715612825/1` remain immutable. Earlier failures +are retained; control rescue is not counted as product repair. + Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [SEED_DV](validation-heal-operations.md#SEED_DV) → [FAULT](validation-heal-operations.md#FAULT) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [VERIFY](validation-heal-operations.md#VERIFY) Source assertions: diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index a4de2e01..01c7fedb 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -9,19 +9,21 @@ live run proves each source assertion on the required topology. ## Current acceptance and remaining work -All 158 canonical IDs are retained. Independent complete live acceptance is **102/158**: the prior 101-case census plus B07 `heal.below-read-quorum`, accepted from [run 38727279347/1](https://github.com/rustfs/rustfs/actions/runs/38727279347). Its complete H1/H2 never-versioned/historical matrix, exact candidate/package/tools and cleanup were independently audited; earlier B07 failures remain retained. +All 158 canonical IDs are retained. Independent complete live acceptance is **103/158**: the prior 101-case census plus B07 `heal.below-read-quorum` ([run 38727279347/1](https://github.com/rustfs/rustfs/actions/runs/38727279347)) and A10 `heal.admin-option-contract` ([run 39197088183/1](https://github.com/rustfs/rustfs/actions/runs/39197088183)). Both complete source matrices, immutable candidate/package/tools and cleanup were independently audited; earlier failures remain retained. | Acceptance state | Cases | Remaining requirement | | --- | ---: | --- | -| Independently accepted complete live PASS | 102 | Preserve original provenance and complete results. | -| Retained live failures with existing executors | 32 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | +| Independently accepted complete live PASS | 103 | Preserve original provenance and complete results. | +| Retained live failures with existing executors | 31 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | | Required native barriers | 7 | A09 and G01–G06 need exact candidate-bound server lock/version/commit barriers. | | Orchestration or evidence blocked | 4 | C11, F07, I05 and J02 need their remaining native or full live evidence closure. | | Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | -The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 43 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. +The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 42 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. -[J02 diagnostic 38874900846/1](https://github.com/rustfs/rustfs/actions/runs/38874900846) completed all 20 H1/H2 × DB/DV branch instances on that exact candidate. Sixteen branches report PASS; the four original all-metadata-corruption probes report permanent Failed rather than Unknown and retain NOT_OBSERVED. Independent archive and provenance audits passed. Independent mixed-branch audits verified all four repaired shards against original complete bytes, exact receipts and counters, surviving below-quorum failures, original terminal preservation and post-rescue S3 inventories. All 50 node namespace finalization receipts passed audit. Real legacy writes separately produce Unknown with zero repairs and unchanged physical files, but do not replace the original source procedure. Full J02 canonical acceptance remains blocked; these results do not increase 102/158. +[J02 diagnostic 38874900846/1](https://github.com/rustfs/rustfs/actions/runs/38874900846) completed all 20 H1/H2 × DB/DV branch instances on that exact candidate. Sixteen branches report PASS; the four original all-metadata-corruption probes report permanent Failed rather than Unknown and retain NOT_OBSERVED. Independent archive and provenance audits passed. Independent mixed-branch audits verified all four repaired shards against original complete bytes, exact receipts and counters, surviving below-quorum failures, original terminal preservation and post-rescue S3 inventories. All 50 node namespace finalization receipts passed audit. Real legacy writes separately produce Unknown with zero repairs and unchanged physical files, but do not replace the original source procedure. Full J02 canonical acceptance remains blocked; these partial results add no canonical acceptance. + +A10 run `39197088183/1` uses immutable helper `b16e6072679d6a60a044b9d4520b30b4a79b5557`, candidate `072b57b37559fbe626aeda8abf61617fdcbdd926`, package `38043283428/1` and inspector `38715612825/1`. All nine source variants across H1/H2 and DB/DV passed their 68 required execution instances. Independent audit checked original SDK request/ACK hashes and version/marker identities, physical set selection, exact task options/receipts/counter partitions, 27,096 complete shard-streaming proofs, physical closure before target GET or control rescue, complete peer inventories and all 170 node cleanup receipts. Full inventories are checked against ACK manifests; their endpoint iteration is established by the immutable executed helper source, since the inventory documents omit endpoint labels. The remove=true branch replays one complete below-quorum original object after exact DELETE and requires whole-orphan removal with unchanged live controls. Rescue restores only post-DELETE authority. The original failed A10 result remains byte-identical. The sections below retain earlier implementation and run history. Historical failed or partial results are not promoted by this summary. From 82babf76c8ad9b6a654e08da1c84beba1d659a21 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 14:19:04 +0800 Subject: [PATCH 229/343] fix(validation): recover durable write owner before rescan --- docs/validation-heal-cases.md | 10 +++ scripts/validation_heal/persistence.py | 84 ++++++++++++++++++++ test_validation_heal_persistence.py | 101 ++++++++++++++++++++++++- 3 files changed, 194 insertions(+), 1 deletion(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index dd48e947..ec014be9 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1222,6 +1222,16 @@ Execution policy: `evidence-blocked`; disabled by default: `false`. Blocked reason (`EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION`): 已实现 H1/H2 checkpoint 截断、未来 schema、真实较早代次回放、committed MRF CRC 故障、私有限额卷的真实 journal ENOSPC 和独立安全重扫；10 项原生回归仅为前置证据。仍缺全部所需 fsync/commit 临界点审核和完整现场分支的独立 live 审计，诊断执行不能提升 canonical 验收。 +Journal ENOSPC recovery preserves the first failed checkpoint/terminal publication +attempt before restarting the original durable owner after filler release. A new +rescan is permitted only after that same responsibility either completes with +full physical and durable terminal proof or retains an exact failed terminal +without a runnable owner. A still-pending scope remains a failure. Successful +original replay must repair the complete remaining manifest before any target +GET; the later distinct rescan must report zero repairs and preserve original +completion bytes. No forceStart, owner deletion or old-record restoration is +used to bypass overlap admission. This flow still requires fresh live evidence. + 1. 先制造并证明真实 durable root/MRF 责任，CHECKPOINT 定位精确本 run task/record，其他系统记录禁止触碰。 2. 停止测试服务，按 FAULT_FORMAT 对这些已审核记录分别截断、坏校验、未来 schema、旧代次；写失败实例用独占限额故障层，不塞满真实共享盘。 3. 重启后核对解析/错误状态：未知或损坏记录保留、可诊断，不能静默删除并 ACK；责任仍能安全重扫处理。 diff --git a/scripts/validation_heal/persistence.py b/scripts/validation_heal/persistence.py index 3472e8b7..60a6eb0f 100644 --- a/scripts/validation_heal/persistence.py +++ b/scripts/validation_heal/persistence.py @@ -367,6 +367,86 @@ def retained_write_failure(observed, roots, frozen, current, events): "failedWriteAcknowledged": False, "originalFailureReclassified": False, "backupRescueUsed": False} +def recover_write_before_rescan(fleet, probe, binding, expected, baseline, original, targets, + frozen, failed, failure_proof, failure_scene, remaining, events, active): + token, bucket = frozen["taskId"], frozen["bucket"] + # Export the first failed attempt before product recovery may legitimately + # replace its runtime status with a successful replay of the same token. + fleet.record("original-write-failure-before-recovery", {"firstFailedAttempt": failed, "proof": failure_proof}) + fleet.stop() + require(physical_content(fleet.snapshot("write-failure-before-original-recovery", quiescent=True)) + == physical_content(failure_scene), "space release changed the original failed repair scene") + retained_write_failure(checkpoints.records(fleet, token, "write-failure-checkpoint-before-recovery", quiescent=True), + checkpoints.records(fleet, token, "write-failure-root-before-recovery", quiescent=True, family="root"), + frozen, failed, events) + fleet.start() + deadline = time.monotonic() + 180 + while True: + require(time.monotonic() < deadline, "released original write responsibility did not reach a bounded recovery outcome") + current = recovery_status(probe, fleet, token) + require(time.monotonic() < deadline, "released original write responsibility did not reach a bounded recovery outcome") + if current.get("summary") != "running": break + time.sleep(.2) + fleet.stop() + roots = checkpoints.records(fleet, token, "write-original-root-after-recovery", quiescent=True, family="root") + if current.get("summary") != "finished": + require(all(canonical(current.get(name)) == canonical(failed.get(name)) + for name in ("summary", "settings", "outcome", "detail")), + "original write recovery neither retained its failure nor completed verified work") + retained = retained_write_failure( + checkpoints.records(fleet, token, "write-original-checkpoint-after-recovery", quiescent=True), + roots, frozen, current, events) + require(retained["originalAdmissionRetained"] is False, + "original write responsibility still owns its durable scope after recovery; separate rescan is not admitted") + require(physical_content(fleet.snapshot("write-original-failed-recovery-physical", quiescent=True)) + == physical_content(failure_scene), "failed original recovery changed its acknowledged repair scene") + fleet.start() + return None + completion = rescan_outcome(current, bucket, expected, + {version_identity(target) for target in remaining}, frozen["bucketIncarnationId"]) + completion.pop("originalFailedTaskReclassified") + durable = completed_original(roots, frozen, current) + healed = fleet.snapshot("write-original-replay-physical-before-rescan-or-GET", quiescent=True) + fleet.record("write-original-replay-complete-physical-proof", + validate_repaired_physical(original, healed, targets[0], repair_targets=targets)) + for target in targets: verify_proof(fleet, target, "write-original-replay-full-shard-integrity") + fleet.start() + admissions = [] + rescanned = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, + option_overrides={"recursive": True}, admissions=admissions) + require(len(admissions) == 1 and admissions[0]["clientToken"] != token, + "write recovery rescan reused or relabelled the original responsibility") + rescan = rescan_outcome(rescanned, bucket, expected, set(), frozen["bucketIncarnationId"]) + repeated = recovery_status(probe, fleet, token) + require(all(canonical(repeated.get(name)) == canonical(current.get(name)) + for name in ("summary", "settings", "outcome", "detail", "truncated")), + "separate rescan rewrote the original write recovery result") + for endpoint in binding["endpoints"]: + peer = Probe(binding, endpoint) + same = status(peer, fleet, checkpoints.PATH, token, "same-original-write-replay-peer") + require(all(canonical(same.get(name)) == canonical(current.get(name)) + for name in ("summary", "settings", "outcome", "detail", "truncated")), + "original write replay did not converge on every peer") + inventory = fleet.record("complete-peer-S3", {"endpoint": endpoint, **peer.inventory(bucket, + unversioned_current_tags=expected[0]["versionId"] == "null")}) + verify_seed(expected, inventory); compare_inventories(baseline, inventory) + fleet.stop() + require(physical_content(healed) == physical_content(fleet.snapshot("write-replay-after-rescan-and-reads", quiescent=True)), + "rescan or business reads repaired work missing from the original write replay proof") + retained = checkpoints.records(fleet, token, "retained-write-original-completion", quiescent=True, family="root") + completed_original(retained, frozen, repeated) + require(canonical(exact_records(retained, token, "root")) == canonical(exact_records(roots, token, "root")), + "rescan or business reads changed the original write replay durable completion") + replay = {"realSameTaskReplayBeforeSeparateRescan": True, "failedFirstAttemptRetained": True, + "separateRescanIsOriginalTaskCompletion": False, "durableOriginalCompletion": durable} + fleet.record("complete-original-journal-write-reconciliation", {"firstFailedAttempt": failed, + "retainedFailureProof": failure_proof, "originalReplayCompletion": current, + "originalReplayPhysicalAndCounters": completion, "separateRescan": rescan, "restartProof": replay}) + fleet.faults.clear() + return {"verdict": "PASS", "originalWriteFailure": failure_proof, "separateRescan": rescan, + "originalAfterRestart": replay, "completePhysicalAndS3ManifestsVerified": True, "canonicalCasePassClaimed": False} + + def journal_write_failure(fleet, probe, binding, expected, baseline, original, targets, frozen, active): token, bucket = frozen["taskId"], frozen["bucket"] require(frozen["value"]["admin"]["outcome"]["counters"]["healed"] == 0, @@ -429,6 +509,10 @@ def journal_write_failure(fleet, probe, binding, expected, baseline, original, t del fleet.frozen_processes[node] active.remove((checkpoints.PATH, token)) idle_fleet(fleet, binding) + if proof["originalAdmissionRetained"]: + recovered = recover_write_before_rescan(fleet, probe, binding, expected, baseline, original, targets, + frozen, current, proof, scene, remaining, window.events, active) + if recovered is not None: return recovered admissions = [] rescanned = run_task(probe, fleet, {"bucket": bucket, "key": ""}, 2, active, option_overrides={"recursive": True}, admissions=admissions) diff --git a/test_validation_heal_persistence.py b/test_validation_heal_persistence.py index a5d9748c..e29c7a9a 100644 --- a/test_validation_heal_persistence.py +++ b/test_validation_heal_persistence.py @@ -444,7 +444,7 @@ def test_write_failure_live_census_requires_proven_frozen_processes(self): persistence.retained_write_failure(observed, roots, frozen, current, events) def test_write_refusal_release_rescan_and_real_original_retry_have_distinct_evidence(self): - for change in (None, "wrong-mount-witness", "release-failure", "late-export-failure"): + for change in (None, "wrong-mount-witness", "release-failure", "late-export-failure", "pending-owner"): observed, roots, frozen, failed, events = self.write_failure() expected = [{"key": f"objects/{i:04d}.bin", "versionId": "null", "deleteMarker": False} for i in range(3)] def finished(heal): @@ -489,8 +489,13 @@ def records(fleet, token, label, **kwargs): if label == "retained-write-admission-after-reads": return final_roots return roots if kwargs.get("family") == "root" else observed def run_task(*args, **kwargs): kwargs["admissions"].append({"clientToken": INCARNATION}); return rescanned + def recover_pending(*args): + issued.append("recover-pending-original-before-new-task") + if change == "pending-owner": raise ValueError("original durable scope remains owned") + return None window = SimpleNamespace(events=events, capture=lambda: events) with patch.object(persistence, "JournalWindow", return_value=window), \ + patch.object(persistence, "recover_write_before_rescan", create=True, side_effect=recover_pending), \ patch.object(persistence, "recovery_status", side_effect=[failed, failed, final]), \ patch.object(persistence.checkpoints, "records", side_effect=records), \ patch.object(persistence, "validate_repaired_physical"), patch.object(persistence, "verify_proof"), \ @@ -510,6 +515,100 @@ def run_task(*args, **kwargs): kwargs["admissions"].append({"clientToken": INCAR with self.assertRaises((ValueError, OSError)): persistence.journal_write_failure(*args) self.assertTrue(fleet.faults) if change == "wrong-mount-witness": self.assertNotIn("bounded-volumes-release", issued) + if change == "pending-owner": + self.assertIn("recover-pending-original-before-new-task", issued) + self.assertNotIn("journal-rescan-complete-physical-before-any-GET", issued) + + def test_pending_write_owner_recovers_before_new_rescan_without_reclassifying_first_failure(self): + cases = (None, "failed-terminal", "still-owned", "wrong-terminal", "streaming-error", "rescan-heals", + "same-token", "peer-disagrees", "read-repair", "changed-terminal", "export-error", + "pre-recovery-drift", "recovery-timeout", "late-recovery-completion") + for change in cases: + with self.subTest(change=change): + observed, failed_roots, frozen, failed, events = self.write_failure() + current, _, expected, _, _ = self.rescan() + final_roots = self.completion_roots(frozen, current) + failed_terminal = copy.deepcopy(failed_roots) + failed_terminal["node"]["records"][3] = {"disk": 1, "relativePath": journal.names("root", TOKEN)[0], "present": False} + terminal = {"schema": 1, "task_id": TOKEN, "heal_type": frozen["rootAdmission"]["value"]["heal_type"], + "options": frozen["rootAdmission"]["value"]["options"], "status": {"Failed": {"error": failed["detail"]}}} + failed_terminal["node"]["records"][4].update(encoded(terminal)) + failed_terminal["node"]["records"][5].update(encoded({"schema": 1, "terminal": terminal, "outcome": failed["outcome"]})) + first_failed = copy.deepcopy(failed) + failure_scene = {"node": {"files": [], "formats": []}} + healed = {"node": {"files": [{"disk": 1, "path": "bound/part.1", "sha256": "a" * 64}], "formats": []}} + issued, exports = [], {} + class ReplayFleet: + faults = [("node", {"faultId": "F07-shard-0"})] + def start(self): issued.append("start") + def stop(self): issued.append("stop") + def record(self, label, value): + issued.append(label) + if change == "export-error" and label == "complete-original-journal-write-reconciliation": raise OSError("lost reconciliation export") + exports[label] = copy.deepcopy(value); return value + def snapshot(self, label, **kwargs): + issued.append(label) + if label == "write-failure-before-original-recovery" and change == "pre-recovery-drift": return copy.deepcopy(healed) + if label in ("write-failure-before-original-recovery", "write-original-failed-recovery-physical"): return copy.deepcopy(failure_scene) + if label == "write-replay-after-rescan-and-reads" and change == "read-repair": return copy.deepcopy(failure_scene) + return copy.deepcopy(healed) + fleet = ReplayFleet() + def records(fleet, token, label, **kwargs): + self.assertEqual(token, TOKEN) + if kwargs.get("family") != "root": return copy.deepcopy(observed) + if label == "write-failure-root-before-recovery": return copy.deepcopy(failed_roots) + if change == "still-owned": return copy.deepcopy(failed_roots) + if change == "failed-terminal": return copy.deepcopy(failed_terminal) + result = copy.deepcopy(final_roots) + if change == "wrong-terminal" or change == "changed-terminal" and label == "retained-write-original-completion": + report = journal.decode_json(checkpoint.bytes_of(result["node"]["records"][5])) + if change == "wrong-terminal": report["outcome"]["counters"]["healed"] = 0 + else: report["unrelated"] = "changed after rescan" + result["node"]["records"][5].update(encoded(report)) + return result + def run_task(*args, **kwargs): + self.assertIn("write-original-replay-complete-physical-proof", issued) + self.assertIn("write-original-replay-full-shard-integrity", issued) + kwargs["admissions"].append({"clientToken": TOKEN if change == "same-token" else INCARNATION}) + issued.append("independent-new-rescan") + rescan = copy.deepcopy(current) + if change != "rescan-heals": rescan["outcome"]["counters"].update(healed=0, unchanged=257) + return rescan + def verify(*args): + issued.append("write-original-replay-full-shard-integrity") + if change == "streaming-error": raise ValueError("actual cohort failed streaming proof") + proof = persistence.retained_write_failure(observed, failed_roots, frozen, failed, events) + target = {"node": "node", "disk": 1, "relativePath": "bound/part.1", "key": expected[-1]["key"], "versionId": "null"} + active = [] + with patch.object(persistence.checkpoints, "records", side_effect=records), \ + patch.object(persistence, "validate_repaired_physical"), patch.object(persistence, "verify_proof", side_effect=verify), \ + patch.object(persistence, "recovery_status", return_value={"summary": "running"} if change == "recovery-timeout" else failed if change in ("failed-terminal", "still-owned") else current), \ + patch.object(persistence, "run_task", side_effect=run_task), \ + patch.object(persistence, "status", return_value=failed if change == "peer-disagrees" else current), \ + patch.object(persistence, "Probe", return_value=SimpleNamespace(inventory=lambda *a, **kw: {})), \ + patch.object(persistence, "verify_seed"), patch.object(persistence, "compare_inventories"), \ + patch.object(persistence.time, "monotonic", return_value=0, + side_effect=[0, .01, 181] if change in ("recovery-timeout", "late-recovery-completion") else None): + args = (fleet, None, {"endpoints": {"node1": "http://owned"}}, expected, {}, healed, [target], + frozen, failed, proof, failure_scene, [target], events, active) + if change in (None, "failed-terminal"): + result = persistence.recover_write_before_rescan(*args) + if change is None: + self.assertTrue(result["originalAfterRestart"]["realSameTaskReplayBeforeSeparateRescan"]) + self.assertFalse(result["originalAfterRestart"]["separateRescanIsOriginalTaskCompletion"]) + self.assertFalse(result["canonicalCasePassClaimed"]) + self.assertFalse(fleet.faults) + self.assertEqual(exports["complete-original-journal-write-reconciliation"]["firstFailedAttempt"], first_failed) + else: + self.assertIsNone(result); self.assertTrue(fleet.faults) + self.assertNotIn("independent-new-rescan", issued) + else: + with self.assertRaises((ValueError, OSError)): persistence.recover_write_before_rescan(*args) + self.assertTrue(fleet.faults) + if change in ("still-owned", "wrong-terminal", "streaming-error"): + self.assertNotIn("independent-new-rescan", issued) + self.assertEqual(failed, first_failed) + self.assertEqual(exports["original-write-failure-before-recovery"]["firstFailedAttempt"], first_failed) def test_successful_replay_cannot_discard_rescue_before_manifest_terminal_and_evidence_agree(self): for change in (None, "read-repair", "late-archive-failure", "rescan-relabels-original"): From c6dcbe27e253e9c996b56351a1bda049f2cde10b Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 15:17:52 +0800 Subject: [PATCH 230/343] test(heal): bind quorum replays to the reviewed GET fix --- docs/validation-heal-cases.md | 4 + scripts/validation_heal/network.py | 26 +++++- scripts/validation_heal/quorum_boundary.py | 27 +++++- test_validation_heal_quorum_candidate.py | 100 +++++++++++++++++++++ 4 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 test_validation_heal_quorum_candidate.py diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index ec014be9..f8379381 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -356,6 +356,8 @@ Source assertions: Topologies: H1,H2. Branches: exact-r-read, exact-r-heal-write-limited, restored-w. +执行器仅接受经逐文件和锁策略复核的 `072b57b37559fbe626aeda8abf61617fdcbdd926` 与 GET quorum 修复候选 `1c7fc818dd858247c45b39542abd8291b1e6e31b`。记录所选候选的精确锁契约；inspector 必须同时匹配候选及当前 helper 源码摘要，未知候选在部署前拒绝。below-R GET 仍必须为候选冻结的 `503/SlowDownRead`，旧 `500/InternalError` 失败记录保留；支持新候选不等于新的 live 验收。 + 1. 记录读取 R、修复写 W、锁 quorum 的来源；选择同一版本保留恰好 R 个权威片，其他片隔离。 2. 第一次请求 GET 验证读取边界并记录 read-repair 干扰；用第二个相同形态新对象执行只手工 deep heal。 3. 如果只有 R 但写/锁 quorum 不足，要求拒绝持久化成功；若拓扑中该组合不可实现，在运行前声明精确原因并选满足条件的拓扑，不伪造离线数。 @@ -1716,6 +1718,8 @@ Source assertions: Topologies: H2. Branches: minority-partition, read-boundary, write-boundary, lock-boundary, rejoin. +候选支持与 B06 相同，但 storageclass 与锁策略分别按精确候选绑定。`1c7fc81` 的锁客户端范围、quorum 阈值及 EC:8 配置已复核；其 RPC 控制通道工厂有变化，必须重新取得真实双向网络探针、admission、partial coverage 和 rejoin 证据，不能由源码等价或 B07 通过推定本例通过。B08 的候选范围保持独立。 + 1. H2 先冻结实际节点/slot/对象R/W和锁quorum，NETWORK规则只匹配本例服务端口并带run ID、超时恢复。 2. 构造少数派及边界分区，双端探针证实隔离；SSH、跳板机和其他服务端口保持连通。 3. 双方执行带nonce的PUT/GET/heal，少数派不能承诺不安全提交；BACKGROUND必须暴露missing peer/partial coverage。 diff --git a/scripts/validation_heal/network.py b/scripts/validation_heal/network.py index 6c230940..37a37560 100644 --- a/scripts/validation_heal/network.py +++ b/scripts/validation_heal/network.py @@ -23,7 +23,8 @@ from .matrix import select from .shards import files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort from .versions import public_version_id, validate_version_receipts, verify_quorum_contract, validate_unrecoverable_get - from .write_quorum import expected_reads, resolve_put, negative_put, LOCK_REFERENCE, STORAGE_REFERENCE + from .write_quorum import expected_reads, resolve_put, negative_put, STORAGE_REFERENCE + from .quorum_boundary import LOCK_REFERENCE, lock_reference from .stop import idle_fleet from .network_gate import TTL_SECONDS except ImportError: @@ -34,7 +35,8 @@ from matrix import select from shards import files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort from versions import public_version_id, validate_version_receipts, verify_quorum_contract, validate_unrecoverable_get - from write_quorum import expected_reads, resolve_put, negative_put, LOCK_REFERENCE, STORAGE_REFERENCE + from write_quorum import expected_reads, resolve_put, negative_put, STORAGE_REFERENCE + from quorum_boundary import LOCK_REFERENCE, lock_reference from stop import idle_fleet from network_gate import TTL_SECONDS @@ -52,6 +54,22 @@ 'writePolicy': 'three-peer groups retain R but use independent private gates to measure W-1; two-peer groups lose W and exclusive lock quorum'} +# Storage geometry is reviewed independently of the namespace lock policy. +STORAGE_REFERENCES = { + STORAGE_REFERENCE['candidateCommit']: STORAGE_REFERENCE, + '1c7fc818dd858247c45b39542abd8291b1e6e31b': { + **STORAGE_REFERENCE, 'candidateCommit': '1c7fc818dd858247c45b39542abd8291b1e6e31b'}, +} + + +def source_reference(candidate_commit): + require(isinstance(candidate_commit, str) and candidate_commit in STORAGE_REFERENCES, + 'network storage policy is not reviewed for this candidate') + return {**copy.deepcopy(REFERENCE), 'candidateCommit': candidate_commit, + 'lock': lock_reference(candidate_commit), + 'storage': copy.deepcopy(STORAGE_REFERENCES[candidate_commit])} + + def plan_for(snapshot, target, branch): nodes = list(snapshot) geometry = target['logical']['erasure'] @@ -250,7 +268,7 @@ def run_branch(args, nodes, branch): run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-h2" fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile='lock-quorum') candidate = json.loads(args.candidate.read_text()) - require(candidate['source_sha'] == REFERENCE['candidateCommit'], 'network lock policy is not reviewed for this candidate') + reference = source_reference(candidate['source_sha']) bucket = 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) @@ -262,7 +280,7 @@ def run_branch(args, nodes, branch): 'rcBinarySha256': hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), 'maxInventoryObjects': 128, 'maxInventoryBytes': 512 * 1024**2} fleet.record('binding', binding) - fleet.record('network-source-contract', REFERENCE) + fleet.record('network-source-contract', reference) contract = verify_quorum_contract(fleet.record('inspector-build', verify_inspector(args.tool_directory, candidate)), candidate) result = {'topology': 'H2', 'branch': branch, 'verdict': 'RUNNING', 'cleanup': 'PENDING'} probe, active, recovery_active, rows = Probe(binding), [], [], [] diff --git a/scripts/validation_heal/quorum_boundary.py b/scripts/validation_heal/quorum_boundary.py index 38ea6d30..97818d48 100644 --- a/scripts/validation_heal/quorum_boundary.py +++ b/scripts/validation_heal/quorum_boundary.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import copy from datetime import UTC, datetime, timedelta import hashlib import json @@ -53,6 +54,28 @@ "crates/lock/src/distributed_lock.rs": "657796708fdc5f826f896147b74a9366dae875fd6b69e4b253994966ed5f6e37"}} +LOCK_REFERENCES = { + LOCK_REFERENCE["candidateCommit"]: LOCK_REFERENCE, + "1c7fc818dd858247c45b39542abd8291b1e6e31b": { + **LOCK_REFERENCE, + "candidateCommit": "1c7fc818dd858247c45b39542abd8291b1e6e31b", + "rpcControlChannelPolicy": "class-aware control channel; audience and replay-scope authentication retained; live admission remains required", + "sourceFiles": { + **LOCK_REFERENCE["sourceFiles"], + "crates/ecstore/src/runtime/sources.rs": "fad5d30e9f76a71c41a4dd5cfa816d3ea49b0fa082ab111118e2ee5a6f8fcc90", + "crates/ecstore/src/set_disk/ops/heal.rs": "fc576119f455848ce279772ffca6d2943cb9f909968cae367860a9887d3e020e", + "crates/ecstore/src/cluster/rpc/remote_locker.rs": "506a16274efc0ac7dd2ffa4e2faf2b2a057deacfb8fb9480f4785768628b9cbf", + "crates/ecstore/src/cluster/rpc/client.rs": "b37e4107e898a3467520c8516608c8eaaa0c1d56a2eac82d11982eb483396291"}, + }, +} + + +def lock_reference(candidate_commit): + require(isinstance(candidate_commit, str) and candidate_commit in LOCK_REFERENCES, + "lock policy is not reviewed for this candidate") + return copy.deepcopy(LOCK_REFERENCES[candidate_commit]) + + def seed_fixtures(recorder, bucket): recorder.call("create_bucket", {"Bucket": bucket}) expected = [] @@ -158,7 +181,7 @@ def run_topology(args, topology, nodes): run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" fleet = Fleet(nodes, directory, run_id, args.tool_directory) candidate = json.loads(args.candidate.read_text()) - require(candidate["source_sha"] == LOCK_REFERENCE["candidateCommit"], "B06 lock policy is not reviewed for this candidate") + locks = lock_reference(candidate["source_sha"]) contract = verify_quorum_contract(fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)), candidate) require(contract.get("writeQuorumRule") == "data-plus-one-if-equal-parity", "B06 write policy is not recognized") require(list(BRANCHES) == select(CASE_ID)[0]["variants"], "B06 executor omitted a source branch") @@ -173,7 +196,7 @@ def run_topology(args, topology, nodes): "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), "maxInventoryObjects": 512, "maxInventoryBytes": 512 * 1024**2} fleet.record("binding", binding) - fleet.record("quorum-source-contract", {"payloadAndWrite": contract, "locks": LOCK_REFERENCE}) + fleet.record("quorum-source-contract", {"payloadAndWrite": contract, "locks": locks}) probe, active, admissions = Probe(binding), [], [] result = {"topology": topology, "fixture": "DB", "verdict": "RUNNING", "branches": {}, "cleanup": "PENDING"} try: diff --git a/test_validation_heal_quorum_candidate.py b/test_validation_heal_quorum_candidate.py new file mode 100644 index 00000000..7f828f2e --- /dev/null +++ b/test_validation_heal_quorum_candidate.py @@ -0,0 +1,100 @@ +"""Candidate selection must bind the real executors before any fleet mutation.""" + +from contextlib import ExitStack +import copy +import json +from pathlib import Path +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import network, quorum_boundary, write_quorum + + +ORIGINAL = '072b57b37559fbe626aeda8abf61617fdcbdd926' +FIXED = '1c7fc818dd858247c45b39542abd8291b1e6e31b' + + +class RecordedSourceContract(Exception): + def __init__(self, records): + self.records = copy.deepcopy(records) + super().__init__("source contract exported") + + +class QuorumCandidateTests(unittest.TestCase): + def execute_until_contract(self, module, source, *, allow_inspector=True): + records = {} + def record(label, value): + records[label] = value + if label in {'quorum-source-contract', 'network-source-contract'}: + raise RecordedSourceContract(records) + return value + fleet = types.SimpleNamespace(record=record) + with tempfile.TemporaryDirectory() as temporary, ExitStack() as stack: + root = Path(temporary) + candidate = root / 'candidate.json' + candidate.write_text(json.dumps({'source_sha': source, 'package_sha256': 'a' * 64})) + rc = root / 'rc' + rc.write_bytes(b'client') + args = types.SimpleNamespace(evidence=root / 'evidence', candidate=candidate, + rc_bin=rc, tool_directory=root, testing_commit='b' * 40, binary_sha='c' * 64) + stack.enter_context(patch.dict('os.environ', {'GITHUB_RUN_ID': '123', 'GITHUB_RUN_ATTEMPT': '1'})) + stack.enter_context(patch.object(module, 'Fleet', return_value=fleet)) + stack.enter_context(patch.object(module, 'verify_inspector', return_value={}, + side_effect=None if allow_inspector else AssertionError('unreviewed candidate reached inspector'))) + stack.enter_context(patch.object(module, 'verify_quorum_contract', return_value={ + 'recognized': True, 'writeQuorumRule': 'data-plus-one-if-equal-parity'})) + # Reaching the contract is the last allowed action; no deploy method + # exists on this fleet and no Probe is constructed by these tests. + if module is quorum_boundary: + module.run_topology(args, 'H1', ['vm000']) + else: + module.run_branch(args, ['vm000', 'vm001', 'vm002', 'vm003'], network.BRANCHES[0]) + return records + + def test_actual_executors_record_the_selected_candidate_before_deploy(self): + for module in (quorum_boundary, network): + for source in (ORIGINAL, FIXED): + with self.subTest(executor=module.CASE_ID, source=source): + with self.assertRaises(RecordedSourceContract) as completed: + self.execute_until_contract(module, source) + captured = completed.exception.records + reference = captured['quorum-source-contract']['locks'] if module is quorum_boundary else captured['network-source-contract'] + self.assertEqual(reference['candidateCommit'], source) + if module is network: + self.assertEqual(reference['lock']['candidateCommit'], source) + self.assertEqual(reference['storage']['candidateCommit'], source) + self.assertEqual(captured['binding']['candidateCommit'], source) + + def test_unknown_and_malformed_sources_never_reach_inspector_or_deploy(self): + for module in (quorum_boundary, network): + for source in ('0' * 40, FIXED[:7], FIXED.upper(), None, [], True): + with self.subTest(executor=module.CASE_ID, source=source): + with self.assertRaises(ValueError): + self.execute_until_contract(module, source, allow_inspector=False) + + def test_network_storage_review_is_separate_from_lock_review(self): + source = 'f' * 40 + with patch.dict(quorum_boundary.LOCK_REFERENCES, {source: { + **quorum_boundary.LOCK_REFERENCE, 'candidateCommit': source}}): + with self.assertRaises(ValueError): + network.source_reference(source) + + def test_reference_exports_cannot_mutate_original_or_other_candidate(self): + original = copy.deepcopy(quorum_boundary.LOCK_REFERENCE) + fixed = quorum_boundary.lock_reference(FIXED) + fixed['sourceFiles'].clear() + fixed['candidateCommit'] = ORIGINAL + reference = network.source_reference(FIXED) + reference['storage']['sourceFiles'].clear() + reference['lock']['sourceFiles'].clear() + self.assertTrue(quorum_boundary.lock_reference(FIXED)['sourceFiles']) + self.assertTrue(network.source_reference(FIXED)['storage']['sourceFiles']) + self.assertEqual(quorum_boundary.LOCK_REFERENCE, original) + self.assertEqual(write_quorum.LOCK_REFERENCE['candidateCommit'], ORIGINAL) + self.assertEqual(write_quorum.STORAGE_REFERENCE['candidateCommit'], ORIGINAL) + + +if __name__ == '__main__': + unittest.main() From 0d93a4cdb88350f7df580c6a33c161fa58acfe22 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 15:28:46 +0800 Subject: [PATCH 231/343] fix(heal): classify detached file reads as bounded churn --- docs/validation-heal-cases.md | 2 + scripts/validation_heal/heal_fault.py | 7 ++- test_validation_heal_journal_fault.py | 73 ++++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index f8379381..6d895fcc 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1220,6 +1220,8 @@ Source assertions: Topologies: H1,H2. Branches: checkpoint-truncate, mrf-checksum, future-schema, stale-generation, journal-write-failure, native-persist-fence. +实时读取遇到原子替换后链接数为零的旧文件句柄时，先核对文件类型、大小与挂载边界，再拒绝旧句柄字节并返回读取变化。只有原有有界观察循环可以重试；停止后的读取和故障注入仍直接失败，硬链接、非普通文件、超限和挂载逃逸仍被拒绝。原 pidfd、60 秒 advance 预算和完整 checkpoint/resume/root 校验不变。Linux 原子替换组件复现仅证明读取竞争，不证明旧 live 失败的唯一原因，也不替代 RustFS 持久化屏障证据。 + Execution policy: `evidence-blocked`; disabled by default: `false`. Blocked reason (`EVIDENCE_DURABLE_JOURNAL_ORCHESTRATION`): 已实现 H1/H2 checkpoint 截断、未来 schema、真实较早代次回放、committed MRF CRC 故障、私有限额卷的真实 journal ENOSPC 和独立安全重扫；10 项原生回归仅为前置证据。仍缺全部所需 fsync/commit 临界点审核和完整现场分支的独立 live 审计，诊断执行不能提升 canonical 验收。 diff --git a/scripts/validation_heal/heal_fault.py b/scripts/validation_heal/heal_fault.py index fd90258b..59dcf651 100644 --- a/scripts/validation_heal/heal_fault.py +++ b/scripts/validation_heal/heal_fault.py @@ -90,13 +90,16 @@ def read_file(fd: int, name: str, limit: int = LIMIT) -> bytes: opened = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd) try: info = os.fstat(opened) - require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size <= limit, + require(stat.S_ISREG(info.st_mode) and info.st_nlink <= 1 and info.st_size <= limit, "expected bounded regular non-hardlinked file") require(mount_id(opened) == mount_id(fd), "file bind-mount escape") + # Atomic publication can unlink the opened predecessor before fstat. + # Reject those bytes as churn; only bounded live observers may retry. + require(info.st_nlink == 1, "file changed while reading") with os.fdopen(opened, "rb", closefd=False) as stream: value = stream.read(limit + 1) after = os.fstat(opened) - require(len(value) <= limit and info.st_size == len(value) + require(after.st_nlink == 1 and len(value) <= limit and info.st_size == len(value) and (info.st_size, info.st_mtime_ns, info.st_ctime_ns) == (after.st_size, after.st_mtime_ns, after.st_ctime_ns), "file changed while reading") return value diff --git a/test_validation_heal_journal_fault.py b/test_validation_heal_journal_fault.py index a9a74c1a..321f3c20 100644 --- a/test_validation_heal_journal_fault.py +++ b/test_validation_heal_journal_fault.py @@ -31,6 +31,58 @@ class DurableRecordFaultTests(unittest.TestCase): incarnation = "22345678-1234-4234-8234-123456789abc" bucket = "rv-heal-0123456789abcdef" + def test_atomic_replace_after_open_rejects_detached_bytes_as_read_churn(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + current, successor = root / 'checkpoint', root / 'successor' + current.write_bytes(b'earlier') + successor.write_bytes(b'later') + original_open = os.open + observed = [] + def replace_after_open(name, flags, **kwargs): + descriptor = original_open(name, flags, **kwargs) + if name == current.name: + os.replace(successor, current) + observed.append(os.fstat(descriptor).st_nlink) + return descriptor + descriptor = original_open(root, os.O_RDONLY | os.O_DIRECTORY) + try: + with patch.object(fault.os, 'open', side_effect=replace_after_open), \ + patch.object(fault, 'mount_id', return_value=1): + with self.assertRaisesRegex(ValueError, '^file changed while reading$'): + fault.read_file(descriptor, current.name) + self.assertEqual(observed, [0]) + with patch.object(fault, 'mount_id', return_value=1): + self.assertEqual(fault.read_file(descriptor, current.name), b'later') + self.assertEqual(current.stat().st_nlink, 1) + finally: + os.close(descriptor) + + def test_detached_inode_does_not_hide_type_size_or_mount_refusals(self): + for poison in ('oversized', 'fifo', 'mount'): + with self.subTest(poison=poison), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + current, successor = root / 'checkpoint', root / 'successor' + if poison == 'fifo': os.mkfifo(current) + else: current.write_bytes(b'12345' if poison == 'oversized' else b'old') + successor.write_bytes(b'new') + original_open = os.open + def replace_after_open(name, flags, **kwargs): + descriptor = original_open(name, flags, **kwargs) + if name == current.name: + os.replace(successor, current) + self.assertEqual(os.fstat(descriptor).st_nlink, 0) + return descriptor + descriptor = original_open(root, os.O_RDONLY | os.O_DIRECTORY) + try: + with patch.object(fault.os, 'open', side_effect=replace_after_open), \ + patch.object(fault, 'mount_id', side_effect=[2, 1] if poison == 'mount' else None, return_value=1): + message = 'file bind-mount escape' if poison == 'mount' else 'expected bounded regular non-hardlinked file' + with self.assertRaisesRegex(ValueError, '^' + message + '$'): + fault.read_file(descriptor, current.name, limit=4) + finally: + os.close(descriptor) + def test_live_read_race_is_retryable_but_stopped_read_and_path_failures_are_not(self): for message in ('file changed while reading', 'file bind-mount escape', 'expected bounded regular non-hardlinked file'): with self.subTest(message=message), patch.object(journal, 'census', side_effect=ValueError(message)): @@ -67,7 +119,7 @@ def test_only_live_observation_refusals_survive_the_remote_boundary_as_retryable with self.assertRaisesRegex(ValueError, '^Heal node'): fleet.remote('node', 'journal-observe') def test_atomic_advance_uses_one_held_pidfd_and_refreezes_on_observation_failure(self): - for change in (None, 'timeout', 'birth', 'corrupt', 'incarnation', 'options', 'counter', 'read-error', 'unstable', 'post-birth', 'late-progress', 'continue-interrupted', 'stop-pending', 'stop-timeout', 'stop-birth'): + for change in (None, 'timeout', 'birth', 'corrupt', 'incarnation', 'options', 'counter', 'read-error', 'unstable', 'unlinked-open', 'unlinked-open-timeout', 'post-birth', 'late-progress', 'continue-interrupted', 'stop-pending', 'stop-timeout', 'stop-birth'): with self.subTest(change=change), tempfile.TemporaryDirectory() as temp, contextlib.ExitStack() as stack: base = Path(temp).resolve(); roots = tuple(base / str(i) for i in range(4)) for root in roots: root.mkdir() @@ -115,6 +167,19 @@ def read(*args, **kwargs): raced[0] = True if change == 'read-error': raise PermissionError('unexpected access denial') raise ValueError('file changed while reading') + if running[0] and not raced[0] and change in ('unlinked-open', 'unlinked-open-timeout'): + raced[0] = True + original_open = os.open + def replace_after_open(name, flags, **params): + descriptor = original_open(name, flags, **params) + if name == path.name: + successor = path.with_name('natural-checkpoint-successor') + successor.write_bytes(next_raw) + os.replace(successor, path) + self.assertEqual(os.fstat(descriptor).st_nlink, 0) + return descriptor + with patch.object(fault.os, 'open', side_effect=replace_after_open): + return real_read(*args, **kwargs) return real_read(*args, **kwargs) for context in (patch.object(fleet_node, 'owned', return_value=(base, roots, owner)), patch.object(fleet_node, 'digest', return_value=owner['binarySha256']), @@ -134,15 +199,19 @@ def close(fd): stack.enter_context(patch.object(Path, 'read_text', read_text)) stack.enter_context(patch.object(journal, 'read', side_effect=read)) if change == 'timeout': stack.enter_context(patch.object(fleet_node.time, 'monotonic', side_effect=[0, 61])) + elif change == 'unlinked-open-timeout': stack.enter_context(patch.object(fleet_node.time, 'monotonic', side_effect=[0, .01, 61])) elif change == 'late-progress': stack.enter_context(patch.object(fleet_node.time, 'monotonic', side_effect=[0, .01, 61])) elif change == 'stop-timeout': stack.enter_context(patch.object(fleet_node.time, 'monotonic', side_effect=[0, .01, .02, 61])) - if change in (None, 'unstable', 'stop-pending'): + if change in (None, 'unstable', 'unlinked-open', 'stop-pending'): result = fleet_node.journal_operation(request) self.assertTrue(result['checkpointAdvancedAndFrozen']) self.assertTrue(result['pidfdBound']) self.assertEqual(result['observedProcessed'], 3) self.assertEqual(path.read_bytes(), next_raw) self.assertEqual(len(process_calls), 3) + elif change == 'unlinked-open-timeout': + with self.assertRaisesRegex(ValueError, 'actual checkpoint advance exceeded its frozen 60-second budget'): + fleet_node.journal_operation(request) else: with self.assertRaises((ValueError, OSError, KeyboardInterrupt)): fleet_node.journal_operation(request) self.assertEqual(events, [fleet_node.signal.SIGCONT, fleet_node.signal.SIGSTOP]) From e9cc06c861e7b3d802827a2704434bb104c83329 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 17:05:57 +0800 Subject: [PATCH 232/343] fix(heal): release armed network timers without unloaded workers --- scripts/validation_heal/network_gate.py | 29 +++++------ test_validation_heal_network.py | 67 +++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/scripts/validation_heal/network_gate.py b/scripts/validation_heal/network_gate.py index 6cc2364b..26e11be3 100644 --- a/scripts/validation_heal/network_gate.py +++ b/scripts/validation_heal/network_gate.py @@ -270,20 +270,20 @@ def release(state, owner, *, expired=False): require(not table_present(table), 'network table has no ownership intent; preserve namespace') return {'state': 'CLEAR', 'gateWasAbsent': True} expired = expired or (state / EXPIRY).exists() - if expired: - for suffix in ('.timer', '.service'): - unit_name = value['timer'] + suffix - load_state = command('systemctl', 'show', unit_name, '-p', 'LoadState', '--value').strip() - require(load_state in {'not-found', 'loaded'}, 'expired network unit identity is unknown') - active_state = command('systemctl', 'show', unit_name, '-p', 'ActiveState', '--value').strip() - if load_state != 'not-found' and active_state not in {'inactive', 'failed', 'unknown'}: - command('systemctl', 'stop', unit_name) - require(command('systemctl', 'show', unit_name, '-p', 'ActiveState', '--value').strip() - in {'inactive', 'failed', 'unknown'}, 'expired network unit remains active') # Stop the expiry worker before inspecting/deleting, avoiding a concurrent - # deletion. A TTL worker does not stop its own unit. - if not expired: - command('systemctl', 'stop', value['timer'] + '.timer', value['timer'] + '.service') + # deletion. An armed timer need not have loaded its service yet. + for suffix in ('.timer', '.service'): + unit_name = value['timer'] + suffix + load_state = command('systemctl', 'show', unit_name, '-p', 'LoadState', '--value').strip() + require(load_state in {'not-found', 'loaded'}, 'network expiry unit identity is unknown') + require(expired or suffix != '.timer' or load_state == 'loaded', + 'network expiry timer disappeared before release') + active_state = command('systemctl', 'show', unit_name, '-p', 'ActiveState', '--value').strip() + if load_state == 'loaded' and active_state not in {'inactive', 'failed', 'unknown'}: + command('systemctl', 'stop', unit_name) + allowed = {'inactive', 'failed', 'unknown'} if expired or load_state == 'not-found' else {'inactive', 'failed'} + require(command('systemctl', 'show', unit_name, '-p', 'ActiveState', '--value').strip() in allowed, + 'network expiry unit remains active') counters = [] if table_present(value['table']): counters = validate_table(json.loads(command('nft', '-j', 'list', 'table', 'inet', value['table'])), @@ -300,9 +300,6 @@ def release(state, owner, *, expired=False): (state / MARKER).unlink() (state / SCRIPT).unlink() else: - for suffix in ('.timer', '.service'): - require(command('systemctl', 'show', value['timer'] + suffix, '-p', 'ActiveState', '--value').strip() in {'inactive', 'failed'}, - 'network expiry unit remains active') (state / MARKER).unlink() (state / SCRIPT).unlink() return receipt diff --git a/test_validation_heal_network.py b/test_validation_heal_network.py index 6556949e..ab5aeb13 100644 --- a/test_validation_heal_network.py +++ b/test_validation_heal_network.py @@ -240,6 +240,71 @@ def test_lost_timer_refuses_activation_before_touching_nft(self): gate.activate(Path(folder), self.owner) self.assertTrue(all(call.args[0] != 'nft' for call in command.call_args_list)) + def release_fixture(self, state, *, service_loaded=False, timer_loaded=True, stop_fails=False): + value, _ = self.prepare(state) + states = {'.timer': 'active' if timer_loaded else 'inactive', '.service': 'active' if service_loaded else 'inactive'} + calls = [] + def command(*args, input=None): + calls.append(args) + if args[:2] == ('systemctl', 'show'): + suffix = '.timer' if args[2].endswith('.timer') else '.service' + loaded = timer_loaded if suffix == '.timer' else service_loaded + if 'LoadState' in args: return 'loaded' if loaded else 'not-found' + return states[suffix] + if args[:2] == ('systemctl', 'stop'): + for unit in args[2:]: + suffix = '.timer' if unit.endswith('.timer') else '.service' + if suffix == '.service' and not service_loaded: + raise ValueError('systemctl network operation failed: Unit expiry.service not loaded') + if stop_fails: raise ValueError('systemctl network operation failed: stop refused') + states[suffix] = 'inactive' + return '' + return value, command, calls + + def test_release_stops_timer_when_its_service_has_never_been_loaded(self): + with tempfile.TemporaryDirectory() as folder: + state = Path(folder) + value, command, calls = self.release_fixture(state) + with patch.object(gate, 'read_marker', return_value=value), patch.object(gate, 'command', side_effect=command), patch.object(gate, 'table_present', return_value=False): + result = gate.release(state, self.owner) + self.assertEqual(result['rulesRemaining'], 0) + self.assertFalse(result['expiryTriggered']) + self.assertIn(('systemctl', 'stop', value['timer'] + '.timer'), calls) + self.assertFalse(any(value['timer'] + '.service' in args for args in calls if args[:2] == ('systemctl', 'stop'))) + self.assertFalse((state / gate.MARKER).exists()) + + def test_release_stops_an_active_worker_after_stopping_its_timer(self): + with tempfile.TemporaryDirectory() as folder: + state = Path(folder) + value, command, calls = self.release_fixture(state, service_loaded=True) + with patch.object(gate, 'read_marker', return_value=value), patch.object(gate, 'command', side_effect=command), patch.object(gate, 'table_present', return_value=False): + gate.release(state, self.owner) + stops = [args[2:] for args in calls if args[:2] == ('systemctl', 'stop')] + self.assertEqual(stops, [(value['timer'] + '.timer',), (value['timer'] + '.service',)]) + + def test_missing_normal_timer_or_failed_stop_preserves_intent_and_rules(self): + for config in ({'timer_loaded': False}, {'stop_fails': True}): + with self.subTest(config=config), tempfile.TemporaryDirectory() as folder: + state = Path(folder) + value, command, calls = self.release_fixture(state, **config) + with patch.object(gate, 'read_marker', return_value=value), patch.object(gate, 'command', side_effect=command), patch.object(gate, 'table_present', return_value=True), self.assertRaises(ValueError): + gate.release(state, self.owner) + self.assertTrue((state / gate.MARKER).exists()) + self.assertFalse(any(args[:4] == ('nft', '-j', '-f', '-') for args in calls)) + + def test_explicit_recovery_accepts_unloaded_timer_and_stops_any_remaining_worker(self): + for service_loaded in (False, True): + with self.subTest(service_loaded=service_loaded), tempfile.TemporaryDirectory() as folder: + state = Path(folder) + value, command, calls = self.release_fixture(state, timer_loaded=False, service_loaded=service_loaded) + with patch.object(gate, 'read_marker', return_value=value), patch.object(gate, 'command', side_effect=command), patch.object(gate, 'table_present', return_value=False): + result = gate.release(state, self.owner, expired=True) + self.assertTrue(result['expiryTriggered']) + self.assertEqual(result['rulesRemaining'], 0) + self.assertTrue((state / gate.EXPIRY).is_file()) + self.assertFalse((state / gate.MARKER).exists()) + self.assertFalse(any(value['timer'] + '.timer' in args for args in calls if args[:2] == ('systemctl', 'stop'))) + def test_foreign_rule_blocks_cleanup_instead_of_deleting_the_table(self): with tempfile.TemporaryDirectory() as folder: state = Path(folder) @@ -249,6 +314,8 @@ def test_foreign_rule_blocks_cleanup_instead_of_deleting_the_table(self): 'comment': 'foreign', 'expr': [{'accept': None}]}}]} def command(*args, input=None): if args[:4] == ('nft', '-j', 'list', 'table'): return json.dumps(broken) + if 'LoadState' in args: return 'loaded' + if 'ActiveState' in args: return 'inactive' return '' with patch.object(gate, 'read_marker', return_value=value), patch.object(gate, 'table_present', return_value=True), \ patch.object(gate, 'command', side_effect=command) as observed, self.assertRaises(ValueError): From c67683cf27eaa22b31493c4f40a483e38927cdbb Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 17:28:18 +0800 Subject: [PATCH 233/343] fix(heal): retain request-bound network failure diagnostics --- scripts/validation_heal/network.py | 48 ++++++++++++++++- test_validation_heal_network.py | 86 ++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 2 deletions(-) diff --git a/scripts/validation_heal/network.py b/scripts/validation_heal/network.py index 37a37560..7765d29e 100644 --- a/scripts/validation_heal/network.py +++ b/scripts/validation_heal/network.py @@ -27,6 +27,7 @@ from .quorum_boundary import LOCK_REFERENCE, lock_reference from .stop import idle_fleet from .network_gate import TTL_SECONDS + from .scanner_repair import JournalWindow except ImportError: from heal_probe import Probe, canonical, compare_inventories, payload, require, save from heal_workload import Recorder, RecordedCallError, verify_seed @@ -39,6 +40,7 @@ from quorum_boundary import LOCK_REFERENCE, lock_reference from stop import idle_fleet from network_gate import TTL_SECONDS + from scanner_repair import JournalWindow CASE_ID = 'heal.network-partition-and-lock-quorum' @@ -262,6 +264,47 @@ def release_network(fleet): require(value.get('expiryTriggered') is False, 'partition expired before its complete evidence window') +def partition_get(probe, fleet, params): + """Retain one GET's raw response and journal window before its oracle can fail.""" + identity_fields = ('pid', 'startTicks', 'bootId', 'unit', 'executable', 'binarySha256', + 'mountNamespace', 'commandSha256', 'environmentFileSha256', 'settings') + first_sequence = fleet.sequence + 1 + before = fleet.record('partition-GET-process-before', {node: fleet.remote(node, 'process') for node in fleet.nodes}) + require(all(all(field in value for field in identity_fields) and type(value['pid']) is int and value['pid'] > 1 + and value['binarySha256'] == probe.binding['candidateBinarySha256'] for value in before.values()), + 'partition GET diagnostic lacks a live candidate process') + # Raw pages are retained; parsed events are not a causal proof or oracle. + window = JournalWindow(fleet, 'partition-GET-journal', event_targets=set()) + observed, request_sequence = None, None + try: + observed = fleet.record('partition-explicit-version-GET', probe.s3('get_object', params)) + request_sequence = fleet.sequence + finally: + errors, after = [], {} + try: + window.capture() + except Exception as exc: + errors.append({'stage': 'journal', 'type': type(exc).__name__, 'message': str(exc)}) + for node in fleet.nodes: + try: + after[node] = fleet.remote(node, 'process') + require(all(field in after[node] and after[node][field] == before[node][field] for field in identity_fields), + 'partition GET crossed a candidate process identity change') + except Exception as exc: + errors.append({'stage': 'process', 'node': node, 'type': type(exc).__name__, 'message': str(exc)}) + fleet.record('partition-GET-process-after', after) + if observed is None: + errors.append({'stage': 'request', 'message': 'GET raised before returning a raw observation'}) + response = (observed or {}).get('response') or {} + diagnostics = fleet.record('partition-GET-diagnostics', { + 'complete': not errors, 'endpoint': probe.endpoint_name, 'params': params, + 'requestId': response.get('ResponseMetadata', {}).get('RequestId'), + 'firstEvidenceSequence': first_sequence, 'requestEvidenceSequence': request_sequence, + 'lastEvidenceSequence': fleet.sequence, 'errors': errors, + 'causalAttributionClaimed': False, 'automaticRequestRetries': 0}) + return observed, diagnostics + + def run_branch(args, nodes, branch): directory = args.evidence / 'H2' / branch directory.mkdir(parents=True) @@ -347,14 +390,15 @@ def witness(label): endpoint = side['endpoint'] local_probe = Probe(binding, endpoint) for version in expected[:2]: - read = fleet.record('partition-explicit-version-GET', local_probe.s3('get_object', { - 'Bucket': bucket, 'Key': version['key'], 'VersionId': version['versionId']})) + read, diagnostics = partition_get(local_probe, fleet, { + 'Bucket': bucket, 'Key': version['key'], 'VersionId': version['versionId']}) available = side['metadataAuthorities'] - int(3 in side['members'] and version['versionId'] == selected['versionId']) if available >= plan['readQuorum']: require(read.get('acknowledged') is True and read.get('resultUnknown') is False and read.get('payload') == version['payload'], 'R-capable partition cannot redeem its exact original ACK') else: validate_unrecoverable_get(read, contract) + require(diagnostics['complete'], 'partition GET diagnostics are incomplete; inspect retained raw response') row = unique_put(local_probe, directory, bucket, 'partition/' + endpoint + '.bin', rows) row['negativeProof'] = negative_put(row['observed']) result.setdefault('partitionTasks', {})[endpoint] = negative_heal(local_probe, fleet, bucket, expected[:2], target, active) diff --git a/test_validation_heal_network.py b/test_validation_heal_network.py index ab5aeb13..0624a6e2 100644 --- a/test_validation_heal_network.py +++ b/test_validation_heal_network.py @@ -10,6 +10,7 @@ import sys import tempfile import unittest +from types import SimpleNamespace from unittest.mock import patch from scripts.validation_heal import network, network_gate as gate, fleet_node @@ -195,6 +196,91 @@ def test_source_branches_and_existing_dispatch_contract_remain_complete(self): self.assertIn(network.CASE_ID + ') driver=network.py', source) +class NetworkGetDiagnosticsTests(unittest.TestCase): + params = {'Bucket': 'owned', 'Key': 'objects/a', 'VersionId': 'exact-version'} + + def fixture(self, *, changed_node=None, journal_failure=False): + class Fleet: + nodes = ('vm000', 'vm001', 'vm002', 'vm003') + def __init__(self): + self.sequence, self.evidence, self.calls, self.process_reads = 0, [], [], {} + def record(self, label, value): + self.sequence += 1 + self.evidence.append((self.sequence, label, copy.deepcopy(value))) + return value + def remote(self, node, action, **params): + self.calls.append((node, action, params)) + if action == 'process': + self.process_reads[node] = self.process_reads.get(node, 0) + 1 + return {'pid': 123, 'startTicks': 7 + int(node == changed_node and self.process_reads[node] > 1), + 'bootId': 'boot', 'unit': 'owned.service', 'executable': '/owned/rustfs', + 'binarySha256': 'a' * 64, 'mountNamespace': 'mnt:1', 'commandSha256': 'command', + 'environmentFileSha256': 'environment', 'settings': {'RUSTFS_SCANNER_ENABLED': 'false'}} + if action != 'logs': raise AssertionError('diagnostics attempted a mutation: ' + action) + if params.get('checkpoint'): + return {'complete': True, 'afterCursor': node + '-before'} + if journal_failure: raise ValueError('original journal read failed') + return {'complete': True, 'beforeCursor': params['afterCursor'], 'afterCursor': node + '-after', + 'hasMore': False, 'journal': json.dumps({'message': 'original error', 'request_id': 'request-500'})} + observed = {'acknowledged': False, 'resultUnknown': False, + 'response': {'Error': {'Code': 'InternalError'}, 'ResponseMetadata': {'HTTPStatusCode': 500, 'RequestId': 'request-500'}}} + probe = SimpleNamespace(binding={'candidateBinarySha256': 'a' * 64}, endpoint_name='node4', + s3=unittest.mock.Mock(return_value=observed)) + return Fleet(), probe, observed + + def test_get500_keeps_four_cursor_windows_and_identity_before_oracle_failure(self): + fleet, probe, original = self.fixture() + observed, diagnostics = network.partition_get(probe, fleet, self.params) + self.assertIs(observed, original) + self.assertTrue(diagnostics['complete']) + self.assertEqual(diagnostics['requestId'], 'request-500') + self.assertFalse(diagnostics['causalAttributionClaimed']) + request = next(row for row in fleet.evidence if row[1] == 'partition-explicit-version-GET') + self.assertEqual(diagnostics['requestEvidenceSequence'], request[0]) + windows = [row for row in fleet.evidence if row[1] == 'partition-GET-journal-window'] + self.assertEqual(len(windows), 4) + self.assertTrue(all(row[0] > request[0] and 'original error' in row[2]['journal'] for row in windows)) + self.assertEqual(fleet.process_reads, dict.fromkeys(fleet.nodes, 2)) + probe.s3.assert_called_once_with('get_object', self.params) + with self.assertRaisesRegex(ValueError, 'below-quorum GET'): + network.validate_unrecoverable_get(observed, {'unrecoverableGet': {'httpStatus': 503, 'code': 'SlowDownRead'}}) + + def test_failed_journal_keeps_response_and_checks_every_after_process(self): + fleet, probe, original = self.fixture(journal_failure=True) + observed, diagnostics = network.partition_get(probe, fleet, self.params) + self.assertIs(observed, original) + self.assertFalse(diagnostics['complete']) + self.assertEqual(diagnostics['errors'][0]['stage'], 'journal') + self.assertEqual(fleet.process_reads, dict.fromkeys(fleet.nodes, 2)) + probe.s3.assert_called_once() + + def test_process_restart_cannot_certify_the_original_request(self): + fleet, probe, _ = self.fixture(changed_node='vm002') + _, diagnostics = network.partition_get(probe, fleet, self.params) + self.assertFalse(diagnostics['complete']) + self.assertEqual(diagnostics['errors'][0]['node'], 'vm002') + self.assertEqual(fleet.process_reads, dict.fromkeys(fleet.nodes, 2)) + probe.s3.assert_called_once() + + def test_request_exception_preserves_original_error_and_still_collects_diagnostics(self): + fleet, probe, _ = self.fixture() + probe.s3.side_effect = ValueError('original request exception') + with self.assertRaisesRegex(ValueError, '^original request exception$'): + network.partition_get(probe, fleet, self.params) + diagnostics = fleet.evidence[-1][2] + self.assertFalse(diagnostics['complete']) + self.assertIsNone(diagnostics['requestEvidenceSequence']) + self.assertEqual(fleet.process_reads, dict.fromkeys(fleet.nodes, 2)) + self.assertEqual(len([row for row in fleet.evidence if row[1] == 'partition-GET-journal-window']), 4) + + def test_wrong_candidate_cannot_send_a_diagnostic_get(self): + fleet, probe, _ = self.fixture() + probe.binding['candidateBinarySha256'] = 'b' * 64 + with self.assertRaisesRegex(ValueError, 'live candidate process'): + network.partition_get(probe, fleet, self.params) + probe.s3.assert_not_called() + + class NetworkExpiryTests(unittest.TestCase): owner = {'runId': 'gh-123-1-h2', 'hostname': 'node1', 'unit': 'owned.service', 'runtimeProfile': 'lock-quorum', 'binarySha256': 'a' * 64} From 4d43a6e96f8b8cfb2e45e1aebc99772d83ae23dc Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 17:47:19 +0800 Subject: [PATCH 234/343] fix(heal): bind network quorum errors and full version recovery --- scripts/validation_heal/network.py | 53 +++++++++++++++++++++++---- test_validation_heal_network.py | 59 ++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 7 deletions(-) diff --git a/scripts/validation_heal/network.py b/scripts/validation_heal/network.py index 7765d29e..241a30a9 100644 --- a/scripts/validation_heal/network.py +++ b/scripts/validation_heal/network.py @@ -64,12 +64,49 @@ } +GET_SOURCE_FILES = { + '072b57b37559fbe626aeda8abf61617fdcbdd926': { + 'rustfs/src/error.rs': '656d38972c06339015c174405571a94f19d8dd64a680e1a7ca631f80205f99a7', + 'crates/ecstore/src/set_disk/ops/object.rs': 'f0f407509b5a12803a191cc1c3a9eb079a15a80a4bbcacca70141150364628ca', + 'crates/heal/src/heal/task/heal_bucket.rs': '2f89bc694d78b68cc2c505fdd854770176fb87cc97d31ea80a1b810f47d8c881'}, + '1c7fc818dd858247c45b39542abd8291b1e6e31b': { + 'rustfs/src/error.rs': 'e7cff0e28978b49d3ee0e8c24da1fa439b3118a7d3f7304edf00dc0535691598', + 'crates/ecstore/src/set_disk/ops/object.rs': 'fdcfda1a8811a93537f9cd47cadac7380bee66b3d14242ca91500bbf38e6bedb', + 'crates/heal/src/heal/task/heal_bucket.rs': 'fa38b21ee7997a909fc5f0671d777fd49983ccaaa18ced85bae8de76b75835ad'}, +} + + def source_reference(candidate_commit): require(isinstance(candidate_commit, str) and candidate_commit in STORAGE_REFERENCES, 'network storage policy is not reviewed for this candidate') return {**copy.deepcopy(REFERENCE), 'candidateCommit': candidate_commit, 'lock': lock_reference(candidate_commit), - 'storage': copy.deepcopy(STORAGE_REFERENCES[candidate_commit])} + 'storage': copy.deepcopy(STORAGE_REFERENCES[candidate_commit]), + 'getPolicy': {'lockBeforeObjectMetadata': True, + 'lockUnavailableGet': {'httpStatus': 503, 'code': 'ServiceUnavailable'}, + 'allVersionRecovery': {'recursive': True}, + 'sourceFiles': {**GET_SOURCE_FILES[candidate_commit], + 'crates/heal/src/heal/channel.rs': '7f669ca43b9559db9b97fba6b419a6c76a94d76acd8526db132d74b7c534d6b0'}}} + + +def negative_get_contract(contract, side, plan): + policy = source_reference(contract['candidateCommit'])['getPolicy'] + require(contract.get('recognized') is True and contract['sourceFiles'].get('rustfs/src/error.rs') + == policy['sourceFiles']['rustfs/src/error.rs'], 'partition GET error mapping differs from exact candidate') + require(type(side.get('lockClients')) is int and 0 <= side['lockClients'] <= 4 + and plan.get('sharedLockQuorum') == 2, 'partition GET has an unknown shared-lock domain') + if side['lockClients'] < plan['sharedLockQuorum']: + return {**contract, 'unrecoverableGet': policy['lockUnavailableGet']} + return contract + + +def recover_versions(probe, fleet, binding, bucket, expected, target, active): + require(len(expected) == 2 and len({entry['key'] for entry in expected}) == 1 + and target['key'] == expected[0]['key'], 'network recovery requires both acknowledged versions of one exact key') + current = run_task(probe, fleet, {'bucket': bucket, 'key': target['key']}, 2, active, + option_overrides={'recursive': True}, status_endpoints=tuple(binding['endpoints'])[1:]) + return validate_version_receipts(current, bucket, expected, 2, + required_repairs={(target['key'], target['versionId'])}) def plan_for(snapshot, target, branch): @@ -180,7 +217,7 @@ def validate_negative_task(current, bucket, expected, target): def negative_heal(probe, fleet, bucket, expected, target, active): path = '/rustfs/admin/v3/heal/' + quote(bucket, safe='') + '/' + quote(target['key'], safe='/') - admission = fleet.record('partition-Heal-admission', probe.admin(path, [], json.dumps(BASE_OPTIONS).encode())) + admission = fleet.record('partition-Heal-admission', probe.admin(path, [], json.dumps({**BASE_OPTIONS, 'recursive': True}).encode())) if admission.get('httpStatus') != 200: # No retry: require an actual structured storage/lock refusal. An # absent token or transport exception alone cannot prove safe refusal. @@ -397,7 +434,12 @@ def witness(label): require(read.get('acknowledged') is True and read.get('resultUnknown') is False and read.get('payload') == version['payload'], 'R-capable partition cannot redeem its exact original ACK') else: - validate_unrecoverable_get(read, contract) + scoped_contract = negative_get_contract(contract, side, plan) + fleet.record('partition-GET-error-contract', {'endpoint': endpoint, 'versionId': version['versionId'], + 'lockClients': side['lockClients'], 'sharedLockQuorum': plan['sharedLockQuorum'], + 'availableSources': available, 'readQuorum': plan['readQuorum'], + 'requiredError': scoped_contract['unrecoverableGet']}) + validate_unrecoverable_get(read, scoped_contract) require(diagnostics['complete'], 'partition GET diagnostics are incomplete; inspect retained raw response') row = unique_put(local_probe, directory, bucket, 'partition/' + endpoint + '.bin', rows) row['negativeProof'] = negative_put(row['observed']) @@ -420,10 +462,7 @@ def witness(label): fleet.start() require(physical_content(scene) == physical_content(fleet.snapshot('all-peer-return-before-independent-repair')), 'network recovery repaired a target before the independent product task') - repaired = run_task(probe, fleet, {'bucket': bucket, 'key': selected['key']}, 2, recovery_active, - status_endpoints=tuple(binding['endpoints'])[1:]) - result['recoveryResponsibility'] = validate_version_receipts(repaired, bucket, expected[:2], 2, - required_repairs={(selected['key'], selected['versionId'])}) + result['recoveryResponsibility'] = recover_versions(probe, fleet, binding, bucket, expected[:2], selected, recovery_active) idle_fleet(fleet, binding) fleet.stop() after = fleet.snapshot('physical-rebuilt-before-any-recovered-GET', quiescent=True) diff --git a/test_validation_heal_network.py b/test_validation_heal_network.py index 0624a6e2..8569faaa 100644 --- a/test_validation_heal_network.py +++ b/test_validation_heal_network.py @@ -281,6 +281,65 @@ def test_wrong_candidate_cannot_send_a_diagnostic_get(self): probe.s3.assert_not_called() +class NetworkScopedContractTests(unittest.TestCase): + def contract(self): + source = '1c7fc818dd858247c45b39542abd8291b1e6e31b' + return {'candidateCommit': source, 'recognized': True, + 'sourceFiles': {'rustfs/src/error.rs': network.GET_SOURCE_FILES[source]['rustfs/src/error.rs']}, + 'unrecoverableGet': {'httpStatus': 503, 'code': 'SlowDownRead'}} + + def test_lock_and_data_losses_require_different_exact503_codes(self): + for clients, expected in [(1, 'ServiceUnavailable'), (2, 'SlowDownRead'), (3, 'SlowDownRead')]: + contract = network.negative_get_contract(self.contract(), {'lockClients': clients}, {'sharedLockQuorum': 2}) + for status, code in [(503, 'ServiceUnavailable'), (503, 'SlowDownRead'), (500, 'InternalError')]: + observed = {'acknowledged': False, 'resultUnknown': False, + 'response': {'Error': {'Code': code}, 'ResponseMetadata': {'HTTPStatusCode': status}}} + with self.subTest(clients=clients, status=status, code=code): + if status == 503 and code == expected: + network.validate_unrecoverable_get(observed, contract) + else: + with self.assertRaises(ValueError): network.validate_unrecoverable_get(observed, contract) + + def test_foreign_source_or_unknown_lock_domain_cannot_select_an_error(self): + for kind in ('source', 'digest', 'unrecognized', 'boolean', 'quorum'): + contract, side, plan = self.contract(), {'lockClients': 1}, {'sharedLockQuorum': 2} + if kind == 'source': contract['candidateCommit'] = 'f' * 40 + elif kind == 'digest': contract['sourceFiles']['rustfs/src/error.rs'] = 'f' * 64 + elif kind == 'unrecognized': contract['recognized'] = False + elif kind == 'boolean': side['lockClients'] = True + else: plan['sharedLockQuorum'] = 1 + with self.subTest(kind=kind), self.assertRaises(ValueError): network.negative_get_contract(contract, side, plan) + + def test_negative_heal_requests_both_versions_without_changing_safe_refusal(self): + fleet = SimpleNamespace(record=lambda label, value: value) + probe = SimpleNamespace(admin=unittest.mock.Mock(return_value={'httpStatus': 503, + 'response': {'Code': 'ServiceUnavailable'}})) + result = network.negative_heal(probe, fleet, 'owned', [], {'key': 'objects/a'}, []) + self.assertTrue(json.loads(probe.admin.call_args.args[2])['recursive']) + self.assertTrue(result['admissionRefused']) + self.assertFalse(result['repairClaimed']) + + def test_recovery_requests_all_versions_and_keeps_the_two_receipt_oracle(self): + status, expected, target = NetworkTests().negative('repaired') + expected = [{**entry, 'deleteMarker': False} for entry in expected] + status['outcome']['counters'].update(healed=1, unchanged=1, skipped=0) + binding = {'endpoints': {'node1': 'one', 'node2': 'two', 'node3': 'three', 'node4': 'four'}} + for missing in (False, True): + current = copy.deepcopy(status) + if missing: + current['outcome']['objects'] = current['outcome']['objects'][1:] + current['outcome']['counters'].update(processed=1, unchanged=0) + with self.subTest(missing=missing), patch.object(network, 'run_task', return_value=current) as task: + if missing: + with self.assertRaisesRegex(ValueError, 'omitted or duplicated'): + network.recover_versions(None, None, binding, 'owned', expected, target, []) + else: + proof = network.recover_versions(None, None, binding, 'owned', expected, target, []) + self.assertEqual((proof['receipts'], proof['repaired']), (2, 1)) + self.assertEqual(task.call_args.kwargs['option_overrides'], {'recursive': True}) + self.assertEqual(task.call_args.kwargs['status_endpoints'], ('node2', 'node3', 'node4')) + + class NetworkExpiryTests(unittest.TestCase): owner = {'runId': 'gh-123-1-h2', 'hostname': 'node1', 'unit': 'owned.service', 'runtimeProfile': 'lock-quorum', 'binarySha256': 'a' * 64} From 008f0d8d7ce4eb050cbfe0d037092a7055b8f212 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 18:15:26 +0800 Subject: [PATCH 235/343] test(heal): cover network version history and null identities --- docs/validation-migration-map.md | 23 +++++ scripts/validation_heal/network.py | 122 ++++++++++++++++++----- test_validation_heal_network.py | 152 ++++++++++++++++++++++++++--- 3 files changed, 256 insertions(+), 41 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 01c7fedb..d2671258 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2024,3 +2024,26 @@ Recovery diagnostic `38704914473/1` ran on helper `9c01cb8` after the independen The 2026-10-15 source reconciliation restores the omitted J02 `readonly-terminal-failure` branch from frozen source script `524eb5b32c3236497fdd8a9271df0494fa3d9974`. Heal retains 84 canonical cases; its operation inventory is now 302 branches and 534 applicable topology templates. The current-token missing-shard fixture uses an actual private service-namespace read-only gate and proves EROFS before admission. Its original task must retain one permanent, unbound key-level failure and exactly one attempt; release must leave the shard missing. A distinct compensating token must repair the original current ACK version at the protected-control incarnation, with physical reconstruction and bitrot evidence captured before GETs. Full public versions/tags and the original failed token are checked again after compensation. Lost gate acknowledgements retain the gate registration for cleanup. This is local implementation evidence, not a live PASS; original corrupt-metadata Unknown and grace Deferred/new-token source procedures remain incomplete, so all current diagnostics passing still cannot claim full J02 completion. + +### I03 fixture completeness follow-up (2026-10-19) + +The frozen source is `524eb5b32c3236497fdd8a9271df0494fa3d9974`. +Its I03 fixture contract includes DB, DV and DL. Run `39232916911/1` +(helper `4d43a6e96f8b8cfb2e45e1aebc99772d83ae23dc`, candidate +`1c7fc818dd858247c45b39542abd8291b1e6e31b`) reported five passing +branches. Independent archive, request, ACK/task/physical/public and cleanup +checks accept that replay's four-key, two-version scope. It does not complete +the frozen fixture contract. The accepted migration total remains **104/158**. +Original failures and original reports remain unchanged. + +| Fixture | Implementation and evidence | Remaining acceptance work | +| --- | --- | --- | +| DV | Reuses auto-test `seed(DV)` and `suspend_fixture`: four keys with three distinct non-null data versions each; real suspended overwrites, null marker creation/removal ACKs; retained live/null marker identities; current and historical markers. Versioning returns to Enabled before partition. The fault targets a retained null data version. Every selected key version participates in GET, negative Heal and complete recursive recovery. All 15 retained data versions receive full physical shard checks; markers receive exact S3 refusal/header checks. HEAD preserves version identity, length, ETag, Last-Modified and metadata. | This expanded fixture still needs a new live run and independent evidence acceptance. It does not claim a partition while the bucket remains Suspended. | +| DB | Existing network objects are external single-part objects of approximately 5 MiB. | Add the frozen zero/one, 4 KiB, actual inline/EC/MPU boundary matrix and physical shape witnesses. External-only controls do not establish DB coverage. | +| DL | Existing partition workload serially probes exact-version GET, unique negative PUT and Heal, followed by explicit write resolution and peer inventories. | Add bounded concurrent PUT/GET/HEAD/LIST/COPY/DELETE, version and MPU operations, with per-request durable ACKs, increasing per-key nonce and resolution of unknown outcomes. Serial PUT/GET is insufficient. | + +The expanded DV fixture stays within the existing 128-identity/512-MiB +inventory limits. Network expiry remains 1,200 seconds; partition task deadlines, +request retry policy, quorum error mapping and complete-version receipt checks +are preserved. Source fixture completeness must be checked separately from a +raw five-branch PASS before accepting I03 as a migrated canonical PASS. diff --git a/scripts/validation_heal/network.py b/scripts/validation_heal/network.py index 241a30a9..26ddb05a 100644 --- a/scripts/validation_heal/network.py +++ b/scripts/validation_heal/network.py @@ -17,12 +17,13 @@ try: from .heal_probe import Probe, canonical, compare_inventories, payload, require, save - from .heal_workload import Recorder, RecordedCallError, verify_seed + from .heal_workload import Recorder, RecordedCallError, seed, verify_seed from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology from .inspector import verify as verify_inspector from .matrix import select from .shards import files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort - from .versions import public_version_id, validate_version_receipts, verify_quorum_contract, validate_unrecoverable_get + from .versions import public_version_id, suspend_fixture, validate_version_receipts, verify_quorum_contract, validate_unrecoverable_get + from .markers import add_marker, verify_markers from .write_quorum import expected_reads, resolve_put, negative_put, STORAGE_REFERENCE from .quorum_boundary import LOCK_REFERENCE, lock_reference from .stop import idle_fleet @@ -30,12 +31,13 @@ from .scanner_repair import JournalWindow except ImportError: from heal_probe import Probe, canonical, compare_inventories, payload, require, save - from heal_workload import Recorder, RecordedCallError, verify_seed + from heal_workload import Recorder, RecordedCallError, seed, verify_seed from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology from inspector import verify as verify_inspector from matrix import select from shards import files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort - from versions import public_version_id, validate_version_receipts, verify_quorum_contract, validate_unrecoverable_get + from versions import public_version_id, suspend_fixture, validate_version_receipts, verify_quorum_contract, validate_unrecoverable_get + from markers import add_marker, verify_markers from write_quorum import expected_reads, resolve_put, negative_put, STORAGE_REFERENCE from quorum_boundary import LOCK_REFERENCE, lock_reference from stop import idle_fleet @@ -51,7 +53,9 @@ 'network': {'servicePort': 9000, 'managementPort': 22, 'ttlSeconds': TTL_SECONDS, 'rules': 'one owned nft table per host; exact peer IPv4 and TCP source/destination port', 'mutation': 'all expiry timers armed before the first atomic table transaction'}, - 'load': {'fixtureObjects': 4, 'versionsPerObject': 2, 'partitionPUTs': 2, 'recoveredPUTs': 1, + 'load': {'fixtureObjects': 4, 'versionsPerObject': 3, 'retainedNullVersions': 4, + 'currentAndHistoricalMarkers': True, 'partitionVersioning': 'Enabled after witnessed Suspended/null operations', + 'partitionPUTs': 2, 'recoveredPUTs': 1, 'automaticRetries': 0, 'negativeTaskDeadlineSeconds': 330}, 'writePolicy': 'three-peer groups retain R but use independent private gates to measure W-1; two-peer groups lose W and exclusive lock quorum'} @@ -101,14 +105,80 @@ def negative_get_contract(contract, side, plan): def recover_versions(probe, fleet, binding, bucket, expected, target, active): - require(len(expected) == 2 and len({entry['key'] for entry in expected}) == 1 - and target['key'] == expected[0]['key'], 'network recovery requires both acknowledged versions of one exact key') + require(len(expected) >= 3 and len({entry['key'] for entry in expected}) == 1 + and len({entry['versionId'] for entry in expected}) == len(expected) + and (target['key'], target['versionId']) in {(entry['key'], entry['versionId']) for entry in expected} + and sum(not entry['deleteMarker'] and entry['versionId'] != 'null' for entry in expected) >= 3, + 'network recovery requires all acknowledged versions of one exact three-version key') current = run_task(probe, fleet, {'bucket': bucket, 'key': target['key']}, 2, active, option_overrides={'recursive': True}, status_endpoints=tuple(binding['endpoints'])[1:]) return validate_version_receipts(current, bucket, expected, 2, required_repairs={(target['key'], target['versionId'])}) +def seed_versions(recorder, fleet, bucket): + expected = seed(recorder, bucket, 'DV', 4, [SIZE]) + expected = suspend_fixture(recorder, bucket, expected) + suspended = fleet.record('suspended-null-S3-baseline', recorder.probe.inventory(bucket)) + fleet.record('suspended-null-ACK-manifest', {'expected': expected, **verify_seed(expected, suspended)}) + # Null overwrites and marker removals are real S3 operations. Re-enable + # before the partition so new writes must return distinct non-null IDs. + recorder.call('put_bucket_versioning', {'Bucket': bucket, 'VersioningConfiguration': {'Status': 'Enabled'}}) + require(recorder.call('get_bucket_versioning', {'Bucket': bucket}).get('Status') == 'Enabled', + 'network fixture did not persist re-enabled versioning') + key = 'objects/0000.bin' + expected = add_marker(expected, key, recorder.call('delete_object', {'Bucket': bucket, 'Key': key})) + fleet.record('version-fixture-coverage', version_coverage(expected)) + return expected + + +def version_coverage(expected): + keys = {f'objects/{index:04d}.bin' for index in range(4)} + require({entry['key'] for entry in expected} == keys + and len({(entry['key'], entry['versionId']) for entry in expected}) == len(expected), + 'network fixture omits a key or duplicates a version identity') + counts = {} + for key in sorted(keys): + entries = [entry for entry in expected if entry['key'] == key] + data = [entry for entry in entries if not entry['deleteMarker'] and entry['versionId'] != 'null'] + require(len(data) == 3 and all(isinstance(entry['versionId'], str) and entry['versionId'] for entry in data) + and len({entry['payload']['sha256'] for entry in data}) == 3 + and sum(entry['isLatest'] is True for entry in entries) == 1 + and sum(entry['versionId'] == 'null' for entry in entries) == 1, + 'network fixture lacks three distinct data ACKs, one latest identity or its retained null version') + counts[key] = len(entries) + require(any(entry['deleteMarker'] and entry['isLatest'] for entry in expected) + and any(entry['deleteMarker'] and not entry['isLatest'] for entry in expected) + and any(entry['versionId'] == 'null' and entry['deleteMarker'] for entry in expected) + and any(entry['versionId'] == 'null' and not entry['deleteMarker'] for entry in expected), + 'network fixture lacks current/historical markers or live/deleted null identities') + return {'acknowledgedIdentitiesPerKey': counts, 'threeDistinctDataVersionsPerKey': True, + 'currentAndHistoricalMarkers': True, 'retainedLiveAndMarkerNullVersions': True, + 'partitionVersioning': 'Enabled', 'suspendedPartitionClaimed': False, + 'remainingSourceFixtures': ['DB boundary matrix', 'DL concurrent operation matrix'], + 'canonicalCasePassClaimed': False} + + +def version_heads(probe, fleet, bucket, expected, label): + result = [] + for entry in expected: + if entry['deleteMarker']: + continue + observed = fleet.record(label, probe.s3('head_object', { + 'Bucket': bucket, 'Key': entry['key'], 'VersionId': entry['versionId']})) + response = observed.get('response') or {} + require(observed.get('acknowledged') is True and observed.get('resultUnknown') is False + and response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 200 + and response.get('ContentLength') == entry['payload']['size'] + and response.get('ETag') and response.get('LastModified') + and response.get('VersionId', 'null') == entry['versionId'], + 'version HEAD lost its ACK identity, length, ETag or Last-Modified') + result.append({'key': entry['key'], 'versionId': entry['versionId'], + 'properties': {name: response.get(name) for name in + ('ContentLength', 'ETag', 'LastModified', 'ContentType', 'Metadata')}}) + return result + + def plan_for(snapshot, target, branch): nodes = list(snapshot) geometry = target['logical']['erasure'] @@ -367,28 +437,21 @@ def run_branch(args, nodes, branch): try: fleet.deploy(args.binary_sha, bucket, access, secret) recorder = Recorder(probe, directory / 'fixture') - recorder.call('create_bucket', {'Bucket': bucket}) - recorder.call('put_bucket_versioning', {'Bucket': bucket, 'VersioningConfiguration': {'Status': 'Enabled'}}) - expected = [] - for index in range(4): - key = f'objects/{index:04d}.bin' - for version in range(2): - body = payload(f'{bucket}/{key}/{version}', SIZE) - ack = recorder.call('put_object', {'Bucket': bucket, 'Key': key, - 'ContentType': 'application/octet-stream', 'Metadata': {'heal-fixture': 'DV', 'nonce': f'{index}-{version}'}, - 'Tagging': 'suite=heal&fixture=DV'}, body) - require(ack.get('VersionId') not in {None, 'null'}, 'versioned network fixture has no exact ACK version') - expected.append({'key': key, 'versionId': ack['VersionId'], 'isLatest': version == 1, 'deleteMarker': False, - 'payload': {'size': SIZE, 'sha256': hashlib.sha256(body).hexdigest()}}) + expected = seed_versions(recorder, fleet, bucket) baseline = fleet.record('all-version-S3-baseline', probe.inventory(bucket)) fleet.record('exact-ACK-manifest', {'expected': expected, **verify_seed(expected, baseline)}) + original_heads = version_heads(probe, fleet, bucket, expected, 'original-version-HEAD') + for key in sorted({entry['key'] for entry in expected if entry['deleteMarker']}): + verify_markers(probe, fleet, bucket, key, expected) fleet.stop() original = fleet.snapshot('physical-baseline', quiescent=True) validate_topology(original, 'H2') - selected = expected[1] - target = map_target(original, bucket, selected['key'], 'data', 1, selected['versionId']) + key_versions = [entry for entry in expected if entry['key'] == 'objects/0001.bin'] + selected = next(entry for entry in key_versions if not entry['deleteMarker'] and entry['versionId'] == 'null') + target = map_target(original, bucket, selected['key'], 'data', 1, selected['versionId'], latest_required=False) target = {**target, **next(row for row in target['cohort'] if row['node'] == nodes[3])} - cohorts = [map_target(original, bucket, entry['key'], 'data', 1, entry['versionId'], latest_required=False) for entry in expected] + cohorts = [map_target(original, bucket, entry['key'], 'data', 1, entry['versionId'], latest_required=False) + for entry in expected if not entry['deleteMarker']] plan = fleet.record('actual-partition-quorum-plan', plan_for(original, target, branch)) params = {'disk': target['disk'], 'faultId': 'I03-missing-target'} registration = target['node'], params @@ -426,7 +489,7 @@ def witness(label): for side in plan['sides']: endpoint = side['endpoint'] local_probe = Probe(binding, endpoint) - for version in expected[:2]: + for version in key_versions: read, diagnostics = partition_get(local_probe, fleet, { 'Bucket': bucket, 'Key': version['key'], 'VersionId': version['versionId']}) available = side['metadataAuthorities'] - int(3 in side['members'] and version['versionId'] == selected['versionId']) @@ -443,7 +506,7 @@ def witness(label): require(diagnostics['complete'], 'partition GET diagnostics are incomplete; inspect retained raw response') row = unique_put(local_probe, directory, bucket, 'partition/' + endpoint + '.bin', rows) row['negativeProof'] = negative_put(row['observed']) - result.setdefault('partitionTasks', {})[endpoint] = negative_heal(local_probe, fleet, bucket, expected[:2], target, active) + result.setdefault('partitionTasks', {})[endpoint] = negative_heal(local_probe, fleet, bucket, key_versions, target, active) status = fleet.record('partition-raw-background', local_probe.admin('/rustfs/admin/v3/background-heal/status', [], b'')) require(status.get('httpStatus') == 200 and isinstance(status.get('response'), dict), 'partition background status unavailable') result.setdefault('coverage', {})[endpoint] = validate_partial(status['response'], len(side['members'])) @@ -462,7 +525,7 @@ def witness(label): fleet.start() require(physical_content(scene) == physical_content(fleet.snapshot('all-peer-return-before-independent-repair')), 'network recovery repaired a target before the independent product task') - result['recoveryResponsibility'] = recover_versions(probe, fleet, binding, bucket, expected[:2], selected, recovery_active) + result['recoveryResponsibility'] = recover_versions(probe, fleet, binding, bucket, key_versions, selected, recovery_active) idle_fleet(fleet, binding) fleet.stop() after = fleet.snapshot('physical-rebuilt-before-any-recovered-GET', quiescent=True) @@ -480,13 +543,18 @@ def witness(label): 'deleteMarker': False, 'payload': released['payload']}] inventories = [] for endpoint in binding['endpoints']: - inventory = fleet.record('all-peer-complete-final-S3', Probe(binding, endpoint).inventory(bucket)) + peer = Probe(binding, endpoint) + inventory = fleet.record('all-peer-complete-final-S3', peer.inventory(bucket)) verify_seed(augmented, inventory) + require(version_heads(peer, fleet, bucket, expected, 'all-peer-original-version-HEAD') == original_heads, + 'network recovery changed an original version HEAD or Last-Modified') subset = {**inventory, 'entries': [row for row in inventory['entries'] if row['key'] != released['key']]} compare_inventories(baseline, subset) inventories.append(inventory) for inventory in inventories[1:]: compare_inventories(inventories[0], inventory) + result['markers'] = [verify_markers(probe, fleet, bucket, key, expected) + for key in sorted({entry['key'] for entry in expected if entry['deleteMarker']})] require(all(row['properties'].get('Metadata') == {'heal-fixture': 'I03', 'nonce': released['key']} and row['tags'] == [{'Key': 'fixture', 'Value': 'I03'}, {'Key': 'suite', 'Value': 'heal'}] for inventory in inventories for row in inventory['entries'] if row['key'] == released['key']), diff --git a/test_validation_heal_network.py b/test_validation_heal_network.py index 8569faaa..2ee9a136 100644 --- a/test_validation_heal_network.py +++ b/test_validation_heal_network.py @@ -2,6 +2,7 @@ import copy import errno +import hashlib import json import os from pathlib import Path @@ -319,25 +320,148 @@ def test_negative_heal_requests_both_versions_without_changing_safe_refusal(self self.assertTrue(result['admissionRefused']) self.assertFalse(result['repairClaimed']) - def test_recovery_requests_all_versions_and_keeps_the_two_receipt_oracle(self): + def test_recovery_requests_every_historical_and_null_version(self): status, expected, target = NetworkTests().negative('repaired') expected = [{**entry, 'deleteMarker': False} for entry in expected] - status['outcome']['counters'].update(healed=1, unchanged=1, skipped=0) + for version in ('v2', 'null'): + expected.append({'key': 'objects/a', 'versionId': version, 'deleteMarker': False}) + status['outcome']['objects'].append({'identity': {'bucket': 'owned', 'object': 'objects/a', + 'versionId': None if version == 'null' else version, 'bucketIncarnationId': 'incarnation', + 'poolIndex': 0, 'setIndex': 0}, 'disposition': {'state': 'verified_healthy'}}) + status['outcome']['counters'].update(processed=4, healed=1, unchanged=3, skipped=0) binding = {'endpoints': {'node1': 'one', 'node2': 'two', 'node3': 'three', 'node4': 'four'}} - for missing in (False, True): - current = copy.deepcopy(status) - if missing: - current['outcome']['objects'] = current['outcome']['objects'][1:] - current['outcome']['counters'].update(processed=1, unchanged=0) - with self.subTest(missing=missing), patch.object(network, 'run_task', return_value=current) as task: + for target_version in ('v1', 'null'): + for missing in (False, True): + current = copy.deepcopy(status) + for row in current['outcome']['objects']: + row['disposition']['state'] = ('repaired' if network.public_version_id(row['identity']['versionId']) + == target_version else 'verified_healthy') if missing: - with self.assertRaisesRegex(ValueError, 'omitted or duplicated'): - network.recover_versions(None, None, binding, 'owned', expected, target, []) + current['outcome']['objects'] = current['outcome']['objects'][1:] + current['outcome']['counters'].update(processed=3, unchanged=2) + selected = {**target, 'versionId': target_version} + with self.subTest(target=target_version, missing=missing), patch.object(network, 'run_task', return_value=current) as task: + if missing: + with self.assertRaisesRegex(ValueError, 'omitted or duplicated'): + network.recover_versions(None, None, binding, 'owned', expected, selected, []) + else: + proof = network.recover_versions(None, None, binding, 'owned', expected, selected, []) + self.assertEqual((proof['receipts'], proof['repaired']), (4, 1)) + self.assertEqual(task.call_args.kwargs['option_overrides'], {'recursive': True}) + self.assertEqual(task.call_args.kwargs['status_endpoints'], ('node2', 'node3', 'node4')) + + + +class NetworkVersionFixtureTests(unittest.TestCase): + def fixture(self, *, wrong_ack=False): + class Recorder: + def __init__(self): + self.mode, self.rows, self.calls, self.nonce = None, [], [], 0 + self.probe = SimpleNamespace(binding={'bucketPrefix': 'owned'}, inventory=self.inventory) + def inventory(self, bucket): + return {'entries': copy.deepcopy(self.rows)} + def call(self, operation, params, body=None): + self.calls.append((operation, copy.deepcopy(params))) + self.nonce += 1 + if operation == 'put_bucket_versioning': + self.mode = params['VersioningConfiguration']['Status'] + if operation == 'get_bucket_versioning': return {'Status': self.mode} + if operation not in {'put_object', 'delete_object'}: return {} + key = params['Key'] + if operation == 'delete_object' and 'VersionId' in params: + self.rows = [entry for entry in self.rows if (entry['key'], entry['versionId']) != (key, params['VersionId'])] + same = [entry for entry in self.rows if entry['key'] == key] + if same: same[-1]['isLatest'] = True + return {'VersionId': params['VersionId'], 'DeleteMarker': True} + version = f'v{self.nonce}' if self.mode == 'Enabled' else 'null' + if version == 'null': + self.rows = [entry for entry in self.rows if (entry['key'], entry['versionId']) != (key, 'null')] + for entry in self.rows: + if entry['key'] == key: entry['isLatest'] = False + marker = operation == 'delete_object' + entry = {'key': key, 'versionId': version, 'isLatest': True, 'deleteMarker': marker} + if body is not None: entry['payload'] = {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()} + self.rows.append(entry) + return {'VersionId': 'null' if wrong_ack and not marker else version, + 'DeleteMarker': marker, 'ResponseMetadata': {'HTTPStatusCode': 204 if marker else 200}} + recorder = Recorder() + evidence = [] + fleet = SimpleNamespace(record=lambda label, value: evidence.append((label, copy.deepcopy(value))) or value) + return recorder, fleet, evidence + + def seed(self): + recorder, fleet, evidence = self.fixture() + with patch.object(network, 'SIZE', 97), patch('scripts.validation_heal.versions.VERSION_SIZE', 123): + expected = network.seed_versions(recorder, fleet, 'owned') + return expected, recorder, evidence + + def test_real_seed_uses_three_versions_and_suspended_null_delete_ack_sequence(self): + expected, recorder, evidence = self.seed() + coverage = network.version_coverage(expected) + self.assertEqual(len(expected), 19) + self.assertEqual(recorder.mode, 'Enabled') + self.assertEqual(len([entry for entry in expected if not entry['deleteMarker']]), 15) + self.assertFalse(coverage['suspendedPartitionClaimed']) + self.assertFalse(coverage['canonicalCasePassClaimed']) + self.assertEqual(len([params for op, params in recorder.calls if op == 'delete_object' and params.get('VersionId') == 'null']), 4) + suspended = next(value for label, value in evidence if label == 'suspended-null-ACK-manifest') + self.assertTrue(suspended['seedAckAndFullGetAgree']) + self.assertEqual(len(suspended['expected']), 18) + self.assertEqual([params['VersioningConfiguration']['Status'] for op, params in recorder.calls + if op == 'put_bucket_versioning'], ['Enabled', 'Suspended', 'Enabled']) + target_versions = [entry for entry in expected if entry['key'] == 'objects/0001.bin'] + self.assertEqual(len(target_versions), 4) + self.assertTrue(any(entry['versionId'] == 'null' and entry['isLatest'] for entry in target_versions)) + + def test_missing_fixture_dimensions_or_duplicate_identities_cannot_claim_coverage(self): + expected, _, _ = self.seed() + for kind in ('third-version', 'duplicate', 'same-content', 'current-marker', 'historical-marker', 'null', 'latest'): + broken = copy.deepcopy(expected) + if kind == 'third-version': broken.pop(0) + elif kind == 'duplicate': broken.append(copy.deepcopy(broken[0])) + elif kind == 'same-content': broken[1]['payload'] = broken[0]['payload'] + elif kind == 'current-marker': + broken = [entry for entry in broken if not (entry['deleteMarker'] and entry['isLatest'])] + elif kind == 'historical-marker': + broken = [entry for entry in broken if not (entry['deleteMarker'] and not entry['isLatest'])] + elif kind == 'null': broken = [entry for entry in broken if entry['versionId'] != 'null'] + else: broken[0]['isLatest'] = True + with self.subTest(kind=kind), self.assertRaises(ValueError): network.version_coverage(broken) + + def test_seed_refuses_missing_enabled_version_ack_before_faults(self): + recorder, fleet, _ = self.fixture(wrong_ack=True) + with patch.object(network, 'SIZE', 97), self.assertRaisesRegex(ValueError, 'real version ID'): + network.seed_versions(recorder, fleet, 'owned') + self.assertEqual(len([op for op, params in recorder.calls if op == 'put_object']), 1) + + def test_two_version_or_foreign_target_recovery_fails_before_task_admission(self): + expected, _, _ = self.seed() + versions = [entry for entry in expected if entry['key'] == 'objects/0001.bin'] + for selected, target in [(versions[:2], versions[0]), (versions, {'key': versions[0]['key'], 'versionId': 'foreign'})]: + with patch.object(network, 'run_task') as task, self.assertRaises(ValueError): + network.recover_versions(None, None, {'endpoints': {}}, 'owned', selected, target, []) + task.assert_not_called() + + def test_head_identity_length_and_last_modified_are_required(self): + entry = {'key': 'objects/a', 'versionId': 'v1', 'deleteMarker': False, 'payload': {'size': 123}} + original = {'acknowledged': True, 'resultUnknown': False, + 'response': {'VersionId': 'v1', 'ContentLength': 123, 'ETag': 'etag', 'LastModified': 'original-time', + 'ResponseMetadata': {'HTTPStatusCode': 200}}} + fleet = SimpleNamespace(record=lambda label, value: value) + for kind in ('valid', 'unknown', 'wrong-version', 'short', 'partial', 'no-modification-time', 'no-etag'): + observed = copy.deepcopy(original) + if kind == 'unknown': observed['resultUnknown'] = True + elif kind == 'wrong-version': observed['response']['VersionId'] = 'v2' + elif kind == 'short': observed['response']['ContentLength'] = 122 + elif kind == 'partial': observed['response']['ResponseMetadata']['HTTPStatusCode'] = 206 + elif kind == 'no-modification-time': observed['response'].pop('LastModified') + elif kind == 'no-etag': observed['response'].pop('ETag') + probe = SimpleNamespace(s3=unittest.mock.Mock(return_value=observed)) + with self.subTest(kind=kind): + if kind == 'valid': self.assertEqual(len(network.version_heads(probe, fleet, 'owned', [entry], 'head')), 1) else: - proof = network.recover_versions(None, None, binding, 'owned', expected, target, []) - self.assertEqual((proof['receipts'], proof['repaired']), (2, 1)) - self.assertEqual(task.call_args.kwargs['option_overrides'], {'recursive': True}) - self.assertEqual(task.call_args.kwargs['status_endpoints'], ('node2', 'node3', 'node4')) + with self.assertRaises(ValueError): network.version_heads(probe, fleet, 'owned', [entry], 'head') + probe.s3.assert_called_once_with('head_object', {'Bucket': 'owned', 'Key': 'objects/a', 'VersionId': 'v1'}) class NetworkExpiryTests(unittest.TestCase): From a2bc3cee9d8a9de6190375aed503b78a56378c02 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 18:56:52 +0800 Subject: [PATCH 236/343] test(heal): preserve network boundary fixture coverage --- docs/validation-migration-map.md | 14 +- scripts/validation_heal/boundaries.py | 10 +- scripts/validation_heal/healthy.py | 5 +- scripts/validation_heal/network.py | 60 ++++++-- scripts/validation_heal/network_fixtures.py | 95 +++++++++++++ test_validation_heal_network_fixtures.py | 149 ++++++++++++++++++++ 6 files changed, 318 insertions(+), 15 deletions(-) create mode 100644 scripts/validation_heal/network_fixtures.py create mode 100644 test_validation_heal_network_fixtures.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index d2671258..f120b049 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2039,7 +2039,7 @@ Original failures and original reports remain unchanged. | Fixture | Implementation and evidence | Remaining acceptance work | | --- | --- | --- | | DV | Reuses auto-test `seed(DV)` and `suspend_fixture`: four keys with three distinct non-null data versions each; real suspended overwrites, null marker creation/removal ACKs; retained live/null marker identities; current and historical markers. Versioning returns to Enabled before partition. The fault targets a retained null data version. Every selected key version participates in GET, negative Heal and complete recursive recovery. All 15 retained data versions receive full physical shard checks; markers receive exact S3 refusal/header checks. HEAD preserves version identity, length, ETag, Last-Modified and metadata. | This expanded fixture still needs a new live run and independent evidence acceptance. It does not claim a partition while the bucket remains Suspended. | -| DB | Existing network objects are external single-part objects of approximately 5 MiB. | Add the frozen zero/one, 4 KiB, actual inline/EC/MPU boundary matrix and physical shape witnesses. External-only controls do not establish DB coverage. | +| DB | Adds a separate never-versioned run-owned bucket with the default 128 controls. The exact candidate size contract and actual EC:8 process/object geometry calibrate zero/one, 4 KiB, inline, EC block and source-bound 5-MiB MPU minimum minus/exact/plus sizes. All 128 controls require decoded full-set shape witnesses; every distinct size has partition GET, physical checksum/no-part and Range evidence. Original/fault/rejoin/repaired snapshots explicitly include both primary and DB buckets. All peers must preserve complete inventories and stable original HEADs. | Fresh live execution and independent acceptance remain required. The MPU-size controls are single PUT objects; they do not claim completed multipart protocol coverage, which remains part of DL. | | DL | Existing partition workload serially probes exact-version GET, unique negative PUT and Heal, followed by explicit write resolution and peer inventories. | Add bounded concurrent PUT/GET/HEAD/LIST/COPY/DELETE, version and MPU operations, with per-request durable ACKs, increasing per-key nonce and resolution of unknown outcomes. Serial PUT/GET is insufficient. | The expanded DV fixture stays within the existing 128-identity/512-MiB @@ -2047,3 +2047,15 @@ inventory limits. Network expiry remains 1,200 seconds; partition task deadlines request retry policy, quorum error mapping and complete-version receipt checks are preserved. Source fixture completeness must be checked separately from a raw five-branch PASS before accepting I03 as a migrated canonical PASS. + + +I03 DB implementation detail (2026-10-19): the existing `boundary_sizes` +helper accepts the explicit EC:8 calibration only when every live process +confirms the exact storage class and the decoded object has eight data and +eight parity members. Default callers keep their original environment guard. +The reviewed multipart minimum is bound to the fixed candidate's full +`set_disk/mod.rs` digest. Current candidate/prior physical calibration yields +15 distinct sizes and 200,127,457 payload bytes for 128 controls, within the +existing 512-MiB budget. This read-only preparation is not fresh DB live evidence. +The existing node-side snapshot bucket guard continues to require the primary +bucket and allows only its owned suffixes; no broader host path is introduced. diff --git a/scripts/validation_heal/boundaries.py b/scripts/validation_heal/boundaries.py index 3067c2c3..bcf23abf 100644 --- a/scripts/validation_heal/boundaries.py +++ b/scripts/validation_heal/boundaries.py @@ -32,12 +32,16 @@ def verify_size_contract(manifest, candidate): return contract -def boundary_sizes(contract, target, processes): +def boundary_sizes(contract, target, processes, *, standard_class=None): + require(standard_class in {None, "EC:8"}, "unreviewed boundary storage class") require(processes and all(entry.get("storageClassEnvironment") == { - "RUSTFS_STORAGE_CLASS_INLINE_BLOCK": None, "RUSTFS_STORAGE_CLASS_STANDARD": None} - for entry in processes.values()), "default size policy requires actual process environment confirmation") + "RUSTFS_STORAGE_CLASS_INLINE_BLOCK": None, "RUSTFS_STORAGE_CLASS_STANDARD": standard_class} + for entry in processes.values()), "size policy requires actual process environment confirmation") geometry = target["logical"]["erasure"] data, block = geometry["data_blocks"], geometry["block_size"] + require(standard_class is None or data == geometry.get("parity_blocks") == 8 + and len(geometry.get("distribution", [])) == 16, + "EC:8 boundary calibration requires the actual balanced sixteen-member object") require(type(data) is int and data > 0 and type(block) is int and 4097 < block < 12 * 1024**2 and target["logical"]["usesLegacyChecksum"] is False, "observed size calibration needs bounded current-codec EC geometry") diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 17f82c2a..da392b20 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -254,9 +254,10 @@ def remote(self, node, action, **params): raise ValueError(message) return response - def snapshot(self, label, *, quiescent=False, metadata_faults=None): + def snapshot(self, label, *, quiescent=False, metadata_faults=None, buckets=None): return self.record(label, {node: self.remote(node, "snapshot", quiescent=quiescent, - metadataFaults=(metadata_faults or {}).get(node, {})) for node in self.nodes}) + metadataFaults=(metadata_faults or {}).get(node, {}), + **({"buckets": buckets} if buckets is not None else {})) for node in self.nodes}) def stop(self): with ThreadPoolExecutor(max_workers=len(self.nodes)) as executor: diff --git a/scripts/validation_heal/network.py b/scripts/validation_heal/network.py index 26ddb05a..36e58dd2 100644 --- a/scripts/validation_heal/network.py +++ b/scripts/validation_heal/network.py @@ -29,6 +29,7 @@ from .stop import idle_fleet from .network_gate import TTL_SECONDS from .scanner_repair import JournalWindow + from .network_fixtures import db_parameters, seed_db, map_db, prove_db, verify_db except ImportError: from heal_probe import Probe, canonical, compare_inventories, payload, require, save from heal_workload import Recorder, RecordedCallError, seed, verify_seed @@ -43,6 +44,7 @@ from stop import idle_fleet from network_gate import TTL_SECONDS from scanner_repair import JournalWindow + from network_fixtures import db_parameters, seed_db, map_db, prove_db, verify_db CASE_ID = 'heal.network-partition-and-lock-quorum' @@ -53,7 +55,7 @@ 'network': {'servicePort': 9000, 'managementPort': 22, 'ttlSeconds': TTL_SECONDS, 'rules': 'one owned nft table per host; exact peer IPv4 and TCP source/destination port', 'mutation': 'all expiry timers armed before the first atomic table transaction'}, - 'load': {'fixtureObjects': 4, 'versionsPerObject': 3, 'retainedNullVersions': 4, + 'load': {'fixtureObjects': 4, 'versionsPerObject': 3, 'retainedNullVersions': 4, 'DBObjects': 128, 'currentAndHistoricalMarkers': True, 'partitionVersioning': 'Enabled after witnessed Suspended/null operations', 'partitionPUTs': 2, 'recoveredPUTs': 1, 'automaticRetries': 0, 'negativeTaskDeadlineSeconds': 330}, @@ -412,6 +414,27 @@ def partition_get(probe, fleet, params): return observed, diagnostics +def partition_db(probe, fleet, bucket, representatives, side, plan, contract): + results = [] + for row in representatives: + entry = row['expected'] + observed, diagnostics = partition_get(probe, fleet, { + 'Bucket': bucket, 'Key': entry['key'], 'VersionId': entry['versionId']}) + read_quorum = row['target']['logical']['erasure']['data_blocks'] + readable = side['lockClients'] >= plan['sharedLockQuorum'] and side['metadataAuthorities'] >= read_quorum + if readable: + require(observed.get('acknowledged') is True and observed.get('resultUnknown') is False + and observed.get('response', {}).get('ResponseMetadata', {}).get('HTTPStatusCode') == 200 + and observed.get('payload') == entry['payload'], + 'R-capable partition lost an acknowledged DB boundary control') + else: + validate_unrecoverable_get(observed, negative_get_contract(contract, side, plan)) + require(diagnostics['complete'], 'DB boundary GET diagnostics are incomplete') + results.append({'key': entry['key'], 'shape': row['shape'], 'size': entry['payload']['size'], + 'readQuorum': read_quorum, 'availableSources': side['metadataAuthorities'], 'readable': readable}) + return results + + def run_branch(args, nodes, branch): directory = args.evidence / 'H2' / branch directory.mkdir(parents=True) @@ -431,34 +454,46 @@ def run_branch(args, nodes, branch): 'maxInventoryBytes': 512 * 1024**2} fleet.record('binding', binding) fleet.record('network-source-contract', reference) - contract = verify_quorum_contract(fleet.record('inspector-build', verify_inspector(args.tool_directory, candidate)), candidate) + manifest = fleet.record('inspector-build', verify_inspector(args.tool_directory, candidate)) + contract = verify_quorum_contract(manifest, candidate) result = {'topology': 'H2', 'branch': branch, 'verdict': 'RUNNING', 'cleanup': 'PENDING'} probe, active, recovery_active, rows = Probe(binding), [], [], [] try: fleet.deploy(args.binary_sha, bucket, access, secret) recorder = Recorder(probe, directory / 'fixture') expected = seed_versions(recorder, fleet, bucket) + key_versions = [entry for entry in expected if entry['key'] == 'objects/0001.bin'] + selected = next(entry for entry in key_versions if not entry['deleteMarker'] and entry['versionId'] == 'null') + calibration = fleet.snapshot('DB-size-calibration-live') + calibration_target = map_target(calibration, bucket, selected['key'], 'data', 1, selected['versionId']) + db_plan = fleet.record('DB-source-size-plan', db_parameters(manifest, candidate, calibration_target, calibration)) + db_bucket = bucket + '-db' + db_expected = seed_db(Recorder(probe, directory / 'DB-fixture'), db_bucket, db_plan) + db_baseline = fleet.record('DB-S3-baseline', probe.inventory(db_bucket, unversioned_current_tags=True)) + fleet.record('DB-ACK-manifest', {'expected': db_expected, **verify_seed(db_expected, db_baseline)}) + db_heads = version_heads(probe, fleet, db_bucket, db_expected, 'DB-original-HEAD') baseline = fleet.record('all-version-S3-baseline', probe.inventory(bucket)) fleet.record('exact-ACK-manifest', {'expected': expected, **verify_seed(expected, baseline)}) original_heads = version_heads(probe, fleet, bucket, expected, 'original-version-HEAD') for key in sorted({entry['key'] for entry in expected if entry['deleteMarker']}): verify_markers(probe, fleet, bucket, key, expected) fleet.stop() - original = fleet.snapshot('physical-baseline', quiescent=True) + physical_buckets = [bucket, db_bucket] + original = fleet.snapshot('physical-baseline', quiescent=True, buckets=physical_buckets) validate_topology(original, 'H2') - key_versions = [entry for entry in expected if entry['key'] == 'objects/0001.bin'] - selected = next(entry for entry in key_versions if not entry['deleteMarker'] and entry['versionId'] == 'null') target = map_target(original, bucket, selected['key'], 'data', 1, selected['versionId'], latest_required=False) target = {**target, **next(row for row in target['cohort'] if row['node'] == nodes[3])} cohorts = [map_target(original, bucket, entry['key'], 'data', 1, entry['versionId'], latest_required=False) for entry in expected if not entry['deleteMarker']] + db_representatives, db_witnesses = map_db(original, db_bucket, db_expected, db_plan) + fleet.record('DB-all128-physical-shapes', {'objects': db_witnesses, 'allBoundariesObserved': True}) plan = fleet.record('actual-partition-quorum-plan', plan_for(original, target, branch)) params = {'disk': target['disk'], 'faultId': 'I03-missing-target'} registration = target['node'], params fleet.faults.append(registration) receipt = fleet.record('exact-missing-target', fleet.remote(target['node'], 'apply-fault', **params, relativePath=target['relativePath'], kind='missing', expectedSha256=target['sha256'])) - faulted = fleet.snapshot('physical-exact-fault', quiescent=True) + faulted = fleet.snapshot('physical-exact-fault', quiescent=True, buckets=physical_buckets) validate_fault(original, faulted, target, receipt) for node, disks in plan['gates'].items(): fleet.quorum_gates.add(node) @@ -506,6 +541,8 @@ def witness(label): require(diagnostics['complete'], 'partition GET diagnostics are incomplete; inspect retained raw response') row = unique_put(local_probe, directory, bucket, 'partition/' + endpoint + '.bin', rows) row['negativeProof'] = negative_put(row['observed']) + result.setdefault('DB-partition-controls', {})[endpoint] = partition_db( + local_probe, fleet, db_bucket, db_representatives, side, plan, contract) result.setdefault('partitionTasks', {})[endpoint] = negative_heal(local_probe, fleet, bucket, key_versions, target, active) status = fleet.record('partition-raw-background', local_probe.admin('/rustfs/admin/v3/background-heal/status', [], b'')) require(status.get('httpStatus') == 200 and isinstance(status.get('response'), dict), 'partition background status unavailable') @@ -513,7 +550,7 @@ def witness(label): result['networkAfter'] = witness('all-directed-peer-probes-after') fleet.record('partition-process-resource-observations', {node: fleet.remote(node, 'process-metrics') for node in nodes}) fleet.stop() - scene = fleet.snapshot('negative-scene-before-release', quiescent=True) + scene = fleet.snapshot('negative-scene-before-release', quiescent=True, buckets=physical_buckets) require(physical_content(faulted) == physical_content(project(scene, bucket, [row['key'] for row in rows])), 'partition writes/Heal changed an original version, control or damaged target') for node in nodes: @@ -523,14 +560,15 @@ def witness(label): fleet.record('release-owned-W-gate', fleet.remote(node, 'quorum-gate', mode='clear')) fleet.quorum_gates.remove(node) fleet.start() - require(physical_content(scene) == physical_content(fleet.snapshot('all-peer-return-before-independent-repair')), + require(physical_content(scene) == physical_content(fleet.snapshot('all-peer-return-before-independent-repair', buckets=physical_buckets)), 'network recovery repaired a target before the independent product task') result['recoveryResponsibility'] = recover_versions(probe, fleet, binding, bucket, key_versions, selected, recovery_active) idle_fleet(fleet, binding) fleet.stop() - after = fleet.snapshot('physical-rebuilt-before-any-recovered-GET', quiescent=True) + after = fleet.snapshot('physical-rebuilt-before-any-recovered-GET', quiescent=True, buckets=physical_buckets) result['physical'] = validate_repaired_physical(original, project(after, bucket, [row['key'] for row in rows]), target) result['streaming'] = [verify_cohort(fleet, cohort, 'every-original-version-streaming') for cohort in cohorts] + result['DB-physical'] = prove_db(fleet, db_representatives) fleet.faults.remove(registration) fleet.start() result['writeResolution'] = resolve_writes(fleet, binding, bucket, rows) @@ -555,6 +593,10 @@ def witness(label): compare_inventories(inventories[0], inventory) result['markers'] = [verify_markers(probe, fleet, bucket, key, expected) for key in sorted({entry['key'] for entry in expected if entry['deleteMarker']})] + result['DB-final'] = verify_db(probe, fleet, db_bucket, db_expected, db_baseline, db_representatives, db_plan) + for endpoint in binding['endpoints']: + require(version_heads(Probe(binding, endpoint), fleet, db_bucket, db_expected, 'DB-final-HEAD') == db_heads, + 'network recovery changed an acknowledged DB HEAD or Last-Modified') require(all(row['properties'].get('Metadata') == {'heal-fixture': 'I03', 'nonce': released['key']} and row['tags'] == [{'Key': 'fixture', 'Value': 'I03'}, {'Key': 'suite', 'Value': 'heal'}] for inventory in inventories for row in inventory['entries'] if row['key'] == released['key']), diff --git a/scripts/validation_heal/network_fixtures.py b/scripts/validation_heal/network_fixtures.py new file mode 100644 index 00000000..85d4cb26 --- /dev/null +++ b/scripts/validation_heal/network_fixtures.py @@ -0,0 +1,95 @@ +"""Source-bound I03 DB fixtures; only public S3 writes and decoded physical witnesses.""" + +try: + from .boundaries import boundary_sizes, verify_size_contract, verify_boundary_ranges + from .heal_probe import require + from .heal_workload import seed, verify_seed + from .shards import map_target + from .versions import map_inline_target, verify_proof +except ImportError: + from boundaries import boundary_sizes, verify_size_contract, verify_boundary_ranges + from heal_probe import require + from heal_workload import seed, verify_seed + from shards import map_target + from versions import map_inline_target, verify_proof + + +DB_COUNT = 128 +MPU_MINIMUM = 5 * 1024**2 +# GLOBAL_MIN_PART_SIZE and is_min_allowed_part_size in these exact source files. +MPU_SOURCE = { + '072b57b37559fbe626aeda8abf61617fdcbdd926': 'e60ff0ab886a5578512a123eb1757352ae3e5204d7b69b5aaf14e9e530ec2d31', + '1c7fc818dd858247c45b39542abd8291b1e6e31b': '6975b05976e88667065dd4fd2d7649ff61ce0887504e2790bc68fc73c9ba048a', +} + + +def db_parameters(manifest, candidate, calibration, processes): + contract = verify_size_contract(manifest, candidate) + require(candidate['source_sha'] in MPU_SOURCE + and contract['sourceFiles']['crates/ecstore/src/set_disk/mod.rs'] == MPU_SOURCE[candidate['source_sha']], + 'network DB multipart boundary is not bound to the exact reviewed candidate source') + parameters = boundary_sizes(contract, calibration, processes, standard_class='EC:8') + parameters['groups']['multipart-minimum-minus-exact-plus'] = [MPU_MINIMUM - 1, MPU_MINIMUM, MPU_MINIMUM + 1] + sizes = [size for group in parameters['groups'].values() for size in group] + require(len(sizes) <= DB_COUNT and max(sizes) <= 64 * 1024**2, 'DB boundary plan exceeds its fixed count or body cap') + return {**parameters, 'objectCount': DB_COUNT, 'sizeCycle': sizes, + 'payloadBytes': sum(sizes[index % len(sizes)] for index in range(DB_COUNT)), + 'multipartMinimumPartBytes': MPU_MINIMUM, 'multipartBoundarySourceSha256': MPU_SOURCE[candidate['source_sha']], + 'multipartBoundaryIsObjectSizeControl': True, + 'completedMultipartProtocolClaimed': False} + + +def seed_db(recorder, bucket, parameters): + require(parameters['objectCount'] == DB_COUNT + and parameters['payloadBytes'] <= recorder.probe.binding['maxInventoryBytes'] + and DB_COUNT <= recorder.probe.binding['maxInventoryObjects'], + 'network DB plan exceeds the frozen inventory budget') + return seed(recorder, bucket, 'DB', DB_COUNT, parameters['sizeCycle']) + + +def map_db(snapshot, bucket, expected, parameters): + require(len(expected) == DB_COUNT and len({entry['key'] for entry in expected}) == DB_COUNT, + 'DB boundary fixture omits or duplicates a control') + representatives, witnessed, selected_sizes = [], [], set() + for index, entry in enumerate(expected): + size = parameters['sizeCycle'][index % len(parameters['sizeCycle'])] + require(entry['key'] == f'objects/{index:04d}.bin' and entry['versionId'] == 'null' + and entry['isLatest'] is True and entry['deleteMarker'] is False and entry['payload']['size'] == size, + 'DB control identity or ACK size differs from its frozen cycle') + shape = 'zero-metadata' if size == 0 else 'inline' if size <= parameters['actualLogicalInlineLimit'] else 'external' + target = (map_target(snapshot, bucket, entry['key'], 'data', 1) if shape == 'external' + else map_inline_target(snapshot, bucket, entry, allow_empty=size == 0)) + require(len(target['cohort']) == 16 and target['logical']['size'] == size + and target['logical']['erasure']['data_blocks'] == target['logical']['erasure']['parity_blocks'] == 8, + 'DB boundary lacks its actual complete sixteen-member size/shape authority') + witnessed.append({'key': entry['key'], 'size': size, 'shape': shape, + 'versionId': entry['versionId'], 'members': len(target['cohort']), + 'dataBlocks': target['logical']['erasure']['data_blocks'], + 'parityBlocks': target['logical']['erasure']['parity_blocks']}) + if size not in selected_sizes: + selected_sizes.add(size) + representatives.append({'expected': entry, 'target': target, 'shape': shape}) + require(selected_sizes == set(parameters['sizeCycle']), 'physical DB representatives omit a required boundary') + return representatives, witnessed + + +def prove_db(fleet, representatives): + return [{'key': row['expected']['key'], 'shape': row['shape'], + 'proof': verify_proof(fleet, row['target'], 'DB-boundary-physical-proof')} + for row in representatives] + + +def verify_db(probe, fleet, bucket, expected, baseline, representatives, parameters): + try: + from .heal_probe import Probe, compare_inventories + except ImportError: + from heal_probe import Probe, compare_inventories + peers = [] + for endpoint in probe.binding['endpoints']: + peer = Probe(probe.binding, endpoint) + inventory = fleet.record('all-peer-DB-final-S3', {'endpoint': endpoint, + **peer.inventory(bucket, unversioned_current_tags=True)}) + peers.append({'endpoint': endpoint, **verify_seed(expected, inventory), **compare_inventories(baseline, inventory)}) + ranges = verify_boundary_ranges(probe, fleet, bucket, + [row['expected'] for row in representatives], parameters['actualECBlockSize']) + return {'allDefault128ControlsPreserved': True, 'peers': peers, 'boundaryRanges': ranges} diff --git a/test_validation_heal_network_fixtures.py b/test_validation_heal_network_fixtures.py new file mode 100644 index 00000000..e4cdd87b --- /dev/null +++ b/test_validation_heal_network_fixtures.py @@ -0,0 +1,149 @@ +"""Network source fixture completeness, real boundaries and fixed resource caps.""" + +import copy +import hashlib +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from scripts.validation_heal import network, network_fixtures as fixtures +from scripts.validation_heal.boundaries import boundary_sizes +from scripts.validation_heal.healthy import Fleet +from scripts.validation_heal.fleet_node import snapshot_buckets + + +class NetworkBoundaryFixturesTest(unittest.TestCase): + def parameters(self): + source = '1c7fc818dd858247c45b39542abd8291b1e6e31b' + files = {name: 'a' * 64 for name in ('crates/ecstore/src/config/storageclass.rs', + 'crates/ecstore/src/set_disk/mod.rs', 'crates/ecstore/src/set_disk/ops/object.rs', + 'crates/ecstore/src/erasure/coding/erasure.rs')} + files['crates/ecstore/src/set_disk/mod.rs'] = fixtures.MPU_SOURCE[source] + manifest = {'sizeContract': {'recognized': True, 'candidateCommit': source, + 'extractorSourceSha256': hashlib.sha256(Path('scripts/validation_heal/prepare_heal_meta_inspector.py').read_bytes()).hexdigest(), + 'inlineRule': 'ceil(size/dataBlocks)<=effectiveInlineBlock AND size<=actualECBlock', + 'effectiveDefaultRule': 'min(ceil(2*defaultInlineBlock/dataBlocks),defaultInlineBlock)', + 'defaultInlineBlock': 128 * 1024, 'defaultObjectBudgetFactor': 2, 'sourceFiles': files}} + target = {'logical': {'usesLegacyChecksum': False, 'erasure': { + 'data_blocks': 8, 'parity_blocks': 8, 'block_size': 1024**2, 'distribution': list(range(1, 17))}}} + processes = {f'node{i}': {'storageClassEnvironment': { + 'RUSTFS_STORAGE_CLASS_INLINE_BLOCK': None, 'RUSTFS_STORAGE_CLASS_STANDARD': 'EC:8'}} for i in range(4)} + candidate = {'source_sha': source} + return manifest, candidate, target, processes + + def test_actual_ec8_size_contract_keeps_all_boundaries_and_default128(self): + manifest, candidate, target, processes = self.parameters() + parameters = fixtures.db_parameters(manifest, candidate, target, processes) + self.assertEqual(parameters['objectCount'], 128) + self.assertEqual(parameters['actualLogicalInlineLimit'], 262144) + self.assertEqual(parameters['groups']['inline-minus-exact-plus'], [262143, 262144, 262145]) + self.assertEqual(parameters['groups']['stripe-minus-exact-plus'], [1048575, 1048576, 1048577]) + self.assertEqual(parameters['groups']['multipart-minimum-minus-exact-plus'], [5242879, 5242880, 5242881]) + self.assertTrue({0, 1, 4095, 4096, 4097} <= set(parameters['sizeCycle'])) + self.assertLess(parameters['payloadBytes'], 512 * 1024**2) + self.assertFalse(parameters['completedMultipartProtocolClaimed']) + + def test_missing_environment_wrong_geometry_or_foreign_source_refuses_calibration(self): + for kind in ('default', 'foreign-class', 'inline-override', 'geometry', 'distribution', 'source-hash', 'candidate'): + manifest, candidate, target, processes = self.parameters() + if kind == 'default': processes['node0']['storageClassEnvironment']['RUSTFS_STORAGE_CLASS_STANDARD'] = None + elif kind == 'foreign-class': processes['node0']['storageClassEnvironment']['RUSTFS_STORAGE_CLASS_STANDARD'] = 'EC:4' + elif kind == 'inline-override': processes['node0']['storageClassEnvironment']['RUSTFS_STORAGE_CLASS_INLINE_BLOCK'] = '1MiB' + elif kind == 'geometry': target['logical']['erasure']['parity_blocks'] = 4 + elif kind == 'distribution': target['logical']['erasure']['distribution'].pop() + elif kind == 'source-hash': manifest['sizeContract']['sourceFiles']['crates/ecstore/src/set_disk/mod.rs'] = 'f' * 64 + else: + candidate['source_sha'] = 'f' * 40 + manifest['sizeContract']['candidateCommit'] = 'f' * 40 + with self.subTest(kind=kind), self.assertRaises(ValueError): + fixtures.db_parameters(manifest, candidate, target, processes) + + def test_default_boundary_callers_cannot_silently_use_ec8_environment(self): + manifest, candidate, target, processes = self.parameters() + with self.assertRaises(ValueError): boundary_sizes(manifest['sizeContract'], target, processes) + for value in ('EC:4', True, 'EC:16'): + with self.subTest(value=value), self.assertRaises(ValueError): + boundary_sizes(manifest['sizeContract'], target, processes, standard_class=value) + + def test_seed_uses_real_db128_cycle_and_refuses_budget_before_first_call(self): + params = fixtures.db_parameters(*self.parameters()) + calls = [] + def call(operation, values, body=None): + calls.append((operation, values, len(body) if body is not None else None)) + return {'ETag': 'ack'} + recorder = SimpleNamespace(probe=SimpleNamespace(binding={'bucketPrefix': 'rv-heal-owned', + 'maxInventoryBytes': 512 * 1024**2, 'maxInventoryObjects': 128}), call=call) + expected = fixtures.seed_db(recorder, 'rv-heal-owned-db', params) + self.assertEqual(len(expected), 128) + self.assertEqual(calls[0][0], 'create_bucket') + self.assertEqual(sum(c[2] for c in calls if c[0] == 'put_object'), params['payloadBytes']) + self.assertEqual({x['payload']['size'] for x in expected}, set(params['sizeCycle'])) + calls.clear() + recorder.probe.binding['maxInventoryBytes'] = params['payloadBytes'] - 1 + with self.assertRaises(ValueError): fixtures.seed_db(recorder, 'rv-heal-owned-db', params) + self.assertFalse(calls) + + def test_each128_control_is_shape_mapped_and_all_unique_sizes_have_representatives(self): + params = fixtures.db_parameters(*self.parameters()) + expected = [{'key': f'objects/{i:04d}.bin', 'versionId': 'null', 'isLatest': True, + 'deleteMarker': False, 'payload': {'size': params['sizeCycle'][i % len(params['sizeCycle'])]}} for i in range(128)] + mapped = [] + def target(entry, shape): + mapped.append((entry['key'], shape)) + return {'cohort': [{}] * 16, 'logical': {'size': entry['payload']['size'], 'erasure': {'data_blocks': 8, 'parity_blocks': 8}}} + def external(snapshot, bucket, key, role, part): + entry = next(x for x in expected if x['key'] == key) + self.assertGreater(entry['payload']['size'], params['actualLogicalInlineLimit']) + return target(entry, 'external') + def inline(snapshot, bucket, entry, *, allow_empty): + self.assertEqual(allow_empty, entry['payload']['size'] == 0) + self.assertLessEqual(entry['payload']['size'], params['actualLogicalInlineLimit']) + return target(entry, 'zero' if allow_empty else 'inline') + with patch.object(fixtures, 'map_target', side_effect=external), patch.object(fixtures, 'map_inline_target', side_effect=inline): + representatives, witnessed = fixtures.map_db({}, 'owned-db', expected, params) + self.assertEqual((len(mapped), len(witnessed)), (128, 128)) + self.assertEqual({r['expected']['payload']['size'] for r in representatives}, set(params['sizeCycle'])) + self.assertEqual(set(shape for key, shape in mapped), {'zero', 'inline', 'external'}) + for wrong in (expected[:-1], [expected[0]] + expected[:-1]): + with self.assertRaises(ValueError): fixtures.map_db({}, 'owned-db', wrong, params) + + def test_full_physical_snapshot_explicitly_includes_the_owned_db_bucket(self): + fleet = Fleet(['node1', 'node2'], Path('/unused'), 'gh-12345-1-h2') + owner = {'bucket': 'rv-heal-owned'} + def remote(node, action, **params): + self.assertEqual(action, 'snapshot') + return {'buckets': snapshot_buckets(owner, params), 'quiescent': params['quiescent']} + with patch.object(fleet, 'record', side_effect=lambda label, value: value), patch.object(fleet, 'remote', side_effect=remote) as call: + current = fleet.snapshot('baseline', quiescent=True, buckets=['rv-heal-owned', 'rv-heal-owned-db']) + self.assertEqual(set(current), {'node1', 'node2'}) + self.assertTrue(all(value == {'buckets': ['rv-heal-owned', 'rv-heal-owned-db'], 'quiescent': True} + for value in current.values())) + self.assertEqual(call.call_count, 2) + default = fleet.snapshot('legacy-default') + self.assertTrue(all(value['buckets'] == ['rv-heal-owned'] for value in default.values())) + for buckets in (['rv-heal-owned-db'], ['rv-heal-owned', 'foreign-bucket']): + with self.assertRaises(ValueError): fleet.snapshot('refuse-escape', buckets=buckets) + + def test_control_read_requires_full_payload_or_exact_source_bound_lock_refusal(self): + source = '1c7fc818dd858247c45b39542abd8291b1e6e31b' + contract = {'candidateCommit': source, 'recognized': True, + 'sourceFiles': {'rustfs/src/error.rs': network.GET_SOURCE_FILES[source]['rustfs/src/error.rs']}, + 'unrecoverableGet': {'httpStatus': 503, 'code': 'SlowDownRead'}} + row = {'expected': {'key': 'control', 'versionId': 'null', 'payload': {'size': 1, 'sha256': 'digest'}}, + 'target': {'logical': {'erasure': {'data_blocks': 1}}}, 'shape': 'inline'} + good = {'acknowledged': True, 'resultUnknown': False, 'payload': row['expected']['payload'], + 'response': {'ResponseMetadata': {'HTTPStatusCode': 200}}} + error = {'acknowledged': False, 'resultUnknown': False, + 'response': {'ResponseMetadata': {'HTTPStatusCode': 503}, 'Error': {'Code': 'ServiceUnavailable'}}} + for clients, observed, okay in [(2, good, True), (1, error, True), (1, good, False), + (2, {**good, 'payload': {'size': 0}}, False), (2, {**good, 'resultUnknown': True}, False)]: + with self.subTest(clients=clients, observation=observed), patch.object(network, 'partition_get', return_value=(observed, {'complete': True})): + args = (None, None, 'owned', [row], {'lockClients': clients, 'metadataAuthorities': 4}, {'sharedLockQuorum': 2}, contract) + if okay: self.assertEqual(len(network.partition_db(*args)), 1) + else: + with self.assertRaises(ValueError): network.partition_db(*args) + + +if __name__ == '__main__': + unittest.main() From d284c27d7f5149c7d717d4d742fe4213edcb3b29 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 19:19:04 +0800 Subject: [PATCH 237/343] test(heal): add request-bound network load journals --- docs/validation-migration-map.md | 30 +- scripts/validation_heal/heal_workload.py | 14 +- scripts/validation_heal/network_load.py | 372 +++++++++++++++++++++++ test_validation_heal_network_load.py | 268 ++++++++++++++++ 4 files changed, 680 insertions(+), 4 deletions(-) create mode 100644 scripts/validation_heal/network_load.py create mode 100644 test_validation_heal_network_load.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index f120b049..4db7461a 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2040,7 +2040,7 @@ Original failures and original reports remain unchanged. | --- | --- | --- | | DV | Reuses auto-test `seed(DV)` and `suspend_fixture`: four keys with three distinct non-null data versions each; real suspended overwrites, null marker creation/removal ACKs; retained live/null marker identities; current and historical markers. Versioning returns to Enabled before partition. The fault targets a retained null data version. Every selected key version participates in GET, negative Heal and complete recursive recovery. All 15 retained data versions receive full physical shard checks; markers receive exact S3 refusal/header checks. HEAD preserves version identity, length, ETag, Last-Modified and metadata. | This expanded fixture still needs a new live run and independent evidence acceptance. It does not claim a partition while the bucket remains Suspended. | | DB | Adds a separate never-versioned run-owned bucket with the default 128 controls. The exact candidate size contract and actual EC:8 process/object geometry calibrate zero/one, 4 KiB, inline, EC block and source-bound 5-MiB MPU minimum minus/exact/plus sizes. All 128 controls require decoded full-set shape witnesses; every distinct size has partition GET, physical checksum/no-part and Range evidence. Original/fault/rejoin/repaired snapshots explicitly include both primary and DB buckets. All peers must preserve complete inventories and stable original HEADs. | Fresh live execution and independent acceptance remain required. The MPU-size controls are single PUT objects; they do not claim completed multipart protocol coverage, which remains part of DL. | -| DL | Existing partition workload serially probes exact-version GET, unique negative PUT and Heal, followed by explicit write resolution and peer inventories. | Add bounded concurrent PUT/GET/HEAD/LIST/COPY/DELETE, version and MPU operations, with per-request durable ACKs, increasing per-key nonce and resolution of unknown outcomes. Serial PUT/GET is insufficient. | +| DL | `network_load.py` adds the request-layer foundation: three distinct control versions per endpoint, response-bound MPU UploadId/part ETag fixtures and four independent worker journals. Its 28-call matrix includes concurrent PUT/GET/HEAD/LIST/COPY/DELETE, version selectors and actual Create/UploadPart/Complete/Abort MPU requests. Requests bind exact selectors, nonce/body/status policy and worker/run/source/endpoint URL fingerprint; overlap uses actual SDK intervals. | This module is not yet wired into the live adapter. Add complete post-rejoin object/version/MPU reconciliation, failure-scene retention and recovery/isolation integration before dispatch. Mocked matrix completion is not DL live or canonical acceptance. | The expanded DV fixture stays within the existing 128-identity/512-MiB inventory limits. Network expiry remains 1,200 seconds; partition task deadlines, @@ -2059,3 +2059,31 @@ The reviewed multipart minimum is bound to the fixed candidate's full existing 512-MiB budget. This read-only preparation is not fresh DB live evidence. The existing node-side snapshot bucket guard continues to require the primary bucket and allows only its owned suffixes; no broader host path is introduced. + + +I03 DL request layer (2026-10-19): `LoadJournal` reuses the auto-test +`Recorder` and serializes one worker's calls; readers and writers never share +mutable sequence state. Two endpoint pairs execute eight mutations and six +reads each, within a frozen 180-second admission budget and the existing SDK +request limits. The controller alone writes Fleet evidence. Every prepared +request stays bound to its original ACK SHA-256; missing ACKs and unknown +results remain unresolved, and a failed worker admits no subsequent calls. + +The fixed candidate maps write-quorum/lock errors to HTTP 503 +`ServiceUnavailable`; that refusal does not prove rollback, especially for +MPU Abort. Partition LIST responses are bounded observations, not global +completeness. A finite HTTP-status tuple (at most three explicit codes) extends +the code-level Recorder API for these LIST observations; existing integer and +default callers retain their checks, and CLI traces still require one integer. +Operation-specific code, payload, version and selector validators remain +mandatory. HEAD properties use the same datetime representation as the durable +ACK JSON. Only actual invoked SDK intervals in one clock domain can prove +client overlap; collector setup windows cannot establish it, and no server +commit fence is claimed. + +The pending live adapter must register the seed Recorder and every worker +journal before invocation, including failures before a complete fixture is +returned. It must preserve the original object and system MPU failure scene, +resolve every invoked mutation without silently retrying PUT/Complete, and +refuse cleanup/acceptance while any result remains unresolved. This request +layer alone neither enables a new fleet dispatch nor completes I03. diff --git a/scripts/validation_heal/heal_workload.py b/scripts/validation_heal/heal_workload.py index 61e89cbf..8d4fe9b9 100644 --- a/scripts/validation_heal/heal_workload.py +++ b/scripts/validation_heal/heal_workload.py @@ -47,8 +47,12 @@ def __init__(self, probe: Probe, output: Path, *, writer=None): self.mutations = MutationOrder() def call(self, operation: str, params: dict, body: bytes | None = None, - identity: dict | None = None, expect_http: int | None = None, *, admission_deadline=None) -> dict: + identity: dict | None = None, expect_http: int | tuple[int, ...] | None = None, *, admission_deadline=None) -> dict: params=copy.deepcopy(params);identity=copy.deepcopy(identity) + require(expect_http is None or type(expect_http) is int and 100 <= expect_http <= 599 + or isinstance(expect_http, tuple) and 1 <= len(expect_http) <= 3 + and all(type(value) is int and 100 <= value <= 599 for value in expect_http) + and len(set(expect_http)) == len(expect_http), 'invalid finite expected HTTP status set') validate_s3_request(self.probe.binding, operation, params) if operation == 'delete_objects':validate_batch_request(params) require(body is None or isinstance(body,bytes) and len(body)<=MAX_BODY and operation in {'put_object','upload_part'}, @@ -86,8 +90,12 @@ def _call(self, operation, params, body, identity, expect_http): "error": {"type": type(exc).__name__, "message": str(exc)}} finished = time.monotonic_ns() status = (result.get("response") or {}).get("ResponseMetadata", {}).get("HTTPStatusCode") - checked = status == expect_http if expect_http is not None else ( - result["acknowledged"] and type(status) is int and 200<=status<300) + if expect_http is None: + checked = result["acknowledged"] and type(status) is int and 200<=status<300 + elif isinstance(expect_http, tuple): + checked = status in expect_http + else: + checked = status == expect_http checked = checked and not result.get('clientCleanupError') per_item = None if operation == 'delete_objects' and status == 200: diff --git a/scripts/validation_heal/network_load.py b/scripts/validation_heal/network_load.py new file mode 100644 index 00000000..73e5d12e --- /dev/null +++ b/scripts/validation_heal/network_load.py @@ -0,0 +1,372 @@ +"""I03 DL request-bound fixtures and bounded concurrent calls; no repair verdict.""" + +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +from pathlib import Path +import threading +import time + +try: + from .heal_probe import payload, require, validate_s3_request + from .heal_workload import Recorder + from .network_fixtures import MPU_MINIMUM, MPU_SOURCE +except ImportError: + from heal_probe import payload, require, validate_s3_request + from heal_workload import Recorder + from network_fixtures import MPU_MINIMUM, MPU_SOURCE + + +BODY_SIZE = 512 * 1024 + 97 +FIRST_PART_SIZE = MPU_MINIMUM + 17 +FINAL_PART_SIZE = 211 +ADMISSION_SECONDS = 180 +SOURCE_FILES = { + '072b57b37559fbe626aeda8abf61617fdcbdd926': { + 'rustfs/src/error.rs': '656d38972c06339015c174405571a94f19d8dd64a680e1a7ca631f80205f99a7', + 'crates/ecstore/src/set_disk/ops/object.rs': 'f0f407509b5a12803a191cc1c3a9eb079a15a80a4bbcacca70141150364628ca'}, + '1c7fc818dd858247c45b39542abd8291b1e6e31b': { + 'rustfs/src/error.rs': 'e7cff0e28978b49d3ee0e8c24da1fa439b3118a7d3f7304edf00dc0535691598', + 'crates/ecstore/src/set_disk/ops/object.rs': 'fdcfda1a8811a93537f9cd47cadac7380bee66b3d14242ca91500bbf38e6bedb'}, +} +MULTIPART_SOURCE = '1ac5c17a94845fb16f6d75b8d3692cb18f81556086d4cc4a96b63deae106ab8d' +MUTATIONS = frozenset({'put_object', 'copy_object', 'delete_object', 'create_multipart_upload', + 'upload_part', 'complete_multipart_upload', 'abort_multipart_upload'}) +READS = frozenset({'get_object', 'head_object', 'list_objects_v2', 'list_object_versions', + 'list_multipart_uploads', 'list_parts'}) + + +def source_reference(candidate): + require(candidate in SOURCE_FILES, 'DL operations are not reviewed for this exact candidate') + return {'candidateCommit': candidate, 'sourceFiles': {**SOURCE_FILES[candidate], + 'crates/ecstore/src/set_disk/mod.rs': MPU_SOURCE[candidate], + 'crates/ecstore/src/set_disk/ops/multipart.rs': MULTIPART_SOURCE}, + 'negativeObjectMutation': {'httpStatus': 503, 'code': 'ServiceUnavailable'}, + 'abortRefusalIsNotRollback': True, 'listDuringPartitionIsNotGlobalCompleteness': True, + 'maximumWorkers': 4, 'admissionSeconds': ADMISSION_SECONDS, + 'automaticMutationRetries': 0, 'serverCommitFenceClaimed': False} + + +def digest(body): + return {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()} + + +def binding_digest(binding): + return hashlib.sha256(json.dumps(binding, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def control_body(bucket, key, nonce): + return payload(f'{bucket}/{key}/{nonce}', BODY_SIZE) + + +def part_body(bucket, key, number, size): + return payload(f'{bucket}/{key}/part-{number}', size) + + +def seed_load(recorder, bucket, endpoints): + reference = source_reference(recorder.probe.binding['candidateCommit']) + require(len(endpoints) == len(set(endpoints)) == 2 + and set(endpoints) <= set(recorder.probe.binding['endpoints']), 'DL requires the two actual partition-side endpoints') + require(recorder.call('get_bucket_versioning', {'Bucket': bucket}).get('Status') == 'Enabled', + 'DL requires the original enabled versioned bucket') + expected, sides, seen_uploads = [], {}, set() + for endpoint in endpoints: + key = f'dl/{endpoint}/control.bin' + versions, heads = [], {} + for nonce in range(3): + body = control_body(bucket, key, nonce) + response = recorder.call('put_object', {'Bucket': bucket, 'Key': key, + 'Metadata': {'heal-fixture': 'DL', 'nonce': str(nonce)}, 'Tagging': 'suite=heal&fixture=DL', + 'ContentType': 'application/octet-stream'}, body, identity={'endpoint': endpoint, 'phase': 'seed', 'nonce': nonce}) + version = response.get('VersionId') + require(isinstance(version, str) and version not in {'', 'null'} + and version not in {entry['versionId'] for entry in versions} and response.get('ETag'), + 'DL control lacks a distinct exact PUT version/ETag ACK') + entry = {'key': key, 'versionId': version, 'deleteMarker': False, + 'isLatest': nonce == 2, 'payload': digest(body)} + versions.append(entry) + expected.append(entry) + head = recorder.call('head_object', {'Bucket': bucket, 'Key': key, 'VersionId': version}) + require(head.get('VersionId') == version and head.get('ContentLength') == len(body) + and head.get('ETag') == response['ETag'] and head.get('LastModified') + and head.get('Metadata') == {'heal-fixture': 'DL', 'nonce': str(nonce)}, + 'DL original HEAD does not redeem its exact data-version ACK') + heads[version] = json.loads(json.dumps({name: head.get(name) for name in + ('ContentLength', 'ContentType', 'ETag', 'LastModified', 'Metadata')}, default=str)) + uploads = {} + for name, sizes in [('complete', [FIRST_PART_SIZE, FINAL_PART_SIZE]), + ('upload', [FIRST_PART_SIZE]), ('abort', [FINAL_PART_SIZE])]: + upload_key = f'dl/{endpoint}/{name}.bin' + response = recorder.call('create_multipart_upload', {'Bucket': bucket, 'Key': upload_key, + 'Metadata': {'heal-fixture': 'DL', 'nonce': name}, 'ContentType': 'application/octet-stream'}) + upload_id = response.get('UploadId') + require(isinstance(upload_id, str) and upload_id and upload_id not in seen_uploads, + 'DL MPU lacks a distinct exact CreateMultipartUpload ACK') + seen_uploads.add(upload_id) + parts, full_digest, total = [], hashlib.sha256(), 0 + for number, size in enumerate(sizes, 1): + body = part_body(bucket, upload_key, number, size) + response = recorder.call('upload_part', {'Bucket': bucket, 'Key': upload_key, + 'UploadId': upload_id, 'PartNumber': number}, body) + require(isinstance(response.get('ETag'), str) and response['ETag'], 'DL part ACK omitted its ETag') + parts.append({'PartNumber': number, 'ETag': response['ETag'], 'payload': digest(body)}) + full_digest.update(body) + total += size + uploads[name] = {'key': upload_key, 'uploadId': upload_id, 'parts': parts, + 'completePayload': {'size': total, 'sha256': full_digest.hexdigest()}} + sides[endpoint] = {'controlKey': key, 'versions': versions, 'heads': heads, 'uploads': uploads} + return {'bucket': bucket, 'runId': recorder.probe.binding['runId'], 'bindingSha256': binding_digest(recorder.probe.binding), + 'sourceContract': reference, + 'expected': expected, 'sides': sides, + 'seedPayloadBytes': recorder.bytes_written, 'canonicalCasePassClaimed': False} + + +def mutation_steps(fixture, endpoint): + bucket, side = fixture['bucket'], fixture['sides'][endpoint] + key, versions, uploads = side['controlKey'], side['versions'], side['uploads'] + steps = [] + for nonce in (3, 4): + steps.append({'operation': 'put_object', 'params': {'Bucket': bucket, 'Key': key, + 'ContentType': 'application/octet-stream', 'Metadata': {'heal-fixture': 'DL', 'nonce': str(nonce)}, + 'Tagging': 'suite=heal&fixture=DL'}, 'body': control_body(bucket, key, nonce), 'nonce': nonce}) + steps.extend([ + {'operation': 'copy_object', 'params': {'Bucket': bucket, 'Key': f'dl/{endpoint}/copy.bin', + 'CopySource': {'Bucket': bucket, 'Key': key, 'VersionId': versions[0]['versionId']}, + 'MetadataDirective': 'REPLACE', 'Metadata': {'heal-fixture': 'DL', 'nonce': '5'}, + 'TaggingDirective': 'REPLACE', 'Tagging': 'suite=heal&fixture=DL'}, 'nonce': 5}, + {'operation': 'delete_object', 'params': {'Bucket': bucket, 'Key': key, 'VersionId': versions[0]['versionId']}, 'nonce': 6}, + {'operation': 'create_multipart_upload', 'params': {'Bucket': bucket, 'Key': f'dl/{endpoint}/create.bin', + 'Metadata': {'heal-fixture': 'DL', 'nonce': '7'}}, 'nonce': 7}, + {'operation': 'upload_part', 'params': {'Bucket': bucket, 'Key': uploads['upload']['key'], + 'UploadId': uploads['upload']['uploadId'], 'PartNumber': 2}, + 'body': part_body(bucket, uploads['upload']['key'], 2, FINAL_PART_SIZE), 'nonce': 8}, + {'operation': 'complete_multipart_upload', 'params': {'Bucket': bucket, 'Key': uploads['complete']['key'], + 'UploadId': uploads['complete']['uploadId'], 'MultipartUpload': {'Parts': [ + {name: part[name] for name in ('PartNumber', 'ETag')} for part in uploads['complete']['parts']]}}, 'nonce': 9}, + {'operation': 'abort_multipart_upload', 'params': {'Bucket': bucket, 'Key': uploads['abort']['key'], + 'UploadId': uploads['abort']['uploadId']}, 'nonce': 10}, + ]) + return steps + + +def read_steps(fixture, endpoint): + bucket, side = fixture['bucket'], fixture['sides'][endpoint] + current = side['versions'][-1] + params = {'Bucket': bucket, 'Key': current['key'], 'VersionId': current['versionId']} + upload = side['uploads']['complete'] + return [ + {'operation': 'get_object', 'params': params}, + {'operation': 'head_object', 'params': params}, + {'operation': 'list_objects_v2', 'params': {'Bucket': bucket, 'Prefix': f'dl/{endpoint}/', 'MaxKeys': 32}}, + {'operation': 'list_object_versions', 'params': {'Bucket': bucket, 'Prefix': side['controlKey'], 'MaxKeys': 32}}, + {'operation': 'list_multipart_uploads', 'params': {'Bucket': bucket, 'Prefix': f'dl/{endpoint}/', 'MaxUploads': 32}}, + {'operation': 'list_parts', 'params': {'Bucket': bucket, 'Key': upload['key'], 'UploadId': upload['uploadId'], 'MaxParts': 32}}, + ] + + +def read_journal(directory, binding, endpoint, role): + """Reconstruct from original durable bytes, including requests with no ACK.""" + require(role in {'writer', 'reader'} and endpoint in binding['endpoints'], 'unknown DL worker ownership') + directory = Path(directory) + requests, acks = sorted(directory.glob('*-request.json')), sorted(directory.glob('*-ack.json')) + require({p.name for p in acks} <= {p.name.replace('-request.json', '-ack.json') for p in requests}, + 'DL has a detached ACK without its exact request') + rows = [] + for index, path in enumerate(requests, 1): + request = json.loads(path.read_text()) + require(path.name == f'{index:06d}-request.json' and type(request['sequence']) is int and request['sequence'] == index + and request.get('schemaVersion') == 2 + and request['runId'] == binding['runId'] and request['identity']['endpoint'] == endpoint + and request['identity']['role'] == role and type(request['identity']['step']) is int + and request['identity']['step'] == index, 'DL request sequence/run/worker ownership differs') + validate_s3_request(binding, request['operation'], request['params']) + require(request['operation'] in (MUTATIONS if role == 'writer' else READS), 'DL operation escaped its worker role') + row = {'request': request, 'requestFile': path.name, 'requestSha256': hashlib.sha256(path.read_bytes()).hexdigest(), + 'prepared': True, 'ackPresent': False, 'resolution': 'UNKNOWN_PREPARED_REQUEST'} + ack_path = path.with_name(path.name.replace('-request.json', '-ack.json')) + if ack_path.exists(): + ack = json.loads(ack_path.read_text()) + require(ack.get('schemaVersion') == 2 and ack['requestFile'] == path.name and ack['requestSha256'] == row['requestSha256'] + and ack['runId'] == request['runId'] and ack['sequence'] == index + and ack['identity'] == request['identity'] and ack['clockDomain'] == request['clockDomain'], + 'DL ACK is not bound to its exact durable request') + observed = ack['observation'] + require(observed.get('resultUnknown') is True or observed.get('operation') == request['operation'] + and observed.get('params') == request['params'] and observed.get('endpoint') == endpoint, + 'DL known observation lost its actual operation/selector/endpoint') + require(all(type(value) is int for value in (request['preparedMonotonicNs'], + ack['collectorCallStartedMonotonicNs'], ack['collectorCallFinishedMonotonicNs'], ack['recordedMonotonicNs'])) + and request['preparedMonotonicNs'] < ack['collectorCallStartedMonotonicNs'] + <= ack['collectorCallFinishedMonotonicNs'] <= ack['recordedMonotonicNs'], + 'DL collector timing differs from its original clock domain') + if observed.get('clientCallInvoked') is True: + require(observed.get('clockDomain') == request['clockDomain'] + and type(observed.get('callStartedMonotonicNs')) is int + and type(observed.get('callFinishedMonotonicNs')) is int + and ack['collectorCallStartedMonotonicNs'] <= observed['callStartedMonotonicNs'] + < observed['callFinishedMonotonicNs'] <= ack['collectorCallFinishedMonotonicNs'], + 'DL actual client interval escaped the original collection window') + row.update(ackPresent=True, ack=ack, resolution='UNKNOWN_RESULT' if observed.get('resultUnknown') is True + else 'REQUIRES_INDEPENDENT_RECONCILIATION') + rows.append(row) + return rows + + +class LoadJournal(Recorder): + """One worker serializes calls; the base Recorder persists every raw ACK.""" + def __init__(self, probe, output, role, *, writer=None): + require(role in {'writer', 'reader'}, 'invalid DL worker role') + super().__init__(probe, output, writer=writer) + self.role, self.serial, self.attempted = role, threading.Lock(), 0 + + def request(self, step, expected_http, deadline): + with self.serial: + self.attempted += 1 + if self.mutations.poisoned is not None: + return {'attempt': self.attempted, 'operation': step['operation'], 'row': None, + 'error': {'type': 'NotAdmitted', 'message': 'DL worker stopped after its original failure'}} + error = None + try: + super().call(step['operation'], step['params'], step.get('body'), + identity={'endpoint': self.probe.endpoint_name, 'role': self.role, + 'step': self.attempted, 'nonce': step.get('nonce')}, + expect_http=expected_http, admission_deadline=deadline) + except Exception as exc: + error = {'type': type(exc).__name__, 'message': str(exc)} + self.mutations.poison('DL original call failed; no further calls') + rows = read_journal(self.output, self.probe.binding, self.probe.endpoint_name, self.role) + # No new durable request means no invocation by Recorder. Do not + # reuse the previous call's receipt after admission/validation fails. + row = rows[-1] if rows and rows[-1]['request']['identity']['step'] == self.attempted else None + return {'attempt': self.attempted, 'operation': step['operation'], 'error': error, 'row': row} + + +def validate_call(result, fixture, endpoint, *, readable): + row = result['row'] + require(row is not None and row['ackPresent'], 'DL call has no exact durable ACK; retain its original request') + observation = row['ack']['observation'] + request, operation = row['request'], row['request']['operation'] + role, ordinal = request['identity']['role'], request['identity']['step'] + steps = mutation_steps(fixture, endpoint) if role == 'writer' else read_steps(fixture, endpoint) + require(type(ordinal) is int and 1 <= ordinal <= len(steps), 'DL request escaped its exact planned ordinal') + step = steps[ordinal - 1] + expected_http = 503 if role == 'writer' else [200, 503] if operation.startswith('list_') else 200 if readable else 503 + require(operation == step['operation'] and request['params'] == step['params'] + and request['identity']['nonce'] == step.get('nonce') and request['expectHttp'] == expected_http + and (request.get('payload') == digest(step['body']) if 'body' in step else 'payload' not in request), + 'DL original selector/nonce/body/status policy differs from its planned request') + response = observation.get('response') or {} + status = response.get('ResponseMetadata', {}).get('HTTPStatusCode') + require(observation.get('clientCallInvoked') is True and observation.get('resultUnknown') is False + and row['ack'].get('operationCheck') == 'PASS' + and not observation.get('clientCleanupError') and type(status) is int + and observation.get('acknowledged') is (200 <= status < 300), + 'DL request is uninvoked, uncertain, inconsistent or incompletely collected') + if operation in MUTATIONS: + require(status == 503 and response.get('Error', {}).get('Code') == 'ServiceUnavailable', + 'DL W/lock-negative mutation returned an unsafe ACK or a different frozen refusal') + return {'kind': 'REFUSED_REQUIRES_RECONCILIATION', 'originalInvocationProved': True, + 'rollbackClaimed': False} + if status == 503: + code = response.get('Error', {}).get('Code') + require((operation == 'head_object' and code == '503') + or operation == 'get_object' and code == 'ServiceUnavailable' + or operation.startswith('list_') and code in {'ServiceUnavailable', 'SlowDownRead'}, + 'DL read lacks its exact structured candidate refusal') + require(not readable or operation.startswith('list_'), 'DL lost its R-capable stable current control') + require('payload' not in observation, 'refused DL read returned a payload') + return {'kind': 'READ_REFUSED', 'globalCompletenessClaimed': False} + require(status == 200, 'DL read returned an unexpected S3 status') + current = fixture['sides'][endpoint]['versions'][-1] + if operation in {'get_object', 'head_object'}: + require(readable and request['params']['VersionId'] == current['versionId'] + and response.get('VersionId') == current['versionId'] + and response.get('ContentLength') == current['payload']['size'], 'DL stable version identity or length differs') + if operation == 'get_object': + require(observation.get('payload') == current['payload'], 'DL GET changed the complete original payload') + else: + require({name: response.get(name) for name in fixture['sides'][endpoint]['heads'][current['versionId']]} + == fixture['sides'][endpoint]['heads'][current['versionId']], + 'DL HEAD changed an original ACK property or Last-Modified') + else: + require(operation in READS and response.get('IsTruncated') is False, 'DL scoped listing is incomplete or unrecognized') + if operation == 'list_parts': + wanted = fixture['sides'][endpoint]['uploads']['complete'] + require(response.get('Key') == wanted['key'] and response.get('UploadId') == wanted['uploadId'], + 'DL parts list refers to another upload') + known = {p['PartNumber']: p for p in wanted['parts']} + identities = [] + for part in response.get('Parts', []): + number = part['PartNumber'];identities.append(number) + require(number in known and part.get('ETag') == known[number]['ETag'] + and part.get('Size') == known[number]['payload']['size'], 'DL parts list changed an acknowledged part') + else: + items = response.get('Contents', []) if operation == 'list_objects_v2' else ( + response.get('Versions', []) + response.get('DeleteMarkers', []) if operation == 'list_object_versions' + else response.get('Uploads', [])) + identities = [] + for item in items: + require(isinstance(item.get('Key'), str) and item['Key'].startswith(f'dl/{endpoint}/'), 'DL list escaped its exact owned prefix') + identities.append((item['Key'], item.get('VersionId'), item.get('UploadId'))) + require(len(identities) <= 32 and len(set(identities)) == len(identities), 'DL list exceeded its cap or duplicated an identity') + return {'kind': 'READ_OBSERVED', 'globalCompletenessClaimed': False} + + +def client_overlap_pairs(readers, writers): + pairs = [] + for a in readers: + for b in writers: + if a['ackPresent'] and b['ackPresent']: + x, y = a['ack']['observation'], b['ack']['observation'] + if (x.get('clientCallInvoked') is True and y.get('clientCallInvoked') is True + and x['clockDomain'] == y['clockDomain'] + and max(x['callStartedMonotonicNs'], y['callStartedMonotonicNs']) + < min(x['callFinishedMonotonicNs'], y['callFinishedMonotonicNs'])): + pairs.append({'readerRequestSha256': a['requestSha256'], 'writerRequestSha256': b['requestSha256']}) + return pairs + + +def concurrent_load(journals, fixture, readable): + """The controller owns Fleet evidence. Workers only append their own ACKs.""" + require(len(journals) == 4 and set(journals) == {(endpoint, role) for endpoint in fixture['sides'] + for role in ('reader', 'writer')}, 'DL requires exactly four independent original endpoint/role journals') + require(len({j.output for j in journals.values()}) == 4 and len({id(j) for j in journals.values()}) == 4, + 'DL workers must not share request sequences or mutable journal state') + require(set(readable) == set(fixture['sides']) and all(type(value) is bool for value in readable.values()) + and all(journal.probe.endpoint_name == endpoint and journal.role == role + and journal.probe.binding['runId'] == fixture['runId'] + and binding_digest(journal.probe.binding) == fixture['bindingSha256'] + and source_reference(journal.probe.binding['candidateCommit']) == fixture['sourceContract'] + for (endpoint, role), journal in journals.items()), 'DL original run/source/endpoint binding differs') + gate = threading.Barrier(4) + deadline = time.monotonic() + ADMISSION_SECONDS + def worker(endpoint, role): + journal = journals[endpoint, role] + steps = mutation_steps(fixture, endpoint) if role == 'writer' else read_steps(fixture, endpoint) + results, error = [], None + try: + gate.wait(timeout=5) + for step in steps: + expected = 503 if role == 'writer' else (200, 503) if step['operation'].startswith('list_') else 200 if readable[endpoint] else 503 + result = journal.request(step, expected, deadline) + results.append(result) + result['observationProof'] = validate_call(result, fixture, endpoint, readable=readable[endpoint]) + require(result['error'] is None, 'DL collector failed; preserve the original bound observation') + except Exception as exc: + journal.mutations.poison('DL worker stopped; drain only') + error = {'type': type(exc).__name__, 'message': str(exc)} + return {'endpoint': endpoint, 'role': role, 'plannedCalls': len(steps), 'results': results, + 'complete': error is None and len(results) == len(steps), 'error': error} + with ThreadPoolExecutor(max_workers=4) as pool: + futures = [pool.submit(worker, endpoint, role) for endpoint, role in journals] + results = [future.result() for future in futures] + overlaps = [] + for endpoint in fixture['sides']: + rows = {role: read_journal(journals[endpoint, role].output, journals[endpoint, role].probe.binding, + endpoint, role) for role in ('reader', 'writer')} + pairs = client_overlap_pairs(rows['reader'], rows['writer']) + overlaps.append({'endpoint': endpoint, 'overlappingClientCalls': pairs, 'serverCommitFenceClaimed': False}) + return {'workers': results, 'overlaps': overlaps, + 'complete': all(r['complete'] for r in results) and all(r['overlappingClientCalls'] for r in overlaps), + 'allCallsDrained': True, 'automaticRetries': 0, 'mutationReconciliationRequired': True, + 'canonicalCasePassClaimed': False} diff --git a/test_validation_heal_network_load.py b/test_validation_heal_network_load.py new file mode 100644 index 00000000..6b4cade0 --- /dev/null +++ b/test_validation_heal_network_load.py @@ -0,0 +1,268 @@ +"""DL ledger binding and actual client overlap; mocks are not live certification.""" + +import copy +from datetime import datetime, timezone, timedelta +import hashlib +import json +from pathlib import Path +import tempfile +import threading +import time +import unittest + +from scripts.validation_heal import network_load as load +from scripts.validation_heal.heal_probe import CLOCK_DOMAIN, save +from scripts.validation_heal.heal_workload import Recorder + + +class Probe: + def __init__(self, binding, endpoint='node1', *, fixture=None, role=None, rendezvous=None, unknown=False, readable=True): + self.binding, self.endpoint_name = binding, endpoint + self.fixture, self.role, self.rendezvous, self.unknown = fixture, role, rendezvous, unknown + self.readable = readable + self.calls, self.objects, self.sequence = [], {}, 0 + + def s3(self, operation, params, body=None): + self.calls.append((operation, copy.deepcopy(params))) + self.sequence += 1 + started = time.monotonic_ns() + if self.rendezvous and self.sequence == 1: + mine, other = self.rendezvous[self.role], self.rendezvous['reader' if self.role == 'writer' else 'writer'] + mine.set() + if not other.wait(2): raise ValueError('test client calls did not overlap') + status, result, uncertain = 200, {}, False + if self.role == 'writer': + status = 503 + result = {'Error': {'Code': 'ServiceUnavailable'}} + if self.unknown: result, uncertain = {}, True + elif self.role == 'reader': + side = self.fixture['sides'][self.endpoint_name] + current = side['versions'][-1] + if operation in {'get_object', 'head_object'}: + result = {**side['heads'][current['versionId']], 'VersionId': current['versionId']} + if not self.readable: + status = 503 + result = {'Error': {'Code': '503' if operation == 'head_object' else 'ServiceUnavailable'}} + else: + result = {'IsTruncated': False} + if operation == 'list_parts': + item = side['uploads']['complete'] + result.update(Key=item['key'], UploadId=item['uploadId'], Parts=[ + {'PartNumber': p['PartNumber'], 'ETag': p['ETag'], 'Size': p['payload']['size']} for p in item['parts']]) + elif operation == 'get_bucket_versioning': result = {'Status': 'Enabled'} + elif operation == 'put_object': + version, etag = f'v{self.sequence}', hashlib.md5(body).hexdigest() + result = {'VersionId': version, 'ETag': etag} + self.objects[params['Key'], version] = {'VersionId': version, 'ETag': etag, + 'LastModified': datetime(2026, 10, 19, tzinfo=timezone.utc) + timedelta(seconds=self.sequence), 'ContentLength': len(body), + 'ContentType': params['ContentType'], 'Metadata': params['Metadata']} + elif operation == 'head_object': result = copy.deepcopy(self.objects[params['Key'], params['VersionId']]) + elif operation == 'create_multipart_upload': result = {'UploadId': f'upload-{self.sequence}'} + elif operation == 'upload_part': result = {'ETag': hashlib.md5(body).hexdigest()} + response = {**result, 'ResponseMetadata': {'HTTPStatusCode': status}} + out = {'operation': operation, 'params': copy.deepcopy(params), 'endpoint': self.endpoint_name, + 'clientCallInvoked': True, 'acknowledged': status < 300 and not uncertain, + 'resultUnknown': uncertain, 'response': None if uncertain else response, + 'clockDomain': CLOCK_DOMAIN, 'callStartedMonotonicNs': started, 'callFinishedMonotonicNs': time.monotonic_ns()} + if body is not None: out['payload'] = load.digest(body) + if operation == 'get_object' and self.fixture and status == 200: out['payload'] = current['payload'] + return out + + +class NetworkLoadTests(unittest.TestCase): + def binding(self): + return {'runId': 'gh-12345-1-h2', 'candidateCommit': '1c7fc818dd858247c45b39542abd8291b1e6e31b', + 'bucketPrefix': 'rv-heal-owned', 'maxInventoryBytes': 512 * 1024**2, + 'maxInventoryObjects': 128, 'endpoints': {'node1': 'http://one:9000', 'node3': 'http://three:9000'}} + + def fixture(self, root): + probe = Probe(self.binding()) + recorder = Recorder(probe, root / 'seed') + fixture = load.seed_load(recorder, 'rv-heal-owned', ['node1', 'node3']) + return fixture, probe + + def journals(self, root, fixture, *, unknown_endpoint=None): + journals, probes = {}, {} + for endpoint in fixture['sides']: + rendezvous = {'writer': threading.Event(), 'reader': threading.Event()} + for role in ('reader', 'writer'): + probe = Probe(self.binding(), endpoint, fixture=fixture, role=role, rendezvous=rendezvous, + unknown=endpoint == unknown_endpoint and role == 'writer') + probes[endpoint, role] = probe + journals[endpoint, role] = load.LoadJournal(probe, root / f'{endpoint}-{role}', role) + return journals, probes + + def test_real_seed_preserves_three_distinct_versions_and_all_actual_mpu_ack_ids(self): + with tempfile.TemporaryDirectory() as d: + fixture, probe = self.fixture(Path(d)) + self.assertEqual(len(fixture['expected']), 6) + self.assertEqual(len({row['payload']['sha256'] for row in fixture['expected']}), 6) + self.assertLess(fixture['seedPayloadBytes'], 32 * 1024**2) + uploads = [u for side in fixture['sides'].values() for u in side['uploads'].values()] + self.assertEqual(len({u['uploadId'] for u in uploads}), 6) + for side in fixture['sides'].values(): + complete = side['uploads']['complete'] + body = b''.join(load.part_body(fixture['bucket'], complete['key'], p['PartNumber'], p['payload']['size']) for p in complete['parts']) + self.assertEqual(complete['completePayload'], load.digest(body)) + self.assertEqual([p['PartNumber'] for p in complete['parts']], [1, 2]) + self.assertEqual(len(side['heads']), 3) + steps = load.mutation_steps(fixture, 'node1') + self.assertEqual([s['nonce'] for s in steps], list(range(3, 11))) + self.assertEqual(steps[2]['params']['CopySource']['VersionId'], fixture['sides']['node1']['versions'][0]['versionId']) + self.assertEqual(steps[3]['params']['VersionId'], fixture['sides']['node1']['versions'][0]['versionId']) + self.assertEqual(steps[6]['params']['MultipartUpload']['Parts'], [ + {k: p[k] for k in ('PartNumber', 'ETag')} for p in fixture['sides']['node1']['uploads']['complete']['parts']]) + + def test_four_independent_journals_drain_full_matrix_with_actual_client_overlap(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d);fixture, _ = self.fixture(root) + journals, probes = self.journals(root, fixture) + result = load.concurrent_load(journals, fixture, {'node1': True, 'node3': True}) + self.assertTrue(result['complete']) + self.assertTrue(result['allCallsDrained']) + self.assertTrue(result['mutationReconciliationRequired']) + self.assertFalse(result['canonicalCasePassClaimed']) + self.assertEqual(sum(len(r['results']) for r in result['workers']), 28) + self.assertTrue(all(row['overlappingClientCalls'] and not row['serverCommitFenceClaimed'] for row in result['overlaps'])) + for (endpoint, role), journal in journals.items(): + rows = load.read_journal(journal.output, journal.probe.binding, endpoint, role) + self.assertEqual(len(rows), 8 if role == 'writer' else 6) + self.assertEqual(len(probes[endpoint, role].calls), len(rows)) + self.assertTrue(all(r['ackPresent'] and r['requestSha256'] == r['ack']['requestSha256'] for r in rows)) + self.assertTrue(all(r['resolution'] == 'REQUIRES_INDEPENDENT_RECONCILIATION' for r in rows)) + + def test_unknown_write_stops_that_worker_without_reusing_or_retrying_its_receipt(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d);fixture, _ = self.fixture(root) + journals, probes = self.journals(root, fixture, unknown_endpoint='node3') + result = load.concurrent_load(journals, fixture, {'node1': True, 'node3': True}) + self.assertFalse(result['complete']) + journal = journals['node3', 'writer'] + rows = load.read_journal(journal.output, journal.probe.binding, 'node3', 'writer') + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]['resolution'], 'UNKNOWN_RESULT') + follow = journal.request(load.mutation_steps(fixture, 'node3')[1], 503, time.monotonic() + 1) + self.assertIsNone(follow['row']) + self.assertIsNotNone(follow['error']) + self.assertEqual(len(probes['node3', 'writer'].calls), 1) + + def test_minority_head_uses_bodyless503_and_current_get_requires_lock_error(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d);fixture, _ = self.fixture(root) + journals, probes = self.journals(root, fixture) + probes['node3', 'reader'].readable = False + result = load.concurrent_load(journals, fixture, {'node1': True, 'node3': False}) + self.assertTrue(result['complete']) + rows = load.read_journal(journals['node3', 'reader'].output, self.binding(), 'node3', 'reader') + for row in rows[:2]: + self.assertEqual(load.validate_call({'row': row}, fixture, 'node3', readable=False)['kind'], 'READ_REFUSED') + wrong = copy.deepcopy(rows[0]);wrong['ack']['observation']['response']['Error']['Code'] = 'SlowDownRead' + with self.assertRaises(ValueError): load.validate_call({'row': wrong}, fixture, 'node3', readable=False) + + def test_changed_url_source_or_run_binding_stops_before_any_invocation(self): + for kind in ('url', 'source', 'run', 'role'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as d: + root = Path(d);fixture, _ = self.fixture(root) + journals, probes = self.journals(root, fixture) + probe = probes['node3', 'writer'] + if kind == 'url': probe.binding['endpoints']['node3'] = 'http://foreign:9000' + elif kind == 'source': probe.binding['candidateCommit'] = '072b57b37559fbe626aeda8abf61617fdcbdd926' + elif kind == 'run': probe.binding['runId'] = 'gh-54321-1-h2' + else: journals['node3', 'writer'].role = 'reader' + with self.assertRaises(ValueError): load.concurrent_load(journals, fixture, {'node1': True, 'node3': True}) + self.assertTrue(all(not p.calls for p in probes.values())) + + def test_lost_ack_file_remains_unknown_and_cannot_become_a_refusal(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d);fixture, _ = self.fixture(root) + probe = Probe(self.binding(), fixture=fixture, role='writer') + def fail_ack(path, value): + if path.name.endswith('-ack.json'): raise OSError('receipt disk unavailable') + save(path, value) + journal = load.LoadJournal(probe, root / 'lost-ack', 'writer', writer=fail_ack) + result = journal.request(load.mutation_steps(fixture, 'node1')[0], 503, time.monotonic() + 1) + self.assertFalse(result['row']['ackPresent']) + self.assertEqual(result['row']['resolution'], 'UNKNOWN_PREPARED_REQUEST') + with self.assertRaises(ValueError): load.validate_call(result, fixture, 'node1', readable=True) + self.assertEqual(len(probe.calls), 1) + + def test_wrong_hash_selector_endpoint_or_actual_clock_cannot_bind_an_ack(self): + for kind in ('hash', 'endpoint', 'selector', 'clock', 'clock-range', 'missing-operation'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as d: + root = Path(d);fixture, _ = self.fixture(root) + probe = Probe(self.binding(), fixture=fixture, role='writer') + journal = load.LoadJournal(probe, root / 'writer', 'writer') + journal.request(load.mutation_steps(fixture, 'node1')[0], 503, time.monotonic() + 1) + path = journal.output / '000001-ack.json';ack = json.loads(path.read_text()) + if kind == 'hash': ack['requestSha256'] = 'f' * 64 + elif kind == 'endpoint': ack['observation']['endpoint'] = 'node3' + elif kind == 'selector': ack['observation']['params']['Key'] = 'other' + elif kind == 'clock': ack['observation']['clockDomain'] = 'foreign' + elif kind == 'clock-range': ack['observation']['callStartedMonotonicNs'] = ack['collectorCallFinishedMonotonicNs'] + 1 + else: ack['observation'].pop('operation') + path.write_text(json.dumps(ack)) + with self.assertRaises(ValueError): load.read_journal(journal.output, journal.probe.binding, 'node1', 'writer') + + def test_success_500_wrong_error_and_uninvoked_write_are_never_safe_refusals(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d);fixture, _ = self.fixture(root) + journal = load.LoadJournal(Probe(self.binding(), fixture=fixture, role='writer'), root / 'writer', 'writer') + result = journal.request(load.mutation_steps(fixture, 'node1')[0], 503, time.monotonic() + 1) + self.assertFalse(load.validate_call(result, fixture, 'node1', readable=True)['rollbackClaimed']) + for kind in ('success', '500', 'read-code', 'uninvoked', 'unknown', 'collector-failed'): + bad = copy.deepcopy(result);obs = bad['row']['ack']['observation'] + if kind == 'success': obs['acknowledged'] = True;obs['response']['ResponseMetadata']['HTTPStatusCode'] = 200 + elif kind == '500': obs['response']['ResponseMetadata']['HTTPStatusCode'] = 500;obs['response']['Error']['Code'] = 'InternalError' + elif kind == 'read-code': obs['response']['Error']['Code'] = 'SlowDownRead' + elif kind == 'uninvoked': obs['clientCallInvoked'] = False + elif kind == 'collector-failed': bad['row']['ack']['operationCheck'] = 'FAIL' + else: obs['resultUnknown'] = True + with self.subTest(kind=kind), self.assertRaises(ValueError): load.validate_call(bad, fixture, 'node1', readable=True) + + def test_rehashed_ack_cannot_substitute_an_unplanned_key_nonce_or_mpu_id(self): + for kind in ('key', 'nonce', 'body', 'mpu'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as d: + root = Path(d);fixture, _ = self.fixture(root) + journal = load.LoadJournal(Probe(self.binding(), fixture=fixture, role='writer'), root / 'writer', 'writer') + steps = load.mutation_steps(fixture, 'node1')[:6 if kind == 'mpu' else 1] + for step in steps: journal.request(step, 503, time.monotonic() + 1) + sequence = len(steps) + request_path = journal.output / f'{sequence:06d}-request.json';ack_path = journal.output / f'{sequence:06d}-ack.json' + request, ack = json.loads(request_path.read_text()), json.loads(ack_path.read_text()) + if kind == 'key': request['params']['Key'] = 'dl/node1/unplanned.bin' + elif kind == 'nonce': request['identity']['nonce'] = 4;ack['identity']['nonce'] = 4 + elif kind == 'mpu': request['params']['UploadId'] = 'unacknowledged-upload-id' + else: request['payload']['sha256'] = 'f' * 64 + ack['observation']['params'] = copy.deepcopy(request['params']) + request_path.write_text(json.dumps(request));ack['requestSha256'] = hashlib.sha256(request_path.read_bytes()).hexdigest() + ack_path.write_text(json.dumps(ack)) + row = load.read_journal(journal.output, self.binding(), 'node1', 'writer')[-1] + with self.assertRaisesRegex(ValueError, 'planned request'): load.validate_call({'row': row}, fixture, 'node1', readable=True) + + def test_finite_status_tuple_is_preserved_and_invalid_sets_fail_before_invocation(self): + with tempfile.TemporaryDirectory() as d: + probe = Probe(self.binding(), role='writer');recorder = Recorder(probe, Path(d) / 'statuses') + params = {'Bucket': 'rv-heal-owned', 'Key': 'one'} + recorder.call('put_object', params, b'body', expect_http=(200, 503)) + request = json.loads((recorder.output / '000001-request.json').read_text()) + self.assertEqual(request['expectHttp'], [200, 503]) + self.assertEqual(len(probe.calls), 1) + for value in ((), (503, 503), (True,), (99,), (200, 201, 202, 503), [200, 503], '503'): + with self.subTest(value=value), self.assertRaises(ValueError): recorder.call('put_object', params, b'body', expect_http=value) + self.assertEqual(len(probe.calls), 1) + + def test_wrapper_overlap_and_distinct_clock_domains_cannot_prove_sdk_overlap(self): + def row(name, start, end, clock=CLOCK_DOMAIN): + return {'requestSha256': name, 'ackPresent': True, 'ack': {'collectorCallStartedMonotonicNs': 1, + 'collectorCallFinishedMonotonicNs': 100, 'observation': {'clientCallInvoked': True, + 'clockDomain': clock, 'callStartedMonotonicNs': start, 'callFinishedMonotonicNs': end}}} + reader, writer = row('read', 10, 20), row('write', 21, 30) + self.assertEqual(load.client_overlap_pairs([reader], [writer]), []) + writer['ack']['observation']['callStartedMonotonicNs'] = 19 + self.assertEqual(load.client_overlap_pairs([reader], [writer]), [{'readerRequestSha256': 'read', 'writerRequestSha256': 'write'}]) + writer['ack']['observation']['clockDomain'] = 'foreign' + self.assertEqual(load.client_overlap_pairs([reader], [writer]), []) + + +if __name__ == '__main__': + unittest.main() From c9179ceb3620f731bcbe62f989ee6aa861f95e4a Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 19:52:52 +0800 Subject: [PATCH 238/343] test(heal): reconcile network load effects before cleanup --- docs/validation-migration-map.md | 35 +- scripts/validation_heal/fleet_node.py | 1 + scripts/validation_heal/healthy.py | 3 + scripts/validation_heal/mpu_scene.py | 101 +++++ scripts/validation_heal/network_load.py | 53 ++- .../validation_heal/network_load_session.py | 355 ++++++++++++++++++ test_validation_heal_mpu_scene.py | 118 ++++++ test_validation_heal_network_load_session.py | 272 ++++++++++++++ 8 files changed, 917 insertions(+), 21 deletions(-) create mode 100644 scripts/validation_heal/mpu_scene.py create mode 100644 scripts/validation_heal/network_load_session.py create mode 100644 test_validation_heal_mpu_scene.py create mode 100644 test_validation_heal_network_load_session.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 4db7461a..f0b8d0d1 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2040,7 +2040,7 @@ Original failures and original reports remain unchanged. | --- | --- | --- | | DV | Reuses auto-test `seed(DV)` and `suspend_fixture`: four keys with three distinct non-null data versions each; real suspended overwrites, null marker creation/removal ACKs; retained live/null marker identities; current and historical markers. Versioning returns to Enabled before partition. The fault targets a retained null data version. Every selected key version participates in GET, negative Heal and complete recursive recovery. All 15 retained data versions receive full physical shard checks; markers receive exact S3 refusal/header checks. HEAD preserves version identity, length, ETag, Last-Modified and metadata. | This expanded fixture still needs a new live run and independent evidence acceptance. It does not claim a partition while the bucket remains Suspended. | | DB | Adds a separate never-versioned run-owned bucket with the default 128 controls. The exact candidate size contract and actual EC:8 process/object geometry calibrate zero/one, 4 KiB, inline, EC block and source-bound 5-MiB MPU minimum minus/exact/plus sizes. All 128 controls require decoded full-set shape witnesses; every distinct size has partition GET, physical checksum/no-part and Range evidence. Original/fault/rejoin/repaired snapshots explicitly include both primary and DB buckets. All peers must preserve complete inventories and stable original HEADs. | Fresh live execution and independent acceptance remain required. The MPU-size controls are single PUT objects; they do not claim completed multipart protocol coverage, which remains part of DL. | -| DL | `network_load.py` adds the request-layer foundation: three distinct control versions per endpoint, response-bound MPU UploadId/part ETag fixtures and four independent worker journals. Its 28-call matrix includes concurrent PUT/GET/HEAD/LIST/COPY/DELETE, version selectors and actual Create/UploadPart/Complete/Abort MPU requests. Requests bind exact selectors, nonce/body/status policy and worker/run/source/endpoint URL fingerprint; overlap uses actual SDK intervals. | This module is not yet wired into the live adapter. Add complete post-rejoin object/version/MPU reconciliation, failure-scene retention and recovery/isolation integration before dispatch. Mocked matrix completion is not DL live or canonical acceptance. | +| DL | `network_load.py` supplies three distinct control versions per endpoint, response-bound MPU fixtures and four independent worker journals for its concurrent 28-call matrix. `network_load_session.py` registers seed/workers before SDK invocation, reconciles original versions and pending MPU effects after rejoin, and withholds cleanup until a stopped physical MPU census proves no files remain. Original request and ACK byte hashes remain frozen. | Session and MPU census are not yet wired into the live adapter. Independent physical repair of partially refused version deletes, retained-scene isolation and recovery integration remain required before dispatch. Modeled matrix/reconciliation tests are not DL live or canonical acceptance. | The expanded DV fixture stays within the existing 128-identity/512-MiB inventory limits. Network expiry remains 1,200 seconds; partition task deadlines, @@ -2081,9 +2081,30 @@ ACK JSON. Only actual invoked SDK intervals in one clock domain can prove client overlap; collector setup windows cannot establish it, and no server commit fence is claimed. -The pending live adapter must register the seed Recorder and every worker -journal before invocation, including failures before a complete fixture is -returned. It must preserve the original object and system MPU failure scene, -resolve every invoked mutation without silently retrying PUT/Complete, and -refuse cleanup/acceptance while any result remains unresolved. This request -layer alone neither enables a new fleet dispatch nor completes I03. +`LoadSession` now durably registers the seed Recorder and all four worker +journals before invocation. Seed failures retain their original unknown +request even when no complete fixture returns. Recovery first verifies all +original business versions on every peer, binds discovered pending upload IDs +only to original prepared Create requests, and records distinct recovery Abort +ACKs. No PUT/Complete retry or fabricated original ACK is allowed. Public +absence alone cannot permit cleanup: a stopped four-node census must also +find no remaining MPU files. The original stopped business snapshot is frozen +before load; the final stopped snapshot must preserve all original files and +formats, with only the existing physical repair oracle's exact acknowledged +version exceptions. Hidden business files also block cleanup. Both original +request and ACK byte hashes must remain unchanged. Unexpected positive mutation ACKs, changed/missing +business versions and hidden MPU orphans retain the namespace. + +`mpu_scene.py` hashes all files under eight exact run-owned bucket/key SHA-256 +prefixes on the original sixteen drives, using the immutable candidate's +multipart layout. It neither decodes opaque UploadIds nor reads credential or +other system-key contents. It requires stopped original services, matching +candidate/owner/mount identities and bounded stable regular files. The census +does not claim payload integrity or EC/bitrot correctness. + +The pending live adapter must preserve the original object and system MPU +failure scene, independently repair partial refused deletes without dropping +version assertions, and isolate unresolved scenes with the existing strict +recovery protocol. Session/census support alone neither enables a new fleet +dispatch nor completes I03. The 807-check Heal suite passes with nine platform +skips; no new live or canonical acceptance is claimed. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 79c4ac0c..208dcc3b 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1740,6 +1740,7 @@ def main(request): for action in ("incarnation-census", "incarnation-capture", "incarnation-install", "incarnation-finalize", "incarnation-phase"): actions[action] = incarnation_operation actions["task-records"] = task_records + actions["mpu-scene"] = lambda value: globals()["mpu_scene_backend"].run(value, globals()) actions["process-clock"] = process_clock actions["environment-capabilities"] = environment_capabilities for action in ("retained-scene-audit", "retained-scene-isolate"): diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index da392b20..6552f651 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -185,6 +185,9 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() + if action == "mpu-scene": + source += "\nimport types\nmpu_scene_backend = types.ModuleType('mpu_scene')\nexec(" + repr( + NODE_SOURCE.with_name("mpu_scene.py").read_text()) + ", mpu_scene_backend.__dict__)\n" if action.startswith(("retained-scene-", "owner-markers-")): source += "\nimport types\nretained_scene_backend = types.ModuleType('retained_scene')\nexec(" + repr( NODE_SOURCE.with_name("retained_scene.py").read_text()) + ", retained_scene_backend.__dict__)\n" diff --git a/scripts/validation_heal/mpu_scene.py b/scripts/validation_heal/mpu_scene.py new file mode 100644 index 00000000..ba7d2b97 --- /dev/null +++ b/scripts/validation_heal/mpu_scene.py @@ -0,0 +1,101 @@ +"""Read-only stopped I03 multipart census, scoped by original bucket/key SHA-256.""" + +import hashlib +import json +import os +from pathlib import Path +import re +import stat + + +NAMES = ('abort', 'complete', 'create', 'upload') +MAX_FILES = 4096 +MAX_BYTES = 256 * 1024**2 + + +def require(value, message): + if not value: + raise ValueError(message) + + +def keys_for(endpoints): + require(len(endpoints) == len(set(endpoints)) == 2 and 'node1' in endpoints + and set(endpoints) <= {'node1', 'node3', 'node4'}, 'I03 MPU scope requires its two actual partition sides') + return sorted(f'dl/{endpoint}/{name}.bin' for endpoint in endpoints for name in NAMES) + + +def stable(info): + return (info.st_mode, info.st_uid, info.st_gid, info.st_dev, info.st_ino, info.st_nlink, + info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +def run(request, node): + require(set(request) == {'runId', 'action', 'bucket', 'keys', 'expectedBinarySha256'} + and request['action'] == 'mpu-scene', 'MPU scene accepts only an exact owned read-only scope') + state, roots, owner = node['owned'](request['runId']) + require(owner.get('runtimeProfile') == 'lock-quorum' and owner.get('bucket') == request['bucket'] + and owner.get('binarySha256') == request['expectedBinarySha256'] + and re.fullmatch(r'[0-9a-f]{64}', request['expectedBinarySha256']) is not None and len(roots) == 4, + 'MPU scene candidate/runtime/primary bucket differs from the owned H2 fixture') + node['stopped'](owner) + require(node['digest'](node['checked'](state / 'rustfs', state)) == owner['binarySha256'], + 'MPU scene candidate binary changed') + keys = request['keys'] + require(isinstance(keys, list) and len(keys) == 8 and all(isinstance(key, str) + and re.fullmatch(r'dl/(node1|node3|node4)/(abort|complete|create|upload)\.bin', key) for key in keys), + 'MPU scene keys escaped the exact DL matrix') + endpoints = sorted({key.split('/')[1] for key in keys}) + require(keys == keys_for(endpoints), 'MPU scene omitted or duplicated an original request key') + files, directories, scopes, total = [], [], [], 0 + for disk, root in enumerate(roots): + data = node['checked'](root / 'data', root) + device = data.lstat().st_dev + for key in keys: + key_hash = hashlib.sha256(f"{request['bucket']}/{key}".encode()).hexdigest() + folder = node['checked'](data / '.rustfs.sys' / 'multipart' / key_hash, data) + scopes.append({'disk': disk, 'key': key, 'keySha256': key_hash, 'present': folder.exists()}) + if not folder.exists(): + continue + def visit(path, depth=0): + nonlocal total + path = node['checked'](path, data) + before = path.lstat() + require(depth <= 8 and before.st_dev == device and (stat.S_ISDIR(before.st_mode) + or stat.S_ISREG(before.st_mode) and before.st_nlink == 1), + 'MPU scene contains a link, mount, special file or excessive depth') + relative = path.relative_to(data).as_posix() + if stat.S_ISDIR(before.st_mode): + require(len(directories) < MAX_FILES, 'MPU scene directory budget exceeded') + directories.append({'disk': disk, 'path': relative, 'key': key}) + for child in sorted(path.iterdir()): + visit(child, depth + 1) + else: + require(len(files) < MAX_FILES and before.st_size <= 64 * 1024**2 + and total + before.st_size <= MAX_BYTES, 'MPU scene file/byte budget exceeded') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + require(stable(os.fstat(fd)) == stable(before), 'MPU file changed before hashing') + checksum, count = hashlib.sha256(), 0 + while chunk := os.read(fd, 1024**2): + checksum.update(chunk) + count += len(chunk) + require(count <= before.st_size, 'MPU file grew during hashing') + require(count == before.st_size and stable(os.fstat(fd)) == stable(before), + 'MPU file changed during hashing') + finally: + os.close(fd) + files.append({'disk': disk, 'path': relative, 'key': key, + 'size': count, 'sha256': checksum.hexdigest()}) + total += count + require(stable(path.lstat()) == stable(before), 'MPU scene changed during its stopped census') + visit(folder) + again = node['owned'](request['runId']) + node['stopped'](again[2]) + require(again == (state, roots, owner), 'MPU scene owner or original roots changed during census') + value = {'bucket': request['bucket'], 'keys': keys, 'scopes': scopes, + 'directories': directories, 'files': files, 'bytesHashed': total} + return {**value, 'runId': request['runId'], 'binarySha256': owner['binarySha256'], + 'quiescent': True, 'originalPhysicalDrives': 4, + 'sceneSha256': hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest(), + 'readOnly': True, 'rawContentsExported': False, 'payloadIntegrityClaimed': False, + 'scope': 'all files under the eight original DL bucket/key SHA-256 prefixes; not a bitrot/EC proof'} diff --git a/scripts/validation_heal/network_load.py b/scripts/validation_heal/network_load.py index 73e5d12e..65818dc1 100644 --- a/scripts/validation_heal/network_load.py +++ b/scripts/validation_heal/network_load.py @@ -30,6 +30,11 @@ 'crates/ecstore/src/set_disk/ops/object.rs': 'fdcfda1a8811a93537f9cd47cadac7380bee66b3d14242ca91500bbf38e6bedb'}, } MULTIPART_SOURCE = '1ac5c17a94845fb16f6d75b8d3692cb18f81556086d4cc4a96b63deae106ab8d' +MPU_LAYOUT_SOURCE = '479a3b3f8c9f7b43b25b3f41d1b6769a67c6dc58c15845331da87e1fbc762679' +DISK_SOURCE = { + '072b57b37559fbe626aeda8abf61617fdcbdd926': '7ab3137947d0a4f9007a40abc3e1735ec83ca820f6dc03a3a9426ae1b4f096d7', + '1c7fc818dd858247c45b39542abd8291b1e6e31b': 'cd045a322ad7e41e2253ac08554f10634192345cbd923f2854553a229c036f4b', +} MUTATIONS = frozenset({'put_object', 'copy_object', 'delete_object', 'create_multipart_upload', 'upload_part', 'complete_multipart_upload', 'abort_multipart_upload'}) READS = frozenset({'get_object', 'head_object', 'list_objects_v2', 'list_object_versions', @@ -40,7 +45,11 @@ def source_reference(candidate): require(candidate in SOURCE_FILES, 'DL operations are not reviewed for this exact candidate') return {'candidateCommit': candidate, 'sourceFiles': {**SOURCE_FILES[candidate], 'crates/ecstore/src/set_disk/mod.rs': MPU_SOURCE[candidate], + 'crates/ecstore/src/set_disk/metadata.rs': MPU_LAYOUT_SOURCE, + 'crates/ecstore/src/disk/mod.rs': DISK_SOURCE[candidate], 'crates/ecstore/src/set_disk/ops/multipart.rs': MULTIPART_SOURCE}, + 'multipartLayout': {'systemBucket': '.rustfs.sys/multipart', 'keyRoot': 'sha256(bucket + "/" + object)', + 'uploadIdDecodedForScope': False}, 'negativeObjectMutation': {'httpStatus': 503, 'code': 'ServiceUnavailable'}, 'abortRefusalIsNotRollback': True, 'listDuringPartitionIsNotGlobalCompleteness': True, 'maximumWorkers': 4, 'admissionSeconds': ADMISSION_SECONDS, @@ -164,9 +173,9 @@ def read_steps(fixture, endpoint): ] -def read_journal(directory, binding, endpoint, role): +def read_receipts(directory, binding, endpoint): """Reconstruct from original durable bytes, including requests with no ACK.""" - require(role in {'writer', 'reader'} and endpoint in binding['endpoints'], 'unknown DL worker ownership') + require(endpoint in binding['endpoints'], 'unknown DL endpoint ownership') directory = Path(directory) requests, acks = sorted(directory.glob('*-request.json')), sorted(directory.glob('*-ack.json')) require({p.name for p in acks} <= {p.name.replace('-request.json', '-ack.json') for p in requests}, @@ -176,18 +185,16 @@ def read_journal(directory, binding, endpoint, role): request = json.loads(path.read_text()) require(path.name == f'{index:06d}-request.json' and type(request['sequence']) is int and request['sequence'] == index and request.get('schemaVersion') == 2 - and request['runId'] == binding['runId'] and request['identity']['endpoint'] == endpoint - and request['identity']['role'] == role and type(request['identity']['step']) is int - and request['identity']['step'] == index, 'DL request sequence/run/worker ownership differs') + and request['runId'] == binding['runId'], 'DL request sequence/run ownership differs') validate_s3_request(binding, request['operation'], request['params']) - require(request['operation'] in (MUTATIONS if role == 'writer' else READS), 'DL operation escaped its worker role') row = {'request': request, 'requestFile': path.name, 'requestSha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'prepared': True, 'ackPresent': False, 'resolution': 'UNKNOWN_PREPARED_REQUEST'} ack_path = path.with_name(path.name.replace('-request.json', '-ack.json')) if ack_path.exists(): - ack = json.loads(ack_path.read_text()) + ack_bytes = ack_path.read_bytes() + ack = json.loads(ack_bytes) require(ack.get('schemaVersion') == 2 and ack['requestFile'] == path.name and ack['requestSha256'] == row['requestSha256'] - and ack['runId'] == request['runId'] and ack['sequence'] == index + and ack['runId'] == request['runId'] and type(ack['sequence']) is int and ack['sequence'] == index and ack['identity'] == request['identity'] and ack['clockDomain'] == request['clockDomain'], 'DL ACK is not bound to its exact durable request') observed = ack['observation'] @@ -206,12 +213,25 @@ def read_journal(directory, binding, endpoint, role): and ack['collectorCallStartedMonotonicNs'] <= observed['callStartedMonotonicNs'] < observed['callFinishedMonotonicNs'] <= ack['collectorCallFinishedMonotonicNs'], 'DL actual client interval escaped the original collection window') - row.update(ackPresent=True, ack=ack, resolution='UNKNOWN_RESULT' if observed.get('resultUnknown') is True + row.update(ackPresent=True, ackFile=ack_path.name, ackSha256=hashlib.sha256(ack_bytes).hexdigest(), + ack=ack, resolution='UNKNOWN_RESULT' if observed.get('resultUnknown') is True else 'REQUIRES_INDEPENDENT_RECONCILIATION') rows.append(row) return rows +def read_journal(directory, binding, endpoint, role): + require(role in {'writer', 'reader'}, 'unknown DL worker role') + rows = read_receipts(directory, binding, endpoint) + for index, row in enumerate(rows, 1): + request, identity = row['request'], row['request']['identity'] + require(isinstance(identity, dict) and identity.get('endpoint') == endpoint and identity.get('role') == role + and type(identity.get('step')) is int and identity['step'] == index, + 'DL request worker ownership or ordinal differs') + require(request['operation'] in (MUTATIONS if role == 'writer' else READS), 'DL operation escaped its worker role') + return rows + + class LoadJournal(Recorder): """One worker serializes calls; the base Recorder persists every raw ACK.""" def __init__(self, probe, output, role, *, writer=None): @@ -241,11 +261,8 @@ def request(self, step, expected_http, deadline): return {'attempt': self.attempted, 'operation': step['operation'], 'error': error, 'row': row} -def validate_call(result, fixture, endpoint, *, readable): - row = result['row'] - require(row is not None and row['ackPresent'], 'DL call has no exact durable ACK; retain its original request') - observation = row['ack']['observation'] - request, operation = row['request'], row['request']['operation'] +def validate_planned_request(request, fixture, endpoint, *, readable): + operation = request['operation'] role, ordinal = request['identity']['role'], request['identity']['step'] steps = mutation_steps(fixture, endpoint) if role == 'writer' else read_steps(fixture, endpoint) require(type(ordinal) is int and 1 <= ordinal <= len(steps), 'DL request escaped its exact planned ordinal') @@ -255,6 +272,14 @@ def validate_call(result, fixture, endpoint, *, readable): and request['identity']['nonce'] == step.get('nonce') and request['expectHttp'] == expected_http and (request.get('payload') == digest(step['body']) if 'body' in step else 'payload' not in request), 'DL original selector/nonce/body/status policy differs from its planned request') + + +def validate_call(result, fixture, endpoint, *, readable): + row = result['row'] + require(row is not None and row['ackPresent'], 'DL call has no exact durable ACK; retain its original request') + observation = row['ack']['observation'] + request, operation = row['request'], row['request']['operation'] + validate_planned_request(request, fixture, endpoint, readable=readable) response = observation.get('response') or {} status = response.get('ResponseMetadata', {}).get('HTTPStatusCode') require(observation.get('clientCallInvoked') is True and observation.get('resultUnknown') is False diff --git a/scripts/validation_heal/network_load_session.py b/scripts/validation_heal/network_load_session.py new file mode 100644 index 00000000..9354a4e4 --- /dev/null +++ b/scripts/validation_heal/network_load_session.py @@ -0,0 +1,355 @@ +"""Own I03 DL intents and reconcile public effects before any namespace cleanup.""" + +import copy +import hashlib +import json +from pathlib import Path +import threading + +try: + from .heal_probe import Probe, compare_inventories, require, save + from .heal_workload import Recorder, verify_seed + from .network_load import (LoadJournal, MUTATIONS, binding_digest, concurrent_load, read_journal, + read_receipts, seed_load, source_reference, validate_planned_request) + from .mpu_scene import keys_for + from .shards import files, validate_repaired_physical +except ImportError: + from heal_probe import Probe, compare_inventories, require, save + from heal_workload import Recorder, verify_seed + from network_load import (LoadJournal, MUTATIONS, binding_digest, concurrent_load, read_journal, + read_receipts, seed_load, source_reference, validate_planned_request) + from mpu_scene import keys_for + from shards import files, validate_repaired_physical + + +def json_hash(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':'), default=str).encode()).hexdigest() + + +class LoadSession: + def __init__(self, binding, bucket, endpoints, directory, *, probe_factory=Probe): + self.binding, self.bucket = copy.deepcopy(binding), bucket + self.endpoints, self.keys = tuple(endpoints), keys_for(endpoints) + require(bucket == binding['bucketPrefix'] and set(endpoints) <= set(binding['endpoints']) + and len(binding['endpoints']) == 4, 'DL session requires its original four-peer primary bucket binding') + self.directory, self.probe_factory = Path(directory), probe_factory + self.directory.mkdir(parents=True, exist_ok=False) + self.serial = threading.Lock() + self.fixture, self.baseline, self.load_result = None, None, None + self.safe_to_cleanup, self.pre_read, self.capture_count, self.attempt = False, None, 0, 0 + self.phase, self.post_mpu, self.business_baseline = 'REGISTERED', None, None + # The intent names every recorder before the first SDK call, including + # a seed that fails before it can return a complete fixture. + save(self.directory / 'intent.json', {'runId': binding['runId'], 'bucket': bucket, + 'bindingSha256': binding_digest(binding), 'sourceContract': source_reference(binding['candidateCommit']), + 'keys': self.keys, 'seedJournal': 'seed', 'workers': [ + {'endpoint': endpoint, 'role': role, 'directory': endpoint + '-' + role} + for endpoint in self.endpoints for role in ('reader', 'writer')], 'cleanupAllowed': False}) + self.seed_recorder = Recorder(probe_factory(self.binding), self.directory / 'seed') + self.journals = {(endpoint, role): LoadJournal(probe_factory(self.binding, endpoint), + self.directory / (endpoint + '-' + role), role) for endpoint in self.endpoints for role in ('reader', 'writer')} + + def seed(self): + with self.serial: + require(self.phase == 'REGISTERED' and self.seed_recorder.sequence == 0, 'DL seed cannot be retried or replaced') + self.phase = 'SEED_STARTED' + try: + self.fixture = seed_load(self.seed_recorder, self.bucket, self.endpoints) + except BaseException: + self.phase = 'SEED_FAILED' + raise + save(self.directory / 'fixture.json', self.fixture) + self.fixture_sha = json_hash(self.fixture) + self.phase = 'SEEDED' + return copy.deepcopy(self.fixture) + + def register_baseline(self, inventory): + with self.serial: + require(self.phase == 'SEEDED' and self.baseline is None and inventory['bucket'] == self.bucket, + 'DL baseline lacks its original seeded bucket or was already frozen') + subset = {**inventory, 'entries': [row for row in inventory['entries'] if row['key'].startswith('dl/')]} + verify_seed(self.fixture['expected'], subset) + self.baseline = copy.deepcopy(inventory) + save(self.directory / 'original-baseline.json', self.baseline) + self.baseline_sha = json_hash(self.baseline) + self.phase = 'BASELINED' + + def _stopped_business_scope(self, snapshot): + require(len(snapshot) == 4 and all(value.get('quiescent') is True + and type(value.get('pid')) is int and value['pid'] == 0 + and value.get('binarySha256') == self.binding['candidateBinarySha256'] + and value.get('unit') == 'rustfs-validation-heal-' + self.binding['runId'] + '.service' + and len(value['formats']) == 4 for value in snapshot.values()), + 'DL business scene lacks the stopped original sixteen-drive candidate') + require(all(entry['path'].split('/')[0] in {self.bucket, self.bucket + '-db'} + for entry in files(snapshot).values()), 'DL business scene escaped its primary/DB buckets') + + def register_business_baseline(self, snapshot): + with self.serial: + require(self.phase == 'BASELINED' and self.business_baseline is None, + 'DL original business scene must be frozen before any load') + self._stopped_business_scope(snapshot) + self.business_baseline = copy.deepcopy(snapshot) + self.business_baseline_sha = json_hash(snapshot) + save(self.directory / 'original-business-physical.json', snapshot) + + def run_load(self, readable): + with self.serial: + require(self.phase == 'BASELINED' and self.business_baseline is not None + and not (self.directory / 'load-started.json').exists(), 'DL requires one original seeded/baselined load') + save(self.directory / 'load-started.json', {'readable': readable, 'bindingSha256': binding_digest(self.binding)}) + self.phase = 'LOAD_STARTED' + try: + self.load_result = concurrent_load(self.journals, self.fixture, readable) + finally: + self.phase = 'LOAD_FINISHED' + save(self.directory / 'load-result.json', self.load_result) + return self.load_result + + def registered_receipts(self): + return {'seed': read_receipts(self.seed_recorder.output, self.binding, self.seed_recorder.probe.endpoint_name), + 'workers': [{'endpoint': endpoint, 'role': role, + 'rows': read_journal(journal.output, self.binding, endpoint, role)} + for (endpoint, role), journal in self.journals.items()]} + + def _validate_seed(self, rows): + require(len(rows) == 27, 'DL complete seed omits or adds an original request') + versions = {(r['key'], r['versionId']): r for r in self.fixture['expected']} + uploads = {u['key']: u for side in self.fixture['sides'].values() for u in side['uploads'].values()} + seen, counts = set(), {} + for row in rows: + require(row['ackPresent'], 'DL complete seed has a prepared request without its ACK') + req, obs = row['request'], row['ack']['observation'] + op, params, response = req['operation'], req['params'], obs.get('response') or {} + require(params['Bucket'] == self.bucket and obs.get('clientCallInvoked') is True + and obs.get('acknowledged') is True and obs.get('resultUnknown') is False + and row['ack']['operationCheck'] == 'PASS', 'DL seed ACK is missing, foreign or uncertain') + counts[op] = counts.get(op, 0) + 1 + if op in {'put_object', 'head_object'}: + pair = params['Key'], response.get('VersionId') if op == 'put_object' else params['VersionId'] + require(pair in versions and (op, *pair) not in seen, 'DL seed omitted or duplicated a control version ACK') + seen.add((op, *pair)) + if op == 'put_object': require(req.get('payload') == versions[pair]['payload'], 'DL original seed body changed') + elif op == 'create_multipart_upload': + require(params['Key'] in uploads and response.get('UploadId') == uploads[params['Key']]['uploadId'] + and (op, params['Key']) not in seen, 'DL seed Create ACK belongs to another upload') + seen.add((op, params['Key'])) + elif op == 'upload_part': + upload = uploads.get(params['Key']) + part = next((p for p in upload['parts'] if p['PartNumber'] == params['PartNumber']), None) if upload else None + require(part is not None and params['UploadId'] == upload['uploadId'] + and req.get('payload') == part['payload'] and response.get('ETag') == part['ETag'] + and (op, params['Key'], params['PartNumber']) not in seen, 'DL seed part ACK changed its upload/number/body') + seen.add((op, params['Key'], params['PartNumber'])) + else: + require(op == 'get_bucket_versioning' and params == {'Bucket': self.bucket} + and response.get('Status') == 'Enabled', 'DL seed operation escaped its original plan') + require(counts == {'get_bucket_versioning': 1, 'put_object': 6, 'head_object': 6, + 'create_multipart_upload': 6, 'upload_part': 8}, 'DL seed request matrix differs') + + def capture_mpu(self, fleet, stage): + with self.serial: + require(stage in {'baseline', 'failure', 'pre-reconcile', 'post-reconcile'} and len(fleet.nodes) == len(set(fleet.nodes)) == 4 + and (stage != 'post-reconcile' or self.phase == 'PUBLIC_RECONCILED'), + 'DL MPU capture requires the complete original stopped fleet and a named phase') + scenes = {node: fleet.remote(node, 'mpu-scene', bucket=self.bucket, keys=self.keys, + expectedBinarySha256=self.binding['candidateBinarySha256']) for node in fleet.nodes} + for node, scene in scenes.items(): + require(scene.get('quiescent') is True and scene.get('readOnly') is True + and scene.get('runId') == self.binding['runId'] and scene.get('bucket') == self.bucket + and scene.get('binarySha256') == self.binding['candidateBinarySha256'] + and scene.get('keys') == self.keys and scene.get('originalPhysicalDrives') == 4 + and scene.get('payloadIntegrityClaimed') is False, + 'DL MPU scene is not the original stopped candidate/key scope') + value = {name: scene[name] for name in ('bucket', 'keys', 'scopes', 'directories', 'files', 'bytesHashed')} + require(json_hash(value) == scene['sceneSha256'] and len(scene['scopes']) == 32 + and {(row['disk'], row['key']) for row in scene['scopes']} + == {(disk, key) for disk in range(4) for key in self.keys}, 'DL MPU census is incomplete or changed') + key_hashes = {key: hashlib.sha256((self.bucket + '/' + key).encode()).hexdigest() for key in self.keys} + require(all(row['keySha256'] == key_hashes[row['key']] and type(row['present']) is bool + for row in scene['scopes']), 'DL MPU census hashes refer to another bucket/key') + for kind in ('files', 'directories'): + rows = scene[kind] + require(len({(row['disk'], row['path']) for row in rows}) == len(rows) + and all(type(row['disk']) is int and 0 <= row['disk'] < 4 and row['key'] in key_hashes + and '..' not in Path(row['path']).parts + and (row['path'] == '.rustfs.sys/multipart/' + key_hashes[row['key']] + or row['path'].startswith('.rustfs.sys/multipart/' + key_hashes[row['key']] + '/')) + for row in rows), 'DL MPU file/directory escaped its original exact key scope') + self.capture_count += 1 + saved = {'stage': stage, 'bindingSha256': binding_digest(self.binding), 'nodes': scenes} + save(self.directory / f'mpu-{self.capture_count:02d}-{stage}.json', saved) + fleet.record('DL-stopped-MPU-' + stage, saved) + if stage == 'pre-reconcile': self.pre_read = json_hash(saved) + if stage == 'post-reconcile': self.post_mpu = saved + return saved + + def _call(self, recorder, operation, params, expected): + recorder.call(operation, params, expect_http=expected) + row = read_receipts(recorder.output, self.binding, recorder.probe.endpoint_name)[-1] + observed = row['ack']['observation'] + require(observed.get('clientCallInvoked') is True and observed.get('resultUnknown') is False + and row['ack']['operationCheck'] == 'PASS' and not observed.get('clientCleanupError'), + 'DL reconciliation call has no exact known invoked response') + return observed + + def _uploads(self, recorders, label, folder): + result = {} + for endpoint, recorder in recorders.items(): + observed = self._call(recorder, 'list_multipart_uploads', { + 'Bucket': self.bucket, 'Prefix': 'dl/', 'MaxUploads': 32}, 200) + response, uploads = observed['response'], [] + require(response.get('IsTruncated') is False, 'DL pending upload listing is incomplete') + for row in response.get('Uploads', []): + require(row.get('Key') in self.keys and isinstance(row.get('UploadId'), str) and row['UploadId'], + 'DL pending upload escaped an original key or omitted its opaque ID') + uploads.append((row['Key'], row['UploadId'])) + require(len(uploads) <= 32 and len(set(uploads)) == len(uploads), 'DL pending upload list duplicates/exceeds its cap') + result[endpoint] = sorted(uploads) + save(folder / (label + '.json'), result) + return result + + def _inventories(self, folder, label): + observed = {} + # Inventory itself verifies full version GET payloads, tags and current + # LIST consistency. Save each peer before checking equality. + for endpoint in self.binding['endpoints']: + value = self.probe_factory(self.binding, endpoint).inventory(self.bucket) + save(folder / (endpoint + '-' + label + '.json'), value) + observed[endpoint] = value + for value in observed.values(): compare_inventories(self.baseline, value) + return observed + + def reconcile(self): + """Retire pending uploads with distinct recovery ACKs, never retry PUT/Complete.""" + with self.serial: + self.safe_to_cleanup = False + require(self.fixture is not None and self.baseline is not None and self.pre_read is not None + and self.phase in {'BASELINED', 'LOAD_FINISHED', 'RECONCILE_FAILED'} + and self.attempt < 2 and json_hash(self.fixture) == self.fixture_sha + and json_hash(self.baseline) == self.baseline_sha, + 'DL reconciliation lacks a complete frozen seed/baseline or original stopped MPU scene') + self.attempt += 1 + self.phase = 'RECONCILING' + folder = self.directory / f'reconcile-{self.attempt:02d}' + folder.mkdir() + original = self.registered_receipts() + self.original_receipts_sha = json_hash(original) + save(folder / 'original-request-states.json', original) + recorders = {endpoint: Recorder(self.probe_factory(self.binding, endpoint), folder / endpoint) + for endpoint in self.binding['endpoints']} + try: + self._validate_seed(original['seed']) + for worker in original['workers']: + if worker['role'] == 'writer': + for row in worker['rows']: + validate_planned_request(row['request'], self.fixture, worker['endpoint'], readable=True) + self._inventories(folder, 'before-retirement') + writers = [row for worker in original['workers'] if worker['role'] == 'writer' for row in worker['rows']] + require(all(not (row['ackPresent'] and row['ack']['observation'].get('acknowledged') is True) for row in writers), + 'DL unexpected positive mutation ACK must be independently redeemed; retain the original failure') + known = {(upload['key'], upload['uploadId']) for side in self.fixture['sides'].values() for upload in side['uploads'].values()} + discovery_keys = {row['request']['params']['Key'] for row in writers if row['request']['operation'] == 'create_multipart_upload'} + pending = self._uploads(recorders, 'pending-before', folder) + discovered = {pair for rows in pending.values() for pair in rows} - known + require(all(key in discovery_keys for key, identity in discovered) + and len({key for key, identity in discovered}) == len(discovered), + 'DL observed an unbound or duplicate unknown CreateMultipartUpload effect') + registered = sorted(known | discovered) + recovery = recorders[self.binding['defaultEndpoint']] + retirements = [] + for key, upload_id in registered: + params = {'Bucket': self.bucket, 'Key': key, 'UploadId': upload_id} + # A refused abort may have partially removed upload/parts. + # These observations precede a distinct, fully restored + # cluster Abort; neither is relabeled as the original ACK. + for recorder in recorders.values(): + part = self._call(recorder, 'list_parts', {**params, 'MaxParts': 32}, (200, 404, 503)) + status = part['response']['ResponseMetadata']['HTTPStatusCode'] + require(status == 200 and part['response'].get('IsTruncated') is False + and part['response'].get('Key') == key and part['response'].get('UploadId') == upload_id + or status == 404 and part['response'].get('Error', {}).get('Code') == 'NoSuchUpload' + or status == 503 and part['response'].get('Error', {}).get('Code') in {'ServiceUnavailable', 'SlowDownRead'}, + 'DL pre-retirement parts observation is foreign or unrecognized') + response = self._call(recovery, 'abort_multipart_upload', params, (204, 404)) + status = response['response']['ResponseMetadata']['HTTPStatusCode'] + require(status == 204 and response['acknowledged'] is True + or status == 404 and response['acknowledged'] is False + and response['response'].get('Error', {}).get('Code') == 'NoSuchUpload', + 'DL recovery Abort has neither its own ACK nor exact upload absence') + retirements.append({'key': key, 'uploadId': upload_id, 'newRecoveryAbortAck': status == 204, + 'observedNoSuchUpload': status == 404, 'originalAbortAckClaimed': False}) + remaining = self._uploads(recorders, 'pending-after', folder) + require(all(not values for values in remaining.values()), 'DL pending upload survived original-intent retirement') + for key, upload_id in registered: + for recorder in recorders.values(): + absent = self._call(recorder, 'list_parts', { + 'Bucket': self.bucket, 'Key': key, 'UploadId': upload_id, 'MaxParts': 32}, 404) + require(absent['response'].get('Error', {}).get('Code') == 'NoSuchUpload' + and absent['acknowledged'] is False, 'DL retired upload still has addressable parts') + self._inventories(folder, 'after-retirement') + result = {'allOriginalBusinessVersionsPreserved': True, 'allOriginalRequestsResolved': False, + 'allPeerPendingUploadsAbsent': True, 'retirements': retirements, + 'discoveredUploadIdsAreNotCreateAcks': sorted(discovered), 'originalRequestBytesRewritten': False, + 'originalUnknownResultsRelabeledAcknowledged': False, 'PUTOrCompleteRetryIssued': False, + 'preReadStoppedMpuEvidenceSha256': self.pre_read, + 'originalLoadComplete': self.load_result is not None and self.load_result['complete'], + 'canonicalCasePassClaimed': False, 'namespaceCleanupAllowed': False, + 'stoppedPostRetirementPhysicalCheckRequired': True} + save(folder / 'public-closure.json', result) + self.public_candidate, self.public_candidate_sha = result, json_hash(result) + self.phase = 'PUBLIC_RECONCILED' + return result + except Exception as exc: + self.phase = 'RECONCILE_FAILED' + save(folder / 'unresolved.json', {'type': type(exc).__name__, 'message': str(exc), + 'namespaceCleanupAllowed': False, 'canonicalCasePassClaimed': False}) + raise + + def finish_reconciliation(self, business_after, *, repair_targets): + with self.serial: + require(self.phase == 'PUBLIC_RECONCILED' and self.post_mpu is not None + and json_hash(self.public_candidate) == self.public_candidate_sha + and self.business_baseline is not None + and json_hash(self.business_baseline) == self.business_baseline_sha, + 'DL closure lacks its original public and stopped post-retirement evidence') + self._stopped_business_scope(business_after) + require(set(business_after) == set(self.business_baseline), 'DL business scene changed original nodes') + expected = {(row['key'], row['versionId']) for row in self.baseline['entries'] if not row['deleteMarker']} + require(isinstance(repair_targets, list) and all(target['bucket'] == self.bucket + and (target['key'], target['versionId']) in expected for target in repair_targets), + 'DL repair exceptions escaped original acknowledged business versions') + if repair_targets: + physical = validate_repaired_physical(self.business_baseline, business_after, repair_targets[0], + repair_targets=repair_targets) + else: + require(files(self.business_baseline) == files(business_after) + and all(value['formats'] == business_after[node]['formats'] + for node, value in self.business_baseline.items()), + 'DL business scene changed files or formats without an independent repair') + physical = {'allOriginalPhysicalFilesAndFormatsExact': True} + save(self.directory / f'reconcile-{self.attempt:02d}' / 'final-business-physical.json', business_after) + require(all(not scene['files'] for scene in self.post_mpu['nodes'].values()), + 'DL unaccounted multipart files remain; retain the original namespace') + original = self.registered_receipts() + require(json_hash(original) == self.original_receipts_sha, 'DL original request journals changed during reconciliation') + effects = [] + rows = [('seed', row) for row in original['seed'] if row['request']['operation'] in MUTATIONS] + rows.extend((worker['endpoint'], row) for worker in original['workers'] if worker['role'] == 'writer' for row in worker['rows']) + for actor, row in rows: + operation = row['request']['operation'] + upload = operation in {'create_multipart_upload', 'upload_part', 'abort_multipart_upload'} + effects.append({'actor': actor, 'operation': operation, 'requestSha256': row['requestSha256'], + 'effect': 'NO_REMAINING_UPLOAD_AFTER_EXPLICIT_RECOVERY' if upload else + 'ORIGINAL_VERSION_PRESERVED' if actor == 'seed' or operation == 'delete_object' else 'NO_BUSINESS_COMMIT', + 'originalAckPresent': row['ackPresent'], + 'originalResultUnknown': not row['ackPresent'] or row['ack']['observation'].get('resultUnknown') is True, + 'originalAcknowledged': row['ackPresent'] and row['ack']['observation'].get('acknowledged') is True}) + result = {**self.public_candidate, 'allOriginalRequestsResolved': True, + 'requestEffectResolutions': effects, 'stoppedPostRetirementPhysicalCheckRequired': False, + 'originalBusinessPhysicalSha256': self.business_baseline_sha, + 'postRetirementBusinessPhysicalSha256': json_hash(business_after), 'businessPhysical': physical, + 'postRetirementMpuEvidenceSha256': json_hash(self.post_mpu), 'namespaceCleanupAllowed': True} + save(self.directory / f'reconcile-{self.attempt:02d}' / 'closure.json', result) + self.safe_to_cleanup, self.phase = True, 'CLOSED' + return result diff --git a/test_validation_heal_mpu_scene.py b/test_validation_heal_mpu_scene.py new file mode 100644 index 00000000..fa5bf1e6 --- /dev/null +++ b/test_validation_heal_mpu_scene.py @@ -0,0 +1,118 @@ +"""Exact stopped MPU census; read-only fixture checks do not certify EC/bitrot.""" + +import hashlib +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from scripts.validation_heal import mpu_scene as scene, fleet_node + + +class MultipartSceneTests(unittest.TestCase): + def fixture(self, root, *, running=False): + root = root.resolve() + state = root / 'state';state.mkdir() + binary = b'exact candidate fixture';(state / 'rustfs').write_bytes(binary) + owner = {'runtimeProfile': 'lock-quorum', 'bucket': 'rv-heal-owned', + 'binarySha256': hashlib.sha256(binary).hexdigest(), 'runId': 'gh-12345-1-h2'} + roots = tuple(root / f'disk-{i}' for i in range(4)) + keys = scene.keys_for(['node1', 'node3']) + raw = {} + for disk, folder in enumerate(roots): + data = folder / 'data';data.mkdir(parents=True) + for key in keys: + if key.endswith('/create.bin'): continue + digest = hashlib.sha256(f"{owner['bucket']}/{key}".encode()).hexdigest() + upload = data / '.rustfs.sys/multipart' / digest / f'opaque-upload-{disk}' + upload.mkdir(parents=True) + for name, body in [('xl.meta', f'original metadata {key}'.encode()), ('part.1', b'original full part' + bytes([disk]))]: + p = upload / name;p.write_bytes(body) + raw[p] = body + # Another key, state credentials and another system namespace are + # deliberately unreadable to the census's digest/open paths. + (data / '.rustfs.sys/config').mkdir() + (data / '.rustfs.sys/config/credentials').write_bytes(b'out of scope secret') + (state / 'credentials.env').write_bytes(b'state secret') + def stopped(value): + if running: raise ValueError('owned process still active') + node = {'owned': lambda run_id: (state, roots, owner), 'stopped': stopped, + 'checked': fleet_node.checked, 'digest': fleet_node.digest} + request = {'runId': owner['runId'], 'action': 'mpu-scene', 'bucket': owner['bucket'], + 'keys': keys, 'expectedBinarySha256': owner['binarySha256']} + return request, node, roots, raw + + def test_census_covers_exact_eight_key_prefixes_all_four_drives_without_reading_credentials(self): + with tempfile.TemporaryDirectory() as d: + request, node, roots, raw = self.fixture(Path(d)) + opened, real_open = [], os.open + def checked_open(path, *args, **kwargs): + opened.append(Path(path)) + self.assertIn(Path(path), raw) + return real_open(path, *args, **kwargs) + with patch.object(scene.os, 'open', side_effect=checked_open): value = scene.run(request, node) + self.assertEqual(len(value['scopes']), 32) + self.assertEqual(len(value['files']), 48) + self.assertEqual(len(opened), 48) + self.assertEqual(len([r for r in value['scopes'] if not r['present']]), 8) + self.assertEqual(value['bytesHashed'], sum(len(b) for b in raw.values())) + self.assertTrue(value['quiescent'] and value['readOnly']) + self.assertFalse(value['rawContentsExported'] or value['payloadIntegrityClaimed']) + expected = {(disk, str(p.relative_to(root / 'data'))): hashlib.sha256(body).hexdigest() + for disk, root in enumerate(roots) for p, body in raw.items() if p.is_relative_to(root)} + self.assertEqual({(r['disk'], r['path']): r['sha256'] for r in value['files']}, expected) + self.assertNotIn('out of scope secret', json.dumps(value)) + self.assertEqual({p: p.read_bytes() for p in raw}, raw) + + def test_capture_distinguishes_partial_refused_abort_and_empty_directories(self): + with tempfile.TemporaryDirectory() as d: + request, node, roots, raw = self.fixture(Path(d));before = scene.run(request, node) + chosen = next(p for p in raw if '/abort.bin' in str(raw[p]) and p.name == 'xl.meta') + chosen.unlink() + after = scene.run(request, node) + self.assertNotEqual(before['sceneSha256'], after['sceneSha256']) + self.assertEqual(len(before['files']) - len(after['files']), 1) + self.assertEqual(before['directories'], after['directories']) + self.assertFalse(after['payloadIntegrityClaimed']) + + def test_live_process_wrong_binary_bucket_runtime_or_key_matrix_refuses_before_file_reads(self): + for kind in ('live', 'binary', 'bucket', 'runtime', 'foreign-key', 'missing-key', 'duplicate-key', 'extra-field'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as d: + request, node, roots, raw = self.fixture(Path(d), running=kind == 'live') + if kind == 'binary': request['expectedBinarySha256'] = 'f' * 64 + elif kind == 'bucket': request['bucket'] = 'rv-heal-foreign' + elif kind == 'runtime': node['owned'](request['runId'])[2]['runtimeProfile'] = 'manual' + elif kind == 'foreign-key': request['keys'][0] = '../credentials' + elif kind == 'missing-key': request['keys'].pop() + elif kind == 'duplicate-key': request['keys'][0] = request['keys'][1] + elif kind == 'extra-field': request['path'] = '/etc' + with patch.object(scene.os, 'open', side_effect=AssertionError('census read outside validated scope')): + with self.assertRaises(ValueError): scene.run(request, node) + + def test_symlinks_hardlinks_special_files_and_budget_overflow_are_not_hashed(self): + for kind in ('symlink', 'hardlink', 'fifo', 'file-cap', 'byte-cap'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as d: + request, node, roots, raw = self.fixture(Path(d));part = next(p for p in raw if p.name == 'part.1') + if kind == 'symlink': part.unlink();part.symlink_to(Path(d).resolve() / 'state/credentials.env') + elif kind == 'hardlink': os.link(part, part.with_name('extra-link')) + elif kind == 'fifo': part.unlink();os.mkfifo(part) + with patch.object(scene, 'MAX_FILES', 1 if kind == 'file-cap' else scene.MAX_FILES), \ + patch.object(scene, 'MAX_BYTES', 1 if kind == 'byte-cap' else scene.MAX_BYTES): + with self.assertRaises(ValueError): scene.run(request, node) + + def test_changed_owner_or_process_during_capture_keeps_scene_unaccepted(self): + with tempfile.TemporaryDirectory() as d: + request, node, roots, raw = self.fixture(Path(d));original = node['owned'];calls = [] + def changing(run_id): + value = original(run_id);calls.append(run_id) + if len(calls) > 1: return value[0], value[1], {**value[2], 'binarySha256': 'f' * 64} + return value + node['owned'] = changing + with self.assertRaisesRegex(ValueError, 'owner or original roots changed'): scene.run(request, node) + self.assertEqual(len(calls), 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/test_validation_heal_network_load_session.py b/test_validation_heal_network_load_session.py new file mode 100644 index 00000000..37db91e9 --- /dev/null +++ b/test_validation_heal_network_load_session.py @@ -0,0 +1,272 @@ +"""DL lifecycle fencing, exact business ACK preservation and post-Abort physical closure.""" + +import copy +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import tempfile +import threading +import time +import unittest + +from scripts.validation_heal import network_load_session as sessions, network_load as load +from scripts.validation_heal.heal_probe import CLOCK_DOMAIN + + +class Server: + def __init__(self, directory): + self.directory = directory + self.objects, self.uploads, self.calls, self.count = {}, {}, [], 0 + self.stopped, self.fault, self.orphan, self.fail_create = False, False, False, False + self.lock = threading.Lock() + self.events = {endpoint: {role: threading.Event() for role in ('reader', 'writer')} for endpoint in ('node1', 'node3')} + self.seen = set() + + def factory(self, binding, endpoint=None): + server = self + class Probe: + def __init__(self): + self.binding, self.endpoint_name = binding, endpoint or binding['defaultEndpoint'] + def s3(self, operation, params, body=None): + assert not server.stopped, 'S3 observation while stopped' + assert (server.directory / 'intent.json').exists() + assert all((server.directory / (ep + '-' + role)).is_dir() for ep in ('node1', 'node3') for role in ('reader', 'writer')) + started = time.monotonic_ns() + with server.lock: + server.calls.append((self.endpoint_name, operation, copy.deepcopy(params))) + server.count += 1 + serial = server.count + role = 'writer' if operation in load.MUTATIONS else 'reader' + event_key = self.endpoint_name, role + if server.fault and event_key not in server.seen: + server.seen.add(event_key) + server.events[self.endpoint_name][role].set() + assert server.events[self.endpoint_name]['reader' if role == 'writer' else 'writer'].wait(2) + code, status, response, unknown = None, 200, {}, False + key = params.get('Key') + if server.fault and operation in load.MUTATIONS: + status, code = 503, 'ServiceUnavailable' + elif operation == 'get_bucket_versioning': response = {'Status': 'Enabled'} + elif operation == 'put_object': + version = f'v{serial}' + for (k, v), item in server.objects.items(): + if k == key: item['isLatest'] = False + properties = {'ETag': hashlib.md5(body).hexdigest(), 'ContentLength': len(body), + 'ContentType': params['ContentType'], 'Metadata': params['Metadata']} + server.objects[key, version] = {'key': key, 'versionId': version, 'isLatest': True, + 'deleteMarker': False, 'payload': load.digest(body), 'properties': properties, + 'tags': [{'Key': 'fixture', 'Value': 'DL'}, {'Key': 'suite', 'Value': 'heal'}]} + response = {'VersionId': version, 'ETag': properties['ETag']} + elif operation in {'head_object', 'get_object'}: + item = server.objects[key, params['VersionId']] + response = {**item['properties'], 'VersionId': item['versionId'], 'LastModified': datetime(2026, 10, 19, tzinfo=timezone.utc)} + elif operation == 'create_multipart_upload': + upload_id = f'upload-{serial}' + server.uploads[key, upload_id] = {} + response = {'UploadId': upload_id} + if server.fail_create: unknown, server.fail_create = True, False + elif operation == 'upload_part': + number = params['PartNumber'];etag = hashlib.md5(body).hexdigest() + server.uploads[key, params['UploadId']][number] = {'PartNumber': number, 'ETag': etag, 'Size': len(body)} + response = {'ETag': etag} + elif operation == 'list_multipart_uploads': + response = {'IsTruncated': False, 'Uploads': [{'Key': k, 'UploadId': u} for k, u in server.uploads if k.startswith(params['Prefix'])]} + elif operation == 'list_parts': + parts = server.uploads.get((key, params['UploadId'])) + if parts is None: status, code = 404, 'NoSuchUpload' + else: response = {'Key': key, 'UploadId': params['UploadId'], 'IsTruncated': False, 'Parts': list(parts.values())} + elif operation == 'abort_multipart_upload': + if (key, params['UploadId']) in server.uploads: + del server.uploads[key, params['UploadId']];status = 204 + else: status, code = 404, 'NoSuchUpload' + elif operation in {'list_objects_v2', 'list_object_versions'}: response = {'IsTruncated': False} + else: raise AssertionError('Unplanned operation: ' + operation) + if code: response = {'Error': {'Code': code}} + response['ResponseMetadata'] = {'HTTPStatusCode': status} + observed = {'operation': operation, 'params': copy.deepcopy(params), 'endpoint': self.endpoint_name, + 'clockDomain': CLOCK_DOMAIN, 'callStartedMonotonicNs': started, + 'clientCallInvoked': True, 'acknowledged': status < 300 and not unknown, 'resultUnknown': unknown, + 'response': None if unknown else response, 'callFinishedMonotonicNs': time.monotonic_ns()} + if body is not None: observed['payload'] = load.digest(body) + if operation == 'get_object': observed['payload'] = item['payload'] + return observed + def inventory(self, bucket): + assert not server.stopped + return {'bucket': bucket, 'entries': sorted(copy.deepcopy(list(server.objects.values())), key=lambda r: (r['key'], r['versionId']))} + return Probe() + + def business_scene(self, binding): + return {node: {'quiescent': self.stopped, 'pid': 0 if self.stopped else 101, + 'unit': 'rustfs-validation-heal-' + binding['runId'] + '.service', + 'binarySha256': binding['candidateBinarySha256'], + 'formats': [{'disk': disk, 'value': {'original': node + str(disk)}} for disk in range(4)], + 'files': [{'disk': 0, 'path': binding['bucketPrefix'] + '/dl/node1/control.bin/xl.meta', + 'size': 10, 'sha256': 'c' * 64}]} + for node in ('vm000', 'vm001', 'vm002', 'vm003')} + + def fleet(self, binding): + server = self + class Fleet: + nodes = ['vm000', 'vm001', 'vm002', 'vm003'] + def record(self, label, value): return value + def remote(self, node, action, **params): + assert server.stopped and action == 'mpu-scene' + keys, scopes, files = params['keys'], [], [] + for disk in range(4): + for key in keys: + sha = hashlib.sha256((params['bucket'] + '/' + key).encode()).hexdigest() + existing = [(k, u) for k, u in server.uploads if k == key] + scopes.append({'disk': disk, 'key': key, 'keySha256': sha, 'present': bool(existing)}) + for k, u in existing: + files.append({'disk': disk, 'key': key, 'path': f'.rustfs.sys/multipart/{sha}/{u}/xl.meta', 'size': 10, 'sha256': 'a' * 64}) + if server.orphan: + key = keys[0];sha = hashlib.sha256((params['bucket'] + '/' + key).encode()).hexdigest() + files.append({'disk': 0, 'key': key, 'path': f'.rustfs.sys/multipart/{sha}/orphan/part.1', 'size': 1, 'sha256': 'b' * 64}) + value = {'bucket': params['bucket'], 'keys': keys, 'scopes': scopes, 'directories': [], + 'files': files, 'bytesHashed': sum(x['size'] for x in files)} + return {**value, 'runId': binding['runId'], 'binarySha256': binding['candidateBinarySha256'], + 'quiescent': True, 'readOnly': True, 'originalPhysicalDrives': 4, + 'payloadIntegrityClaimed': False, 'sceneSha256': sessions.json_hash(value)} + return Fleet() + + +class LoadSessionTests(unittest.TestCase): + def session(self, root): + directory = root / 'DL';server = Server(directory) + binding = {'runId': 'gh-12345-1-h2', 'candidateCommit': '1c7fc818dd858247c45b39542abd8291b1e6e31b', + 'bucketPrefix': 'rv-heal-owned', 'maxInventoryBytes': 512 * 1024**2, 'maxInventoryObjects': 128, + 'candidateBinarySha256': 'a' * 64, 'defaultEndpoint': 'node1', + 'endpoints': {f'node{i}': f'http://node{i}:9000' for i in range(1, 5)}} + value = sessions.LoadSession(binding, binding['bucketPrefix'], ['node1', 'node3'], directory, probe_factory=server.factory) + self.assertEqual(server.calls, []) + return value, server, binding + + def prepared(self, root): + session, server, binding = self.session(root) + fixture = session.seed() + session.register_baseline(server.factory(binding).inventory(binding['bucketPrefix'])) + server.stopped = True + session.register_business_baseline(server.business_scene(binding)) + server.stopped = False + server.fault = True + self.assertTrue(session.run_load({'node1': True, 'node3': True})['complete']) + server.fault, server.stopped = False, True + session.capture_mpu(server.fleet(binding), 'pre-reconcile') + server.stopped = False + return session, server, binding + + def test_full_closure_requires_distinct_recovery_acks_and_stopped_empty_mpu_scene(self): + with tempfile.TemporaryDirectory() as d: + value, server, binding = self.prepared(Path(d)) + original = sessions.json_hash(value.registered_receipts()) + public = value.reconcile() + self.assertFalse(public['namespaceCleanupAllowed'] or value.safe_to_cleanup) + self.assertEqual(len(public['retirements']), 6) + self.assertTrue(all(r['newRecoveryAbortAck'] and not r['originalAbortAckClaimed'] for r in public['retirements'])) + with self.assertRaises(ValueError): value.finish_reconciliation(server.business_scene(binding), repair_targets=[]) + server.stopped = True + value.capture_mpu(server.fleet(binding), 'post-reconcile') + closure = value.finish_reconciliation(server.business_scene(binding), repair_targets=[]) + self.assertTrue(value.safe_to_cleanup and closure['namespaceCleanupAllowed']) + self.assertEqual(len(closure['requestEffectResolutions']), 36) + self.assertTrue(closure['allOriginalRequestsResolved']) + self.assertFalse(closure['canonicalCasePassClaimed'] or closure['originalUnknownResultsRelabeledAcknowledged']) + self.assertEqual(original, sessions.json_hash(value.registered_receipts())) + self.assertEqual(server.uploads, {}) + count = len(server.calls) + with self.assertRaises(ValueError): value.run_load({'node1': True, 'node3': True}) + self.assertEqual(len(server.calls), count) + + def test_seed_failure_keeps_original_unknown_create_even_without_returned_fixture(self): + with tempfile.TemporaryDirectory() as d: + value, server, binding = self.session(Path(d));server.fail_create = True + with self.assertRaises(ValueError): value.seed() + self.assertIsNone(value.fixture) + self.assertEqual(value.phase, 'SEED_FAILED') + receipts = value.registered_receipts() + self.assertEqual(receipts['seed'][-1]['request']['operation'], 'create_multipart_upload') + self.assertEqual(receipts['seed'][-1]['resolution'], 'UNKNOWN_RESULT') + self.assertTrue(all(not r['rows'] for r in receipts['workers'])) + count = len(server.calls) + with self.assertRaises(ValueError): value.reconcile() + self.assertFalse(value.safe_to_cleanup) + self.assertEqual(count, len(server.calls)) + self.assertTrue(server.uploads) + + def test_missing_original_version_or_unexpected_version_blocks_before_abort(self): + for kind in ('missing', 'extra'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as d: + value, server, binding = self.prepared(Path(d)) + pair = next(iter(server.objects)) + if kind == 'missing': server.objects.pop(pair) + else: + row = copy.deepcopy(server.objects[pair]);row['versionId'] = 'unexpected';server.objects[row['key'], row['versionId']] = row + before = set(server.uploads) + with self.assertRaisesRegex(ValueError, 'versions/content/metadata/tags'): value.reconcile() + self.assertFalse(value.safe_to_cleanup) + self.assertEqual(set(server.uploads), before) + self.assertTrue((value.directory / 'reconcile-01/unresolved.json').exists()) + + def test_unknown_create_discovery_is_bound_to_original_request_and_never_fabricates_ack(self): + with tempfile.TemporaryDirectory() as d: + value, server, binding = self.prepared(Path(d)) + server.uploads['dl/node1/create.bin', 'unacknowledged-original-id'] = {} + public = value.reconcile() + self.assertEqual(public['discoveredUploadIdsAreNotCreateAcks'], [('dl/node1/create.bin', 'unacknowledged-original-id')]) + server.stopped = True;value.capture_mpu(server.fleet(binding), 'post-reconcile') + closure = value.finish_reconciliation(server.business_scene(binding), repair_targets=[]) + create = [r for r in closure['requestEffectResolutions'] if r['actor'] == 'node1' and r['operation'] == 'create_multipart_upload'] + self.assertEqual(len(create), 1) + self.assertFalse(create[0]['originalAcknowledged']) + self.assertTrue(value.safe_to_cleanup) + + def test_unbound_pending_id_and_post_abort_orphan_never_allow_cleanup(self): + for kind in ('unbound-id', 'physical-orphan'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as d: + value, server, binding = self.prepared(Path(d)) + if kind == 'unbound-id': + server.uploads['dl/node1/complete.bin', 'foreign-id'] = {} + with self.assertRaisesRegex(ValueError, 'unbound'): value.reconcile() + else: + value.reconcile();server.stopped, server.orphan = True, True + value.capture_mpu(server.fleet(binding), 'post-reconcile') + with self.assertRaisesRegex(ValueError, 'multipart files remain'): value.finish_reconciliation(server.business_scene(binding), repair_targets=[]) + self.assertFalse(value.safe_to_cleanup) + + def test_hidden_business_orphan_or_changed_format_blocks_even_after_public_and_mpu_closure(self): + for kind in ('hidden-business-file', 'format', 'live', 'foreign-candidate'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as d: + value, server, binding = self.prepared(Path(d));value.reconcile() + server.stopped = True;value.capture_mpu(server.fleet(binding), 'post-reconcile') + after = server.business_scene(binding) + if kind == 'hidden-business-file': + after['vm001']['files'].append({'disk': 1, + 'path': binding['bucketPrefix'] + '/dl/node1/copy.bin/uncommitted/part.1', + 'size': 1, 'sha256': 'd' * 64}) + elif kind == 'format': after['vm001']['formats'][0]['value'] = {'foreign': True} + elif kind == 'live': after['vm001']['quiescent'] = False + else: after['vm001']['binarySha256'] = 'f' * 64 + with self.assertRaises(ValueError): + value.finish_reconciliation(after, repair_targets=[]) + self.assertFalse(value.safe_to_cleanup) + + def test_changed_original_journal_or_missing_stopped_capture_blocks_closure(self): + with tempfile.TemporaryDirectory() as d: + value, server, binding = self.session(Path(d));value.seed() + value.register_baseline(server.factory(binding).inventory(binding['bucketPrefix'])) + count = len(server.calls) + with self.assertRaises(ValueError): value.reconcile() + self.assertEqual(count, len(server.calls)) + for suffix in ('request', 'ack'): + with self.subTest(suffix=suffix), tempfile.TemporaryDirectory() as d: + value, server, binding = self.prepared(Path(d));value.reconcile() + server.stopped = True;value.capture_mpu(server.fleet(binding), 'post-reconcile') + path = value.seed_recorder.output / ('000001-' + suffix + '.json') + path.write_text(path.read_text() + ' ') + with self.assertRaises(ValueError): value.finish_reconciliation(server.business_scene(binding), repair_targets=[]) + self.assertFalse(value.safe_to_cleanup) + + +if __name__ == '__main__': + unittest.main() From c08462664bca011898068ad430841ea973ee1152 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 20:01:10 +0800 Subject: [PATCH 239/343] test(heal): preserve unresolved network load scenes --- docs/validation-migration-map.md | 22 ++ scripts/validation_heal/healthy.py | 2 +- scripts/validation_heal/network_retention.py | 147 ++++++++++++ .../validation_heal/recover_environment.py | 9 +- scripts/validation_heal/retained_scene.py | 44 +++- test_validation_heal_network_retention.py | 223 ++++++++++++++++++ test_validation_heal_retained_scene.py | 1 + 7 files changed, 437 insertions(+), 11 deletions(-) create mode 100644 scripts/validation_heal/network_retention.py create mode 100644 test_validation_heal_network_retention.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index f0b8d0d1..89d85f09 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2108,3 +2108,25 @@ version assertions, and isolate unresolved scenes with the existing strict recovery protocol. Session/census support alone neither enables a new fleet dispatch nor completes I03. The 807-check Heal suite passes with nine platform skips; no new live or canonical acceptance is claimed. + +I03 unresolved-scene isolation (2026-10-19): `network_retention.py` now freezes +the original four reservation owners and stopped business/MPU scene digests. +All four read-only preflights precede any move. The existing durable isolation +backend supports `lock-quorum` with private per-run archive parents and +non-overwriting same-filesystem renames. It hashes the primary and DB fixture, +original DL MPU key prefixes and original I03 shard-injection backups while +retaining credential files by inode/metadata without reading their contents. +Interrupted moves resume against the same durable census; missing roots, +changed archived bytes, active gates, owner/mount drift and destination +collisions block progress. Every original file and inode is retained; no shard +restoration, S3 Abort or namespace deletion is performed by isolation. + +The recovery entry point accepts this path only for one archived I03 +FAIL/cleanup-FAIL branch with an exact candidate/package/binary/helper/run +binding and four matching original reservations. It reports `ISOLATED`, never +`PASS` or `alreadyClean`, including retries after all active paths are absent. +Historical expiry-only recovery keeps its existing strict boundary. The live +I03 adapter still does not emit this new intent or invoke DL; integration, +independent partial-delete repair and fresh full-fixture live acceptance +remain required. Local filesystem/controller tests do not certify live +isolation or increase the canonical count. diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 6552f651..fe978568 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -185,7 +185,7 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() - if action == "mpu-scene": + if action == "mpu-scene" or action.startswith("retained-scene-"): source += "\nimport types\nmpu_scene_backend = types.ModuleType('mpu_scene')\nexec(" + repr( NODE_SOURCE.with_name("mpu_scene.py").read_text()) + ", mpu_scene_backend.__dict__)\n" if action.startswith(("retained-scene-", "owner-markers-")): diff --git a/scripts/validation_heal/network_retention.py b/scripts/validation_heal/network_retention.py new file mode 100644 index 00000000..6eef009e --- /dev/null +++ b/scripts/validation_heal/network_retention.py @@ -0,0 +1,147 @@ +"""Preserve unresolved I03 business and MPU bytes without claiming restoration.""" + +import copy +import hashlib +import json +import re + +try: + from .heal_probe import require + from .healthy import physical_content + from .mpu_scene import keys_for +except ImportError: + from heal_probe import require + from healthy import physical_content + from mpu_scene import keys_for + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def validate_plan(plan): + require(set(plan) == {'kind', 'runId', 'nodes', 'candidateCommit', 'harnessCommit', 'binarySha256', + 'bucket', 'owners', 'businessSceneSha256', 'mpuSceneSha256', 'mpuKeys'} + and plan['kind'] == 'unresolved-network-load' + and re.fullmatch(r'gh-[1-9][0-9]*-[1-9][0-9]*-h2', plan['runId']) + and len(plan['nodes']) == len(set(plan['nodes'])) == 4 + and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in plan['nodes']) + and re.fullmatch(r'rv-heal-[0-9a-f]{16}', plan['bucket']) + and all(re.fullmatch(r'[0-9a-f]{40}', plan[field]) for field in ('candidateCommit', 'harnessCommit')) + and re.fullmatch(r'[0-9a-f]{64}', plan['binarySha256']), 'I03 retention plan identity or scope differs') + for field in ('owners', 'businessSceneSha256', 'mpuSceneSha256'): + require(set(plan[field]) == set(plan['nodes']), 'I03 retention omits an original peer') + endpoints = sorted({key.split('/')[1] for key in plan['mpuKeys']}) + require(plan['mpuKeys'] == keys_for(endpoints), 'I03 retention MPU scope differs from its two original sides') + for node in plan['nodes']: + owner = plan['owners'][node] + require(owner.get('runId') == plan['runId'] and owner.get('bucket') == plan['bucket'] + and owner.get('runtimeProfile') == 'lock-quorum' and owner.get('binarySha256') == plan['binarySha256'] + and owner.get('unit') == 'rustfs-validation-heal-' + plan['runId'] + '.service' + and len(owner.get('mounts', [])) == 4 + and all(re.fullmatch(r'[0-9a-f]{64}', plan[field][node]) for field in ('businessSceneSha256', 'mpuSceneSha256')), + 'I03 retention lacks the exact reserved candidate/runtime owner and stopped scene digests') + require(len({owner['hostname'] for owner in plan['owners'].values()}) == 4, + 'I03 retention aliases do not identify four distinct reserved hosts') + + +def prepare(fleet, binding, keys, business, mpu_nodes): + """Freeze the actual stopped failure scene after gate release, before moving.""" + require(fleet.run_id == binding['runId'] and not fleet.network_gates and not fleet.quorum_gates + and set(business) == set(mpu_nodes) == set(fleet.nodes) and len(fleet.nodes) == 4, + 'I03 retention requires all stopped peers and released owned network/write gates') + reservations = [json.loads(path.read_text()) for path in fleet.evidence.glob('*-reserved.json')] + require(len(reservations) == 4 and {row.get('node') for row in reservations} == set(fleet.nodes) + and all(row.get('reserved') is True for row in reservations), 'I03 retention lacks its original reservations') + plan = {'kind': 'unresolved-network-load', 'runId': fleet.run_id, 'nodes': list(fleet.nodes), + 'candidateCommit': binding['candidateCommit'], 'harnessCommit': binding['harnessCommit'], + 'binarySha256': binding['candidateBinarySha256'], 'bucket': binding['bucketPrefix'], 'mpuKeys': list(keys), + 'owners': {row['node']: row['owner'] for row in reservations}, + 'businessSceneSha256': {node: digest(value) for node, value in physical_content(business).items()}, + 'mpuSceneSha256': {node: value['sceneSha256'] for node, value in mpu_nodes.items()}} + validate_plan(plan) + for node in fleet.nodes: + scene, mpu = business[node], mpu_nodes[node] + require(scene.get('quiescent') is True and type(scene.get('pid')) is int and scene['pid'] == 0 + and scene.get('unit') == plan['owners'][node]['unit'] + and scene.get('binarySha256') == plan['binarySha256'] + and len(scene['formats']) == 4 and mpu.get('keys') == keys + and mpu.get('bucket') == plan['bucket'] and mpu.get('runId') == fleet.run_id + and mpu.get('binarySha256') == plan['binarySha256'] and mpu.get('quiescent') is True + and mpu.get('readOnly') is True and mpu.get('originalPhysicalDrives') == 4 + and digest({field: mpu[field] for field in ('bucket', 'keys', 'scopes', 'directories', 'files', 'bytesHashed')}) + == mpu['sceneSha256'], 'I03 retention lacks its stopped original business/MPU capture') + return fleet.record('network-retention-intent', copy.deepcopy(plan)) + + +def isolate(fleet, plan): + """Retry the same archive intent; never recover a shard, Abort or delete data.""" + validate_plan(plan) + require(fleet.run_id == plan['runId'] and list(fleet.nodes) == plan['nodes'], 'I03 retention controller changed its original fleet') + def params(node): + return {'expectedOwner': plan['owners'][node], 'expectedOriginalSceneSha256': plan['businessSceneSha256'][node], + 'expectedMpuSceneSha256': plan['mpuSceneSha256'][node], 'mpuKeys': plan['mpuKeys']} + audits = {} + for node in fleet.nodes: + audits[node] = fleet.record('network-retention-preflight', {'node': node, + **fleet.remote(node, 'retained-scene-audit', **params(node))}) + require(audits[node].get('inspectionOnly') is True and audits[node].get('mutated') is False + and audits[node].get('credentialContentsRead') is False, + 'I03 retention preflight did not preserve read-only credential boundaries') + for node in fleet.nodes: + result = fleet.record('network-retention-isolation', {'node': node, + **fleet.remote(node, 'retained-scene-isolate', expectedBaseline=audits[node]['baseline'], **params(node))}) + require(result.get('activeNamespaceIsolated') is True and result.get('baseline') == audits[node]['baseline'] + and result.get('originalSceneRestored') is False and result.get('backupRestorationUsed') is False + and result.get('canonicalPassClaimed') is False and result.get('credentialContentsRead') is False, + 'I03 retained scene move did not preserve the exact original failure') + for node in fleet.nodes: + proof = fleet.record('network-retention-post-audit', {'node': node, + **fleet.remote(node, 'retained-scene-audit', **params(node))}) + require(proof.get('inspectionOnly') is True and proof.get('mutated') is False + and proof.get('activeNamespaceIsolated') is True and proof.get('baseline') == audits[node]['baseline'] + and len(proof.get('privateArchiveParents', [])) == 5 + and all(row.get('exists') is True and row.get('uid') == 0 and row.get('mode') == 0o700 + for row in proof['privateArchiveParents']), 'I03 retained scene post-audit drifted') + return {'runId': plan['runId'], 'cleanup': 'ISOLATED', 'activeNamespaceIsolated': True, + 'originalSceneRestored': False, 'originalEvidenceComplete': False, 'originalVerdictRewritten': False, + 'canonicalPassClaimed': False, 'namespaceDeletionPerformed': False, 'intentSha256': digest(plan)} + + +def archived_plan(artifact, failed_run, candidate): + """Select one retained branch; an isolated namespace is never already-clean.""" + case = 'heal.network-partition-and-lock-quorum' + results = list(artifact.glob(f'*/run/{case}/cases/{case}.json')) + require(type(failed_run) is int and failed_run > 0 and len(results) == 1, 'I03 isolation needs one exact failed run') + result = json.loads(results[0].read_text()) + provenance = result.get('provenance') or {} + require(result.get('case_id') == case and result.get('verdict') == 'FAIL' and result.get('cleanup_status') == 'FAIL' + and provenance.get('source_commit') == candidate['source_sha'] + and provenance.get('candidate_sha256') == candidate['package_sha256'], + 'I03 isolation failed-result candidate or cleanup differs') + evidence = results[0].parents[1] / 'evidence' / case + summary = json.loads((evidence / 'healthy-summary.json').read_text()) + retained = [row for row in summary['results'] if row.get('cleanup') != 'PASS'] + require(summary.get('cleanup') == 'FAIL' and len(retained) == 1 + and retained[0].get('verdict') == 'FAIL' and retained[0].get('topology') == 'H2' + and retained[0].get('branch') in {'minority-partition', 'read-boundary', 'write-boundary', 'lock-boundary', 'rejoin'}, + 'I03 isolation must preserve exactly one failed branch') + folder = evidence / 'H2' / retained[0]['branch'] + def one(label): + paths = list(folder.glob('*-' + label + '.json')) + require(len(paths) == 1, 'I03 isolation lacks its original ' + label) + return json.loads(paths[0].read_text()) + plan, binding = one('network-retention-intent'), one('binding') + validate_plan(plan) + require(re.fullmatch(rf'gh-{failed_run}-[1-9][0-9]*-h2', plan['runId']) + and plan['runId'] == binding.get('runId') and plan['bucket'] == binding.get('bucketPrefix') + and plan['candidateCommit'] == binding.get('candidateCommit') == candidate['source_sha'] + and plan['harnessCommit'] == binding.get('harnessCommit') == provenance.get('testing_commit') + and plan['binarySha256'] == binding.get('candidateBinarySha256') == provenance.get('binary_sha256') + and binding.get('candidatePackageSha256') == candidate['package_sha256'], + 'I03 isolation original plan/binding/provenance differs') + reservations = [json.loads(path.read_text()) for path in folder.glob('*-reserved.json')] + require(len(reservations) == 4 and {row.get('node') for row in reservations} == set(plan['nodes']) + and all(row.get('reserved') is True and row.get('owner') == plan['owners'][row['node']] for row in reservations), + 'I03 isolation owner differs from its original reservation') + return [plan] diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index cc6ce463..4ed14e3e 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -14,11 +14,13 @@ from .heal_probe import require, save from .interrupted_recovery import checkpoint_plan, restore_and_isolate from .outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace, interrupted_opaque_plan, acknowledged_legacy_plan + from .network_retention import archived_plan as network_retention_plan, isolate as isolate_network_scene except ImportError: from healthy import Fleet, physical_content from heal_probe import require, save from interrupted_recovery import checkpoint_plan, restore_and_isolate from outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace, interrupted_opaque_plan, acknowledged_legacy_plan + from network_retention import archived_plan as network_retention_plan, isolate as isolate_network_scene def quota_recovery_plan(artifact, failed_run, candidate): @@ -93,8 +95,10 @@ def one(suffix): def network_recovery_plan(artifact, failed_run, candidate): - """Recover only an I03 gate whose independent expiry unit is gone.""" + """Select exact I03 expiry recovery or unresolved-load scene isolation.""" case = "heal.network-partition-and-lock-quorum" + if list(artifact.glob(f"*/run/{case}/evidence/{case}/H2/*/*-network-retention-intent.json")): + return network_retention_plan(artifact, failed_run, candidate) results = list(artifact.glob(f"*/run/{case}/cases/{case}.json")) require(len(results) == 1, "network recovery requires one exact I03 result") result = json.loads(results[0].read_text()) @@ -639,6 +643,9 @@ def main(): if plan.get("buckets"): fleet.snapshot_buckets = plan["buckets"] fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) + if plan.get("kind") == "unresolved-network-load": + complete.append({"failedRunId": args.failed_run_id, **isolate_network_scene(fleet, plan)}) + continue if plan.get("kind") == "network-expiry-recovery": for node in plan["nodes"]: fleet.record("network-recovery-stop", {"node": node, **fleet.remote(node, "stop")}) diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py index fbde6284..4e8952f9 100644 --- a/scripts/validation_heal/retained_scene.py +++ b/scripts/validation_heal/retained_scene.py @@ -92,6 +92,13 @@ def census(folder, owner, *, state, node): """Hash the complete inode inventory; read only explicitly scoped content.""" rows, content_count = [], 0 boundary_device = folder.lstat().st_dev + network = owner.get("runtimeProfile") == "lock-quorum" + fixture_buckets = {owner["bucket"]} + if network: + fixture_buckets.add(owner["bucket"] + "-db") + multipart_keys = {hashlib.sha256(f"{owner['bucket']}/dl/{endpoint}/{name}.bin".encode()).hexdigest() + for endpoint in ("node1", "node3", "node4") + for name in ("abort", "complete", "create", "upload")} if network else set() def visit(path): nonlocal content_count node["checked"](path, folder) @@ -115,7 +122,11 @@ def visit(path): require(stable(info) == stable(after), "retained scene directory changed during audit") else: parts = Path(relative).parts - fixture = not state and len(parts) > 2 and parts[:2] == ("data", owner["bucket"]) + fixture = not state and len(parts) > 2 and parts[0] == "data" and parts[1] in fixture_buckets + network_mpu = (not state and network and len(parts) > 4 + and parts[:3] == ("data", ".rustfs.sys", "multipart") and parts[3] in multipart_keys) + network_backup = (not state and network and len(parts) == 2 and parts[0] == "quarantine" + and re.fullmatch(r"I03-missing-target\.(original|intent\.json|applied\.json)", parts[1])) mrf = not state and relative in { "data/.rustfs.sys/.heal-mrf-snapshot.0.bin", "data/.rustfs.sys/.heal-mrf-snapshot.1.bin", "data/.rustfs.sys/.heal-mrf-manifest.json"} @@ -125,7 +136,7 @@ def visit(path): opaque_backup = (not state and owner.get("runtimeProfile") == "outcome-legacy" and len(parts) == 2 and parts[0] == "quarantine" and re.fullmatch(r"J02-[0-9a-f]{32}\.(original|intent\.json|applied\.json|failed-[0-9a-f]{64})", parts[1])) - if (state and relative == "rustfs") or fixture or mrf or backup or opaque_backup: + if (state and relative == "rustfs") or fixture or mrf or backup or opaque_backup or network_mpu or network_backup: fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) try: require(stable(os.fstat(fd)) == stable(info) and info.st_size <= 2 * 1024**3, @@ -157,17 +168,17 @@ def mounted_paths(): def inspect(request, node): state, roots = node["paths"](request["runId"]) expected = request["expectedOwner"] - per_run = expected.get("runtimeProfile") == "outcome-legacy" + per_run = expected.get("runtimeProfile") in {"outcome-legacy", "lock-quorum"} retained_state, retained_roots = destinations(state, roots, per_run_parent=per_run) if per_run: old_state, old_roots = destinations(state, roots) require(not any(node["checked"](path, state.parent if index == 4 else node["MOUNTS"][index]).exists() for index, path in enumerate((*old_roots, old_state))), - "J02 shared archive already exists; refusing another retention layout") + "shared archive already exists; refusing another retention layout") require(isinstance(expected, dict) and expected.get("runId") == request["runId"] and expected.get("unit") == node["unit"](request["runId"]) and expected.get("hostname") == node["socket"].gethostname() - and expected.get("runtimeProfile") in {"mrf-crash", "root-restart", "outcome-grace", "outcome-legacy"}, "retained scene archived owner differs") + and expected.get("runtimeProfile") in {"mrf-crash", "root-restart", "outcome-grace", "outcome-legacy", "lock-quorum"}, "retained scene archived owner differs") current = node["mounts"]() require(len(current) == len(expected.get("mounts", [])) == 4 and all(all(actual[key] == old[key] for key in ("target", "source", "fstype", "uuid", "maj:min")) @@ -210,19 +221,32 @@ def inspect(request, node): baseline["archiveLayout"] = "per-run-parent" intent = retained_state.parent / (request["runId"] + ".intent.json") node["checked"](intent, state.parent) - if expected["runtimeProfile"] in {"root-restart", "outcome-grace", "outcome-legacy"}: + network = expected["runtimeProfile"] == "lock-quorum" + if expected["runtimeProfile"] in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum"}: original_sha = request.get("expectedOriginalSceneSha256", "") require(re.fullmatch(r"[0-9a-f]{64}", original_sha) is not None, "restored fixture requires its original physical scene digest") baseline["originalSceneSha256"] = original_sha + if network: + require(re.fullmatch(r"[0-9a-f]{64}", request.get("expectedMpuSceneSha256", "")) is not None, + "network retention requires its stopped original MPU scene digest") + baseline["originalMpuSceneSha256"] = request["expectedMpuSceneSha256"] + baseline["originalMpuKeysSha256"] = sha(request["mpuKeys"]) require("absent" not in locations, "restored fixture cannot omit an original root") if not intent.exists(): require(all(location == "active" for location in locations), "unbound retained fixture") - scene = node["snapshot"]({"runId": request["runId"], "action": "snapshot", "quiescent": True}) + scene = node["snapshot"]({"runId": request["runId"], "action": "snapshot", "quiescent": True, + **({"buckets": [expected["bucket"], expected["bucket"] + "-db"]} if network else {})}) require(scene.get("quiescent") is True and type(scene.get("pid")) is int and scene["pid"] == 0 and scene.get("binarySha256") == expected["binarySha256"] and sha({key: scene[key] for key in ("files", "formats")}) == original_sha, "restored fixture differs from the original business scene") + if network: + mpu = node["mpu_scene_backend"].run({"runId": request["runId"], "action": "mpu-scene", + "bucket": expected["bucket"], "keys": request["mpuKeys"], + "expectedBinarySha256": expected["binarySha256"]}, node) + require(mpu["sceneSha256"] == request["expectedMpuSceneSha256"], + "network retained multipart scene differs from the original stopped census") if intent.exists(): require(encoded(private_json(intent)) == encoded(baseline), "retained scene drifted from its durable isolation intent") else: @@ -232,8 +256,10 @@ def inspect(request, node): def run(request, node): fields = {"runId", "action", "expectedOwner"} - if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace", "outcome-legacy"}: + if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum"}: fields.add("expectedOriginalSceneSha256") + if request.get("expectedOwner", {}).get("runtimeProfile") == "lock-quorum": + fields.update({"expectedMpuSceneSha256", "mpuKeys"}) if request["action"] == "retained-scene-isolate": fields.add("expectedBaseline") require(set(request) == fields, @@ -267,7 +293,7 @@ def run(request, node): "originalSceneRestored": False, "canonicalPassClaimed": False, "backupRestorationUsed": False, "credentialContentsRead": False, "inspectionOnly": request["action"] == "retained-scene-audit", "mutated": mutated, "activeNamespaceIsolated": "active" not in locations} - if request["expectedOwner"].get("runtimeProfile") == "outcome-legacy": + if request["expectedOwner"].get("runtimeProfile") in {"outcome-legacy", "lock-quorum"}: result["privateArchiveParents"] = [] for target in (*retained_roots, retained_state): info = target.parent.lstat() if target.parent.exists() else None diff --git a/test_validation_heal_network_retention.py b/test_validation_heal_network_retention.py new file mode 100644 index 00000000..d76334d6 --- /dev/null +++ b/test_validation_heal_network_retention.py @@ -0,0 +1,223 @@ +"""Stopped I03 isolation binds business/MPU bytes and retains interrupted moves.""" + +import copy +import hashlib +import json +import os +from pathlib import Path +import shutil +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import retained_scene as retained, mpu_scene, network_retention as controller, recover_environment as recovery +import test_validation_heal_retained_scene as retained_tests + + +class NetworkRetentionTests(unittest.TestCase): + def scene(self, directory): + helper = retained_tests.RetainedSceneTests() + state, roots, node, request = helper.scene(directory) + shutil.copytree(roots[1], roots[0]) + owner = request['expectedOwner'];owner['runtimeProfile'] = 'lock-quorum' + for folder in (*roots, state): (folder / 'owner.json').write_text(json.dumps(owner)) + keys = mpu_scene.keys_for(['node1', 'node3']) + for root in roots: + db = root / 'data' / (owner['bucket'] + '-db');db.mkdir() + (db / 'part.1').write_bytes(b'original DB payload') + for key in keys: + key_hash = hashlib.sha256((owner['bucket'] + '/' + key).encode()).hexdigest() + upload = root / 'data/.rustfs.sys/multipart' / key_hash / 'original-opaque-upload' + upload.mkdir(parents=True) + (upload / 'xl.meta').write_bytes(b'original uncompleted MPU metadata') + (upload / 'part.1').write_bytes(b'original full MPU part') + for suffix in ('original', 'intent.json', 'applied.json'): + (root / 'quarantine' / ('I03-missing-target.' + suffix)).write_bytes(('original fault ' + suffix).encode()) + node['owned'] = lambda run: (state, roots, owner) + node['mpu_scene_backend'] = mpu_scene + def snapshot(value): + self.assertEqual(value['buckets'], [owner['bucket'], owner['bucket'] + '-db']) + files = [] + for disk, root in enumerate(roots): + for bucket in value['buckets']: + for path in sorted((root / 'data' / bucket).rglob('*')): + if path.is_file(): files.append({'disk': disk, 'path': path.relative_to(root / 'data').as_posix(), + 'size': path.stat().st_size, 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}) + return {'quiescent': True, 'pid': 0, 'unit': owner['unit'], 'binarySha256': owner['binarySha256'], + 'files': files, 'formats': [{'disk': disk, 'original': True} for disk in range(4)]} + node['snapshot'] = snapshot + scene = snapshot({'buckets': [owner['bucket'], owner['bucket'] + '-db']}) + mpu = mpu_scene.run({'runId': request['runId'], 'action': 'mpu-scene', 'bucket': owner['bucket'], + 'keys': keys, 'expectedBinarySha256': owner['binarySha256']}, node) + request.update(expectedOriginalSceneSha256=retained.sha({name: scene[name] for name in ('files', 'formats')}), + expectedMpuSceneSha256=mpu['sceneSha256'], mpuKeys=keys) + return helper, state, roots, node, request + + def test_original_business_mpu_backups_and_unread_credentials_survive_each_interrupted_move(self): + for interrupt_after in range(1, 6): + with self.subTest(interrupt_after=interrupt_after), tempfile.TemporaryDirectory() as directory: + helper, state, roots, node, request = self.scene(directory) + original = {p: (p.stat().st_ino, p.read_bytes()) for folder in (*roots, state) + for p in folder.rglob('*') if p.is_file()} + opened, real_open = [], os.open + def tracked(path, *args, **kwargs): + opened.append(str(path));return real_open(path, *args, **kwargs) + with helper.root_stats(), patch.object(os, 'open', side_effect=tracked): + before = retained.run(request, node) + self.assertEqual(before['baseline']['originalMpuSceneSha256'], request['expectedMpuSceneSha256']) + self.assertEqual(before['baseline']['archiveLayout'], 'per-run-parent') + change = {**request, 'action': 'retained-scene-isolate', 'expectedBaseline': before['baseline']} + count = 0 + def interrupted(source, target): + nonlocal count + helper.local_rename(source, target) + if target.is_dir(): + count += 1 + if count == interrupt_after: raise RuntimeError('lost SSH after atomic rename') + with patch.object(retained, 'rename_new', side_effect=interrupted), self.assertRaises(RuntimeError): + retained.run(change, node) + with patch.object(retained, 'rename_new', side_effect=helper.local_rename): + result = retained.run(change, node) + self.assertFalse(retained.run(change, node)['mutated']) + self.assertTrue(result['activeNamespaceIsolated']) + self.assertFalse(result['originalSceneRestored'] or result['canonicalPassClaimed'] or result['credentialContentsRead']) + self.assertEqual(result['baseline'], before['baseline']) + self.assertEqual(result['locations'], ['retained'] * 5) + retained_state, retained_roots = retained.destinations(state, roots, per_run_parent=True) + for old, expected in original.items(): + source = next(folder for folder in (*roots, state) if old.is_relative_to(folder)) + target = retained_state if source == state else retained_roots[roots.index(source)] + moved = target / old.relative_to(source) + self.assertEqual((moved.stat().st_ino, moved.read_bytes()), expected) + self.assertTrue(all(p['uid'] == 0 and p['mode'] == 0o700 for p in result['privateArchiveParents'])) + self.assertFalse(any(p.endswith('service.env') or p.endswith('/credential') for p in opened)) + self.assertTrue(any('/multipart/' in p for p in opened)) + self.assertTrue(any('-db/' in p for p in opened)) + self.assertTrue(any('I03-missing-target.original' in p for p in opened)) + + def test_missing_or_changed_archived_business_mpu_scope_and_active_gates_cannot_move(self): + for kind in ('business-sha', 'mpu-sha', 'keys', 'missing-field', 'db-drift', 'mpu-drift', 'backup-drift', 'gate', 'missing-root'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as directory: + helper, state, roots, node, request = self.scene(directory) + with helper.root_stats(): + before = retained.run(request, node) + change = {**request, 'action': 'retained-scene-isolate', 'expectedBaseline': before['baseline']} + if kind == 'business-sha': change['expectedOriginalSceneSha256'] = 'f' * 64 + elif kind == 'mpu-sha': change['expectedMpuSceneSha256'] = 'f' * 64 + elif kind == 'keys': change['mpuKeys'] = change['mpuKeys'][:-1] + elif kind == 'missing-field': change.pop('expectedMpuSceneSha256') + elif kind == 'db-drift': (roots[1] / 'data' / (request['expectedOwner']['bucket'] + '-db') / 'part.1').write_bytes(b'changed') + elif kind == 'mpu-drift': next((roots[1] / 'data/.rustfs.sys/multipart').rglob('part.1')).write_bytes(b'changed') + elif kind == 'backup-drift': (roots[1] / 'quarantine/I03-missing-target.original').write_bytes(b'changed') + elif kind == 'gate': (state / 'network-gate.json').write_text('{}') + else: shutil.rmtree(roots[1]) + with patch.object(retained, 'rename_new') as move, self.assertRaises(ValueError): retained.run(change, node) + move.assert_not_called() + self.assertTrue(state.exists()) + + def test_mpu_content_drift_in_retained_archive_is_not_reclassified_as_clean(self): + with tempfile.TemporaryDirectory() as directory: + helper, state, roots, node, request = self.scene(directory) + with helper.root_stats(), patch.object(retained, 'rename_new', side_effect=helper.local_rename): + baseline = retained.run(request, node)['baseline'] + retained.run({**request, 'action': 'retained-scene-isolate', 'expectedBaseline': baseline}, node) + retained_roots = retained.destinations(state, roots, per_run_parent=True)[1] + next((retained_roots[0] / 'data/.rustfs.sys/multipart').rglob('part.1')).write_bytes(b'drift') + with self.assertRaisesRegex(ValueError, 'durable isolation intent'): retained.run(request, node) + + +class NetworkRetentionControllerTests(unittest.TestCase): + def plan(self, folder): + helper, state, roots, node, request = NetworkRetentionTests().scene(str(folder)) + owner = request['expectedOwner'];nodes = [f'vm00{i}' for i in range(4)] + binding = {'runId': request['runId'], 'candidateCommit': '1' * 40, 'harnessCommit': '2' * 40, + 'candidateBinarySha256': owner['binarySha256'], 'candidatePackageSha256': '3' * 64, + 'bucketPrefix': owner['bucket']} + evidence = folder / 'evidence';evidence.mkdir() + for i, name in enumerate(nodes): + (evidence / f'{i:05d}-reserved.json').write_text(json.dumps({'node': name, 'reserved': True, + 'owner': {**copy.deepcopy(owner), 'hostname': 'original-host-' + str(i)}})) + value = node['snapshot']({'buckets': [owner['bucket'], owner['bucket'] + '-db']}) + mpu = mpu_scene.run({'runId': request['runId'], 'action': 'mpu-scene', 'bucket': owner['bucket'], + 'keys': request['mpuKeys'], 'expectedBinarySha256': owner['binarySha256']}, node) + fleet = types.SimpleNamespace(run_id=request['runId'], nodes=nodes, evidence=evidence, + network_gates=set(), quorum_gates=set(), record=lambda label, value: value) + plan = controller.prepare(fleet, binding, request['mpuKeys'], + {n: copy.deepcopy(value) for n in nodes}, {n: copy.deepcopy(mpu) for n in nodes}) + return fleet, plan, binding + + def test_all_four_preflights_precede_any_move_and_no_restoration_or_cleanup_is_issued(self): + for failure in (None, 'preflight-vm003', 'move-vm001', 'post-drift-vm002'): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory: + fleet, plan, binding = self.plan(Path(directory));events, moved = [], set() + def remote(node, action, **params): + self.assertEqual(params['expectedOwner'], plan['owners'][node]) + self.assertEqual(params['expectedMpuSceneSha256'], plan['mpuSceneSha256'][node]) + events.append((node, action)) + if failure == 'preflight-' + node and action.endswith('audit') and not moved: + raise ValueError('last peer preflight failed') + if action.endswith('isolate'): + moved.add(node) + if failure == 'move-' + node: raise ValueError('lost SSH after move') + return {'baseline': {'original': node if failure != 'post-drift-' + node or len(moved) < 4 else 'drift'}, + 'inspectionOnly': action.endswith('audit'), 'mutated': action.endswith('isolate'), + 'credentialContentsRead': False, 'activeNamespaceIsolated': node in moved, + 'originalSceneRestored': False, 'backupRestorationUsed': False, 'canonicalPassClaimed': False, + 'privateArchiveParents': [{'exists': True, 'uid': 0, 'mode': 0o700}] * 5} + fleet.remote = remote + if failure: + with self.assertRaises(ValueError): controller.isolate(fleet, plan) + else: + result = controller.isolate(fleet, plan) + self.assertEqual(result['cleanup'], 'ISOLATED') + self.assertFalse(result['originalSceneRestored'] or result['namespaceDeletionPerformed'] or result['canonicalPassClaimed']) + self.assertEqual(events[:4], [(n, 'retained-scene-audit') for n in fleet.nodes]) + self.assertTrue(all(action in {'retained-scene-audit', 'retained-scene-isolate'} for n, action in events)) + if failure == 'preflight-vm003': self.assertFalse(moved) + + def test_archived_plan_requires_original_failed_case_binding_and_all_reserved_owners(self): + for kind in ('valid', 'source', 'helper', 'run', 'result', 'owner', 'other-retained'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as directory: + root = Path(directory);fleet, plan, binding = self.plan(root) + case = 'heal.network-partition-and-lock-quorum';base = root / 'artifact/run' / case + folder = base / 'evidence' / case / 'H2/read-boundary';folder.mkdir(parents=True) + (base / 'cases').mkdir() + result = {'case_id': case, 'verdict': 'FAIL', 'cleanup_status': 'FAIL', 'provenance': { + 'source_commit': binding['candidateCommit'], 'candidate_sha256': binding['candidatePackageSha256'], + 'binary_sha256': binding['candidateBinarySha256'], 'testing_commit': binding['harnessCommit']}} + candidate = {'source_sha': binding['candidateCommit'], 'package_sha256': binding['candidatePackageSha256']} + summary = {'cleanup': 'FAIL', 'results': [{'topology': 'H2', 'branch': 'read-boundary', 'verdict': 'FAIL', 'cleanup': 'FAIL'}]} + if kind == 'source': candidate['source_sha'] = 'f' * 40 + elif kind == 'helper': binding['harnessCommit'] = 'e' * 40 + elif kind == 'run': plan['runId'] = 'gh-54321-1-h2' + elif kind == 'result': result['verdict'] = 'PASS' + elif kind == 'other-retained': summary['results'].append({'branch': 'rejoin', 'cleanup': 'FAIL'}) + (base / 'cases' / (case + '.json')).write_text(json.dumps(result)) + (folder.parent.parent / 'healthy-summary.json').write_text(json.dumps(summary)) + (folder / '00001-network-retention-intent.json').write_text(json.dumps(plan)) + (folder / '00002-binding.json').write_text(json.dumps(binding)) + for p in fleet.evidence.glob('*-reserved.json'): shutil.copy2(p, folder / p.name) + if kind == 'owner': + p = next(folder.glob('*-reserved.json'));row = json.loads(p.read_text());row['owner']['runtimeProfile'] = 'manual';p.write_text(json.dumps(row)) + if kind == 'valid': self.assertEqual(recovery.network_recovery_plan(root, 12345, candidate), [plan]) + else: + with self.assertRaises(ValueError): recovery.network_recovery_plan(root, 12345, candidate) + + def test_recovery_never_enters_already_clean_or_deletes_an_isolated_scene(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory);fleet, plan, binding = self.plan(root) + candidate = root / 'candidate.json';candidate.write_text('{}') + argv = ['recover', '--failed-run-id', '12345', '--artifact', str(root), '--candidate', str(candidate), '--evidence', str(root / 'out')] + result = {'cleanup': 'ISOLATED', 'originalSceneRestored': False, 'canonicalPassClaimed': False} + with patch('sys.argv', argv), patch.object(recovery, 'recovery_plan', return_value=[plan]), \ + patch.object(recovery, 'isolate_network_scene', return_value=result) as isolate, \ + patch.object(recovery.Fleet, 'remote', side_effect=AssertionError('ordinary cleanup/namespace shortcut')): + recovery.main() + isolate.assert_called_once() + output = json.loads((root / 'out/recovery-summary.json').read_text()) + self.assertEqual(output['cleanup'], 'ISOLATED') + self.assertFalse(output['canonicalPassClaimed']) + + +if __name__ == '__main__': unittest.main() diff --git a/test_validation_heal_retained_scene.py b/test_validation_heal_retained_scene.py index 495cc63a..8e1e2b53 100644 --- a/test_validation_heal_retained_scene.py +++ b/test_validation_heal_retained_scene.py @@ -159,6 +159,7 @@ def execute(*args, **kwargs): scope = {"__name__": "__node_test__"} exec(compile(source, "retained-node", "exec"), scope) self.assertTrue(callable(scope["retained_scene_backend"].run)) + self.assertTrue(callable(scope["mpu_scene_backend"].run)) return types.SimpleNamespace(returncode=0, stdout="{}", stderr="") with tempfile.TemporaryDirectory() as directory, patch("scripts.validation_heal.healthy.subprocess.run", side_effect=execute): Fleet(["vm000"], Path(directory), "gh-12345-1-h2").remote("vm000", "retained-scene-audit", expectedOwner={}) From 36c11db1ce8cf98eb1feca4b365246f9464f46e9 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 20:29:21 +0800 Subject: [PATCH 240/343] test(heal): run complete network partition fixtures --- docs/validation-migration-map.md | 62 +++++- scripts/validation_heal/fleet_node.py | 1 + scripts/validation_heal/healthy.py | 3 + scripts/validation_heal/network.py | 106 ++++++++-- .../validation_heal/network_delete_records.py | 113 ++++++++++ .../validation_heal/network_load_repair.py | 186 ++++++++++++++++ test_validation_heal_network_adapter.py | 184 ++++++++++++++++ ..._validation_heal_network_delete_records.py | 113 ++++++++++ test_validation_heal_network_load_repair.py | 199 ++++++++++++++++++ 9 files changed, 935 insertions(+), 32 deletions(-) create mode 100644 scripts/validation_heal/network_delete_records.py create mode 100644 scripts/validation_heal/network_load_repair.py create mode 100644 test_validation_heal_network_adapter.py create mode 100644 test_validation_heal_network_delete_records.py create mode 100644 test_validation_heal_network_load_repair.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 89d85f09..c01516bb 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2040,7 +2040,7 @@ Original failures and original reports remain unchanged. | --- | --- | --- | | DV | Reuses auto-test `seed(DV)` and `suspend_fixture`: four keys with three distinct non-null data versions each; real suspended overwrites, null marker creation/removal ACKs; retained live/null marker identities; current and historical markers. Versioning returns to Enabled before partition. The fault targets a retained null data version. Every selected key version participates in GET, negative Heal and complete recursive recovery. All 15 retained data versions receive full physical shard checks; markers receive exact S3 refusal/header checks. HEAD preserves version identity, length, ETag, Last-Modified and metadata. | This expanded fixture still needs a new live run and independent evidence acceptance. It does not claim a partition while the bucket remains Suspended. | | DB | Adds a separate never-versioned run-owned bucket with the default 128 controls. The exact candidate size contract and actual EC:8 process/object geometry calibrate zero/one, 4 KiB, inline, EC block and source-bound 5-MiB MPU minimum minus/exact/plus sizes. All 128 controls require decoded full-set shape witnesses; every distinct size has partition GET, physical checksum/no-part and Range evidence. Original/fault/rejoin/repaired snapshots explicitly include both primary and DB buckets. All peers must preserve complete inventories and stable original HEADs. | Fresh live execution and independent acceptance remain required. The MPU-size controls are single PUT objects; they do not claim completed multipart protocol coverage, which remains part of DL. | -| DL | `network_load.py` supplies three distinct control versions per endpoint, response-bound MPU fixtures and four independent worker journals for its concurrent 28-call matrix. `network_load_session.py` registers seed/workers before SDK invocation, reconciles original versions and pending MPU effects after rejoin, and withholds cleanup until a stopped physical MPU census proves no files remain. Original request and ACK byte hashes remain frozen. | Session and MPU census are not yet wired into the live adapter. Independent physical repair of partially refused version deletes, retained-scene isolation and recovery integration remain required before dispatch. Modeled matrix/reconciliation tests are not DL live or canonical acceptance. | +| DL | `network_load.py` supplies three distinct control versions per endpoint, response-bound MPU fixtures and four independent worker journals for its concurrent 28-call matrix. `network_load_session.py` registers seed/workers before SDK invocation, reconciles original versions and pending MPU effects after rejoin, and withholds cleanup until a stopped physical MPU census proves no files remain. Original request and ACK byte hashes remain frozen. | The complete adapter now invokes the session, physical version recovery, MPU closure and unresolved-scene isolation for the fixed `1c7fc818dd858247c45b39542abd8291b1e6e31b` candidate. Fresh same-candidate live execution and independent full-fixture acceptance remain required. Modeled matrix/reconciliation tests are not DL live or canonical acceptance. | The expanded DV fixture stays within the existing 128-identity/512-MiB inventory limits. Network expiry remains 1,200 seconds; partition task deadlines, @@ -2102,12 +2102,11 @@ other system-key contents. It requires stopped original services, matching candidate/owner/mount identities and bounded stable regular files. The census does not claim payload integrity or EC/bitrot correctness. -The pending live adapter must preserve the original object and system MPU -failure scene, independently repair partial refused deletes without dropping -version assertions, and isolate unresolved scenes with the existing strict -recovery protocol. Session/census support alone neither enables a new fleet -dispatch nor completes I03. The 807-check Heal suite passes with nine platform -skips; no new live or canonical acceptance is claimed. +The live adapter now preserves original object and system MPU failure scenes, +checks partial refused deletes without dropping version assertions, and routes +unresolved effects into the strict isolation/recovery protocol. The earlier +807-check session delivery remains scoped evidence; current full-adapter +regressions and fresh live acceptance are recorded separately below. I03 unresolved-scene isolation (2026-10-19): `network_retention.py` now freezes the original four reservation owners and stopped business/MPU scene digests. @@ -2126,7 +2125,48 @@ FAIL/cleanup-FAIL branch with an exact candidate/package/binary/helper/run binding and four matching original reservations. It reports `ISOLATED`, never `PASS` or `alreadyClean`, including retries after all active paths are absent. Historical expiry-only recovery keeps its existing strict boundary. The live -I03 adapter still does not emit this new intent or invoke DL; integration, -independent partial-delete repair and fresh full-fixture live acceptance -remain required. Local filesystem/controller tests do not certify live -isolation or increase the canonical count. +I03 adapter now emits this intent when a seed or original DL effect remains +unresolved. Local filesystem/controller tests do not certify live isolation +or increase the canonical count; full-fixture live acceptance remains pending. + + +I03 complete DL adapter (2026-10-19): the fixed `1c7fc818...` candidate now +executes all DB/DV/DL stages in one original branch. The session exists before +any DL SDK call. Its six acknowledged data versions join the primary baseline, +all original HEADs, recursive version responsibility and all sixteen-drive +streaming checks. Four workers issue the 28-call matrix inside the actual +partition, between independent network witnesses. DL stable reads use their +own intact data and actual shared-lock quorum, independently of the separate +missing DV shard. The 128-object DB fixture remains included in every complete +business snapshot. + +`network_delete_records.py` reads only bounded `delete-prepared` JSON files +under the two original stopped DL control keys. It exports selectors and +hashes, never raw records or nested metadata. Exact source hashes bind the +version-1 journal, FileInfo serialization, delete rollback and Heal settlement +behavior. Duplicate JSON fields, unsafe files, wrong runtime/owner, unbound +versions/data directories/metadata hashes/target bytes and changed +incarnations fail closed. The exact original refused historical-version +request can explain only its reachable writable slots. Other versions, DB/DV +controls, readonly or disconnected roots, new business commits and altered +part bytes keep their original strict assertions. + +A complete prepared original backup is a product settlement obligation, not +proof of commit or rollback. After rejoin, independent recursive product tasks +must report all three original versions per DL key. Unbacked version loss +requires an actual repaired receipt; source-bound product delete settlement +may precede a healthy shard receipt only for that exact original generation +and incarnation. All final physical files/formats, original shard bytes and +public versions/metadata/tags still must match before cleanup. MPU retirement +uses separate recovery ACKs and stopped final MPU/business checks. These occur +before the recovered unique PUT. An incomplete seed/load or unresolved effect +instead stops services, releases owned gates, captures the original scene and +isolates every inode; the case and cleanup remain FAIL until separately audited. + +The full adapter intentionally requires the exact fixed candidate above. +Historical `072b57b...` scoped source/contracts are retained, but its different +delete protocol does not certify this expanded adapter. No product source, +compiled collector, inspector pin, native barrier, request retry or time budget +changes in this delivery. Local modeled orchestration and filesystem tests +remain distinct from new full-fixture live evidence. Canonical acceptance +remains **104/158** until that evidence is independently verified. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 208dcc3b..41472c3e 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1741,6 +1741,7 @@ def main(request): actions[action] = incarnation_operation actions["task-records"] = task_records actions["mpu-scene"] = lambda value: globals()["mpu_scene_backend"].run(value, globals()) + actions["network-delete-records"] = lambda value: globals()["network_delete_records_backend"].run(value, globals()) actions["process-clock"] = process_clock actions["environment-capabilities"] = environment_capabilities for action in ("retained-scene-audit", "retained-scene-isolate"): diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index fe978568..f8b97a9a 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -185,6 +185,9 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() + if action == "network-delete-records": + source += "\nimport types\nnetwork_delete_records_backend = types.ModuleType('network_delete_records')\nexec(" + repr( + NODE_SOURCE.with_name("network_delete_records.py").read_text()) + ", network_delete_records_backend.__dict__)\n" if action == "mpu-scene" or action.startswith("retained-scene-"): source += "\nimport types\nmpu_scene_backend = types.ModuleType('mpu_scene')\nexec(" + repr( NODE_SOURCE.with_name("mpu_scene.py").read_text()) + ", mpu_scene_backend.__dict__)\n" diff --git a/scripts/validation_heal/network.py b/scripts/validation_heal/network.py index 36e58dd2..60a15453 100644 --- a/scripts/validation_heal/network.py +++ b/scripts/validation_heal/network.py @@ -30,6 +30,9 @@ from .network_gate import TTL_SECONDS from .scanner_repair import JournalWindow from .network_fixtures import db_parameters, seed_db, map_db, prove_db, verify_db + from .network_load_session import LoadSession + from .network_load_repair import source_reference as delete_source_reference, classify_deletes, recover_controls + from .network_retention import prepare as prepare_retention, isolate as isolate_retention except ImportError: from heal_probe import Probe, canonical, compare_inventories, payload, require, save from heal_workload import Recorder, RecordedCallError, seed, verify_seed @@ -45,6 +48,9 @@ from network_gate import TTL_SECONDS from scanner_repair import JournalWindow from network_fixtures import db_parameters, seed_db, map_db, prove_db, verify_db + from network_load_session import LoadSession + from network_load_repair import source_reference as delete_source_reference, classify_deletes, recover_controls + from network_retention import prepare as prepare_retention, isolate as isolate_retention CASE_ID = 'heal.network-partition-and-lock-quorum' @@ -454,16 +460,21 @@ def run_branch(args, nodes, branch): 'maxInventoryBytes': 512 * 1024**2} fleet.record('binding', binding) fleet.record('network-source-contract', reference) + fleet.record('DL-delete-recovery-source-contract', delete_source_reference(candidate['source_sha'])) manifest = fleet.record('inspector-build', verify_inspector(args.tool_directory, candidate)) contract = verify_quorum_contract(manifest, candidate) result = {'topology': 'H2', 'branch': branch, 'verdict': 'RUNNING', 'cleanup': 'PENDING'} - probe, active, recovery_active, rows = Probe(binding), [], [], [] + probe, active, recovery_active, rows, dl = Probe(binding), [], [], [], None try: fleet.deploy(args.binary_sha, bucket, access, secret) + dl_endpoints = ['node1', 'node3' if branch in {'read-boundary', 'lock-boundary'} else 'node4'] + dl = LoadSession(binding, bucket, dl_endpoints, directory / 'DL') recorder = Recorder(probe, directory / 'fixture') expected = seed_versions(recorder, fleet, bucket) key_versions = [entry for entry in expected if entry['key'] == 'objects/0001.bin'] selected = next(entry for entry in key_versions if not entry['deleteMarker'] and entry['versionId'] == 'null') + dl_fixture = dl.seed() + expected += dl_fixture['expected'] calibration = fleet.snapshot('DB-size-calibration-live') calibration_target = map_target(calibration, bucket, selected['key'], 'data', 1, selected['versionId']) db_plan = fleet.record('DB-source-size-plan', db_parameters(manifest, candidate, calibration_target, calibration)) @@ -474,12 +485,15 @@ def run_branch(args, nodes, branch): db_heads = version_heads(probe, fleet, db_bucket, db_expected, 'DB-original-HEAD') baseline = fleet.record('all-version-S3-baseline', probe.inventory(bucket)) fleet.record('exact-ACK-manifest', {'expected': expected, **verify_seed(expected, baseline)}) + dl.register_baseline(baseline) original_heads = version_heads(probe, fleet, bucket, expected, 'original-version-HEAD') for key in sorted({entry['key'] for entry in expected if entry['deleteMarker']}): verify_markers(probe, fleet, bucket, key, expected) fleet.stop() physical_buckets = [bucket, db_bucket] original = fleet.snapshot('physical-baseline', quiescent=True, buckets=physical_buckets) + dl.register_business_baseline(original) + dl.capture_mpu(fleet, 'baseline') validate_topology(original, 'H2') target = map_target(original, bucket, selected['key'], 'data', 1, selected['versionId'], latest_required=False) target = {**target, **next(row for row in target['cohort'] if row['node'] == nodes[3])} @@ -488,6 +502,8 @@ def run_branch(args, nodes, branch): db_representatives, db_witnesses = map_db(original, db_bucket, db_expected, db_plan) fleet.record('DB-all128-physical-shapes', {'objects': db_witnesses, 'allBoundariesObserved': True}) plan = fleet.record('actual-partition-quorum-plan', plan_for(original, target, branch)) + require(tuple(side['endpoint'] for side in plan['sides']) == dl.endpoints, + 'DL endpoints differ from the actual original cohort partition') params = {'disk': target['disk'], 'faultId': 'I03-missing-target'} registration = target['node'], params fleet.faults.append(registration) @@ -521,6 +537,11 @@ def witness(label): require(sum(row['writable'] for index in side['members'] for row in gates[nodes[index]]['disks']) == side['writableDrives'] and all(row['readableMetadata'] and row['sameServiceUidAndMountNamespace'] for index in side['members'] for row in gates[nodes[index]]['disks']), 'actual independent R/W witnesses differ from partition plan') + result['DL-load'] = fleet.record('DL-partition-concurrent-load', dl.run_load({ + side['endpoint']: side['lockClients'] >= plan['sharedLockQuorum'] + and side['metadataAuthorities'] >= plan['readQuorum'] for side in plan['sides']})) + require(result['DL-load']['complete'] is True, + 'DL partition load is incomplete or has an uncertain/unexpected original response') for side in plan['sides']: endpoint = side['endpoint'] local_probe = Probe(binding, endpoint) @@ -551,8 +572,12 @@ def witness(label): fleet.record('partition-process-resource-observations', {node: fleet.remote(node, 'process-metrics') for node in nodes}) fleet.stop() scene = fleet.snapshot('negative-scene-before-release', quiescent=True, buckets=physical_buckets) - require(physical_content(faulted) == physical_content(project(scene, bucket, [row['key'] for row in rows])), - 'partition writes/Heal changed an original version, control or damaged target') + dl.capture_mpu(fleet, 'failure') + delete_records = fleet.record('DL-stopped-original-delete-records', {node: fleet.remote(node, 'network-delete-records', + bucket=bucket, keys=sorted(side['controlKey'] for side in dl_fixture['sides'].values()), + expectedBinarySha256=binding['candidateBinarySha256']) for node in nodes}) + damage = fleet.record('DL-original-delete-failure-obligations', classify_deletes(faulted, + project(scene, bucket, [row['key'] for row in rows]), dl_fixture, dl.registered_receipts(), plan, nodes, delete_records)) for node in nodes: fleet.record('partition-peer-logs', {'node': node, **fleet.remote(node, 'logs')}) release_network(fleet) @@ -563,16 +588,27 @@ def witness(label): require(physical_content(scene) == physical_content(fleet.snapshot('all-peer-return-before-independent-repair', buckets=physical_buckets)), 'network recovery repaired a target before the independent product task') result['recoveryResponsibility'] = recover_versions(probe, fleet, binding, bucket, key_versions, selected, recovery_active) + result['DL-recoveryResponsibility'] = recover_controls(probe, fleet, binding, dl_fixture, damage, recovery_active) idle_fleet(fleet, binding) fleet.stop() after = fleet.snapshot('physical-rebuilt-before-any-recovered-GET', quiescent=True, buckets=physical_buckets) - result['physical'] = validate_repaired_physical(original, project(after, bucket, [row['key'] for row in rows]), target) + repair_targets = [target, *damage['repairTargets']] + result['physical'] = validate_repaired_physical(original, project(after, bucket, [row['key'] for row in rows]), target, + repair_targets=repair_targets) result['streaming'] = [verify_cohort(fleet, cohort, 'every-original-version-streaming') for cohort in cohorts] result['DB-physical'] = prove_db(fleet, db_representatives) fleet.faults.remove(registration) + dl.capture_mpu(fleet, 'pre-reconcile') fleet.start() result['writeResolution'] = resolve_writes(fleet, binding, bucket, rows) require(all(row['resolution']['state'] == 'ABSENT' for row in rows), 'a W/lock-negative partition nevertheless committed a new version') + result['DL-public-reconciliation'] = dl.reconcile() + fleet.stop() + dl.capture_mpu(fleet, 'post-reconcile') + final_business = fleet.snapshot('DL-stopped-final-business-closure', quiescent=True, buckets=physical_buckets) + result['DL-closure'] = dl.finish_reconciliation(final_business, repair_targets=repair_targets) + require(dl.safe_to_cleanup, 'DL original mutation effects remain unresolved') + fleet.start() released = unique_put(probe, directory, bucket, 'recovered/unique.bin', rows) require(released['observed']['acknowledged'] is True and released['observed']['response'].get('VersionId') not in {None, 'null'}, 'fully recovered cluster cannot acknowledge a distinct safe PUT') @@ -612,28 +648,56 @@ def witness(label): fleet.record('stop-original-partition-task', Probe(binding, endpoint).admin(path, [('clientToken', token), ('forceStop', 'true')], b'')) except Exception as exc: fleet.record('stop-partition-task-error', {'error': str(exc)}) - try: - if fleet.network_gates or fleet.quorum_gates: + if dl is not None and not dl.safe_to_cleanup: + # No S3 observation, rescue or ordinary cleanup may erase an + # incomplete seed or uncertain mutation. Preserve the stopped + # original scene, including refused Abort effects, by inode. + dirty = True + try: fleet.stop() - release_network(fleet) + for node in tuple(fleet.network_gates): + proof = fleet.record('DL-failure-owned-network-release', {'node': node, + **fleet.remote(node, 'network-release', expired=True)}) + require(proof.get('state') == 'CLEAR' and proof.get('rulesRemaining') == 0, + 'DL failure retained an owned network rule') + fleet.network_gates.remove(node) for node in tuple(fleet.quorum_gates): - fleet.record('failed-branch-W-release', fleet.remote(node, 'quorum-gate', mode='clear')) + proof = fleet.record('DL-failure-owned-W-release', {'node': node, + **fleet.remote(node, 'quorum-gate', mode='clear')}) + require(proof.get('state') == 'CLEAR', 'DL failure retained its private W gate') fleet.quorum_gates.remove(node) - fleet.start() - pending = [row for row in rows if 'resolution' not in row] - if pending: - fleet.stop() - fleet.start() - resolve_writes(fleet, binding, bucket, pending) - except Exception as exc: - dirty = True - result['recoveryError'] = str(exc) - if dirty: - fleet.stop() - fleet.record('retain-unresolved-network-or-write', {'state': 'DIRTY', 'namespacesRetained': True, 'writes': rows}) + failed_scene = fleet.snapshot('DL-original-unresolved-business-scene', quiescent=True, buckets=[bucket, bucket + '-db']) + failed_mpu = dl.capture_mpu(fleet, 'failure') + for node in nodes: + fleet.record('DL-original-unresolved-service-logs', {'node': node, **fleet.remote(node, 'logs')}) + retained_plan = prepare_retention(fleet, binding, dl.keys, failed_scene, failed_mpu['nodes']) + result['retention'] = isolate_retention(fleet, retained_plan) + except Exception as exc: + result['retentionError'] = {'type': type(exc).__name__, 'message': str(exc)} clean = False else: - clean = fleet.cleanup() + try: + if fleet.network_gates or fleet.quorum_gates: + fleet.stop() + release_network(fleet) + for node in tuple(fleet.quorum_gates): + fleet.record('failed-branch-W-release', fleet.remote(node, 'quorum-gate', mode='clear')) + fleet.quorum_gates.remove(node) + fleet.start() + pending = [row for row in rows if 'resolution' not in row] + if pending: + fleet.stop() + fleet.start() + resolve_writes(fleet, binding, bucket, pending) + except Exception as exc: + dirty = True + result['recoveryError'] = str(exc) + if dirty: + fleet.stop() + fleet.record('retain-unresolved-network-or-write', {'state': 'DIRTY', 'namespacesRetained': True, 'writes': rows}) + clean = False + else: + clean = fleet.cleanup() result.update(cleanup='PASS' if clean else 'FAIL', writes=rows) fleet.record('branch-result', result) return result diff --git a/scripts/validation_heal/network_delete_records.py b/scripts/validation_heal/network_delete_records.py new file mode 100644 index 00000000..083a5b0e --- /dev/null +++ b/scripts/validation_heal/network_delete_records.py @@ -0,0 +1,113 @@ +"""Read only the original stopped DL controls' bounded delete-prepared records.""" + +import hashlib +import json +import os +import re +import stat +import uuid + + +FIELDS = {'protocol_version', 'volume', 'object', 'transaction', 'bucket_incarnation', 'original_digest', + 'target_digest', 'data_dirs', 'deleted_versions', 'requests', 'original_versions', 'predecessors'} + + +def require(value, message): + if not value: raise ValueError(message) + + +def stable(info): + return (info.st_dev, info.st_ino, info.st_mode, info.st_nlink, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +def canonical_uuid(value): + require(isinstance(value, str) and str(uuid.UUID(value)) == value and uuid.UUID(value).int != 0, + 'DL delete record has a missing/noncanonical generation or transaction') + return value + + +def checksum(value): + require(isinstance(value, list) and len(value) == 32 and all(type(x) is int and 0 <= x <= 255 for x in value), + 'DL delete record digest is not an exact SHA-256 byte array') + return bytes(value).hex() + + +def unique_fields(pairs): + result = {} + for key, value in pairs: + require(key not in result, 'DL prepared delete contains duplicate JSON fields') + result[key] = value + return result + + +def run(request, node): + require(set(request) == {'runId', 'action', 'bucket', 'keys', 'expectedBinarySha256'} + and request['action'] == 'network-delete-records', 'DL delete census accepts only its exact owned controls') + state, roots, owner = node['owned'](request['runId']) + require(owner.get('runtimeProfile') == 'lock-quorum' and owner.get('bucket') == request['bucket'] + and owner.get('binarySha256') == request['expectedBinarySha256'] and len(roots) == 4 + and node['digest'](node['checked'](state / 'rustfs', state)) == owner['binarySha256'], + 'DL delete census candidate/runtime/bucket differs') + node['stopped'](owner) + keys = request['keys'] + require(isinstance(keys, list) and len(keys) == len(set(keys)) == 2 and keys == sorted(keys) + and keys[0] == 'dl/node1/control.bin' and keys[1] in {'dl/node3/control.bin', 'dl/node4/control.bin'}, + 'DL delete census escaped its original two control keys') + records = [] + for disk, root in enumerate(roots): + data = node['checked'](root / 'data', root) + for key in keys: + folder = node['checked'](data / request['bucket'] / key, data) + if not folder.exists(): continue + children = sorted(folder.iterdir()) + require(len(children) <= 64, 'DL control directory census exceeds its source fixture bound') + for child in children: + child = node['checked'](child, data) + if not stat.S_ISDIR(child.lstat().st_mode): continue + # Data directories without a prepared delete are not read. + path = node['checked'](child / 'delete-prepared', data) + if not path.exists(): continue + canonical_uuid(child.name) + before = path.lstat() + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_size <= 1024**2 + and len(records) < 32, 'DL prepared delete has an unsafe file type or excessive size/count') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + require(stable(os.fstat(fd)) == stable(before), 'DL prepared delete changed before reading') + raw = bytearray() + while chunk := os.read(fd, min(65536, before.st_size + 1 - len(raw))): + raw.extend(chunk) + require(len(raw) <= before.st_size, 'DL prepared delete grew while reading') + require(len(raw) == before.st_size and stable(os.fstat(fd)) == stable(before), + 'DL prepared delete changed while reading') + finally: + os.close(fd) + require(stable(path.lstat()) == stable(before), 'DL prepared delete changed after reading') + value = json.loads(raw, object_pairs_hook=unique_fields) + require(isinstance(value, dict) and set(value) == FIELDS and type(value['protocol_version']) is int + and value['protocol_version'] == 1 and value['volume'] == request['bucket'] + and value['object'] == key and value['transaction'] == child.name + and isinstance(value['requests'], list) and len(value['requests']) == 1 + and len(value['original_versions']) == 3 and value['predecessors'] == [], + 'DL prepared delete escaped the reviewed one-request three-version protocol') + request_row = value['requests'][0] + require(request_row.get('name') == key and request_row.get('deleted') is False + and request_row.get('mark_deleted') is False + and len(value['data_dirs']) == len(value['deleted_versions']) == 1 + and all(row.get('deleted') is False for row in value['original_versions']), + 'DL prepared delete is not an exact ordinary historical-version delete') + records.append({'disk': disk, 'key': key, 'path': path.relative_to(data).as_posix(), + 'size': len(raw), 'sha256': hashlib.sha256(raw).hexdigest(), 'protocolVersion': 1, + 'transaction': child.name, 'bucketIncarnation': canonical_uuid(value['bucket_incarnation']), + 'originalMetadataSha256': checksum(value['original_digest']), + 'targetMetadataSha256': checksum(value['target_digest']), + 'dataDirs': [canonical_uuid(x) for x in value['data_dirs']], + 'deletedVersions': [canonical_uuid(x) for x in value['deleted_versions']], + 'requestVersion': canonical_uuid(request_row.get('version_id')), + 'originalVersions': [canonical_uuid(x.get('version_id')) for x in value['original_versions']], + 'originalDataDirs': [canonical_uuid(x.get('data_dir')) for x in value['original_versions']]}) + again = node['owned'](request['runId']);node['stopped'](again[2]) + require(again == (state, roots, owner), 'DL delete census original owner/roots changed') + return {'runId': request['runId'], 'bucket': request['bucket'], 'keys': keys, + 'binarySha256': owner['binarySha256'], 'quiescent': True, 'readOnly': True, 'records': records, + 'rawContentsExported': False, 'credentialContentsRead': False, 'commitOrRollbackClaimed': False} diff --git a/scripts/validation_heal/network_load_repair.py b/scripts/validation_heal/network_load_repair.py new file mode 100644 index 00000000..f50753c5 --- /dev/null +++ b/scripts/validation_heal/network_load_repair.py @@ -0,0 +1,186 @@ +"""Bind refused DL deletes to exact historical generations and product recovery.""" + +import copy +import uuid + +try: + from .heal_probe import require + from .network_load import validate_call + from .shards import files, map_target, run_task + from .versions import public_version_id, validate_version_receipts +except ImportError: + from heal_probe import require + from network_load import validate_call + from shards import files, map_target, run_task + from versions import public_version_id, validate_version_receipts + + +CANDIDATE = '1c7fc818dd858247c45b39542abd8291b1e6e31b' +SOURCE_FILES = { + 'crates/ecstore/src/disk/local.rs': 'c715c122c06eafb5861ab560e1acac087e5326cd72e2063f0a437786fb4a965b', + 'crates/ecstore/src/disk/local/delete_recovery.rs': 'dc631db4794b0189d42636247f407e32d1d94fc65253c17b6f4942655f8ae7e3', + 'crates/ecstore/src/set_disk/ops/heal.rs': 'fc576119f455848ce279772ffca6d2943cb9f909968cae367860a9887d3e020e', + 'crates/ecstore/src/disk/mod.rs': 'cd045a322ad7e41e2253ac08554f10634192345cbd923f2854553a229c036f4b', + 'crates/filemeta/src/fileinfo.rs': 'e2553d6b0210ad15d723896735c5a5a40528d9bce10d9a6e02f7ac2fff271c24', +} + + +def source_reference(candidate): + require(candidate == CANDIDATE, 'full I03 DL delete recovery is reviewed only for the exact fixed candidate') + return {'candidateCommit': candidate, 'sourceFiles': SOURCE_FILES.copy(), + 'deleteWriteQuorum': 'configured set / 2 + 1', 'failedDeleteAttemptsProductRollback': True, + 'rollbackFailureIsNotSuccessfulDelete': True, 'healSettlesDeleteBeforeReadingVersion': True, + 'preparedTransactionAloneProvesNeitherCommitNorRollback': True, + 'finalOriginalBusinessPhysicalAndPayloadProofRequired': True} + + +def classify_deletes(before, after, fixture, receipts, plan, nodes, delete_records): + """Preserve all control assertions; only the original refused old-version delete may leave residue.""" + source_reference(fixture['sourceContract']['candidateCommit']) + require(list(before) == list(after) == list(nodes), 'DL failure scene changed original node ordering') + left, right = files(before), files(after) + scopes, targets = {}, [] + for worker in receipts['workers']: + if worker['role'] != 'writer': continue + endpoint = worker['endpoint'] + for row in worker['rows']: + if row['request']['operation'] != 'delete_object': continue + validate_call({'row': row}, fixture, endpoint, readable=True) + side = next(side for side in plan['sides'] if side['endpoint'] == endpoint) + if side['lockClients'] < plan['exclusiveLockQuorum']: continue + entry = fixture['sides'][endpoint]['versions'][0] + require(row['request']['params']['VersionId'] == entry['versionId'], 'DL delete escaped its first exact historical ACK') + target = map_target(before, fixture['bucket'], entry['key'], 'data', 1, entry['versionId'], latest_required=False) + allowed = {(nodes[index], disk) for index in side['members'] for disk in range(4) + if disk not in plan['gates'].get(nodes[index], [])} + scopes[entry['key']] = {'target': target, 'allowed': allowed, 'changed': set(), 'lost': set(), 'transactions': {}} + for node in before: + require(before[node]['formats'] == after[node]['formats'], 'DL refused mutation changed an original disk format') + def scope_for(identity): + node, disk, path = identity + for key, scope in scopes.items(): + prefix = fixture['bucket'] + '/' + key + '/' + if path.startswith(prefix) and (node, disk) in scope['allowed']: + return scope, path[len(prefix):] + return None, None + for identity, original in left.items(): + actual = right.get(identity) + if actual == original: continue + scope, relative = scope_for(identity) + require(scope is not None, 'DL refused mutation changed an unrelated original version, DB control or damaged DV target') + slot = identity[:2];target = scope['target'] + if relative == 'xl.meta': + require(actual is not None and original.get('metadata') and actual.get('metadata'), 'DL delete lost all original control metadata') + old, new = original['metadata'], actual['metadata'] + reduced = {**old, 'versions': [version for version in old['versions'] + if public_version_id(version.get('versionId')) != target['versionId']]} + require(new == old or new == reduced, 'DL delete changed another original generation or introduced metadata/version content') + if new == reduced: scope['lost'].add(slot) + else: + require(relative.startswith(target['dataDir'] + '/') and actual is None, + 'DL delete changed bytes or removed a different acknowledged data directory') + scope['lost'].add(slot) + scope['changed'].add(slot) + for identity in set(right) - set(left): + scope, relative = scope_for(identity) + require(scope is not None, 'DL refused mutation created an unbound business file') + parts = relative.split('/');target = scope['target'];slot = identity[:2] + require(len(parts) in {2, 3} and str(uuid.UUID(parts[0])) == parts[0] + and parts[0] not in {version['dataDir'] for version in left[(identity[0], identity[1], + fixture['bucket'] + '/' + target['key'] + '/xl.meta')]['metadata']['versions']}, + 'DL refused delete residue is not a distinct source transaction directory') + transaction = scope['transactions'].setdefault(parts[0], {}) + transaction.setdefault(slot, set()).add('/'.join(parts[1:])) + if len(parts) == 2: + require(parts[1] in {'xl.meta.bkp', 'delete-prepared', 'delete-target', 'delete-commit-finished'}, + 'DL delete residue contains a foreign transaction file') + if parts[1] == 'xl.meta.bkp': + original = left[(identity[0], identity[1], fixture['bucket'] + '/' + target['key'] + '/xl.meta')] + require(all(right[identity][name] == original[name] for name in ('size', 'sha256')), + 'DL transaction backup is not the exact original metadata bytes') + else: + require(parts[1] == target['dataDir'], 'DL transaction staged a different acknowledged generation') + original = left.get((identity[0], identity[1], fixture['bucket'] + '/' + target['key'] + '/' + '/'.join(parts[1:]))) + require(original is not None and all(right[identity][name] == original[name] for name in ('size', 'sha256')), + 'DL transaction staged bytes differ from the exact original file') + scope['changed'].add(slot) + require(set(delete_records) == set(before), 'DL delete record census omitted an original peer') + prepared = {} + control_keys = sorted(side['controlKey'] for side in fixture['sides'].values()) + for node, census in delete_records.items(): + require(census.get('readOnly') is True and census.get('quiescent') is True + and census.get('runId') == fixture['runId'] and census.get('bucket') == fixture['bucket'] + and census.get('binarySha256') == before[node]['binarySha256'] and census.get('keys') == control_keys + and census.get('credentialContentsRead') is False, + 'DL prepared delete records lack original stopped candidate/control ownership') + for row in census['records']: + identity = (node, row['disk'], row['path']) + scope, relative = scope_for(identity) + require(identity not in prepared and scope is not None and identity in right and identity not in left, + 'DL prepared delete record is duplicated, pre-existing or unrelated to the original invocation') + target = scope['target'];parts = relative.split('/') + original = left[(node, row['disk'], fixture['bucket'] + '/' + target['key'] + '/xl.meta')] + versions = original['metadata']['versions'] + require(len(parts) == 2 and parts[1] == 'delete-prepared' and row['transaction'] == parts[0] + and row.get('protocolVersion') == 1 and row['key'] == target['key'] + and all(row[name] == right[identity][name] for name in ('size', 'sha256')) + and row['requestVersion'] == target['versionId'] and row['deletedVersions'] == [target['versionId']] + and row['dataDirs'] == [target['dataDir']] and row['originalMetadataSha256'] == original['sha256'] + and len(row['originalVersions']) == len(row['originalDataDirs']) == 3 + and set(zip(row['originalVersions'], row['originalDataDirs'])) + == {(public_version_id(version.get('versionId')), version['dataDir']) for version in versions} + and isinstance(row.get('bucketIncarnation'), str) and row['bucketIncarnation'], + 'DL prepared delete journal does not bind its raw bytes to the exact original generation/backup') + target_identity = (node, row['disk'], fixture['bucket'] + '/' + target['key'] + '/' + parts[0] + '/delete-target') + require(target_identity in right and row['targetMetadataSha256'] == right[target_identity]['sha256'], + 'DL prepared delete target bytes differ from the original journal digest') + prepared[identity] = row + require(set(prepared) == {identity for identity in set(right) - set(left) if identity[2].endswith('/delete-prepared')}, + 'DL original prepared delete file lacks an independent stopped raw-byte record') + policies = {} + for key, scope in scopes.items(): + require(len(scope['transactions']) <= 1, 'one original DL delete cannot explain multiple transaction IDs') + complete_backups = {slot for transaction, locations in scope['transactions'].items() for slot, names in locations.items() + if {'xl.meta.bkp', 'delete-prepared', 'delete-target'} <= names + and (*slot, fixture['bucket'] + '/' + key + '/' + transaction + '/delete-prepared') in prepared} + # A staged journal is only a recovery obligation. The subsequent product + # task and final physical/payload proofs must redeem the original ACKs. + missing = scope['lost'] + policies[key] = {'versionId': scope['target']['versionId'], + 'requiredRepair': bool(missing - complete_backups), + 'productSettlementAllowed': bool(missing) and missing <= complete_backups, + 'changedOriginalSlots': sorted(scope['changed']), + 'originalBucketIncarnations': sorted({row['bucketIncarnation'] for identity, row in prepared.items() + if row['key'] == key}), + 'originalTransactionIds': sorted(scope['transactions']), 'finalPhysicalProofRequired': True} + for slot in scope['changed']: + member = next(member for member in scope['target']['cohort'] if (member['node'], member['disk']) == slot) + targets.append({**scope['target'], **member}) + return {'policies': policies, 'repairTargets': targets, 'refusalProvesRollback': False, + 'originalFailureSceneRetained': True, 'canonicalCasePassClaimed': False} + + +def recover_controls(probe, fleet, binding, fixture, damage, active): + result = {} + for side in fixture['sides'].values(): + expected, key = side['versions'], side['controlKey'] + policy = damage['policies'].get(key, {}) + status = run_task(probe, fleet, {'bucket': fixture['bucket'], 'key': key}, 2, active, + option_overrides={'recursive': True}, status_endpoints=tuple(binding['endpoints'])[1:]) + pair = (key, expected[0]['versionId']) + required = {pair} if policy.get('requiredRepair') else set() + settlement = policy.get('productSettlementAllowed') + if settlement: + for row in status.get('outcome', {}).get('objects', []): + identity = row.get('identity', {}) + if (identity.get('object'), public_version_id(identity.get('versionId'))) == pair: + require(row.get('disposition', {}).get('state') in {'repaired', 'verified_healthy'}, + 'DL original delete generation remains unresolved after product settlement') + if row['disposition']['state'] == 'repaired': required.add(pair) + proof = validate_version_receipts(status, fixture['bucket'], expected, 2, required_repairs=required) + if policy.get('originalBucketIncarnations'): + require(policy['originalBucketIncarnations'] == [proof['bucketIncarnationId']], + 'DL recovered task responsibility differs from the original delete journal incarnation') + result[key] = {**proof, 'originalDeletePolicy': copy.deepcopy(policy), + 'productSettlementMayPrecedeShardCheck': bool(settlement), 'finalPhysicalProofRequired': True} + return result diff --git a/test_validation_heal_network_adapter.py b/test_validation_heal_network_adapter.py new file mode 100644 index 00000000..470a2fc1 --- /dev/null +++ b/test_validation_heal_network_adapter.py @@ -0,0 +1,184 @@ +"""Execute live-adapter control flow with modeled services; no live acceptance.""" + +import copy +from contextlib import ExitStack +import json +import os +from pathlib import Path +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import network +from scripts.validation_heal.mpu_scene import keys_for +from scripts.validation_heal.network_load_repair import CANDIDATE + + +class NetworkAdapterTests(unittest.TestCase): + def run_mode(self, directory, mode): + events, state = [], {'running': False, 'recovered': None, 'session': None} + root = Path(directory);candidate = root / 'candidate.json';binary = root / 'candidate' + candidate.write_text(json.dumps({'source_sha': CANDIDATE, 'package_sha256': 'b' * 64}));binary.write_bytes(b'candidate') + args = types.SimpleNamespace(evidence=root / 'evidence', candidate=candidate, rc_bin=binary, + tool_directory=root, testing_commit='c' * 40, binary_sha='d' * 64) + nodes = ['vm000', 'vm001', 'vm002', 'vm003'] + payload = {'size': 100, 'sha256': 'original'} + def entry(key, version, latest): + return {'key': key, 'versionId': version, 'deleteMarker': False, 'isLatest': latest, 'payload': payload.copy()} + dv = [entry('objects/0001.bin', version, version == 'null') for version in ('v0', 'v1', 'v2', 'null')] + dl_expected = [entry(f'dl/{endpoint}/control.bin', f'{endpoint}-v{i}', i == 2) + for endpoint in ('node1', 'node3') for i in range(3)] + fixture = {'expected': dl_expected, 'sides': {endpoint: {'controlKey': f'dl/{endpoint}/control.bin', + 'versions': [row for row in dl_expected if row['key'] == f'dl/{endpoint}/control.bin']} + for endpoint in ('node1', 'node3')}} + class Fleet: + def __init__(self, nodes, evidence, run_id, *args, **kwargs): + self.nodes, self.evidence, self.run_id = nodes, evidence, run_id + self.network_gates, self.quorum_gates, self.faults = set(), set(), [] + self.reserved = nodes[:] + def record(self, label, value): events.append(('record', label));return value + def deploy(self, *args): state['running'] = True;events.append(('deploy',)) + def start(self): state['running'] = True;events.append(('start',)) + def stop(self): state['running'] = False;events.append(('stop',)) + def snapshot(self, label, **kwargs): + events.append(('snapshot', label, copy.deepcopy(kwargs))) + return {node: {'files': [], 'formats': []} for node in nodes} + def remote(self, node, action, **params): + events.append(('remote', node, action)) + if action == 'network-preflight': + return {'addresses': ['10.0.0.' + str(i) for i in range(1, 5)], 'localIndex': nodes.index(node), 'exclusiveServicePort': 9000} + if action == 'quorum-gate-probe': + return {'disks': [{'writable': True, 'readableMetadata': True, 'sameServiceUidAndMountNamespace': True}] * 4} + return {'state': 'CLEAR', 'rulesRemaining': 0, 'expiryTriggered': False} + def cleanup(self): + assert state['session'].safe_to_cleanup + events.append(('cleanup',));return True + class Probe: + def __init__(self, binding, endpoint='node1'): self.endpoint_name, self.binding = endpoint, binding + def inventory(self, bucket, **kwargs): + events.append(('inventory', bucket)) + rows = [] if bucket.endswith('-db') else copy.deepcopy(dv + dl_expected) + if state['recovered'] and not bucket.endswith('-db'): + row = state['recovered'] + rows.append({**entry(row['key'], row['versionId'], True), 'payload': row['payload'], + 'properties': {'Metadata': {'heal-fixture': 'I03', 'nonce': row['key']}}, + 'tags': [{'Key': 'fixture', 'Value': 'I03'}, {'Key': 'suite', 'Value': 'heal'}]}) + return {'bucket': bucket, 'entries': rows} + def admin(self, *args): return {'httpStatus': 200, 'response': {}} + test = self + class Session: + def __init__(self, binding, bucket, endpoints, directory): + self.binding, self.bucket, self.endpoints = binding, bucket, tuple(endpoints) + self.keys, self.safe_to_cleanup = keys_for(endpoints), False + self.fixture, self.stages = fixture, [] + state['session'] = self;events.append(('register-session',)) + def seed(self): + events.append(('seed-DL',)) + if mode == 'seed-unknown': raise ValueError('original seed Create result UNKNOWN') + return fixture + def register_baseline(self, value): + test.assertEqual({(r['key'], r['versionId']) for r in value['entries']}, + {(r['key'], r['versionId']) for r in dv + dl_expected}) + events.append(('freeze-public',)) + def register_business_baseline(self, value): + test.assertFalse(state['running']);events.append(('freeze-business',)) + def capture_mpu(self, fleet, stage): + test.assertFalse(state['running']);self.stages.append(stage) + events.append(('capture-MPU', stage));return {'nodes': {n: {} for n in nodes}} + def run_load(self, readable): + test.assertTrue(state['running']);test.assertEqual(readable, {'node1': True, 'node3': True}) + events.append(('load-inside-partition',)) + return {'complete': mode == 'pass'} + def registered_receipts(self): return {'original': 'preserved'} + def reconcile(self): + test.assertTrue(state['running']);test.assertEqual(self.stages[-1], 'pre-reconcile') + events.append(('reconcile-original-effects',));return {'namespaceCleanupAllowed': False} + def finish_reconciliation(self, scene, *, repair_targets): + test.assertFalse(state['running']);test.assertEqual(self.stages[-1], 'post-reconcile') + events.append(('close-original-effects',));self.safe_to_cleanup = True + return {'namespaceCleanupAllowed': True} + def map_target(snapshot, bucket, key, role, number, version, **kwargs): + return {'bucket': bucket, 'key': key, 'versionId': version, + 'node': 'vm000', 'disk': 0, 'relativePath': bucket + '/' + key + '/part.1', 'sha256': 'original', + 'cohort': [{'node': n, 'disk': 0} for n in nodes]} + def unique(probe, directory, bucket, key, rows): + events.append(('unique-PUT', key));test.assertTrue(state['running']) + if key.startswith('recovered/'): + test.assertTrue(state['session'].safe_to_cleanup) + row = {'key': key, 'versionId': 'new-original-ACK', 'endpoint': probe.endpoint_name, 'payload': payload.copy(), + 'observed': {'acknowledged': key.startswith('recovered/'), 'response': {'VersionId': 'new-original-ACK'}}} + rows.append(row) + if key.startswith('recovered/'): state['recovered'] = row + return row + def resolve(fleet, binding, bucket, rows): + for row in rows: row['resolution'] = {'state': 'REDEEMED' if row['observed']['acknowledged'] else 'ABSENT'} + return rows + def retain(fleet, binding, keys, business, mpu): + test.assertFalse(state['running']);test.assertFalse(fleet.network_gates or fleet.quorum_gates) + events.append(('freeze-retention-intent',));return {'original': 'intent'} + def isolate(fleet, plan): + events.append(('isolate-original-inodes',)) + if mode == 'isolation-error': raise ValueError('retained bytes drifted') + return {'cleanup': 'ISOLATED', 'originalSceneRestored': False} + plan = {'readQuorum': 8, 'sharedLockQuorum': 2, 'exclusiveLockQuorum': 3, 'gates': {}, + 'blocked': {node: [] for node in nodes}, 'sides': [ + {'endpoint': endpoint, 'members': members, 'metadataAuthorities': 8, + 'readableSources': 8 if index == 0 else 7, 'writableDrives': 8, 'lockClients': 2} + for index, (endpoint, members) in enumerate((('node1', [0, 1]), ('node3', [2, 3])))]} + replacements = {'Fleet': Fleet, 'Probe': Probe, 'LoadSession': Session, 'Recorder': lambda *args: None, + 'verify_inspector': lambda *args: {}, 'verify_quorum_contract': lambda *args: { + 'candidateCommit': CANDIDATE, 'recognized': True, 'sourceFiles': network.GET_SOURCE_FILES[CANDIDATE], + 'unrecoverableGet': {'httpStatus': 503, 'code': 'SlowDownRead'}}, + 'seed_versions': lambda *args: copy.deepcopy(dv), 'db_parameters': lambda *args: {'count': 128}, + 'seed_db': lambda *args: [], 'verify_seed': lambda *args: {}, 'version_heads': lambda *args: [], + 'validate_topology': lambda *args: {}, 'map_target': map_target, 'map_db': lambda *args: ([], []), + 'plan_for': lambda *args: plan, 'validate_fault': lambda *args: {}, + 'validate_witness': lambda *args: {}, 'expected_reads': lambda *args: [], + 'partition_get': lambda *args: ({'acknowledged': True, 'resultUnknown': False, 'payload': payload.copy()}, {'complete': True}), + 'validate_unrecoverable_get': lambda *args: None, 'unique_put': unique, 'negative_put': lambda *args: {}, + 'partition_db': lambda *args: {}, 'negative_heal': lambda *args: {}, 'validate_partial': lambda *args: {}, + 'classify_deletes': lambda *args: {'policies': {}, 'repairTargets': []}, + 'recover_versions': lambda *args: {}, 'recover_controls': lambda *args: {}, 'idle_fleet': lambda *args: None, + 'validate_repaired_physical': lambda *args, **kwargs: {}, + 'verify_cohort': lambda fleet, cohort, label: events.append(('streaming-cohort', cohort['key'], cohort['versionId'])), + 'prove_db': lambda *args: {}, 'resolve_writes': resolve, 'compare_inventories': lambda *args: {}, + 'verify_db': lambda *args: {}, 'prepare_retention': retain, 'isolate_retention': isolate} + with ExitStack() as stack: + stack.enter_context(patch.dict(os.environ, {'GITHUB_RUN_ID': '12345', 'GITHUB_RUN_ATTEMPT': '1'})) + for name, value in replacements.items(): stack.enter_context(patch.object(network, name, value)) + result = network.run_branch(args, nodes, 'read-boundary') + return result, events + + def test_complete_adapter_keeps_all_dl_versions_physical_then_closes_before_new_safe_put(self): + with tempfile.TemporaryDirectory() as d: result, events = self.run_mode(d, 'pass') + self.assertEqual((result['verdict'], result['cleanup']), ('PASS', 'PASS')) + cohorts = [event for event in events if event[0] == 'streaming-cohort' and event[1].startswith('dl/')] + self.assertEqual(len(cohorts), 6) + self.assertLess(events.index(('freeze-business',)), events.index(('load-inside-partition',))) + self.assertLess(events.index(('load-inside-partition',)), events.index(('record', 'all-directed-peer-probes-after'))) + self.assertLess(events.index(('streaming-cohort', 'dl/node3/control.bin', 'node3-v2')), + events.index(('reconcile-original-effects',))) + self.assertLess(events.index(('close-original-effects',)), events.index(('unique-PUT', 'recovered/unique.bin'))) + self.assertEqual([e for e in events if e[0] == 'cleanup'], [('cleanup',)]) + self.assertFalse(any(e[0] == 'isolate-original-inodes' for e in events)) + for event in events: + if event[0] == 'snapshot' and event[1] != 'DB-size-calibration-live': + self.assertEqual(len(event[2]['buckets']), 2) + + def test_unknown_seed_or_load_never_runs_ordinary_cleanup_or_retries_original_mutation(self): + for mode in ('seed-unknown', 'unknown-load', 'isolation-error'): + with self.subTest(mode=mode), tempfile.TemporaryDirectory() as d: + result, events = self.run_mode(d, mode) + self.assertEqual((result['verdict'], result['cleanup']), ('FAIL', 'FAIL')) + self.assertNotIn(('cleanup',), events) + self.assertNotIn(('unique-PUT', 'recovered/unique.bin'), events) + self.assertNotIn(('reconcile-original-effects',), events) + self.assertIn(('isolate-original-inodes',), events) + if mode == 'seed-unknown': + self.assertFalse(any(event[0] == 'inventory' for event in events)) + if mode == 'isolation-error': self.assertIn('retentionError', result) + else: self.assertEqual(result['retention']['cleanup'], 'ISOLATED') + + +if __name__ == '__main__': unittest.main() diff --git a/test_validation_heal_network_delete_records.py b/test_validation_heal_network_delete_records.py new file mode 100644 index 00000000..940e1564 --- /dev/null +++ b/test_validation_heal_network_delete_records.py @@ -0,0 +1,113 @@ +"""Real stopped fixture journal bytes bind safe selectors without exporting metadata.""" + +import copy +from contextlib import redirect_stdout +import hashlib +import io +import json +import os +from pathlib import Path +import tempfile +import types +import unittest +from unittest.mock import patch + +from scripts.validation_heal import network_delete_records as records +from scripts.validation_heal.healthy import Fleet +import test_validation_heal_mpu_scene as mpu_tests + + +class DeleteRecordTests(unittest.TestCase): + def scene(self, directory): + request, node, roots, ignored = mpu_tests.MultipartSceneTests().fixture(Path(directory)) + keys = ['dl/node1/control.bin', 'dl/node3/control.bin'] + request.update(action='network-delete-records', keys=keys) + transaction = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' + versions = [f'12345678-1234-1234-1234-{i + 1:012d}' for i in range(3)] + dirs = [f'22345678-1234-1234-1234-{i + 1:012d}' for i in range(3)] + paths = [] + for root in roots: + for key in keys: + folder = root / 'data' / request['bucket'] / key / transaction;folder.mkdir(parents=True) + value = {'protocol_version': 1, 'volume': request['bucket'], 'object': key, 'transaction': transaction, + 'bucket_incarnation': '33333333-3333-3333-3333-333333333333', + 'original_digest': [1] * 32, 'target_digest': [2] * 32, + 'data_dirs': dirs[:1], 'deleted_versions': versions[:1], + 'requests': [{'name': key, 'version_id': versions[0], 'deleted': False, 'mark_deleted': False, + 'metadata': {'fixture-private-value': 'must-not-be-exported'}}], + 'original_versions': [{'version_id': v, 'data_dir': d, 'deleted': False, 'metadata_digest': [3] * 32} + for v, d in zip(versions, dirs)], 'predecessors': []} + path = folder / 'delete-prepared';path.write_text(json.dumps(value));paths.append(path) + (folder / 'do-not-read').write_bytes(b'private payload') + return request, node, roots, paths + + def test_all_four_original_disks_two_keys_raw_shas_are_retained_without_exporting_nested_metadata(self): + with tempfile.TemporaryDirectory() as directory: + request, node, roots, paths = self.scene(directory) + before = {p: p.read_bytes() for p in paths};opened, real_open = [], os.open + def checked(path, *args, **kwargs): + self.assertIn(Path(path), paths);opened.append(Path(path));return real_open(path, *args, **kwargs) + with patch.object(records.os, 'open', side_effect=checked): value = records.run(request, node) + self.assertEqual(len(value['records']), 8) + self.assertEqual(set(opened), set(paths)) + for row in value['records']: + path = roots[row['disk']] / 'data' / row['path'] + self.assertEqual(row['sha256'], hashlib.sha256(before[path]).hexdigest()) + self.assertEqual(row['size'], len(before[path])) + self.assertEqual(row['originalMetadataSha256'], '01' * 32) + self.assertEqual(row['targetMetadataSha256'], '02' * 32) + self.assertNotIn('must-not-be-exported', json.dumps(value)) + self.assertTrue(value['readOnly'] and value['quiescent']) + self.assertFalse(value['commitOrRollbackClaimed'] or value['rawContentsExported'] or value['credentialContentsRead']) + self.assertEqual({p: p.read_bytes() for p in paths}, before) + + def test_wrong_scope_schema_digest_selector_type_duplicate_fields_links_and_live_service_refuse(self): + for kind in ('bucket', 'keys', 'runtime', 'live', 'schema', 'duplicate-json', 'digest', 'extra-field', 'symlink', 'hardlink', 'oversize'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as directory: + request, node, roots, paths = self.scene(directory);path = paths[0] + if kind == 'bucket': request['bucket'] = 'foreign' + elif kind == 'keys': request['keys'][1] = 'dl/node2/control.bin' + elif kind == 'runtime': node['owned'](request['runId'])[2]['runtimeProfile'] = 'manual' + elif kind == 'live': node['stopped'] = lambda owner: (_ for _ in ()).throw(ValueError('running')) + elif kind == 'symlink': path.unlink();path.symlink_to(Path(directory).resolve() / 'state/credentials.env') + elif kind == 'hardlink': os.link(path, path.with_name('extra-link')) + elif kind == 'oversize': path.write_bytes(b'x' * (1024**2 + 1)) + else: + value = json.loads(path.read_text()) + if kind == 'schema': value['protocol_version'] = True + elif kind == 'digest': value['original_digest'][0] = True + elif kind == 'extra-field': value['unknown'] = 'unbound' + if kind == 'duplicate-json': path.write_text(json.dumps(value)[:-1] + ',"protocol_version":1}') + else: path.write_text(json.dumps(value)) + with self.assertRaises(ValueError): records.run(request, node) + + def test_owner_change_after_read_refuses_original_scene_claim(self): + with tempfile.TemporaryDirectory() as directory: + request, node, roots, paths = self.scene(directory);original = node['owned'];calls = [] + def changed(run_id): + state, roots, owner = original(run_id);calls.append(run_id) + return state, roots, owner if len(calls) == 1 else {**owner, 'bucket': 'foreign'} + node['owned'] = changed + with self.assertRaisesRegex(ValueError, 'owner/roots changed'): records.run(request, node) + + def test_actual_remote_source_injection_and_action_dispatch_use_the_read_only_backend(self): + with tempfile.TemporaryDirectory() as directory: + request, node, roots, paths = self.scene(directory) + def execute(*args, **kwargs): + source = kwargs['input'].rsplit('\nmain(', 1)[0] + scope = {'__name__': '__owned_node_test__'} + exec(compile(source, 'network-delete-node', 'exec'), scope) + scope.update(node) + out = io.StringIO() + with redirect_stdout(out), patch.object(scope['os'], 'geteuid', return_value=0): + scope['main'](request) + return types.SimpleNamespace(returncode=0, stdout=out.getvalue(), stderr='') + with patch('scripts.validation_heal.healthy.subprocess.run', side_effect=execute): + result = Fleet(['vm000'], Path(directory), request['runId'], runtime_profile='lock-quorum').remote( + 'vm000', 'network-delete-records', bucket=request['bucket'], keys=request['keys'], + expectedBinarySha256=request['expectedBinarySha256']) + self.assertEqual(len(result['records']), 8) + self.assertFalse(result['credentialContentsRead'] or result['commitOrRollbackClaimed']) + + +if __name__ == '__main__': unittest.main() diff --git a/test_validation_heal_network_load_repair.py b/test_validation_heal_network_load_repair.py new file mode 100644 index 00000000..65801c98 --- /dev/null +++ b/test_validation_heal_network_load_repair.py @@ -0,0 +1,199 @@ +"""Exact historical DL deletion effects cannot hide corruption or new commits.""" + +import copy +import hashlib +import unittest +from unittest.mock import patch + +import test_validation_heal as oracle +from scripts.validation_heal import network_load_repair as repair, network_load as load, network +from scripts.validation_heal.shards import map_target + + +class LoadRepairTests(unittest.TestCase): + def scene(self): + base, _ = oracle.HealReadQuorumBoundaryTest().scene(total=16, data=8) + bucket = oracle.HealReadQuorumBoundaryTest.bucket + snapshot = {node: {**copy.deepcopy(value), 'files': [], 'binarySha256': 'a' * 64} for node, value in base.items()} + fixture = {'bucket': bucket, 'runId': 'gh-12345-1-h2', 'sourceContract': load.source_reference(repair.CANDIDATE), 'sides': {}} + for side_index, endpoint in enumerate(('node1', 'node4')): + key = 'dl/' + endpoint + '/control.bin' + expected = [{'key': key, 'versionId': f'v{side_index}{nonce}', 'isLatest': nonce == 2, + 'deleteMarker': False, 'payload': {'size': 100, 'sha256': 'original' + str(nonce)}} for nonce in range(3)] + fixture['sides'][endpoint] = {'controlKey': key, 'versions': expected, + 'uploads': {name: {'key': f'dl/{endpoint}/{name}.bin', 'uploadId': name + '-original', + 'parts': [{'PartNumber': 1, 'ETag': 'original'}]} for name in ('complete', 'upload', 'abort')}} + for node, value in base.items(): + for original in value['files']: + if 'metadata' not in original: continue + disk = original['disk'];meta = copy.deepcopy(original['metadata']);meta['key'] = key + template = meta['versions'][0];versions = [] + for nonce, entry in enumerate(expected): + version = copy.deepcopy(template) + version.update(versionId=entry['versionId'], isLatest=entry['isLatest'], + dataDir=f'12345678-1234-1234-1234-{side_index * 10 + nonce:012d}') + versions.append(version) + snapshot[node]['files'].append({'disk': disk, 'path': f"{bucket}/{key}/{version['dataDir']}/part.1", + 'size': 82, 'sha256': hashlib.sha256(f'{node}/{disk}/{key}/{nonce}'.encode()).hexdigest()}) + meta['versions'] = versions + snapshot[node]['files'].append({'disk': disk, 'path': f'{bucket}/{key}/xl.meta', 'size': 600, + 'sha256': hashlib.sha256(f'meta/{node}/{disk}/{key}'.encode()).hexdigest(), 'metadata': meta}) + first = fixture['sides']['node1']['versions'][0] + target = map_target(snapshot, bucket, first['key'], 'data', 1, first['versionId'], latest_required=False) + target = {**target, **next(row for row in target['cohort'] if row['node'] == 'node4')} + plan = network.plan_for(snapshot, target, 'minority-partition') + workers = [] + for endpoint in fixture['sides']: + step = load.mutation_steps(fixture, endpoint)[3] + request = {**copy.deepcopy(step), 'identity': {'role': 'writer', 'endpoint': endpoint, 'step': 4, 'nonce': 6}, + 'expectHttp': 503} + row = {'request': request, 'ackPresent': True, 'ack': {'operationCheck': 'PASS', 'observation': { + 'clientCallInvoked': True, 'resultUnknown': False, 'acknowledged': False, + 'response': {'ResponseMetadata': {'HTTPStatusCode': 503}, 'Error': {'Code': 'ServiceUnavailable'}}}}} + workers.append({'endpoint': endpoint, 'role': 'writer', 'rows': [row]}) + return snapshot, fixture, {'workers': workers}, plan + + def deleted(self, snapshot, fixture, *, staged=True): + after = copy.deepcopy(snapshot);node, disk = 'node2', 0 + key = fixture['sides']['node1']['controlKey'];bucket = fixture['bucket'] + meta = next(row for row in after[node]['files'] if row['disk'] == disk and row['path'] == f'{bucket}/{key}/xl.meta') + original = copy.deepcopy(meta);version = meta['metadata']['versions'].pop(0);meta['sha256'] = 'new-metadata' + path = f"{bucket}/{key}/{version['dataDir']}/part.1" + after[node]['files'] = [row for row in after[node]['files'] if (row['disk'], row['path']) != (disk, path)] + if staged: + prefix = f'{bucket}/{key}/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa/' + after[node]['files'].extend([ + {'disk': disk, 'path': prefix + 'xl.meta.bkp', 'size': original['size'], 'sha256': original['sha256']}, + {'disk': disk, 'path': prefix + 'delete-prepared', 'size': 300, 'sha256': 'prepared'}, + {'disk': disk, 'path': prefix + 'delete-target', 'size': 500, 'sha256': 'target'}]) + return after + + def records(self, before, after, fixture): + keys = sorted(side['controlKey'] for side in fixture['sides'].values()) + result = {node: {'readOnly': True, 'quiescent': True, 'runId': fixture['runId'], 'bucket': fixture['bucket'], + 'binarySha256': 'a' * 64, 'keys': keys, 'credentialContentsRead': False, 'records': []} for node in before} + for node, value in after.items(): + for file in value['files']: + if not file['path'].endswith('/delete-prepared'): continue + path = file['path'];prefix, transaction, _ = path.rsplit('/', 2) + key = prefix[len(fixture['bucket']) + 1:] + original = next(row for row in before[node]['files'] if row['disk'] == file['disk'] and row['path'] == prefix + '/xl.meta') + target = next(row for row in value['files'] if row['disk'] == file['disk'] and row['path'] == prefix + '/' + transaction + '/delete-target') + versions = original['metadata']['versions'];first = versions[0] + result[node]['records'].append({'disk': file['disk'], 'key': key, 'path': path, 'size': file['size'], + 'sha256': file['sha256'], 'protocolVersion': 1, 'transaction': transaction, + 'requestVersion': first['versionId'], 'deletedVersions': [first['versionId']], 'dataDirs': [first['dataDir']], + 'originalMetadataSha256': original['sha256'], 'targetMetadataSha256': target['sha256'], + 'bucketIncarnation': 'original-incarnation', + 'originalVersions': [row['versionId'] for row in versions], 'originalDataDirs': [row['dataDir'] for row in versions]}) + return result + + def status(self, fixture, key, repaired=()): + expected = next(side['versions'] for side in fixture['sides'].values() if side['controlKey'] == key) + objects = [{'identity': {'bucket': fixture['bucket'], 'object': key, 'versionId': row['versionId'], + 'bucketIncarnationId': 'original-incarnation', 'poolIndex': 0, 'setIndex': 0}, + 'disposition': {'state': 'repaired' if row['versionId'] in repaired else 'verified_healthy'}} for row in expected] + return {'summary': 'finished', 'truncated': False, 'outcome': {'execution': {'state': 'completed'}, + 'coverage': 'complete', 'objectsTruncated': False, 'objects': objects, 'counters': { + 'processed': 3, 'healed': len(repaired), 'unchanged': 3 - len(repaired), 'skipped': 0, + 'failed': 0, 'unknown': 0, 'attemptFailures': 0, 'overflowed': False}}} + + def test_source_contract_is_exact_and_no_change_means_both_three_version_keys_remain_healthy(self): + before, fixture, receipts, plan = self.scene() + damage = repair.classify_deletes(before, copy.deepcopy(before), fixture, receipts, plan, list(before), self.records(before, before, fixture)) + self.assertEqual(damage['repairTargets'], []) + with self.assertRaises(ValueError): repair.source_reference('072b57b37559fbe626aeda8abf61617fdcbdd926') + calls = [] + def task(probe, fleet, selection, mode, active, **kwargs): + calls.append(selection);self.assertTrue(kwargs['option_overrides']['recursive']) + return self.status(fixture, selection['key']) + with patch.object(repair, 'run_task', side_effect=task): + result = repair.recover_controls(None, None, {'endpoints': {f'node{i}': 'url' for i in range(1, 5)}}, fixture, damage, []) + self.assertEqual(len(calls), 2) + self.assertTrue(all(value['receipts'] == 3 and value['repaired'] == 0 for value in result.values())) + + def test_staged_original_delete_can_be_product_settled_but_does_not_itself_claim_rollback(self): + before, fixture, receipts, plan = self.scene() + after = self.deleted(before, fixture) + damage = repair.classify_deletes(before, after, fixture, receipts, plan, list(before), self.records(before, after, fixture)) + key = fixture['sides']['node1']['controlKey'];policy = damage['policies'][key] + self.assertFalse(policy['requiredRepair'] or damage['refusalProvesRollback'] or damage['canonicalCasePassClaimed']) + self.assertTrue(policy['productSettlementAllowed'] and policy['finalPhysicalProofRequired']) + self.assertEqual(policy['changedOriginalSlots'], [('node2', 0)]) + self.assertEqual(len(damage['repairTargets']), 1) + for state in ('healthy-after-product-rollback', 'shard-repaired'): + with self.subTest(state=state): + def task(probe, fleet, selection, mode, active, **kwargs): + repaired = ['v00'] if state == 'shard-repaired' and selection['key'] == key else [] + return self.status(fixture, selection['key'], repaired) + with patch.object(repair, 'run_task', side_effect=task): + result = repair.recover_controls(None, None, {'endpoints': {'node1': '1', 'node2': '2', 'node3': '3', 'node4': '4'}}, fixture, damage, []) + self.assertTrue(result[key]['productSettlementMayPrecedeShardCheck']) + self.assertTrue(result[key]['finalPhysicalProofRequired']) + + def test_loss_without_exact_prepared_backup_requires_real_repaired_responsibility(self): + before, fixture, receipts, plan = self.scene();after = self.deleted(before, fixture, staged=False) + damage = repair.classify_deletes(before, after, fixture, receipts, plan, list(before), self.records(before, after, fixture)) + key = fixture['sides']['node1']['controlKey'] + self.assertTrue(damage['policies'][key]['requiredRepair']) + with patch.object(repair, 'run_task', return_value=self.status(fixture, key)): + with self.assertRaisesRegex(ValueError, 'repaired responsibility'): + repair.recover_controls(None, None, {'endpoints': {'node1': '1'}}, fixture, damage, []) + + def test_other_versions_roots_foreign_files_and_bad_original_request_never_gain_an_exception(self): + for kind in ('other-version', 'changed-bytes', 'foreign-version', 'read-only-slot', 'disconnected-slot', + 'new-key', 'bad-backup', 'foreign-staged-version', 'unknown-transaction-file', 'wrong-request', 'unknown-response'): + with self.subTest(kind=kind): + before, fixture, receipts, plan = self.scene();after = self.deleted(before, fixture) + rows = after['node2']['files'];bucket = fixture['bucket'];key = fixture['sides']['node1']['controlKey'] + if kind == 'other-version': + chosen = next(row for row in rows if row['disk'] == 0 and row['path'].endswith('000000000001/part.1')) + rows.remove(chosen) + elif kind == 'changed-bytes': next(row for row in rows if row['path'].endswith('000000000001/part.1'))['sha256'] = 'changed' + elif kind == 'foreign-version': next(row for row in rows if row.get('metadata'))['metadata']['versions'][0]['versionId'] = 'foreign' + elif kind in {'read-only-slot', 'disconnected-slot'}: + node = 'node1' if kind == 'read-only-slot' else 'node4' + after[node]['files'] = [row for row in after[node]['files'] if not (row['disk'] == 0 and row['path'].endswith('000000000000/part.1'))] + elif kind == 'new-key': rows.append({'disk': 0, 'path': bucket + '/dl/node1/copy.bin/part.1', 'size': 1, 'sha256': 'new'}) + elif kind == 'bad-backup': next(row for row in rows if '/aaaaaaaa-' in row['path'] and row['path'].endswith('xl.meta.bkp'))['sha256'] = 'foreign' + elif kind == 'foreign-staged-version': rows.append({'disk': 0, 'path': f'{bucket}/{key}/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa/12345678-1234-1234-1234-000000000001/part.1', 'size': 82, 'sha256': 'foreign'}) + elif kind == 'unknown-transaction-file': rows.append({'disk': 0, 'path': f'{bucket}/{key}/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa/unknown', 'size': 1, 'sha256': 'foreign'}) + elif kind == 'wrong-request': receipts['workers'][0]['rows'][0]['request']['params']['VersionId'] = 'v01' + else: receipts['workers'][0]['rows'][0]['ack']['observation']['resultUnknown'] = True + with self.assertRaises(ValueError): repair.classify_deletes(before, after, fixture, receipts, plan, list(before), self.records(before, after, fixture)) + + def test_raw_journal_census_must_match_exact_original_ack_selector_backup_target_and_incarnation(self): + for kind in ('missing-peer', 'missing-record', 'wrong-selector', 'wrong-backup', 'wrong-target', 'wrong-generation', 'wrong-incarnation'): + with self.subTest(kind=kind): + before, fixture, receipts, plan = self.scene();after = self.deleted(before, fixture) + records = self.records(before, after, fixture);row = records['node2']['records'][0] + if kind == 'missing-peer': records.pop('node4') + elif kind == 'missing-record': records['node2']['records'] = [] + elif kind == 'wrong-selector': row['requestVersion'] = 'unbound' + elif kind == 'wrong-backup': row['originalMetadataSha256'] = 'unbound' + elif kind == 'wrong-target': row['targetMetadataSha256'] = 'unbound' + elif kind == 'wrong-generation': row['originalDataDirs'][1] = 'unbound' + else: row['bucketIncarnation'] = 'foreign-incarnation' + if kind == 'wrong-incarnation': + damage = repair.classify_deletes(before, after, fixture, receipts, plan, list(before), records) + key = fixture['sides']['node1']['controlKey'] + with patch.object(repair, 'run_task', return_value=self.status(fixture, key)), self.assertRaisesRegex(ValueError, 'incarnation'): + repair.recover_controls(None, None, {'endpoints': {'node1': 'url'}}, fixture, damage, []) + else: + with self.assertRaises(ValueError): repair.classify_deletes(before, after, fixture, receipts, plan, list(before), records) + + def test_hidden_status_failure_wrong_counters_and_control_repairs_are_not_settlement(self): + for kind in ('unknown', 'counters', 'other-repair', 'missing-version'): + with self.subTest(kind=kind): + before, fixture, receipts, plan = self.scene();after = self.deleted(before, fixture) + damage = repair.classify_deletes(before, after, fixture, receipts, plan, list(before), self.records(before, after, fixture)) + key = fixture['sides']['node1']['controlKey'];status = self.status(fixture, key) + if kind == 'unknown': status['outcome']['objects'][0]['disposition']['state'] = 'unknown' + elif kind == 'counters': status['outcome']['counters']['healed'] = 1 + elif kind == 'other-repair': status['outcome']['objects'][1]['disposition']['state'] = 'repaired' + else: status['outcome']['objects'].pop() + with patch.object(repair, 'run_task', return_value=status), self.assertRaises(ValueError): + repair.recover_controls(None, None, {'endpoints': {'node1': '1'}}, fixture, damage, []) + + +if __name__ == '__main__': unittest.main() From 3b064c90bc24a11003c1721a964a9c30b831751a Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 20:39:39 +0800 Subject: [PATCH 241/343] test(heal): freeze failed network scene before gate recovery --- docs/validation-migration-map.md | 9 ++++++-- scripts/validation_heal/network.py | 10 +++++++-- test_validation_heal_network_adapter.py | 28 ++++++++++++++++++++----- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index c01516bb..ea1e065f 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2160,8 +2160,13 @@ and incarnation. All final physical files/formats, original shard bytes and public versions/metadata/tags still must match before cleanup. MPU retirement uses separate recovery ACKs and stopped final MPU/business checks. These occur before the recovered unique PUT. An incomplete seed/load or unresolved effect -instead stops services, releases owned gates, captures the original scene and -isolates every inode; the case and cleanup remain FAIL until separately audited. +instead stops services, captures the original scene, releases owned gates and +isolates unchanged inodes; the case and cleanup remain FAIL until separately audited. + +Failure capture order is explicit: stop services, save original stopped +business/MPU bytes and logs while fault intents still exist, release only +owned network/private-write gates, and recapture the stopped scene. Byte +hashes must remain identical across gate recovery before isolation can begin. The full adapter intentionally requires the exact fixed candidate above. Historical `072b57b...` scoped source/contracts are retained, but its different diff --git a/scripts/validation_heal/network.py b/scripts/validation_heal/network.py index 60a15453..d4e19cf6 100644 --- a/scripts/validation_heal/network.py +++ b/scripts/validation_heal/network.py @@ -655,6 +655,11 @@ def witness(label): dirty = True try: fleet.stop() + original_failure = fleet.snapshot('DL-original-unresolved-business-before-gate-release', + quiescent=True, buckets=[bucket, bucket + '-db']) + original_mpu = dl.capture_mpu(fleet, 'failure') + for node in nodes: + fleet.record('DL-original-unresolved-service-logs', {'node': node, **fleet.remote(node, 'logs')}) for node in tuple(fleet.network_gates): proof = fleet.record('DL-failure-owned-network-release', {'node': node, **fleet.remote(node, 'network-release', expired=True)}) @@ -668,8 +673,9 @@ def witness(label): fleet.quorum_gates.remove(node) failed_scene = fleet.snapshot('DL-original-unresolved-business-scene', quiescent=True, buckets=[bucket, bucket + '-db']) failed_mpu = dl.capture_mpu(fleet, 'failure') - for node in nodes: - fleet.record('DL-original-unresolved-service-logs', {'node': node, **fleet.remote(node, 'logs')}) + require(physical_content(original_failure) == physical_content(failed_scene) + and all(original_mpu['nodes'][node]['sceneSha256'] == failed_mpu['nodes'][node]['sceneSha256'] + for node in nodes), 'DL gate recovery changed the original failed business or MPU bytes') retained_plan = prepare_retention(fleet, binding, dl.keys, failed_scene, failed_mpu['nodes']) result['retention'] = isolate_retention(fleet, retained_plan) except Exception as exc: diff --git a/test_validation_heal_network_adapter.py b/test_validation_heal_network_adapter.py index 470a2fc1..2f720e9b 100644 --- a/test_validation_heal_network_adapter.py +++ b/test_validation_heal_network_adapter.py @@ -43,7 +43,11 @@ def start(self): state['running'] = True;events.append(('start',)) def stop(self): state['running'] = False;events.append(('stop',)) def snapshot(self, label, **kwargs): events.append(('snapshot', label, copy.deepcopy(kwargs))) - return {node: {'files': [], 'formats': []} for node in nodes} + value = {node: {'files': [], 'formats': []} for node in nodes} + if mode == 'business-gate-drift' and label == 'DL-original-unresolved-business-scene': + value[nodes[0]]['files'].append({'disk': 0, 'path': state['session'].bucket + '/unexpected-part', + 'size': 1, 'sha256': 'changed-during-gate-release'}) + return value def remote(self, node, action, **params): events.append(('remote', node, action)) if action == 'network-preflight': @@ -85,7 +89,11 @@ def register_business_baseline(self, value): test.assertFalse(state['running']);events.append(('freeze-business',)) def capture_mpu(self, fleet, stage): test.assertFalse(state['running']);self.stages.append(stage) - events.append(('capture-MPU', stage));return {'nodes': {n: {} for n in nodes}} + events.append(('capture-MPU', stage)) + value = {'nodes': {n: {'sceneSha256': 'f' * 64} for n in nodes}} + if mode == 'mpu-gate-drift' and self.stages.count('failure') == 2: + value['nodes'][nodes[0]]['sceneSha256'] = 'e' * 64 + return value def run_load(self, readable): test.assertTrue(state['running']);test.assertEqual(readable, {'node1': True, 'node3': True}) events.append(('load-inside-partition',)) @@ -167,18 +175,28 @@ def test_complete_adapter_keeps_all_dl_versions_physical_then_closes_before_new_ self.assertEqual(len(event[2]['buckets']), 2) def test_unknown_seed_or_load_never_runs_ordinary_cleanup_or_retries_original_mutation(self): - for mode in ('seed-unknown', 'unknown-load', 'isolation-error'): + for mode in ('seed-unknown', 'unknown-load', 'isolation-error', 'business-gate-drift', 'mpu-gate-drift'): with self.subTest(mode=mode), tempfile.TemporaryDirectory() as d: result, events = self.run_mode(d, mode) self.assertEqual((result['verdict'], result['cleanup']), ('FAIL', 'FAIL')) self.assertNotIn(('cleanup',), events) self.assertNotIn(('unique-PUT', 'recovered/unique.bin'), events) self.assertNotIn(('reconcile-original-effects',), events) - self.assertIn(('isolate-original-inodes',), events) + if mode.endswith('gate-drift'): + self.assertNotIn(('isolate-original-inodes',), events) + self.assertIn('gate recovery changed', result['retentionError']['message']) + else: + self.assertIn(('isolate-original-inodes',), events) if mode == 'seed-unknown': self.assertFalse(any(event[0] == 'inventory' for event in events)) if mode == 'isolation-error': self.assertIn('retentionError', result) - else: self.assertEqual(result['retention']['cleanup'], 'ISOLATED') + elif not mode.endswith('gate-drift'): self.assertEqual(result['retention']['cleanup'], 'ISOLATED') + if mode != 'seed-unknown': + before = next(i for i, event in enumerate(events) if event[:2] == + ('snapshot', 'DL-original-unresolved-business-before-gate-release')) + releases = [i for i, event in enumerate(events) if event[0] == 'remote' and event[2] == 'network-release'] + self.assertEqual(len(releases), 4) + self.assertLess(before, min(releases)) if __name__ == '__main__': unittest.main() From 293fe2c353f6a04f6094e28e2dc7a1119188c66e Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 21:50:25 +0800 Subject: [PATCH 242/343] docs(validation): record full I03 failure and retained recovery --- docs/validation-migration-map.md | 35 ++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index ea1e065f..cc63e9b6 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -9,17 +9,17 @@ live run proves each source assertion on the required topology. ## Current acceptance and remaining work -All 158 canonical IDs are retained. Independent complete live acceptance is **103/158**: the prior 101-case census plus B07 `heal.below-read-quorum` ([run 38727279347/1](https://github.com/rustfs/rustfs/actions/runs/38727279347)) and A10 `heal.admin-option-contract` ([run 39197088183/1](https://github.com/rustfs/rustfs/actions/runs/39197088183)). Both complete source matrices, immutable candidate/package/tools and cleanup were independently audited; earlier failures remain retained. +All 158 canonical IDs are retained. Independent complete live acceptance is **104/158**: the prior 101-case census plus B07 `heal.below-read-quorum` ([run 38727279347/1](https://github.com/rustfs/rustfs/actions/runs/38727279347)), A10 `heal.admin-option-contract` ([run 39197088183/1](https://github.com/rustfs/rustfs/actions/runs/39197088183)) and B06 `heal.read-quorum-boundary` ([run 39223475622/1](https://github.com/rustfs/rustfs/actions/runs/39223475622)). Their complete source matrices, immutable candidate/package/tools and cleanup were independently audited; earlier failures remain retained. | Acceptance state | Cases | Remaining requirement | | --- | ---: | --- | -| Independently accepted complete live PASS | 103 | Preserve original provenance and complete results. | -| Retained live failures with existing executors | 31 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | +| Independently accepted complete live PASS | 104 | Preserve original provenance and complete results. | +| Retained live failures with existing executors | 30 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | | Required native barriers | 7 | A09 and G01–G06 need exact candidate-bound server lock/version/commit barriers. | | Orchestration or evidence blocked | 4 | C11, F07, I05 and J02 need their remaining native or full live evidence closure. | | Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | -The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 42 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. +The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 41 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. [J02 diagnostic 38874900846/1](https://github.com/rustfs/rustfs/actions/runs/38874900846) completed all 20 H1/H2 × DB/DV branch instances on that exact candidate. Sixteen branches report PASS; the four original all-metadata-corruption probes report permanent Failed rather than Unknown and retain NOT_OBSERVED. Independent archive and provenance audits passed. Independent mixed-branch audits verified all four repaired shards against original complete bytes, exact receipts and counters, surviving below-quorum failures, original terminal preservation and post-rescue S3 inventories. All 50 node namespace finalization receipts passed audit. Real legacy writes separately produce Unknown with zero repairs and unchanged physical files, but do not replace the original source procedure. Full J02 canonical acceptance remains blocked; these partial results add no canonical acceptance. @@ -2175,3 +2175,30 @@ compiled collector, inspector pin, native barrier, request retry or time budget changes in this delivery. Local modeled orchestration and filesystem tests remain distinct from new full-fixture live evidence. Canonical acceptance remains **104/158** until that evidence is independently verified. + + +I03 full-fixture live run [39254429889/1](https://github.com/rustfs/rustfs/actions/runs/39254429889) +used helper `3b064c90bc24a11003c1721a964a9c30b831751a`, fixed candidate +`1c7fc818dd858247c45b39542abd8291b1e6e31b`, package `38288322026/1` +and inspector `39214576404/1`. Its first minority-partition branch failed: +19 DL requests were actually invoked and fully drained, with zero unknown +results. The node1 historical DELETE and node4 GET returned definite +`500/InternalError`. Original request IDs, exact version selectors, seed +payloads and current server PIDs were independently correlated. The remaining +four branches did not run. These responses retain their original FAIL; +no expected-status relaxation or product root-cause claim is made. + +The original stopped business/MPU/log scene was captured before owned-gate +removal, with unchanged business and MPU hashes afterward. All original +root/state trees on the four nodes were isolated by inode-preserving same-filesystem moves into +private per-run archives. Separate recovery re-audit +[39257565469/1](https://github.com/rustfs/rustfs/actions/runs/39257565469) +verified all four preflight, no-op isolation and post-audit receipts. Independent +audit checked the official ZIP digests, all 723 unchanged original evidence +members, all 20 original root/state content and inode censuses, unchanged +private archive parents and the exact executed stopped-service/binary/mount +guards. Two sequential reviews passed before fleet ownership was released. +Recovery reports `ISOLATED`; it performs no S3 requests, namespace deletion, +old-shard restoration or original verdict rewrite. Original case SHA-256 is +`13d06d16e8a05d544ec5dd6df5ae8bdabcd6c204c85f8ca4f411196569807099`. +The complete I03 case remains unaccepted; the canonical total stays **104/158**. From 536a0a13b7a428a557dbe20d03faf72808da4abd Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 22:21:01 +0800 Subject: [PATCH 243/343] test(heal): pin the reviewed I03 diagnostic candidate --- docs/validation-migration-map.md | 28 +++++++++++++++++++ scripts/validation_heal/network.py | 9 ++++++ scripts/validation_heal/network_fixtures.py | 1 + scripts/validation_heal/network_load.py | 7 +++++ .../validation_heal/network_load_repair.py | 5 +++- scripts/validation_heal/quorum_boundary.py | 5 ++++ 6 files changed, 54 insertions(+), 1 deletion(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index cc63e9b6..dc3f34ee 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2202,3 +2202,31 @@ Recovery reports `ISOLATED`; it performs no S3 requests, namespace deletion, old-shard restoration or original verdict rewrite. Original case SHA-256 is `13d06d16e8a05d544ec5dd6df5ae8bdabcd6c204c85f8ca4f411196569807099`. The complete I03 case remains unaccepted; the canonical total stays **104/158**. + +I03 diagnostic candidate `85c706cac09d53d0649487b8f1865f18beb53225` +adds bounded GET/DELETE error classification at the existing S3 handler return +boundary. It preserves the ingress request ID and original response. The +classifier uses `S3Error`'s inherent source accessor, including nested wrappers; +its standard error trait does not expose that payload. Two actual development +regressions caught empty source/RPC classifications before this correction. +The final three classification checks and one actual JSON-event check pass, +including nested/cyclic errors, numeric errno, unchanged response properties +and sensitive-message/RPC-metadata exclusion. Two fresh final-diff source +reviews passed. This diagnoses errors without changing HTTP mapping, storage, +RPC/authentication, retries, timeouts or the old live result. + +The adapter explicitly admits this additional full commit after checking all +20 referenced product source files against committed Git objects. Eighteen +original contract files remain byte-identical; the error source hash is +updated and the diagnostic S3 boundary is additionally bound. Independent +extractor checks cover 21 distinct files in total and retain the original +`503/SlowDownRead` and size rules. Original `1c7fc818...` contracts are unchanged; +unknown neighboring SHAs and the historical incompatible delete protocol stay +rejected. The 825 Heal checks pass with nine platform skips. + +Separate hosted package-only run +[39267250075/1](https://github.com/rustfs/rustfs/actions/runs/39267250075) +is building this exact source. The complete published package, opaque build +bytes and a newly built matching inspector still require independent audits +before a new full-fixture live run. No previous tool artifact is treated as a +new-candidate build, and no original `500` or canonical count is rewritten. diff --git a/scripts/validation_heal/network.py b/scripts/validation_heal/network.py index d4e19cf6..d968be3a 100644 --- a/scripts/validation_heal/network.py +++ b/scripts/validation_heal/network.py @@ -88,6 +88,15 @@ } +STORAGE_REFERENCES['85c706cac09d53d0649487b8f1865f18beb53225'] = { + **STORAGE_REFERENCES['1c7fc818dd858247c45b39542abd8291b1e6e31b'], + 'candidateCommit': '85c706cac09d53d0649487b8f1865f18beb53225'} +GET_SOURCE_FILES['85c706cac09d53d0649487b8f1865f18beb53225'] = { + **GET_SOURCE_FILES['1c7fc818dd858247c45b39542abd8291b1e6e31b'], + 'rustfs/src/error.rs': 'c6a8d569b295659cf5a26a44bb3d8ba809c4ca3405d87525624699d161e08970', + 'rustfs/src/storage/ecfs.rs': '68d8ae2428504010adeb7bc51d2309496260898e4c74e5577a0af4f1bfe13c85'} + + def source_reference(candidate_commit): require(isinstance(candidate_commit, str) and candidate_commit in STORAGE_REFERENCES, 'network storage policy is not reviewed for this candidate') diff --git a/scripts/validation_heal/network_fixtures.py b/scripts/validation_heal/network_fixtures.py index 85d4cb26..95f8a83c 100644 --- a/scripts/validation_heal/network_fixtures.py +++ b/scripts/validation_heal/network_fixtures.py @@ -20,6 +20,7 @@ MPU_SOURCE = { '072b57b37559fbe626aeda8abf61617fdcbdd926': 'e60ff0ab886a5578512a123eb1757352ae3e5204d7b69b5aaf14e9e530ec2d31', '1c7fc818dd858247c45b39542abd8291b1e6e31b': '6975b05976e88667065dd4fd2d7649ff61ce0887504e2790bc68fc73c9ba048a', + '85c706cac09d53d0649487b8f1865f18beb53225': '6975b05976e88667065dd4fd2d7649ff61ce0887504e2790bc68fc73c9ba048a', } diff --git a/scripts/validation_heal/network_load.py b/scripts/validation_heal/network_load.py index 65818dc1..7fdaaff5 100644 --- a/scripts/validation_heal/network_load.py +++ b/scripts/validation_heal/network_load.py @@ -41,6 +41,13 @@ 'list_multipart_uploads', 'list_parts'}) +SOURCE_FILES['85c706cac09d53d0649487b8f1865f18beb53225'] = { + **SOURCE_FILES['1c7fc818dd858247c45b39542abd8291b1e6e31b'], + 'rustfs/src/error.rs': 'c6a8d569b295659cf5a26a44bb3d8ba809c4ca3405d87525624699d161e08970', + 'rustfs/src/storage/ecfs.rs': '68d8ae2428504010adeb7bc51d2309496260898e4c74e5577a0af4f1bfe13c85'} +DISK_SOURCE['85c706cac09d53d0649487b8f1865f18beb53225'] = DISK_SOURCE['1c7fc818dd858247c45b39542abd8291b1e6e31b'] + + def source_reference(candidate): require(candidate in SOURCE_FILES, 'DL operations are not reviewed for this exact candidate') return {'candidateCommit': candidate, 'sourceFiles': {**SOURCE_FILES[candidate], diff --git a/scripts/validation_heal/network_load_repair.py b/scripts/validation_heal/network_load_repair.py index f50753c5..d291b6ed 100644 --- a/scripts/validation_heal/network_load_repair.py +++ b/scripts/validation_heal/network_load_repair.py @@ -25,8 +25,11 @@ } +CANDIDATES = frozenset((CANDIDATE, '85c706cac09d53d0649487b8f1865f18beb53225')) + + def source_reference(candidate): - require(candidate == CANDIDATE, 'full I03 DL delete recovery is reviewed only for the exact fixed candidate') + require(candidate in CANDIDATES, 'full I03 DL delete recovery is not reviewed for this exact candidate') return {'candidateCommit': candidate, 'sourceFiles': SOURCE_FILES.copy(), 'deleteWriteQuorum': 'configured set / 2 + 1', 'failedDeleteAttemptsProductRollback': True, 'rollbackFailureIsNotSuccessfulDelete': True, 'healSettlesDeleteBeforeReadingVersion': True, diff --git a/scripts/validation_heal/quorum_boundary.py b/scripts/validation_heal/quorum_boundary.py index 97818d48..e69c8654 100644 --- a/scripts/validation_heal/quorum_boundary.py +++ b/scripts/validation_heal/quorum_boundary.py @@ -70,6 +70,11 @@ } +LOCK_REFERENCES['85c706cac09d53d0649487b8f1865f18beb53225'] = { + **LOCK_REFERENCES['1c7fc818dd858247c45b39542abd8291b1e6e31b'], + 'candidateCommit': '85c706cac09d53d0649487b8f1865f18beb53225'} + + def lock_reference(candidate_commit): require(isinstance(candidate_commit, str) and candidate_commit in LOCK_REFERENCES, "lock policy is not reviewed for this candidate") From 7189c4b6493bc0cddb227e2f2670d38351f1ce7f Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 22:41:27 +0800 Subject: [PATCH 244/343] test(heal): calibrate C02 and C03 DB boundary controls --- docs/validation-migration-map.md | 20 +++ scripts/validation_heal/shard_boundaries.py | 84 +++++++++++ scripts/validation_heal/shards.py | 29 +++- test_validation_heal_shard_boundaries.py | 155 ++++++++++++++++++++ 4 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 scripts/validation_heal/shard_boundaries.py create mode 100644 test_validation_heal_shard_boundaries.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index dc3f34ee..3e3e6f9b 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2230,3 +2230,23 @@ is building this exact source. The complete published package, opaque build bytes and a newly built matching inspector still require independent audits before a new full-fixture live run. No previous tool artifact is treated as a new-candidate build, and no original `500` or canonical count is rewritten. + + +C02/C03 DB boundary completion (2026-10-19): the shard executor retains all +128 original DB controls and every frozen first/middle/last corruption branch, +including C02's completed multipart-data branch. Before any fault, the exact +candidate size contract, actual default-class process environment and decoded +full-set calibration object now derive inline and EC block minus/exact/plus +sizes. Supplemental public PUT controls also cover zero/one, 4-KiB and the +source-bound 5-MiB minimum-part-size boundary. These single-PUT size controls do +not claim completed multipart protocol coverage or a Cartesian C03/DM matrix. +The original count, bytes and MPU controls remain in the full baseline; the +combined fixture must fit the existing 256-object and 512-MiB budgets. Every +supplemental control requires a decoded full-set shape and original physical +checksum/no-part proof, repeated after each repair, plus independent full GET +and Range verification. The runtime integration regression rejects the previous +uncalibrated C02 and C03 callers. Six focused tests and the 831-test Heal suite +pass (nine platform skips). Fresh candidate-bound H1/H2 execution and independent +acceptance remain required; the original C02/C03 failures and canonical +104/158 count are unchanged. I03's inspector and replay remain pinned to the +separately reviewed `536a0a13b7a428a557dbe20d03faf72808da4abd` helper. diff --git a/scripts/validation_heal/shard_boundaries.py b/scripts/validation_heal/shard_boundaries.py new file mode 100644 index 00000000..b809c72b --- /dev/null +++ b/scripts/validation_heal/shard_boundaries.py @@ -0,0 +1,84 @@ +"""Supplement C02/C03 DB controls with actual default-class size boundaries.""" + +try: + from .boundaries import boundary_sizes, seed_boundaries, verify_boundary_ranges, verify_size_contract + from .heal_probe import require +except ImportError: + from boundaries import boundary_sizes, seed_boundaries, verify_boundary_ranges, verify_size_contract + from heal_probe import require + + +MPU_MINIMUM = 5 * 1024**2 +# Exact reviewed GLOBAL_MIN_PART_SIZE and is_min_allowed_part_size source. +MPU_SOURCES = { + "1c7fc818dd858247c45b39542abd8291b1e6e31b": "6975b05976e88667065dd4fd2d7649ff61ce0887504e2790bc68fc73c9ba048a", + "85c706cac09d53d0649487b8f1865f18beb53225": "6975b05976e88667065dd4fd2d7649ff61ce0887504e2790bc68fc73c9ba048a", +} + + +def parameters(manifest, candidate, calibration, processes, recorder, original): + contract = verify_size_contract(manifest, candidate) + require(candidate["source_sha"] in MPU_SOURCES and + contract["sourceFiles"]["crates/ecstore/src/set_disk/mod.rs"] == MPU_SOURCES[candidate["source_sha"]], + "shard DB multipart size controls require an exact reviewed candidate source") + require({row["key"] for row in original if row["key"].startswith("objects/")} == + {f"objects/{index:04d}.bin" for index in range(128)}, + "size controls must retain every original default DB object") + plan = boundary_sizes(contract, calibration, processes) + plan["groups"]["multipart-minimum-minus-exact-plus"] = [MPU_MINIMUM - 1, MPU_MINIMUM, MPU_MINIMUM + 1] + sizes = [size for group in plan["groups"].values() for size in group] + require(max(sizes) <= 64 * 1024**2 and recorder.bytes_written + sum(sizes) <= recorder.probe.binding["maxInventoryBytes"] + and len(original) + len(sizes) <= recorder.probe.binding["maxInventoryObjects"], + "supplemental DB controls exceed the original complete fixture budget") + return {**plan, "originalDefaultDBCount": 128, "supplementalObjectCount": len(sizes), + "supplementalPayloadBytes": sum(sizes), "multipartMinimumPartBytes": MPU_MINIMUM, + "multipartBoundarySourceSha256": MPU_SOURCES[candidate["source_sha"]], + "multipartBoundaryIsObjectSizeControl": True, "completedMultipartProtocolClaimed": False, + "faultBranchMatrixChanged": False} + + +def seed_controls(recorder, bucket, plan): + expected, _ = seed_boundaries(recorder, bucket, plan) + return expected + + +def map_controls(snapshot, bucket, expected, plan): + # Lazy imports avoid the existing versions -> shards dependency at startup. + try: + from .shards import map_target + from .versions import map_inline_target + except ImportError: + from shards import map_target + from versions import map_inline_target + sizes = [size for group in plan["groups"].values() for size in group] + require(len(expected) == len(sizes) == plan["supplementalObjectCount"], + "supplemental DB manifest omitted a required boundary") + targets = [] + for index, (entry, size) in enumerate(zip(expected, sizes)): + require(entry["key"] == f"boundaries/{index:04d}-{size}.bin" and entry["versionId"] == "null" + and entry["deleteMarker"] is False and entry["isLatest"] is True and entry["payload"]["size"] == size, + "supplemental DB ACK identity or size differs from the frozen plan") + shape = "zero-metadata" if size == 0 else "inline" if size <= plan["actualLogicalInlineLimit"] else "external" + target = (map_target(snapshot, bucket, entry["key"], "data", 1) if shape == "external" else + map_inline_target(snapshot, bucket, entry, allow_empty=size == 0)) + geometry = target["logical"]["erasure"] + require(target["logical"]["size"] == size and target["logical"]["usesLegacyChecksum"] is False + and geometry["data_blocks"] == plan["actualDataBlocks"] + and geometry["block_size"] == plan["actualECBlockSize"] + and len(target["cohort"]) == geometry["data_blocks"] + geometry["parity_blocks"], + "supplemental DB control differs from the actual calibrated full-set geometry") + targets.append({"expected": entry, "shape": shape, "target": target}) + return targets + + +def prove_controls(fleet, targets, label): + try: + from .versions import verify_proof + except ImportError: + from versions import verify_proof + return [{"key": row["expected"]["key"], "size": row["expected"]["payload"]["size"], "shape": row["shape"], + "physical": verify_proof(fleet, row["target"], label)} for row in targets] + + +def verify_ranges(probe, fleet, bucket, expected, plan): + return verify_boundary_ranges(probe, fleet, bucket, expected, plan["actualECBlockSize"]) diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index 28ebd1b5..2bae48bb 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -506,7 +506,7 @@ def run_topology(args, topology, nodes, *, fixture="DB"): run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" fleet = Fleet(nodes, directory, run_id, args.tool_directory) candidate = json.loads(args.candidate.read_text()) - fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + tool_manifest = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) @@ -532,6 +532,18 @@ def run_topology(args, topology, nodes, *, fixture="DB"): needs_multipart = "DM" in recipe["contract"]["probes"]["fixture"].split(",") if needs_multipart: expected += seed(recorder, bucket, "DM", 4, SIZES, existing_bucket=True) + boundary_plan, boundary_expected, boundary_targets = None, [], [] + if args.case_id in {"heal.data-shard-bitrot", "heal.parity-shard-bitrot"}: + try: + from . import shard_boundaries + except ImportError: + import shard_boundaries + calibration = fleet.snapshot("DB-size-calibration-live") + target = map_target(calibration, bucket, "objects/0008.bin", "data", 1) + boundary_plan = fleet.record("DB-source-size-plan", shard_boundaries.parameters( + tool_manifest, candidate, target, calibration, recorder, expected)) + boundary_expected = shard_boundaries.seed_controls(recorder, bucket, boundary_plan) + expected += boundary_expected baseline = fleet.record("s3-baseline", probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) fleet.record("fixture-manifest", {"expected": expected, **verify_seed(expected, baseline)}) pending = fleet.record("mpu-baseline", pending_inventory(probe, bucket)) if needs_multipart else None @@ -544,6 +556,10 @@ def run_topology(args, topology, nodes, *, fixture="DB"): require(all(head.get(name) == value for name, value in entry["properties"].items()), "HEAD differs from GET baseline") fleet.stop() original = fleet.snapshot("physical-baseline", quiescent=True) + if boundary_plan is not None: + boundary_targets = shard_boundaries.map_controls(original, bucket, boundary_expected, boundary_plan) + fleet.record("DB-boundary-original-proofs", shard_boundaries.prove_controls( + fleet, boundary_targets, "DB-boundary-original-bitrot")) cross_node = args.case_id == "heal.admin-token-cross-node-status" if cross_node: fleet.record("cross-node-binding", {"coordinator": "node1", "peer": "node2", "stableEndpoint": "node3", @@ -612,6 +628,10 @@ def run_topology(args, topology, nodes, *, fixture="DB"): physical = {"dryRunPhysicalUnchanged": True, "repairedAck": False, "backupUsedOnlyAfterNoMutationProof": True} else: physical = validate_repaired_physical(before, repaired, target) + if boundary_plan is not None: + shard_boundaries.map_controls(repaired, bucket, boundary_expected, boundary_plan) + fleet.record("DB-boundary-repaired-proofs-" + name, shard_boundaries.prove_controls( + fleet, boundary_targets, "DB-boundary-preserved-bitrot-" + name)) bitrot = verify_cohort(fleet, target, ("recovered-bitrot-" if dry_run else "repaired-bitrot-") + name) fleet.start() current = fleet.record("s3-after-" + name, probe.inventory(bucket, unversioned_current_tags=fixture == "DB")) @@ -623,6 +643,8 @@ def run_topology(args, topology, nodes, *, fixture="DB"): compare_inventories(baseline, peer_s3) ranges = verify_ranges(probe, fleet, bucket, key, version_id=target["versionId"], expected_body=payload(f"{bucket}/{key}/0", 5 * 1024**2 + 123) if fixture == "DV" else None) + boundary_ranges = (shard_boundaries.verify_ranges(probe, fleet, bucket, boundary_expected, boundary_plan) + if boundary_plan is not None else None) condition_proof = verify_conditions(probe, fleet, branch, baseline, heads[key], "repaired-condition") if conditions else None if pending is not None: require(fleet.record("mpu-after-" + name, pending_inventory(probe, bucket)) == pending, @@ -631,6 +653,11 @@ def run_topology(args, topology, nodes, *, fixture="DB"): validate_repaired_status(status, target) result["branches"][name] = {"verdict": "PASS", "physical": physical, "bitrot": bitrot, "s3": comparison, "ranges": ranges, "trigger": "manual-normal" if mode == 1 else "manual-deep"} + if boundary_plan is not None: + result["branches"][name]["DBBoundaryControls"] = { + "originalDefaultDBCount": 128, "supplementalObjectCount": len(boundary_expected), + "allBoundaryShapesPhysicallyVerified": True, "ranges": boundary_ranges, + "completedMultipartProtocolClaimed": False} if condition_proof is not None: result["branches"][name]["conditions"] = condition_proof except (Exception, KeyboardInterrupt) as exc: diff --git a/test_validation_heal_shard_boundaries.py b/test_validation_heal_shard_boundaries.py new file mode 100644 index 00000000..77c06099 --- /dev/null +++ b/test_validation_heal_shard_boundaries.py @@ -0,0 +1,155 @@ +"""Default-class C02/C03 boundaries retain the full original DB and strict witnesses.""" + +from contextlib import ExitStack +import copy +import json +import hashlib +from types import SimpleNamespace +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import test_validation_heal as physical_fixtures +import test_validation_heal_network_fixtures as contract_fixtures +from scripts.validation_heal import shard_boundaries as controls, shards + + +class ShardBoundaryControlsTest(unittest.TestCase): + def inputs(self, data=2): + manifest, candidate, target, processes = contract_fixtures.NetworkBoundaryFixturesTest().parameters() + target['logical']['erasure'].update(data_blocks=data, parity_blocks=data, distribution=list(range(1, data * 2 + 1))) + for value in processes.values(): + value['storageClassEnvironment']['RUSTFS_STORAGE_CLASS_STANDARD'] = None + original = [{'key': f'objects/{i:04d}.bin'} for i in range(128)] + recorder = SimpleNamespace(bytes_written=100 * 1024**2, + probe=SimpleNamespace(binding={'maxInventoryBytes': 512 * 1024**2, 'maxInventoryObjects': 256})) + return manifest, candidate, target, processes, recorder, original + + def test_actual_default_class_geometry_preserves128_and_all_required_boundaries(self): + for data, inline in ((2, 262144), (6, 262146)): + with self.subTest(data=data): + plan = controls.parameters(*self.inputs(data)) + self.assertEqual(plan['actualLogicalInlineLimit'], inline) + self.assertEqual(plan['groups']['inline-minus-exact-plus'], [inline - 1, inline, inline + 1]) + self.assertEqual(plan['groups']['stripe-minus-exact-plus'], [1048575, 1048576, 1048577]) + self.assertEqual(plan['groups']['multipart-minimum-minus-exact-plus'], [5242879, 5242880, 5242881]) + self.assertEqual(plan['originalDefaultDBCount'], 128) + self.assertFalse(plan['faultBranchMatrixChanged']) + self.assertFalse(plan['completedMultipartProtocolClaimed']) + + def test_source_configuration_original_controls_and_resource_refusals_precede_writes(self): + for kind in ('unknown-source', 'source-hash', 'class', 'inline-override', 'missing-original', 'bytes', 'count'): + values = self.inputs() + manifest, candidate, target, processes, recorder, original = values + if kind == 'unknown-source': + candidate['source_sha'] = 'f' * 40 + manifest['sizeContract']['candidateCommit'] = candidate['source_sha'] + elif kind == 'source-hash': manifest['sizeContract']['sourceFiles']['crates/ecstore/src/set_disk/mod.rs'] = 'f' * 64 + elif kind == 'class': processes['node0']['storageClassEnvironment']['RUSTFS_STORAGE_CLASS_STANDARD'] = 'EC:8' + elif kind == 'inline-override': processes['node0']['storageClassEnvironment']['RUSTFS_STORAGE_CLASS_INLINE_BLOCK'] = '1MiB' + elif kind == 'missing-original': original.pop() + elif kind == 'bytes': recorder.probe.binding['maxInventoryBytes'] = recorder.bytes_written + else: recorder.probe.binding['maxInventoryObjects'] = 128 + with self.subTest(kind=kind), self.assertRaises(ValueError): controls.parameters(*values) + + def test_seed_is_public_put_with_independent_payloads_and_disjoint_keys(self): + plan = controls.parameters(*self.inputs()) + calls = [] + def call(operation, params, body): + calls.append((operation, params, body)) + return {'ETag': 'ack'} + expected = controls.seed_controls(SimpleNamespace(call=call), 'owned', plan) + self.assertEqual(len(expected), plan['supplementalObjectCount']) + self.assertEqual(sum(len(body) for _, _, body in calls), plan['supplementalPayloadBytes']) + for entry, (operation, params, body) in zip(expected, calls): + self.assertEqual(operation, 'put_object') + self.assertEqual(params['Key'], entry['key']) + self.assertTrue(entry['key'].startswith('boundaries/')) + self.assertEqual(entry['payload'], {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}) + + def scene(self): + original = physical_fixtures.HealShardOracleTest().snapshot() + bucket = 'rv-heal-0123456789abcdef' + size = 1048577 + key = f'boundaries/0000-{size}.bin' + for row in original['node1']['files']: + row['path'] = row['path'].replace('objects/0008.bin', key) + if 'metadata' in row: + row['metadata']['key'] = key + version = row['metadata']['versions'][0] + version['size'] = size + version['parts'][0].update(size=size, actualSize=size) + plan = {'groups': {'stripe-plus': [size]}, 'supplementalObjectCount': 1, + 'actualLogicalInlineLimit': 262144, 'actualDataBlocks': 2, 'actualECBlockSize': 1048576} + expected = [{'key': key, 'versionId': 'null', 'isLatest': True, 'deleteMarker': False, 'payload': {'size': size}}] + return original, bucket, expected, plan + + def test_real_decoded_full_set_mapping_rejects_missing_parts_changed_shapes_and_ack(self): + snapshot, bucket, expected, plan = self.scene() + targets = controls.map_controls(snapshot, bucket, expected, plan) + self.assertEqual((targets[0]['shape'], len(targets[0]['target']['cohort'])), ('external', 4)) + for kind in ('missing-part', 'size', 'geometry', 'key', 'version', 'duplicate'): + value, ack, parameters = copy.deepcopy(snapshot), copy.deepcopy(expected), copy.deepcopy(plan) + if kind == 'missing-part': value['node1']['files'] = [row for row in value['node1']['files'] if not row['path'].endswith('part.1')] + elif kind == 'size': ack[0]['payload']['size'] -= 1 + elif kind == 'geometry': parameters['actualDataBlocks'] = 6 + elif kind == 'key': ack[0]['key'] += '-other' + elif kind == 'version': ack[0]['versionId'] = 'other' + else: ack += copy.deepcopy(ack) + with self.subTest(kind=kind), self.assertRaises(ValueError): controls.map_controls(value, bucket, ack, parameters) + + def test_original_byte_proof_refuses_different_surviving_shard(self): + snapshot, bucket, expected, plan = self.scene() + targets = controls.map_controls(snapshot, bucket, expected, plan) + fleet = SimpleNamespace(record=lambda label, value: value) + def remote(node, action, **params): + shard = next(row for row in targets[0]['target']['cohort'] if row['disk'] == params['disk']) + return {'sha256': shard['sha256'], 'physicalSize': shard['size']} + fleet.remote = remote + self.assertTrue(controls.prove_controls(fleet, targets, 'proof')[0]['physical']['sameRoleOriginalShardBytesRestored']) + fleet.remote = lambda *args, **kwargs: {'sha256': 'changed', 'physicalSize': 82} + with self.assertRaises(ValueError): controls.prove_controls(fleet, targets, 'proof') + + + def test_real_c02_and_c03_callers_require_calibrated_controls_before_any_fault(self): + class ReachedControls(BaseException): + pass + for case in ("heal.data-shard-bitrot", "heal.parity-shard-bitrot"): + with self.subTest(case=case), tempfile.TemporaryDirectory() as directory, ExitStack() as stack: + root = Path(directory) + candidate = root / "candidate.json" + candidate.write_text(json.dumps({"source_sha": "1c7fc818dd858247c45b39542abd8291b1e6e31b", "package_sha256": "a" * 64})) + rc = root / "rc" + rc.write_bytes(b"client") + args = SimpleNamespace(evidence=root, candidate=candidate, rc_bin=rc, tool_directory=root, + testing_commit="b" * 40, binary_sha="c" * 64, case_id=case) + records = {} + def record(label, value): + records[label] = value + return value + fleet = SimpleNamespace(record=record, deploy=lambda *args: None, + snapshot=lambda label: {"calibration": label}, cleanup=lambda: True) + stack.enter_context(patch.dict('os.environ', {'GITHUB_RUN_ID': '123', 'GITHUB_RUN_ATTEMPT': '1'})) + stack.enter_context(patch.object(shards, 'Fleet', return_value=fleet)) + stack.enter_context(patch.object(shards, 'Probe', return_value=None)) + stack.enter_context(patch.object(shards, 'Recorder', return_value="recorder")) + stack.enter_context(patch.object(shards, 'verify_inspector', return_value="exact-manifest")) + stack.enter_context(patch.object(shards, 'validate_topology', return_value={})) + def seed(recorder, bucket, fixture, count, sizes, **kwargs): + return [{'key': f'objects/{i:04d}.bin'} for i in range(128)] if fixture == 'DB' else [] + stack.enter_context(patch.object(shards, 'seed', side_effect=seed)) + stack.enter_context(patch.object(shards, 'map_target', return_value="actual-target")) + plan = stack.enter_context(patch.object(controls, 'parameters', return_value="source-size-plan")) + seeded = stack.enter_context(patch.object(controls, 'seed_controls', side_effect=ReachedControls)) + with self.assertRaises(ReachedControls): shards.run_topology(args, 'H1', ['vm000']) + self.assertEqual(plan.call_args.args[:3], ("exact-manifest", json.loads(candidate.read_text()), "actual-target")) + self.assertEqual(plan.call_args.args[3], {"calibration": "DB-size-calibration-live"}) + self.assertEqual(plan.call_args.args[4], "recorder") + self.assertEqual(len(plan.call_args.args[5]), 128) + self.assertEqual(seeded.call_args.args[2], "source-size-plan") + self.assertEqual(records["DB-source-size-plan"], "source-size-plan") + + +if __name__ == '__main__': + unittest.main() From 111b2cec613ee6cd7bc2661044ae5a42539718be Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 19 Oct 2026 22:50:57 +0800 Subject: [PATCH 245/343] fix(heal): include all I04 buckets in physical evidence --- docs/validation-migration-map.md | 17 +++++ scripts/validation_heal/queue_pressure.py | 1 + test_validation_heal_queue_bucket_scope.py | 83 ++++++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 test_validation_heal_queue_bucket_scope.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 3e3e6f9b..bec87573 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2250,3 +2250,20 @@ pass (nine platform skips). Fresh candidate-bound H1/H2 execution and independen acceptance remain required; the original C02/C03 failures and canonical 104/158 count are unchanged. I03's inspector and replay remain pinned to the separately reviewed `536a0a13b7a428a557dbe20d03faf72808da4abd` helper. + + +I04 physical-scope correction (2026-10-19): original run `38119569421/1` +recorded successful 769/129/129/129-object queue manifests on both H1 and H2, +but its physical baselines contain zero business files on every node. The exact +original helper calls `Fleet.snapshot` without a bucket list, selecting only the +unseeded primary owner bucket instead of its four queue buckets and durable +control bucket. All six baseline, fault, attribution, pre-GET, post-GET and +failure snapshots now explicitly include the primary prefix and those five +owned buckets through the existing frozen fleet binding and bounded scope validator. No queue admission, +resource limit, MRF retention/retirement, fairness, physical comparison, candidate +source or cleanup oracle changes. Runtime regression tests execute the actual +H1/H2 caller and `Fleet.snapshot` adapter; the previous caller fails. All 13 +focused checks and the 832-test Heal suite pass (nine platform skips). The +original FAIL remains unchanged; later pressure behavior and complete fresh +candidate-bound H1/H2 execution remain unproven. Canonical acceptance stays +104/158, and I03 still uses its separate frozen helper. diff --git a/scripts/validation_heal/queue_pressure.py b/scripts/validation_heal/queue_pressure.py index f89dcb52..1b205d4a 100644 --- a/scripts/validation_heal/queue_pressure.py +++ b/scripts/validation_heal/queue_pressure.py @@ -233,6 +233,7 @@ def run_topology(args, topology, nodes, **unused): fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) fleet.record("MRF-contract", verify_reference(candidate)) prefix = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] + fleet.snapshot_buckets = [prefix, *[f"{prefix}-queue-{index}" for index in range(len(COUNTS))], prefix + "-durable"] access, secret = secrets.token_hex(16), secrets.token_hex(24) os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) binding = {"schemaVersion": 1, "runId": run_id, "jumpHostOnly": True, "exclusiveHealService": True, diff --git a/test_validation_heal_queue_bucket_scope.py b/test_validation_heal_queue_bucket_scope.py new file mode 100644 index 00000000..57aad72f --- /dev/null +++ b/test_validation_heal_queue_bucket_scope.py @@ -0,0 +1,83 @@ +"""I04 physical oracles must include every run-owned pressure and durable bucket.""" + +import ast +from contextlib import ExitStack +import json +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from scripts.validation_heal import queue_pressure as queue +from scripts.validation_heal.fleet_node import snapshot_buckets +from scripts.validation_heal.healthy import Fleet + + +class QueueBucketScopeTest(unittest.TestCase): + def test_real_executor_passes_all_owned_buckets_through_actual_fleet_snapshot(self): + class ReachedPhysicalMapping(BaseException): + pass + for topology, nodes in [('H1', ['vm000']), ('H2', ['vm000', 'vm001', 'vm002', 'vm003'])]: + with self.subTest(topology=topology), tempfile.TemporaryDirectory() as directory, ExitStack() as stack: + root = Path(directory) + candidate = root / 'candidate.json' + candidate.write_text(json.dumps({'source_sha': queue.REFERENCE['candidateCommit'], 'package_sha256': 'a' * 64})) + rc = root / 'rc' + rc.write_bytes(b'client') + args = SimpleNamespace(evidence=root, candidate=candidate, rc_bin=rc, tool_directory=root, + testing_commit='b' * 40, binary_sha='c' * 64) + records, mapped, owner = {}, [], {} + fleet = Fleet(nodes, root, 'not-used') + fleet.record = lambda label, value: records.setdefault(label, value) + def deploy(binary, prefix, *credentials): + owner['bucket'] = prefix + fleet.deploy = deploy + fleet.stop = lambda: None + fleet.cleanup = lambda: True + requests = [] + def execute(command, *, input, **kwargs): + request = ast.literal_eval(ast.parse(input.rsplit('\nmain(', 1)[1].rsplit(')', 1)[0], mode='eval').body) + self.assertEqual(request['action'], 'snapshot') + buckets = snapshot_buckets(owner, request) + requests.append(request) + return SimpleNamespace(returncode=0, stderr='', stdout=json.dumps({'formats': ['format'], + 'files': [{'path': bucket + '/objects/0000.bin/xl.meta'} for bucket in buckets]})) + stack.enter_context(patch('scripts.validation_heal.healthy.subprocess.run', side_effect=execute)) + probe = SimpleNamespace(inventory=lambda *args, **kwargs: {}) + stack.enter_context(patch.dict('os.environ', {'GITHUB_RUN_ID': '123', 'GITHUB_RUN_ATTEMPT': '1'})) + stack.enter_context(patch.object(queue, 'Fleet', return_value=fleet)) + stack.enter_context(patch.object(queue, 'Probe', return_value=probe)) + stack.enter_context(patch.object(queue, 'Recorder', return_value=None)) + stack.enter_context(patch.object(queue, 'seed', return_value=[])) + stack.enter_context(patch.object(queue, 'verify_seed', return_value={})) + stack.enter_context(patch.object(queue, 'verify_reference', return_value={})) + stack.enter_context(patch.object(queue, 'verify_inspector', return_value={})) + stack.enter_context(patch.object(queue, 'validate_topology', return_value={})) + def map_cohorts(snapshot, bucket, expected, **kwargs): + mapped.append(snapshot) + raise ReachedPhysicalMapping + stack.enter_context(patch.object(queue, 'map_cohorts', side_effect=map_cohorts)) + with self.assertRaises(ReachedPhysicalMapping): queue.run_topology(args, topology, nodes) + expected = {owner['bucket'], *[owner['bucket'] + f'-queue-{i}' for i in range(4)], owner['bucket'] + '-durable'} + baseline = records['physical-baseline'] + self.assertEqual(set(baseline), set(nodes)) + for observed in baseline.values(): + self.assertEqual({row['path'].split('/')[0] for row in observed['files']}, expected) + for observed in mapped[0].values(): + self.assertEqual([row['path'].split('/')[0] for row in observed['files']], [owner['bucket'] + '-queue-0']) + + # Every later phase uses the same actual remote adapter and + # cannot silently fall back to the empty primary bucket. + for label in ['physical-fault', 'physical-before-pressure', 'physical-product-result-before-GET', + 'physical-after-S3', 'physical-failure']: + fleet.snapshot(label, quiescent=label != 'physical-before-pressure') + self.assertEqual(len(requests), 6 * len(nodes)) + for request in requests: + self.assertEqual(set(request['buckets']), expected) + with self.assertRaisesRegex(ValueError, 'frozen fleet binding'): + fleet.snapshot('changed-scope', buckets=[owner['bucket']]) + + +if __name__ == '__main__': + unittest.main() From 2b20cde723f727c395f2dcca888af78fd1dfbec9 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 00:48:22 +0800 Subject: [PATCH 246/343] test(heal): bind RPC refusal candidate to exact I03 sources --- docs/validation-heal-cases.md | 2 ++ scripts/validation_heal/network.py | 8 ++++++++ scripts/validation_heal/network_fixtures.py | 1 + scripts/validation_heal/network_load.py | 6 ++++++ scripts/validation_heal/network_load_repair.py | 3 ++- scripts/validation_heal/quorum_boundary.py | 4 ++++ test_validation_heal_quorum_candidate.py | 7 +++++-- 7 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index 6d895fcc..daf2a807 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1722,6 +1722,8 @@ Topologies: H2. Branches: minority-partition, read-boundary, write-boundary, loc 候选支持与 B06 相同，但 storageclass 与锁策略分别按精确候选绑定。`1c7fc81` 的锁客户端范围、quorum 阈值及 EC:8 配置已复核；其 RPC 控制通道工厂有变化，必须重新取得真实双向网络探针、admission、partial coverage 和 rejoin 证据，不能由源码等价或 B07 通过推定本例通过。B08 的候选范围保持独立。 +The adapter additionally binds exact source `62ff96774fa4139d45f14d70b70519aae8070ac0` and its typed RPC `Unavailable` response mapping to `503/ServiceUnavailable`. Every referenced lock, EC, multipart and delete-recovery source file remains independently checked against that committed source; neighboring or abbreviated revisions are rejected. This source requires its matching immutable package and source-bound inspector before live execution. All five original branches, complete DB/DV/DL fixtures, refused-mutation accounting, physical-before-GET proofs, task responsibility and cleanup requirements remain mandatory; the source admission alone does not establish live acceptance. + 1. H2 先冻结实际节点/slot/对象R/W和锁quorum，NETWORK规则只匹配本例服务端口并带run ID、超时恢复。 2. 构造少数派及边界分区，双端探针证实隔离；SSH、跳板机和其他服务端口保持连通。 3. 双方执行带nonce的PUT/GET/heal，少数派不能承诺不安全提交；BACKGROUND必须暴露missing peer/partial coverage。 diff --git a/scripts/validation_heal/network.py b/scripts/validation_heal/network.py index d968be3a..73b91fa1 100644 --- a/scripts/validation_heal/network.py +++ b/scripts/validation_heal/network.py @@ -96,6 +96,14 @@ 'rustfs/src/error.rs': 'c6a8d569b295659cf5a26a44bb3d8ba809c4ca3405d87525624699d161e08970', 'rustfs/src/storage/ecfs.rs': '68d8ae2428504010adeb7bc51d2309496260898e4c74e5577a0af4f1bfe13c85'} +STORAGE_REFERENCES['62ff96774fa4139d45f14d70b70519aae8070ac0'] = { + **STORAGE_REFERENCES['85c706cac09d53d0649487b8f1865f18beb53225'], + 'candidateCommit': '62ff96774fa4139d45f14d70b70519aae8070ac0'} +GET_SOURCE_FILES['62ff96774fa4139d45f14d70b70519aae8070ac0'] = { + **GET_SOURCE_FILES['85c706cac09d53d0649487b8f1865f18beb53225'], + 'rustfs/src/error.rs': '267cf292f0af47bca782ebb4de119a5bbe119af82111dac0bbf60f1311ef8383', + 'rustfs/src/storage/ecfs.rs': 'e0c4ce6569f75771c73de043f8416645bcd1de86a1192a79f45d7418953086f9'} + def source_reference(candidate_commit): require(isinstance(candidate_commit, str) and candidate_commit in STORAGE_REFERENCES, diff --git a/scripts/validation_heal/network_fixtures.py b/scripts/validation_heal/network_fixtures.py index 95f8a83c..f1e9fa5b 100644 --- a/scripts/validation_heal/network_fixtures.py +++ b/scripts/validation_heal/network_fixtures.py @@ -21,6 +21,7 @@ '072b57b37559fbe626aeda8abf61617fdcbdd926': 'e60ff0ab886a5578512a123eb1757352ae3e5204d7b69b5aaf14e9e530ec2d31', '1c7fc818dd858247c45b39542abd8291b1e6e31b': '6975b05976e88667065dd4fd2d7649ff61ce0887504e2790bc68fc73c9ba048a', '85c706cac09d53d0649487b8f1865f18beb53225': '6975b05976e88667065dd4fd2d7649ff61ce0887504e2790bc68fc73c9ba048a', + '62ff96774fa4139d45f14d70b70519aae8070ac0': '6975b05976e88667065dd4fd2d7649ff61ce0887504e2790bc68fc73c9ba048a', } diff --git a/scripts/validation_heal/network_load.py b/scripts/validation_heal/network_load.py index 7fdaaff5..5ca3442c 100644 --- a/scripts/validation_heal/network_load.py +++ b/scripts/validation_heal/network_load.py @@ -47,6 +47,12 @@ 'rustfs/src/storage/ecfs.rs': '68d8ae2428504010adeb7bc51d2309496260898e4c74e5577a0af4f1bfe13c85'} DISK_SOURCE['85c706cac09d53d0649487b8f1865f18beb53225'] = DISK_SOURCE['1c7fc818dd858247c45b39542abd8291b1e6e31b'] +SOURCE_FILES['62ff96774fa4139d45f14d70b70519aae8070ac0'] = { + **SOURCE_FILES['85c706cac09d53d0649487b8f1865f18beb53225'], + 'rustfs/src/error.rs': '267cf292f0af47bca782ebb4de119a5bbe119af82111dac0bbf60f1311ef8383', + 'rustfs/src/storage/ecfs.rs': 'e0c4ce6569f75771c73de043f8416645bcd1de86a1192a79f45d7418953086f9'} +DISK_SOURCE['62ff96774fa4139d45f14d70b70519aae8070ac0'] = DISK_SOURCE['85c706cac09d53d0649487b8f1865f18beb53225'] + def source_reference(candidate): require(candidate in SOURCE_FILES, 'DL operations are not reviewed for this exact candidate') diff --git a/scripts/validation_heal/network_load_repair.py b/scripts/validation_heal/network_load_repair.py index d291b6ed..2e0ef258 100644 --- a/scripts/validation_heal/network_load_repair.py +++ b/scripts/validation_heal/network_load_repair.py @@ -25,7 +25,8 @@ } -CANDIDATES = frozenset((CANDIDATE, '85c706cac09d53d0649487b8f1865f18beb53225')) +CANDIDATES = frozenset((CANDIDATE, '85c706cac09d53d0649487b8f1865f18beb53225', + '62ff96774fa4139d45f14d70b70519aae8070ac0')) def source_reference(candidate): diff --git a/scripts/validation_heal/quorum_boundary.py b/scripts/validation_heal/quorum_boundary.py index e69c8654..b3e55d17 100644 --- a/scripts/validation_heal/quorum_boundary.py +++ b/scripts/validation_heal/quorum_boundary.py @@ -74,6 +74,10 @@ **LOCK_REFERENCES['1c7fc818dd858247c45b39542abd8291b1e6e31b'], 'candidateCommit': '85c706cac09d53d0649487b8f1865f18beb53225'} +LOCK_REFERENCES['62ff96774fa4139d45f14d70b70519aae8070ac0'] = { + **LOCK_REFERENCES['85c706cac09d53d0649487b8f1865f18beb53225'], + 'candidateCommit': '62ff96774fa4139d45f14d70b70519aae8070ac0'} + def lock_reference(candidate_commit): require(isinstance(candidate_commit, str) and candidate_commit in LOCK_REFERENCES, diff --git a/test_validation_heal_quorum_candidate.py b/test_validation_heal_quorum_candidate.py index 7f828f2e..84bb7e4e 100644 --- a/test_validation_heal_quorum_candidate.py +++ b/test_validation_heal_quorum_candidate.py @@ -14,6 +14,8 @@ ORIGINAL = '072b57b37559fbe626aeda8abf61617fdcbdd926' FIXED = '1c7fc818dd858247c45b39542abd8291b1e6e31b' +DIAGNOSTIC = '85c706cac09d53d0649487b8f1865f18beb53225' +RPC_FIXED = '62ff96774fa4139d45f14d70b70519aae8070ac0' class RecordedSourceContract(Exception): @@ -55,7 +57,7 @@ def record(label, value): def test_actual_executors_record_the_selected_candidate_before_deploy(self): for module in (quorum_boundary, network): - for source in (ORIGINAL, FIXED): + for source in (ORIGINAL, FIXED, DIAGNOSTIC, RPC_FIXED): with self.subTest(executor=module.CASE_ID, source=source): with self.assertRaises(RecordedSourceContract) as completed: self.execute_until_contract(module, source) @@ -69,7 +71,8 @@ def test_actual_executors_record_the_selected_candidate_before_deploy(self): def test_unknown_and_malformed_sources_never_reach_inspector_or_deploy(self): for module in (quorum_boundary, network): - for source in ('0' * 40, FIXED[:7], FIXED.upper(), None, [], True): + for source in ('0' * 40, FIXED[:7], FIXED.upper(), RPC_FIXED[:7], RPC_FIXED.upper(), + RPC_FIXED[:-1] + '1', None, [], True): with self.subTest(executor=module.CASE_ID, source=source): with self.assertRaises(ValueError): self.execute_until_contract(module, source, allow_inspector=False) From fc30b4af32c2890eea481b81acf8af5f1bafb7b5 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 02:56:44 +0800 Subject: [PATCH 247/343] test(validation): inspect fleet services and port ownership read-only --- docs/validation-heal-environment-gates.md | 6 ++ scripts/validation_heal/environment.py | 3 + scripts/validation_heal/fleet_node.py | 52 ++++++++++++++++ test_validation_heal_environment.py | 72 +++++++++++++++++++++++ 4 files changed, 133 insertions(+) diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index 5f0b26dc..84b79753 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -1,5 +1,11 @@ # Heal 环境门控与原生阻断 +## Read-only fleet inspection + +The Functional Chain `environment.inspect` entry records the current four Heal hosts' mount capabilities, fixed `rustfs.service` status, loaded validation service names and listening PIDs on port 9000. Service and listener observations must agree across two samples. Process arguments, service descriptions and credential files are excluded. Inspection never stops services or changes retained scenes. + +These observations do not certify a clean environment. Independently audit retained validation scenes and the complete shared-workflow census before dispatching another case. A completed workflow's success or failure alone does not prove recovery. The regular case installer remains responsible for stopping the standard service, and its existing namespace/port reservation guards remain authoritative. + 第一批 13 个用例保留全部 recipe、分支和 source assertions，并新增 `environment_cases.py` 的控制流程与断言，接入 validation 调度。它们默认关闭，只产生 `BLOCKED` 结果与未满足原因；默认路径不会连接或修改 fleet。真实执行沿用 auto-testing 的 S3/Admin collector、ACK ledger、候选包绑定、物理故障 oracle 和恢复模式，专用资源操作由部署环境的适配器提供。当前没有把适配器协议或未运行代码计为 canonical PASS。 设置 `AUTO_TESTING_ENABLE_ENV_GATED_HEAL=1` 并通过 `VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES` 指向 JSON 能力清单后，门控会检查全部能力。缺项仍返回 `BLOCKED`。能力齐全后还要用 `VALIDATION_HEAL_ENVIRONMENT_RUNTIME` 指向 runtime JSON；只有候选/checkout/client 摘要、全部拓扑、当前 Heal SSH 节点和适配器摘要校验通过后才会运行分支。未绑定适配器仍为 `BLOCKED`；执行结果需要 source assertion 审核，当前实现不会自动生成 canonical PASS。 diff --git a/scripts/validation_heal/environment.py b/scripts/validation_heal/environment.py index 93969f29..bb9081ab 100644 --- a/scripts/validation_heal/environment.py +++ b/scripts/validation_heal/environment.py @@ -58,6 +58,9 @@ def summarize(observed): 'H3': {'mountsAvailable': available(nodes[:3], 4), 'requiredEndpoints': 12}, 'H4': {'mountsAvailable': available(nodes[:3], 8), 'requiredEndpoints': 24}, 'H6': {'platformRestartAuthorityVerified': False}}, + 'runtimeObservationsComplete': all(value.get('runtime', {}).get('stableObservation') is True + for value in observed.values()), + 'cleanStateCertified': False, 'globalWorkflowIdleCertified': False, 'actualErasureLayoutVerified': False, 'faultBackendVerified': False, 'compatibleReleasePairVerified': False, 'canonicalPassClaimed': False} diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 41472c3e..c406338d 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -312,6 +312,57 @@ def mounts(): return observed +def environment_service_state(): + """Observe fixed service/port names; never stop or certify a clean scene.""" + fields = {'LoadState', 'ActiveState', 'SubState', 'MainPID'} + + def service(): + raw = command('systemctl', 'show', 'rustfs.service', '--no-pager', + '--property=LoadState,ActiveState,SubState,MainPID', check=False) + values = dict(line.split('=', 1) for line in raw.stdout.splitlines() if '=' in line) + require(set(values) == fields and re.fullmatch(r'[0-9]+', values['MainPID']), + 'environment service observation is incomplete') + require(raw.returncode == 0 or values['LoadState'] == 'not-found', + 'environment service observation failed') + return {**values, 'MainPID': int(values['MainPID'])} + + def validation_units(): + raw = command('systemctl', 'list-units', '--all', '--type=service', '--plain', + '--no-legend', '--no-pager', 'rustfs-validation-heal-*.service') + rows = [] + for line in raw.stdout.splitlines(): + parts = line.split(None, 4) + require(len(parts) >= 4 and parts[0].startswith('rustfs-validation-heal-') + and parts[0].endswith('.service'), 'ambiguous validation service inventory') + run_id = parts[0][len('rustfs-validation-heal-'):-len('.service')] + require(RUN_PATTERN.fullmatch(run_id), 'unrecognized validation service requires audit') + rows.append(dict(zip(('unit', 'load', 'active', 'sub'), parts[:4]))) + require(len(rows) <= 256 and len({row['unit'] for row in rows}) == len(rows), + 'validation service inventory is unbounded or duplicated') + return sorted(rows, key=lambda row: row['unit']) + + def listeners(): + raw = command('ss', '-H', '-ltnp', '( sport = :9000 )') + rows = [] + for line in raw.stdout.splitlines(): + parts = line.split(None, 5) + require(len(parts) >= 5 and parts[0] == 'LISTEN' and parts[3].endswith(':9000'), + 'ambiguous port 9000 listener') + rows.append({'localAddress': parts[3], + 'pids': sorted({int(pid) for pid in re.findall(r'\bpid=([1-9][0-9]*)\b', line)})}) + require(len(rows) <= 256, 'port listener inventory is unbounded') + return sorted(rows, key=lambda row: (row['localAddress'], row['pids'])) + + before = (service(), validation_units(), listeners()) + after = (service(), validation_units(), listeners()) + require(before == after, 'environment runtime changed during read-only observation') + standard, units, port = after + return {'standardService': standard, 'loadedValidationUnits': units, 'port9000Listeners': port, + 'stableObservation': True, 'inspectionOnly': True, 'mutated': False, + 'cleanStateCertified': False, 'globalWorkflowIdleCertified': False, + 'retainedSceneRecoveryCertified': False} + + def environment_capabilities(request): """Read existing mounts and kernel interfaces without reserving a run.""" paths(request['runId']) @@ -344,6 +395,7 @@ def environment_capabilities(request): observed.append(row) binary = Path('/usr/bin/rustfs') return {'hostname': socket.gethostname(), 'mounts': observed, 'inspectionOnly': True, 'mutated': False, + 'runtime': environment_service_state(), 'kernel': {'pidfd': hasattr(os, 'pidfd_open'), 'cgroupV2': Path('/sys/fs/cgroup/cgroup.controllers').is_file(), 'fuseDevice': Path('/dev/fuse').exists() and stat.S_ISCHR(Path('/dev/fuse').stat().st_mode)}, 'commands': {name: shutil.which(name) is not None for name in ('docker', 'fusermount3', 'nft', 'az')}, diff --git a/test_validation_heal_environment.py b/test_validation_heal_environment.py index e83f88a2..1d9e73a1 100644 --- a/test_validation_heal_environment.py +++ b/test_validation_heal_environment.py @@ -6,6 +6,7 @@ from unittest.mock import patch from scripts.validation_heal import environment +from scripts.validation_heal import fleet_node from scripts.validation_heal.healthy import Fleet @@ -70,5 +71,76 @@ def invoke(argv, **params): fleet = Fleet(['n0'], Path(name), 'gh-1234-1-h2') self.assertFalse(fleet.remote('n0', 'environment-capabilities')['mutated']) + def runtime_command(self, *, units='', listeners='', state=None): + state = state or 'LoadState=loaded\nActiveState=active\nSubState=running\nMainPID=42\n' + def invoke(*args, **params): + if args[:2] == ('systemctl', 'show'): output = state + elif args[:2] == ('systemctl', 'list-units'): output = units + elif args[0] == 'ss': output = listeners + else: self.fail(f'Unexpected runtime command: {args}') + return types.SimpleNamespace(returncode=0, stdout=output, stderr='') + return invoke + + def test_runtime_records_service_and_port_without_exposing_process_arguments(self): + command = self.runtime_command( + units='rustfs-validation-heal-gh-123-1-h2.service loaded inactive dead private fixture\n', + listeners='LISTEN 0 4096 [::]:9000 [::]:* users:(("private-label",pid=42,fd=7))\n') + with patch.object(fleet_node, 'command', side_effect=command) as calls: + result = fleet_node.environment_service_state() + self.assertEqual(result['standardService']['MainPID'], 42) + self.assertEqual(result['port9000Listeners'], [{'localAddress': '[::]:9000', 'pids': [42]}]) + self.assertEqual(result['loadedValidationUnits'][0]['active'], 'inactive') + self.assertNotIn('private-label', str(result)) + self.assertNotIn('private fixture', str(result)) + self.assertEqual(len(calls.call_args_list), 6) + for field in ('mutated', 'cleanStateCertified', 'globalWorkflowIdleCertified', 'retainedSceneRecoveryCertified'): + self.assertFalse(result[field]) + + def test_idle_runtime_and_unattributed_listener_never_certify_clean_state(self): + for listener in ('', 'LISTEN 0 4096 0.0.0.0:9000 0.0.0.0:*\n'): + with self.subTest(listener=listener), patch.object(fleet_node, 'command', side_effect=self.runtime_command( + state='LoadState=not-found\nActiveState=inactive\nSubState=dead\nMainPID=0\n', listeners=listener)): + result = fleet_node.environment_service_state() + self.assertFalse(result['cleanStateCertified']) + self.assertEqual(result['standardService']['MainPID'], 0) + if listener: self.assertEqual(result['port9000Listeners'][0]['pids'], []) + + def test_runtime_refuses_partial_failed_ambiguous_or_unrecognized_observations(self): + scenarios = [ + self.runtime_command(state='LoadState=loaded\nMainPID=42\n'), + self.runtime_command(units='rustfs-validation-heal-gh-123-1-h2.service loaded active running\n' * 2), + self.runtime_command(units='rustfs-validation-heal-unrecognized.service loaded active running\n'), + self.runtime_command(listeners='LISTEN 0 4096 0.0.0.0:9001 0.0.0.0:*\n'), + ] + for command in scenarios: + with patch.object(fleet_node, 'command', side_effect=command), self.assertRaises(ValueError): + fleet_node.environment_service_state() + failed = types.SimpleNamespace(returncode=1, stdout='LoadState=loaded\nActiveState=active\nSubState=running\nMainPID=42\n', stderr='') + with patch.object(fleet_node, 'command', return_value=failed), self.assertRaises(ValueError): + fleet_node.environment_service_state() + + def test_runtime_refuses_service_or_listener_changes_between_observations(self): + for changing in ('service', 'listener', 'validation-unit'): + command = self.runtime_command() + counts = {} + def invoke(*args, **params): + result = command(*args, **params) + counts[args] = counts.get(args, 0) + 1 + if counts[args] == 2: + if changing == 'service' and args[:2] == ('systemctl', 'show'): + result.stdout = result.stdout.replace('MainPID=42', 'MainPID=43') + elif changing == 'listener' and args[0] == 'ss': + result.stdout = 'LISTEN 0 4096 0.0.0.0:9000 0.0.0.0:*\n' + elif changing == 'validation-unit' and args[:2] == ('systemctl', 'list-units'): + result.stdout = 'rustfs-validation-heal-gh-123-1-h2.service loaded active running\n' + return result + with self.subTest(changing=changing), patch.object(fleet_node, 'command', side_effect=invoke), self.assertRaises(ValueError): + fleet_node.environment_service_state() + + def test_legacy_mount_only_inventory_does_not_imply_runtime_or_clean_proof(self): + result = environment.summarize(self.fixture()) + self.assertFalse(result['runtimeObservationsComplete']) + self.assertFalse(result['cleanStateCertified']) + if __name__ == '__main__': unittest.main() From 50ba02ded5e0e072792074158639c49c86106ad2 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 04:28:57 +0800 Subject: [PATCH 248/343] test(validation): collect and admit exact-source race native subsets --- .../workflows/validation-catalog-tests.yml | 3 + docs/validation-heal-operations.md | 8 + scripts/validation_heal/race_native.py | 298 ++++++++++++++++++ .../race_native_contracts.json | 91 ++++++ .../validation_heal/race_native_evidence.py | 109 +++++++ test_validation_heal_race_native.py | 170 ++++++++++ 6 files changed, 679 insertions(+) create mode 100644 scripts/validation_heal/race_native.py create mode 100644 scripts/validation_heal/race_native_contracts.json create mode 100644 scripts/validation_heal/race_native_evidence.py create mode 100644 test_validation_heal_race_native.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index ae4befb8..e894c384 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -33,6 +33,7 @@ on: - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py + - test_validation_heal_race_native.py - test_validation_heal_incarnation_native.py - test_validation_heal_manager_native.py - test_validation_heal_manager_fixture.py @@ -98,6 +99,7 @@ on: - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py + - test_validation_heal_race_native.py - test_validation_heal_incarnation_native.py - test_validation_heal_manager_native.py - test_validation_heal_manager_fixture.py @@ -160,6 +162,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py - run: python3 -B -m unittest -v test_validation_heal_incarnation.py test_validation_heal_incarnation_replay.py - run: python3 -B -m unittest -v test_validation_heal_incarnation_native.py + - run: python3 -B -m unittest -v test_validation_heal_race_native.py - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh - run: bash test_validation_scanner_usage.sh diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 09a871dc..5e7314e4 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -646,6 +646,14 @@ J06 的自动入口为 `heal.performance-abba`。执行器在部署前冻结三 A02 的自动入口为 `heal.admin-scope-isolation`。H2 的专用 `scope-isolation` runtime 使用当前四节点/十六盘，按盘序交织节点并设置 `RUSTFS_ERASURE_SET_DRIVE_COUNT=8`，要求真实两个 set 各跨全部四个节点。普通执行器仍只接受原单 set。两个桶的完整 ACK/实际物理 set 清单冻结后，root/bucket/prefix/object/pool-set 各自新部署、注入选中与控制故障。完整物理和 streaming 证明必须先于 GET 和控制备份恢复。H1 保留原四盘单 set，明确记录不存在未选 set 的适用边界；不能拿同盘目录冒充第二个 set。未限制选择器的原始 receipt 在该固定候选中允许 null，单独保存实际物理映射，显式 pool/set 则必须严格匹配原始字段。 +### Exact-source native race subsets + +`race_native.py` builds and runs the reviewed tests in `race_native_contracts.json` on a hosted tools runner. The candidate checkout must match the exact commit, tree, lock and reviewed Rust files; the producer and registry must match the exact auto-testing checkout. The build uses a fresh target directory and rejects compiler wrappers or injected Rust flags. Each selected regression runs in a separate process with one exact nonignored test. Original logs, executable digest and ordered barrier milestones are retained. These artifacts are separate from `native_regressions.py`; existing F07/C11 artifact contracts remain unchanged. + +The first registered source is `6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca`. G01 includes real Heal during committed overwrite cleanup in four disk/parity/mode scenarios and the existing stale read-repair final-rename test. The G02 subset covers the version-specific shared Heal owner and queued successor PUT. The I05 subset reuses the actual cancelled physical unlink owner and stale read-repair fences. G02's complete race matrix and I05's TCP disconnect, timeout and file-descriptor matrix still require their own executor and live evidence. + +`race_native_evidence.py` admits the downloaded artifact without executing it. It checks the successful hosted workflow attempt, exact candidate/driver/helper identity, selected source contract, native executable, complete unique regression set and every original log. Only the twelve real G01 milestone lines are classified as emitted barrier evidence; the other existing tests use their reviewed in-test assertions. The result states `requiredNativeSubsetPassed`; whole-case, fleet and canonical PASS remain false. Missing source groups, old-source artifacts, ignored or zero-test output and partial evidence are rejected. Updating another candidate's registry entry does not invalidate an unchanged selected contract. Changing the producer or the selected contract requires a fresh tools artifact. Canonical race dispatch remains blocked until the separate fleet consumer and full source procedures are integrated and verified. + ### J02 original opaque metadata probe The `unknown-negative` diagnostic includes the source procedure before the supplemental legacy-object diagnostic. A separate acknowledged object first receives an independently healthy Deep receipt. With the owned services stopped, the existing descriptor-relative fault backend flips byte zero of every original `xl.meta` copy: four copies in H1 and sixteen in H2. Each mutation has its own original backup, receipt and complete physical delta; the candidate-bound decoder must refuse every exact damaged copy. No other metadata, part or format may change. diff --git a/scripts/validation_heal/race_native.py b/scripts/validation_heal/race_native.py new file mode 100644 index 00000000..ef3d95ce --- /dev/null +++ b/scripts/validation_heal/race_native.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +"""Exact-source race fences, separate from retained legacy native artifacts.""" +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import subprocess +import sys + +CONTRACTS = Path(__file__).with_name('race_native_contracts.json') +BUILD_ENV = {'CARGO_BUILD_JOBS': '2', 'CARGO_INCREMENTAL': '0', + 'CARGO_PROFILE_RELEASE_DEBUG': '0', 'CARGO_PROFILE_RELEASE_STRIP': 'debuginfo'} +BUILD_COMMAND = ['cargo', 'test', '--locked', '--release', '-p', 'rustfs-ecstore', + '--lib', '--no-run', '--message-format=json'] +RUNTIME_ENV = {'RUST_MIN_STACK': '33554432'} +BINARY = 'ecstore-race-native-tests' +KIND = 'exact-source-race-native-not-release-service' +CASE_IDS = {'heal.admin-nolock-safety', 'heal.race-unversioned-overwrite', 'heal.race-versioned-put', + 'heal.race-version-delete', 'heal.race-markers-and-batch-delete', + 'heal.race-multipart-complete-abort', 'heal.race-copy-and-metadata-update', + 'heal.lock-timeout-and-caller-abort'} + + +def require(value, message): + if not value: + raise ValueError(message) + + +def sha(path): + value = hashlib.sha256() + with path.open('rb') as stream: + while chunk := stream.read(1024**2): + value.update(chunk) + return value.hexdigest() + + +def save(path, value): + with path.open('x') as out: + json.dump(value, out, indent=2) + out.write('\n') + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def registry(): + require(CONTRACTS.is_file() and not CONTRACTS.is_symlink() and CONTRACTS.stat().st_size < 1024**2, + 'bounded regular race native registry required') + value = json.loads(CONTRACTS.read_text()) + require(isinstance(value, dict) and type(value.get('schemaVersion')) is int and value['schemaVersion'] == 1 + and set(value) == {'schemaVersion', 'candidates'} and isinstance(value['candidates'], dict) + and len(value['candidates']) <= 256 + and all(re.fullmatch(r'[0-9a-f]{40}', key) for key in value['candidates']), + 'race native registry schema or candidate identity differs') + return value['candidates'] + + +def contract_for(commit): + require(isinstance(commit, str) and re.fullmatch(r'[0-9a-f]{40}', commit), + 'native source must be an exact lowercase commit SHA') + contract = registry().get(commit) + require(contract is not None, 'race native producer source is not reviewed') + require(isinstance(contract, dict) and set(contract) == {'sourceTree', 'cargoLockSha256', 'sourceFiles', 'regressions', 'cases'}, + 'race native contract fields differ') + require(re.fullmatch(r'[0-9a-f]{40}', contract['sourceTree']) + and re.fullmatch(r'[0-9a-f]{64}', contract['cargoLockSha256']) + and isinstance(contract['sourceFiles'], dict) and 1 <= len(contract['sourceFiles']) <= 64 + and isinstance(contract['regressions'], dict) and 1 <= len(contract['regressions']) <= 16 + and isinstance(contract['cases'], dict) and 1 <= len(contract['cases']) <= len(CASE_IDS), + 'incomplete reviewed race native contract') + for path, digest in contract['sourceFiles'].items(): + relative = PurePosixPath(path) + require(len(path) <= 256 and not relative.is_absolute() and '..' not in relative.parts + and '\\' not in path and path.startswith('crates/') and path.endswith('.rs') + and re.fullmatch(r'[0-9a-f]{64}', digest), 'native source path or digest differs') + for label, spec in contract['regressions'].items(): + require(re.fullmatch(r'[a-z][a-z0-9-]{0,63}', label) and isinstance(spec, dict) + and set(spec) == {'test', 'sourcePath', 'sourceAssertions', 'witnessKind', 'milestonePrefix', 'milestones'}, + 'native regression fields differ') + require(isinstance(spec['test'], str) and len(spec['test']) <= 512 + and re.fullmatch(r'(set_disk|store)(::[A-Za-z_][A-Za-z0-9_]*)+', spec['test']) + and spec['sourcePath'] in contract['sourceFiles'] + and isinstance(spec['sourceAssertions'], list) and 1 <= len(spec['sourceAssertions']) <= 32 + and all(isinstance(item, str) and 0 < len(item) <= 512 for item in spec['sourceAssertions']), + 'native test identity or source assertions differ') + require(isinstance(spec['milestones'], list) and len(spec['milestones']) <= 128 + and all(isinstance(item, str) and 0 < len(item) <= 512 and '\n' not in item and '\r' not in item + for item in spec['milestones']) + and len(set(spec['milestones'])) == len(spec['milestones']), 'native milestone contract is malformed') + if spec['witnessKind'] == 'ordered-barrier-milestones': + require(isinstance(spec['milestonePrefix'], str) + and re.fullmatch(r'native_(a09|g0[1-6]|i05): ', spec['milestonePrefix']) + and spec['milestones'] and all(item.startswith(spec['milestonePrefix']) for item in spec['milestones']), + 'native ordered milestone namespace differs') + else: + require(spec['witnessKind'] == 'reviewed-in-test-assertions' + and spec['milestonePrefix'] is None and spec['milestones'] == [], 'native witness kind differs') + used = set() + for case, labels in contract['cases'].items(): + require(case in CASE_IDS and isinstance(labels, list) and labels + and all(isinstance(label, str) and label in contract['regressions'] for label in labels) + and len(labels) == len(set(labels)), 'native case lost its exact required regression group') + used.update(labels) + require(used == set(contract['regressions']), 'native regressions contain an unbound case subset') + return contract + + +def exact_result(output, name, code): + require(type(code) is int and code == 0 + and re.findall(r'(?m)^running .*', output) == ['running 1 test'] + and re.findall(r'(?m)^test (?!result:).*', output) == [f'test {name} ... ok'] + and len(re.findall(r'(?m)^test result:.*', output)) == 1 + and re.search(r'(?m)^test result: ok\. 1 passed; 0 failed; 0 ignored;', output), + 'native execution must run one exact nonignored passing test') + + +def milestones(output, spec): + if spec.get('witnessKind') == 'reviewed-in-test-assertions': + require(spec.get('milestones') == [] and spec.get('milestonePrefix') is None + and bool(spec.get('sourceAssertions')), 'native in-test assertion contract is incomplete') + return [] + require(spec.get('witnessKind') == 'ordered-barrier-milestones', 'unreviewed native witness kind') + prefix = spec['milestonePrefix'] + require(re.fullmatch(r'native_(a09|g0[1-6]|i05): ', prefix), 'unreviewed native milestone namespace') + observed = [line for line in output.splitlines() if line.startswith(prefix)] + require(bool(spec['milestones']) and observed == spec['milestones'], + 'actual native milestones are absent, duplicated or out of order') + return observed + + +def verify(directory): + require(directory.is_dir() and not directory.is_symlink(), 'native artifact directory is not regular') + path = directory / 'race-native.json' + require(path.is_file() and not path.is_symlink() and path.stat().st_size < 1024**2, + 'bounded regular race native manifest required') + value = json.loads(path.read_text()) + contract = contract_for(value.get('candidateCommit')) + require(type(value.get('schemaVersion')) is int and value['schemaVersion'] == 1 and value.get('artifactKind') == KIND + and value.get('candidateSourceTree') == contract['sourceTree'] + and value.get('candidateCargoLockSha256') == contract['cargoLockSha256'] + and value.get('sourceFiles') == contract['sourceFiles'] + and value.get('regressions') == contract['regressions'] + and value.get('cases') == contract['cases'] + and value.get('buildEnvironment') == BUILD_ENV and value.get('buildCommand') == BUILD_COMMAND + and value.get('runtimeEnvironment') == RUNTIME_ENV and value.get('optimized') is True + and value.get('sourceUnmodified') is True and value.get('binaryName') == BINARY + and value.get('collectorSha256') == sha(Path(__file__)) + and value.get('selectedContractSha256') == hashlib.sha256(canonical(contract)).hexdigest() + and value.get('canonicalCasePassClaimed') is False and value.get('fleetExecutionClaimed') is False, + 'race native artifact identity or reviewed contract differs') + require(isinstance(value.get('testingCommit'), str) + and re.fullmatch(r'[0-9a-f]{40}', value['testingCommit']), 'native helper identity is absent') + require(isinstance(value.get('driverCommit'), str) and re.fullmatch(r'[0-9a-f]{40}', value['driverCommit']) + and isinstance(value.get('registrySourceSha256'), str) + and re.fullmatch(r'[0-9a-f]{64}', value['registrySourceSha256']), 'native driver or registry source identity is absent') + require(all(isinstance(value.get(key), str) and re.fullmatch(r'[1-9][0-9]*', value[key]) + for key in ('runId', 'runAttempt')), 'native build attempt identity is absent') + binary = directory / BINARY + require(binary.is_file() and not binary.is_symlink() and sha(binary) == value.get('binarySha256'), + 'exact race native executable changed') + return value, binary + + +def build(source, commit, testing_commit, out): + contract = contract_for(commit) + helper_root = Path(__file__).resolve().parents[2] + git = lambda *args: subprocess.check_output(['git', '-C', str(source), *args], text=True).strip() + require(git('rev-parse', 'HEAD') == commit and git('rev-parse', 'HEAD^{tree}') == contract['sourceTree'], + 'candidate checkout differs from reviewed source') + require(not git('status', '--porcelain', '--untracked-files=all'), 'native source has modified or untracked files') + require(all(re.fullmatch(r'[1-9][0-9]*', os.environ.get(key, '')) + for key in ('GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT')) + and re.fullmatch(r'[0-9a-f]{40}', os.environ.get('GITHUB_SHA', '')), + 'native producer requires an exact workflow build attempt') + require(re.fullmatch(r'[0-9a-f]{40}', testing_commit) + and subprocess.check_output(['git', '-C', str(helper_root), 'rev-parse', 'HEAD'], text=True).strip() == testing_commit, + 'native helper checkout differs from exact requested commit') + require(sha(source / 'Cargo.lock') == contract['cargoLockSha256'] + and all((source / path).is_file() and not (source / path).is_symlink() + and (source / path).resolve() == source / path and sha(source / path) == digest + for path, digest in contract['sourceFiles'].items()), + 'candidate lock or runtime-fence source differs') + collector_digest, registry_digest = sha(Path(__file__)), sha(CONTRACTS) + for path in [Path(__file__), CONTRACTS]: + relative = path.relative_to(helper_root).as_posix() + committed = subprocess.check_output(['git', '-C', str(helper_root), 'show', testing_commit + ':' + relative]) + require(committed == path.read_bytes(), 'native helper registry or collector has tracked changes') + for key in ('RUSTFLAGS', 'CARGO_ENCODED_RUSTFLAGS', 'RUSTC_WRAPPER', 'RUSTC_WORKSPACE_WRAPPER', + 'CARGO_BUILD_RUSTFLAGS', 'CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS'): + require(not os.environ.get(key), 'native build has an unbound compiler override') + require(not out.exists() and shutil.disk_usage(source).free >= 12 * 1024**3, + 'fresh native output and build disk space required') + out.mkdir(parents=True) + target = out / 'target' + request = {'candidateCommit': commit, 'candidateSourceTree': contract['sourceTree'], + 'candidateCargoLockSha256': contract['cargoLockSha256'], 'testingCommit': testing_commit, + 'sourceFiles': contract['sourceFiles'], 'regressions': contract['regressions'], + 'cases': contract['cases'], + 'runId': os.environ.get('GITHUB_RUN_ID'), 'runAttempt': os.environ.get('GITHUB_RUN_ATTEMPT'), + 'artifactKind': KIND, 'canonicalCasePassClaimed': False, 'fleetExecutionClaimed': False} + request.update(selectedContractSha256=hashlib.sha256(canonical(contract)).hexdigest(), + registrySourceSha256=registry_digest, driverCommit=os.environ.get('GITHUB_SHA')) + save(out / 'race-native-build-request.json', request) + env = {**os.environ, **BUILD_ENV, 'CARGO_TARGET_DIR': str(target)} + with (out / 'cargo.jsonl').open('x') as stdout, (out / 'cargo.stderr').open('x') as stderr: + result = subprocess.run(BUILD_COMMAND, cwd=source, env=env, stdout=stdout, stderr=stderr, timeout=9000) + require(result.returncode == 0, 'native build failed; preserve original diagnostics') + require(not git('status', '--porcelain', '--untracked-files=all'), 'native build changed candidate source') + require(sha(Path(__file__)) == collector_digest and sha(CONTRACTS) == registry_digest, + 'native build changed its helper source or registry') + executables = [] + for line in (out / 'cargo.jsonl').read_text().splitlines(): + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if (row.get('reason') == 'compiler-artifact' and row.get('profile', {}).get('test') is True + and row.get('target', {}).get('name') == 'rustfs_ecstore' and row.get('executable')): + executables.append(Path(row['executable'])) + require(len(executables) == 1 and not executables[0].is_symlink() + and executables[0].resolve().is_relative_to(target.resolve()), 'native build executable escaped its target') + binary = out / BINARY + shutil.copyfile(executables[0], binary) + binary.chmod(0o500) + listing = subprocess.run([str(binary), '--list'], capture_output=True, text=True, timeout=60) + (out / 'test-inventory.log').write_text(listing.stdout + listing.stderr) + require(listing.returncode == 0, 'native test inventory failed') + names = [line.removesuffix(': test') for line in listing.stdout.splitlines() if line.endswith(': test')] + require(len(names) == len(set(names)), 'native test inventory duplicates an identity') + require(all(names.count(spec['test']) == 1 for spec in contract['regressions'].values()), + 'required exact native test is absent or repeated') + save(out / 'race-native.json', {**request, 'schemaVersion': 1, 'optimized': True, 'sourceUnmodified': True, + 'binaryName': BINARY, 'binarySha256': sha(binary), 'collectorSha256': sha(Path(__file__)), + 'buildEnvironment': BUILD_ENV, 'buildCommand': BUILD_COMMAND, 'runtimeEnvironment': RUNTIME_ENV}) + + +def run(directory, out): + value, binary = verify(directory) + require(not out.exists(), 'retain original native run; no automatic retries') + out.mkdir(parents=True) + rows = [] + for label, spec in value['regressions'].items(): + require(verify(directory)[0] == value, 'native identity changed before execution') + require(re.fullmatch(r'[a-z][a-z0-9-]*', label), 'invalid reviewed native label') + command = [str(binary), spec['test'], '--exact', '--show-output', '--test-threads=1'] + result = subprocess.run(command, env={**os.environ, **RUNTIME_ENV}, capture_output=True, text=True, timeout=300) + log = out / (label + '.log') + log.write_text(result.stdout + result.stderr) + require(verify(directory)[0] == value, 'native identity changed during execution') + exact_result(result.stdout, spec['test'], result.returncode) + observed = milestones(result.stdout, spec) + rows.append({'label': label, 'test': spec['test'], 'command': command, 'nativeRegressionPassed': True, + 'exitCode': result.returncode, 'runtimeEnvironment': RUNTIME_ENV, + 'milestones': observed, 'originalLogSha256': sha(log)}) + save(out / 'race-native-results.json', {'schemaVersion': 1, 'candidateCommit': value['candidateCommit'], + 'candidateSourceTree': value['candidateSourceTree'], 'binarySha256': value['binarySha256'], + 'nativeManifestSha256': sha(directory / 'race-native.json'), 'results': rows, + 'requiredNativeSubsetsComplete': True, 'canonicalCasePassClaimed': False, 'fleetExecutionClaimed': False}) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='action', required=True) + supports_parser = sub.add_parser('supports') + supports_parser.add_argument('--commit', required=True) + supports_parser.add_argument('--case-id', choices=sorted(CASE_IDS)) + build_parser = sub.add_parser('build') + build_parser.add_argument('--source', type=Path, required=True) + build_parser.add_argument('--commit', required=True) + build_parser.add_argument('--testing-commit', required=True) + build_parser.add_argument('--out', type=Path, required=True) + run_parser = sub.add_parser('run') + run_parser.add_argument('--directory', type=Path, required=True) + run_parser.add_argument('--out', type=Path, required=True) + args = parser.parse_args() + if args.action == 'supports': + require(re.fullmatch(r'[0-9a-f]{40}', args.commit), 'exact native source SHA required') + if args.commit not in registry(): + sys.exit(2) + contract = contract_for(args.commit) + if args.case_id and args.case_id not in contract['cases']: + sys.exit(2) + elif args.action == 'build': + build(args.source.resolve(), args.commit, args.testing_commit, args.out.resolve()) + else: + run(args.directory.absolute(), args.out.resolve()) + + +if __name__ == '__main__': + try: + main() + except (ValueError, OSError, subprocess.SubprocessError, KeyError, TypeError, json.JSONDecodeError) as error: + print(f'race native refusal: {error}', file=sys.stderr) + sys.exit(1) diff --git a/scripts/validation_heal/race_native_contracts.json b/scripts/validation_heal/race_native_contracts.json new file mode 100644 index 00000000..ded686bc --- /dev/null +++ b/scripts/validation_heal/race_native_contracts.json @@ -0,0 +1,91 @@ +{ + "schemaVersion": 1, + "candidates": { + "6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca": { + "sourceTree": "65abe5d45aa4d632499960cff01bbf8d80f8a26a", + "cargoLockSha256": "b4d959f964c13d8ed507657fa4b52752769f96e5cce932845d3d8cec69a5cd56", + "sourceFiles": { + "crates/ecstore/src/set_disk/ops/object.rs": "9af09ff2c969975dd5196bda8006745d13c5cc6a8f520f6e1b67cc1b2dd0ca3a", + "crates/ecstore/src/set_disk/ops/heal.rs": "4029248422352e178b92cf26f6b96cc88b6a521eefd7c57f2985f9da1daab2e8", + "crates/ecstore/src/set_disk/core/io_primitives.rs": "6e50c8ea8b374b8a2d73da740321af25fed6e26c34e5117feca3e5f5891782ab" + }, + "regressions": { + "committed-overwrite-cleanup": { + "test": "set_disk::ops::object::put_object_tmp_cleanup_tests::deep_heal_preserves_committed_generation_during_old_data_cleanup", + "sourcePath": "crates/ecstore/src/set_disk/ops/object.rs", + "sourceAssertions": [ + "Actual PHASE_CLEANUP pause after new physical generation commit while TailDrained overwrite ACK remains pending.", + "Real Deep and shared read-repair each reconstruct missing new-generation encoded part before any GET, across 4/2 and 16/4 geometries.", + "All unchanged replica metadata, original overwrite ACK and final new-generation physical/full-body identity are preserved." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g01: ", + "milestones": [ + "native_g01: geometry=4/2 read_repair=false old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=4/2 read_repair=false locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=4/2 read_repair=false overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=4/2 read_repair=true old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=4/2 read_repair=true locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=4/2 read_repair=true overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=16/4 read_repair=false old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=16/4 read_repair=false locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=16/4 read_repair=false overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=16/4 read_repair=true old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=16/4 read_repair=true locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=16/4 read_repair=true overwrite_ack_preserved final_physical_generation_and_full_body_verified" + ] + }, + "stale-read-repair-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::read_repair_commit_aborts_when_object_changes_before_final_rename", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Real missing external shard forces actual shared read-repair data reconstruction.", + "Pause immediately before final commit fence; a successor overwrite ACK precedes release.", + "Obsolete read-repair returns exact DiskError::Timeout and final current ETag equals acknowledged overwrite." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "versioned-read-repair-lock-owner": { + "test": "set_disk::ops::heal::heal_result_report_tests::object_lock_diagnostics_distinguish_read_repair_from_queued_put", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Historical missing external shard is repaired under actual shared Data-phase lock.", + "Queued PUT cannot acquire namespace and later historical GET times out while version-specific owner remains held.", + "After release, successor PUT ACK and every original/new version full body remain correct." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "cancelled-physical-unlink-owner": { + "test": "set_disk::ops::heal::heal_result_report_tests::cancelled_heal_backup_cleanup_retains_key_owner_until_unlink_finishes", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Actual external Heal backup unlink pauses in outer, inner and shared read-repair modes.", + "Caller abort does not retire namespace commit owner; successor write lock remains pending until physical unlink drains.", + "After physical completion, successor obtains lock, owned backup is absent and committed full body is preserved." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + } + }, + "cases": { + "heal.race-unversioned-overwrite": [ + "committed-overwrite-cleanup", + "stale-read-repair-final-rename" + ], + "heal.race-versioned-put": [ + "versioned-read-repair-lock-owner" + ], + "heal.lock-timeout-and-caller-abort": [ + "versioned-read-repair-lock-owner", + "cancelled-physical-unlink-owner", + "stale-read-repair-final-rename" + ] + } + } + } +} diff --git a/scripts/validation_heal/race_native_evidence.py b/scripts/validation_heal/race_native_evidence.py new file mode 100644 index 00000000..314d14a3 --- /dev/null +++ b/scripts/validation_heal/race_native_evidence.py @@ -0,0 +1,109 @@ +"""Admit exact-source native subsets without executing the downloaded binary.""" + +import argparse +import json +from pathlib import Path +import re + +try: + from . import race_native as suite +except ImportError: + import race_native as suite + + +def read(path, limit=1024**2): + suite.require(path.is_file() and not path.is_symlink() and path.stat().st_size < limit, + 'native evidence requires a bounded regular original file') + return json.loads(path.read_text()) + + +def verify(directory, candidate, helper_sha, case_id): + directory = Path(directory) + suite.require(directory.is_dir() and not directory.is_symlink(), 'native evidence directory differs') + suite.require(isinstance(helper_sha, str) and re.fullmatch(r'[0-9a-f]{40}', helper_sha), + 'native consumer requires an exact runtime helper') + suite.require(isinstance(candidate, dict) and isinstance(candidate.get('source_sha'), str) + and re.fullmatch(r'[0-9a-f]{40}', candidate['source_sha']) + and isinstance(candidate.get('package_sha256'), str) + and re.fullmatch(r'[0-9a-f]{64}', candidate['package_sha256']), + 'native consumer has no immutable candidate identity') + api = read(directory / 'tool-run.json') + suite.require(type(api.get('id')) is int and api['id'] > 0 + and type(api.get('run_attempt')) is int and api['run_attempt'] > 0 + and api.get('status') == 'completed' and api.get('conclusion') == 'success' + and api.get('event') == 'workflow_dispatch' + and api.get('repository', {}).get('full_name') == 'rustfs/rustfs' + and api.get('path') == '.github/workflows/rustfs-functional-chain.yml', + 'native tools lack a successful exact hosted workflow attempt') + tools = directory / 'native-tools' + evidence = directory / 'native-evidence' + suite.require(evidence.is_dir() and not evidence.is_symlink(), 'native result directory differs') + manifest, binary = suite.verify(tools) + suite.require(manifest['candidateCommit'] == candidate['source_sha'] + and manifest['driverCommit'] == api.get('head_sha') + and manifest['testingCommit'] == helper_sha + and manifest['runId'] == str(api['id']) + and manifest['runAttempt'] == str(api['run_attempt']), + 'native artifact changed candidate, driver, helper or original attempt') + contract = suite.contract_for(candidate['source_sha']) + suite.require(case_id in contract['cases'], 'native subset is not reviewed for the selected case') + summary = read(evidence / 'race-native-results.json') + suite.require(type(summary.get('schemaVersion')) is int and summary['schemaVersion'] == 1 + and summary.get('candidateCommit') == manifest['candidateCommit'] + and summary.get('candidateSourceTree') == manifest['candidateSourceTree'] + and summary.get('binarySha256') == manifest['binarySha256'] + and summary.get('nativeManifestSha256') == suite.sha(tools / 'race-native.json') + and summary.get('requiredNativeSubsetsComplete') is True + and summary.get('canonicalCasePassClaimed') is False + and summary.get('fleetExecutionClaimed') is False, + 'native results changed source, binary, manifest or proof scope') + rows = summary.get('results') + regressions = contract['regressions'] + suite.require(isinstance(rows, list) and len(rows) == len(regressions) + and all(isinstance(row, dict) for row in rows) + and {row.get('label') for row in rows} == set(regressions), + 'native results omit or duplicate required exact tests') + observed = {} + for row in rows: + label = row['label'] + spec = regressions[label] + command = row.get('command') + suite.require(row.get('test') == spec['test'] and row.get('nativeRegressionPassed') is True + and type(row.get('exitCode')) is int and row['exitCode'] == 0 + and row.get('runtimeEnvironment') == suite.RUNTIME_ENV + and isinstance(command, list) and len(command) == 5 + and isinstance(command[0], str) and Path(command[0]).name == binary.name + and command[1:] == [spec['test'], '--exact', '--show-output', '--test-threads=1'], + 'native result has no exact one-test passing invocation') + log = evidence / (label + '.log') + suite.require(log.is_file() and not log.is_symlink() and log.stat().st_size < 16 * 1024**2 + and suite.sha(log) == row.get('originalLogSha256'), 'original native log bytes differ') + output = log.read_text() + suite.exact_result(output, spec['test'], row['exitCode']) + events = suite.milestones(output, spec) + suite.require(row.get('milestones') == events, 'native result invented or omitted barrier events') + observed[label] = {'test': spec['test'], 'witnessKind': spec['witnessKind'], 'milestones': events} + suite.require(suite.verify(tools)[0] == manifest, 'native artifact changed during read-only admission') + return {'caseId': case_id, 'candidateCommit': manifest['candidateCommit'], + 'candidatePackageSha256': candidate['package_sha256'], 'candidateSourceTree': manifest['candidateSourceTree'], + 'candidateCargoLockSha256': manifest['candidateCargoLockSha256'], 'binarySha256': manifest['binarySha256'], + 'runId': api['id'], 'runAttempt': api['run_attempt'], 'driverCommit': api['head_sha'], + 'testingCommit': helper_sha, 'selectedContractSha256': manifest['selectedContractSha256'], + 'requiredRegressionLabels': contract['cases'][case_id], + 'observedRegressions': {label: observed[label] for label in contract['cases'][case_id]}, + 'requiredNativeSubsetPassed': True, 'wholeCasePassClaimed': False, + 'fleetExecutionClaimed': False, 'canonicalCasePassClaimed': False} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--directory', type=Path, required=True) + parser.add_argument('--candidate', type=Path, required=True) + parser.add_argument('--testing-commit', required=True) + parser.add_argument('--case-id', choices=sorted(suite.CASE_IDS), required=True) + args = parser.parse_args() + print(json.dumps(verify(args.directory, read(args.candidate), args.testing_commit, args.case_id), sort_keys=True)) + + +if __name__ == '__main__': + main() diff --git a/test_validation_heal_race_native.py b/test_validation_heal_race_native.py new file mode 100644 index 00000000..bd4c1ad4 --- /dev/null +++ b/test_validation_heal_race_native.py @@ -0,0 +1,170 @@ +"""Negative admission checks; synthetic files never constitute native evidence.""" + +import copy +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from scripts.validation_heal import race_native as suite +from scripts.validation_heal import race_native_evidence as consumer + + +SOURCE = '6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca' +HELPER = 'a' * 40 +CASE = 'heal.race-unversioned-overwrite' +CANDIDATE = {'source_sha': SOURCE, 'package_sha256': 'b' * 64} + + +class RaceNativeTests(unittest.TestCase): + def fixture(self, root): + tools, evidence = root / 'native-tools', root / 'native-evidence' + tools.mkdir() + evidence.mkdir() + contract = suite.contract_for(SOURCE) + binary = tools / suite.BINARY + binary.write_bytes(b'synthetic nonexecutable artifact; must never run') + manifest = {'schemaVersion': 1, 'candidateCommit': SOURCE, 'artifactKind': suite.KIND, + 'candidateSourceTree': contract['sourceTree'], 'candidateCargoLockSha256': contract['cargoLockSha256'], + 'sourceFiles': contract['sourceFiles'], 'regressions': contract['regressions'], 'cases': contract['cases'], + 'buildEnvironment': suite.BUILD_ENV, 'buildCommand': suite.BUILD_COMMAND, + 'runtimeEnvironment': suite.RUNTIME_ENV, 'optimized': True, 'sourceUnmodified': True, + 'binaryName': suite.BINARY, 'binarySha256': suite.sha(binary), + 'collectorSha256': suite.sha(Path(suite.__file__)), + 'selectedContractSha256': hashlib.sha256(suite.canonical(contract)).hexdigest(), + 'canonicalCasePassClaimed': False, 'fleetExecutionClaimed': False, 'testingCommit': HELPER, + 'driverCommit': 'c' * 40, 'registrySourceSha256': suite.sha(suite.CONTRACTS), + 'runId': '123', 'runAttempt': '2'} + self.write(tools / 'race-native.json', manifest) + rows = [] + for label, spec in contract['regressions'].items(): + output = f"running 1 test\ntest {spec['test']} ... ok\n" + output += '\n'.join(spec['milestones']) + '\n' + output += 'test result: ok. 1 passed; 0 failed; 0 ignored; 99 filtered out; finished in 0.1s\n' + log = evidence / (label + '.log') + log.write_text(output) + rows.append({'label': label, 'test': spec['test'], 'nativeRegressionPassed': True, + 'exitCode': 0, 'runtimeEnvironment': suite.RUNTIME_ENV, + 'command': ['/synthetic/' + suite.BINARY, spec['test'], '--exact', '--show-output', '--test-threads=1'], + 'milestones': spec['milestones'], 'originalLogSha256': suite.sha(log)}) + self.write(evidence / 'race-native-results.json', {'schemaVersion': 1, 'candidateCommit': SOURCE, + 'candidateSourceTree': manifest['candidateSourceTree'], 'binarySha256': manifest['binarySha256'], + 'nativeManifestSha256': suite.sha(tools / 'race-native.json'), 'results': rows, + 'requiredNativeSubsetsComplete': True, 'canonicalCasePassClaimed': False, 'fleetExecutionClaimed': False}) + self.write(root / 'tool-run.json', {'id': 123, 'run_attempt': 2, 'status': 'completed', + 'conclusion': 'success', 'event': 'workflow_dispatch', 'repository': {'full_name': 'rustfs/rustfs'}, + 'head_sha': 'c' * 40, 'path': '.github/workflows/rustfs-functional-chain.yml'}) + + @staticmethod + def write(path, value): + path.write_text(json.dumps(value) + '\n') + + def test_read_only_admission_selects_only_reviewed_subset_and_never_claims_case_pass(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.fixture(root) + with patch.object(suite.subprocess, 'run', side_effect=AssertionError('must not execute')): + proof = consumer.verify(root, CANDIDATE, HELPER, CASE) + self.assertEqual(len(proof['observedRegressions']), 2) + self.assertTrue(proof['requiredNativeSubsetPassed']) + self.assertFalse(proof['wholeCasePassClaimed']) + self.assertFalse(proof['canonicalCasePassClaimed']) + self.assertFalse(proof['fleetExecutionClaimed']) + + def test_wrong_api_source_attempt_driver_helper_or_package_is_rejected(self): + mutations = [('id', 124), ('run_attempt', 1), ('run_attempt', True), ('head_sha', 'd' * 40), + ('status', 'in_progress'), ('conclusion', 'failure'), ('event', 'push'), + ('path', '.github/workflows/other.yml'), ('repository', {'full_name': 'other/rustfs'})] + for key, value in mutations: + with self.subTest(key=key, value=value), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.fixture(root) + api = json.loads((root / 'tool-run.json').read_text()) + api[key] = value + self.write(root / 'tool-run.json', api) + with self.assertRaises(ValueError): consumer.verify(root, CANDIDATE, HELPER, CASE) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.fixture(root) + for candidate, helper, case in [({**CANDIDATE, 'source_sha': 'd' * 40}, HELPER, CASE), + ({**CANDIDATE, 'package_sha256': 'invalid'}, HELPER, CASE), + (CANDIDATE, 'd' * 40, CASE), (CANDIDATE, HELPER, 'heal.race-version-delete')]: + with self.subTest(candidate=candidate, helper=helper, case=case), self.assertRaises(ValueError): + consumer.verify(root, candidate, helper, case) + + def test_incomplete_duplicate_false_scope_wrong_command_and_log_tampering_rejected(self): + for kind in ('missing', 'duplicate', 'case-pass', 'fleet-pass', 'scope', 'exit', 'bool-exit', + 'command', 'test', 'environment', 'milestones', 'log', 'binary', 'manifest', 'symlink'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.fixture(root) + path = root / 'native-evidence/race-native-results.json' + value = json.loads(path.read_text()) + row = value['results'][0] + if kind == 'missing': value['results'].pop() + elif kind == 'duplicate': value['results'][-1] = copy.deepcopy(row) + elif kind == 'case-pass': value['canonicalCasePassClaimed'] = True + elif kind == 'fleet-pass': value['fleetExecutionClaimed'] = True + elif kind == 'scope': value['requiredNativeSubsetsComplete'] = False + elif kind == 'exit': row['exitCode'] = 1 + elif kind == 'bool-exit': row['exitCode'] = False + elif kind == 'command': row['command'].remove('--exact') + elif kind == 'test': row['test'] = 'set_disk::other' + elif kind == 'environment': row['runtimeEnvironment'] = {} + elif kind == 'milestones': row['milestones'].reverse() + elif kind == 'log': (root / 'native-evidence' / (row['label'] + '.log')).write_text('changed') + elif kind == 'binary': (root / 'native-tools' / suite.BINARY).write_bytes(b'changed') + elif kind == 'manifest': value['nativeManifestSha256'] = '0' * 64 + else: + log = root / 'native-evidence' / (row['label'] + '.log') + moved = log.with_suffix('.original') + log.rename(moved) + log.symlink_to(moved) + self.write(path, value) + with self.assertRaises(ValueError): consumer.verify(root, CANDIDATE, HELPER, CASE) + + def test_exact_output_parser_rejects_zero_multiple_ignored_and_wrong_tests(self): + name = 'set_disk::ops::test' + valid = f'running 1 test\ntest {name} ... ok\ntest result: ok. 1 passed; 0 failed; 0 ignored;\n' + suite.exact_result(valid, name, 0) + for output, code in [(valid, 1), (valid, False), (valid.replace('running 1 test', 'running 0 tests'), 0), + (valid.replace('0 ignored', '1 ignored'), 0), (valid + valid, 0), (valid.replace(name, name + '_other'), 0)]: + with self.subTest(output=output, code=code), self.assertRaises(ValueError): + suite.exact_result(output, name, code) + + def test_ordered_events_cannot_be_missing_duplicated_reordered_or_invented(self): + spec = suite.contract_for(SOURCE)['regressions']['committed-overwrite-cleanup'] + events = spec['milestones'] + self.assertEqual(suite.milestones('\n'.join(events), spec), events) + for rows in [events[:-1], list(reversed(events)), events + events[:1], events + ['native_g01: invented']]: + with self.subTest(rows=rows), self.assertRaises(ValueError): suite.milestones('\n'.join(rows), spec) + + def test_registry_additions_preserve_selected_contract_but_edits_invalidate_artifact(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.fixture(root) + registry = json.loads(suite.CONTRACTS.read_text()) + registry['candidates']['e' * 40] = copy.deepcopy(registry['candidates'][SOURCE]) + p = root / 'registry.json' + self.write(p, registry) + with patch.object(suite, 'CONTRACTS', p): consumer.verify(root, CANDIDATE, HELPER, CASE) + registry['candidates'][SOURCE]['sourceTree'] = 'f' * 40 + self.write(p, registry) + with patch.object(suite, 'CONTRACTS', p), self.assertRaises(ValueError): + consumer.verify(root, CANDIDATE, HELPER, CASE) + + def test_supports_unknown_and_other_case_are_distinct_from_malformed_input(self): + for commit, case, code in [(SOURCE, CASE, 0), (SOURCE, 'heal.race-version-delete', 2), + ('e' * 40, CASE, 2), ('main', CASE, 1)]: + with self.subTest(commit=commit, case=case): + result = subprocess.run([sys.executable, '-B', suite.__file__, 'supports', '--commit', commit, + '--case-id', case], capture_output=True, text=True) + self.assertEqual(result.returncode, code, result.stderr) + + +if __name__ == '__main__': + unittest.main() From 3a2e16eea1b7e043cfe832609d882db0c1c9933d Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 04:35:09 +0800 Subject: [PATCH 249/343] test(validation): require exact G01 native proof before fleet bootstrap --- docs/validation-heal-operations.md | 4 +- scripts/validation_heal/race.py | 103 +++++++++++++++++++++------- scripts/validation_heal/run_case.sh | 5 ++ test_validation_heal_race_native.py | 25 +++++++ 4 files changed, 111 insertions(+), 26 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 5e7314e4..1adad4dc 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -652,7 +652,9 @@ A02 的自动入口为 `heal.admin-scope-isolation`。H2 的专用 `scope-isolat The first registered source is `6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca`. G01 includes real Heal during committed overwrite cleanup in four disk/parity/mode scenarios and the existing stale read-repair final-rename test. The G02 subset covers the version-specific shared Heal owner and queued successor PUT. The I05 subset reuses the actual cancelled physical unlink owner and stale read-repair fences. G02's complete race matrix and I05's TCP disconnect, timeout and file-descriptor matrix still require their own executor and live evidence. -`race_native_evidence.py` admits the downloaded artifact without executing it. It checks the successful hosted workflow attempt, exact candidate/driver/helper identity, selected source contract, native executable, complete unique regression set and every original log. Only the twelve real G01 milestone lines are classified as emitted barrier evidence; the other existing tests use their reviewed in-test assertions. The result states `requiredNativeSubsetPassed`; whole-case, fleet and canonical PASS remain false. Missing source groups, old-source artifacts, ignored or zero-test output and partial evidence are rejected. Updating another candidate's registry entry does not invalidate an unchanged selected contract. Changing the producer or the selected contract requires a fresh tools artifact. Canonical race dispatch remains blocked until the separate fleet consumer and full source procedures are integrated and verified. +`race_native_evidence.py` admits the downloaded artifact without executing it. It checks the successful hosted workflow attempt, exact candidate/driver/helper identity, selected source contract, native executable, complete unique regression set and every original log. Only the twelve real G01 milestone lines are classified as emitted barrier evidence; the other existing tests use their reviewed in-test assertions. The result states `requiredNativeSubsetPassed`; whole-case, fleet and canonical PASS remain false. Missing source groups, old-source artifacts, ignored or zero-test output and partial evidence are rejected. Updating another candidate's registry entry does not invalidate an unchanged selected contract. Changing the producer or the selected contract requires a fresh tools artifact. + +G01's source-specific fleet route requires that native bundle both in the workflow and in a read-only preflight before package installation or service changes. The same four black-box H1/H2 branches still use all 257 original objects, three ordered overwrite ACKs, exact original Admin settings, overlap witnesses and complete physical/S3 verification. Native logical disk shapes are explicitly distinct from distributed VM execution. The original `072b57b37559fbe626aeda8abf61617fdcbdd926` route retains its blocked native rows. Other race cases are not admitted under the new candidate merely because their native subsets exist. Complete G01 acceptance still requires a fresh successful hosted native bundle, all live branches and independent original-evidence audit. ### J02 original opaque metadata probe diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index eef47fab..79c7f80c 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -29,6 +29,7 @@ from .shards import files, map_target, validate_repaired_physical from .cursor import validate_missing_batch, verify_part_manifest from .versions import public_version_id + from .race_native_evidence import verify as verify_native except ImportError: from heal_probe import Probe, canonical, compare_inventories, payload, require, save from heal_workload import Recorder, verify_seed @@ -39,6 +40,7 @@ from shards import files, map_target, validate_repaired_physical from cursor import validate_missing_batch, verify_part_manifest from versions import public_version_id + from race_native_evidence import verify as verify_native CASE_IDS = ("heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-markers-and-batch-delete", "heal.race-multipart-complete-abort", "heal.race-copy-and-metadata-update") @@ -73,6 +75,37 @@ "effective": {**BASE_OPTIONS, "recursive": True, "nolock": False, "pool": None, "set": None}, "nativeFence": "pinned release has no reviewed e2e-test-hooks artifact or exact Admin commit/cleanup barrier", } +NATIVE_CANDIDATE = '6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca' +NATIVE_REFERENCE = { + 'candidateCommit': NATIVE_CANDIDATE, + 'sourceFiles': { + 'rustfs/src/admin/handlers/heal.rs': '6187d9331555ea8421e0d4fe1449d310dd31d0c48f4e57ff43365701d3c6621b', + 'crates/heal/src/heal/channel.rs': 'b806939f7101c88ec0746b1b377f25ee5e88cc4871527bd348c7ee5f3f7f195d'}, + 'requested': {**BASE_OPTIONS, 'recursive': True, 'nolock': False, 'pool': None, 'set': None}, + 'effective': {**BASE_OPTIONS, 'recursive': True, 'nolock': False, 'pool': None, 'set': None}, + 'nativeFence': 'exact-source real Heal during committed overwrite cleanup plus stale read-repair final rename', +} + + +def source_contract(candidate, case_id): + require(case_id in CASE_IDS, 'unregistered race case') + if candidate.get('source_sha') == REFERENCE['candidateCommit']: + return REFERENCE + require(candidate.get('source_sha') == NATIVE_CANDIDATE + and case_id == 'heal.race-unversioned-overwrite', 'race candidate/case contract is not reviewed') + return NATIVE_REFERENCE + + +def native_preflight(candidate, testing_commit, case_id, directory): + reference = source_contract(candidate, case_id) + if reference is REFERENCE: + return None + require(bool(directory), 'G01 requires exact-source native proof before any fleet mutation') + proof = verify_native(directory, candidate, testing_commit, case_id) + require(proof['requiredRegressionLabels'] == ['committed-overwrite-cleanup', 'stale-read-repair-final-rename'] + and proof['requiredNativeSubsetPassed'] is True and proof['wholeCasePassClaimed'] is False, + 'G01 native commit/cleanup group is incomplete') + return proof class WindowNotObserved(ValueError): @@ -140,20 +173,20 @@ def call(self, operation, params, body=None, **kwargs): return result -def settings(observed): +def settings(observed, reference=REFERENCE): require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), "G01/G03 status has an unknown or failed HTTP response") value = observed["response"] require(value.get("summary") in {"running", "finished"}, "G01/G03 original task disappeared, stopped or failed") # Canonical JSON distinguishes bool/int and preserves raw null selectors. - require(canonical(value.get("settings")) == canonical(REFERENCE["effective"]), + require(canonical(value.get("settings")) == canonical(reference["effective"]), "G01/G03 Admin nolock was not safely normalized or changed unrelated options") return value["settings"] -def sample_overlap(ack, observed, path, token): +def sample_overlap(ack, observed, path, token, reference=REFERENCE): foreground = ack.get("observation") or {} - settings(observed) + settings(observed, reference) value = observed["response"] a, z = foreground.get("callStartedMonotonicNs"), foreground.get("callFinishedMonotonicNs") b, c = observed.get("callStartedMonotonicNs"), observed.get("callFinishedMonotonicNs") @@ -593,16 +626,16 @@ def terminal_proof(status, bucket, before, final): "counters": counters, "exactRaceEnumerationClaimed": False} -def observe(probe, fleet, path, token): +def observe(probe, fleet, path, token, reference=REFERENCE): value = fleet.record("original-token-status", probe.admin(path, [("clientToken", token)], b"")) - settings(value) + settings(value, reference) return value -def wait_running(probe, fleet, path, token, total): +def wait_running(probe, fleet, path, token, total, reference=REFERENCE): deadline = time.monotonic() + 30 while True: - observed = observe(probe, fleet, path, token) + observed = observe(probe, fleet, path, token, reference) value = observed["response"] processed = (value.get("outcome") or {}).get("counters", {}).get("processed") if value["summary"] == "finished": @@ -613,10 +646,10 @@ def wait_running(probe, fleet, path, token, total): time.sleep(.05) -def wait_terminal(probe, fleet, path, token): +def wait_terminal(probe, fleet, path, token, reference=REFERENCE): deadline = time.monotonic() + 240 while True: - current = observe(probe, fleet, path, token) + current = observe(probe, fleet, path, token, reference) if current["response"]["summary"] == "finished": return current["response"] require(time.monotonic() < deadline, "G01/G03 original task exceeded its terminal budget") time.sleep(.2) @@ -655,7 +688,7 @@ def foreground(recorder, plan): return responses, {"uploadIds": upload_ids, "partEtags": part_etags} -def race(recorder, plan, probe, fleet, path, token): +def race(recorder, plan, probe, fleet, path, token, reference=REFERENCE): samples = [] # Only the worker mutates Recorder state. All Fleet.record writes run on # the controller, so evidence sequence allocation never races. @@ -685,7 +718,7 @@ def read_repair(): deadline = time.monotonic() + 120 try: while not worker.done() and time.monotonic() < deadline: - samples.append(observe(probe, fleet, path, token)) + samples.append(observe(probe, fleet, path, token, reference)) time.sleep(.01) except BaseException: recorder.mutations.poison("status observation failed; drain in-flight mutations only") @@ -699,7 +732,7 @@ def read_repair(): acks = [json.loads(path.read_text()) for path in sorted(recorder.output.glob("*-ack.json"))] require(len(acks) == len(plan) and all(row.get("operationCheck") == "PASS" for row in acks), "G01/G03 lacks every durable foreground ACK") - overlaps = [any(sample_overlap(ack, sample, path, token) for sample in samples) for ack in acks] + overlaps = [any(sample_overlap(ack, sample, path, token, reference) for sample in samples) for ack in acks] fleet.record("read-repair-overlap-proof", {"branch": plan[0].get("branch") if plan else None, "readCalls": len(readers), "allReadsObserved": bool(readers) if read_branch else True, "nativeFenceProved": False}) @@ -746,7 +779,7 @@ def run_branch(args, topology, nodes, branch): run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="manual") candidate = json.loads(args.candidate.read_text()) - require(candidate["source_sha"] == REFERENCE["candidateCommit"], "G01/G03 candidate contract is not reviewed") + reference = source_contract(candidate, args.case_id) fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) @@ -760,7 +793,7 @@ def run_branch(args, topology, nodes, branch): "maxInventoryObjects": 512, "maxInventoryBytes": 512 * 1024**2} fleet.record("binding", binding) current_target_key, current_reference_key = target_key(branch), reference_key(branch) - fleet.record("source-contract", {**REFERENCE, "branch": branch, "objectCount": OBJECTS, "objectSize": SIZE, + fleet.record("source-contract", {**reference, "branch": branch, "objectCount": OBJECTS, "objectSize": SIZE, "requiredPUTs": 3 if branch in (*G01_BRANCHES, *G02_BRANCHES) else 0, "serverCommitOverlapProved": False}) probe, active, recorder, writer = Probe(binding), [], None, None result = {"topology": topology, "branch": branch, @@ -807,12 +840,12 @@ def run_branch(args, topology, nodes, branch): require(physical_content(faulted) == physical_content(fleet.snapshot("attribution-before-START")), "NOT_OBSERVED: G01/G03 missing shards changed before manual admission") path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") - fleet.record("prepared-Admin-request", {"path": path, "query": [], "options": REFERENCE["requested"]}) - start = fleet.record("original-Admin-start", probe.admin(path, [], json.dumps(REFERENCE["requested"]).encode())) + fleet.record("prepared-Admin-request", {"path": path, "query": [], "options": reference["requested"]}) + start = fleet.record("original-Admin-start", probe.admin(path, [], json.dumps(reference["requested"]).encode())) token = start.get("response", {}).get("clientToken") if isinstance(start.get("response"), dict) else None require(start.get("httpStatus") == 200 and isinstance(token, str) and token, "G01/G03 admission lacks an original token") active.append((path, token)) - wait_running(probe, fleet, path, token, len(expected)) + wait_running(probe, fleet, path, token, len(expected), reference) witness = fleet.snapshot("target-missing-before-foreground") target = next(target for target in targets if target["key"] == current_target_key) if (target["node"], target["disk"], target["relativePath"]) in files(witness): @@ -823,16 +856,16 @@ def run_branch(args, topology, nodes, branch): writer.seen.update((row["key"], row["versionId"]) for row in expected) if branch == "batch-version-delete": writer.batch_expected_ids = {item["VersionId"] for item in plan[0]["params"]["Delete"]["Objects"]} - responses, overlapped = race(writer, plan, probe, fleet, path, token) + responses, overlapped = race(writer, plan, probe, fleet, path, token, reference) mpu_runtime = plan[0].get("runtime") if branch in G05_BRANCHES else None if branch in G05_BRANCHES: verify_multipart_state(probe, fleet, bucket, branch, mpu_runtime, [(1, G05_PART_SIZES[0])] if branch != "complete" else list(enumerate(G05_PART_SIZES, 1))) final = fleet.record("ACK-derived-final-manifest", final_inventory(baseline, branch, plan, responses)) - terminal = wait_terminal(probe, fleet, path, token) + terminal = wait_terminal(probe, fleet, path, token, reference) active.remove((path, token)) for _ in range(2): - repeated = observe(probe, fleet, path, token)["response"] + repeated = observe(probe, fleet, path, token, reference)["response"] require(canonical(repeated.get("outcome")) == canonical(terminal.get("outcome")) and repeated["summary"] == terminal["summary"], "G01/G03 terminal responsibility changed on repetition") # Collect physical evidence even if terminal counters fail. Old target @@ -924,6 +957,17 @@ def complete_results(results, case_id): def main(): + if sys.argv[1:2] == ['preflight']: + parser = argparse.ArgumentParser(description='Read-only native admission before fleet bootstrap') + parser.add_argument('--candidate', type=Path, required=True) + parser.add_argument('--case-id', choices=list(CASE_BRANCHES), required=True) + parser.add_argument('--testing-commit', required=True) + args = parser.parse_args(sys.argv[2:]) + candidate = json.loads(args.candidate.read_text()) + proof = native_preflight(candidate, args.testing_commit, args.case_id, + os.environ.get('VALIDATION_HEAL_NATIVE_DIRECTORY')) + print(json.dumps({'nativeProof': proof, 'fleetMutationRequested': False, 'canonicalCasePassClaimed': False})) + return def interrupted(signum, frame): raise KeyboardInterrupt(f"G01/G03 interrupted by signal {signum}") signal.signal(signal.SIGTERM, interrupted) parser = argparse.ArgumentParser(description=__doc__) @@ -942,8 +986,17 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"G01/G03 interrupted by and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "G01/G03 requires four distinct current Heal nodes") candidate = json.loads(args.candidate.read_text()) - require(candidate["source_sha"] == REFERENCE["candidateCommit"], "G01/G03 candidate contract not reviewed") - results = [native_blocked(candidate, topology, case_id) for topology in ("H1", "H2")] + native = native_preflight(candidate, args.testing_commit, case_id, + os.environ.get('VALIDATION_HEAL_NATIVE_DIRECTORY')) + if native is None: + results = [native_blocked(candidate, topology, case_id) for topology in ("H1", "H2")] + else: + save(args.evidence / 'independent-native-race-subset.json', native) + results = [{'caseId': case_id, 'topology': topology, 'branch': branches[-1], 'verdict': 'PASS', + 'cleanup': 'PASS', 'candidateCommit': candidate['source_sha'], 'nativeFenceProved': True, + 'nativeProof': native, 'nativePhysicalShapes': ['4/2', '16/4'], + 'distributedFleetExecutionClaimed': False, 'fleetExecutionClaimed': False, + 'canonicalCasePassClaimed': False} for topology in ('H1', 'H2')] try: stop = False for topology in ("H1", "H2"): @@ -957,8 +1010,8 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"G01/G03 interrupted by complete = complete_results(results, case_id) save(args.evidence / "healthy-summary.json", {"caseId": case_id, "results": results, "complete": complete, "cleanup": "PASS" if all(row["cleanup"] == "PASS" for row in results) else "FAIL", - "canonicalCasePassClaimed": False, "nativePrerequisitesBlocked": True}) - require(complete, "G01/G03 native prerequisites remain BLOCKED; every H1/H2 branch stays in the denominator") + "canonicalCasePassClaimed": False, "nativePrerequisitesBlocked": native is None}) + require(complete, "race coverage or native prerequisites remain incomplete; every H1/H2 branch stays in the denominator") if __name__ == "__main__": diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index ae119e93..11762b12 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -96,6 +96,11 @@ fi [ "${driver}" = healthy.py ] || [ "${#tool_args[@]}" -gt 0 ] || die "physical Heal cases require candidate-bound bitrot tools" evidence="${artifacts}/evidence/${case_id}" mkdir -p "${artifacts}/cases" "${evidence}" +if [ "${driver}" = race.py ]; then + python3 -B "${ROOT}/scripts/validation_heal/race.py" preflight \ + --candidate "${candidate}" --case-id "${case_id}" --testing-commit "${testing_commit}" \ + > "${evidence}/native-preflight.json" +fi python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ --result-file "${artifacts}/cases/${case_id}.json" --verdict RUNNING --cleanup-status PENDING diff --git a/test_validation_heal_race_native.py b/test_validation_heal_race_native.py index bd4c1ad4..d542e9c9 100644 --- a/test_validation_heal_race_native.py +++ b/test_validation_heal_race_native.py @@ -12,6 +12,7 @@ from scripts.validation_heal import race_native as suite from scripts.validation_heal import race_native_evidence as consumer +from scripts.validation_heal import race SOURCE = '6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca' @@ -21,6 +22,30 @@ class RaceNativeTests(unittest.TestCase): + def test_new_source_g01_requires_exact_bundle_and_other_race_cases_remain_closed(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.fixture(root) + proof = race.native_preflight(CANDIDATE, HELPER, CASE, root) + self.assertTrue(proof['requiredNativeSubsetPassed']) + with self.assertRaises(ValueError): race.native_preflight(CANDIDATE, HELPER, CASE, '') + for case in race.CASE_IDS[1:]: + with self.subTest(case=case), self.assertRaises(ValueError): + race.native_preflight(CANDIDATE, HELPER, case, root) + self.assertIsNone(race.native_preflight({'source_sha': race.REFERENCE['candidateCommit']}, HELPER, CASE, '')) + self.assertFalse(race.complete_results([], CASE)) + shell = Path('scripts/validation_heal/run_case.sh').read_text() + self.assertLess(shell.index('race.py" preflight'), shell.index('(bootstrap)')) + + def test_source_specific_settings_keep_exact_types_selectors_and_lock_policy(self): + reference = race.source_contract(CANDIDATE, CASE) + value = {'httpStatus': 200, 'response': {'summary': 'running', 'settings': copy.deepcopy(reference['effective'])}} + self.assertEqual(race.settings(value, reference), reference['effective']) + for key, changed in [('nolock', True), ('pool', 0), ('set', 0), ('recursive', 1), ('readRepair', True)]: + rejected = copy.deepcopy(value) + rejected['response']['settings'][key] = changed + with self.subTest(key=key), self.assertRaises(ValueError): race.settings(rejected, reference) + def fixture(self, root): tools, evidence = root / 'native-tools', root / 'native-evidence' tools.mkdir() From d5e328a21a12be0d9e95d6c1c25e482fdbd6e314 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 04:42:39 +0800 Subject: [PATCH 250/343] fix(validation): verify package removal during Heal finalization --- scripts/validation_heal/run_case.sh | 12 +++++- test_validation_heal.py | 61 +++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 11762b12..dfebb414 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -137,7 +137,17 @@ finalize_package() { for node in "${NODES[@]}"; do (stop_existing "${node}") || failed=1 ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ - 'if dpkg-query -W -f="${Status}" rustfs 2>/dev/null | grep -q "install ok installed"; then sudo -n dpkg -P rustfs; fi' || failed=1 + 'set -eu +status=0 +state="$(dpkg-query -W -f="\${db:Status-Status}" rustfs 2>/dev/null)" || status=$? +case "${status}:${state}" in + 0:installed|0:config-files) sudo -n dpkg -P rustfs ;; + 1:) ;; + *) printf "unexpected rustfs package query state\n" >&2; exit 1 ;; +esac +status=0 +state="$(dpkg-query -W -f="\${db:Status-Status}" rustfs 2>/dev/null)" || status=$? +test "${status}:${state}" = "1:"' || failed=1 ssh "${SSH_OPTS[@]}" "${SSH_USER}@${node}" \ '! systemctl is-active --quiet rustfs' || failed=1 done diff --git a/test_validation_heal.py b/test_validation_heal.py index 3e9017ab..b89eff90 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -3433,6 +3433,67 @@ def invoke(args, **kwargs): self.assertEqual(len(observed), 5) +class HealPackageFinalizationTest(unittest.TestCase): + def run_finalizer(self, mode, *, installed=True): + import os + import subprocess + source = Path('scripts/validation_heal/run_case.sh').read_text() + function = 'finalize_package() {' + source.split('finalize_package() {', 1)[1].split('\n}\n', 1)[0] + '\n}\n' + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for node in ('n0', 'n1', 'n2', 'n3'): + if installed: (root / node).touch() + query = root / 'dpkg-query' + query.write_text('''#!/usr/bin/env python3 +import os,pathlib,sys +if sys.argv[1:]==['-W','-f=','rustfs']: sys.exit(0) +if sys.argv[1:] != ['-W','-f=${db:Status-Status}','rustfs']: sys.exit(2) +if os.environ['MODE']=='query-error': sys.exit(42) +if (pathlib.Path(os.environ['SCENE'])/os.environ['NODE']).exists(): print('installed');sys.exit(0) +sys.exit(1) +''') + purge = root / 'dpkg' + purge.write_text('''#!/usr/bin/env python3 +import os,pathlib,sys +assert sys.argv[1:]==['-P','rustfs'] +p=pathlib.Path(os.environ['SCENE']) +with (p/'purges').open('a') as out: out.write(os.environ['NODE']+'\\n') +if os.environ['MODE']=='purge-error':sys.exit(42) +if os.environ['MODE']!='left-installed':(p/os.environ['NODE']).unlink() +''') + sudo = root / 'sudo' + sudo.write_text('#!/bin/sh\n[ "$1" = -n ] || exit 1\nshift\nexec "$@"\n') + systemctl = root / 'systemctl' + systemctl.write_text('#!/bin/sh\nexit 3\n') + for p in (query, purge, sudo, systemctl): p.chmod(0o700) + shell = '''set -euo pipefail +NODES=(n0 n1 n2 n3); SSH_OPTS=(-o BatchMode=yes); SSH_USER=test +stop_existing() { return 0; } +ssh() { local arg node=""; for arg in "$@"; do case "$arg" in test@*) node="${arg##*@}" ;; esac; done; NODE="$node" bash -c "${@: -1}"; } +''' + function + '\nfinalize_package\n' + env = dict(os.environ, PATH=str(root) + os.pathsep + os.environ['PATH'], SCENE=str(root), MODE=mode) + env.pop('BASH_ENV', None) + result = subprocess.run(['bash', '-c', shell], env=env, capture_output=True, text=True) + purges = (root / 'purges').read_text().splitlines() if (root / 'purges').exists() else [] + return result, purges, [n for n in ('n0','n1','n2','n3') if (root / n).exists()] + + def test_actual_nested_shell_passes_literal_dpkg_format_and_purges_all_four_packages(self): + result, purges, remaining = self.run_finalizer('success') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(purges, ['n0','n1','n2','n3']) + self.assertEqual(remaining, []) + result, purges, remaining = self.run_finalizer('success', installed=False) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(purges, []) + + def test_query_errors_failed_purge_and_still_installed_packages_cannot_report_clean(self): + for mode in ('query-error', 'purge-error', 'left-installed'): + with self.subTest(mode=mode): + result, _, remaining = self.run_finalizer(mode) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(remaining, ['n0','n1','n2','n3']) + + class HealAdminResponseBudgetTest(unittest.TestCase): def test_complete_reviewed_8mib_response_and_strict_oversize_and_content_length(self): from io import BytesIO From 686eea0f837ae643127cfd13f954f7dcadc595ef Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 05:30:19 +0800 Subject: [PATCH 251/343] fix(validation): preserve system metadata during I04 degraded writes --- docs/validation-heal-operations.md | 6 ++ scripts/validation_heal/drive_gate.py | 15 +-- scripts/validation_heal/fleet_node.py | 3 +- scripts/validation_heal/queue_pressure.py | 23 ++++- test_validation_heal_queue_pressure.py | 118 ++++++++++++++++++++++ 5 files changed, 152 insertions(+), 13 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 1adad4dc..df0f004d 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -646,6 +646,12 @@ J06 的自动入口为 `heal.performance-abba`。执行器在部署前冻结三 A02 的自动入口为 `heal.admin-scope-isolation`。H2 的专用 `scope-isolation` runtime 使用当前四节点/十六盘，按盘序交织节点并设置 `RUSTFS_ERASURE_SET_DRIVE_COUNT=8`，要求真实两个 set 各跨全部四个节点。普通执行器仍只接受原单 set。两个桶的完整 ACK/实际物理 set 清单冻结后，root/bucket/prefix/object/pool-set 各自新部署、注入选中与控制故障。完整物理和 streaming 证明必须先于 GET 和控制备份恢复。H1 保留原四盘单 set，明确记录不存在未选 set 的适用边界；不能拿同盘目录冒充第二个 set。未限制选择器的原始 receipt 在该固定候选中允许 null，单独保存实际物理映射，显式 pool/set 则必须严格匹配原始字段。 +### I04 durable responsibility during queue pressure + +The live `queue-pressure` permission fault targets only `-durable` on the first owned drive. The other four queue buckets and `.rustfs.sys` retain their original permissions. The live arm derives this fixed suffix from the root-owned reservation; callers cannot supply another bucket or path. Flapping keeps its separate whole-drive fault and F07 keeps its original dedicated bucket scope. + +Before the three degraded-write ACKs, the same original service/mount-namespace identity must observe business read/write `EACCES` and successful system metadata read/write controls with DAC bypass removed. A whole-drive refusal or unavailable system control is rejected before those writes. Full, Merged, Dropped, all three committed MRF responsibilities, exact source-task retirement, complete six-bucket snapshots and original `769/129/129/129` fixtures remain required. Blocking the MRF journal cannot isolate queue-Full retention. A corrected fault does not establish canonical PASS until a separate complete live run is independently audited; original failed results remain retained. + ### Exact-source native race subsets `race_native.py` builds and runs the reviewed tests in `race_native_contracts.json` on a hosted tools runner. The candidate checkout must match the exact commit, tree, lock and reviewed Rust files; the producer and registry must match the exact auto-testing checkout. The build uses a fresh target directory and rejects compiler wrappers or injected Rust flags. Each selected regression runs in a separate process with one exact nonignored test. Original logs, executable digest and ordered barrier milestones are retained. These artifacts are separate from `native_regressions.py`; existing F07/C11 artifact contracts remain unchanged. diff --git a/scripts/validation_heal/drive_gate.py b/scripts/validation_heal/drive_gate.py index 11be8206..c013787a 100644 --- a/scripts/validation_heal/drive_gate.py +++ b/scripts/validation_heal/drive_gate.py @@ -43,19 +43,20 @@ def require(value, message): def gate_folder(state, roots, disk, scope): folder = roots[disk] / "data" if scope == "disk": return folder - require(scope == "bucket", "invalid drive gate scope") + require(scope in {"bucket", "durable-bucket"}, "invalid drive gate scope") marker = state / "owner.json" info = marker.lstat() require(stat.S_ISREG(info.st_mode) and info.st_uid == 0 and info.st_nlink == 1 and info.st_size < 16384 and info.st_mode & 0o022 == 0, "invalid bucket gate owner marker") owner = json.loads(marker.read_text()) - require(owner.get("runtimeProfile") == "mrf-crash" + expected_profile = "queue-pressure" if scope == "durable-bucket" else "mrf-crash" + require(owner.get("runtimeProfile") == expected_profile and isinstance(owner.get("bucket"), str) and re.fullmatch(r"rv-heal-[0-9a-f]{16}", owner["bucket"]), "bucket gate requires the dedicated owned MRF fixture") info = folder.lstat() require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and stat.S_IMODE(info.st_mode) == 0o700, "bucket gate private data root changed") - return folder / owner["bucket"] + return folder / (owner["bucket"] + ("-durable" if scope == "durable-bucket" else "")) def checked_gate(state, roots): @@ -88,7 +89,7 @@ def apply_gate(state, roots, disk, mode, *, scope="disk"): and (original_mode == 0o700 if scope == "disk" else mode == "permission-denied" and original_mode in {0o700, 0o755, 0o777}), "expected owned gate directory with supported original permissions") - if scope == "bucket": + if scope in {"bucket", "durable-bucket"}: objects = (folder / "objects").lstat() probe = (folder / "objects/0000.bin").lstat() read_target = (folder / "objects/0000.bin/xl.meta").lstat() @@ -99,7 +100,7 @@ def apply_gate(state, roots, disk, mode, *, scope="disk"): value = {"schemaVersion": 1, "disk": disk, "mode": mode, "dataPath": str(folder), "originalMode": original_mode, "device": info.st_dev, "inode": info.st_ino, "phase": "armed" if mode == "permission-denied" else "active"} - if scope == "bucket": value["scope"] = scope + if scope != "disk": value["scope"] = scope # Intent precedes chmod/start. Cleanup can restore a partially applied gate. fd = os.open(state / "drive-gate.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) with os.fdopen(fd, "w") as stream: @@ -203,7 +204,7 @@ def probe_gate(state, roots, pid): read = None if value["mode"] == "permission-denied": # A denied write alone cannot prove a stale member cannot read. - relative = "objects/0000.bin/xl.meta" if value.get("scope") == "bucket" else ".rustfs.sys/format.json" + relative = "objects/0000.bin/xl.meta" if value.get("scope") in {"bucket", "durable-bucket"} else ".rustfs.sys/format.json" read_args = [*args[:-4], "python3", "-c", READ_PROBE_SOURCE, value["dataPath"] + "/" + relative] result = subprocess.run(read_args, capture_output=True, text=True, timeout=15) @@ -211,7 +212,7 @@ def probe_gate(state, roots, pid): read = json.loads(result.stdout) validate_probe(read, value["mode"], identity) control = None - if value.get("scope") == "bucket": + if value.get("scope") in {"bucket", "durable-bucket"}: system = roots[value["disk"]] / "data/.rustfs.sys" info = system.lstat() require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0, "MRF control root is not an owned non-linked directory") diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index c406338d..3f96d6ab 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -951,7 +951,8 @@ def drive_gate_arm_live(request): module = gate_module() module.validate_live_boundary(before["pid"]) require(quorum_gate_module().checked(state, roots) is None, "live drive gate cannot share a quorum gate") - receipt = module.apply_gate(state, roots, request["disk"], "permission-denied") + gate_args = {"scope": "durable-bucket"} if owner.get("runtimeProfile") == "queue-pressure" else {} + receipt = module.apply_gate(state, roots, request["disk"], "permission-denied", **gate_args) after = process(request) require(all(before.get(name) == after.get(name) for name in PROCESS_FIELDS), "live arm crossed its owned process identity") return {**receipt, "processBefore": before, "processAfter": after, "sameOriginalProcess": True} diff --git a/scripts/validation_heal/queue_pressure.py b/scripts/validation_heal/queue_pressure.py index 1b205d4a..111b3f05 100644 --- a/scripts/validation_heal/queue_pressure.py +++ b/scripts/validation_heal/queue_pressure.py @@ -66,6 +66,21 @@ "nativeCriticalWindowClaimed": False} +def validate_durable_gate(value, run_id, bucket): + control = value.get("systemMetadataControl") or {} + require(value.get("scope") == "durable-bucket" and type(value.get("disk")) is int and value["disk"] == 0 + and value.get("dataPath") == f"/data/rustfs1/heal-validation/{run_id}/data/{bucket}" + and value.get("systemMetadataAccessible") is True and control.get("sameServiceIdentity") is True + and all(type(control.get(label, {}).get("errno")) is int and control[label]["errno"] == 0 + and type(control[label].get("uid")) is int and control[label]["uid"] == 0 + and type(control[label].get("gid")) is int and control[label]["gid"] == 0 for label in ("read", "write")) + and value.get("sameServiceIdentity") is True and value.get("privateMountNamespace") is True + and value.get("dacBypassRemoved") is True and value.get("name") == "EACCES" + and value.get("errno") == 13 and value.get("readProbe", {}).get("errno") == 13, + "queue pressure needs exact durable-bucket refusal and readable/writable system metadata") + return value + + def rejection(observed, *, admission, reason, code, http): require(observed.get("httpStatus") == http and not observed.get("transportError"), "queue rejection has a wrong status or an unknown transport result") @@ -311,17 +326,15 @@ def run_topology(args, topology, nodes, **unused): require(len({scope["token"] for scope in scopes[:3]}) == 3, "independent buckets reused a task identity") rejected = rejection(pressure.call(scopes[3]["path"]), admission="full", reason="none", code="SlowDown", http=503) require(pressure.sample()["queueLength"] == CAPACITY, "NOT_OBSERVED: Full was not witnessed with the actual two-slot queue") - # Restrict the outage to one run-owned drive in the existing service's - # private mount namespace. No scheduler/process is stopped to fake Full. + # Deny only the dedicated durable bucket on one owned drive. System + # metadata must remain readable/writable for the real MRF journal. gate_node, gate_disk = nodes[0], 0 owner = fleet.record("original-gated-process", fleet.remote(gate_node, "process")) fleet.drive_gates.add(gate_node) fleet.record("permission-gate-armed", fleet.remote(gate_node, "drive-gate-arm-live", disk=gate_disk, expected=owner)) fleet.record("permission-gate-activated", fleet.remote(gate_node, "drive-gate-activate")) gate_proof = fleet.record("actual-service-refusal", fleet.remote(gate_node, "drive-gate-probe")) - require(gate_proof.get("sameServiceIdentity") is True and gate_proof.get("privateMountNamespace") is True - and gate_proof.get("name") == "EACCES" and gate_proof.get("readProbe", {}).get("errno") == 13, - "pressure degraded write has no same-namespace refusal witness") + validate_durable_gate(gate_proof, run_id, durable_bucket) earliest_ms, ack_start = int(time.time() * 1000) - 5000, datetime.now(timezone.utc) durable_expected, obligations, _ = acknowledge(durable_recorder, durable_bucket, "offline-put", durable_expected, fixture="DB") fleet.record("durable-ACK-ledger", {"expected": durable_expected, "obligations": obligations}) diff --git a/test_validation_heal_queue_pressure.py b/test_validation_heal_queue_pressure.py index 71fd6d85..bb486fdf 100644 --- a/test_validation_heal_queue_pressure.py +++ b/test_validation_heal_queue_pressure.py @@ -3,15 +3,133 @@ import copy from datetime import datetime, timezone import json +from pathlib import Path +import stat +import tempfile import types import unittest +from unittest.mock import patch from scripts.validation_heal import queue_pressure as queue from scripts.validation_heal.fleet_node import runtime_config from scripts.validation_heal.matrix import AUTOMATIC_CASES, select +from scripts.validation_heal import drive_gate, fleet_node +import test_validation_heal_persistence_mrf as mrf_gate_fixtures class QueuePressureTests(unittest.TestCase): + def durable_gate_fixture(self, root): + state, disk, bucket, system = mrf_gate_fixtures.MrfCorruptionTests().bucket_gate_fixture(root) + prefix = bucket.name + durable = bucket.with_name(prefix + '-durable') + bucket.rename(durable) + (state / 'owner.json').write_text(json.dumps({'runtimeProfile': 'queue-pressure', 'bucket': prefix})) + for i in range(4): (disk / 'data' / (prefix + f'-queue-{i}')).mkdir() + return state, disk, durable, system + + def test_actual_durable_gate_preserves_system_and_queue_buckets_through_activation_and_recovery(self): + with tempfile.TemporaryDirectory() as tmp: + state, disk, durable, system = self.durable_gate_fixture(Path(tmp)) + controls = [system, disk / 'data', *list((disk / 'data').glob('*-queue-*'))] + original = {p: (p.stat().st_mode, p.stat().st_ino) for p in controls} + mode = stat.S_IMODE(durable.stat().st_mode) + with mrf_gate_fixtures.MrfCorruptionTests().rooted_lstat(): + value = drive_gate.apply_gate(state, [disk], 0, 'permission-denied', scope='durable-bucket') + self.assertEqual(value['dataPath'], str(durable)) + self.assertEqual(value['scope'], 'durable-bucket') + drive_gate.activate_gate(state, [disk]) + self.assertEqual(stat.S_IMODE(durable.stat().st_mode), 0) + self.assertEqual({p: (p.stat().st_mode, p.stat().st_ino) for p in controls}, original) + self.assertEqual((system / 'format.json').read_bytes(), b'format-control') + self.assertTrue(drive_gate.release_gate(state, [disk])['originalPermissionsRestored']) + self.assertEqual(stat.S_IMODE(durable.stat().st_mode), mode) + + def test_durable_gate_rejects_foreign_owner_scope_symlink_and_marker_redirection(self): + for kind in ('profile', 'prefix', 'symlink', 'marker'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp: + state, disk, durable, system = self.durable_gate_fixture(Path(tmp)) + owner = json.loads((state / 'owner.json').read_text()) + if kind == 'profile': owner['runtimeProfile'] = 'flapping' + elif kind == 'prefix': owner['bucket'] = '../foreign' + elif kind == 'symlink': + saved = durable.with_name(durable.name + '-saved') + durable.rename(saved) + durable.symlink_to(saved, target_is_directory=True) + (state / 'owner.json').write_text(json.dumps(owner)) + with mrf_gate_fixtures.MrfCorruptionTests().rooted_lstat(): + if kind == 'marker': + drive_gate.apply_gate(state, [disk], 0, 'permission-denied', scope='durable-bucket') + p = state / 'drive-gate.json' + value = json.loads(p.read_text()) + value['dataPath'] = str(system) + p.write_text(json.dumps(value));p.chmod(0o600) + with self.assertRaises(ValueError): drive_gate.activate_gate(state, [disk]) + with self.assertRaises(ValueError): drive_gate.release_gate(state, [disk]) + else: + with self.assertRaises(ValueError): + drive_gate.apply_gate(state, [disk], 0, 'permission-denied', scope='durable-bucket') + self.assertFalse((state / 'drive-gate.json').exists()) + + def test_live_arm_selects_fixed_durable_scope_only_for_queue_pressure(self): + before = {name: {'pid': 10, 'startTicks': 100}.get(name, name) for name in fleet_node.PROCESS_FIELDS} + request = {'runId': 'owned', 'action': 'drive-gate-arm-live', 'disk': 0, 'expected': before} + for profile, expected in [('queue-pressure', {'scope': 'durable-bucket'}), ('flapping', {})]: + calls = [] + gate = types.SimpleNamespace(validate_live_boundary=lambda pid: None, + apply_gate=lambda *args, **kwargs: calls.append(kwargs) or {'state': 'ARMED'}) + with patch.object(fleet_node, 'owned', return_value=(Path('/owned'), [Path('/disk')], {'runtimeProfile': profile})), \ + patch.object(fleet_node, 'process', return_value=before), \ + patch.object(fleet_node, 'gate_module', return_value=gate), \ + patch.object(fleet_node, 'quorum_gate_module', return_value=types.SimpleNamespace(checked=lambda *args: None)): + self.assertTrue(fleet_node.drive_gate_arm_live(request)['sameOriginalProcess']) + self.assertEqual(calls, [expected]) + + def test_actual_durable_scope_probes_business_denial_and_both_system_controls_in_same_namespace(self): + with tempfile.TemporaryDirectory() as tmp: + state, disk, durable, _ = self.durable_gate_fixture(Path(tmp)) + denied = {'uid': 0, 'gid': 0, 'errno': 13} + control = {'uid': 0, 'gid': 0, 'errno': 0} + value = {'mode': 'permission-denied', 'scope': 'durable-bucket', 'phase': 'active', + 'dataPath': str(durable), 'disk': 0} + outputs = [types.SimpleNamespace(returncode=0, stdout=json.dumps(v)) for v in (denied, denied, control, control)] + with mrf_gate_fixtures.MrfCorruptionTests().rooted_lstat(), \ + patch.object(drive_gate, 'checked_gate', return_value=value), \ + patch.object(drive_gate, 'process_identity', return_value={'uid': 0, 'gid': 0, 'capEff': 0}), \ + patch.object(Path, 'readlink', side_effect=[Path('mnt:[candidate]'), Path('mnt:[host]')]), \ + patch.object(drive_gate.subprocess, 'run', side_effect=outputs) as invoke: + proof = drive_gate.probe_gate(state, [disk], 123) + self.assertTrue(proof['systemMetadataAccessible']) + self.assertEqual(proof['scope'], 'durable-bucket') + self.assertEqual(len(invoke.call_args_list), 4) + for call in invoke.call_args_list: + self.assertIn('--mount=/proc/123/ns/mnt', call.args[0]) + self.assertIn('--bounding-set=-dac_override,-dac_read_search', call.args[0]) + self.assertTrue(invoke.call_args_list[1].args[0][-1].endswith('-durable/objects/0000.bin/xl.meta')) + self.assertTrue(invoke.call_args_list[2].args[0][-1].endswith('/data/.rustfs.sys/.validation-mrf-write-probe')) + self.assertTrue(invoke.call_args_list[3].args[0][-1].endswith('/data/.rustfs.sys/format.json')) + + def test_durable_refusal_requires_both_successful_system_controls_and_exact_owned_path(self): + run_id, bucket = 'gh-123-1-h1', 'rv-heal-0123456789abcdef-durable' + control = {'errno': 0, 'uid': 0, 'gid': 0} + value = {'scope': 'durable-bucket', 'disk': 0, 'errno': 13, 'name': 'EACCES', 'readProbe': {'errno': 13}, + 'dataPath': f'/data/rustfs1/heal-validation/{run_id}/data/{bucket}', 'systemMetadataAccessible': True, + 'systemMetadataControl': {'sameServiceIdentity': True, 'read': dict(control), 'write': dict(control)}, + 'sameServiceIdentity': True, 'privateMountNamespace': True, 'dacBypassRemoved': True} + queue.validate_durable_gate(value, run_id, bucket) + for kind in ('scope', 'path', 'disk', 'read', 'write', 'control', 'identity', 'namespace', 'dac', 'flag', 'false-success'): + bad = copy.deepcopy(value) + if kind == 'scope': bad['scope'] = 'disk' + elif kind == 'path': bad['dataPath'] = bad['dataPath'].removesuffix('-durable') + elif kind == 'disk': bad['disk'] = 1 + elif kind in {'read', 'write'}: bad['systemMetadataControl'][kind]['errno'] = 13 + elif kind == 'control': bad['systemMetadataControl'].pop('read') + elif kind == 'identity': bad['systemMetadataControl']['write']['uid'] = 1000 + elif kind == 'namespace': bad['privateMountNamespace'] = False + elif kind == 'dac': bad['dacBypassRemoved'] = False + elif kind == 'flag': bad['systemMetadataAccessible'] = False + else: bad['systemMetadataControl']['read']['errno'] = False + with self.subTest(kind=kind), self.assertRaises(ValueError): queue.validate_durable_gate(bad, run_id, bucket) + def status(self): return {"clusterStatusComplete": True, "coverage": {"expected": 1, "responded": 1, "unknown": 0}, "healQueueLength": 2, "healActiveTasks": 1, "healOperations": {"queueLength": 2, From 8a4e3f4d656f553b20ef4cc95ad91b6164e821d3 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 06:03:49 +0800 Subject: [PATCH 252/343] fix(validation): preserve I04 pressure windows and null identities --- docs/validation-heal-operations.md | 2 + scripts/validation_heal/queue_pressure.py | 26 ++++--- test_validation_heal_queue_pressure.py | 82 +++++++++++++++++++++++ 3 files changed, 100 insertions(+), 10 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index df0f004d..c1b37beb 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -652,6 +652,8 @@ The live `queue-pressure` permission fault targets only `-durable` Before the three degraded-write ACKs, the same original service/mount-namespace identity must observe business read/write `EACCES` and successful system metadata read/write controls with DAC bypass removed. A whole-drive refusal or unavailable system control is rejected before those writes. Full, Merged, Dropped, all three committed MRF responsibilities, exact source-task retirement, complete six-bucket snapshots and original `769/129/129/129` fixtures remain required. Blocking the MRF journal cannot isolate queue-Full retention. A corrected fault does not establish canonical PASS until a separate complete live run is independently audited; original failed results remain retained. +Complete the owned permission probes before admitting the finite pressure workload, so SSH setup does not consume its queue window. The later actual MRF Full event after the SDK ACKs remains mandatory; scheduler pauses, deadline increases and a prior Admin Full response cannot replace that event. Terminal object identities use the existing source-reviewed mapping between Admin's all-zero UUID and S3's `null` version. Real version UUIDs, incarnation/set boundaries, complete counters and independent physical/S3 proofs remain exact; untyped version identities are rejected. + ### Exact-source native race subsets `race_native.py` builds and runs the reviewed tests in `race_native_contracts.json` on a hosted tools runner. The candidate checkout must match the exact commit, tree, lock and reviewed Rust files; the producer and registry must match the exact auto-testing checkout. The build uses a fresh target directory and rejects compiler wrappers or injected Rust flags. Each selected regression runs in a separate process with one exact nonignored test. Original logs, executable digest and ordered barrier milestones are retained. These artifacts are separate from `native_regressions.py`; existing F07/C11 artifact contracts remain unchanged. diff --git a/scripts/validation_heal/queue_pressure.py b/scripts/validation_heal/queue_pressure.py index 111b3f05..71d7a8b9 100644 --- a/scripts/validation_heal/queue_pressure.py +++ b/scripts/validation_heal/queue_pressure.py @@ -26,6 +26,7 @@ from .stop import status, idle_fleet from .scanner_repair import JournalWindow, event_time from .mrf_records import verify_reference, REFERENCE as MRF_REFERENCE + from .versions import public_version_id from .mrf_replay import (acknowledge, identity, pending_bindings, read_checkpoints, retirement, source_tasks, verified_receipt) except ImportError: @@ -41,6 +42,7 @@ from stop import status, idle_fleet from scanner_repair import JournalWindow, event_time from mrf_records import verify_reference, REFERENCE as MRF_REFERENCE + from versions import public_version_id from mrf_replay import (acknowledge, identity, pending_bindings, read_checkpoints, retirement, source_tasks, verified_receipt) @@ -156,7 +158,9 @@ def terminal(current, bucket, expected, missing): seen, incarnation = set(), None for row in rows: item = row.get("identity") or {} - pair = item.get("object"), item.get("versionId") or "null" + version = item.get("versionId") + require(version is None or isinstance(version, str), "queue receipt has an untyped version identity") + pair = item.get("object"), public_version_id(version) require(item.get("bucket") == bucket and pair in wanted and pair not in seen and type(item.get("poolIndex")) is int and item["poolIndex"] == 0 and type(item.get("setIndex")) is int and item["setIndex"] == 0 @@ -302,6 +306,17 @@ def run_topology(args, topology, nodes, **unused): "ATTRIBUTION_NOT_OBSERVED: an owned fault changed before pressure admission") initial_checkpoints = read_checkpoints(fleet, nodes, "MRF-empty-baseline") require(all(not item["intents"] for item in initial_checkpoints.values()), "pressure fixture has unrelated durable responsibility") + # Deny only the dedicated durable bucket on one owned drive. System + # metadata must remain readable/writable for the real MRF journal. + # Finish SSH permission probes before admitting the finite pressure work; + # the actual MRF Full event after the SDK ACKs is still mandatory. + gate_node, gate_disk = nodes[0], 0 + owner = fleet.record("original-gated-process", fleet.remote(gate_node, "process")) + fleet.drive_gates.add(gate_node) + fleet.record("permission-gate-armed", fleet.remote(gate_node, "drive-gate-arm-live", disk=gate_disk, expected=owner)) + fleet.record("permission-gate-activated", fleet.remote(gate_node, "drive-gate-activate")) + gate_proof = fleet.record("actual-service-refusal", fleet.remote(gate_node, "drive-gate-probe")) + validate_durable_gate(gate_proof, run_id, durable_bucket) window = JournalWindow(fleet, "pressure-admission", event_targets={"rustfs::heal::manager", "rustfs::heal::task"}) pressure = Pressure(probe, fleet, binding, active) first = scopes[0] @@ -326,15 +341,6 @@ def run_topology(args, topology, nodes, **unused): require(len({scope["token"] for scope in scopes[:3]}) == 3, "independent buckets reused a task identity") rejected = rejection(pressure.call(scopes[3]["path"]), admission="full", reason="none", code="SlowDown", http=503) require(pressure.sample()["queueLength"] == CAPACITY, "NOT_OBSERVED: Full was not witnessed with the actual two-slot queue") - # Deny only the dedicated durable bucket on one owned drive. System - # metadata must remain readable/writable for the real MRF journal. - gate_node, gate_disk = nodes[0], 0 - owner = fleet.record("original-gated-process", fleet.remote(gate_node, "process")) - fleet.drive_gates.add(gate_node) - fleet.record("permission-gate-armed", fleet.remote(gate_node, "drive-gate-arm-live", disk=gate_disk, expected=owner)) - fleet.record("permission-gate-activated", fleet.remote(gate_node, "drive-gate-activate")) - gate_proof = fleet.record("actual-service-refusal", fleet.remote(gate_node, "drive-gate-probe")) - validate_durable_gate(gate_proof, run_id, durable_bucket) earliest_ms, ack_start = int(time.time() * 1000) - 5000, datetime.now(timezone.utc) durable_expected, obligations, _ = acknowledge(durable_recorder, durable_bucket, "offline-put", durable_expected, fixture="DB") fleet.record("durable-ACK-ledger", {"expected": durable_expected, "obligations": obligations}) diff --git a/test_validation_heal_queue_pressure.py b/test_validation_heal_queue_pressure.py index bb486fdf..21325aba 100644 --- a/test_validation_heal_queue_pressure.py +++ b/test_validation_heal_queue_pressure.py @@ -1,6 +1,7 @@ """Reject false queue admission, responsibility retirement and resource proofs.""" import copy +from contextlib import ExitStack from datetime import datetime, timezone import json from pathlib import Path @@ -215,6 +216,87 @@ def test_complete_scope_counters_and_dispositions_are_independent_of_idle(self): else: outcome["objectsTruncated"] = True with self.subTest(name=name), self.assertRaises(ValueError): queue.terminal(value, "owned", expected, missing) + def test_terminal_maps_only_reviewed_null_version_forms_to_exact_sdk_identity(self): + for version in (None, "", "null", "00000000-0000-0000-0000-000000000000"): + value, expected, missing = self.terminal() + for row in value['outcome']['objects']: row['identity']['versionId'] = version + with self.subTest(version=version): + self.assertEqual(queue.terminal(value, 'owned', expected, missing)['diagnosticObjects'], 2) + for version in (False, True, 0, [], {}, 'another-version', '01234567-89ab-4cde-8fab-0123456789ab'): + value, expected, missing = self.terminal() + value['outcome']['objects'][0]['identity']['versionId'] = version + with self.subTest(version=version), self.assertRaises(ValueError): + queue.terminal(value, 'owned', expected, missing) + + def test_real_orchestrator_finishes_durable_gate_before_finite_pressure_admission(self): + for system_accessible in (True, False): + with self.subTest(system_accessible=system_accessible), tempfile.TemporaryDirectory() as tmp: + calls = [] + + class Fleet: + def __init__(self, nodes, directory, run_id, *args, **kwargs): + self.nodes, self.run_id = nodes, run_id + self.faults, self.drive_gates, self.reserved = [], set(), True + + def record(self, label, value): + if label == 'binding': self.binding = value + return value + + def remote(self, node, action, **kwargs): + calls.append(action) + if action == 'drive-gate-probe': + control = {name: {'uid': 0, 'gid': 0, 'errno': 0} for name in ('read', 'write')} + return {'scope': 'durable-bucket', 'disk': 0, + 'dataPath': f'/data/rustfs1/heal-validation/{self.run_id}/data/{self.binding["bucketPrefix"]}-durable', + 'systemMetadataAccessible': system_accessible, + 'systemMetadataControl': {**control, 'sameServiceIdentity': True}, + 'sameServiceIdentity': True, 'privateMountNamespace': True, 'dacBypassRemoved': True, + 'name': 'EACCES', 'errno': 13, 'readProbe': {'errno': 13}} + return {} + + def deploy(self, *args): pass + def snapshot(self, *args, **kwargs): return {} + def start(self): pass + def stop(self): pass + def cleanup(self): return True + + class Pressure: + def __init__(self, *args): pass + def admit(self, path): + calls.append('pressure-admission') + raise ValueError('end test at first pressure admission') + + root = Path(tmp) + (root / 'evidence').mkdir() + (root / 'candidate.json').write_text(json.dumps({ + 'source_sha': queue.REFERENCE['candidateCommit'], 'package_sha256': 'c' * 64})) + (root / 'rc').write_bytes(b'unit-test-only') + args = types.SimpleNamespace(evidence=root / 'evidence', candidate=root / 'candidate.json', + rc_bin=root / 'rc', tool_directory=root, testing_commit='a' * 40, binary_sha='b' * 64) + target = {'node': 'vm000', 'disk': 0, 'relativePath': 'owned/objects/0/generation/part.1', + 'sha256': 'a' * 64, 'key': 'objects/0', 'versionId': 'null'} + with ExitStack() as patches: + replacements = {'Fleet': Fleet, 'Pressure': Pressure, + 'Probe': lambda *args: types.SimpleNamespace(inventory=lambda *args, **kwargs: {}), + 'Recorder': lambda *args: None, 'seed': lambda *args: [{'key': 'objects/0', 'versionId': 'null'}], + 'verify_inspector': lambda *args: {}, 'verify_reference': lambda *args: {}, + 'verify_seed': lambda *args: {}, 'validate_topology': lambda *args: {}, + 'map_cohorts': lambda *args, **kwargs: [target], 'verify_part_manifest': lambda *args, **kwargs: {}, + 'validate_missing_batch': lambda *args: {}, 'physical_content': lambda *args: 'unchanged', + 'read_checkpoints': lambda *args, **kwargs: {'vm000': {'intents': []}}, + 'JournalWindow': lambda *args, **kwargs: None} + for name, value in replacements.items(): patches.enter_context(patch.object(queue, name, value)) + patches.enter_context(patch.dict(queue.os.environ, {'GITHUB_RUN_ID': '321', 'GITHUB_RUN_ATTEMPT': '1'})) + result = queue.run_topology(args, 'H1', ['vm000']) + self.assertEqual(result['verdict'], 'FAIL') + for action in ('drive-gate-arm-live', 'drive-gate-activate', 'drive-gate-probe'): + self.assertIn(action, calls) + if system_accessible: + self.assertLess(calls.index('drive-gate-probe'), calls.index('pressure-admission')) + self.assertEqual(result['failure']['message'], 'end test at first pressure admission') + else: + self.assertNotIn('pressure-admission', calls) + def test_running_window_requires_partial_actual_processing(self): value = {"summary": "running", "outcome": {"execution": {"state": "running"}, "counters": {"processed": 5}}} self.assertTrue(queue.active_window(value, 10)) From fc6b5d69402af44a4c06011e526ad854ce9a8155 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 08:59:50 +0800 Subject: [PATCH 253/343] test(validation): admit source-bound G02 and G03 native fences --- docs/validation-heal-operations.md | 8 +- scripts/validation_heal/race.py | 29 +++- .../race_native_contracts.json | 134 ++++++++++++++++++ test_validation_heal_race_native.py | 47 +++++- 4 files changed, 206 insertions(+), 12 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index c1b37beb..b8ec4fc9 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -658,11 +658,15 @@ Complete the owned permission probes before admitting the finite pressure worklo `race_native.py` builds and runs the reviewed tests in `race_native_contracts.json` on a hosted tools runner. The candidate checkout must match the exact commit, tree, lock and reviewed Rust files; the producer and registry must match the exact auto-testing checkout. The build uses a fresh target directory and rejects compiler wrappers or injected Rust flags. Each selected regression runs in a separate process with one exact nonignored test. Original logs, executable digest and ordered barrier milestones are retained. These artifacts are separate from `native_regressions.py`; existing F07/C11 artifact contracts remain unchanged. -The first registered source is `6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca`. G01 includes real Heal during committed overwrite cleanup in four disk/parity/mode scenarios and the existing stale read-repair final-rename test. The G02 subset covers the version-specific shared Heal owner and queued successor PUT. The I05 subset reuses the actual cancelled physical unlink owner and stale read-repair fences. G02's complete race matrix and I05's TCP disconnect, timeout and file-descriptor matrix still require their own executor and live evidence. +The first registered source is `6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca`. G01 includes real Heal during committed overwrite cleanup in four disk/parity/mode scenarios and the existing stale read-repair final-rename test. The G02 subset covers the version-specific shared Heal owner and queued successor PUT. The I05 subset reuses the actual cancelled physical unlink owner and stale read-repair fences. These legacy subsets do not complete G02 or I05; I05 also needs its TCP disconnect, timeout and file-descriptor matrix. + +Source `bd0671ba781d8ebf509f4f03533228dfb44496f1` adds complete G02 and G03 native groups. The versioned PUT group requires both the historical lock owner and an actual new-version commit while historical reconstruction is paused. The deletion group requires exact historical and current version deletion before releasing stale reconstruction. Both use real `4/2` and `16/4` local geometries, physical write-tail drain, exact surviving encoded shards and complete version bodies. Its G01 fixture pauses every shuffled cleanup slot and checks every new encoded shard again after cleanup and before GET. These source contracts require fresh same-source hosted artifacts; local tests or an older source's artifacts cannot supply them. `race_native_evidence.py` admits the downloaded artifact without executing it. It checks the successful hosted workflow attempt, exact candidate/driver/helper identity, selected source contract, native executable, complete unique regression set and every original log. Only the twelve real G01 milestone lines are classified as emitted barrier evidence; the other existing tests use their reviewed in-test assertions. The result states `requiredNativeSubsetPassed`; whole-case, fleet and canonical PASS remain false. Missing source groups, old-source artifacts, ignored or zero-test output and partial evidence are rejected. Updating another candidate's registry entry does not invalidate an unchanged selected contract. Changing the producer or the selected contract requires a fresh tools artifact. -G01's source-specific fleet route requires that native bundle both in the workflow and in a read-only preflight before package installation or service changes. The same four black-box H1/H2 branches still use all 257 original objects, three ordered overwrite ACKs, exact original Admin settings, overlap witnesses and complete physical/S3 verification. Native logical disk shapes are explicitly distinct from distributed VM execution. The original `072b57b37559fbe626aeda8abf61617fdcbdd926` route retains its blocked native rows. Other race cases are not admitted under the new candidate merely because their native subsets exist. Complete G01 acceptance still requires a fresh successful hosted native bundle, all live branches and independent original-evidence audit. +Runtime sources opt in to specific cases separately from the native registry. The `6af` source enables only G01; `bd0671b` enables G01/G02/G03 with their complete, distinct native groups. The workflow verifies the bundle and invokes the pinned helper's read-only runtime preflight before exporting it to canonical execution. A collector-only helper without the corresponding runtime adapter fails before fleet mutation. The executor repeats admission before package installation or service changes. + +All original H1/H2 black-box branches, SDK ACKs, manifests, overlap witnesses, exact Admin settings and physical/S3 checks remain required. Native logical disk shapes do not certify distributed VM execution. The original `072b57b37559fbe626aeda8abf61617fdcbdd926` route retains its blocked native rows. A09, G04/G05/G06 and I05 still require their own complete native groups and executor evidence. No source or case gains canonical acceptance from a registry entry or local native test; a successful exact-source hosted bundle, every live branch and independent original-evidence audit are required. ### J02 original opaque metadata probe diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index 79c7f80c..a57195de 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -85,26 +85,43 @@ 'effective': {**BASE_OPTIONS, 'recursive': True, 'nolock': False, 'pool': None, 'set': None}, 'nativeFence': 'exact-source real Heal during committed overwrite cleanup plus stale read-repair final rename', } +VERSION_CANDIDATE = 'bd0671ba781d8ebf509f4f03533228dfb44496f1' +VERSION_REFERENCE = {**NATIVE_REFERENCE, 'candidateCommit': VERSION_CANDIDATE, + 'nativeFence': 'exact-source Heal cleanup and explicit-version commit and deletion fences'} +NATIVE_RUNTIME = { + NATIVE_CANDIDATE: {'reference': NATIVE_REFERENCE, 'cases': ('heal.race-unversioned-overwrite',)}, + VERSION_CANDIDATE: {'reference': VERSION_REFERENCE, + 'cases': ('heal.race-unversioned-overwrite', 'heal.race-versioned-put', 'heal.race-version-delete')}, +} +# Registry subsets do not enable a fleet case. Its reviewed runtime source and +# complete case-specific native group must both be present before bootstrap. +FULL_NATIVE_GROUPS = { + 'heal.race-unversioned-overwrite': ['committed-overwrite-cleanup', 'stale-read-repair-final-rename'], + 'heal.race-versioned-put': ['versioned-read-repair-lock-owner', 'explicit-version-append-final-rename'], + 'heal.race-version-delete': ['explicit-version-delete-final-rename'], +} def source_contract(candidate, case_id): require(case_id in CASE_IDS, 'unregistered race case') if candidate.get('source_sha') == REFERENCE['candidateCommit']: return REFERENCE - require(candidate.get('source_sha') == NATIVE_CANDIDATE - and case_id == 'heal.race-unversioned-overwrite', 'race candidate/case contract is not reviewed') - return NATIVE_REFERENCE + entry = NATIVE_RUNTIME.get(candidate.get('source_sha')) + require(entry is not None and case_id in entry['cases'] and case_id in FULL_NATIVE_GROUPS + and entry['reference']['candidateCommit'] == candidate['source_sha'], + 'race candidate/case contract is not reviewed') + return entry['reference'] def native_preflight(candidate, testing_commit, case_id, directory): reference = source_contract(candidate, case_id) if reference is REFERENCE: return None - require(bool(directory), 'G01 requires exact-source native proof before any fleet mutation') + require(bool(directory), 'race requires exact-source native proof before any fleet mutation') proof = verify_native(directory, candidate, testing_commit, case_id) - require(proof['requiredRegressionLabels'] == ['committed-overwrite-cleanup', 'stale-read-repair-final-rename'] + require(proof['requiredRegressionLabels'] == FULL_NATIVE_GROUPS[case_id] and proof['requiredNativeSubsetPassed'] is True and proof['wholeCasePassClaimed'] is False, - 'G01 native commit/cleanup group is incomplete') + 'case-specific native commit group is incomplete') return proof diff --git a/scripts/validation_heal/race_native_contracts.json b/scripts/validation_heal/race_native_contracts.json index ded686bc..02a66948 100644 --- a/scripts/validation_heal/race_native_contracts.json +++ b/scripts/validation_heal/race_native_contracts.json @@ -86,6 +86,140 @@ "stale-read-repair-final-rename" ] } + }, + "bd0671ba781d8ebf509f4f03533228dfb44496f1": { + "sourceTree": "d1ee045065b69ab9298fb7f3d5d7bba0bd0b5a42", + "cargoLockSha256": "b4d959f964c13d8ed507657fa4b52752769f96e5cce932845d3d8cec69a5cd56", + "sourceFiles": { + "crates/ecstore/src/set_disk/ops/object.rs": "d033df47054f054413c007d5e5b94212676411fbb3670403e08358f9152ef443", + "crates/ecstore/src/set_disk/ops/heal.rs": "7693b55f5417991f6f5ef7d6a234f6e645d5dbbf39921e722b2825301b24b747", + "crates/ecstore/src/set_disk/core/io_primitives.rs": "6e50c8ea8b374b8a2d73da740321af25fed6e26c34e5117feca3e5f5891782ab" + }, + "regressions": { + "committed-overwrite-cleanup": { + "test": "set_disk::ops::object::put_object_tmp_cleanup_tests::deep_heal_preserves_committed_generation_during_old_data_cleanup", + "sourcePath": "crates/ecstore/src/set_disk/ops/object.rs", + "sourceAssertions": [ + "Every actual shuffled cleanup slot pauses after new physical generation commit while TailDrained overwrite ACK remains pending.", + "Real Deep and shared read-repair reconstruct a missing new-generation encoded part before GET in4/2 and16/4 geometries.", + "All old physical generations disappear after cleanup; every new encoded shard, unchanged metadata, overwrite ACK and complete final body remain exact." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g01: ", + "milestones": [ + "native_g01: geometry=4/2 read_repair=false old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=4/2 read_repair=false locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=4/2 read_repair=false overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=4/2 read_repair=true old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=4/2 read_repair=true locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=4/2 read_repair=true overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=16/4 read_repair=false old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=16/4 read_repair=false locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=16/4 read_repair=false overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=16/4 read_repair=true old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=16/4 read_repair=true locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=16/4 read_repair=true overwrite_ack_preserved final_physical_generation_and_full_body_verified" + ] + }, + "stale-read-repair-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::read_repair_commit_aborts_when_object_changes_before_final_rename", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Real missing external shard forces actual shared read-repair data reconstruction.", + "Pause immediately before final commit fence; a successor overwrite ACK precedes release.", + "Obsolete read-repair returns exact DiskError::Timeout and final current ETag equals acknowledged overwrite." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "versioned-read-repair-lock-owner": { + "test": "set_disk::ops::heal::heal_result_report_tests::object_lock_diagnostics_distinguish_read_repair_from_queued_put", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Historical missing external shard is repaired under actual shared Data-phase lock.", + "Queued PUT cannot acquire namespace and later historical GET times out while version-specific owner remains held.", + "After release, successor PUT ACK and every original/new version full body remain correct." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "cancelled-physical-unlink-owner": { + "test": "set_disk::ops::heal::heal_result_report_tests::cancelled_heal_backup_cleanup_retains_key_owner_until_unlink_finishes", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Actual external Heal backup unlink pauses in outer, inner and shared read-repair modes.", + "Caller abort does not retire namespace commit owner; successor write lock remains pending until physical unlink drains.", + "After physical completion, successor obtains lock, owned backup is absent and committed full body is preserved." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "explicit-version-append-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_explicit_version_commit_preserves_acknowledged_new_version", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Actual selected-version shared Deep reconstruction pauses before its final rename after releasing the read guard.", + "Foreground version mutation ACK and same-key physical write-tail drain precede releasing stale Heal; append succeeds or deleted version remains typed FileVersionNotFound.", + "All surviving acknowledged version UUIDs, data dirs, original encoded shards and complete exact/current bodies remain intact in4/2 and16/4; deleted generations do not reappear." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g02: ", + "milestones": [ + "native_g02: geometry=4/2 mutation=append reconstruction_paused_before_final_rename", + "native_g02: geometry=4/2 mutation=append foreground_ack_and_physical_tail_verified_before_release", + "native_g02: geometry=4/2 mutation=append exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g02: geometry=16/4 mutation=append reconstruction_paused_before_final_rename", + "native_g02: geometry=16/4 mutation=append foreground_ack_and_physical_tail_verified_before_release", + "native_g02: geometry=16/4 mutation=append exact_version_outcome_physical_generations_and_full_bodies_verified" + ] + }, + "explicit-version-delete-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_explicit_version_commit_refuses_deleted_historical_and_current_versions", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Actual selected-version shared Deep reconstruction pauses before its final rename after releasing the read guard.", + "Foreground version mutation ACK and same-key physical write-tail drain precede releasing stale Heal; append succeeds or deleted version remains typed FileVersionNotFound.", + "All surviving acknowledged version UUIDs, data dirs, original encoded shards and complete exact/current bodies remain intact in4/2 and16/4; deleted generations do not reappear." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g03: ", + "milestones": [ + "native_g03: geometry=4/2 mutation=delete-historical reconstruction_paused_before_final_rename", + "native_g03: geometry=4/2 mutation=delete-historical foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=4/2 mutation=delete-historical exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g03: geometry=16/4 mutation=delete-historical reconstruction_paused_before_final_rename", + "native_g03: geometry=16/4 mutation=delete-historical foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=16/4 mutation=delete-historical exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g03: geometry=4/2 mutation=delete-current reconstruction_paused_before_final_rename", + "native_g03: geometry=4/2 mutation=delete-current foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=4/2 mutation=delete-current exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g03: geometry=16/4 mutation=delete-current reconstruction_paused_before_final_rename", + "native_g03: geometry=16/4 mutation=delete-current foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=16/4 mutation=delete-current exact_version_outcome_physical_generations_and_full_bodies_verified" + ] + } + }, + "cases": { + "heal.race-unversioned-overwrite": [ + "committed-overwrite-cleanup", + "stale-read-repair-final-rename" + ], + "heal.race-versioned-put": [ + "versioned-read-repair-lock-owner", + "explicit-version-append-final-rename" + ], + "heal.lock-timeout-and-caller-abort": [ + "versioned-read-repair-lock-owner", + "cancelled-physical-unlink-owner", + "stale-read-repair-final-rename" + ], + "heal.race-version-delete": [ + "explicit-version-delete-final-rename" + ] + } } } } diff --git a/test_validation_heal_race_native.py b/test_validation_heal_race_native.py index d542e9c9..212807a2 100644 --- a/test_validation_heal_race_native.py +++ b/test_validation_heal_race_native.py @@ -46,14 +46,53 @@ def test_source_specific_settings_keep_exact_types_selectors_and_lock_policy(sel rejected['response']['settings'][key] = changed with self.subTest(key=key), self.assertRaises(ValueError): race.settings(rejected, reference) - def fixture(self, root): + def test_a_registered_historical_owner_subset_cannot_certify_the_full_version_put_case(self): + case = 'heal.race-versioned-put' + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.fixture(root) + subset = consumer.verify(root, CANDIDATE, HELPER, case) + self.assertEqual(subset['requiredRegressionLabels'], ['versioned-read-repair-lock-owner']) + self.assertTrue(subset['requiredNativeSubsetPassed']) + with patch.dict(race.NATIVE_RUNTIME, {SOURCE: {'reference': race.NATIVE_REFERENCE, 'cases': (case,)}}): + with self.assertRaisesRegex(ValueError, 'case-specific native commit group is incomplete'): + race.native_preflight(CANDIDATE, HELPER, case, root) + + def test_runtime_case_opt_in_does_not_change_the_exact_source_or_enable_other_native_groups(self): + wrong_source = copy.deepcopy(race.NATIVE_REFERENCE) + wrong_source['candidateCommit'] = 'e' * 40 + for reference, case in [(wrong_source, CASE), (race.NATIVE_REFERENCE, 'heal.race-markers-and-batch-delete')]: + with self.subTest(case=case), patch.dict(race.NATIVE_RUNTIME, {SOURCE: {'reference': reference, 'cases': (case,)}}): + with self.assertRaisesRegex(ValueError, 'race candidate/case contract is not reviewed'): + race.native_preflight(CANDIDATE, HELPER, case, '/never-used') + + def test_version_source_admits_only_three_complete_native_groups_without_executing_artifacts(self): + source = race.VERSION_CANDIDATE + candidate = {**CANDIDATE, 'source_sha': source} + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.fixture(root, source=source) + with patch.object(suite.subprocess, 'run', side_effect=AssertionError('must not execute native artifacts')): + for case in race.CASE_IDS[:3]: + with self.subTest(case=case): + proof = race.native_preflight(candidate, HELPER, case, root) + self.assertEqual(proof['candidateCommit'], source) + self.assertEqual(proof['requiredRegressionLabels'], race.FULL_NATIVE_GROUPS[case]) + self.assertTrue(proof['requiredNativeSubsetPassed']) + self.assertFalse(proof['wholeCasePassClaimed']) + self.assertFalse(proof['canonicalCasePassClaimed']) + for case in race.CASE_IDS[3:]: + with self.subTest(case=case), self.assertRaises(ValueError): + race.native_preflight(candidate, HELPER, case, root) + + def fixture(self, root, *, source=SOURCE): tools, evidence = root / 'native-tools', root / 'native-evidence' tools.mkdir() evidence.mkdir() - contract = suite.contract_for(SOURCE) + contract = suite.contract_for(source) binary = tools / suite.BINARY binary.write_bytes(b'synthetic nonexecutable artifact; must never run') - manifest = {'schemaVersion': 1, 'candidateCommit': SOURCE, 'artifactKind': suite.KIND, + manifest = {'schemaVersion': 1, 'candidateCommit': source, 'artifactKind': suite.KIND, 'candidateSourceTree': contract['sourceTree'], 'candidateCargoLockSha256': contract['cargoLockSha256'], 'sourceFiles': contract['sourceFiles'], 'regressions': contract['regressions'], 'cases': contract['cases'], 'buildEnvironment': suite.BUILD_ENV, 'buildCommand': suite.BUILD_COMMAND, @@ -76,7 +115,7 @@ def fixture(self, root): 'exitCode': 0, 'runtimeEnvironment': suite.RUNTIME_ENV, 'command': ['/synthetic/' + suite.BINARY, spec['test'], '--exact', '--show-output', '--test-threads=1'], 'milestones': spec['milestones'], 'originalLogSha256': suite.sha(log)}) - self.write(evidence / 'race-native-results.json', {'schemaVersion': 1, 'candidateCommit': SOURCE, + self.write(evidence / 'race-native-results.json', {'schemaVersion': 1, 'candidateCommit': source, 'candidateSourceTree': manifest['candidateSourceTree'], 'binarySha256': manifest['binarySha256'], 'nativeManifestSha256': suite.sha(tools / 'race-native.json'), 'results': rows, 'requiredNativeSubsetsComplete': True, 'canonicalCasePassClaimed': False, 'fleetExecutionClaimed': False}) From 5c6adccf4e110b8229a0427b79a60bbcc09a4671 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 09:14:14 +0800 Subject: [PATCH 254/343] fix(validation): reject rate-limited source journals --- docs/validation-heal-operations.md | 2 ++ scripts/validation_heal/fleet_node.py | 14 +++++++++++--- test_validation_heal.py | 25 +++++++++++++++++++++++++ test_validation_heal_capacity.py | 2 ++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index b8ec4fc9..e47d9092 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -654,6 +654,8 @@ Before the three degraded-write ACKs, the same original service/mount-namespace Complete the owned permission probes before admitting the finite pressure workload, so SSH setup does not consume its queue window. The later actual MRF Full event after the SDK ACKs remains mandatory; scheduler pauses, deadline increases and a prior Admin Full response cannot replace that event. Terminal object identities use the existing source-reviewed mapping between Admin's all-zero UUID and S3's `null` version. Real version UUIDs, incarnation/set boundaries, complete counters and independent physical/S3 proofs remain exact; untyped version identities are rejected. +Owned Heal services disable journald rate limiting through their transient unit properties. Host-wide journal settings remain unchanged. Cursor continuity alone does not establish complete source attribution: the reader retains journald's original dropped-message notice and marks that page incomplete, including notices identified by the stable journal message ID. Source observers reject the page before accepting any task events. Existing page/event limits, resource ceilings, workload counts and deadlines remain enforced; absent source receipts in a rate-limited original run cannot establish either successful repair or a product responsibility-loss diagnosis. + ### Exact-source native race subsets `race_native.py` builds and runs the reviewed tests in `race_native_contracts.json` on a hosted tools runner. The candidate checkout must match the exact commit, tree, lock and reviewed Rust files; the producer and registry must match the exact auto-testing checkout. The build uses a fresh target directory and rejects compiler wrappers or injected Rust flags. Each selected regression runs in a separate process with one exact nonignored test. Original logs, executable digest and ordered barrier milestones are retained. These artifacts are separate from `native_regressions.py`; existing F07/C11 artifact contracts remain unchanged. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 3f96d6ab..060b0587 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -548,6 +548,7 @@ def start(request): runtime_configuration(state, owner) command("systemd-run", "--unit=" + owner["unit"], "--property=Type=exec", "--property=KillMode=control-group", "--property=TimeoutStopSec=30", + "--property=LogRateLimitIntervalSec=0", "--property=LogRateLimitBurst=0", "--property=RuntimeMaxSec=" + str(resource_limits(owner.get("resourceProfile", "standard"))["runtimeMaxSeconds"]), "--property=MemoryMax=4G", *(["--property=PrivateMounts=true", "--property=CapabilityBoundingSet=~CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH"] @@ -1173,12 +1174,12 @@ def logs(request): require(cursor is None or isinstance(cursor, str) and re.fullmatch(r"[A-Za-z0-9=;_-]{1,512}", cursor), "invalid journal cursor") args = ["journalctl", "-u", owner["unit"], "--quiet", "--no-pager", "--all", "-o", "json", - "--output-fields=__CURSOR,MESSAGE"] + "--output-fields=__CURSOR,MESSAGE,MESSAGE_ID"] # +N selects the FIRST N entries after the cursor. A plain -n N # selects the tail and could silently drop the beginning of a window. args += ["--after-cursor", cursor, "-n", "+10000"] if cursor else ["-n", "1"] output = command(*args).stdout - lines, observed_cursor, entries = [], None, 0 + lines, observed_cursor, entries, source_message_loss = [], None, 0, [] for line in output.splitlines(): row = json.loads(line) entry_cursor = row.get("__CURSOR") @@ -1188,6 +1189,12 @@ def logs(request): if isinstance(message, list) and all(type(value) is int and 0 <= value <= 255 for value in message): message = bytes(message).decode("utf-8") require(isinstance(message, str), "journal message is not complete text") + # Cursor continuity cannot recover entries discarded by journald. + # Keep the original notice so the caller records the failed page. + if (row.get("MESSAGE_ID") == "a596d6fe7bfa4994828e72309e95d61e" + or re.fullmatch(r"Suppressed [1-9][0-9]* messages from .+", message)): + source_message_loss.append({"cursor": entry_cursor, "message": message, + "messageId": row.get("MESSAGE_ID")}) entries += 1 require(entries <= 10000, "journal page exceeds its bound; source attribution is incomplete") observed_cursor = entry_cursor @@ -1195,7 +1202,8 @@ def logs(request): require(observed_cursor is not None or cursor is not None and not lines, "journal did not expose a complete cursor window") return {"unit": owner["unit"], "journal": "\n".join(lines), "beforeCursor": cursor, - "afterCursor": observed_cursor or cursor, "complete": True, + "afterCursor": observed_cursor or cursor, "complete": not source_message_loss, + "sourceMessageLoss": source_message_loss, "entries": entries, "hasMore": cursor is not None and entries == 10000} return {"unit": owner["unit"], "journal": command("journalctl", "-u", owner["unit"], "--no-pager", "-n", "2000").stdout} diff --git a/test_validation_heal.py b/test_validation_heal.py index b89eff90..001366a4 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -1439,6 +1439,31 @@ def test_remote_log_window_rejects_truncation_and_invalid_cursor(self): with self.assertRaisesRegex(ValueError, "incomplete"): node_logs({"runId": "unit", "afterCursor": "s=previous;i=1"}) + def test_remote_log_window_retains_suppression_notice_and_refuses_complete_attribution(self): + owner = {"unit": "private.service"} + for notice in ( + {"MESSAGE": "Suppressed 893 messages from private.service"}, + {"MESSAGE": "localized journal loss notice", "MESSAGE_ID": "a596d6fe7bfa4994828e72309e95d61e"}): + with self.subTest(notice=notice), patch.object(fleet_node, "owned", return_value=(None, None, owner)), \ + patch.object(fleet_node, "command") as command: + command.return_value = types.SimpleNamespace(stdout=json.dumps({**notice, "__CURSOR": "s=abc;i=loss"}) + "\n") + value = node_logs({"runId": "unit", "afterCursor": "s=abc;i=before"}) + self.assertFalse(value["complete"]) + self.assertEqual(value["journal"], notice["MESSAGE"]) + self.assertEqual(value["afterCursor"], "s=abc;i=loss") + self.assertTrue(value["sourceMessageLoss"]) + self.assertIn("MESSAGE_ID", next(arg for arg in command.call_args.args if arg.startswith("--output-fields="))) + records = [] + def remote(node, action, **params): + return {"complete": True, "afterCursor": "s=abc;i=before"} if params.get("checkpoint") else value + fleet = types.SimpleNamespace(nodes=["host"], remote=remote, + record=lambda label, row: records.append(row) or row) + window = JournalWindow(fleet, "suppressed", event_targets={"rustfs::heal::manager"}) + with self.assertRaisesRegex(ValueError, "completeness"): + window.capture() + self.assertEqual(records[-1], {"node": "host", **value}) + self.assertFalse(window.events["host"]) + def test_cursor_pagination_keeps_early_attribution_and_does_not_skip_pages(self): events = self.events() observations = [{"complete": True, "afterCursor": "checkpoint"}, diff --git a/test_validation_heal_capacity.py b/test_validation_heal_capacity.py index 0f8759d2..12fdd919 100644 --- a/test_validation_heal_capacity.py +++ b/test_validation_heal_capacity.py @@ -234,6 +234,8 @@ def test_bounded_journal_start_uses_only_verified_private_service_bindings(self) fleet_node.start({"runId": "gh-1-1-h1"}) issued = command.call_args.args self.assertEqual(issued[0], 'systemd-run') + self.assertIn('--property=LogRateLimitIntervalSec=0', issued) + self.assertIn('--property=LogRateLimitBurst=0', issued) for property in bindings: self.assertIn(property, issued) self.assertIn('--property=RemainAfterExit=yes', issued) self.assertEqual(issued[-4:], tuple(owner["volumes"])) From 888e5c63c8611b727eed54e5b5a699edcc929440 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 09:31:57 +0800 Subject: [PATCH 255/343] test(validation): admit exact-source marker race fences --- docs/validation-heal-operations.md | 8 +- scripts/validation_heal/race.py | 7 + .../race_native_contracts.json | 174 ++++++++++++++++++ test_validation_heal_race_native.py | 30 ++- 4 files changed, 215 insertions(+), 4 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index e47d9092..abe26d54 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -664,11 +664,13 @@ The first registered source is `6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca`. G01 i Source `bd0671ba781d8ebf509f4f03533228dfb44496f1` adds complete G02 and G03 native groups. The versioned PUT group requires both the historical lock owner and an actual new-version commit while historical reconstruction is paused. The deletion group requires exact historical and current version deletion before releasing stale reconstruction. Both use real `4/2` and `16/4` local geometries, physical write-tail drain, exact surviving encoded shards and complete version bodies. Its G01 fixture pauses every shuffled cleanup slot and checks every new encoded shard again after cleanup and before GET. These source contracts require fresh same-source hosted artifacts; local tests or an older source's artifacts cannot supply them. -`race_native_evidence.py` admits the downloaded artifact without executing it. It checks the successful hosted workflow attempt, exact candidate/driver/helper identity, selected source contract, native executable, complete unique regression set and every original log. Only the twelve real G01 milestone lines are classified as emitted barrier evidence; the other existing tests use their reviewed in-test assertions. The result states `requiredNativeSubsetPassed`; whole-case, fleet and canonical PASS remain false. Missing source groups, old-source artifacts, ignored or zero-test output and partial evidence are rejected. Updating another candidate's registry entry does not invalidate an unchanged selected contract. Changing the producer or the selected contract requires a fresh tools artifact. +Source `8380466f3a27c3c3ef4115084178a0ab99a78f32` additionally requires the complete G04 marker/batch-delete group. In each geometry, creating a marker, removing an exact marker, deleting two exact data versions with separate item receipts and deleting an unversioned object all commit while older reconstruction is paused. The foreground physical tail drains before release. Every disk must retain the exact data/marker version set without duplicates, every surviving encoded shard and complete exact/current bodies; removed markers and data directories cannot reappear. Its 24 ordered milestones supplement the existing G01/G02/G03 groups. -Runtime sources opt in to specific cases separately from the native registry. The `6af` source enables only G01; `bd0671b` enables G01/G02/G03 with their complete, distinct native groups. The workflow verifies the bundle and invokes the pinned helper's read-only runtime preflight before exporting it to canonical execution. A collector-only helper without the corresponding runtime adapter fails before fleet mutation. The executor repeats admission before package installation or service changes. +`race_native_evidence.py` admits the downloaded artifact without executing it. It checks the successful hosted workflow attempt, exact candidate/driver/helper identity, selected source contract, native executable, complete unique regression set and every original log. Each ordered-barrier regression must emit exactly the complete source-registered milestone sequence. Regressions marked as in-test assertions retain that narrower witness type; assertion-only evidence cannot invent emitted barrier events. The result states `requiredNativeSubsetPassed`; whole-case, fleet and canonical PASS remain false. Missing source groups, old-source artifacts, ignored or zero-test output and partial evidence are rejected. Updating another candidate's registry entry does not invalidate an unchanged selected contract. Changing the producer or the selected contract requires a fresh tools artifact. -All original H1/H2 black-box branches, SDK ACKs, manifests, overlap witnesses, exact Admin settings and physical/S3 checks remain required. Native logical disk shapes do not certify distributed VM execution. The original `072b57b37559fbe626aeda8abf61617fdcbdd926` route retains its blocked native rows. A09, G04/G05/G06 and I05 still require their own complete native groups and executor evidence. No source or case gains canonical acceptance from a registry entry or local native test; a successful exact-source hosted bundle, every live branch and independent original-evidence audit are required. +Runtime sources opt in to specific cases separately from the native registry. The `6af` source enables only G01; `bd0671b` enables G01/G02/G03; `8380466` additionally enables G04 with its complete, distinct native group. The workflow verifies the bundle and invokes the pinned helper's read-only runtime preflight before exporting it to canonical execution. A collector-only helper without the corresponding runtime adapter fails before fleet mutation. The executor repeats admission before package installation or service changes. + +All original H1/H2 black-box branches, SDK ACKs, manifests, overlap witnesses, exact Admin settings and physical/S3 checks remain required. Native logical disk shapes do not certify distributed VM execution. The original `072b57b37559fbe626aeda8abf61617fdcbdd926` route retains its blocked native rows. A09, G05/G06 and I05 still require their own complete native groups and executor evidence. G04 also remains unaccepted until its same-source hosted evidence and every original live branch pass independent audit. No source or case gains canonical acceptance from a registry entry or local native test; a successful exact-source hosted bundle, every live branch and independent original-evidence audit are required. ### J02 original opaque metadata probe diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index a57195de..8d4c897c 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -88,10 +88,16 @@ VERSION_CANDIDATE = 'bd0671ba781d8ebf509f4f03533228dfb44496f1' VERSION_REFERENCE = {**NATIVE_REFERENCE, 'candidateCommit': VERSION_CANDIDATE, 'nativeFence': 'exact-source Heal cleanup and explicit-version commit and deletion fences'} +MARKER_CANDIDATE = '8380466f3a27c3c3ef4115084178a0ab99a78f32' +MARKER_REFERENCE = {**VERSION_REFERENCE, 'candidateCommit': MARKER_CANDIDATE, + 'nativeFence': 'exact-source Heal cleanup, version and marker/batch-delete commit fences'} NATIVE_RUNTIME = { NATIVE_CANDIDATE: {'reference': NATIVE_REFERENCE, 'cases': ('heal.race-unversioned-overwrite',)}, VERSION_CANDIDATE: {'reference': VERSION_REFERENCE, 'cases': ('heal.race-unversioned-overwrite', 'heal.race-versioned-put', 'heal.race-version-delete')}, + MARKER_CANDIDATE: {'reference': MARKER_REFERENCE, + 'cases': ('heal.race-unversioned-overwrite', 'heal.race-versioned-put', 'heal.race-version-delete', + 'heal.race-markers-and-batch-delete')}, } # Registry subsets do not enable a fleet case. Its reviewed runtime source and # complete case-specific native group must both be present before bootstrap. @@ -99,6 +105,7 @@ 'heal.race-unversioned-overwrite': ['committed-overwrite-cleanup', 'stale-read-repair-final-rename'], 'heal.race-versioned-put': ['versioned-read-repair-lock-owner', 'explicit-version-append-final-rename'], 'heal.race-version-delete': ['explicit-version-delete-final-rename'], + 'heal.race-markers-and-batch-delete': ['marker-and-batch-delete-final-rename'], } diff --git a/scripts/validation_heal/race_native_contracts.json b/scripts/validation_heal/race_native_contracts.json index 02a66948..9e0c66a3 100644 --- a/scripts/validation_heal/race_native_contracts.json +++ b/scripts/validation_heal/race_native_contracts.json @@ -220,6 +220,180 @@ "explicit-version-delete-final-rename" ] } + }, + "8380466f3a27c3c3ef4115084178a0ab99a78f32": { + "sourceTree": "48a90bf754f51d1f743b2809d484dd0b4c0575d1", + "cargoLockSha256": "b4d959f964c13d8ed507657fa4b52752769f96e5cce932845d3d8cec69a5cd56", + "sourceFiles": { + "crates/ecstore/src/set_disk/ops/object.rs": "d033df47054f054413c007d5e5b94212676411fbb3670403e08358f9152ef443", + "crates/ecstore/src/set_disk/ops/heal.rs": "b606f01db029cf6eb909a809094f5696b8eb92b2c90fdf5a23cc116185c45583", + "crates/ecstore/src/set_disk/core/io_primitives.rs": "6e50c8ea8b374b8a2d73da740321af25fed6e26c34e5117feca3e5f5891782ab" + }, + "regressions": { + "committed-overwrite-cleanup": { + "test": "set_disk::ops::object::put_object_tmp_cleanup_tests::deep_heal_preserves_committed_generation_during_old_data_cleanup", + "sourcePath": "crates/ecstore/src/set_disk/ops/object.rs", + "sourceAssertions": [ + "Every actual shuffled cleanup slot pauses after new physical generation commit while TailDrained overwrite ACK remains pending.", + "Real Deep and shared read-repair reconstruct a missing new-generation encoded part before GET in4/2 and16/4 geometries.", + "All old physical generations disappear after cleanup; every new encoded shard, unchanged metadata, overwrite ACK and complete final body remain exact." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g01: ", + "milestones": [ + "native_g01: geometry=4/2 read_repair=false old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=4/2 read_repair=false locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=4/2 read_repair=false overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=4/2 read_repair=true old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=4/2 read_repair=true locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=4/2 read_repair=true overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=16/4 read_repair=false old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=16/4 read_repair=false locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=16/4 read_repair=false overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=16/4 read_repair=true old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=16/4 read_repair=true locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=16/4 read_repair=true overwrite_ack_preserved final_physical_generation_and_full_body_verified" + ] + }, + "stale-read-repair-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::read_repair_commit_aborts_when_object_changes_before_final_rename", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Real missing external shard forces actual shared read-repair data reconstruction.", + "Pause immediately before final commit fence; a successor overwrite ACK precedes release.", + "Obsolete read-repair returns exact DiskError::Timeout and final current ETag equals acknowledged overwrite." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "versioned-read-repair-lock-owner": { + "test": "set_disk::ops::heal::heal_result_report_tests::object_lock_diagnostics_distinguish_read_repair_from_queued_put", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Historical missing external shard is repaired under actual shared Data-phase lock.", + "Queued PUT cannot acquire namespace and later historical GET times out while version-specific owner remains held.", + "After release, successor PUT ACK and every original/new version full body remain correct." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "cancelled-physical-unlink-owner": { + "test": "set_disk::ops::heal::heal_result_report_tests::cancelled_heal_backup_cleanup_retains_key_owner_until_unlink_finishes", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Actual external Heal backup unlink pauses in outer, inner and shared read-repair modes.", + "Caller abort does not retire namespace commit owner; successor write lock remains pending until physical unlink drains.", + "After physical completion, successor obtains lock, owned backup is absent and committed full body is preserved." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "explicit-version-append-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_explicit_version_commit_preserves_acknowledged_new_version", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Actual selected-version shared Deep reconstruction pauses before its final rename after releasing the read guard.", + "Foreground version mutation ACK and same-key physical write-tail drain precede releasing stale Heal; append succeeds or deleted version remains typed FileVersionNotFound.", + "All surviving acknowledged version UUIDs, data dirs, original encoded shards and complete exact/current bodies remain intact in4/2 and16/4; deleted generations do not reappear." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g02: ", + "milestones": [ + "native_g02: geometry=4/2 mutation=append reconstruction_paused_before_final_rename", + "native_g02: geometry=4/2 mutation=append foreground_ack_and_physical_tail_verified_before_release", + "native_g02: geometry=4/2 mutation=append exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g02: geometry=16/4 mutation=append reconstruction_paused_before_final_rename", + "native_g02: geometry=16/4 mutation=append foreground_ack_and_physical_tail_verified_before_release", + "native_g02: geometry=16/4 mutation=append exact_version_outcome_physical_generations_and_full_bodies_verified" + ] + }, + "explicit-version-delete-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_explicit_version_commit_refuses_deleted_historical_and_current_versions", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Actual selected-version shared Deep reconstruction pauses before its final rename after releasing the read guard.", + "Foreground version mutation ACK and same-key physical write-tail drain precede releasing stale Heal; append succeeds or deleted version remains typed FileVersionNotFound.", + "All surviving acknowledged version UUIDs, data dirs, original encoded shards and complete exact/current bodies remain intact in4/2 and16/4; deleted generations do not reappear." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g03: ", + "milestones": [ + "native_g03: geometry=4/2 mutation=delete-historical reconstruction_paused_before_final_rename", + "native_g03: geometry=4/2 mutation=delete-historical foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=4/2 mutation=delete-historical exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g03: geometry=16/4 mutation=delete-historical reconstruction_paused_before_final_rename", + "native_g03: geometry=16/4 mutation=delete-historical foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=16/4 mutation=delete-historical exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g03: geometry=4/2 mutation=delete-current reconstruction_paused_before_final_rename", + "native_g03: geometry=4/2 mutation=delete-current foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=4/2 mutation=delete-current exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g03: geometry=16/4 mutation=delete-current reconstruction_paused_before_final_rename", + "native_g03: geometry=16/4 mutation=delete-current foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=16/4 mutation=delete-current exact_version_outcome_physical_generations_and_full_bodies_verified" + ] + }, + "marker-and-batch-delete-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_marker_and_batch_delete_fence_preserves_acknowledged_version_set", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Real protected external-data reconstruction pauses before final metadata rename in4/2 and16/4 geometries for four independent marker/delete mutations.", + "Foreground create-marker, exact marker removal, two-item version deletion with exact per-item ACKs or unversioned deletion and same-key physical tail drain all precede releasing stale Heal.", + "Every member retains the exact acknowledged marker/data version set without duplicates; surviving encoded shards and full exact/current bodies remain unchanged, and deleted directories or markers cannot resurrect." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g04: ", + "milestones": [ + "native_g04: geometry=4/2 mutation=create-marker reconstruction_paused_before_final_rename", + "native_g04: geometry=4/2 mutation=create-marker exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=4/2 mutation=create-marker marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=4/2 mutation=delete-marker reconstruction_paused_before_final_rename", + "native_g04: geometry=4/2 mutation=delete-marker exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=4/2 mutation=delete-marker marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=4/2 mutation=batch-version-delete reconstruction_paused_before_final_rename", + "native_g04: geometry=4/2 mutation=batch-version-delete exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=4/2 mutation=batch-version-delete marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=4/2 mutation=unversioned-delete reconstruction_paused_before_final_rename", + "native_g04: geometry=4/2 mutation=unversioned-delete exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=4/2 mutation=unversioned-delete marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=16/4 mutation=create-marker reconstruction_paused_before_final_rename", + "native_g04: geometry=16/4 mutation=create-marker exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=16/4 mutation=create-marker marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=16/4 mutation=delete-marker reconstruction_paused_before_final_rename", + "native_g04: geometry=16/4 mutation=delete-marker exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=16/4 mutation=delete-marker marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=16/4 mutation=batch-version-delete reconstruction_paused_before_final_rename", + "native_g04: geometry=16/4 mutation=batch-version-delete exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=16/4 mutation=batch-version-delete marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=16/4 mutation=unversioned-delete reconstruction_paused_before_final_rename", + "native_g04: geometry=16/4 mutation=unversioned-delete exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=16/4 mutation=unversioned-delete marker_set_deleted_directories_surviving_shards_and_full_bodies_verified" + ] + } + }, + "cases": { + "heal.race-unversioned-overwrite": [ + "committed-overwrite-cleanup", + "stale-read-repair-final-rename" + ], + "heal.race-versioned-put": [ + "versioned-read-repair-lock-owner", + "explicit-version-append-final-rename" + ], + "heal.lock-timeout-and-caller-abort": [ + "versioned-read-repair-lock-owner", + "cancelled-physical-unlink-owner", + "stale-read-repair-final-rename" + ], + "heal.race-version-delete": [ + "explicit-version-delete-final-rename" + ], + "heal.race-markers-and-batch-delete": [ + "marker-and-batch-delete-final-rename" + ] + } } } } diff --git a/test_validation_heal_race_native.py b/test_validation_heal_race_native.py index 212807a2..144dc940 100644 --- a/test_validation_heal_race_native.py +++ b/test_validation_heal_race_native.py @@ -61,7 +61,7 @@ def test_a_registered_historical_owner_subset_cannot_certify_the_full_version_pu def test_runtime_case_opt_in_does_not_change_the_exact_source_or_enable_other_native_groups(self): wrong_source = copy.deepcopy(race.NATIVE_REFERENCE) wrong_source['candidateCommit'] = 'e' * 40 - for reference, case in [(wrong_source, CASE), (race.NATIVE_REFERENCE, 'heal.race-markers-and-batch-delete')]: + for reference, case in [(wrong_source, CASE), (race.NATIVE_REFERENCE, 'heal.race-multipart-complete-abort')]: with self.subTest(case=case), patch.dict(race.NATIVE_RUNTIME, {SOURCE: {'reference': reference, 'cases': (case,)}}): with self.assertRaisesRegex(ValueError, 'race candidate/case contract is not reviewed'): race.native_preflight(CANDIDATE, HELPER, case, '/never-used') @@ -85,6 +85,34 @@ def test_version_source_admits_only_three_complete_native_groups_without_executi with self.subTest(case=case), self.assertRaises(ValueError): race.native_preflight(candidate, HELPER, case, root) + def test_marker_source_requires_the_complete_four_mutation_group_and_keeps_later_cases_closed(self): + source = race.MARKER_CANDIDATE + candidate = {**CANDIDATE, 'source_sha': source} + case = 'heal.race-markers-and-batch-delete' + contract = suite.contract_for(source) + label = 'marker-and-batch-delete-final-rename' + self.assertEqual(contract['cases'][case], [label]) + self.assertEqual(len(contract['regressions'][label]['milestones']), 24) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.fixture(root, source=source) + with patch.object(suite.subprocess, 'run', side_effect=AssertionError('must not execute artifacts')): + for selected in race.CASE_IDS[:4]: + proof = race.native_preflight(candidate, HELPER, selected, root) + self.assertEqual(proof['requiredRegressionLabels'], race.FULL_NATIVE_GROUPS[selected]) + self.assertTrue(proof['requiredNativeSubsetPassed']) + self.assertFalse(proof['wholeCasePassClaimed']) + self.assertFalse(proof['canonicalCasePassClaimed']) + for selected in race.CASE_IDS[4:]: + with self.subTest(case=selected), self.assertRaises(ValueError): + race.native_preflight(candidate, HELPER, selected, root) + summary = root / 'native-evidence/race-native-results.json' + value = json.loads(summary.read_text()) + value['results'] = [row for row in value['results'] if row['label'] != label] + self.write(summary, value) + with self.assertRaisesRegex(ValueError, 'omit or duplicate'): + race.native_preflight(candidate, HELPER, case, root) + def fixture(self, root, *, source=SOURCE): tools, evidence = root / 'native-tools', root / 'native-evidence' tools.mkdir() From 33b30bf4053c67a4931e8be79806aca101cbdfc5 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 09:36:25 +0800 Subject: [PATCH 256/343] fix(validation): verify effective owned journal budget --- docs/validation-heal-operations.md | 2 +- scripts/validation_heal/fleet_node.py | 18 ++++++++++-- test_validation_heal_capacity.py | 40 +++++++++++++++++++++++---- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index abe26d54..85ab4d29 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -654,7 +654,7 @@ Before the three degraded-write ACKs, the same original service/mount-namespace Complete the owned permission probes before admitting the finite pressure workload, so SSH setup does not consume its queue window. The later actual MRF Full event after the SDK ACKs remains mandatory; scheduler pauses, deadline increases and a prior Admin Full response cannot replace that event. Terminal object identities use the existing source-reviewed mapping between Admin's all-zero UUID and S3's `null` version. Real version UUIDs, incarnation/set boundaries, complete counters and independent physical/S3 proofs remain exact; untyped version identities are rejected. -Owned Heal services disable journald rate limiting through their transient unit properties. Host-wide journal settings remain unchanged. Cursor continuity alone does not establish complete source attribution: the reader retains journald's original dropped-message notice and marks that page incomplete, including notices identified by the stable journal message ID. Source observers reject the page before accepting any task events. Existing page/event limits, resource ceilings, workload counts and deadlines remain enforced; absent source receipts in a rate-limited original run cannot establish either successful repair or a product responsibility-loss diagnosis. +Owned Heal services request a per-unit journal budget of 1,000,000 messages per second, then read back both effective properties and record the systemd version before source attribution. Missing, zero or different settings are rejected; merely passing launch arguments does not establish effective configuration. Host-wide journal settings remain unchanged. Cursor continuity alone does not establish complete source attribution: the reader retains journald's original dropped-message notice and marks that page incomplete, including notices identified by the stable journal message ID. Source observers reject the page before accepting any task events. Existing page/event limits, resource ceilings, workload counts and deadlines remain enforced; absent source receipts in a rate-limited original run cannot establish either successful repair or a product responsibility-loss diagnosis. ### Exact-source native race subsets diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 060b0587..a36908cc 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -548,7 +548,7 @@ def start(request): runtime_configuration(state, owner) command("systemd-run", "--unit=" + owner["unit"], "--property=Type=exec", "--property=KillMode=control-group", "--property=TimeoutStopSec=30", - "--property=LogRateLimitIntervalSec=0", "--property=LogRateLimitBurst=0", + "--property=LogRateLimitIntervalSec=1s", "--property=LogRateLimitBurst=1000000", "--property=RuntimeMaxSec=" + str(resource_limits(owner.get("resourceProfile", "standard"))["runtimeMaxSeconds"]), "--property=MemoryMax=4G", *(["--property=PrivateMounts=true", "--property=CapabilityBoundingSet=~CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH"] @@ -560,7 +560,21 @@ def start(request): *quorum_gate_module().properties(state, roots), *(volume_module().properties(state, roots, owner) if owner.get("runtimeProfile") in {"bounded-capacity", "journal-bounded"} else []), *replacement_module().start_properties(state, roots), str(state / "rustfs"), *owner["volumes"]) - return {"unit": owner["unit"], "started": True} + return {"unit": owner["unit"], "started": True, "journalPolicy": journal_policy(owner)} + + +def journal_policy(owner): + """Verify the owned unit's effective policy before source attribution.""" + output = command("systemctl", "show", owner["unit"], "-p", "LogRateLimitIntervalUSec", + "-p", "LogRateLimitBurst").stdout + values = dict(line.split("=", 1) for line in output.splitlines() if "=" in line) + require(values.get("LogRateLimitIntervalUSec") in {"1s", "1000000"} + and values.get("LogRateLimitBurst") == "1000000", + "owned service journal rate policy differs; source attribution cannot start") + version = command("systemctl", "--version").stdout.splitlines() + require(version and re.fullmatch(r"systemd [0-9]+(?: .*)?", version[0]), "systemd version is unavailable") + return {"unit": owner["unit"], "rateIntervalSeconds": 1, "rateBurst": 1000000, + "effectiveProperties": values, "systemdVersion": version[0], "hostWideSettingsChanged": False} def stopped(owner): diff --git a/test_validation_heal_capacity.py b/test_validation_heal_capacity.py index 12fdd919..84f0471e 100644 --- a/test_validation_heal_capacity.py +++ b/test_validation_heal_capacity.py @@ -224,22 +224,52 @@ def test_bounded_journal_start_uses_only_verified_private_service_bindings(self) owner = {"runtimeProfile": "journal-bounded", "unit": "owned-test.service", "binarySha256": "a" * 64, "volumes": [str(root / 'data') for root in roots]} bindings = ['--property=PrivateMounts=true', '--property=BindPaths=/verified/private/data:/owned/disk0/data'] + def execute(*args, **kwargs): + if args[:2] == ('systemctl', 'show'): + return SimpleNamespace(stdout='LogRateLimitIntervalUSec=1s\nLogRateLimitBurst=1000000\n') + return SimpleNamespace(stdout='systemd 249 (249.11)\n') with patch.object(fleet_node, "owned", return_value=(state, roots, owner)), patch.object(fleet_node, "stopped"), \ patch.object(fleet_node, "digest", return_value=owner["binarySha256"]), \ - patch.object(fleet_node, "command") as command, \ + patch.object(fleet_node, "command", side_effect=execute) as command, \ patch.object(fleet_node, "gate_module", return_value=SimpleNamespace(checked_gate=lambda *args: None, start_properties=lambda *args: [])), \ patch.object(fleet_node, "quorum_gate_module", return_value=SimpleNamespace(properties=lambda *args: [])), \ patch.object(fleet_node, "replacement_module", return_value=SimpleNamespace(start_properties=lambda *args: [])), \ patch.object(fleet_node, "volume_module", return_value=SimpleNamespace(properties=lambda *args: bindings)): - fleet_node.start({"runId": "gh-1-1-h1"}) - issued = command.call_args.args + started = fleet_node.start({"runId": "gh-1-1-h1"}) + issued = next(call.args for call in command.call_args_list if call.args[0] == 'systemd-run') self.assertEqual(issued[0], 'systemd-run') - self.assertIn('--property=LogRateLimitIntervalSec=0', issued) - self.assertIn('--property=LogRateLimitBurst=0', issued) + self.assertIn('--property=LogRateLimitIntervalSec=1s', issued) + self.assertIn('--property=LogRateLimitBurst=1000000', issued) + self.assertEqual(started['journalPolicy']['rateBurst'], 1000000) + self.assertEqual(started['journalPolicy']['systemdVersion'], 'systemd 249 (249.11)') + self.assertFalse(started['journalPolicy']['hostWideSettingsChanged']) for property in bindings: self.assertIn(property, issued) self.assertIn('--property=RemainAfterExit=yes', issued) self.assertEqual(issued[-4:], tuple(owner["volumes"])) + def test_owned_journal_policy_rejects_zero_ignored_missing_or_different_effective_values(self): + for output in ('', 'LogRateLimitIntervalUSec=0\nLogRateLimitBurst=0\n', + 'LogRateLimitIntervalUSec=30s\nLogRateLimitBurst=10000\n', + 'LogRateLimitIntervalUSec=1s\nLogRateLimitBurst=999999\n', + 'LogRateLimitBurst=1000000\n'): + with self.subTest(output=output), patch.object(fleet_node, 'command', + return_value=SimpleNamespace(stdout=output)) as command: + with self.assertRaisesRegex(ValueError, 'journal rate policy differs'): + fleet_node.journal_policy({'unit': 'owned.service'}) + self.assertEqual(command.call_count, 1) + self.assertEqual(command.call_args.args[:3], ('systemctl', 'show', 'owned.service')) + for interval in ('1s', '1000000'): + with self.subTest(interval=interval), patch.object(fleet_node, 'command', side_effect=[ + SimpleNamespace(stdout=f'LogRateLimitIntervalUSec={interval}\nLogRateLimitBurst=1000000\n'), + SimpleNamespace(stdout='systemd 249 (249.11)\n')]): + self.assertEqual(fleet_node.journal_policy({'unit': 'owned.service'})['rateIntervalSeconds'], 1) + for version in ('', 'unknown daemon\n'): + with self.subTest(version=version), patch.object(fleet_node, 'command', side_effect=[ + SimpleNamespace(stdout='LogRateLimitIntervalUSec=1s\nLogRateLimitBurst=1000000\n'), + SimpleNamespace(stdout=version)]): + with self.assertRaisesRegex(ValueError, 'systemd version is unavailable'): + fleet_node.journal_policy({'unit': 'owned.service'}) + def test_bounded_journal_capacity_refuses_arbitrary_limits_paths_or_process_birth(self): owner = {"runtimeProfile": "journal-bounded"} identity = {field: field for field in fleet_node.PROCESS_FIELDS}; identity['pid'] = 50 From 0eec80ef2aa40c2619400f867cba2bbc42fe8328 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 10:09:41 +0800 Subject: [PATCH 257/343] fix(validation): preserve frozen multipart middle part size --- scripts/validation_heal/race.py | 2 +- test_validation_heal_race.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index 8d4c897c..715f101c 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -65,7 +65,7 @@ G06_TARGET_KEY = "actual/slash/0256.bin" G06_REFERENCE_KEY = "space key/0000.bin" G05_TARGET_KEY = "multipart/target.bin" -G05_PART_SIZES = (5 * 1024 * 1024 + 17, 1024 * 1024 + 93, 1024 * 1024 + 47) +G05_PART_SIZES = (5 * 1024 * 1024 + 17, 5 * 1024 * 1024 + 93, 1024 * 1024 + 47) REFERENCE = { "candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", "sourceFiles": { diff --git a/test_validation_heal_race.py b/test_validation_heal_race.py index 05302a86..57b8d89b 100644 --- a/test_validation_heal_race.py +++ b/test_validation_heal_race.py @@ -48,6 +48,9 @@ def test_g06_keeps_copy_metadata_and_tag_operations_distinct(self): self.assertEqual(race.reference_key("copy-source"), race.G06_REFERENCE_KEY) def test_g05_binds_multipart_operations_to_acknowledged_upload(self): + # The frozen DM fixture requires two nonfinal parts above 5 MiB. + self.assertEqual(race.G05_PART_SIZES, + (5 * 1024 * 1024 + 17, 5 * 1024 * 1024 + 93, 1024 * 1024 + 47)) expected = [{"key": race.G05_TARGET_KEY, "versionId": "null", "isLatest": True}] continued = race.operation_plan("rv-heal-0123456789abcdef", "upload-continue", expected) completed = race.operation_plan("rv-heal-0123456789abcdef", "complete", expected) From ce92ba8d76a62d9c3569ca8d5a12cf632b0c3ed3 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 10:16:39 +0800 Subject: [PATCH 258/343] fix(validation): verify marker and batch-delete physical transitions --- scripts/validation_heal/race.py | 39 +++++++--- test_validation_heal_race_physical.py | 103 ++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 9 deletions(-) create mode 100644 test_validation_heal_race_physical.py diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index 715f101c..ffb253cb 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -455,15 +455,6 @@ def final_inventory(baseline, branch, plan, responses): require(responses[0].get("VersionId") in {None, "null"} and responses[0].get("DeleteMarker") in {None, False}, "G04 unversioned delete ACK changed object identity") result["entries"] = [row for row in result["entries"] if row["key"] != TARGET_KEY] - elif branch in G04_BRANCHES: - if branch == "unversioned-delete": - require(len(original_target) == 1 and not current_target_values, - "G04 unversioned DELETE retained or replaced the committed target") - else: - require(len(original_target) >= 1 and len(current_target_values) >= 1, - "G04 marker operation lost the retained data generation") - require(current_target_values[0]["dataDir"] in {target["dataDir"] for target in original_target}, - "G04 marker operation changed the committed data generation") elif branch in G05_BRANCHES: runtime = plan[0].get("runtime") or {} upload_id = runtime.get("uploadIds", {}).get(branch.split("-", 1)[-1]) @@ -561,6 +552,36 @@ def physical_transition(before, after, bucket, mapped, final, branch): require(all(any(part["size"] == SIZE for part in target["cohort"]) for target in current_target_values if target["dataDir"] in new_dirs), "G02 new version physical parts are incomplete") + elif branch in G04_BRANCHES: + require(len(original_target) == (1 if branch == "unversioned-delete" else 3), + "G04 baseline has the wrong acknowledged data-version count") + old_versions = {target["versionId"] for target in original_target} + remaining = {target["versionId"] for target in current_target_values} + expected_remaining = (set() if branch == "unversioned-delete" else + {original_target[-1]["versionId"]} if branch == "batch-version-delete" else old_versions) + require(remaining == expected_remaining, + "G04 operation lost, replaced or retained the wrong acknowledged data versions") + for target in current_target_values: + original = mapped[current_target, target["versionId"]] + require({name: value for name, value in target["logical"].items() if name != "isLatest"} + == {name: value for name, value in original["logical"].items() if name != "isLatest"} + and [(part["index"], part["sha256"], part["size"]) for part in target["cohort"]] + == [(part["index"], part["sha256"], part["size"]) for part in original["cohort"]], + "G04 marker/delete changed another retained data generation or shard") + if current_target_values: + expected_versions = sorted((row["versionId"], row["isLatest"], row["deleteMarker"]) + for row in final["entries"] if row["key"] == current_target) + indexed = files(after) + for node, value in after.items(): + for drive in value["formats"]: + metadata = indexed[node, drive["disk"], f"{bucket}/{current_target}/xl.meta"]["metadata"] + versions = metadata.get("versions", []) + require(all(type(version.get("isLatest")) is bool and type(version.get("deleted")) is bool + for version in versions), "G04 physical version flags are missing or untyped") + actual_versions = sorted((public_version_id(version.get("versionId")), + version["isLatest"], version["deleted"]) for version in versions) + require(actual_versions == expected_versions, + "G04 physical data/marker version set differs from exact foreground ACKs") elif branch in G05_BRANCHES: require(len(original_target) == len(current_target_values) == 1, "G05 multipart transition changed the target identity count") diff --git a/test_validation_heal_race_physical.py b/test_validation_heal_race_physical.py new file mode 100644 index 00000000..5f34921e --- /dev/null +++ b/test_validation_heal_race_physical.py @@ -0,0 +1,103 @@ +"""Exercise the real G04 physical transition oracle with decoded disk scenes.""" +import copy +import unittest +import uuid + +import test_validation_heal as physical_fixtures +from scripts.validation_heal import race + + +class MarkerPhysicalTests(unittest.TestCase): + bucket = "rv-heal-0123456789abcdef" + + def scene(self, branch): + before = physical_fixtures.HealShardOracleTest().snapshot() + count = 1 if branch == "unversioned-delete" else 3 + expected = [{"key": race.TARGET_KEY, "versionId": "null" if count == 1 else str(uuid.UUID(int=i + 1)), + "isLatest": i == count - 1, "deleteMarker": False} for i in range(count)] + for value in before.values(): + original = value["files"] + value["files"] = [] + for meta, part in zip(original[::3], original[1::3], strict=True): + meta = copy.deepcopy(meta) + meta["path"] = f"{self.bucket}/{race.TARGET_KEY}/xl.meta" + metadata = meta["metadata"] + metadata["key"] = race.TARGET_KEY + template = metadata["versions"][0] + metadata["versions"] = [] + for i, row in enumerate(expected): + version = copy.deepcopy(template) + version.update(versionId=None if count == 1 else row["versionId"], + dataDir=str(uuid.UUID(int=100 + i)), isLatest=row["isLatest"]) + metadata["versions"].append(version) + value["files"].append({**part, "path": f"{self.bucket}/{race.TARGET_KEY}/{version['dataDir']}/part.1", + "sha256": part["sha256"] + f"-version-{i}"}) + value["files"].append(meta) + marker = {"key": race.TARGET_KEY, "versionId": str(uuid.UUID(int=99)), + "isLatest": True, "deleteMarker": True} + if branch == "delete-marker": + for row in expected: row["isLatest"] = False + expected.append(marker) + for value in before.values(): + for f in value["files"]: + if not f["path"].endswith("xl.meta"): continue + for v in f["metadata"]["versions"]: v["isLatest"] = False + f["metadata"]["versions"].append({"versionId": marker["versionId"], "deleted": True, "isLatest": True}) + mapped = race.map_fixture(before, self.bucket, expected) + final = copy.deepcopy(expected) + if branch == "create-marker": + for row in final: row["isLatest"] = False + final.append(marker) + elif branch == "delete-marker": + final.pop() + final[-1]["isLatest"] = True + elif branch == "batch-version-delete": final = final[-1:] + else: final = [] + after = copy.deepcopy(before) + retained = {r["versionId"] for r in final if not r["deleteMarker"]} + keep_dirs = {v["dataDir"] for v in mapped.values() if v["versionId"] in retained} + for value in after.values(): + if not final: + value["files"] = [] + continue + value["files"] = [f for f in value["files"] if f["path"].endswith("xl.meta") + or f["path"].rsplit("/", 2)[1] in keep_dirs] + for f in value["files"]: + if not f["path"].endswith("xl.meta"): continue + old = {v["versionId"] or "null": v for v in f["metadata"]["versions"]} + f["metadata"]["versions"] = [ + {**old[r["versionId"]], "isLatest": r["isLatest"]} if not r["deleteMarker"] else + {"versionId": r["versionId"], "isLatest": r["isLatest"], "deleted": True} + for r in final] + return before, after, mapped, {"entries": final} + + def test_real_g04_transition_accepts_markers_batch_delete_and_unversioned_delete(self): + for branch in race.G04_BRANCHES: + with self.subTest(branch=branch): + before, after, mapped, final = self.scene(branch) + survivors, proof = race.physical_transition(before, after, self.bucket, mapped, final, branch) + self.assertEqual(len(survivors), {"create-marker": 3, "delete-marker": 3, + "batch-version-delete": 1, "unversioned-delete": 0}[branch]) + self.assertTrue(proof["physicalProofBeforeGETOrRescue"]) + + def test_g04_rejects_retained_part_drift_marker_drift_and_deleted_generation_resurrection(self): + for branch in race.G04_BRANCHES[:3]: + for fault in ("part", "data-dir", "duplicate-version", "unknown-marker", "marker-latest", "old-directory"): + with self.subTest(branch=branch, fault=fault): + before, after, mapped, final = self.scene(branch) + value = after["node1"] + meta = next(f for f in value["files"] if f["path"].endswith("xl.meta")) + part = next(f for f in value["files"] if f["path"].endswith("part.1")) + if fault == "part": part["sha256"] = "changed" + elif fault == "data-dir": meta["metadata"]["versions"][0]["dataDir"] = str(uuid.UUID(int=999)) + elif fault == "duplicate-version": meta["metadata"]["versions"].append(copy.deepcopy(meta["metadata"]["versions"][0])) + elif fault == "unknown-marker": + meta["metadata"]["versions"].append({"versionId": str(uuid.UUID(int=888)), "deleted": True, "isLatest": False}) + elif fault == "marker-latest": meta["metadata"]["versions"][0]["isLatest"] = not meta["metadata"]["versions"][0]["isLatest"] + else: + value["files"].append({**part, "path": f"{self.bucket}/{race.TARGET_KEY}/{uuid.UUID(int=999)}/part.1"}) + with self.assertRaises(ValueError): race.physical_transition(before, after, self.bucket, mapped, final, branch) + + +if __name__ == "__main__": + unittest.main() From 10e55719a0dc8f7213ef7e7fa1482dff37030e94 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 10:19:49 +0800 Subject: [PATCH 259/343] fix(validation): bind race fault IDs to registered case codes --- scripts/validation_heal/race.py | 3 +- test_validation_heal_race_physical.py | 45 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index ffb253cb..e7a5e846 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -872,8 +872,9 @@ def run_branch(args, topology, nodes, branch): if row["key"] != current_target_key or row["versionId"] == target_version["versionId"]] require(len(targets) == OBJECTS, "G01/G03 fault plan omitted a key or changed its source dimension") receipts = [] + case_code = select(args.case_id)[0]["code"] for index, target in enumerate(targets): - params = {"disk": target["disk"], "faultId": bounded_fault_id("G01/G03", branch, index)} + params = {"disk": target["disk"], "faultId": bounded_fault_id(case_code, branch, index)} fleet.faults.append((target["node"], params)) if target["key"] == current_target_key: changed_target_registered = target["node"], params receipts.append(fleet.record("exact-missing-part", {"node": target["node"], **fleet.remote( diff --git a/test_validation_heal_race_physical.py b/test_validation_heal_race_physical.py index 5f34921e..e24692ae 100644 --- a/test_validation_heal_race_physical.py +++ b/test_validation_heal_race_physical.py @@ -1,6 +1,12 @@ """Exercise the real G04 physical transition oracle with decoded disk scenes.""" import copy +from contextlib import ExitStack +from pathlib import Path +import re +import tempfile +from types import SimpleNamespace import unittest +from unittest.mock import Mock, patch import uuid import test_validation_heal as physical_fixtures @@ -99,5 +105,44 @@ def test_g04_rejects_retained_part_drift_marker_drift_and_deleted_generation_res with self.assertRaises(ValueError): race.physical_transition(before, after, self.bucket, mapped, final, branch) +class RaceFaultAdmissionTests(unittest.TestCase): + def test_real_runner_uses_each_registered_case_code_before_first_fault(self): + class FaultBoundaryReached(BaseException): + pass + for case_id, branches in race.CASE_BRANCHES.items(): + with self.subTest(case_id=case_id), tempfile.TemporaryDirectory() as folder, ExitStack() as stack: + root = Path(folder) + candidate = root / "candidate.json" + candidate.write_text('{"source_sha":"072b57b37559fbe626aeda8abf61617fdcbdd926","package_sha256":"digest"}') + rc = root / "rc" + rc.write_bytes(b"fixture-client") + branch = branches[0] + target_key = race.target_key(branch) + expected = [{"key": key, "versionId": "null", "isLatest": True} + for key in ("control/a", "control/b", target_key)] + mapped = {(row["key"], "null"): {**row, "disk": 0, "node": "vm000", "relativePath": row["key"], + "sha256": "original"} for row in expected} + fleet = Mock() + fleet.record.side_effect = lambda label, value: value + fleet.snapshot.return_value = {"vm000": {"files": []}} + fleet.cleanup.return_value = True + def remote(node, action, **params): + self.assertEqual(action, "apply-fault") + self.assertTrue(re.fullmatch(r"G0[1-6]-[0-9a-f]{32}", params["faultId"])) + self.assertTrue(params["faultId"].startswith(race.select(case_id)[0]["code"] + "-")) + raise FaultBoundaryReached() + fleet.remote.side_effect = remote + for name, value in {"Fleet": Mock(return_value=fleet), "verify_inspector": Mock(return_value={}), + "validate_topology": Mock(return_value={}), "Probe": Mock(), + "seed_fixture": Mock(return_value=expected), "verify_seed": Mock(), + "operation_plan": Mock(return_value=[]), "map_fixture": Mock(return_value=mapped), + "verify_part_manifest": Mock(), "OBJECTS": 3}.items(): + stack.enter_context(patch.object(race, name, value)) + stack.enter_context(patch.dict(race.os.environ, {"GITHUB_RUN_ID": "12345", "GITHUB_RUN_ATTEMPT": "1"})) + args = SimpleNamespace(evidence=root / "evidence", candidate=candidate, rc_bin=rc, + tool_directory=root, testing_commit="helper", binary_sha="binary", case_id=case_id) + with self.assertRaises(FaultBoundaryReached): race.run_branch(args, "H1", ["vm000"], branch) + + if __name__ == "__main__": unittest.main() From ea894f4deae486ca5a1d487cef011e0d5a148c0c Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 10:25:42 +0800 Subject: [PATCH 260/343] test(validation): migrate same-key multipart fixture and native contract --- docs/validation-heal-operations.md | 10 + scripts/validation_heal/mpu_race_fixture.py | 200 +++++++++++++++++ .../race_native_contracts.json | 201 +++++++++++++++++ test_validation_heal_mpu_race_fixture.py | 212 ++++++++++++++++++ test_validation_heal_race_native.py | 26 +++ 5 files changed, 649 insertions(+) create mode 100644 scripts/validation_heal/mpu_race_fixture.py create mode 100644 test_validation_heal_mpu_race_fixture.py diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 85ab4d29..5a8fe2a1 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -672,6 +672,16 @@ Runtime sources opt in to specific cases separately from the native registry. Th All original H1/H2 black-box branches, SDK ACKs, manifests, overlap witnesses, exact Admin settings and physical/S3 checks remain required. Native logical disk shapes do not certify distributed VM execution. The original `072b57b37559fbe626aeda8abf61617fdcbdd926` route retains its blocked native rows. A09, G05/G06 and I05 still require their own complete native groups and executor evidence. G04 also remains unaccepted until its same-source hosted evidence and every original live branch pass independent audit. No source or case gains canonical acceptance from a registry entry or local native test; a successful exact-source hosted bundle, every live branch and independent original-evidence audit are required. +### G05 same-key multipart components + +`mpu_race_fixture.py` migrates the frozen validation repository's `heal_mpu_race_fixture.py` model at `524eb5b32c3236497fdd8a9271df0494fa3d9974`. It uses the existing auto-testing `Recorder` and probe directly. Imports are local; no source validation checkout is needed at runtime. Part sizes follow the frozen common DM recipe: `5MiB+17`, `5MiB+93`, and `1MiB+47`. + +The caller supplies a fresh versioned owned bucket. One completed three-part object and two distinct pending uploads share `objects-0256.bin`: A initially has parts 1/2, B has parts 1/2/3. The model binds each mutation to its actual UploadId/part ETags and durable request/ACK files. A continues with part 3 and completes; B aborts. Unknown or malformed mutation results poison the model and prevent later sibling changes. Bounded paginated listings, exact terminal `404/NoSuchUpload`, all acknowledged historical/current versions and complete GET payloads verify sibling isolation. The model does not clean up uploads with uncertain outcomes. + +Immutable native source `09fb813a003e7964bd1b893efb7446a3c41bd3bf` adds `multipart-continue-complete-abort-final-rename`: ten ordered milestones across 4/2 and 16/4 logical geometries, using real three-part commits, same-key A/B pending parts and a paused shared Deep Heal. Completion tails drain before stale Heal release; exact retry disposition, old/sibling preservation, new committed bytes and upload retirement are asserted. The native test's unversioned logical scope is supplemental to the full versioned SDK/fleet case. + +Its native registry entry permits collecting this evidence only. G05 runtime admission remains closed until the executor integrates the complete same-key model, all pending/committed physical part witnesses, real public overlap and both original H1/H2 instances. Component tests and native bundles do not add canonical PASS. + ### J02 original opaque metadata probe The `unknown-negative` diagnostic includes the source procedure before the supplemental legacy-object diagnostic. A separate acknowledged object first receives an independently healthy Deep receipt. With the owned services stopped, the existing descriptor-relative fault backend flips byte zero of every original `xl.meta` copy: four copies in H1 and sixteen in H2. Each mutation has its own original backup, receipt and complete physical delta; the candidate-bound decoder must refuse every exact damaged copy. No other metadata, part or format may change. diff --git a/scripts/validation_heal/mpu_race_fixture.py b/scripts/validation_heal/mpu_race_fixture.py new file mode 100644 index 00000000..1d36761c --- /dev/null +++ b/scripts/validation_heal/mpu_race_fixture.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""G05 ACK-bound same-key multipart fixture; no topology or Heal PASS claim. + +One previously committed MPU plus two independent uploads for the same key. +Caller owns the fresh versioned bucket, endpoint, physical witnesses and race. +""" +import copy +import hashlib +import json +import re +import threading +import uuid + +try: + from .heal_probe import require, save +except ImportError: + from heal_probe import require, save + +# Match the frozen common DM recipe while retaining its same-key A/B model. +PART_SIZES=(5*1024**2+17,5*1024**2+93,1024**2+47) +ROLES=('committed','a','b') +KEY='objects-0256.bin' + + +def payload(bucket,role,number): + require(role in ROLES and type(number) is int and 1<=number<=3,'registered multipart payload required') + return hashlib.shake_256(f'G05:{bucket}:{role}:{number}'.encode()).digest(PART_SIZES[number-1]) + + +def etag(body):return '"'+hashlib.md5(body,usedforsecurity=False).hexdigest()+'"' + + +def multipart_etag(parts): + require([p['number'] for p in parts]==[1,2,3],'exact ordered three-part ACK manifest required') + require(all(re.fullmatch(r'"[0-9a-f]{32}"',p['etag']) for p in parts),'plain fixture part MD5 required') + raw=b''.join(bytes.fromhex(p['etag'][1:-1]) for p in parts) + return '"'+hashlib.md5(raw,usedforsecurity=False).hexdigest()+'-3"' + + +def pages(recorder,operation,params): + markers={'list_parts':('PartNumberMarker','NextPartNumberMarker'), + 'list_multipart_uploads':('KeyMarker','NextKeyMarker','UploadIdMarker','NextUploadIdMarker'), + 'list_object_versions':('KeyMarker','NextKeyMarker','VersionIdMarker','NextVersionIdMarker')} + require(operation in markers,'only bounded owned multipart/version listings allowed') + request=copy.deepcopy(params);seen=set();out=[] + for _ in range(32): + value=recorder.call(operation,request);out.append(value) + require(type(value.get('IsTruncated')) is bool,'listing omitted typed truncation flag') + if not value['IsTruncated']:return out + fields=markers[operation];next_values={fields[i]:value.get(fields[i+1]) for i in range(0,len(fields),2)} + if operation=='list_parts': + require(type(next_values['PartNumberMarker']) is int and next_values['PartNumberMarker']>request.get('PartNumberMarker',0), + 'part pagination did not advance') + else:require(all(isinstance(v,str) and v for v in next_values.values()),'listing omitted continuation identity') + identity=json.dumps(next_values,sort_keys=True) + require(identity not in seen,'listing repeated a continuation cursor');seen.add(identity) + request.update(next_values) + raise ValueError('owned listing exceeds 32-page budget') + + +class MpuRaceFixture: + def __init__(self,recorder,bucket,key=KEY): + require(key==KEY and isinstance(bucket,str) and re.fullmatch('[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]',bucket), + 'fixed G05 key and owned fixture bucket required') + self.recorder=recorder;self.bucket=bucket;self.key=key + self.items={};self.events=[];self.poisoned=None;self.lock=threading.RLock();self.seed_started=False + + def manifest(self): + with self.lock:return copy.deepcopy(dict(bucket=self.bucket,key=self.key,items=self.items,events=self.events, + poisoned=self.poisoned,fixture='DM',topologyProved=False,physicalRepairProved=False,overlapProved=False, + casePassClaimed=False,matrix84PassClaimed=False)) + + def _persist(self): + (getattr(self.recorder,'writer',None) or save)(self.recorder.output/f'mpu-manifest-{len(self.events):04d}.json',self.manifest()) + + def _mutation(self,role,operation,params,apply,body=None): + require(self.poisoned is None,'multipart fixture halted; do not retry or mutate sibling upload') + response=None + try: + response=self.recorder.call(operation,params,body,identity={'mpuRole':role}, + expect_http=204 if operation=='abort_multipart_upload' else 200) + apply(response) + except BaseException as exc: + self.poisoned=f'{type(exc).__name__}: {exc}' + self.events.append(dict(role=role,operation=operation,params=copy.deepcopy(params), + state='UNRESOLVED',responseReceived=response is not None,error=self.poisoned)) + self._persist();raise + self.events.append(dict(role=role,operation=operation,params=copy.deepcopy(params),state='ACK',response=copy.deepcopy(response))) + try:self._persist() + except BaseException as exc: + self.poisoned=f'manifest persistence failed: {exc}';raise + return response + + def create(self,role): + with self.lock: + require(len(self.items)<3 and role==ROLES[len(self.items)],'unknown/repeated/out-of-order fixture upload role') + if role!='committed':require(self.items['committed']['state']=='complete','original committed object must exist first') + metadata={'case':'G05','role':role,'nonce':hashlib.sha256(f'{self.bucket}:{role}'.encode()).hexdigest()[:24]} + def apply(response): + upload=response.get('UploadId') + require(isinstance(upload,str) and 1<=len(upload)<=256 and + upload not in {v['uploadId'] for v in self.items.values()},'missing/reused UploadId') + self.items[role]=dict(role=role,uploadId=upload,state='pending',parts=[],metadata=metadata) + return self._mutation(role,'create_multipart_upload',{'Bucket':self.bucket,'Key':self.key, + 'Metadata':metadata,'ContentType':'application/octet-stream'},apply) + + def upload(self,role,number): + with self.lock: + item=self.items[role] + require(item['state']=='pending' and type(number) is int and number==len(item['parts'])+1 and number<=3, + 'only next known pending part can be uploaded; no retry/overwrite') + body=payload(self.bucket,role,number) + def apply(response): + require(response.get('ETag')==etag(body),'UploadPart ACK differs from exact submitted bytes') + item['parts'].append(dict(number=number,size=len(body),etag=response['ETag'],sha256=hashlib.sha256(body).hexdigest())) + return self._mutation(role,'upload_part',{'Bucket':self.bucket,'Key':self.key,'UploadId':item['uploadId'], + 'PartNumber':number},apply,body) + + def complete(self,role): + with self.lock: + require(role in ('committed','a'),'B is the independently aborted upload') + item=self.items[role];require(item['state']=='pending','upload is already terminal') + if role=='committed':require(set(self.items)=={'committed'},'original MPU must complete before pending pair') + expected=multipart_etag(item['parts']) + def apply(response): + vid=response.get('VersionId') + require(isinstance(vid,str) and str(uuid.UUID(vid))==vid and uuid.UUID(vid).int!=0 and + vid not in {v.get('versionId') for v in self.items.values()},'missing/reused committed VersionId') + require(response.get('ETag')==expected,'Complete ACK multipart ETag differs from ordered ACK parts') + body=b''.join(payload(self.bucket,role,n) for n in (1,2,3)) + item.update(state='complete',versionId=vid,etag=expected,payload={'size':len(body),'sha256':hashlib.sha256(body).hexdigest()}) + return self._mutation(role,'complete_multipart_upload',{'Bucket':self.bucket,'Key':self.key, + 'UploadId':item['uploadId'],'MultipartUpload':{'Parts':[{'PartNumber':p['number'],'ETag':p['etag']} for p in item['parts']]}},apply) + + def abort_b(self): + with self.lock: + item=self.items['b'];require(item['state']=='pending','B is already terminal') + return self._mutation('b','abort_multipart_upload',{'Bucket':self.bucket,'Key':self.key,'UploadId':item['uploadId']}, + lambda response:item.update(state='aborted')) + + def seed(self): + with self.lock: + require(not self.seed_started and not self.items,'fresh fixture required');self.seed_started=True + self.create('committed') + for number in (1,2,3):self.upload('committed',number) + self.complete('committed') + self.create('a') + for number in (1,2):self.upload('a',number) + self.create('b') + for number in (1,2,3):self.upload('b',number) + return self.manifest() + + def verify_s3(self,recorder): + """Verify one endpoint; parent repeats for required topology endpoints. + + This deliberately requires current/old object identity while A/B are + pending, unlike merely checking HEAD404 on separate unused keys. + """ + with self.lock: + require(self.poisoned is None and set(self.items)==set(ROLES),'complete known fixture model required') + items=copy.deepcopy(self.items) + uploads=[v for page in pages(recorder,'list_multipart_uploads',{'Bucket':self.bucket,'Prefix':self.key,'MaxUploads':1}) + for v in page.get('Uploads',[])] + require(all(v.get('Key')==self.key for v in uploads) and len({v['UploadId'] for v in uploads})==len(uploads) and + {v['UploadId'] for v in uploads}=={v['uploadId'] for v in items.values() if v['state']=='pending'}, + 'pending UploadId population differs; sibling upload affected') + for item in items.values(): + params={'Bucket':self.bucket,'Key':self.key,'UploadId':item['uploadId'],'MaxParts':1} + if item['state']=='pending': + parts=[p for page in pages(recorder,'list_parts',params) for p in page.get('Parts',[])] + require([{'number':p['PartNumber'],'etag':p['ETag'],'size':p['Size']} for p in parts]== + [{k:p[k] for k in ('number','etag','size')} for p in item['parts']], 'pending part order/ETag/length differs') + else: + response=recorder.call('list_parts',params,expect_http=404) + require(response.get('Error',{}).get('Code')=='NoSuchUpload','terminal UploadId failed for wrong reason') + committed=[v for v in items.values() if v['state']=='complete'] + latest=next(v for v in reversed(list(items.values())) if v['state']=='complete') + listed=pages(recorder,'list_object_versions',{'Bucket':self.bucket,'Prefix':self.key,'MaxKeys':1}) + versions=[v for page in listed for v in page.get('Versions',[])] + require(not any(page.get('DeleteMarkers') for page in listed) and len(versions)==len(committed) and + all(v.get('Key')==self.key for v in versions) and + {v['VersionId'] for v in versions}=={v['versionId'] for v in committed}, + 'Complete duplicated/lost a version or pending upload became visible') + for row in versions: + item=next(v for v in committed if v['versionId']==row['VersionId']) + require(row.get('IsLatest') is (item['role']==latest['role']) and row.get('ETag')==item['etag'] and + row.get('Size')==item['payload']['size'],'committed version order/ETag/size differs') + for item,selector in [(v,{'VersionId':v['versionId']}) for v in committed]+[(latest,{})]: + params={'Bucket':self.bucket,'Key':self.key,**selector} + expected={'VersionId':item['versionId'],'ETag':item['etag'],'Metadata':item['metadata'], + 'ContentLength':item['payload']['size'],'ContentType':'application/octet-stream'} + head=recorder.call('head_object',params) + require(all(head.get(k)==v for k,v in expected.items()),'HEAD changed committed/current object') + recorder.call('get_object',params) + observed=json.loads((recorder.output/f'{recorder.sequence:06d}-ack.json').read_text())['observation'] + require(observed.get('params')==params and observed.get('payload')==item['payload'] and + all(observed['response'].get(k)==v for k,v in expected.items()),'GET committed/current bytes or properties differ') + with self.lock:require(self.poisoned is None and self.items==items,'multipart model changed during verification') + return dict(committedVersions=len(committed),pendingUploads=len(uploads),sameKeySiblingIsolationVerified=True, + physicalRepairProved=False,overlapProved=False,casePassClaimed=False) diff --git a/scripts/validation_heal/race_native_contracts.json b/scripts/validation_heal/race_native_contracts.json index 9e0c66a3..eedce4cd 100644 --- a/scripts/validation_heal/race_native_contracts.json +++ b/scripts/validation_heal/race_native_contracts.json @@ -394,6 +394,207 @@ "marker-and-batch-delete-final-rename" ] } + }, + "09fb813a003e7964bd1b893efb7446a3c41bd3bf": { + "sourceTree": "754738c3500a70ecd560994a414343072094caf9", + "cargoLockSha256": "b4d959f964c13d8ed507657fa4b52752769f96e5cce932845d3d8cec69a5cd56", + "sourceFiles": { + "crates/ecstore/src/set_disk/ops/object.rs": "d033df47054f054413c007d5e5b94212676411fbb3670403e08358f9152ef443", + "crates/ecstore/src/set_disk/ops/heal.rs": "381b63e6206fee996260e2d0a213114140957a5fb4fc26feb0f4533d2be9ec68", + "crates/ecstore/src/set_disk/core/io_primitives.rs": "6e50c8ea8b374b8a2d73da740321af25fed6e26c34e5117feca3e5f5891782ab" + }, + "regressions": { + "committed-overwrite-cleanup": { + "test": "set_disk::ops::object::put_object_tmp_cleanup_tests::deep_heal_preserves_committed_generation_during_old_data_cleanup", + "sourcePath": "crates/ecstore/src/set_disk/ops/object.rs", + "sourceAssertions": [ + "Every actual shuffled cleanup slot pauses after new physical generation commit while TailDrained overwrite ACK remains pending.", + "Real Deep and shared read-repair reconstruct a missing new-generation encoded part before GET in4/2 and16/4 geometries.", + "All old physical generations disappear after cleanup; every new encoded shard, unchanged metadata, overwrite ACK and complete final body remain exact." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g01: ", + "milestones": [ + "native_g01: geometry=4/2 read_repair=false old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=4/2 read_repair=false locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=4/2 read_repair=false overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=4/2 read_repair=true old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=4/2 read_repair=true locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=4/2 read_repair=true overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=16/4 read_repair=false old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=16/4 read_repair=false locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=16/4 read_repair=false overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=16/4 read_repair=true old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=16/4 read_repair=true locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=16/4 read_repair=true overwrite_ack_preserved final_physical_generation_and_full_body_verified" + ] + }, + "stale-read-repair-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::read_repair_commit_aborts_when_object_changes_before_final_rename", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Real missing external shard forces actual shared read-repair data reconstruction.", + "Pause immediately before final commit fence; a successor overwrite ACK precedes release.", + "Obsolete read-repair returns exact DiskError::Timeout and final current ETag equals acknowledged overwrite." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "versioned-read-repair-lock-owner": { + "test": "set_disk::ops::heal::heal_result_report_tests::object_lock_diagnostics_distinguish_read_repair_from_queued_put", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Historical missing external shard is repaired under actual shared Data-phase lock.", + "Queued PUT cannot acquire namespace and later historical GET times out while version-specific owner remains held.", + "After release, successor PUT ACK and every original/new version full body remain correct." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "cancelled-physical-unlink-owner": { + "test": "set_disk::ops::heal::heal_result_report_tests::cancelled_heal_backup_cleanup_retains_key_owner_until_unlink_finishes", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Actual external Heal backup unlink pauses in outer, inner and shared read-repair modes.", + "Caller abort does not retire namespace commit owner; successor write lock remains pending until physical unlink drains.", + "After physical completion, successor obtains lock, owned backup is absent and committed full body is preserved." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "explicit-version-append-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_explicit_version_commit_preserves_acknowledged_new_version", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Actual selected-version shared Deep reconstruction pauses before its final rename after releasing the read guard.", + "Foreground version mutation ACK and same-key physical write-tail drain precede releasing stale Heal; append succeeds or deleted version remains typed FileVersionNotFound.", + "All surviving acknowledged version UUIDs, data dirs, original encoded shards and complete exact/current bodies remain intact in4/2 and16/4; deleted generations do not reappear." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g02: ", + "milestones": [ + "native_g02: geometry=4/2 mutation=append reconstruction_paused_before_final_rename", + "native_g02: geometry=4/2 mutation=append foreground_ack_and_physical_tail_verified_before_release", + "native_g02: geometry=4/2 mutation=append exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g02: geometry=16/4 mutation=append reconstruction_paused_before_final_rename", + "native_g02: geometry=16/4 mutation=append foreground_ack_and_physical_tail_verified_before_release", + "native_g02: geometry=16/4 mutation=append exact_version_outcome_physical_generations_and_full_bodies_verified" + ] + }, + "explicit-version-delete-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_explicit_version_commit_refuses_deleted_historical_and_current_versions", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Actual selected-version shared Deep reconstruction pauses before its final rename after releasing the read guard.", + "Foreground version mutation ACK and same-key physical write-tail drain precede releasing stale Heal; append succeeds or deleted version remains typed FileVersionNotFound.", + "All surviving acknowledged version UUIDs, data dirs, original encoded shards and complete exact/current bodies remain intact in4/2 and16/4; deleted generations do not reappear." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g03: ", + "milestones": [ + "native_g03: geometry=4/2 mutation=delete-historical reconstruction_paused_before_final_rename", + "native_g03: geometry=4/2 mutation=delete-historical foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=4/2 mutation=delete-historical exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g03: geometry=16/4 mutation=delete-historical reconstruction_paused_before_final_rename", + "native_g03: geometry=16/4 mutation=delete-historical foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=16/4 mutation=delete-historical exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g03: geometry=4/2 mutation=delete-current reconstruction_paused_before_final_rename", + "native_g03: geometry=4/2 mutation=delete-current foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=4/2 mutation=delete-current exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g03: geometry=16/4 mutation=delete-current reconstruction_paused_before_final_rename", + "native_g03: geometry=16/4 mutation=delete-current foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=16/4 mutation=delete-current exact_version_outcome_physical_generations_and_full_bodies_verified" + ] + }, + "marker-and-batch-delete-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_marker_and_batch_delete_fence_preserves_acknowledged_version_set", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Real protected external-data reconstruction pauses before final metadata rename in4/2 and16/4 geometries for four independent marker/delete mutations.", + "Foreground create-marker, exact marker removal, two-item version deletion with exact per-item ACKs or unversioned deletion and same-key physical tail drain all precede releasing stale Heal.", + "Every member retains the exact acknowledged marker/data version set without duplicates; surviving encoded shards and full exact/current bodies remain unchanged, and deleted directories or markers cannot resurrect." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g04: ", + "milestones": [ + "native_g04: geometry=4/2 mutation=create-marker reconstruction_paused_before_final_rename", + "native_g04: geometry=4/2 mutation=create-marker exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=4/2 mutation=create-marker marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=4/2 mutation=delete-marker reconstruction_paused_before_final_rename", + "native_g04: geometry=4/2 mutation=delete-marker exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=4/2 mutation=delete-marker marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=4/2 mutation=batch-version-delete reconstruction_paused_before_final_rename", + "native_g04: geometry=4/2 mutation=batch-version-delete exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=4/2 mutation=batch-version-delete marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=4/2 mutation=unversioned-delete reconstruction_paused_before_final_rename", + "native_g04: geometry=4/2 mutation=unversioned-delete exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=4/2 mutation=unversioned-delete marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=16/4 mutation=create-marker reconstruction_paused_before_final_rename", + "native_g04: geometry=16/4 mutation=create-marker exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=16/4 mutation=create-marker marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=16/4 mutation=delete-marker reconstruction_paused_before_final_rename", + "native_g04: geometry=16/4 mutation=delete-marker exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=16/4 mutation=delete-marker marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=16/4 mutation=batch-version-delete reconstruction_paused_before_final_rename", + "native_g04: geometry=16/4 mutation=batch-version-delete exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=16/4 mutation=batch-version-delete marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=16/4 mutation=unversioned-delete reconstruction_paused_before_final_rename", + "native_g04: geometry=16/4 mutation=unversioned-delete exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=16/4 mutation=unversioned-delete marker_set_deleted_directories_surviving_shards_and_full_bodies_verified" + ] + }, + "multipart-continue-complete-abort-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_multipart_continue_complete_abort_fences_stale_heal", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Real three-part committed unversioned object and distinct same-key A/B pending uploads: A parts1/2, B parts1/2/3, independent payloads and ACK identities.", + "Actual missing committed part forces shared Deep Heal to pause before final rename in4/2 and16/4 geometries; A part3 and B Abort ACK preserve all other committed and A pending shard bytes.", + "A Complete ACK and namespace physical tail drain precede release; stale Heal returns exact DiskError::Timeout. All new metadata/parts remain byte exact, old generation and both staging directories absent, final full body equals ordered A ACK payloads.", + "Native logical/unversioned scope does not prove full versioned SDK H1/H2 overlap, protocol or fleet execution." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g05: ", + "milestones": [ + "native_g05: geometry=4/2 committed_multipart_and_two_distinct_pending_uploads_seeded", + "native_g05: geometry=4/2 real_committed_object_heal_paused_before_final_rename", + "native_g05: geometry=4/2 continue_ack_and_other_abort_preserve_pending_parts_and_original_commit", + "native_g05: geometry=4/2 complete_ack_and_all_physical_tails_verified_before_release", + "native_g05: geometry=4/2 stale_heal_rejected_exact_completed_parts_full_body_and_upload_retirement_verified", + "native_g05: geometry=16/4 committed_multipart_and_two_distinct_pending_uploads_seeded", + "native_g05: geometry=16/4 real_committed_object_heal_paused_before_final_rename", + "native_g05: geometry=16/4 continue_ack_and_other_abort_preserve_pending_parts_and_original_commit", + "native_g05: geometry=16/4 complete_ack_and_all_physical_tails_verified_before_release", + "native_g05: geometry=16/4 stale_heal_rejected_exact_completed_parts_full_body_and_upload_retirement_verified" + ] + } + }, + "cases": { + "heal.race-unversioned-overwrite": [ + "committed-overwrite-cleanup", + "stale-read-repair-final-rename" + ], + "heal.race-versioned-put": [ + "versioned-read-repair-lock-owner", + "explicit-version-append-final-rename" + ], + "heal.lock-timeout-and-caller-abort": [ + "versioned-read-repair-lock-owner", + "cancelled-physical-unlink-owner", + "stale-read-repair-final-rename" + ], + "heal.race-version-delete": [ + "explicit-version-delete-final-rename" + ], + "heal.race-markers-and-batch-delete": [ + "marker-and-batch-delete-final-rename" + ], + "heal.race-multipart-complete-abort": [ + "multipart-continue-complete-abort-final-rename" + ] + } } } } diff --git a/test_validation_heal_mpu_race_fixture.py b/test_validation_heal_mpu_race_fixture.py new file mode 100644 index 00000000..c4abfc9a --- /dev/null +++ b/test_validation_heal_mpu_race_fixture.py @@ -0,0 +1,212 @@ +import copy +import hashlib +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import Mock,patch +import uuid + +from scripts.validation_heal import mpu_race_fixture as m +from scripts.validation_heal.heal_workload import Recorder as DurableRecorder + + +class Recorder: + def __init__(self,folder): + self.output=Path(folder);self.sequence=0;self.uploads={};self.versions=[];self.calls=[];self.fail=None;self.corrupt=None + + def call(self,operation,params,body=None,**kwargs): + self.calls.append((operation,copy.deepcopy(params),kwargs));self.sequence+=1 + if self.fail==operation:raise ValueError('uncertain call; durable production Recorder owns raw receipt') + response={'ResponseMetadata':{'HTTPStatusCode':204 if operation=='abort_multipart_upload' else 200}} + if operation=='create_multipart_upload': + uid=f'upload-{len(self.uploads)+1}';self.uploads[uid]={'params':copy.deepcopy(params),'parts':{},'state':'pending'} + response['UploadId']=uid + elif operation=='upload_part': + self.uploads[params['UploadId']]['parts'][params['PartNumber']]=body;response['ETag']=m.etag(body) + elif operation=='complete_multipart_upload': + item=self.uploads[params['UploadId']];parts=[{'number':n,'etag':m.etag(b)} for n,b in sorted(item['parts'].items())] + item['state']='complete';raw=b''.join(v for _,v in sorted(item['parts'].items())) + version={'VersionId':str(uuid.UUID(int=len(self.versions)+1)),'ETag':m.multipart_etag(parts), + 'Metadata':item['params']['Metadata'],'ContentType':'application/octet-stream', + 'ContentLength':len(raw),'body':raw} + self.versions.append(version);response.update({k:version[k] for k in ('VersionId','ETag')}) + elif operation=='abort_multipart_upload':self.uploads[params['UploadId']]['state']='aborted' + elif operation=='list_multipart_uploads': + response.update(IsTruncated=False,Uploads=[{'Key':m.KEY,'UploadId':u} for u,v in self.uploads.items() if v['state']=='pending']) + elif operation=='list_parts': + item=self.uploads[params['UploadId']] + if item['state']=='pending':response.update(IsTruncated=False,Parts=[{'PartNumber':n,'ETag':m.etag(b),'Size':len(b)} for n,b in sorted(item['parts'].items())]) + else:response={'ResponseMetadata':{'HTTPStatusCode':404},'Error':{'Code':'NoSuchUpload'}} + elif operation=='list_object_versions': + response.update(IsTruncated=False,Versions=[{'Key':m.KEY,'VersionId':v['VersionId'],'ETag':v['ETag'], + 'Size':v['ContentLength'],'IsLatest':v is self.versions[-1]} for v in self.versions]) + elif operation in ('head_object','get_object'): + version=next(v for v in self.versions if v['VersionId']==params['VersionId']) if 'VersionId' in params else self.versions[-1] + response.update({k:v for k,v in version.items() if k!='body'}) + else:raise AssertionError(operation) + if self.corrupt:response=self.corrupt(operation,response) + observation={'params':copy.deepcopy(params),'response':copy.deepcopy(response)} + if operation=='get_object':observation['payload']={'size':len(version['body']),'sha256':hashlib.sha256(version['body']).hexdigest()} + m.save(self.output/f'{self.sequence:06d}-ack.json',{'observation':observation}) + return response + + +class MpuRaceFixtureTests(unittest.TestCase): + def setUp(self): + patcher=patch.object(m,'PART_SIZES',(12,13,14));patcher.start();self.addCleanup(patcher.stop) + tmp=tempfile.TemporaryDirectory();self.addCleanup(tmp.cleanup) + self.rec=Recorder(tmp.name);self.fixture=m.MpuRaceFixture(self.rec,'rv-g05-owned') + + def test_seed_same_key_original_and_two_distinct_pending_uploads(self): + value=self.fixture.seed();items=value['items'] + self.assertEqual([items[r]['state'] for r in m.ROLES],['complete','pending','pending']) + self.assertEqual([len(items[r]['parts']) for r in m.ROLES],[3,2,3]) + self.assertEqual(len({v['uploadId'] for v in items.values()}),3) + self.assertEqual({v[1]['Key'] for v in self.rec.calls},{m.KEY}) + self.assertEqual(len({items[r]['parts'][0]['etag'] for r in m.ROLES}),3) + self.assertFalse(value['casePassClaimed']);self.assertFalse(value['physicalRepairProved']) + self.assertEqual(self.fixture.verify_s3(self.rec)['pendingUploads'],2) + + def test_continue_complete_abort_preserve_sibling_and_old_committed_version_both_orders(self): + for abort_first in (False,True): + with self.subTest(abort_first=abort_first),tempfile.TemporaryDirectory() as tmp: + rec=Recorder(tmp);fixture=m.MpuRaceFixture(rec,'rv-g05-owned');fixture.seed() + if abort_first:fixture.abort_b();fixture.verify_s3(rec) + fixture.upload('a',3);fixture.verify_s3(rec) + fixture.complete('a');fixture.verify_s3(rec) + if not abort_first:fixture.abort_b() + proof=fixture.verify_s3(rec) + self.assertEqual(proof['committedVersions'],2);self.assertEqual(proof['pendingUploads'],0) + self.assertTrue(proof['sameKeySiblingIsolationVerified']);self.assertFalse(proof['overlapProved']) + abort=next(c for c in rec.calls if c[0]=='abort_multipart_upload') + self.assertEqual(abort[1]['UploadId'],fixture.items['b']['uploadId']);self.assertEqual(abort[2]['expect_http'],204) + + def test_unknown_mutation_halts_sibling_and_never_retries(self): + self.fixture.seed();self.rec.fail='upload_part' + with self.assertRaisesRegex(ValueError,'uncertain'):self.fixture.upload('a',3) + calls=len(self.rec.calls) + with self.assertRaisesRegex(ValueError,'halted'):self.fixture.abort_b() + self.assertEqual(len(self.rec.calls),calls);self.assertIsNotNone(self.fixture.poisoned) + self.assertEqual(self.fixture.items['a']['state'],'pending') + self.assertEqual(self.fixture.events[-1]['state'],'UNRESOLVED') + with self.assertRaises(ValueError):self.fixture.verify_s3(self.rec) + + def test_malformed_successful_ack_keeps_durable_failure_and_prevents_retry(self): + self.fixture.seed();self.fixture.upload('a',3) + self.rec.corrupt=lambda op,r:{**r,'ETag':'"wrong"'} if op=='complete_multipart_upload' else r + with self.assertRaisesRegex(ValueError,'multipart ETag'):self.fixture.complete('a') + self.assertTrue(self.fixture.events[-1]['responseReceived']) + self.assertEqual(self.fixture.items['a']['state'],'pending') + with self.assertRaisesRegex(ValueError,'halted'):self.fixture.abort_b() + + def test_out_of_order_duplicate_or_unprepared_calls_never_go_to_network(self): + with self.assertRaises(ValueError):self.fixture.create('b') + self.assertFalse(self.rec.calls);self.fixture.seed();count=len(self.rec.calls) + for method in (lambda:self.fixture.seed(),lambda:self.fixture.create('a'),lambda:self.fixture.upload('a',2), + lambda:self.fixture.upload('a',True),lambda:self.fixture.complete('a'),lambda:self.fixture.complete('b')): + with self.assertRaises(ValueError):method() + self.assertEqual(len(self.rec.calls),count) + + def test_wrong_part_ack_or_reused_upload_identity_is_not_accepted(self): + self.fixture.seed();self.rec.corrupt=lambda op,r:{**r,'ETag':'"'+'0'*32+'"'} if op=='upload_part' else r + with self.assertRaisesRegex(ValueError,'exact submitted'):self.fixture.upload('a',3) + self.assertEqual(len(self.fixture.items['a']['parts']),2) + with tempfile.TemporaryDirectory() as tmp: + rec=Recorder(tmp);fixture=m.MpuRaceFixture(rec,'rv-g05-owned') + rec.corrupt=lambda op,r:{**r,'UploadId':'same'} if op=='create_multipart_upload' else r + fixture.create('committed') + fixture.items['committed']['state']='complete' + with self.assertRaisesRegex(ValueError,'reused UploadId'):fixture.create('a') + + def test_s3_oracle_rejects_sibling_loss_part_corruption_and_version_resurrection(self): + self.fixture.seed() + for kind in ('upload','part','version','marker','head','get','terminal-error'): + def corrupt(op,r): + r=copy.deepcopy(r) + if kind=='upload' and op=='list_multipart_uploads':r['Uploads'].pop() + if kind=='part' and op=='list_parts' and r.get('Parts'):r['Parts'][0]['ETag']='wrong' + if kind=='version' and op=='list_object_versions':r['Versions'].append(copy.deepcopy(r['Versions'][0])) + if kind=='marker' and op=='list_object_versions':r['DeleteMarkers']=[{'Key':m.KEY}] + if kind=='head' and op=='head_object':r['Metadata']={} + if kind=='get' and op=='get_object':r['VersionId']=str(uuid.UUID(int=999)) + if kind=='terminal-error' and op=='list_parts' and 'Error' in r:r['Error']['Code']='NoSuchKey' + return r + self.rec.corrupt=corrupt + with self.subTest(kind=kind),self.assertRaises(ValueError):self.fixture.verify_s3(self.rec) + + def test_manifest_persistence_failure_stops_later_mutation(self): + self.fixture.seed() + with patch.object(self.fixture,'_persist',side_effect=OSError('full')): + with self.assertRaises(OSError):self.fixture.upload('a',3) + self.assertIsNotNone(self.fixture.poisoned) + count=len(self.rec.calls) + with self.assertRaises(ValueError):self.fixture.abort_b() + self.assertEqual(count,len(self.rec.calls)) + + def test_multipart_digest_requires_all_ordered_plain_parts(self): + parts=[{'number':n,'etag':m.etag(m.payload('rv-owned','a',n))} for n in (1,2,3)] + self.assertTrue(m.multipart_etag(parts).endswith('-3"')) + for bad in (parts[:2],parts[::-1],[{**p,'etag':'wrong'} for p in parts]): + with self.assertRaises(ValueError):m.multipart_etag(bad) + + def test_pagination_rejects_missing_typed_flags_missing_cursor_and_cycles(self): + recorder=Mock();recorder.call.return_value={} + with self.assertRaises(ValueError):m.pages(recorder,'list_parts',{}) + recorder.call.return_value={'IsTruncated':True,'NextPartNumberMarker':1} + with self.assertRaisesRegex(ValueError,'advance'):m.pages(recorder,'list_parts',{}) + recorder.call.return_value={'IsTruncated':True,'NextKeyMarker':m.KEY,'NextUploadIdMarker':'u'} + with self.assertRaisesRegex(ValueError,'repeated'):m.pages(recorder,'list_multipart_uploads',{}) + recorder.call.side_effect=[{'IsTruncated':True,'NextPartNumberMarker':1},{'IsTruncated':False}] + self.assertEqual(len(m.pages(recorder,'list_parts',{})),2) + + +class MigratedRecorderTests(unittest.TestCase): + def test_frozen_common_dm_sizes_are_retained(self): + self.assertEqual(m.PART_SIZES, (5 * 1024**2 + 17, 5 * 1024**2 + 93, 1024**2 + 47)) + + def test_model_uses_actual_auto_test_request_ack_ledger_and_halts_unknown_writes(self): + with tempfile.TemporaryDirectory() as folder, patch.object(m, 'PART_SIZES', (12, 13, 14)): + root = Path(folder) + backend_dir = root / 'backend' + backend_dir.mkdir() + backend = Recorder(backend_dir) + class Probe: + binding = {'runId': 'gh-12345-1-h1', 'bucketPrefix': 'rv-g05-owned', + 'maxInventoryBytes': 64 * 1024**2} + fail_upload = False + def s3(self, operation, params, body=None): + if self.fail_upload and operation == 'upload_part': + return {'operation': operation, 'params': copy.deepcopy(params), + 'acknowledged': False, 'resultUnknown': True, + 'error': {'type': 'TimeoutError', 'message': 'unknown write'}} + backend.call(operation, params, body) + observation = json.loads((backend.output / f'{backend.sequence:06d}-ack.json').read_text())['observation'] + status = observation['response']['ResponseMetadata']['HTTPStatusCode'] + return {**observation, 'operation': operation, 'acknowledged': 200 <= status < 300, + 'resultUnknown': False} + probe = Probe() + recorder = DurableRecorder(probe, root / 'durable') + fixture = m.MpuRaceFixture(recorder, 'rv-g05-owned') + fixture.seed() + self.assertTrue(fixture.verify_s3(recorder)['sameKeySiblingIsolationVerified']) + probe.fail_upload = True + with self.assertRaisesRegex(ValueError, 'unexpected upload_part'): + fixture.upload('a', 3) + sequence = recorder.sequence + with self.assertRaisesRegex(ValueError, 'halted'): + fixture.abort_b() + self.assertEqual(recorder.sequence, sequence) + ack = json.loads((recorder.output / f'{sequence:06d}-ack.json').read_text()) + request = recorder.output / ack['requestFile'] + self.assertEqual(ack['requestSha256'], hashlib.sha256(request.read_bytes()).hexdigest()) + self.assertEqual(ack['identity'], {'mpuRole': 'a'}) + self.assertEqual(ack['operationCheck'], 'FAIL') + self.assertTrue(ack['observation']['resultUnknown']) + self.assertEqual(fixture.items['b']['state'], 'pending') + self.assertEqual(fixture.items['committed']['state'], 'complete') + self.assertEqual(fixture.events[-1]['state'], 'UNRESOLVED') + + +if __name__=='__main__':unittest.main() diff --git a/test_validation_heal_race_native.py b/test_validation_heal_race_native.py index 144dc940..5ebe8f60 100644 --- a/test_validation_heal_race_native.py +++ b/test_validation_heal_race_native.py @@ -113,6 +113,32 @@ def test_marker_source_requires_the_complete_four_mutation_group_and_keeps_later with self.assertRaisesRegex(ValueError, 'omit or duplicate'): race.native_preflight(candidate, HELPER, case, root) + def test_multipart_native_source_collects_all_ten_events_without_enabling_incomplete_runtime(self): + source = '09fb813a003e7964bd1b893efb7446a3c41bd3bf' + candidate = {**CANDIDATE, 'source_sha': source} + case = 'heal.race-multipart-complete-abort' + label = 'multipart-continue-complete-abort-final-rename' + contract = suite.contract_for(source) + self.assertEqual(contract['cases'][case], [label]) + self.assertEqual(len(contract['regressions'][label]['milestones']), 10) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.fixture(root, source=source) + with patch.object(suite.subprocess, 'run', side_effect=AssertionError('must not execute artifacts')): + proof = consumer.verify(root, candidate, HELPER, case) + self.assertTrue(proof['requiredNativeSubsetPassed']) + self.assertFalse(proof['wholeCasePassClaimed']) + self.assertFalse(proof['canonicalCasePassClaimed']) + with self.assertRaisesRegex(ValueError, 'race candidate/case contract is not reviewed'): + race.native_preflight(candidate, HELPER, case, root) + summary = root / 'native-evidence/race-native-results.json' + value = json.loads(summary.read_text()) + row = next(row for row in value['results'] if row['label'] == label) + row['milestones'].pop() + self.write(summary, value) + with self.assertRaisesRegex(ValueError, 'invented or omitted'): + consumer.verify(root, candidate, HELPER, case) + def fixture(self, root, *, source=SOURCE): tools, evidence = root / 'native-tools', root / 'native-evidence' tools.mkdir() From f1ff47e492b133aecef09a991c41051e371f456a Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 10:38:59 +0800 Subject: [PATCH 261/343] test(validation): migrate owned multipart staging witnesses --- docs/validation-heal-operations.md | 2 + scripts/validation_heal/fleet_node.py | 1 + scripts/validation_heal/healthy.py | 3 + scripts/validation_heal/mpu_race_physical.py | 230 +++++++++++++++++++ test_validation_heal_mpu_race_physical.py | 161 +++++++++++++ 5 files changed, 397 insertions(+) create mode 100644 scripts/validation_heal/mpu_race_physical.py create mode 100644 test_validation_heal_mpu_race_physical.py diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 5a8fe2a1..938ead56 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -682,6 +682,8 @@ Immutable native source `09fb813a003e7964bd1b893efb7446a3c41bd3bf` adds `multipa Its native registry entry permits collecting this evidence only. G05 runtime admission remains closed until the executor integrates the complete same-key model, all pending/committed physical part witnesses, real public overlap and both original H1/H2 instances. Component tests and native bundles do not add canonical PASS. +The additive `mpu-race-scene` node action in `mpu_race_physical.py` adapts the frozen `heal_h1_mpu_physical.py` upload-ID and byte-continuity oracle to the current owned four-drive hosts. Each request binds the manual runtime, bucket, candidate binary and all three ACK-selected uploads. It inventories the complete fixed key prefix, hashes only selected staging files under bounded entry/depth/byte limits, rejects unknown uploads and links, and preserves original process identity during a live capture. Stopped captures additionally verify the owned service remains stopped. Across H1/H2, pending part/sidecar populations and existing sibling bytes remain exact; a completed upload must lose its staging directory and retain all three previously witnessed encoded parts in committed storage. These are staging/continuity witnesses; independent inspector coding/bitrot checks, product Heal proof, real overlap and full SDK inventories remain required. + ### J02 original opaque metadata probe The `unknown-negative` diagnostic includes the source procedure before the supplemental legacy-object diagnostic. A separate acknowledged object first receives an independently healthy Deep receipt. With the owned services stopped, the existing descriptor-relative fault backend flips byte zero of every original `xl.meta` copy: four copies in H1 and sixteen in H2. Each mutation has its own original backup, receipt and complete physical delta; the candidate-bound decoder must refuse every exact damaged copy. No other metadata, part or format may change. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index a36908cc..ef90d0e7 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1816,6 +1816,7 @@ def main(request): actions[action] = incarnation_operation actions["task-records"] = task_records actions["mpu-scene"] = lambda value: globals()["mpu_scene_backend"].run(value, globals()) + actions["mpu-race-scene"] = lambda value: globals()["mpu_race_backend"].run(value, globals()) actions["network-delete-records"] = lambda value: globals()["network_delete_records_backend"].run(value, globals()) actions["process-clock"] = process_clock actions["environment-capabilities"] = environment_capabilities diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index f8b97a9a..3038d268 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -185,6 +185,9 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() + if action == "mpu-race-scene": + source += "\nimport types\nmpu_race_backend = types.ModuleType('mpu_race_physical')\nexec(" + repr( + NODE_SOURCE.with_name("mpu_race_physical.py").read_text()) + ", mpu_race_backend.__dict__)\n" if action == "network-delete-records": source += "\nimport types\nnetwork_delete_records_backend = types.ModuleType('network_delete_records')\nexec(" + repr( NODE_SOURCE.with_name("network_delete_records.py").read_text()) + ", network_delete_records_backend.__dict__)\n" diff --git a/scripts/validation_heal/mpu_race_physical.py b/scripts/validation_heal/mpu_race_physical.py new file mode 100644 index 00000000..e7a45725 --- /dev/null +++ b/scripts/validation_heal/mpu_race_physical.py @@ -0,0 +1,230 @@ +"""ACK-selected G05 staging census and continuity; not a Heal/EC proof.""" +import base64 +import hashlib +import os +import re +import stat +import uuid + +KEY = 'objects-0256.bin' +ROLES = ('committed', 'a', 'b') +MAX_ENTRIES = 128 +MAX_BYTES = 256 * 1024**2 + + +def require(value, message): + if not value: + raise ValueError(message) + + +def selected(bucket, items): + """Retain the frozen H1 upload-ID decoder, bound to the auto-test owner.""" + require(re.fullmatch(r'rv-heal-[0-9a-f]{16}', bucket) and isinstance(items, list) and len(items) == 3, + 'G05 requires one owned bucket and all three ACK-selected uploads') + out, roles, uploads = [], set(), set() + for item in items: + require(set(item) == {'role', 'uploadId', 'bucket', 'key'} and item['role'] in ROLES + and item['role'] not in roles and item['bucket'] == bucket and item['key'] == KEY, + 'foreign or duplicate G05 upload selection') + encoded = item['uploadId'] + require(isinstance(encoded, str) and re.fullmatch(r'[A-Za-z0-9_-]{1,256}', encoded) + and encoded not in uploads, 'invalid or reused G05 upload identity') + raw = base64.b64decode(encoded + '=' * ((-len(encoded)) % 4), altchars=b'-_', validate=True) + require(base64.urlsafe_b64encode(raw).decode().rstrip('=') == encoded, 'noncanonical upload encoding') + deployment, suffix = raw.decode().split('.', 1) + require(str(uuid.UUID(deployment)) == deployment and uuid.UUID(deployment).int != 0, + 'invalid upload deployment UUID') + match = re.fullmatch(r'([0-9a-f-]{36})x([0-9]{1,30})', suffix) + require(match is not None and str(uuid.UUID(match[1])) == match[1] and uuid.UUID(match[1]).int != 0, + 'invalid upload suffix') + key_hash = hashlib.sha256((bucket + '/' + KEY).encode()).hexdigest() + out.append({**item, 'relative': '.rustfs.sys/multipart/' + key_hash + '/' + suffix}) + roles.add(item['role']); uploads.add(encoded) + require(roles == set(ROLES), 'G05 upload selection omitted an original role') + return out + + +def stable(info): + return (info.st_mode, info.st_uid, info.st_gid, info.st_dev, info.st_ino, info.st_nlink, + info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +def run(request, node): + require(set(request) == {'runId', 'action', 'bucket', 'items', 'expectedBinarySha256', 'quiescent'} + and request['action'] == 'mpu-race-scene' and type(request['quiescent']) is bool, + 'G05 census accepts an exact owned read-only upload scope') + state, roots, owner = node['owned'](request['runId']) + require(owner.get('runtimeProfile') == 'manual' and owner.get('bucket') == request['bucket'] + and owner.get('binarySha256') == request['expectedBinarySha256'] + and re.fullmatch(r'[0-9a-f]{64}', request['expectedBinarySha256']) and len(roots) == 4, + 'G05 census owner/runtime/binary differs') + items = selected(request['bucket'], request['items']) + process = None + if request['quiescent']: + node['stopped'](owner) + else: + process = node['process'](request) + require(process['pid'] > 1 and process['binarySha256'] == owner['binarySha256'], + 'live G05 staging census lacks its original owned process') + require(node['digest'](node['checked'](state / 'rustfs', state)) == owner['binarySha256'], + 'G05 census candidate changed') + rows, prefixes, total, count = [], [], 0, 0 + for disk, root in enumerate(roots): + data = node['checked'](root / 'data', root) + device = data.lstat().st_dev + parent = node['checked']((data / items[0]['relative']).parent, data) + parent_before = parent.lstat() if parent.exists() else None + children = [] + if parent_before: + for child in parent.iterdir(): + require(len(children) < len(items), 'G05 key prefix exceeds its three original upload identities') + children.append(child.name) + children.sort() + require(set(children) <= {item['relative'].rsplit('/', 1)[-1] for item in items}, + 'G05 key prefix contains an unacknowledged upload directory') + prefixes.append({'disk': disk, 'path': parent.relative_to(data).as_posix(), + 'present': parent_before is not None, 'children': children}) + for item in items: + folder = node['checked'](data / item['relative'], data) + files, directories = {}, [] + def visit(path, depth=0): + nonlocal total, count + path = node['checked'](path, data) + before = path.lstat() + count += 1 + require(count <= MAX_ENTRIES, 'G05 staging entry budget exceeded') + require(depth <= 2 and before.st_dev == device and (stat.S_ISDIR(before.st_mode) + or stat.S_ISREG(before.st_mode) and before.st_nlink == 1), + 'G05 staging contains links, mounts, special files or excessive depth') + if stat.S_ISDIR(before.st_mode): + if path != folder: directories.append(path.relative_to(folder).as_posix()) + children = [] + for child in path.iterdir(): + require(len(children) < MAX_ENTRIES, 'G05 staging directory population exceeds its entry budget') + children.append(child) + for child in sorted(children): visit(child, depth + 1) + else: + require(before.st_size <= 64 * 1024**2 + and total + before.st_size <= MAX_BYTES, 'G05 staging file/byte budget exceeded') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + require(stable(os.fstat(fd)) == stable(before), 'G05 file changed before hashing') + digest, size = hashlib.sha256(), 0 + while chunk := os.read(fd, 1024**2): + digest.update(chunk); size += len(chunk) + require(size <= before.st_size, 'G05 staging file grew while hashing') + require(size == before.st_size and stable(os.fstat(fd)) == stable(before), + 'G05 staging changed while hashing') + finally: + os.close(fd) + files[path.relative_to(folder).as_posix()] = {'size': size, 'sha256': digest.hexdigest()} + total += size + require(stable(path.lstat()) == stable(before), 'G05 staging population changed during census') + exists = folder.exists() + if exists: visit(folder) + rows.append({'disk': disk, 'role': item['role'], 'uploadId': item['uploadId'], + 'relative': item['relative'], 'exists': exists, 'files': files, 'directories': directories}) + parent_after = parent.lstat() if parent.exists() else None + require((parent_before is None and parent_after is None) or + (parent_before is not None and parent_after is not None and stable(parent_before) == stable(parent_after)), + 'G05 key prefix changed during staging census') + require(node['owned'](request['runId']) == (state, roots, owner), 'G05 owner or roots changed during census') + require(node['digest'](node['checked'](state / 'rustfs', state)) == owner['binarySha256'], + 'G05 candidate changed during staging census') + if request['quiescent']: + node['stopped'](owner) + else: + after = node['process'](request) + require(all(after.get(k) == process.get(k) for k in node['PROCESS_FIELDS']), + 'G05 owned process changed during live staging census') + return {'rows': rows, 'keyPrefixes': prefixes, 'bytes': total, 'binarySha256': owner['binarySha256'], 'quiescent': request['quiescent'], + 'process': process, 'readOnly': True, 'rawContentsExported': False, + 'physicalRepairProved': False, 'codingLayoutProved': False, 'overlapProved': False} + + +def verify(manifest, evidence, nodes, binary_sha, *, previous=None, committed=None): + """Generalize the frozen four-disk continuity oracle to each H1/H2 host.""" + require(len(nodes) in (1, 4) and len(set(nodes)) == len(nodes) and set(evidence) == set(nodes), + 'G05 staging proof omitted or duplicated an original host') + items = [{'role': role, 'uploadId': item['uploadId'], 'bucket': manifest['bucket'], 'key': manifest['key']} + for role, item in manifest['items'].items()] + requests = {item['role']: item for item in selected(manifest['bucket'], items)} + old = {} if previous is None else {(node, row['role'], row['disk']): row + for node, value in previous.items() for row in value['rows']} + if previous is not None: + require(set(previous) == set(nodes) and len(old) == 12 * len(nodes) + and all(value.get('readOnly') is True and value.get('binarySha256') == binary_sha + and len(value['rows']) == 12 for value in previous.values()), + 'G05 previous staging proof has incomplete ownership/candidate coverage') + live = {} if committed is None else {(node, row['disk'], row['path']): row + for node, value in committed.items() for row in value['files']} + data_dirs, completed, witnesses = {role: set() for role in ROLES}, 0, 0 + for node, value in evidence.items(): + rows = value['rows']; expected = {(role, disk) for role in ROLES for disk in range(4)} + require(value.get('readOnly') is True and value.get('binarySha256') == binary_sha + and type(value.get('quiescent')) is bool and len(rows) == len(expected) + and all(type(row['disk']) is int and row['role'] in ROLES for row in rows) + and {(row['role'], row['disk']) for row in rows} == expected, + 'G05 staging proof omitted or duplicated a drive/role') + prefixes = value.get('keyPrefixes', []) + require(len(prefixes) == 4 and all(type(p['disk']) is int for p in prefixes) + and {p['disk'] for p in prefixes} == set(range(4)), + 'G05 staging proof omitted or duplicated a complete key prefix') + expected_children = sorted(item['relative'].rsplit('/', 1)[-1] for role, item in requests.items() + if manifest['items'][role]['state'] == 'pending') + for prefix in prefixes: + require(prefix['path'] == requests['a']['relative'].rsplit('/', 1)[0] + and type(prefix['present']) is bool and prefix['children'] == expected_children + and (prefix['present'] or not expected_children), + 'G05 complete key-prefix population differs from acknowledged pending uploads') + for row in rows: + item, identity = manifest['items'][row['role']], requests[row['role']] + require(row['relative'] == identity['relative'] and row['uploadId'] == identity['uploadId'], + 'G05 staging identity differs from its original ACK') + require(all(type(v['size']) is int and v['size'] >= 0 and re.fullmatch(r'[0-9a-f]{64}', v['sha256']) + for v in row['files'].values()), 'G05 staging digest/size is invalid') + prior = old.get((node, row['role'], row['disk'])) + if prior: + require(prior['uploadId'] == identity['uploadId'] and prior['relative'] == identity['relative'], + 'G05 previous staging identity belongs to another upload') + if item['state'] == 'pending': + require(row['exists'] is True and row['files'].get('xl.meta', {}).get('size', 0) > 0, + 'G05 pending upload lost its metadata') + parts, dirs = {}, set() + for path, part in row['files'].items(): + match = re.fullmatch(r'([0-9a-f-]{36})/part\.([1-3])', path) + if match: + directory, number = match[1], int(match[2]) + require(str(uuid.UUID(directory)) == directory and uuid.UUID(directory).int and part['size'] > 32 + and number not in parts, 'G05 pending part has an invalid/duplicate generation') + parts[number] = path; dirs.add(directory) + require(set(parts) == {p['number'] for p in item['parts']} and len(dirs) == 1, + 'G05 pending physical parts differ from acknowledged parts') + require(set(row['files']) == {'xl.meta', *parts.values(), *(path + '.meta' for path in parts.values())} + and all(row['files'][path + '.meta']['size'] > 0 for path in parts.values()) + and set(row['directories']) == dirs and len(row['directories']) == 1, + 'G05 pending upload has an unknown or missing staging leaf/directory') + data_dirs[row['role']].update(dirs) + if prior and prior['exists']: + require(all(row['files'].get(path) == part for path, part in prior['files'].items() if path != 'xl.meta'), + 'G05 continuation or sibling operation changed an acknowledged staging part') + if set(prior['files']) == set(row['files']): + require(prior['files'] == row['files'] and prior['directories'] == row['directories'], + 'G05 unchanged sibling metadata or population changed') + else: + require(item['state'] in ('complete', 'aborted') and row['exists'] is False + and not row['files'] and not row['directories'], 'G05 terminal upload staging remains') + if item['state'] == 'complete' and prior and prior['exists']: + paths = [p for p in prior['files'] if re.fullmatch(r'[0-9a-f-]{36}/part\.[1-3]', p)] + require(len(paths) == 3 and {p.rsplit('.', 1)[-1] for p in paths} == {'1', '2', '3'}, + 'G05 Complete lacks three prior physical pending parts') + for path in paths: + current = live.get((node, row['disk'], manifest['bucket'] + '/' + KEY + '/' + path), {}) + require({k: current.get(k) for k in ('size', 'sha256')} == prior['files'][path], + 'G05 completed part differs from original pending shard bytes') + completed += 1 + witnesses += 1 + require(all(len(dirs) <= 1 for dirs in data_dirs.values()), 'G05 pending dataDir differs across original hosts/drives') + return {'pendingAndTerminalWitnesses': witnesses, 'completedMembersCompared': completed, + 'codingLayoutProved': False, 'physicalRepairProved': False, 'overlapProved': False, + 'canonicalCasePassClaimed': False} diff --git a/test_validation_heal_mpu_race_physical.py b/test_validation_heal_mpu_race_physical.py new file mode 100644 index 00000000..6b0d0859 --- /dev/null +++ b/test_validation_heal_mpu_race_physical.py @@ -0,0 +1,161 @@ +"""Real read-only G05 filesystem census and cross-host continuation oracles.""" +import base64 +import copy +import hashlib +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch +from types import SimpleNamespace +import uuid + +from scripts.validation_heal import fleet_node, mpu_race_physical as scene +from scripts.validation_heal.healthy import Fleet + + +class MultipartRacePhysicalTests(unittest.TestCase): + def test_actual_remote_adapter_embeds_the_local_module(self): + def execute(*args, **kwargs): + source = kwargs['input'].rsplit('\nmain(', 1)[0] + scope = {'__name__': '__g05_node_test__'} + exec(compile(source, 'g05-node', 'exec'), scope) + self.assertTrue(callable(scope['mpu_race_backend'].run)) + self.assertEqual(scope['mpu_race_backend'].KEY, scene.KEY) + return SimpleNamespace(returncode=0, stdout='{}', stderr='') + with tempfile.TemporaryDirectory() as d, patch('scripts.validation_heal.healthy.subprocess.run', side_effect=execute): + Fleet(['vm000'], Path(d), 'gh-12345-1-h1').remote('vm000', 'mpu-race-scene') + + def fixture(self, root, *, live=False): + root = root.resolve() + state = root / 'state'; state.mkdir() + binary = b'exact owned G05 candidate' + (state / 'rustfs').write_bytes(binary) + owner = {'runtimeProfile': 'manual', 'bucket': 'rv-heal-0123456789abcdef', + 'binarySha256': hashlib.sha256(binary).hexdigest()} + items = [] + for i, role in enumerate(scene.ROLES): + raw = f'{uuid.UUID(int=9)}.{uuid.UUID(int=i+1)}x123456'.encode() + items.append({'role': role, 'uploadId': base64.urlsafe_b64encode(raw).decode().rstrip('='), + 'bucket': owner['bucket'], 'key': scene.KEY}) + selected = scene.selected(owner['bucket'], items) + roots, raw = tuple(root / f'disk-{i}' for i in range(4)), {} + for disk, folder in enumerate(roots): + (folder / 'data').mkdir(parents=True) + for item in selected: + if item['role'] == 'committed': continue + upload = folder / 'data' / item['relative'] + upload.mkdir(parents=True) + p = upload / 'xl.meta'; p.write_bytes(b'owned pending metadata'); raw[p] = p.read_bytes() + for number in range(1, 3 if item['role'] == 'a' else 4): + part = upload / str(uuid.UUID(int=100 if item['role'] == 'a' else 200)) / f'part.{number}' + part.parent.mkdir(exist_ok=True) + body = (f'{item["role"]}/{number}/{disk}'.encode() * 20) + part.write_bytes(body); raw[part] = body + sidecar = part.with_name(part.name + '.meta') + sidecar.write_bytes(b'original part ACK metadata'); raw[sidecar] = sidecar.read_bytes() + (state / 'credentials.env').write_bytes(b'never export this secret') + process = {k: k for k in fleet_node.PROCESS_FIELDS} + process.update(pid=1234, binarySha256=owner['binarySha256']) + def stopped(value): + if live: raise ValueError('owned service active') + node = {'owned': lambda run: (state, roots, owner), 'checked': fleet_node.checked, + 'digest': fleet_node.digest, 'stopped': stopped, + 'process': lambda request: copy.deepcopy(process), 'PROCESS_FIELDS': fleet_node.PROCESS_FIELDS} + request = {'runId': 'gh-12345-1-h1', 'action': 'mpu-race-scene', 'bucket': owner['bucket'], + 'items': items, 'expectedBinarySha256': owner['binarySha256'], 'quiescent': not live} + manifest = {'bucket': owner['bucket'], 'key': scene.KEY, 'items': { + item['role']: {'uploadId': item['uploadId'], 'state': 'complete' if item['role'] == 'committed' else 'pending', + 'parts': [{'number': n} for n in range(1, 3 if item['role'] == 'a' else 4)]} + for item in items}} + return request, node, roots, raw, manifest + + def test_real_census_hashes_only_three_ack_selected_scopes_and_preserves_all_bytes(self): + for live in (False, True): + with self.subTest(live=live), tempfile.TemporaryDirectory() as d: + request, node, roots, raw, manifest = self.fixture(Path(d), live=live) + real_open, opened = os.open, [] + def checked_open(path, *args, **kwargs): + self.assertIn(Path(path), raw); opened.append(Path(path)) + return real_open(path, *args, **kwargs) + with patch.object(scene.os, 'open', side_effect=checked_open): + value = scene.run(request, node) + self.assertEqual(len(value['rows']), 12) + self.assertEqual(len(opened), 48) + self.assertEqual(value['bytes'], sum(len(body) for body in raw.values())) + self.assertEqual(value['quiescent'], not live) + self.assertEqual({p: p.read_bytes() for p in raw}, raw) + self.assertNotIn('never export', json.dumps(value)) + proof = scene.verify(manifest, {'vm000': value}, ['vm000'], request['expectedBinarySha256']) + self.assertEqual(proof['pendingAndTerminalWitnesses'], 12) + self.assertFalse(proof['codingLayoutProved'] or proof['canonicalCasePassClaimed']) + + def test_cross_host_sibling_and_terminal_continuity_requires_every_original_drive(self): + with tempfile.TemporaryDirectory() as d: + request, node, roots, raw, manifest = self.fixture(Path(d)) + value = scene.run(request, node) + nodes = [f'vm00{i}' for i in range(4)] + baseline = {n: copy.deepcopy(value) for n in nodes} + continued = copy.deepcopy(baseline) + manifest['items']['a']['parts'].append({'number': 3}) + for n, observed in continued.items(): + for row in observed['rows']: + if row['role'] != 'a': continue + for suffix in ['', '.meta']: + body = f'A/3/{n}/{row["disk"]}/{suffix}'.encode() + row['files'][f'{uuid.UUID(int=100)}/part.3{suffix}'] = { + 'size': 64 if not suffix else len(body), 'sha256': hashlib.sha256(body).hexdigest()} + scene.verify(manifest, continued, nodes, request['expectedBinarySha256'], previous=baseline) + committed, completed = {n: {'files': []} for n in nodes}, copy.deepcopy(continued) + manifest['items']['a']['state'] = 'complete'; manifest['items']['b']['state'] = 'aborted' + for n, observed in completed.items(): + for prefix in observed['keyPrefixes']: prefix['children'] = [] + for row in observed['rows']: + if row['role'] == 'a': + for path, entry in row['files'].items(): + if path.endswith(('/part.1', '/part.2', '/part.3')): + committed[n]['files'].append({'disk': row['disk'], 'path': manifest['bucket']+'/'+scene.KEY+'/'+path, **entry}) + row.update(exists=False, files={}, directories=[]) + proof = scene.verify(manifest, completed, nodes, request['expectedBinarySha256'], previous=continued, committed=committed) + self.assertEqual(proof['completedMembersCompared'], 16) + for fault in ('host', 'drive', 'duplicate', 'retired', 'candidate', 'completed-bytes', 'unlisted-upload'): + broken, live = copy.deepcopy(completed), copy.deepcopy(committed) + if fault == 'host': broken.pop('vm003') + elif fault == 'drive': broken['vm000']['rows'].pop() + elif fault == 'duplicate': broken['vm000']['rows'][-1] = copy.deepcopy(broken['vm000']['rows'][0]) + elif fault == 'retired': broken['vm000']['rows'][1]['exists'] = True + elif fault == 'candidate': broken['vm000']['binarySha256'] = 'f'*64 + elif fault == 'unlisted-upload': broken['vm000']['keyPrefixes'][0]['children'].append('unknown-upload') + else: live['vm000']['files'][0]['sha256'] = 'f'*64 + with self.subTest(fault=fault), self.assertRaises(ValueError): + scene.verify(manifest, broken, nodes, request['expectedBinarySha256'], previous=continued, committed=live) + + def test_scope_escape_links_budgets_and_changed_process_are_rejected(self): + for fault in ('role', 'bucket', 'runtime', 'binary', 'unknown-field', 'upload-path', + 'symlink', 'hardlink', 'fifo', 'entries', 'bytes', 'process', 'unlisted-upload'): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as d: + request, node, roots, raw, manifest = self.fixture(Path(d), live=fault=='process') + part = next(p for p in raw if p.name=='part.1') + if fault=='role': request['items'][1]['role']='committed' + elif fault=='bucket': request['bucket']='rv-heal-ffffffffffffffff' + elif fault=='runtime': node['owned'](request['runId'])[2]['runtimeProfile']='lock-quorum' + elif fault=='binary': request['expectedBinarySha256']='f'*64 + elif fault=='unknown-field': request['path']='/etc' + elif fault=='upload-path': request['items'][1]['uploadId']='../outside' + elif fault=='symlink': part.unlink(); part.symlink_to(Path(d).resolve()/'state/credentials.env') + elif fault=='hardlink': os.link(part,part.with_name('extra-link')) + elif fault=='fifo': part.unlink();os.mkfifo(part) + elif fault=='unlisted-upload': + selected=scene.selected(request['bucket'],request['items']) + (roots[0]/'data'/selected[0]['relative']).parent.joinpath('unknown-upload').mkdir() + elif fault=='process': + first=node['process'](request);second={**first,'startTicks':'another birth'} + node['process']=Mock(side_effect=[first,second]) + with patch.object(scene,'MAX_ENTRIES',1 if fault=='entries' else scene.MAX_ENTRIES), \ + patch.object(scene,'MAX_BYTES',1 if fault=='bytes' else scene.MAX_BYTES): + with self.assertRaises(ValueError): scene.run(request,node) + + +if __name__ == '__main__': + unittest.main() From 4707873c46fb36da03be85de3a3ae60c4c1fccc1 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 11:44:17 +0800 Subject: [PATCH 262/343] fix(validation): preserve frozen copy race fixtures and direction --- docs/validation-heal-operations.md | 6 + scripts/validation_heal/race.py | 57 ++-- .../race_native_contracts.json | 275 ++++++++++++++++++ test_validation_heal_g06_source.py | 127 ++++++++ test_validation_heal_race_native.py | 47 +++ 5 files changed, 484 insertions(+), 28 deletions(-) create mode 100644 test_validation_heal_g06_source.py diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 938ead56..aa688d8b 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -672,6 +672,12 @@ Runtime sources opt in to specific cases separately from the native registry. Th All original H1/H2 black-box branches, SDK ACKs, manifests, overlap witnesses, exact Admin settings and physical/S3 checks remain required. Native logical disk shapes do not certify distributed VM execution. The original `072b57b37559fbe626aeda8abf61617fdcbdd926` route retains its blocked native rows. A09, G05/G06 and I05 still require their own complete native groups and executor evidence. G04 also remains unaccepted until its same-source hosted evidence and every original live branch pass independent audit. No source or case gains canonical acceptance from a registry entry or local native test; a successful exact-source hosted bundle, every live branch and independent original-evidence audit are required. +### A09/G06 native groups and frozen G06 public fixtures + +Source `757b751482bb970bc321e92059690a7698554358` adds two collector-only groups. `default-heal-exclusive-owner` observes actual PUT and exact historical DELETE lock futures becoming pending while default Deep Heal exclusively owns a real staged external repair. Physical reconstruction precedes foreground publication; all acknowledged successor or retained-current shards and full bodies are checked in 4/2 and 16/4. Its sixteen events prove the storage fence, while Admin `nolock=true` normalization and public request timing still require separate evidence. `copy-and-metadata-final-rename` covers real streamed cross-key COPY source/destination, self metadata replacement and tags. Twenty-four events bind actual ACK and physical-tail completion before releasing shared reconstruction, exact stale refusal or unchanged-source repair, final source/target/control payloads, metadata and tags. The registry binds all five storage fixture/barrier source files. This candidate remains outside runtime admission until the complete executor, same-source hosted artifacts and workflow admission are ready. + +The G06 public adapter uses the existing DS workload: all 257 original keys, including Chinese, spaces, literal `%2F`, actual slashes and parent-like prefixes. The observed missing key is `中文/文件/0252.bin`; its companion is `literal%2Fslash/0254.bin`. `copy-source` copies from the observed key into the companion; `copy-destination` copies in the reverse direction. Default COPY expectations inherit the source payload, content properties, metadata and tags. Both directions verify the unchanged source, exact destination coding-index shard bytes and a distinct destination generation before GET, rejecting retained old destination directories. Registry, fixture and local transition tests do not establish public overlap or complete canonical PASS. + ### G05 same-key multipart components `mpu_race_fixture.py` migrates the frozen validation repository's `heal_mpu_race_fixture.py` model at `524eb5b32c3236497fdd8a9271df0494fa3d9974`. It uses the existing auto-testing `Recorder` and probe directly. Imports are local; no source validation checkout is needed at runtime. Part sizes follow the frozen common DM recipe: `5MiB+17`, `5MiB+93`, and `1MiB+47`. diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index e7a5e846..db440f25 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -21,7 +21,7 @@ try: from .heal_probe import Probe, canonical, compare_inventories, payload, require, save - from .heal_workload import Recorder, verify_seed + from .heal_workload import Recorder, seed as seed_workload, verify_seed from .healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology from .inspector import verify as verify_inspector from .matrix import select @@ -32,7 +32,7 @@ from .race_native_evidence import verify as verify_native except ImportError: from heal_probe import Probe, canonical, compare_inventories, payload, require, save - from heal_workload import Recorder, verify_seed + from heal_workload import Recorder, seed as seed_workload, verify_seed from healthy import BASE_OPTIONS, Fleet, physical_content, validate_topology from inspector import verify as verify_inspector from matrix import select @@ -62,8 +62,8 @@ SIZE = 257 * 1024 + 123 TARGET_KEY = "objects/0256.bin" REFERENCE_KEY = "objects/0000.bin" -G06_TARGET_KEY = "actual/slash/0256.bin" -G06_REFERENCE_KEY = "space key/0000.bin" +G06_TARGET_KEY = "中文/文件/0252.bin" +G06_REFERENCE_KEY = "literal%2Fslash/0254.bin" G05_TARGET_KEY = "multipart/target.bin" G05_PART_SIZES = (5 * 1024 * 1024 + 17, 5 * 1024 * 1024 + 93, 1024 * 1024 + 47) REFERENCE = { @@ -224,6 +224,9 @@ def sample_overlap(ack, observed, path, token, reference=REFERENCE): def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES, *G05_BRANCHES, *G06_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G02/G03/G04/G05/G06 unregistered fixture") + if branch in G06_BRANCHES: + require(count == OBJECTS, "G06 requires the frozen complete 257-key DS population") + return seed_workload(recorder, bucket, "DS", count, [SIZE]) recorder.call("create_bucket", {"Bucket": bucket}) if branch in G04_BRANCHES: versioned = branch != "unversioned-delete" @@ -285,10 +288,7 @@ def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): if branch in (*G02_BRANCHES, *G03_BRANCHES): recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) entries = [] - if branch in G06_BRANCHES: - keys = [G06_REFERENCE_KEY] + [f"ds/{index:04d}.bin" for index in range(1, count - 1)] + [G06_TARGET_KEY] - else: - keys = [f"objects/{index:04d}.bin" for index in range(count - 1)] + [TARGET_KEY] + keys = [f"objects/{index:04d}.bin" for index in range(count - 1)] + [TARGET_KEY] current_target = target_key(branch) for key in keys: versions = 3 if key == current_target and branch in (*G02_BRANCHES, *G03_BRANCHES) else 1 @@ -379,11 +379,11 @@ def operation_plan(bucket, branch, expected): ] if branch in G06_BRANCHES: if branch == "copy-source": - return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": current_target, - "CopySource": {"Bucket": bucket, "Key": current_reference, "VersionId": "null"}}, "nonce": "copy-source"}] - if branch == "copy-destination": return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": current_reference, - "CopySource": {"Bucket": bucket, "Key": current_target, "VersionId": "null"}}, "nonce": "copy-destination"}] + "CopySource": {"Bucket": bucket, "Key": current_target, "VersionId": "null"}}, "nonce": "copy-source"}] + if branch == "copy-destination": + return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": current_target, + "CopySource": {"Bucket": bucket, "Key": current_reference, "VersionId": "null"}}, "nonce": "copy-destination"}] if branch == "replace-metadata": return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": current_target, "CopySource": {"Bucket": bucket, "Key": current_target, "VersionId": "null"}, @@ -480,16 +480,15 @@ def final_inventory(baseline, branch, plan, responses): entry = next(row for row in result["entries"] if row["key"] == current_target) copy_result = responses[-1].get("CopyObjectResult") or {} copy_etag = copy_result.get("ETag") or responses[-1].get("ETag") - if branch == "copy-source": - reference = next(row for row in result["entries"] if row["key"] == current_reference) - entry["payload"] = copy.deepcopy(reference["payload"]) - require(isinstance(copy_etag, str) and copy_etag, "G06 copy-source ACK lacks its committed ETag") - entry["properties"].update(ETag=copy_etag) - elif branch == "copy-destination": - destination = next(row for row in result["entries"] if row["key"] == current_reference) - destination["payload"] = copy.deepcopy(entry["payload"]) - require(isinstance(copy_etag, str) and copy_etag, "G06 copy-destination ACK lacks its committed ETag") - destination["properties"].update(ETag=copy_etag) + if branch in {"copy-source", "copy-destination"}: + source_key, destination_key = ((current_target, current_reference) if branch == "copy-source" + else (current_reference, current_target)) + source = next(row for row in result["entries"] if row["key"] == source_key) + destination = next(row for row in result["entries"] if row["key"] == destination_key) + require(isinstance(copy_etag, str) and copy_etag, "G06 COPY ACK lacks its committed ETag") + for field in ("payload", "properties", "tags"): + destination[field] = copy.deepcopy(source[field]) + destination["properties"]["ETag"] = copy_etag elif branch == "replace-metadata": require(isinstance(copy_etag, str) and copy_etag, "G06 metadata replacement ACK lacks its committed ETag") entry["properties"].update(Metadata=plan[0]["params"]["Metadata"], ContentType=plan[0]["params"]["ContentType"]) @@ -528,8 +527,9 @@ def physical_transition(before, after, bucket, mapped, final, branch): require(all(any(row["path"].startswith(f"{bucket}/{key}/") for key in known) for value in after.values() for row in value["files"]), "G01/G03 created an unacknowledged physical key") current_target, current_reference = target_key(branch), reference_key(branch) + changed_key = (current_reference if branch == "copy-source" else current_target) for (key, version), target in mapped.items(): - if key == current_target: continue + if key == (changed_key if branch in G06_BRANCHES else current_target): continue validate_repaired_physical(object_scene(before, bucket, key), object_scene(after, bucket, key), target) survivors = map_fixture(after, bucket, final["entries"]) original_target = [value for (key, _), value in mapped.items() if key == current_target] @@ -598,11 +598,10 @@ def physical_transition(before, after, bucket, mapped, final, branch): == [(part["index"], part["sha256"], part["size"]) for part in current_target_values[0]["cohort"]], "G05 pending/aborted MPU changed committed object bytes") elif branch in G06_BRANCHES: - changed_key = current_reference if branch == "copy-destination" else current_target target_after = next(value for (key, _), value in survivors.items() if key == changed_key) target_before = next(value for (key, _), value in mapped.items() if key == changed_key) if branch in {"copy-source", "copy-destination"}: - source_key = current_reference if branch == "copy-source" else current_target + source_key = current_target if branch == "copy-source" else current_reference source = next(value for (key, _), value in mapped.items() if key == source_key) require(target_after["dataDir"] != target_before["dataDir"] and {part["index"]: (part["sha256"], part["size"]) for part in target_after["cohort"]} @@ -623,11 +622,13 @@ def physical_transition(before, after, bucket, mapped, final, branch): and [(row["index"], row["sha256"], row["size"]) for row in target["cohort"]] == [(row["index"], row["sha256"], row["size"]) for row in original["cohort"]], "G01/G03 delete changed another retained ACK generation") - current_dirs = {target["dataDir"] for target in current_target_values} + inspected_keys = {current_target, changed_key} if branch in G06_BRANCHES else {current_target} for value in after.values(): for row in value["files"]: - prefix = f"{bucket}/{current_target}/" - if not row["path"].startswith(prefix): continue + key = next((key for key in inspected_keys if row["path"].startswith(f"{bucket}/{key}/")), None) + if key is None: continue + prefix = f"{bucket}/{key}/" + current_dirs = {target["dataDir"] for (name, _), target in survivors.items() if name == key} if branch == "unversioned-delete": raise ValueError("G04 unversioned DELETE left a physical target leaf") suffix = row["path"][len(prefix):] diff --git a/scripts/validation_heal/race_native_contracts.json b/scripts/validation_heal/race_native_contracts.json index eedce4cd..2955367a 100644 --- a/scripts/validation_heal/race_native_contracts.json +++ b/scripts/validation_heal/race_native_contracts.json @@ -595,6 +595,281 @@ "multipart-continue-complete-abort-final-rename" ] } + }, + "757b751482bb970bc321e92059690a7698554358": { + "sourceTree": "069b7f321084c52d00ce873d932b8e7f6ca17319", + "cargoLockSha256": "b4d959f964c13d8ed507657fa4b52752769f96e5cce932845d3d8cec69a5cd56", + "sourceFiles": { + "crates/ecstore/src/set_disk/ops/object.rs": "38f3fc4ab7562d4c14202cafb75f36d3e6ea63c376526702042f1139eebc9fa0", + "crates/ecstore/src/set_disk/ops/heal.rs": "0b706ac476e02202d5ad217a166eb523abfd3cfd123902a9fb7edf27bd2bf376", + "crates/ecstore/src/set_disk/core/io_primitives.rs": "6e50c8ea8b374b8a2d73da740321af25fed6e26c34e5117feca3e5f5891782ab", + "crates/ecstore/src/bucket/metadata_sys.rs": "35313244379c20bcfea44fae79c8b4f74fa7cecd60c7ea33cae47399951e8de3", + "crates/ecstore/src/set_disk/mod.rs": "4a20398f08dab92f83b2e6fd649de31c5a57c395c9184e2a896cf8990a986de8" + }, + "regressions": { + "committed-overwrite-cleanup": { + "test": "set_disk::ops::object::put_object_tmp_cleanup_tests::deep_heal_preserves_committed_generation_during_old_data_cleanup", + "sourcePath": "crates/ecstore/src/set_disk/ops/object.rs", + "sourceAssertions": [ + "Every actual shuffled cleanup slot pauses after new physical generation commit while TailDrained overwrite ACK remains pending.", + "Real Deep and shared read-repair reconstruct a missing new-generation encoded part before GET in4/2 and16/4 geometries.", + "All old physical generations disappear after cleanup; every new encoded shard, unchanged metadata, overwrite ACK and complete final body remain exact." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g01: ", + "milestones": [ + "native_g01: geometry=4/2 read_repair=false old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=4/2 read_repair=false locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=4/2 read_repair=false overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=4/2 read_repair=true old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=4/2 read_repair=true locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=4/2 read_repair=true overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=16/4 read_repair=false old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=16/4 read_repair=false locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=16/4 read_repair=false overwrite_ack_preserved final_physical_generation_and_full_body_verified", + "native_g01: geometry=16/4 read_repair=true old_cleanup_paused new_generation_committed target_part_missing", + "native_g01: geometry=16/4 read_repair=true locked_heal_completed new_generation_part_rebuilt_before_get", + "native_g01: geometry=16/4 read_repair=true overwrite_ack_preserved final_physical_generation_and_full_body_verified" + ] + }, + "stale-read-repair-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::read_repair_commit_aborts_when_object_changes_before_final_rename", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Real missing external shard forces actual shared read-repair data reconstruction.", + "Pause immediately before final commit fence; a successor overwrite ACK precedes release.", + "Obsolete read-repair returns exact DiskError::Timeout and final current ETag equals acknowledged overwrite." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "versioned-read-repair-lock-owner": { + "test": "set_disk::ops::heal::heal_result_report_tests::object_lock_diagnostics_distinguish_read_repair_from_queued_put", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Historical missing external shard is repaired under actual shared Data-phase lock.", + "Queued PUT cannot acquire namespace and later historical GET times out while version-specific owner remains held.", + "After release, successor PUT ACK and every original/new version full body remain correct." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "cancelled-physical-unlink-owner": { + "test": "set_disk::ops::heal::heal_result_report_tests::cancelled_heal_backup_cleanup_retains_key_owner_until_unlink_finishes", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Actual external Heal backup unlink pauses in outer, inner and shared read-repair modes.", + "Caller abort does not retire namespace commit owner; successor write lock remains pending until physical unlink drains.", + "After physical completion, successor obtains lock, owned backup is absent and committed full body is preserved." + ], + "witnessKind": "reviewed-in-test-assertions", + "milestonePrefix": null, + "milestones": [] + }, + "explicit-version-append-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_explicit_version_commit_preserves_acknowledged_new_version", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Actual selected-version shared Deep reconstruction pauses before its final rename after releasing the read guard.", + "Foreground version mutation ACK and same-key physical write-tail drain precede releasing stale Heal; append succeeds or deleted version remains typed FileVersionNotFound.", + "All surviving acknowledged version UUIDs, data dirs, original encoded shards and complete exact/current bodies remain intact in4/2 and16/4; deleted generations do not reappear." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g02: ", + "milestones": [ + "native_g02: geometry=4/2 mutation=append reconstruction_paused_before_final_rename", + "native_g02: geometry=4/2 mutation=append foreground_ack_and_physical_tail_verified_before_release", + "native_g02: geometry=4/2 mutation=append exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g02: geometry=16/4 mutation=append reconstruction_paused_before_final_rename", + "native_g02: geometry=16/4 mutation=append foreground_ack_and_physical_tail_verified_before_release", + "native_g02: geometry=16/4 mutation=append exact_version_outcome_physical_generations_and_full_bodies_verified" + ] + }, + "explicit-version-delete-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_explicit_version_commit_refuses_deleted_historical_and_current_versions", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Actual selected-version shared Deep reconstruction pauses before its final rename after releasing the read guard.", + "Foreground version mutation ACK and same-key physical write-tail drain precede releasing stale Heal; append succeeds or deleted version remains typed FileVersionNotFound.", + "All surviving acknowledged version UUIDs, data dirs, original encoded shards and complete exact/current bodies remain intact in4/2 and16/4; deleted generations do not reappear." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g03: ", + "milestones": [ + "native_g03: geometry=4/2 mutation=delete-historical reconstruction_paused_before_final_rename", + "native_g03: geometry=4/2 mutation=delete-historical foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=4/2 mutation=delete-historical exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g03: geometry=16/4 mutation=delete-historical reconstruction_paused_before_final_rename", + "native_g03: geometry=16/4 mutation=delete-historical foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=16/4 mutation=delete-historical exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g03: geometry=4/2 mutation=delete-current reconstruction_paused_before_final_rename", + "native_g03: geometry=4/2 mutation=delete-current foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=4/2 mutation=delete-current exact_version_outcome_physical_generations_and_full_bodies_verified", + "native_g03: geometry=16/4 mutation=delete-current reconstruction_paused_before_final_rename", + "native_g03: geometry=16/4 mutation=delete-current foreground_ack_and_physical_tail_verified_before_release", + "native_g03: geometry=16/4 mutation=delete-current exact_version_outcome_physical_generations_and_full_bodies_verified" + ] + }, + "marker-and-batch-delete-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_marker_and_batch_delete_fence_preserves_acknowledged_version_set", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Real protected external-data reconstruction pauses before final metadata rename in4/2 and16/4 geometries for four independent marker/delete mutations.", + "Foreground create-marker, exact marker removal, two-item version deletion with exact per-item ACKs or unversioned deletion and same-key physical tail drain all precede releasing stale Heal.", + "Every member retains the exact acknowledged marker/data version set without duplicates; surviving encoded shards and full exact/current bodies remain unchanged, and deleted directories or markers cannot resurrect." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g04: ", + "milestones": [ + "native_g04: geometry=4/2 mutation=create-marker reconstruction_paused_before_final_rename", + "native_g04: geometry=4/2 mutation=create-marker exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=4/2 mutation=create-marker marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=4/2 mutation=delete-marker reconstruction_paused_before_final_rename", + "native_g04: geometry=4/2 mutation=delete-marker exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=4/2 mutation=delete-marker marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=4/2 mutation=batch-version-delete reconstruction_paused_before_final_rename", + "native_g04: geometry=4/2 mutation=batch-version-delete exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=4/2 mutation=batch-version-delete marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=4/2 mutation=unversioned-delete reconstruction_paused_before_final_rename", + "native_g04: geometry=4/2 mutation=unversioned-delete exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=4/2 mutation=unversioned-delete marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=16/4 mutation=create-marker reconstruction_paused_before_final_rename", + "native_g04: geometry=16/4 mutation=create-marker exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=16/4 mutation=create-marker marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=16/4 mutation=delete-marker reconstruction_paused_before_final_rename", + "native_g04: geometry=16/4 mutation=delete-marker exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=16/4 mutation=delete-marker marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=16/4 mutation=batch-version-delete reconstruction_paused_before_final_rename", + "native_g04: geometry=16/4 mutation=batch-version-delete exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=16/4 mutation=batch-version-delete marker_set_deleted_directories_surviving_shards_and_full_bodies_verified", + "native_g04: geometry=16/4 mutation=unversioned-delete reconstruction_paused_before_final_rename", + "native_g04: geometry=16/4 mutation=unversioned-delete exact_foreground_acks_and_physical_tail_verified_before_release", + "native_g04: geometry=16/4 mutation=unversioned-delete marker_set_deleted_directories_surviving_shards_and_full_bodies_verified" + ] + }, + "multipart-continue-complete-abort-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_multipart_continue_complete_abort_fences_stale_heal", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Real three-part committed unversioned object and distinct same-key A/B pending uploads: A parts1/2, B parts1/2/3, independent payloads and ACK identities.", + "Actual missing committed part forces shared Deep Heal to pause before final rename in4/2 and16/4 geometries; A part3 and B Abort ACK preserve all other committed and A pending shard bytes.", + "A Complete ACK and namespace physical tail drain precede release; stale Heal returns exact DiskError::Timeout. All new metadata/parts remain byte exact, old generation and both staging directories absent, final full body equals ordered A ACK payloads.", + "Native logical/unversioned scope does not prove full versioned SDK H1/H2 overlap, protocol or fleet execution." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g05: ", + "milestones": [ + "native_g05: geometry=4/2 committed_multipart_and_two_distinct_pending_uploads_seeded", + "native_g05: geometry=4/2 real_committed_object_heal_paused_before_final_rename", + "native_g05: geometry=4/2 continue_ack_and_other_abort_preserve_pending_parts_and_original_commit", + "native_g05: geometry=4/2 complete_ack_and_all_physical_tails_verified_before_release", + "native_g05: geometry=4/2 stale_heal_rejected_exact_completed_parts_full_body_and_upload_retirement_verified", + "native_g05: geometry=16/4 committed_multipart_and_two_distinct_pending_uploads_seeded", + "native_g05: geometry=16/4 real_committed_object_heal_paused_before_final_rename", + "native_g05: geometry=16/4 continue_ack_and_other_abort_preserve_pending_parts_and_original_commit", + "native_g05: geometry=16/4 complete_ack_and_all_physical_tails_verified_before_release", + "native_g05: geometry=16/4 stale_heal_rejected_exact_completed_parts_full_body_and_upload_retirement_verified" + ] + }, + "default-heal-exclusive-owner": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_default_heal_owner_fences_queued_put_and_version_delete", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Default locked Deep Heal owns the real exclusive namespace during external part staging before any writer queues; a positive reader control distinguishes exclusivity.", + "Real PUT and exact historical DELETE acquisition futures each poll Pending under that owner; after release Heal reconstructs every original shard before foreground publication.", + "ACK successor or retained-current generation, exact version population, independent coding-index shard bytes and complete bodies remain correct across4/2 and16/4; this does not claim Admin normalization or live API timing." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_a09: ", + "milestones": [ + "native_a09: geometry=4/2 mutation=put default_deep_heal_staged_exclusive_owner_read_blocked", + "native_a09: geometry=4/2 mutation=put actual_foreground_namespace_future_pending_original_bytes_unchanged", + "native_a09: geometry=4/2 mutation=put heal_reconstructed_before_foreground_commit", + "native_a09: geometry=4/2 mutation=put foreground_ack_final_versions_parts_and_full_body_verified", + "native_a09: geometry=4/2 mutation=version-delete default_deep_heal_staged_exclusive_owner_read_blocked", + "native_a09: geometry=4/2 mutation=version-delete actual_foreground_namespace_future_pending_original_bytes_unchanged", + "native_a09: geometry=4/2 mutation=version-delete heal_reconstructed_before_foreground_commit", + "native_a09: geometry=4/2 mutation=version-delete foreground_ack_final_versions_parts_and_full_body_verified", + "native_a09: geometry=16/4 mutation=put default_deep_heal_staged_exclusive_owner_read_blocked", + "native_a09: geometry=16/4 mutation=put actual_foreground_namespace_future_pending_original_bytes_unchanged", + "native_a09: geometry=16/4 mutation=put heal_reconstructed_before_foreground_commit", + "native_a09: geometry=16/4 mutation=put foreground_ack_final_versions_parts_and_full_body_verified", + "native_a09: geometry=16/4 mutation=version-delete default_deep_heal_staged_exclusive_owner_read_blocked", + "native_a09: geometry=16/4 mutation=version-delete actual_foreground_namespace_future_pending_original_bytes_unchanged", + "native_a09: geometry=16/4 mutation=version-delete heal_reconstructed_before_foreground_commit", + "native_a09: geometry=16/4 mutation=version-delete foreground_ack_final_versions_parts_and_full_body_verified" + ] + }, + "copy-and-metadata-final-rename": { + "test": "set_disk::ops::heal::heal_result_report_tests::native_copy_and_metadata_mutations_preserve_acknowledged_state_during_heal", + "sourcePath": "crates/ecstore/src/set_disk/ops/heal.rs", + "sourceAssertions": [ + "Real ECStore cross-key streamed COPY, self metadata replacement and tag update ACK before releasing actual shared Deep reconstruction final-rename pause.", + "Unchanged COPY source completes repair; changed destination or metadata/tag generation rejects stale Heal with exact DiskError::Timeout, preserving all acknowledged physical bytes and reclaiming replaced generation.", + "Independent fresh Heal reconstructs metadata/tag targets before finalGET; exact source/destination/control payloads, user metadata/content type/tags and acknowledged shard bytes remain preserved in4/2 and16/4." + ], + "witnessKind": "ordered-barrier-milestones", + "milestonePrefix": "native_g06: ", + "milestones": [ + "native_g06: geometry=4/2 mutation=copy-source real_reconstruction_paused_before_final_rename", + "native_g06: geometry=4/2 mutation=copy-source foreground_ack_and_all_physical_tails_verified_before_release", + "native_g06: geometry=4/2 mutation=copy-source exact_copy_metadata_tags_generations_and_full_bodies_verified", + "native_g06: geometry=4/2 mutation=copy-destination real_reconstruction_paused_before_final_rename", + "native_g06: geometry=4/2 mutation=copy-destination foreground_ack_and_all_physical_tails_verified_before_release", + "native_g06: geometry=4/2 mutation=copy-destination exact_copy_metadata_tags_generations_and_full_bodies_verified", + "native_g06: geometry=4/2 mutation=replace-metadata real_reconstruction_paused_before_final_rename", + "native_g06: geometry=4/2 mutation=replace-metadata foreground_ack_and_all_physical_tails_verified_before_release", + "native_g06: geometry=4/2 mutation=replace-metadata exact_copy_metadata_tags_generations_and_full_bodies_verified", + "native_g06: geometry=4/2 mutation=update-tags real_reconstruction_paused_before_final_rename", + "native_g06: geometry=4/2 mutation=update-tags foreground_ack_and_all_physical_tails_verified_before_release", + "native_g06: geometry=4/2 mutation=update-tags exact_copy_metadata_tags_generations_and_full_bodies_verified", + "native_g06: geometry=16/4 mutation=copy-source real_reconstruction_paused_before_final_rename", + "native_g06: geometry=16/4 mutation=copy-source foreground_ack_and_all_physical_tails_verified_before_release", + "native_g06: geometry=16/4 mutation=copy-source exact_copy_metadata_tags_generations_and_full_bodies_verified", + "native_g06: geometry=16/4 mutation=copy-destination real_reconstruction_paused_before_final_rename", + "native_g06: geometry=16/4 mutation=copy-destination foreground_ack_and_all_physical_tails_verified_before_release", + "native_g06: geometry=16/4 mutation=copy-destination exact_copy_metadata_tags_generations_and_full_bodies_verified", + "native_g06: geometry=16/4 mutation=replace-metadata real_reconstruction_paused_before_final_rename", + "native_g06: geometry=16/4 mutation=replace-metadata foreground_ack_and_all_physical_tails_verified_before_release", + "native_g06: geometry=16/4 mutation=replace-metadata exact_copy_metadata_tags_generations_and_full_bodies_verified", + "native_g06: geometry=16/4 mutation=update-tags real_reconstruction_paused_before_final_rename", + "native_g06: geometry=16/4 mutation=update-tags foreground_ack_and_all_physical_tails_verified_before_release", + "native_g06: geometry=16/4 mutation=update-tags exact_copy_metadata_tags_generations_and_full_bodies_verified" + ] + } + }, + "cases": { + "heal.race-unversioned-overwrite": [ + "committed-overwrite-cleanup", + "stale-read-repair-final-rename" + ], + "heal.race-versioned-put": [ + "versioned-read-repair-lock-owner", + "explicit-version-append-final-rename" + ], + "heal.lock-timeout-and-caller-abort": [ + "versioned-read-repair-lock-owner", + "cancelled-physical-unlink-owner", + "stale-read-repair-final-rename" + ], + "heal.race-version-delete": [ + "explicit-version-delete-final-rename" + ], + "heal.race-markers-and-batch-delete": [ + "marker-and-batch-delete-final-rename" + ], + "heal.race-multipart-complete-abort": [ + "multipart-continue-complete-abort-final-rename" + ], + "heal.admin-nolock-safety": [ + "default-heal-exclusive-owner" + ], + "heal.race-copy-and-metadata-update": [ + "copy-and-metadata-final-rename" + ] + } } } } diff --git a/test_validation_heal_g06_source.py b/test_validation_heal_g06_source.py new file mode 100644 index 00000000..9974801d --- /dev/null +++ b/test_validation_heal_g06_source.py @@ -0,0 +1,127 @@ +"""Frozen DS and COPY direction contracts; synthetic scenes are not live proof.""" +import copy +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch +import uuid + +import test_validation_heal as physical_fixtures +from scripts.validation_heal import race + + +class G06SourceTests(unittest.TestCase): + bucket = 'rv-heal-0123456789abcdef' + + def baseline(self): + return {'entries': [ + {'key': key, 'versionId': 'null', 'isLatest': True, 'deleteMarker': False, + 'payload': {'size': size, 'sha256': role}, + 'properties': {'ETag': '"' + role + '"', 'ContentLength': size, + 'ContentType': 'application/' + role, 'Metadata': {'role': role}}, + 'tags': [{'Key': 'role', 'Value': role}]} + for key, role, size in [(race.G06_TARGET_KEY, 'selected', 17), + (race.G06_REFERENCE_KEY, 'other', 23)]]} + + def test_frozen_ds_keys_and_seed_use_the_existing_workload(self): + self.assertEqual(race.G06_TARGET_KEY, '中文/文件/0252.bin') + self.assertEqual(race.G06_REFERENCE_KEY, 'literal%2Fslash/0254.bin') + recorder = Mock() + recorder.probe = SimpleNamespace(binding={'bucketPrefix': self.bucket}) + recorder.call.return_value = {'ETag': '"ack"'} + with patch.object(race, 'SIZE', 31): + expected = race.seed_fixture(recorder, self.bucket, 'copy-source') + prefixes = ('中文/文件', 'space key', 'literal%2Fslash', 'actual/slash', 'a/child', 'ab/control', 'b/other') + keys = [f'{prefixes[i % 7]}/{i:04d}.bin' for i in range(257)] + self.assertEqual([r['key'] for r in expected], keys) + puts = [c for c in recorder.call.call_args_list if c.args[0] == 'put_object'] + self.assertEqual(len(puts), 257) + self.assertEqual([c.args[1]['Key'] for c in puts], keys) + self.assertEqual(puts[252].args[1]['Metadata'], {'heal-fixture': 'DS', 'nonce': '252-0'}) + self.assertEqual(puts[254].args[1]['Tagging'], 'suite=heal&fixture=DS') + + def test_copy_source_or_destination_is_the_original_missing_selected_key(self): + for branch in ('copy-source', 'copy-destination'): + with self.subTest(branch=branch): + params = race.operation_plan(self.bucket, branch, self.baseline()['entries'])[0]['params'] + source, destination = ((race.G06_TARGET_KEY, race.G06_REFERENCE_KEY) if branch == 'copy-source' + else (race.G06_REFERENCE_KEY, race.G06_TARGET_KEY)) + self.assertEqual(params['Key'], destination) + self.assertEqual(params['CopySource'], {'Bucket': self.bucket, 'Key': source, 'VersionId': 'null'}) + + def test_copy_final_s3_follows_source_content_metadata_and_tags(self): + for branch in ('copy-source', 'copy-destination'): + with self.subTest(branch=branch): + before = self.baseline() + plan = race.operation_plan(self.bucket, branch, before['entries']) + source, destination = ((race.G06_TARGET_KEY, race.G06_REFERENCE_KEY) if branch == 'copy-source' + else (race.G06_REFERENCE_KEY, race.G06_TARGET_KEY)) + original = {r['key']: r for r in before['entries']} + after = race.final_inventory(before, branch, plan, [{'CopyObjectResult': {'ETag': '"copied"'}}]) + actual = {r['key']: r for r in after['entries']} + self.assertEqual(actual[source], original[source]) + self.assertEqual(actual[destination]['payload'], original[source]['payload']) + self.assertEqual(actual[destination]['tags'], original[source]['tags']) + self.assertEqual(actual[destination]['properties'], {**original[source]['properties'], 'ETag': '"copied"'}) + + def scene(self, branch): + before = physical_fixtures.HealShardOracleTest().snapshot() + for value in before.values(): + original = value['files'] + value['files'] = [] + for meta, part in zip(original[::3], original[1::3], strict=True): + for key, generation, role in [(race.G06_TARGET_KEY, 100, 'selected'), + (race.G06_REFERENCE_KEY, 200, 'other')]: + m, p = copy.deepcopy(meta), copy.deepcopy(part) + m['path'] = f'{self.bucket}/{key}/xl.meta' + m['metadata']['key'] = key + m['metadata']['versions'][0]['dataDir'] = str(uuid.UUID(int=generation)) + p['path'] = f'{self.bucket}/{key}/{uuid.UUID(int=generation)}/part.1' + p['sha256'] += '-' + role + value['files'] += [m, p] + expected = self.baseline()['entries'] + mapped = race.map_fixture(before, self.bucket, expected) + source, destination = ((race.G06_TARGET_KEY, race.G06_REFERENCE_KEY) if branch == 'copy-source' + else (race.G06_REFERENCE_KEY, race.G06_TARGET_KEY)) + after = copy.deepcopy(before) + for value in after.values(): + by_path = {(f['disk'], f['path']): f for f in value['files']} + for f in value['files']: + if not f['path'].startswith(f'{self.bucket}/{destination}/'): continue + if f['path'].endswith('/xl.meta'): + f['metadata']['versions'][0]['dataDir'] = str(uuid.UUID(int=300)) + else: + slot = 100 if source == race.G06_TARGET_KEY else 200 + f['sha256'] = by_path[f['disk'], f'{self.bucket}/{source}/{uuid.UUID(int=slot)}/part.1']['sha256'] + f['path'] = f'{self.bucket}/{destination}/{uuid.UUID(int=300)}/part.1' + return before, after, mapped, {'entries': expected}, destination + + def test_real_copy_transition_accepts_both_directions_before_get(self): + for branch in ('copy-source', 'copy-destination'): + with self.subTest(branch=branch): + before, after, mapped, final, _ = self.scene(branch) + _, proof = race.physical_transition(before, after, self.bucket, mapped, final, branch) + self.assertTrue(proof['physicalProofBeforeGETOrRescue']) + + def test_copy_rejects_wrong_source_shards_and_resurrected_destination_generation(self): + for branch in ('copy-source', 'copy-destination'): + for fault in ('wrong-shard', 'old-generation', 'source-shard'): + with self.subTest(branch=branch, fault=fault): + before, after, mapped, final, destination = self.scene(branch) + part = next(f for f in after['node1']['files'] if f['path'].startswith(f'{self.bucket}/{destination}/') + and f['path'].endswith('/part.1')) + if fault == 'wrong-shard': part['sha256'] = 'wrong-copy' + elif fault == 'old-generation': + old = next(f for f in before['node1']['files'] if f['path'].startswith(f'{self.bucket}/{destination}/') + and f['path'].endswith('/part.1')) + after['node1']['files'].append(copy.deepcopy(old)) + else: + source = race.G06_TARGET_KEY if branch == 'copy-source' else race.G06_REFERENCE_KEY + original = next(f for f in after['node1']['files'] if f['path'].startswith(f'{self.bucket}/{source}/') + and f['path'].endswith('/part.1')) + original['sha256'] = 'changed-copy-source' + with self.assertRaises(ValueError): + race.physical_transition(before, after, self.bucket, mapped, final, branch) + + +if __name__ == '__main__': + unittest.main() diff --git a/test_validation_heal_race_native.py b/test_validation_heal_race_native.py index 5ebe8f60..270c2c87 100644 --- a/test_validation_heal_race_native.py +++ b/test_validation_heal_race_native.py @@ -139,6 +139,53 @@ def test_multipart_native_source_collects_all_ten_events_without_enabling_incomp with self.assertRaisesRegex(ValueError, 'invented or omitted'): consumer.verify(root, candidate, HELPER, case) + def test_default_owner_and_copy_source_collect_exact_groups_without_runtime_admission(self): + source = '757b751482bb970bc321e92059690a7698554358' + candidate = {**CANDIDATE, 'source_sha': source} + contract = suite.contract_for(source) + self.assertEqual(len(contract['regressions']), 10) + self.assertEqual(sum(len(v['milestones']) for v in contract['regressions'].values()), 104) + self.assertIn('crates/ecstore/src/bucket/metadata_sys.rs', contract['sourceFiles']) + self.assertIn('crates/ecstore/src/set_disk/mod.rs', contract['sourceFiles']) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.fixture(root, source=source) + with patch.object(suite.subprocess, 'run', side_effect=AssertionError('must not execute artifacts')): + for case, label, count in [ + ('heal.admin-nolock-safety', 'default-heal-exclusive-owner', 16), + ('heal.race-copy-and-metadata-update', 'copy-and-metadata-final-rename', 24), + ]: + with self.subTest(case=case): + self.assertEqual(contract['cases'][case], [label]) + self.assertEqual(len(contract['regressions'][label]['milestones']), count) + proof = consumer.verify(root, candidate, HELPER, case) + self.assertEqual(proof['requiredRegressionLabels'], [label]) + self.assertTrue(proof['requiredNativeSubsetPassed']) + self.assertFalse(proof['wholeCasePassClaimed']) + self.assertFalse(proof['canonicalCasePassClaimed']) + for case in race.CASE_IDS: + with self.subTest(case=case), self.assertRaisesRegex(ValueError, 'case contract is not reviewed'): + race.native_preflight(candidate, HELPER, case, root) + + def test_default_owner_or_copy_cannot_omit_reverse_or_invent_native_events(self): + source = '757b751482bb970bc321e92059690a7698554358' + candidate = {**CANDIDATE, 'source_sha': source} + for case, label in [('heal.admin-nolock-safety', 'default-heal-exclusive-owner'), + ('heal.race-copy-and-metadata-update', 'copy-and-metadata-final-rename')]: + for mutation in ('omit', 'reverse', 'invent'): + with self.subTest(case=case, mutation=mutation), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.fixture(root, source=source) + summary = root / 'native-evidence/race-native-results.json' + value = json.loads(summary.read_text()) + row = next(row for row in value['results'] if row['label'] == label) + if mutation == 'omit': row['milestones'].pop() + elif mutation == 'reverse': row['milestones'].reverse() + else: row['milestones'].append(row['milestones'][0] + '-invented') + self.write(summary, value) + with self.assertRaisesRegex(ValueError, 'invented or omitted'): + consumer.verify(root, candidate, HELPER, case) + def fixture(self, root, *, source=SOURCE): tools, evidence = root / 'native-tools', root / 'native-evidence' tools.mkdir() From 123d862d1e04bd47399d8e13221f966c9b91177b Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 11:59:43 +0800 Subject: [PATCH 263/343] fix(validation): migrate owned race write bandwidth controls --- docs/validation-heal-operations.md | 2 + scripts/validation_heal/fleet_node.py | 1 + scripts/validation_heal/healthy.py | 3 + scripts/validation_heal/race.py | 19 ++- scripts/validation_heal/race_io.py | 120 +++++++++++++++ test_validation_heal_race_io.py | 212 ++++++++++++++++++++++++++ 6 files changed, 354 insertions(+), 3 deletions(-) create mode 100644 scripts/validation_heal/race_io.py create mode 100644 test_validation_heal_race_io.py diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index aa688d8b..156c4963 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -672,6 +672,8 @@ Runtime sources opt in to specific cases separately from the native registry. Th All original H1/H2 black-box branches, SDK ACKs, manifests, overlap witnesses, exact Admin settings and physical/S3 checks remain required. Native logical disk shapes do not certify distributed VM execution. The original `072b57b37559fbe626aeda8abf61617fdcbdd926` route retains its blocked native rows. A09, G05/G06 and I05 still require their own complete native groups and executor evidence. G04 also remains unaccepted until its same-source hosted evidence and every original live branch pass independent audit. No source or case gains canonical acceptance from a registry entry or local native test; a successful exact-source hosted bundle, every live branch and independent original-evidence audit are required. +The G01-G04/G06 public race adapter retains the frozen fourth-physical-drive fault selection, independent of shuffled data/parity role. H2 selects that local drive on its first original host. After faults and restart, `race_io.py` applies the source-defined `65536` bytes/s write cap only to the run-owned unit on actual fault hosts. Exact process birth/candidate, all four original mount identities, systemd cgroup and `/proc` membership are checked; the actual kernel `io.max` must show the original fourth device and rate. Existing foreign limits, changed identity, wrong devices and unbound release reject. Intent is recorded before each single control request. After the foreground sequence, the cap is removed and kernel readback is verified before waiting for the same original task. This adds no sleep to SDK timing and changes no payload, overlap oracle or task timeout. An interrupted or uncertain control still uses existing owned-service stop and retained-scene rules. G05 refuses this cap and retains its source-defined unrestricted process I/O. + ### A09/G06 native groups and frozen G06 public fixtures Source `757b751482bb970bc321e92059690a7698554358` adds two collector-only groups. `default-heal-exclusive-owner` observes actual PUT and exact historical DELETE lock futures becoming pending while default Deep Heal exclusively owns a real staged external repair. Physical reconstruction precedes foreground publication; all acknowledged successor or retained-current shards and full bodies are checked in 4/2 and 16/4. Its sixteen events prove the storage fence, while Admin `nolock=true` normalization and public request timing still require separate evidence. `copy-and-metadata-final-rename` covers real streamed cross-key COPY source/destination, self metadata replacement and tags. Twenty-four events bind actual ACK and physical-tail completion before releasing shared reconstruction, exact stale refusal or unchanged-source repair, final source/target/control payloads, metadata and tags. The registry binds all five storage fixture/barrier source files. This candidate remains outside runtime admission until the complete executor, same-source hosted artifacts and workflow admission are ready. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index ef90d0e7..7933f246 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1818,6 +1818,7 @@ def main(request): actions["mpu-scene"] = lambda value: globals()["mpu_scene_backend"].run(value, globals()) actions["mpu-race-scene"] = lambda value: globals()["mpu_race_backend"].run(value, globals()) actions["network-delete-records"] = lambda value: globals()["network_delete_records_backend"].run(value, globals()) + actions["race-io-cap"] = lambda value: globals()["race_io_backend"].run(value, globals()) actions["process-clock"] = process_clock actions["environment-capabilities"] = environment_capabilities for action in ("retained-scene-audit", "retained-scene-isolate"): diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 3038d268..fbc06eac 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -185,6 +185,9 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() + if action == "race-io-cap": + source += "\nimport types\nrace_io_backend = types.ModuleType('race_io')\nexec(" + repr( + NODE_SOURCE.with_name("race_io.py").read_text()) + ", race_io_backend.__dict__)\n" if action == "mpu-race-scene": source += "\nimport types\nmpu_race_backend = types.ModuleType('mpu_race_physical')\nexec(" + repr( NODE_SOURCE.with_name("mpu_race_physical.py").read_text()) + ", mpu_race_backend.__dict__)\n" diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index db440f25..1cb00e1c 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -30,6 +30,7 @@ from .cursor import validate_missing_batch, verify_part_manifest from .versions import public_version_id from .race_native_evidence import verify as verify_native + from .race_io import set_fleet as set_race_io_cap except ImportError: from heal_probe import Probe, canonical, compare_inventories, payload, require, save from heal_workload import Recorder, seed as seed_workload, verify_seed @@ -41,6 +42,7 @@ from cursor import validate_missing_batch, verify_part_manifest from versions import public_version_id from race_native_evidence import verify as verify_native + from race_io import set_fleet as set_race_io_cap CASE_IDS = ("heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-markers-and-batch-delete", "heal.race-multipart-complete-abort", "heal.race-copy-and-metadata-update") @@ -512,9 +514,16 @@ def object_scene(snapshot, bucket, key): def map_fixture(snapshot, bucket, expected): live_expected = [row for row in expected if not row.get("deleteMarker")] - mapped = {(row["key"], row["versionId"]): map_target( - object_scene(snapshot, bucket, row["key"]), bucket, row["key"], "data", 1, - row["versionId"], latest_required=False) for row in live_expected} + mapped = {} + for row in live_expected: + target = map_target(object_scene(snapshot, bucket, row["key"]), bucket, row["key"], "data", 1, + row["versionId"], latest_required=False) + # Frozen H1 removes the fourth physical drive's shard, regardless of + # shuffled coding role. H2 uses the same local slot on its first host. + selected = next((part for part in target["cohort"] + if part["node"] == sorted(snapshot)[0] and part["disk"] == 3), None) + require(selected is not None, "race fixture lacks its original fourth-drive member") + mapped[row["key"], row["versionId"]] = {**target, **selected} require(len(mapped) == len(live_expected), "G01/G04 physical map omitted/duplicated an ACK") return mapped @@ -884,6 +893,8 @@ def run_branch(args, topology, nodes, branch): faulted = fleet.snapshot("complete-fault-physical", quiescent=True) fleet.record("fault-batch-proof", validate_missing_batch(before, faulted, targets, receipts)) fleet.start() + if branch not in G05_BRANCHES: + set_race_io_cap(fleet, args.case_id, branch, True, nodes=sorted({target['node'] for target in targets})) require(physical_content(faulted) == physical_content(fleet.snapshot("attribution-before-START")), "NOT_OBSERVED: G01/G03 missing shards changed before manual admission") path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") @@ -904,6 +915,8 @@ def run_branch(args, topology, nodes, branch): if branch == "batch-version-delete": writer.batch_expected_ids = {item["VersionId"] for item in plan[0]["params"]["Delete"]["Objects"]} responses, overlapped = race(writer, plan, probe, fleet, path, token, reference) + if branch not in G05_BRANCHES: + set_race_io_cap(fleet, args.case_id, branch, False, nodes=sorted({target['node'] for target in targets})) mpu_runtime = plan[0].get("runtime") if branch in G05_BRANCHES else None if branch in G05_BRANCHES: verify_multipart_state(probe, fleet, bucket, branch, mpu_runtime, diff --git a/scripts/validation_heal/race_io.py b/scripts/validation_heal/race_io.py new file mode 100644 index 00000000..ea7ff823 --- /dev/null +++ b/scripts/validation_heal/race_io.py @@ -0,0 +1,120 @@ +"""Frozen race write bandwidth control on one owned unit and its fourth disk.""" +from pathlib import Path +import re + +RATE = '65536' +CAP_BRANCHES = { + 'heal.admin-nolock-safety': ('overwrite-nolock', 'delete-nolock'), + 'heal.race-unversioned-overwrite': ('overwrite-vs-deep', 'overwrite-vs-read-repair'), + 'heal.race-versioned-put': ('new-version-vs-historical-heal', 'read-all-versions-during-heal'), + 'heal.race-version-delete': ('delete-old-version-vs-deep', 'delete-current-version-vs-deep'), + 'heal.race-markers-and-batch-delete': ('create-marker', 'delete-marker', 'batch-version-delete', 'unversioned-delete'), + 'heal.race-copy-and-metadata-update': ('copy-source', 'copy-destination', 'replace-metadata', 'update-tags'), +} + + +def require(value, message): + if not value: + raise ValueError(message) + + +def limits(raw): + require(isinstance(raw, str) and len(raw) <= 65536, 'bounded cgroup IO evidence required') + out = {} + for line in raw.splitlines(): + parts = line.split() + require(len(out) < 64 and len(parts) == 5 and re.fullmatch(r'[0-9]+:[0-9]+', parts[0]) + and parts[0] not in out, 'invalid or duplicate cgroup IO device') + values = {} + for item in parts[1:]: + pair = item.split('=') + require(len(pair) == 2 and pair[0] in {'rbps', 'wbps', 'riops', 'wiops'} and pair[0] not in values + and (pair[1] == 'max' or re.fullmatch(r'[1-9][0-9]{0,19}', pair[1])), + 'invalid or duplicate cgroup IO limit') + values[pair[0]] = pair[1] + out[parts[0]] = values + return out + + +def run(request, node): + require(set(request) == {'runId', 'action', 'caseId', 'branch', 'enabled', 'expectedProcess'} + and request['action'] == 'race-io-cap' and type(request['enabled']) is bool + and request['branch'] in CAP_BRANCHES.get(request['caseId'], ()), + 'write cap requires a registered frozen public race; G05 forbids this cap') + state, roots, owner = node['owned'](request['runId']) + require(owner.get('runtimeProfile') == 'manual' and len(roots) == len(node['MOUNTS']) == 4, + 'write cap requires the original four-drive manual owner') + expected = request['expectedProcess'] + def process(): + actual = node['process'](request) + require(isinstance(expected, dict) and type(actual.get('pid')) is int and actual['pid'] > 1 + and all(k in expected and actual.get(k) == expected[k] for k in node['PROCESS_FIELDS']), + 'write cap refuses a changed process birth or candidate') + require(actual['unit'] == owner['unit'] and actual['binarySha256'] == owner['binarySha256'], + 'write cap process escaped its original owner') + return actual + original = process() + require(node['mounts']() == owner['mounts'], 'write cap original mount identities changed') + target = owner['mounts'][3] + require(target['target'] == str(node['MOUNTS'][3]) and re.fullmatch(r'[0-9]+:[0-9]+', target['maj:min']), + 'write cap must target the original fourth physical disk') + group = node['command']('systemctl', 'show', '--value', '-p', 'ControlGroup', owner['unit']).stdout.strip() + require(group == '/system.slice/' + owner['unit'] and '/' not in owner['unit'], + 'write cap requires the exact owned systemd cgroup') + cgroup = Path('/sys/fs/cgroup') / group.lstrip('/') + def scope(): + systemd_group = node['command']('systemctl', 'show', '--value', '-p', 'ControlGroup', owner['unit']).stdout.strip() + membership = node['command']('cat', f'/proc/{original["pid"]}/cgroup').stdout + require(systemd_group == group and + [line for line in membership.splitlines() if line.startswith('0:')] == ['0::' + group], + 'write cap process or unit moved to another cgroup') + return {'systemdControlGroup': systemd_group, 'processCgroup': membership} + scope() + def evidence(): + value = node['command']('cat', str(cgroup / 'io.max'), check=False) + if value.returncode: + # A fresh unthrottled unit need not have its IO controller enabled. + # Verify the group still exists; any other cat error remains fatal. + require(value.returncode == 1 and 'No such file or directory' in value.stderr + and node['command']('test', '-d', str(cgroup), check=False).returncode == 0, + 'owned cgroup IO limits could not be read') + return '', {} + return value.stdout, limits(value.stdout) + raw_before, before = evidence() + device = target['maj:min'] + defaults = dict.fromkeys(('rbps','wbps','riops','wiops'), 'max') + prior = before.get(device, defaults) + require(all(v == {**defaults, 'wbps': RATE if d == device and not request['enabled'] else 'max'} + for d,v in before.items()) and (request['enabled'] or prior['wbps'] == RATE), + 'write cap refuses an existing foreign limit or an unbound release') + process() + scope_before = scope() + node['command']('systemctl', 'set-property', '--runtime', owner['unit'], + 'IOWriteBandwidthMax=' + (target['target'] + ' ' + RATE if request['enabled'] else '')) + raw_after, after = evidence() + wanted = {**defaults, 'wbps': RATE if request['enabled'] else 'max'} + require(after.get(device, defaults) == wanted and + all(v == (wanted if d == device else defaults) for d,v in after.items()), + 'kernel cgroup IO write cap did not match the exact original device/rate') + process() + scope_after = scope() + require(node['owned'](request['runId']) == (state, roots, owner) and node['mounts']() == owner['mounts'], + 'write cap owner or mount changed during control') + return {'enabled': request['enabled'], 'writeBytesPerSecond': int(RATE) if request['enabled'] else None, + 'disk': 3, 'device': device, 'mount': target['target'], 'controlGroup': group, + 'beforeIoMax': raw_before, 'afterIoMax': raw_after, 'process': original, + 'beforeScope': scope_before, 'afterScope': scope_after, + 'actualKernelLimitVerified': True, 'publicOverlapProved': False, 'canonicalCasePassClaimed': False} + + +def set_fleet(fleet, case_id, branch, enabled, *, nodes): + require(type(enabled) is bool and branch in CAP_BRANCHES.get(case_id, ()) + and isinstance(nodes, list) and nodes and len(set(nodes)) == len(nodes) + and set(nodes) <= set(fleet.nodes), + 'write cap is not part of this frozen public race') + for host in nodes: + process = fleet.record('race-io-process', {'node': host, **fleet.remote(host, 'process')}) + expected = {key: value for key, value in process.items() if key != 'node'} + request = {'caseId': case_id, 'branch': branch, 'enabled': enabled, 'expectedProcess': expected} + fleet.record('race-io-cap-intent', {'node': host, **request}) + fleet.record('race-io-cap-applied', {'node': host, **fleet.remote(host, 'race-io-cap', **request)}) diff --git a/test_validation_heal_race_io.py b/test_validation_heal_race_io.py new file mode 100644 index 00000000..bb089636 --- /dev/null +++ b/test_validation_heal_race_io.py @@ -0,0 +1,212 @@ +"""Owned cgroup control boundaries; mock kernel evidence is not live proof.""" +import copy +from contextlib import ExitStack +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +from scripts.validation_heal import fleet_node, race, race_io +from scripts.validation_heal.healthy import Fleet +import test_validation_heal as physical_fixtures + + +class RaceIoTests(unittest.TestCase): + def test_actual_mapper_selects_fourth_physical_drive_independent_of_coding_role(self): + scene = physical_fixtures.HealShardOracleTest().snapshot() + entry = next(v for v in scene.values())['files'][0] + bucket, key = entry['metadata']['bucket'], entry['metadata']['key'] + mapped = race.map_fixture(scene,bucket,[{'key':key,'versionId':'null','deleteMarker':False}]) + target = mapped[key,'null'] + self.assertEqual((target['node'],target['disk']),('node1',3)) + self.assertEqual(target['role'],'parity') + self.assertEqual(len(target['cohort']),4) + + def fixture(self): + process = {k: k for k in fleet_node.PROCESS_FIELDS} + process.update(pid=1234, startTicks=17, unit='owned.service', binarySha256='candidate') + mounts = [{'target': f'/data/rustfs{i+1}', 'maj:min': f'8:{i+1}', 'uuid': str(i)} for i in range(4)] + owner = {'runtimeProfile':'manual', 'unit':'owned.service', 'binarySha256':'candidate', 'mounts':mounts} + kernel = {'raw':'', 'writes':[], 'wrongReadback':False, 'membership':'0::/system.slice/owned.service\n', + 'group':'/system.slice/owned.service\n'} + def command(*args, **kwargs): + if args[:2] == ('systemctl', 'show'): raw = kernel['group'] + elif args[:2] == ('cat', '/proc/1234/cgroup'): raw = kernel['membership'] + elif args[:1] == ('cat',): + self.assertEqual(args[1], '/sys/fs/cgroup/system.slice/owned.service/io.max') + raw = kernel['raw'] + elif args[:2] == ('systemctl', 'set-property'): + self.assertEqual(args[2:5], ('--runtime', 'owned.service', args[4])) + kernel['writes'].append(args) + enabled = args[4] == 'IOWriteBandwidthMax=/data/rustfs4 65536' + self.assertIn(args[4], ('IOWriteBandwidthMax=/data/rustfs4 65536', 'IOWriteBandwidthMax=')) + kernel['raw'] = '8:4 rbps=max wbps=' + ('65536' if enabled and not kernel['wrongReadback'] else 'max') + ' riops=max wiops=max\n' + raw = '' + else: raise AssertionError(args) + return SimpleNamespace(stdout=raw, stderr='', returncode=0) + node = {'owned':lambda run:(Path('/state'), tuple(Path(f'/root{i}') for i in range(4)), copy.deepcopy(owner)), + 'MOUNTS':fleet_node.MOUNTS, 'PROCESS_FIELDS':fleet_node.PROCESS_FIELDS, + 'process':lambda request:copy.deepcopy(process), 'mounts':lambda:copy.deepcopy(mounts), 'command':command} + request = {'runId':'gh-12345-1-h1', 'action':'race-io-cap', 'caseId':race.CASE_IDS[0], + 'branch':'overwrite-vs-deep', 'enabled':True, 'expectedProcess':copy.deepcopy(process)} + return request, node, kernel, process, owner + + def test_enable_and_release_require_exact_kernel_fourth_device_and_original_process(self): + request, node, kernel, process, owner = self.fixture() + enabled = race_io.run(request, node) + self.assertTrue(enabled['actualKernelLimitVerified']) + self.assertEqual((enabled['disk'], enabled['device'], enabled['writeBytesPerSecond']), (3,'8:4',65536)) + self.assertEqual(enabled['process'], process) + self.assertFalse(enabled['publicOverlapProved'] or enabled['canonicalCasePassClaimed']) + released = race_io.run({**request, 'enabled':False}, node) + self.assertIsNone(released['writeBytesPerSecond']) + self.assertEqual(len(kernel['writes']), 2) + self.assertEqual(owner['mounts'][3]['target'], '/data/rustfs4') + + def test_unregistered_g05_bad_identity_membership_or_foreign_limit_refuse_before_write(self): + for fault in ('g05','unknown-branch','boolean','extra','birth','runtime','membership','group','foreign','unbound-release'): + with self.subTest(fault=fault): + request, node, kernel, process, owner = self.fixture() + if fault == 'g05': request.update(caseId=race.CASE_IDS[4], branch='complete') + elif fault == 'unknown-branch': request['branch'] = 'unknown' + elif fault == 'boolean': request['enabled'] = 1 + elif fault == 'extra': request['device'] = '/etc/passwd' + elif fault == 'birth': process['startTicks'] += 1 + elif fault == 'runtime': owner['runtimeProfile'] = 'root-restart' + elif fault == 'membership': kernel['membership'] = '0::/system.slice/another.service\n' + elif fault == 'group': kernel['group'] = '/system.slice/owned.service/../../foreign.service\n' + elif fault == 'foreign': kernel['raw'] = '8:1 rbps=max wbps=65536 riops=max wiops=max\n' + else: request['enabled'] = False + with self.assertRaises(ValueError): race_io.run(request, node) + self.assertFalse(kernel['writes']) + + def test_wrong_kernel_readback_cannot_claim_effective_cap(self): + request, node, kernel, process, _ = self.fixture() + kernel['wrongReadback'] = True + with self.assertRaisesRegex(ValueError, 'kernel cgroup'): race_io.run(request, node) + self.assertEqual(len(kernel['writes']), 1) + + def test_process_or_unit_cgroup_move_during_control_cannot_claim_effective_cap(self): + for move in ('process','unit'): + with self.subTest(move=move): + request, node, kernel, _, _ = self.fixture() + original = node['command'] + def command(*args, **kwargs): + value = original(*args,**kwargs) + if args[:2] == ('systemctl','set-property'): + if move == 'process': kernel['membership'] = '0::/system.slice/other.service\n' + else: kernel['group'] = '/system.slice/other.service\n' + return value + node['command'] = command + with self.assertRaisesRegex(ValueError,'cgroup'): race_io.run(request,node) + self.assertEqual(len(kernel['writes']),1) + + def test_process_birth_change_during_control_cannot_claim_effective_cap(self): + request, node, kernel, process, _ = self.fixture() + original = node['process']; calls = 0 + def changing(request): + nonlocal calls + calls += 1 + if calls == 3: process['startTicks'] += 1 + return original(request) + node['process'] = changing + with self.assertRaisesRegex(ValueError, 'process birth'): race_io.run(request, node) + self.assertEqual(len(kernel['writes']), 1) + + def test_kernel_parser_rejects_incomplete_duplicate_or_unbounded_fields(self): + for raw in ('8:4 wbps=65536', '8:4 rbps=max wbps=0 riops=max wiops=max', + '8:4 rbps=max wbps=65536 riops=max rbps=max', + '8:4 rbps=max wbps=max riops=max wiops=max\n'*2, 'x'*65537): + with self.subTest(raw=raw[:80]), self.assertRaises(ValueError): race_io.limits(raw) + + def test_missing_unenabled_controller_is_accepted_only_for_an_existing_owned_group(self): + for group_exists in (True, False): + with self.subTest(group_exists=group_exists): + request, node, kernel, _, _ = self.fixture() + original = node['command']; first = True + def command(*args, **kwargs): + nonlocal first + if args == ('cat','/sys/fs/cgroup/system.slice/owned.service/io.max') and first: + first = False + return SimpleNamespace(returncode=1,stdout='',stderr='cat: io.max: No such file or directory') + if args == ('test','-d','/sys/fs/cgroup/system.slice/owned.service'): + return SimpleNamespace(returncode=0 if group_exists else 1,stdout='',stderr='') + return original(*args,**kwargs) + node['command'] = command + if group_exists: + self.assertTrue(race_io.run(request,node)['actualKernelLimitVerified']) + else: + with self.assertRaisesRegex(ValueError,'limits could not be read'): race_io.run(request,node) + self.assertFalse(kernel['writes']) + + def test_real_remote_adapter_embeds_the_reviewed_node_module(self): + def execute(*args, **kwargs): + source = kwargs['input'].rsplit('\nmain(',1)[0] + scope = {'__name__':'__race_io_test__'} + exec(compile(source,'race-io-node','exec'),scope) + self.assertEqual(scope['race_io_backend'].RATE,'65536') + self.assertTrue(callable(scope['race_io_backend'].run)) + return SimpleNamespace(returncode=0,stdout='{}',stderr='') + with tempfile.TemporaryDirectory() as tmp, patch('scripts.validation_heal.healthy.subprocess.run',side_effect=execute): + Fleet(['vm000'],Path(tmp),'gh-12345-1-h1').remote('vm000','race-io-cap') + + def test_controller_records_intent_before_single_call_and_never_retries_a_lost_result(self): + fleet = Mock(); fleet.nodes = ['vm000','vm001']; trace = [] + def record(label,value): trace.append(label); return value + def remote(host,action,**params): + self.assertEqual(host,'vm001') + if action == 'process': return {'pid':1234,'birth':'original'} + self.assertEqual(action,'race-io-cap') + self.assertEqual(params['expectedProcess'],{'pid':1234,'birth':'original'}) + self.assertEqual(trace,['race-io-process','race-io-cap-intent']) + raise OSError('lost response; caller must stop its owned service') + fleet.record.side_effect = record; fleet.remote.side_effect = remote + with self.assertRaises(OSError): + race_io.set_fleet(fleet,race.CASE_IDS[0],'overwrite-vs-deep',True,nodes=['vm001']) + self.assertEqual(fleet.remote.call_count,2) + self.assertEqual(trace,['race-io-process','race-io-cap-intent']) + for nodes in ([],['foreign'],['vm001','vm001']): + fleet.remote.reset_mock() + with self.subTest(nodes=nodes), self.assertRaises(ValueError): + race_io.set_fleet(fleet,race.CASE_IDS[0],'overwrite-vs-deep',True,nodes=nodes) + fleet.remote.assert_not_called() + + def test_real_runner_arms_before_admin_and_releases_after_foreground_before_terminal(self): + class ReachedTerminalBoundary(BaseException): pass + for case_id, branch in [(race.CASE_IDS[0],'overwrite-vs-deep'),(race.CASE_IDS[4],'upload-continue')]: + with self.subTest(case=case_id), tempfile.TemporaryDirectory() as tmp, ExitStack() as stack: + root = Path(tmp); candidate = root/'candidate.json' + candidate.write_text('{"source_sha":"072b57b37559fbe626aeda8abf61617fdcbdd926","package_sha256":"digest"}') + rc = root/'rc'; rc.write_bytes(b'client') + selected = race.target_key(branch) + expected = [{'key':key,'versionId':'null','isLatest':True} for key in ('control/a','control/b',selected)] + mapped = {(r['key'],'null'):{**r,'disk':0,'node':'vm000','relativePath':r['key'],'sha256':'original'} for r in expected} + trace = [] + fleet = Mock(); fleet.nodes = ['vm000'] + fleet.record.side_effect = lambda label,value:value + fleet.snapshot.return_value = {'vm000':{'files':[],'formats':[]}} + fleet.remote.return_value = {} + fleet.cleanup.return_value = True + probe = Mock(); probe.admin.side_effect = lambda *a:(trace.append('admin') or {'httpStatus':200,'response':{'clientToken':'token'}}) + probe.inventory.return_value = {'entries':expected} + def cap(f,c,b,enabled,*,nodes): + self.assertIs(f,fleet); self.assertEqual(nodes,['vm000']); trace.append('cap-on' if enabled else 'cap-off') + def foreground(*args): trace.append('foreground'); return [],False + def terminal(*args): trace.append('terminal'); raise ReachedTerminalBoundary() + values = {'Fleet':Mock(return_value=fleet),'Probe':Mock(return_value=probe),'verify_inspector':Mock(return_value={}), + 'validate_topology':Mock(),'seed_fixture':Mock(return_value=expected),'verify_seed':Mock(), + 'operation_plan':Mock(return_value=[{}]),'map_fixture':Mock(return_value=mapped),'verify_part_manifest':Mock(), + 'validate_missing_batch':Mock(),'wait_running':Mock(),'race':Mock(side_effect=foreground), + 'final_inventory':Mock(return_value={'entries':expected}),'verify_multipart_state':Mock(), + 'wait_terminal':Mock(side_effect=terminal),'set_race_io_cap':Mock(side_effect=cap),'OBJECTS':3} + for name,value in values.items(): stack.enter_context(patch.object(race,name,value)) + stack.enter_context(patch.dict(race.os.environ,{'GITHUB_RUN_ID':'12345','GITHUB_RUN_ATTEMPT':'1'})) + args = SimpleNamespace(evidence=root/'evidence',candidate=candidate,rc_bin=rc,tool_directory=root, + testing_commit='helper',binary_sha='binary',case_id=case_id) + with self.assertRaises(ReachedTerminalBoundary): race.run_branch(args,'H1',['vm000'],branch) + self.assertEqual(trace[:trace.index('terminal')+1],['cap-on','admin','foreground','cap-off','terminal'] if case_id == race.CASE_IDS[0] + else ['admin','foreground','terminal']) + + +if __name__ == '__main__': unittest.main() From ec822848152035e1ab396345fb5a532eeaeaa10f Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 12:16:00 +0800 Subject: [PATCH 264/343] test(validation): admit exact owner and copy native groups --- docs/validation-heal-operations.md | 6 +- scripts/validation_heal/nolock.py | 110 ++++++++++++++++++++-------- scripts/validation_heal/race.py | 5 ++ scripts/validation_heal/run_case.sh | 4 +- test_validation_heal_nolock.py | 90 +++++++++++++++++++++++ test_validation_heal_race_io.py | 19 +++-- test_validation_heal_race_native.py | 14 +++- 7 files changed, 200 insertions(+), 48 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 156c4963..738039ec 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -668,15 +668,15 @@ Source `8380466f3a27c3c3ef4115084178a0ab99a78f32` additionally requires the comp `race_native_evidence.py` admits the downloaded artifact without executing it. It checks the successful hosted workflow attempt, exact candidate/driver/helper identity, selected source contract, native executable, complete unique regression set and every original log. Each ordered-barrier regression must emit exactly the complete source-registered milestone sequence. Regressions marked as in-test assertions retain that narrower witness type; assertion-only evidence cannot invent emitted barrier events. The result states `requiredNativeSubsetPassed`; whole-case, fleet and canonical PASS remain false. Missing source groups, old-source artifacts, ignored or zero-test output and partial evidence are rejected. Updating another candidate's registry entry does not invalidate an unchanged selected contract. Changing the producer or the selected contract requires a fresh tools artifact. -Runtime sources opt in to specific cases separately from the native registry. The `6af` source enables only G01; `bd0671b` enables G01/G02/G03; `8380466` additionally enables G04 with its complete, distinct native group. The workflow verifies the bundle and invokes the pinned helper's read-only runtime preflight before exporting it to canonical execution. A collector-only helper without the corresponding runtime adapter fails before fleet mutation. The executor repeats admission before package installation or service changes. +Runtime sources opt in to specific cases separately from the native registry. The `6af` source enables only G01; `bd0671b` enables G01/G02/G03; `8380466` additionally enables G04 with its complete, distinct native group. Source `757b751` enables only A09 and G06 on their respective `nolock.py` and `race.py` runtime paths; it does not enable G05 or substitute a different source into G01-G04. The workflow verifies the bundle and invokes the pinned helper's read-only runtime preflight before exporting it to canonical execution. A collector-only helper without the corresponding runtime adapter fails before fleet mutation. The executor repeats admission before package installation or service changes. All original H1/H2 black-box branches, SDK ACKs, manifests, overlap witnesses, exact Admin settings and physical/S3 checks remain required. Native logical disk shapes do not certify distributed VM execution. The original `072b57b37559fbe626aeda8abf61617fdcbdd926` route retains its blocked native rows. A09, G05/G06 and I05 still require their own complete native groups and executor evidence. G04 also remains unaccepted until its same-source hosted evidence and every original live branch pass independent audit. No source or case gains canonical acceptance from a registry entry or local native test; a successful exact-source hosted bundle, every live branch and independent original-evidence audit are required. -The G01-G04/G06 public race adapter retains the frozen fourth-physical-drive fault selection, independent of shuffled data/parity role. H2 selects that local drive on its first original host. After faults and restart, `race_io.py` applies the source-defined `65536` bytes/s write cap only to the run-owned unit on actual fault hosts. Exact process birth/candidate, all four original mount identities, systemd cgroup and `/proc` membership are checked; the actual kernel `io.max` must show the original fourth device and rate. Existing foreign limits, changed identity, wrong devices and unbound release reject. Intent is recorded before each single control request. After the foreground sequence, the cap is removed and kernel readback is verified before waiting for the same original task. This adds no sleep to SDK timing and changes no payload, overlap oracle or task timeout. An interrupted or uncertain control still uses existing owned-service stop and retained-scene rules. G05 refuses this cap and retains its source-defined unrestricted process I/O. +The A09/G01-G04/G06 public race adapters retain the frozen fourth-physical-drive fault selection, independent of shuffled data/parity role. H2 selects that local drive on its first original host. After faults and restart, `race_io.py` applies the source-defined `65536` bytes/s write cap only to the run-owned unit on actual fault hosts. Exact process birth/candidate, all four original mount identities, systemd cgroup and `/proc` membership are checked; the actual kernel `io.max` must show the original fourth device and rate. Existing foreign limits, changed identity, wrong devices and unbound release reject. Intent is recorded before each single control request. After the foreground sequence, the cap is removed and kernel readback is verified before waiting for the same original task. This adds no sleep to SDK timing and changes no payload, overlap oracle or task timeout. An interrupted or uncertain control still uses existing owned-service stop and retained-scene rules. G05 refuses this cap and retains its source-defined unrestricted process I/O. ### A09/G06 native groups and frozen G06 public fixtures -Source `757b751482bb970bc321e92059690a7698554358` adds two collector-only groups. `default-heal-exclusive-owner` observes actual PUT and exact historical DELETE lock futures becoming pending while default Deep Heal exclusively owns a real staged external repair. Physical reconstruction precedes foreground publication; all acknowledged successor or retained-current shards and full bodies are checked in 4/2 and 16/4. Its sixteen events prove the storage fence, while Admin `nolock=true` normalization and public request timing still require separate evidence. `copy-and-metadata-final-rename` covers real streamed cross-key COPY source/destination, self metadata replacement and tags. Twenty-four events bind actual ACK and physical-tail completion before releasing shared reconstruction, exact stale refusal or unchanged-source repair, final source/target/control payloads, metadata and tags. The registry binds all five storage fixture/barrier source files. This candidate remains outside runtime admission until the complete executor, same-source hosted artifacts and workflow admission are ready. +Source `757b751482bb970bc321e92059690a7698554358` adds two collector-only groups. `default-heal-exclusive-owner` observes actual PUT and exact historical DELETE lock futures becoming pending while default Deep Heal exclusively owns a real staged external repair. Physical reconstruction precedes foreground publication; all acknowledged successor or retained-current shards and full bodies are checked in 4/2 and 16/4. Its sixteen events prove the storage fence, while Admin `nolock=true` normalization and public request timing still require separate evidence. `copy-and-metadata-final-rename` covers real streamed cross-key COPY source/destination, self metadata replacement and tags. Twenty-four events bind actual ACK and physical-tail completion before releasing shared reconstruction, exact stale refusal or unchanged-source repair, final source/target/control payloads, metadata and tags. The registry binds all five storage fixture/barrier source files. This candidate admits A09 and G06 only with their exact case-specific native group and matching hosted helper/source artifact. Both workflow and executor run read-only preflight before bootstrap. A09 always sends wire `nolock=true` and requires effective `nolock=false`, plus both original H1/H2 public branches and the separate storage-native proof. G05 remains outside runtime admission. The G06 public adapter uses the existing DS workload: all 257 original keys, including Chinese, spaces, literal `%2F`, actual slashes and parent-like prefixes. The observed missing key is `中文/文件/0252.bin`; its companion is `literal%2Fslash/0254.bin`. `copy-source` copies from the observed key into the companion; `copy-destination` copies in the reverse direction. Default COPY expectations inherit the source payload, content properties, metadata and tags. Both directions verify the unchanged source, exact destination coding-index shard bytes and a distinct destination generation before GET, rejecting retained old destination directories. Registry, fixture and local transition tests do not establish public overlap or complete canonical PASS. diff --git a/scripts/validation_heal/nolock.py b/scripts/validation_heal/nolock.py index c65e08bb..ce498b40 100644 --- a/scripts/validation_heal/nolock.py +++ b/scripts/validation_heal/nolock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""A09 public-call races; a blocked native fence never certifies the case.""" +"""A09 normalized public races and separately bound native storage fences.""" from __future__ import annotations @@ -26,9 +26,12 @@ from .inspector import verify as verify_inspector from .matrix import select from .heal_fault import bounded_fault_id - from .shards import files, map_target, validate_repaired_physical + from .shards import files, validate_repaired_physical from .cursor import validate_missing_batch, verify_part_manifest from .versions import public_version_id + from .race import COPY_CANDIDATE, COPY_REFERENCE, map_fixture as map_race_fixture + from .race_native_evidence import verify as verify_native + from .race_io import set_fleet as set_race_io_cap except ImportError: from heal_probe import Probe, canonical, compare_inventories, payload, require, save from heal_workload import Recorder, verify_seed @@ -36,9 +39,12 @@ from inspector import verify as verify_inspector from matrix import select from heal_fault import bounded_fault_id - from shards import files, map_target, validate_repaired_physical + from shards import files, validate_repaired_physical from cursor import validate_missing_batch, verify_part_manifest from versions import public_version_id + from race import COPY_CANDIDATE, COPY_REFERENCE, map_fixture as map_race_fixture + from race_native_evidence import verify as verify_native + from race_io import set_fleet as set_race_io_cap CASE_ID = "heal.admin-nolock-safety" @@ -57,6 +63,29 @@ "effective": {**BASE_OPTIONS, "recursive": True, "nolock": False, "pool": None, "set": None}, "nativeFence": "pinned release has no reviewed e2e-test-hooks artifact or exact Admin lock/commit barrier", } +NATIVE_REFERENCE = {**REFERENCE, 'candidateCommit': COPY_CANDIDATE, + 'sourceFiles': dict(COPY_REFERENCE['sourceFiles']), + 'nativeFence': 'exact-source default Deep exclusive owner fences queued PUT and exact historical DELETE'} +NATIVE_GROUP = ['default-heal-exclusive-owner'] + + +def source_contract(candidate): + for reference in (REFERENCE, NATIVE_REFERENCE): + if candidate.get('source_sha') == reference['candidateCommit']: + return reference + raise ValueError('A09 candidate contract is not reviewed') + + +def native_preflight(candidate, testing_commit, directory): + reference = source_contract(candidate) + if reference is REFERENCE: + return None + require(bool(directory), 'A09 requires exact-source native proof before fleet mutation') + proof = verify_native(directory, candidate, testing_commit, CASE_ID) + require(proof['requiredRegressionLabels'] == NATIVE_GROUP and proof['requiredNativeSubsetPassed'] is True + and proof['wholeCasePassClaimed'] is False and proof['canonicalCasePassClaimed'] is False, + 'A09 exact default-owner native group is incomplete') + return proof class WindowNotObserved(ValueError): @@ -91,20 +120,20 @@ def call(self, operation, params, body=None, **kwargs): return result -def settings(observed): +def settings(observed, reference=REFERENCE): require(observed.get("httpStatus") == 200 and isinstance(observed.get("response"), dict), "A09 status has an unknown or failed HTTP response") value = observed["response"] require(value.get("summary") in {"running", "finished"}, "A09 original task disappeared, stopped or failed") # Canonical JSON distinguishes bool/int and preserves raw null selectors. - require(canonical(value.get("settings")) == canonical(REFERENCE["effective"]), + require(canonical(value.get("settings")) == canonical(reference["effective"]), "A09 Admin nolock was not safely normalized or changed unrelated options") return value["settings"] -def sample_overlap(ack, observed, path, token): +def sample_overlap(ack, observed, path, token, reference=REFERENCE): foreground = ack.get("observation") or {} - settings(observed) + settings(observed, reference) value = observed["response"] a, z = foreground.get("callStartedMonotonicNs"), foreground.get("callFinishedMonotonicNs") b, c = observed.get("callStartedMonotonicNs"), observed.get("callFinishedMonotonicNs") @@ -184,9 +213,7 @@ def object_scene(snapshot, bucket, key): def map_fixture(snapshot, bucket, expected): - mapped = {(row["key"], row["versionId"]): map_target( - object_scene(snapshot, bucket, row["key"]), bucket, row["key"], "data", 1, - row["versionId"], latest_required=False) for row in expected} + mapped = map_race_fixture(snapshot, bucket, expected) require(len(mapped) == len(expected), "A09 physical map omitted/duplicated an ACK") return mapped @@ -268,16 +295,16 @@ def terminal_proof(status, bucket, before, final): "counters": counters, "exactRaceEnumerationClaimed": False} -def observe(probe, fleet, path, token): +def observe(probe, fleet, path, token, reference=REFERENCE): value = fleet.record("original-token-status", probe.admin(path, [("clientToken", token)], b"")) - settings(value) + settings(value, reference) return value -def wait_running(probe, fleet, path, token, total): +def wait_running(probe, fleet, path, token, total, reference=REFERENCE): deadline = time.monotonic() + 30 while True: - observed = observe(probe, fleet, path, token) + observed = observe(probe, fleet, path, token, reference) value = observed["response"] processed = (value.get("outcome") or {}).get("counters", {}).get("processed") if value["summary"] == "finished": @@ -288,10 +315,10 @@ def wait_running(probe, fleet, path, token, total): time.sleep(.05) -def wait_terminal(probe, fleet, path, token): +def wait_terminal(probe, fleet, path, token, reference=REFERENCE): deadline = time.monotonic() + 240 while True: - current = observe(probe, fleet, path, token) + current = observe(probe, fleet, path, token, reference) if current["response"]["summary"] == "finished": return current["response"] require(time.monotonic() < deadline, "A09 original task exceeded its terminal budget") time.sleep(.2) @@ -305,7 +332,7 @@ def foreground(recorder, plan): return responses -def race(recorder, plan, probe, fleet, path, token): +def race(recorder, plan, probe, fleet, path, token, reference=REFERENCE): samples = [] # Only the worker mutates Recorder state. All Fleet.record writes run on # the controller, so evidence sequence allocation never races. @@ -314,7 +341,7 @@ def race(recorder, plan, probe, fleet, path, token): deadline = time.monotonic() + 120 try: while not worker.done() and time.monotonic() < deadline: - samples.append(observe(probe, fleet, path, token)) + samples.append(observe(probe, fleet, path, token, reference)) time.sleep(.01) except BaseException: recorder.mutations.poison("status observation failed; drain in-flight mutations only") @@ -324,7 +351,7 @@ def race(recorder, plan, probe, fleet, path, token): acks = [json.loads(path.read_text()) for path in sorted(recorder.output.glob("*-ack.json"))] require(len(acks) == len(plan) and all(row.get("operationCheck") == "PASS" for row in acks), "A09 lacks every durable foreground ACK") - overlaps = [any(sample_overlap(ack, sample, path, token) for sample in samples) for ack in acks] + overlaps = [any(sample_overlap(ack, sample, path, token, reference) for sample in samples) for ack in acks] fleet.record("public-call-overlap-proof", {"everyMutationOverlapped": all(overlaps), "mutations": overlaps, "nativeFenceProved": False, "serverCommitOverlapProved": False}) return responses, all(overlaps) @@ -336,7 +363,7 @@ def run_branch(args, topology, nodes, branch): run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="manual") candidate = json.loads(args.candidate.read_text()) - require(candidate["source_sha"] == REFERENCE["candidateCommit"], "A09 candidate contract is not reviewed") + reference = source_contract(candidate) fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) @@ -349,7 +376,7 @@ def run_branch(args, topology, nodes, branch): "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), "maxInventoryObjects": 512, "maxInventoryBytes": 512 * 1024**2} fleet.record("binding", binding) - fleet.record("source-contract", {**REFERENCE, "branch": branch, "objectCount": OBJECTS, "objectSize": SIZE, + fleet.record("source-contract", {**reference, "branch": branch, "objectCount": OBJECTS, "objectSize": SIZE, "requiredPUTs": 3 if branch == BLACK_BOX[0] else 0, "serverCommitOverlapProved": False}) probe, active, recorder, writer = Probe(binding), [], None, None result = {"topology": topology, "branch": branch, "fixture": "DL" if branch == BLACK_BOX[0] else "DV", @@ -383,27 +410,30 @@ def run_branch(args, topology, nodes, branch): faulted = fleet.snapshot("complete-fault-physical", quiescent=True) fleet.record("fault-batch-proof", validate_missing_batch(before, faulted, targets, receipts)) fleet.start() + fault_hosts = sorted({target['node'] for target in targets}) + set_race_io_cap(fleet, CASE_ID, branch, True, nodes=fault_hosts) require(physical_content(faulted) == physical_content(fleet.snapshot("attribution-before-START")), "NOT_OBSERVED: A09 missing shards changed before manual admission") path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") - fleet.record("prepared-Admin-request", {"path": path, "query": [], "options": REFERENCE["requested"]}) - start = fleet.record("original-Admin-start", probe.admin(path, [], json.dumps(REFERENCE["requested"]).encode())) + fleet.record("prepared-Admin-request", {"path": path, "query": [], "options": reference["requested"]}) + start = fleet.record("original-Admin-start", probe.admin(path, [], json.dumps(reference["requested"]).encode())) token = start.get("response", {}).get("clientToken") if isinstance(start.get("response"), dict) else None require(start.get("httpStatus") == 200 and isinstance(token, str) and token, "A09 admission lacks an original token") active.append((path, token)) - wait_running(probe, fleet, path, token, len(expected)) + wait_running(probe, fleet, path, token, len(expected), reference) witness = fleet.snapshot("target-missing-before-foreground") target = next(target for target in targets if target["key"] == TARGET_KEY) if (target["node"], target["disk"], target["relativePath"]) in files(witness): raise WindowNotObserved("A09 old target already repaired before foreground mutations") writer = RaceRecorder(probe, directory / "foreground-ACKs", versioned=branch == BLACK_BOX[1]) writer.seen.update((row["key"], row["versionId"]) for row in expected) - responses, overlapped = race(writer, plan, probe, fleet, path, token) + responses, overlapped = race(writer, plan, probe, fleet, path, token, reference) + set_race_io_cap(fleet, CASE_ID, branch, False, nodes=fault_hosts) final = fleet.record("ACK-derived-final-manifest", final_inventory(baseline, branch, plan, responses)) - terminal = wait_terminal(probe, fleet, path, token) + terminal = wait_terminal(probe, fleet, path, token, reference) active.remove((path, token)) for _ in range(2): - repeated = observe(probe, fleet, path, token)["response"] + repeated = observe(probe, fleet, path, token, reference)["response"] require(canonical(repeated.get("outcome")) == canonical(terminal.get("outcome")) and repeated["summary"] == terminal["summary"], "A09 terminal responsibility changed on repetition") # Collect physical evidence even if terminal counters fail. Old target @@ -483,6 +513,16 @@ def complete_results(results): def main(): + if sys.argv[1:2] == ['preflight']: + parser = argparse.ArgumentParser(description='Read-only A09 native admission before bootstrap') + parser.add_argument('--candidate', type=Path, required=True) + parser.add_argument('--case-id', choices=[CASE_ID], required=True) + parser.add_argument('--testing-commit', required=True) + args = parser.parse_args(sys.argv[2:]) + proof = native_preflight(json.loads(args.candidate.read_text()), args.testing_commit, + os.environ.get('VALIDATION_HEAL_NATIVE_DIRECTORY')) + print(json.dumps({'nativeProof':proof,'fleetMutationRequested':False,'canonicalCasePassClaimed':False})) + return def interrupted(signum, frame): raise KeyboardInterrupt(f"A09 interrupted by signal {signum}") signal.signal(signal.SIGTERM, interrupted) parser = argparse.ArgumentParser(description=__doc__) @@ -500,8 +540,16 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"A09 interrupted by sig and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes), "A09 requires four distinct current Heal nodes") candidate = json.loads(args.candidate.read_text()) - require(candidate["source_sha"] == REFERENCE["candidateCommit"], "A09 candidate contract not reviewed") - results = [native_blocked(candidate, topology) for topology in ("H1", "H2")] + native = native_preflight(candidate, args.testing_commit, os.environ.get('VALIDATION_HEAL_NATIVE_DIRECTORY')) + if native is None: + results = [native_blocked(candidate, topology) for topology in ("H1", "H2")] + else: + save(args.evidence/'independent-native-nolock-subset.json', native) + results = [{'caseId':CASE_ID,'topology':topology,'branch':BRANCHES[-1],'verdict':'PASS','cleanup':'PASS', + 'candidateCommit':candidate['source_sha'],'nativeFenceProved':True,'nativeProof':native, + 'nativePhysicalShapes':['4/2','16/4'],'fleetExecutionClaimed':False, + 'distributedFleetExecutionClaimed':False,'canonicalCasePassClaimed':False} + for topology in ('H1','H2')] try: stop = False for topology in ("H1", "H2"): @@ -517,8 +565,8 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"A09 interrupted by sig complete = complete_results(results) save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "complete": complete, "cleanup": "PASS" if all(row["cleanup"] == "PASS" for row in results) else "FAIL", - "canonicalCasePassClaimed": False, "nativePrerequisitesBlocked": True}) - require(complete, "A09 native prerequisites remain BLOCKED; every H1/H2 branch stays in the denominator") + "canonicalCasePassClaimed": False, "nativePrerequisitesBlocked": native is None}) + require(complete, "A09 full race/native coverage incomplete; every H1/H2 branch stays in the denominator") if __name__ == "__main__": diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index 1cb00e1c..f3f565fe 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -93,6 +93,9 @@ MARKER_CANDIDATE = '8380466f3a27c3c3ef4115084178a0ab99a78f32' MARKER_REFERENCE = {**VERSION_REFERENCE, 'candidateCommit': MARKER_CANDIDATE, 'nativeFence': 'exact-source Heal cleanup, version and marker/batch-delete commit fences'} +COPY_CANDIDATE = '757b751482bb970bc321e92059690a7698554358' +COPY_REFERENCE = {**MARKER_REFERENCE, 'candidateCommit': COPY_CANDIDATE, + 'nativeFence': 'exact-source streamed COPY, metadata and tag final-rename fences'} NATIVE_RUNTIME = { NATIVE_CANDIDATE: {'reference': NATIVE_REFERENCE, 'cases': ('heal.race-unversioned-overwrite',)}, VERSION_CANDIDATE: {'reference': VERSION_REFERENCE, @@ -100,6 +103,7 @@ MARKER_CANDIDATE: {'reference': MARKER_REFERENCE, 'cases': ('heal.race-unversioned-overwrite', 'heal.race-versioned-put', 'heal.race-version-delete', 'heal.race-markers-and-batch-delete')}, + COPY_CANDIDATE: {'reference': COPY_REFERENCE, 'cases': ('heal.race-copy-and-metadata-update',)}, } # Registry subsets do not enable a fleet case. Its reviewed runtime source and # complete case-specific native group must both be present before bootstrap. @@ -108,6 +112,7 @@ 'heal.race-versioned-put': ['versioned-read-repair-lock-owner', 'explicit-version-append-final-rename'], 'heal.race-version-delete': ['explicit-version-delete-final-rename'], 'heal.race-markers-and-batch-delete': ['marker-and-batch-delete-final-rename'], + 'heal.race-copy-and-metadata-update': ['copy-and-metadata-final-rename'], } diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index dfebb414..09a7a42d 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -96,8 +96,8 @@ fi [ "${driver}" = healthy.py ] || [ "${#tool_args[@]}" -gt 0 ] || die "physical Heal cases require candidate-bound bitrot tools" evidence="${artifacts}/evidence/${case_id}" mkdir -p "${artifacts}/cases" "${evidence}" -if [ "${driver}" = race.py ]; then - python3 -B "${ROOT}/scripts/validation_heal/race.py" preflight \ +if [ "${driver}" = race.py ] || [ "${driver}" = nolock.py ]; then + python3 -B "${ROOT}/scripts/validation_heal/${driver}" preflight \ --candidate "${candidate}" --case-id "${case_id}" --testing-commit "${testing_commit}" \ > "${evidence}/native-preflight.json" fi diff --git a/test_validation_heal_nolock.py b/test_validation_heal_nolock.py index 4d2bb8eb..1330f4b6 100644 --- a/test_validation_heal_nolock.py +++ b/test_validation_heal_nolock.py @@ -1,13 +1,103 @@ import copy +from contextlib import redirect_stdout +import io import json from pathlib import Path +import tempfile import unittest +from unittest.mock import patch from scripts.validation_heal import nolock from scripts.validation_heal.matrix import AUTOMATIC_CASES, select +import test_validation_heal_race_native as native_fixtures + +HELPER = native_fixtures.HELPER class NolockTests(unittest.TestCase): + def test_source_specific_nolock_wire_true_and_effective_false_are_exact(self): + for source in (nolock.REFERENCE['candidateCommit'],nolock.COPY_CANDIDATE): + with self.subTest(source=source): + ref = nolock.source_contract({'source_sha':source}) + self.assertEqual(ref['candidateCommit'],source) + self.assertIs(ref['requested']['nolock'],True) + self.assertIs(ref['effective']['nolock'],False) + value = {'httpStatus':200,'response':{'summary':'running','settings':copy.deepcopy(ref['effective'])}} + nolock.settings(value,ref) + for field,wrong in [('nolock',True),('pool',0),('set',False),('recursive',1),('readRepair',True)]: + bad = copy.deepcopy(value);bad['response']['settings'][field] = wrong + with self.subTest(field=field),self.assertRaises(ValueError):nolock.settings(bad,ref) + + def test_exact_owner_group_admits_a09_only_and_never_executes_downloaded_native(self): + candidate = {'source_sha':nolock.COPY_CANDIDATE,'package_sha256':'b'*64} + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + native_fixtures.RaceNativeTests().fixture(root,source=nolock.COPY_CANDIDATE) + with patch('scripts.validation_heal.race_native.subprocess.run',side_effect=AssertionError('no executable')): + proof = nolock.native_preflight(candidate,HELPER,root) + self.assertEqual(proof['requiredRegressionLabels'],['default-heal-exclusive-owner']) + self.assertTrue(proof['requiredNativeSubsetPassed']) + self.assertFalse(proof['wholeCasePassClaimed'] or proof['canonicalCasePassClaimed']) + self.assertIsNone(nolock.native_preflight({'source_sha':nolock.REFERENCE['candidateCommit']},HELPER,'')) + for source in ('e'*40,'bd0671ba781d8ebf509f4f03533228dfb44496f1'): + with self.subTest(source=source),self.assertRaises(ValueError): + nolock.native_preflight({**candidate,'source_sha':source},HELPER,root) + with self.assertRaises(ValueError):nolock.native_preflight(candidate,HELPER,'') + with self.assertRaises(ValueError):nolock.native_preflight(candidate,'f'*40,root) + summary = root/'native-evidence/race-native-results.json' + value = json.loads(summary.read_text()) + value['results'] = [r for r in value['results'] if r['label'] != 'default-heal-exclusive-owner'] + summary.write_text(json.dumps(value)) + with self.assertRaisesRegex(ValueError,'omit or duplicate'):nolock.native_preflight(candidate,HELPER,root) + + def test_real_preflight_cli_returns_before_fleet_creation(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp);native = root/'native';native.mkdir() + native_fixtures.RaceNativeTests().fixture(native,source=nolock.COPY_CANDIDATE) + candidate = root/'candidate.json' + candidate.write_text(json.dumps({'source_sha':nolock.COPY_CANDIDATE,'package_sha256':'b'*64})) + argv = ['nolock.py','preflight','--candidate',str(candidate),'--case-id',nolock.CASE_ID,'--testing-commit',HELPER] + output = io.StringIO() + with patch.object(nolock.sys,'argv',argv),patch.dict(nolock.os.environ,{'VALIDATION_HEAL_NATIVE_DIRECTORY':str(native)}),\ + patch.object(nolock,'Fleet',side_effect=AssertionError('preflight must not create fleet')),redirect_stdout(output): + nolock.main() + proof = json.loads(output.getvalue()) + self.assertFalse(proof['fleetMutationRequested'] or proof['canonicalCasePassClaimed']) + self.assertEqual(proof['nativeProof']['requiredRegressionLabels'],nolock.NATIVE_GROUP) + + def test_real_main_keeps_every_public_branch_and_stops_after_dirty_cleanup(self): + for dirty in (False,True): + with self.subTest(dirty=dirty),tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);native=root/'native';native.mkdir();evidence=root/'evidence';evidence.mkdir() + native_fixtures.RaceNativeTests().fixture(native,source=nolock.COPY_CANDIDATE) + candidate=root/'candidate.json' + candidate.write_text(json.dumps({'source_sha':nolock.COPY_CANDIDATE,'package_sha256':'b'*64})) + argv=['nolock.py','--candidate',str(candidate),'--rc-bin',str(root/'rc'),'--evidence',str(evidence), + '--tool-directory',str(root),'--case-id',nolock.CASE_ID,'--testing-commit',HELPER,'--binary-sha','c'*64] + calls=[] + def run(args,topology,nodes,branch): + calls.append((topology,len(nodes),branch)) + return {'topology':topology,'branch':branch,'verdict':'FAIL' if dirty else 'PASS', + 'cleanup':'FAIL' if dirty else 'PASS','canonicalCasePassClaimed':False} + with patch.object(nolock.sys,'argv',argv),patch.object(nolock.signal,'signal'),\ + patch.dict(nolock.os.environ,{'VALIDATION_HEAL_NATIVE_DIRECTORY':str(native),'GITHUB_RUN_ID':'12345', + 'RUSTFS_NODES':'vm000 vm001 vm002 vm003'}),\ + patch.object(nolock,'run_branch',side_effect=run),\ + patch.object(nolock,'Fleet',side_effect=AssertionError('mock public branches do not create fleet')): + if dirty: + with self.assertRaisesRegex(ValueError,'coverage incomplete'):nolock.main() + else:nolock.main() + summary=json.loads((evidence/'healthy-summary.json').read_text()) + self.assertIs(summary['complete'],not dirty) + self.assertFalse(summary['canonicalCasePassClaimed'] or summary['nativePrerequisitesBlocked']) + self.assertEqual(len(calls),1 if dirty else 4) + if not dirty:self.assertEqual(calls,[(top,len(nodes),branch) for top,nodes in + [('H1',['vm000']),('H2',['vm000','vm001','vm002','vm003'])] for branch in nolock.BLACK_BOX]) + native_rows=[r for r in summary['results'] if r['branch']==nolock.BRANCHES[-1]] + self.assertEqual(len(native_rows),2) + self.assertTrue(all(r['nativeFenceProved'] and not r['fleetExecutionClaimed'] + and r['nativeProof']['requiredRegressionLabels']==nolock.NATIVE_GROUP for r in native_rows)) + def test_source_branches_and_fixture_mapping_are_preserved(self): recipe = select(nolock.CASE_ID)[0] self.assertEqual(recipe["variants"], list(nolock.BRANCHES)) diff --git a/test_validation_heal_race_io.py b/test_validation_heal_race_io.py index bb089636..6dae96b1 100644 --- a/test_validation_heal_race_io.py +++ b/test_validation_heal_race_io.py @@ -7,7 +7,7 @@ import unittest from unittest.mock import Mock, patch -from scripts.validation_heal import fleet_node, race, race_io +from scripts.validation_heal import fleet_node, nolock, race, race_io from scripts.validation_heal.healthy import Fleet import test_validation_heal as physical_fixtures @@ -174,12 +174,14 @@ def remote(host,action,**params): def test_real_runner_arms_before_admin_and_releases_after_foreground_before_terminal(self): class ReachedTerminalBoundary(BaseException): pass - for case_id, branch in [(race.CASE_IDS[0],'overwrite-vs-deep'),(race.CASE_IDS[4],'upload-continue')]: + for driver, case_id, branch in [(race,race.CASE_IDS[0],'overwrite-vs-deep'), + (race,race.CASE_IDS[4],'upload-continue'), + (nolock,nolock.CASE_ID,'overwrite-nolock')]: with self.subTest(case=case_id), tempfile.TemporaryDirectory() as tmp, ExitStack() as stack: root = Path(tmp); candidate = root/'candidate.json' candidate.write_text('{"source_sha":"072b57b37559fbe626aeda8abf61617fdcbdd926","package_sha256":"digest"}') rc = root/'rc'; rc.write_bytes(b'client') - selected = race.target_key(branch) + selected = race.target_key(branch) if driver is race else nolock.TARGET_KEY expected = [{'key':key,'versionId':'null','isLatest':True} for key in ('control/a','control/b',selected)] mapped = {(r['key'],'null'):{**r,'disk':0,'node':'vm000','relativePath':r['key'],'sha256':'original'} for r in expected} trace = [] @@ -198,15 +200,16 @@ def terminal(*args): trace.append('terminal'); raise ReachedTerminalBoundary() 'validate_topology':Mock(),'seed_fixture':Mock(return_value=expected),'verify_seed':Mock(), 'operation_plan':Mock(return_value=[{}]),'map_fixture':Mock(return_value=mapped),'verify_part_manifest':Mock(), 'validate_missing_batch':Mock(),'wait_running':Mock(),'race':Mock(side_effect=foreground), - 'final_inventory':Mock(return_value={'entries':expected}),'verify_multipart_state':Mock(), + 'final_inventory':Mock(return_value={'entries':expected}), 'wait_terminal':Mock(side_effect=terminal),'set_race_io_cap':Mock(side_effect=cap),'OBJECTS':3} - for name,value in values.items(): stack.enter_context(patch.object(race,name,value)) + if driver is race: values['verify_multipart_state'] = Mock() + for name,value in values.items(): stack.enter_context(patch.object(driver,name,value)) stack.enter_context(patch.dict(race.os.environ,{'GITHUB_RUN_ID':'12345','GITHUB_RUN_ATTEMPT':'1'})) args = SimpleNamespace(evidence=root/'evidence',candidate=candidate,rc_bin=rc,tool_directory=root, testing_commit='helper',binary_sha='binary',case_id=case_id) - with self.assertRaises(ReachedTerminalBoundary): race.run_branch(args,'H1',['vm000'],branch) - self.assertEqual(trace[:trace.index('terminal')+1],['cap-on','admin','foreground','cap-off','terminal'] if case_id == race.CASE_IDS[0] - else ['admin','foreground','terminal']) + with self.assertRaises(ReachedTerminalBoundary): driver.run_branch(args,'H1',['vm000'],branch) + self.assertEqual(trace[:trace.index('terminal')+1],['admin','foreground','terminal'] if case_id == race.CASE_IDS[4] + else ['cap-on','admin','foreground','cap-off','terminal']) if __name__ == '__main__': unittest.main() diff --git a/test_validation_heal_race_native.py b/test_validation_heal_race_native.py index 270c2c87..284eea3e 100644 --- a/test_validation_heal_race_native.py +++ b/test_validation_heal_race_native.py @@ -35,7 +35,8 @@ def test_new_source_g01_requires_exact_bundle_and_other_race_cases_remain_closed self.assertIsNone(race.native_preflight({'source_sha': race.REFERENCE['candidateCommit']}, HELPER, CASE, '')) self.assertFalse(race.complete_results([], CASE)) shell = Path('scripts/validation_heal/run_case.sh').read_text() - self.assertLess(shell.index('race.py" preflight'), shell.index('(bootstrap)')) + self.assertLess(shell.index('${driver}" preflight'), shell.index('(bootstrap)')) + self.assertIn('[ "${driver}" = nolock.py ]', shell) def test_source_specific_settings_keep_exact_types_selectors_and_lock_policy(self): reference = race.source_contract(CANDIDATE, CASE) @@ -139,7 +140,7 @@ def test_multipart_native_source_collects_all_ten_events_without_enabling_incomp with self.assertRaisesRegex(ValueError, 'invented or omitted'): consumer.verify(root, candidate, HELPER, case) - def test_default_owner_and_copy_source_collect_exact_groups_without_runtime_admission(self): + def test_default_owner_and_copy_source_keep_native_groups_separate_and_only_g06_race_runtime(self): source = '757b751482bb970bc321e92059690a7698554358' candidate = {**CANDIDATE, 'source_sha': source} contract = suite.contract_for(source) @@ -164,8 +165,13 @@ def test_default_owner_and_copy_source_collect_exact_groups_without_runtime_admi self.assertFalse(proof['wholeCasePassClaimed']) self.assertFalse(proof['canonicalCasePassClaimed']) for case in race.CASE_IDS: - with self.subTest(case=case), self.assertRaisesRegex(ValueError, 'case contract is not reviewed'): - race.native_preflight(candidate, HELPER, case, root) + with self.subTest(case=case): + if case == 'heal.race-copy-and-metadata-update': + proof = race.native_preflight(candidate, HELPER, case, root) + self.assertEqual(proof['requiredRegressionLabels'], ['copy-and-metadata-final-rename']) + else: + with self.assertRaisesRegex(ValueError, 'case contract is not reviewed'): + race.native_preflight(candidate, HELPER, case, root) def test_default_owner_or_copy_cannot_omit_reverse_or_invent_native_events(self): source = '757b751482bb970bc321e92059690a7698554358' From 6a0d33b3b731457667c49fb284d704f6a61aaae3 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 12:59:46 +0800 Subject: [PATCH 265/343] fix(validation): isolate unresolved acknowledged race scenes --- scripts/validation_heal/race_retention.py | 144 +++++++++++++ .../validation_heal/recover_environment.py | 7 + scripts/validation_heal/retained_scene.py | 15 +- test_validation_heal_race_retention.py | 203 ++++++++++++++++++ 4 files changed, 363 insertions(+), 6 deletions(-) create mode 100644 scripts/validation_heal/race_retention.py create mode 100644 test_validation_heal_race_retention.py diff --git a/scripts/validation_heal/race_retention.py b/scripts/validation_heal/race_retention.py new file mode 100644 index 00000000..7d1ed886 --- /dev/null +++ b/scripts/validation_heal/race_retention.py @@ -0,0 +1,144 @@ +"""Isolate a stopped G01 scene without replaying writes or restoring old shards.""" + +import hashlib +import json +import re + +try: + from .heal_probe import require + from .healthy import physical_content +except ImportError: + from heal_probe import require + from healthy import physical_content + + +CASE = 'heal.race-unversioned-overwrite' + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def validate(plan): + require(set(plan) == {'kind', 'runId', 'nodes', 'bucket', 'owners', 'businessSceneSha256', + 'candidateCommit', 'harnessCommit', 'binarySha256', 'ackFilesSha256'} + and plan['kind'] == 'unresolved-g01-race' + and re.fullmatch(r'gh-[1-9][0-9]*-[1-9][0-9]*-h[12]', plan['runId']) + and len(plan['nodes']) == len(set(plan['nodes'])) == (1 if plan['runId'].endswith('-h1') else 4) + and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in plan['nodes']) + and re.fullmatch(r'rv-heal-[0-9a-f]{16}', plan['bucket']) + and all(re.fullmatch(r'[0-9a-f]{40}', plan[field]) for field in ('candidateCommit', 'harnessCommit')) + and all(re.fullmatch(r'[0-9a-f]{64}', plan[field]) for field in ('binarySha256', 'ackFilesSha256')) + and set(plan['owners']) == set(plan['businessSceneSha256']) == set(plan['nodes']), + 'G01 isolation plan identity, scope or evidence digest differs') + for node in plan['nodes']: + owner = plan['owners'][node] + require(owner.get('runId') == plan['runId'] and owner.get('bucket') == plan['bucket'] + and owner.get('runtimeProfile') == 'manual' and owner.get('binarySha256') == plan['binarySha256'] + and owner.get('unit') == 'rustfs-validation-heal-' + plan['runId'] + '.service' + and len(owner.get('mounts', [])) == 4 + and re.fullmatch(r'[0-9a-f]{64}', plan['businessSceneSha256'][node]), + 'G01 isolation lost its exact reserved owner or stopped physical scene') + require(len({owner['hostname'] for owner in plan['owners'].values()}) == len(plan['nodes']), + 'G01 isolation host aliases overlap') + + +def archived_plan(artifact, failed_run, candidate): + require(type(failed_run) is int and failed_run > 0, 'G01 isolation needs one exact failed run') + paths = list(artifact.glob('*/run/*/cases/*.json')) + require(len(paths) == 1 and paths[0].name == CASE + '.json', 'G01 isolation cannot omit another case') + result = json.loads(paths[0].read_text()) + provenance = result.get('provenance') or {} + require(result.get('case_id') == CASE and result.get('verdict') == result.get('cleanup_status') == 'FAIL' + and provenance.get('source_commit') == candidate['source_sha'] + and provenance.get('candidate_sha256') == candidate['package_sha256'], + 'G01 isolation original failed result or candidate differs') + evidence = paths[0].parents[1] / 'evidence' / CASE + summary = json.loads((evidence / 'healthy-summary.json').read_text()) + rows = summary.get('results', []) + retained = [row for row in rows if row.get('cleanup') != 'PASS'] + require(summary.get('caseId') == CASE and summary.get('cleanup') == 'FAIL' and len(retained) == 1 + and retained[0].get('verdict') == 'FAIL' and retained[0].get('topology') in {'H1', 'H2'} + and retained[0].get('branch') in {'overwrite-vs-deep', 'overwrite-vs-read-repair'} + and all(row.get('caseId') == CASE for row in rows), + 'G01 isolation requires one unmixed retained public branch') + row = retained[0] + folder = evidence / row['topology'] / row['branch'] + def one(label): + matches = list(folder.glob('*-' + label + '.json')) + require(len(matches) == 1, 'G01 isolation lacks one original ' + label) + return json.loads(matches[0].read_text()) + binding, scene, dirty = one('binding'), one('retained-failure-scene'), one('last-ACK-rescue-required') + require(re.fullmatch(rf'gh-{failed_run}-[1-9][0-9]*-{row["topology"].lower()}', binding.get('runId', '')) + and binding.get('candidateCommit') == candidate['source_sha'] + and binding.get('candidatePackageSha256') == candidate['package_sha256'] + and binding.get('candidateBinarySha256') == provenance.get('binary_sha256') + and binding.get('harnessCommit') == provenance.get('testing_commit') + and dirty.get('state') == 'DIRTY' and dirty.get('namespacesRetained') is True + and dirty.get('lastACKPhysicalAuthorityVerified') is False + and dirty.get('originalGenerationBlindRestorationProhibited') is True, + 'G01 isolation run/provenance or last-ACK refusal differs') + reservations = [json.loads(path.read_text()) for path in folder.glob('*-reserved.json')] + owners = {item['node']: item['owner'] for item in reservations} + require(len(owners) == len(reservations) == len(scene) and set(owners) == set(scene) + and all(item.get('reserved') is True for item in reservations), 'G01 isolation lacks all reserved hosts') + ack_files = {} + for name in ('seed-ACKs', 'foreground-ACKs'): + journal = folder / name + entries = sorted(journal.glob('*.json')) + require(0 < len(entries) <= 1024, 'G01 isolation lacks its bounded original request/ACK journal') + for path in entries: + require(re.fullmatch(r'[0-9]{6}-(request|ack)\.json', path.name) and path.stat().st_size <= 1024**2, + 'G01 isolation journal contains an unexpected member') + item = json.loads(path.read_text()) + require(item.get('runId') == binding['runId'], 'G01 isolation journal belongs to another run') + # A missing/unknown ACK remains unknown. Isolation never infers its + # effect or sends another request, including for explicit refusals. + operation = item.get('operation') if path.name.endswith('-request.json') else (item.get('observation') or {}).get('operation') + require(operation in {'create_bucket', 'put_object'}, 'G01 isolation cannot omit a multipart mutation') + ack_files[name + '/' + path.name] = hashlib.sha256(path.read_bytes()).hexdigest() + plan = {'kind': 'unresolved-g01-race', 'runId': binding['runId'], 'nodes': sorted(owners), + 'bucket': binding['bucketPrefix'], 'owners': owners, 'candidateCommit': candidate['source_sha'], + 'harnessCommit': binding['harnessCommit'], 'binarySha256': binding['candidateBinarySha256'], + 'businessSceneSha256': {node: digest(value) for node, value in physical_content(scene).items()}, + 'ackFilesSha256': digest(ack_files)} + validate(plan) + for node, value in scene.items(): + require(value.get('quiescent') is True and type(value.get('pid')) is int and value['pid'] == 0 + and value.get('unit') == owners[node]['unit'] and value.get('hostname') == owners[node]['hostname'] + and value.get('binarySha256') == plan['binarySha256'] + and len(value['formats']) == 4 and all(type(part.get('disk')) is int for part in value['formats']) + and {part['disk'] for part in value['formats']} == set(range(4)), + 'G01 isolation lacks a stopped four-drive original failure scene') + return [plan] + + +def isolate(fleet, plan): + validate(plan) + require(fleet.run_id == plan['runId'] and list(fleet.nodes) == plan['nodes'], 'G01 isolation changed its fleet') + def params(node): + return {'expectedOwner': plan['owners'][node], 'expectedOriginalSceneSha256': plan['businessSceneSha256'][node]} + audits = {} + for node in fleet.nodes: + audit = audits[node] = fleet.record('race-retention-preflight', {'node': node, + **fleet.remote(node, 'retained-scene-audit', **params(node))}) + require(audit.get('inspectionOnly') is True and audit.get('mutated') is False + and audit.get('credentialContentsRead') is False, 'G01 isolation preflight did not preserve its read boundary') + for node in fleet.nodes: + fleet.record('race-retained-service-logs', {'node': node, **fleet.remote(node, 'logs')}) + proof = fleet.record('race-retention-isolation', {'node': node, + **fleet.remote(node, 'retained-scene-isolate', expectedBaseline=audits[node]['baseline'], **params(node))}) + require(proof.get('activeNamespaceIsolated') is True and proof.get('baseline') == audits[node]['baseline'] + and all(proof.get(field) is False for field in ('originalSceneRestored', 'backupRestorationUsed', + 'canonicalPassClaimed', 'credentialContentsRead')), 'G01 isolation did not preserve its original failure') + for node in fleet.nodes: + proof = fleet.record('race-retention-post-audit', {'node': node, + **fleet.remote(node, 'retained-scene-audit', **params(node))}) + require(proof.get('inspectionOnly') is True and proof.get('mutated') is False + and proof.get('activeNamespaceIsolated') is True and proof.get('baseline') == audits[node]['baseline'] + and len(proof.get('privateArchiveParents', [])) == 5 + and all(row.get('exists') is True and row.get('uid') == 0 and row.get('mode') == 0o700 + for row in proof['privateArchiveParents']), 'G01 isolation post-audit drifted') + return {'runId': plan['runId'], 'cleanup': 'ISOLATED', 'activeNamespaceIsolated': True, + 'originalSceneRestored': False, 'backupRestorationUsed': False, 'originalVerdictRewritten': False, + 'canonicalPassClaimed': False, 'namespaceDeletionPerformed': False, 'intentSha256': digest(plan)} diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index 4ed14e3e..8c106443 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -15,12 +15,14 @@ from .interrupted_recovery import checkpoint_plan, restore_and_isolate from .outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace, interrupted_opaque_plan, acknowledged_legacy_plan from .network_retention import archived_plan as network_retention_plan, isolate as isolate_network_scene + from .race_retention import archived_plan as race_retention_plan, isolate as isolate_race_scene except ImportError: from healthy import Fleet, physical_content from heal_probe import require, save from interrupted_recovery import checkpoint_plan, restore_and_isolate from outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace, interrupted_opaque_plan, acknowledged_legacy_plan from network_retention import archived_plan as network_retention_plan, isolate as isolate_network_scene + from race_retention import archived_plan as race_retention_plan, isolate as isolate_race_scene def quota_recovery_plan(artifact, failed_run, candidate): @@ -147,6 +149,8 @@ def network_recovery_plan(artifact, failed_run, candidate): def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") + if list(artifact.glob("*/run/heal.race-unversioned-overwrite/cases/*.json")): + return race_retention_plan(artifact, failed_run, candidate) if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/cases/*.json")): if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/evidence/heal.outcome-counters-and-error-truth/DB/H1/unknown-negative/*-one-way-legacy-write-phase.json")): return acknowledged_legacy_plan(artifact, failed_run, candidate) @@ -643,6 +647,9 @@ def main(): if plan.get("buckets"): fleet.snapshot_buckets = plan["buckets"] fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) + if plan.get("kind") == "unresolved-g01-race": + complete.append({"failedRunId": args.failed_run_id, **isolate_race_scene(fleet, plan)}) + continue if plan.get("kind") == "unresolved-network-load": complete.append({"failedRunId": args.failed_run_id, **isolate_network_scene(fleet, plan)}) continue diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py index 4e8952f9..778a9c23 100644 --- a/scripts/validation_heal/retained_scene.py +++ b/scripts/validation_heal/retained_scene.py @@ -136,7 +136,10 @@ def visit(path): opaque_backup = (not state and owner.get("runtimeProfile") == "outcome-legacy" and len(parts) == 2 and parts[0] == "quarantine" and re.fullmatch(r"J02-[0-9a-f]{32}\.(original|intent\.json|applied\.json|failed-[0-9a-f]{64})", parts[1])) - if (state and relative == "rustfs") or fixture or mrf or backup or opaque_backup or network_mpu or network_backup: + race_backup = (not state and owner.get("runtimeProfile") == "manual" + and len(parts) == 2 and parts[0] == "quarantine" + and re.fullmatch(r"G01-[a-z0-9-]{1,48}\.(original|intent\.json|applied\.json)", parts[1])) + if (state and relative == "rustfs") or fixture or mrf or backup or opaque_backup or network_mpu or network_backup or race_backup: fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) try: require(stable(os.fstat(fd)) == stable(info) and info.st_size <= 2 * 1024**3, @@ -168,7 +171,7 @@ def mounted_paths(): def inspect(request, node): state, roots = node["paths"](request["runId"]) expected = request["expectedOwner"] - per_run = expected.get("runtimeProfile") in {"outcome-legacy", "lock-quorum"} + per_run = expected.get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual"} retained_state, retained_roots = destinations(state, roots, per_run_parent=per_run) if per_run: old_state, old_roots = destinations(state, roots) @@ -178,7 +181,7 @@ def inspect(request, node): require(isinstance(expected, dict) and expected.get("runId") == request["runId"] and expected.get("unit") == node["unit"](request["runId"]) and expected.get("hostname") == node["socket"].gethostname() - and expected.get("runtimeProfile") in {"mrf-crash", "root-restart", "outcome-grace", "outcome-legacy", "lock-quorum"}, "retained scene archived owner differs") + and expected.get("runtimeProfile") in {"mrf-crash", "root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual"}, "retained scene archived owner differs") current = node["mounts"]() require(len(current) == len(expected.get("mounts", [])) == 4 and all(all(actual[key] == old[key] for key in ("target", "source", "fstype", "uuid", "maj:min")) @@ -222,7 +225,7 @@ def inspect(request, node): intent = retained_state.parent / (request["runId"] + ".intent.json") node["checked"](intent, state.parent) network = expected["runtimeProfile"] == "lock-quorum" - if expected["runtimeProfile"] in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum"}: + if expected["runtimeProfile"] in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual"}: original_sha = request.get("expectedOriginalSceneSha256", "") require(re.fullmatch(r"[0-9a-f]{64}", original_sha) is not None, "restored fixture requires its original physical scene digest") @@ -256,7 +259,7 @@ def inspect(request, node): def run(request, node): fields = {"runId", "action", "expectedOwner"} - if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum"}: + if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual"}: fields.add("expectedOriginalSceneSha256") if request.get("expectedOwner", {}).get("runtimeProfile") == "lock-quorum": fields.update({"expectedMpuSceneSha256", "mpuKeys"}) @@ -293,7 +296,7 @@ def run(request, node): "originalSceneRestored": False, "canonicalPassClaimed": False, "backupRestorationUsed": False, "credentialContentsRead": False, "inspectionOnly": request["action"] == "retained-scene-audit", "mutated": mutated, "activeNamespaceIsolated": "active" not in locations} - if request["expectedOwner"].get("runtimeProfile") in {"outcome-legacy", "lock-quorum"}: + if request["expectedOwner"].get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual"}: result["privateArchiveParents"] = [] for target in (*retained_roots, retained_state): info = target.parent.lstat() if target.parent.exists() else None diff --git a/test_validation_heal_race_retention.py b/test_validation_heal_race_retention.py new file mode 100644 index 00000000..35e91ba4 --- /dev/null +++ b/test_validation_heal_race_retention.py @@ -0,0 +1,203 @@ +"""Stopped G01 isolation preserves ACK uncertainty, bytes and credentials.""" + +import copy +import hashlib +import json +import os +from pathlib import Path +import shutil +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from scripts.validation_heal import race_retention as controller, retained_scene as retained, recover_environment as recovery +import test_validation_heal_retained_scene as retained_tests + + +class RaceRetentionTests(unittest.TestCase): + def scene(self, directory): + helper = retained_tests.RetainedSceneTests() + state, roots, node, request = helper.scene(directory) + shutil.copytree(roots[1], roots[0]) + owner = request['expectedOwner']; owner['runtimeProfile'] = 'manual' + for folder in (*roots, state): (folder / 'owner.json').write_text(json.dumps(owner)) + for root in roots: + for suffix in ('original', 'intent.json', 'applied.json'): + (root / 'quarantine' / ('G01-object-0256.' + suffix)).write_bytes(('old shard ' + suffix).encode()) + def snapshot(value): + self.assertEqual(value, {'runId': request['runId'], 'action': 'snapshot', 'quiescent': True}) + files = [] + for disk, root in enumerate(roots): + for path in sorted((root / 'data' / owner['bucket']).rglob('*')): + if path.is_file(): files.append({'disk': disk, 'path': path.relative_to(root / 'data').as_posix(), + 'size': path.stat().st_size, 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}) + return {'hostname': owner['hostname'], 'quiescent': True, 'pid': 0, 'unit': owner['unit'], + 'binarySha256': owner['binarySha256'], 'files': files, + 'formats': [{'disk': disk, 'original': True} for disk in range(4)]} + node['snapshot'] = snapshot + scene = snapshot({'runId': request['runId'], 'action': 'snapshot', 'quiescent': True}) + request['expectedOriginalSceneSha256'] = retained.sha({key: scene[key] for key in ('files', 'formats')}) + return helper, state, roots, node, request, scene + + def test_all_five_interrupted_moves_preserve_real_bytes_inodes_and_unread_credentials(self): + for interrupt_after in range(1, 6): + with self.subTest(interrupt_after=interrupt_after), tempfile.TemporaryDirectory() as directory: + helper, state, roots, node, request, _ = self.scene(directory) + original = {path: (path.stat().st_ino, path.read_bytes()) for folder in (*roots, state) + for path in folder.rglob('*') if path.is_file()} + opened, real_open = [], os.open + def tracked(path, *args, **kwargs): + opened.append(str(path)); return real_open(path, *args, **kwargs) + with helper.root_stats(), patch.object(os, 'open', side_effect=tracked): + before = retained.run(request, node) + change = {**request, 'action': 'retained-scene-isolate', 'expectedBaseline': before['baseline']} + count = 0 + def interrupted(source, target): + nonlocal count + helper.local_rename(source, target) + if target.is_dir(): + count += 1 + if count == interrupt_after: raise RuntimeError('SSH response lost') + with patch.object(retained, 'rename_new', side_effect=interrupted), self.assertRaises(RuntimeError): + retained.run(change, node) + with patch.object(retained, 'rename_new', side_effect=helper.local_rename): + proof = retained.run(change, node) + self.assertFalse(retained.run(change, node)['mutated']) + self.assertEqual(proof['locations'], ['retained'] * 5) + self.assertEqual(proof['baseline'], before['baseline']) + self.assertFalse(proof['originalSceneRestored'] or proof['backupRestorationUsed'] or proof['canonicalPassClaimed']) + target_state, target_roots = retained.destinations(state, roots, per_run_parent=True) + for path, expected in original.items(): + source = next(folder for folder in (*roots, state) if path.is_relative_to(folder)) + target = target_state if source == state else target_roots[roots.index(source)] + moved = target / path.relative_to(source) + self.assertEqual((moved.stat().st_ino, moved.read_bytes()), expected) + self.assertTrue(all(row['uid'] == 0 and row['mode'] == 0o700 for row in proof['privateArchiveParents'])) + self.assertTrue(any(path.endswith('G01-object-0256.original') for path in opened)) + self.assertFalse(any(path.endswith('service.env') or path.endswith('/credential') for path in opened)) + + def test_changed_business_backup_scope_gate_and_missing_root_refuse_before_move(self): + for kind in ('scene-sha', 'missing-field', 'business-drift', 'backup-drift', 'gate', 'missing-root'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as directory: + helper, state, roots, node, request, _ = self.scene(directory) + with helper.root_stats(): + baseline = retained.run(request, node)['baseline'] + change = {**request, 'action': 'retained-scene-isolate', 'expectedBaseline': baseline} + if kind == 'scene-sha': change['expectedOriginalSceneSha256'] = '0' * 64 + elif kind == 'missing-field': del change['expectedOriginalSceneSha256'] + elif kind == 'business-drift': (roots[0] / 'data' / request['expectedOwner']['bucket'] / 'part.1').write_bytes(b'new ACK') + elif kind == 'backup-drift': (roots[0] / 'quarantine/G01-object-0256.original').write_bytes(b'changed backup') + elif kind == 'gate': (state / 'drive-gate.json').write_text('{}') + else: shutil.rmtree(roots[0]) + with patch.object(retained, 'rename_new') as move, self.assertRaises(ValueError): retained.run(change, node) + move.assert_not_called() + + def archive(self, directory): + source = Path(directory) / 'source'; source.mkdir() + _, _, _, _, request, scene = self.scene(source) + owner = request['expectedOwner']; run = Path(directory) / 'artifact/rustfs-validation-12345-1/run' / controller.CASE + evidence = run / 'evidence' / controller.CASE; folder = evidence / 'H2/overwrite-vs-deep' + folder.mkdir(parents=True); (run / 'cases').mkdir() + nodes = [f'vm00{i}' for i in range(4)] + binding = {'runId': request['runId'], 'candidateCommit': '1' * 40, 'harnessCommit': '2' * 40, + 'candidateBinarySha256': owner['binarySha256'], 'candidatePackageSha256': '3' * 64, 'bucketPrefix': owner['bucket']} + candidate = {'source_sha': binding['candidateCommit'], 'package_sha256': binding['candidatePackageSha256']} + def write(path, value): path.write_text(json.dumps(value)) + write(run / 'cases' / (controller.CASE + '.json'), {'case_id': controller.CASE, 'verdict': 'FAIL', + 'cleanup_status': 'FAIL', 'provenance': {'source_commit': binding['candidateCommit'], + 'candidate_sha256': binding['candidatePackageSha256'], 'binary_sha256': owner['binarySha256'], + 'testing_commit': binding['harnessCommit']}}) + write(evidence / 'healthy-summary.json', {'caseId': controller.CASE, 'cleanup': 'FAIL', 'results': [ + {'caseId': controller.CASE, 'cleanup': 'FAIL', 'verdict': 'FAIL', 'topology': 'H2', 'branch': 'overwrite-vs-deep'}]}) + write(folder / '00001-binding.json', binding) + failure = {} + for i, name in enumerate(nodes): + host = 'original-host-' + str(i) + write(folder / f'{i:05d}-reserved.json', {'node': name, 'reserved': True, 'owner': {**owner, 'hostname': host}}) + failure[name] = {**copy.deepcopy(scene), 'hostname': host} + write(folder / '00005-retained-failure-scene.json', failure) + write(folder / '00006-last-ACK-rescue-required.json', {'state': 'DIRTY', 'namespacesRetained': True, + 'lastACKPhysicalAuthorityVerified': False, 'originalGenerationBlindRestorationProhibited': True}) + for name in ('seed-ACKs', 'foreground-ACKs'): + journal = folder / name; journal.mkdir() + write(journal / '000001-request.json', {'runId': binding['runId'], 'operation': 'put_object'}) + # Unknown/missing results are retained, never reconstructed or retried. + root = Path(directory) / 'artifact' + return root, candidate, folder + + def test_archived_unknown_or_explicit_refusal_can_only_select_isolation_with_exact_identity(self): + for kind in ('valid', 'unknown', 'refusal', 'candidate', 'binary', 'host', 'live', 'mpu', 'omitted-owner', 'mixed-case', 'extra-failure'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as directory: + root, candidate, folder = self.archive(directory) + if kind in ('unknown', 'refusal'): + (folder / 'foreground-ACKs/000001-ack.json').write_text(json.dumps({'runId': 'gh-12345-1-h2', + 'observation': {'operation': 'put_object', 'acknowledged': False, 'resultUnknown': kind == 'unknown', + 'response': {'ResponseMetadata': {'HTTPStatusCode': 503}}}})) + elif kind == 'candidate': candidate['source_sha'] = '4' * 40 + elif kind in ('binary', 'host', 'live'): + path = folder / '00005-retained-failure-scene.json'; value = json.loads(path.read_text()) + value['vm003'][{'binary': 'binarySha256', 'host': 'hostname', 'live': 'pid'}[kind]] = 1 if kind == 'live' else 'foreign' + path.write_text(json.dumps(value)) + elif kind == 'mpu': + (folder / 'foreground-ACKs/000001-request.json').write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'complete_multipart_upload'})) + elif kind == 'omitted-owner': (folder / '00003-reserved.json').unlink() + elif kind == 'mixed-case': + path = next(root.glob('*/run/*/cases')).parent / 'cases/another-case.json'; path.write_text('{}') + elif kind == 'extra-failure': + path = folder.parent.parent / 'healthy-summary.json'; value = json.loads(path.read_text()); value['results'] *= 2 + path.write_text(json.dumps(value)) + if kind in ('valid', 'unknown', 'refusal'): + plan = recovery.recovery_plan(root, 12345, candidate)[0] + self.assertEqual(plan['kind'], 'unresolved-g01-race'); self.assertEqual(plan['nodes'], [f'vm00{i}' for i in range(4)]) + else: + with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) + + def test_all_peers_preflight_before_any_move_and_failed_checks_never_issue_cleanup_or_restart(self): + for failure in (None, 'preflight-vm003', 'isolate-vm001', 'post-vm002'): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory: + root, candidate, _ = self.archive(directory); plan = recovery.recovery_plan(root, 12345, candidate)[0] + calls, audits = [], {} + def remote(node, action, **params): + calls.append((node, action)) + if action == 'logs': return {'retained': True} + if action == 'retained-scene-isolate': + self.assertEqual(len(audits), 4) + if failure == 'isolate-' + node: raise RuntimeError('SSH response lost') + return {'baseline': audits[node], 'activeNamespaceIsolated': True, 'originalSceneRestored': False, + 'backupRestorationUsed': False, 'canonicalPassClaimed': False, 'credentialContentsRead': False} + self.assertEqual(action, 'retained-scene-audit') + post = node in audits + if failure == ('post-' if post else 'preflight-') + node: raise RuntimeError('drift') + audits[node] = {'node': node} + return {'baseline': audits[node], 'inspectionOnly': True, 'mutated': False, 'credentialContentsRead': False, + 'activeNamespaceIsolated': post, 'privateArchiveParents': [{'exists': True, 'uid': 0, 'mode': 0o700}] * 5} + fleet = SimpleNamespace(run_id=plan['runId'], nodes=plan['nodes'], remote=remote, record=lambda label, value: value) + if failure is None: + proof = controller.isolate(fleet, plan); self.assertEqual(proof['cleanup'], 'ISOLATED') + self.assertFalse(proof['namespaceDeletionPerformed'] or proof['backupRestorationUsed'] or proof['canonicalPassClaimed']) + else: + with self.assertRaises(RuntimeError): controller.isolate(fleet, plan) + self.assertTrue(all(action in {'retained-scene-audit', 'retained-scene-isolate', 'logs'} for _, action in calls)) + if failure == 'preflight-vm003': self.assertFalse(any(action == 'retained-scene-isolate' for _, action in calls)) + + def test_recovery_routes_isolated_scene_before_absent_namespace_cleanup(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, _ = self.archive(directory) + plan = recovery.recovery_plan(root, 12345, candidate)[0] + candidate_path = Path(directory) / 'candidate.json'; candidate_path.write_text(json.dumps(candidate)) + evidence = Path(directory) / 'recovery' + fleet = SimpleNamespace(nodes=plan['nodes'], run_id=plan['runId'], record=lambda label, value: value) + proof = {'runId': plan['runId'], 'cleanup': 'ISOLATED', 'canonicalPassClaimed': False} + with patch('sys.argv', ['recover', '--failed-run-id', '12345', '--artifact', str(root), + '--candidate', str(candidate_path), '--evidence', str(evidence)]), \ + patch.object(recovery, 'Fleet', return_value=fleet), \ + patch.object(recovery, 'isolate_race_scene', return_value=proof) as isolate: + recovery.main() + isolate.assert_called_once_with(fleet, plan) + summary = json.loads((evidence / 'recovery-summary.json').read_text()) + self.assertIn('ISOLATED', json.dumps(summary)) + + +if __name__ == '__main__': + unittest.main() From ec210969adc60a90c7d4294cb34701ba5ccd4757 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 13:27:04 +0800 Subject: [PATCH 266/343] fix(validation): preserve version race ACKs and backup witnesses --- scripts/validation_heal/fleet_node.py | 2 + scripts/validation_heal/healthy.py | 5 +- scripts/validation_heal/mpu_race_backups.py | 184 +++++++++ scripts/validation_heal/mpu_race_schedule.py | 61 +++ scripts/validation_heal/race.py | 253 ++++++++++-- scripts/validation_heal/race_retention.py | 66 +-- .../validation_heal/recover_environment.py | 5 +- scripts/validation_heal/retained_scene.py | 2 +- test_validation_heal_g02_source.py | 387 ++++++++++++++++++ test_validation_heal_mpu_race_backups.py | 151 +++++++ test_validation_heal_mpu_race_schedule.py | 66 +++ test_validation_heal_race.py | 3 +- test_validation_heal_race_retention.py | 43 ++ 13 files changed, 1152 insertions(+), 76 deletions(-) create mode 100644 scripts/validation_heal/mpu_race_backups.py create mode 100644 scripts/validation_heal/mpu_race_schedule.py create mode 100644 test_validation_heal_g02_source.py create mode 100644 test_validation_heal_mpu_race_backups.py create mode 100644 test_validation_heal_mpu_race_schedule.py diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 7933f246..6acdb69c 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1819,6 +1819,8 @@ def main(request): actions["mpu-race-scene"] = lambda value: globals()["mpu_race_backend"].run(value, globals()) actions["network-delete-records"] = lambda value: globals()["network_delete_records_backend"].run(value, globals()) actions["race-io-cap"] = lambda value: globals()["race_io_backend"].run(value, globals()) + actions["mpu-race-backups"] = lambda value: globals()["mpu_race_backups_backend"].run(value, globals()) + actions["version-race-backups"] = lambda value: globals()["mpu_race_backups_backend"].run(value, globals()) actions["process-clock"] = process_clock actions["environment-capabilities"] = environment_capabilities for action in ("retained-scene-audit", "retained-scene-isolate"): diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index fbc06eac..d14ccc1b 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -185,6 +185,9 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() + if action in {"mpu-race-backups", "version-race-backups"}: + source += "\nimport types\nmpu_race_backups_backend = types.ModuleType('mpu_race_backups')\nexec(" + repr( + NODE_SOURCE.with_name("mpu_race_backups.py").read_text()) + ", mpu_race_backups_backend.__dict__)\n" if action == "race-io-cap": source += "\nimport types\nrace_io_backend = types.ModuleType('race_io')\nexec(" + repr( NODE_SOURCE.with_name("race_io.py").read_text()) + ", race_io_backend.__dict__)\n" @@ -220,7 +223,7 @@ def remote(self, node, action, **params): source += "\nimport types\ngeneration_backend = types.ModuleType('generation_fault')\nexec(" + repr( NODE_SOURCE.with_name("generation_fault.py").read_text()) + ", generation_backend.__dict__)\n" if action in {"apply-fault", "recover-fault", "verify-part", "verify-parts", "verify-inline", - "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure", "outcome-legacy-phase", "start"} or action.startswith("generation-") or action.startswith("orphan-") or action.startswith(("journal-", "incarnation-", "owner-markers-")): + "bucket-record-snapshot", "bucket-record-apply", "bucket-record-recover", "format-schema-fault", "encryption-configure", "outcome-legacy-phase", "mpu-race-backups", "version-race-backups", "start"} or action.startswith("generation-") or action.startswith("orphan-") or action.startswith(("journal-", "incarnation-", "owner-markers-")): source += "\nimport types\nfault = types.ModuleType('heal_fault')\nexec(" + repr( NODE_SOURCE.with_name("heal_fault.py").read_text()) + ", fault.__dict__)\n" if action.startswith(("journal-", "incarnation-")): diff --git a/scripts/validation_heal/mpu_race_backups.py b/scripts/validation_heal/mpu_race_backups.py new file mode 100644 index 00000000..cb31ebe3 --- /dev/null +++ b/scripts/validation_heal/mpu_race_backups.py @@ -0,0 +1,184 @@ +"""Stopped G02/G05 backups, decoded from the same held file that was hashed.""" +import hashlib +import json +import os +import re +import stat +import uuid + +KEY = 'objects-0256.bin' +LIMIT = 1024**2 + + +def require(value, message): + if not value: + raise ValueError(message) + + +def stable(info): + return (info.st_mode,info.st_uid,info.st_gid,info.st_dev,info.st_ino,info.st_nlink, + info.st_size,info.st_mtime_ns,info.st_ctime_ns) + + +def run(request,node): + require(set(request)=={'runId','action','bucket','expectedBinarySha256','expectedInspectorSha256'} + and request['action'] in {'mpu-race-backups','version-race-backups'} + and re.fullmatch(r'rv-heal-[0-9a-f]{16}',request['bucket']), + 'G05 backup census requires the exact original owned key') + key = KEY if request['action']=='mpu-race-backups' else 'objects/0256.bin' + generations = 3 if request['action']=='mpu-race-backups' else 6 + state,roots,owner=node['owned'](request['runId']) + require(owner.get('runtimeProfile')=='manual' and owner.get('bucket')==request['bucket'] + and owner.get('binarySha256')==request['expectedBinarySha256'] + and owner.get('inspectorSha256')==request['expectedInspectorSha256'] + and all(re.fullmatch(r'[0-9a-f]{64}',request[k]) for k in ('expectedBinarySha256','expectedInspectorSha256')) + and len(roots)==len(node['MOUNTS'])==4,'G05 backup owner/binary/inspector differs') + node['stopped'](owner) + tool=node['checked'](state/'heal-meta-inspect',state) + require(node['digest'](tool)==owner['inspectorSha256'] and node['digest'](state/'rustfs')==owner['binarySha256'], + 'G05 original candidate or inspector changed') + fault=node['fault'];rows=[] + for disk,(root,mount) in enumerate(zip(roots,node['MOUNTS'],strict=True)): + require(root==mount/'heal-validation'/request['runId'],'G05 backup root escaped the original mount') + with fault.directory(mount) as mount_fd, fault.parent(mount_fd,['heal-validation',request['runId'],'data',request['bucket'],*key.split('/')]) as key_fd: + before_key=os.fstat(key_fd);names=sorted(os.listdir(key_fd)) + require(len(names)<=generations+1 and 'xl.meta' in names,'G02/G05 key population differs from its bounded version fixture') + for name in names: + if name=='xl.meta':continue + require(str(uuid.UUID(name))==name and uuid.UUID(name).int!=0,'G05 backup parent is not an original generation UUID') + with fault.parent(key_fd,[name]) as part_fd: + before_parent=os.fstat(part_fd) + try:fd=os.open('xl.meta.bkp',os.O_RDONLY|os.O_NOFOLLOW|os.O_NONBLOCK,dir_fd=part_fd) + except FileNotFoundError: + require(stable(os.fstat(part_fd))==stable(before_parent), + 'G05 backup population changed during absent-file census') + continue + try: + before=os.fstat(fd) + require(stat.S_ISREG(before.st_mode) and before.st_nlink==1 and before.st_size<=LIMIT + and fault.mount_id(fd)==fault.mount_id(part_fd), + 'G05 backup is oversized, linked, special or on a different mount') + raw=os.read(fd,LIMIT+1) + require(len(raw)==before.st_size and stable(before)==stable(os.fstat(fd)), + 'G05 backup changed while hashing') + os.lseek(fd,0,os.SEEK_SET) + # Linux parent-fd access keeps the decoder on the checked + # inode even if an attacker replaces its pathname. + observed=node['command'](str(tool),f'/proc/{os.getpid()}/fd/{fd}',request['bucket'],key,check=False) + require(observed.returncode==0 and len(observed.stdout)<=LIMIT, + 'G05 backup metadata decoder failed or exceeded its bound') + decoded=json.loads(observed.stdout) + require(isinstance(decoded,dict) and decoded.get('bucket')==request['bucket'] + and decoded.get('key')==key and decoded.get('physicalBitrotVerified') is False, + 'G05 backup decoder identity or proof role differs') + require(stable(os.fstat(fd))==stable(before) + and stable(os.stat('xl.meta.bkp',dir_fd=part_fd,follow_symlinks=False))==stable(before), + 'G05 backup inode changed during descriptor-bound decode') + rows.append({'disk':disk,'path':f"{request['bucket']}/{key}/{name}/xl.meta.bkp", + 'size':len(raw),'sha256':hashlib.sha256(raw).hexdigest(),'metadata':decoded}) + finally:os.close(fd) + require(stable(os.fstat(part_fd))==stable(before_parent),'G05 backup generation changed during census') + require(stable(os.fstat(key_fd))==stable(before_key) and sorted(os.listdir(key_fd))==names, + 'G05 committed key population changed during backup census') + node['stopped'](owner) + require(node['owned'](request['runId'])==(state,roots,owner) + and node['digest'](tool)==owner['inspectorSha256'] and node['digest'](state/'rustfs')==owner['binarySha256'], + 'G05 original ownership or tools changed during backup census') + return {'hostname':owner['hostname'],'rows':rows,'inspectorSha256':owner['inspectorSha256'], + 'binarySha256':owner['binarySha256'],'quiescent':True,'readOnly':True, + 'sameHeldFileHashedAndDecoded':True,'rawContentsExported':False,'liveAuthorityClaimed':False, + 'physicalRepairProved':False,'canonicalCasePassClaimed':False} + + +def version_map(metadata): + versions=metadata.get('versions') + require(isinstance(versions,list) and 1<=len(versions)<=6,'G02/G05 backup needs bounded acknowledged data versions') + result={} + for version in versions: + vid=version.get('versionId') + require(isinstance(vid,str) and str(uuid.UUID(vid))==vid and uuid.UUID(vid).int!=0 + and vid not in result and version.get('deleted') is False and type(version.get('isLatest')) is bool, + 'G05 backup version identity, kind or latest flag differs') + result[vid]=version + require(sum(v['isLatest'] for v in result.values())==1,'G05 backup needs exactly one latest data version') + return result + + +def verify(before,after,observations,nodes,bucket,old_version,new_version,binary_sha,inspector_sha): + return verify_versions(before,after,observations,nodes,bucket,KEY,[old_version],[new_version],binary_sha,inspector_sha) + + +def verify_versions(before,after,observations,nodes,bucket,key,old_versions,new_versions,binary_sha,inspector_sha): + """Adapt frozen additive_backups to H1/H2; backups never supply live authority.""" + require(re.fullmatch(r'rv-heal-[0-9a-f]{16}',bucket) + and all(isinstance(v,str) and re.fullmatch(r'[0-9a-f]{64}',v) for v in (binary_sha,inspector_sha)) + and len(nodes) in (1,4) and len(set(nodes))==len(nodes) + and set(before)==set(after)==set(observations)==set(nodes) + and (key==KEY and len(old_versions)==len(new_versions)==1 + or key=='objects/0256.bin' and len(old_versions)==len(new_versions)==3) + and len(set(old_versions+new_versions))==len(old_versions)+len(new_versions), + 'G05 backup proof omitted or mixed an original host/version') + proofs=[];prefix=f'{bucket}/{key}/' + for node in nodes: + evidence=observations[node] + require(evidence.get('quiescent') is True and evidence.get('readOnly') is True + and evidence.get('sameHeldFileHashedAndDecoded') is True + and evidence.get('binarySha256')==before[node].get('binarySha256')==after[node].get('binarySha256')==binary_sha + and evidence.get('inspectorSha256')==inspector_sha + and evidence.get('hostname')==before[node].get('hostname')==after[node].get('hostname') + and before[node].get('quiescent') is True and after[node].get('quiescent') is True + and evidence.get('liveAuthorityClaimed') is False and evidence.get('rawContentsExported') is False, + 'G05 backup proof lost its candidate/inspector/stopped binding') + def index(scene): + rows=[r for r in scene[node]['files'] if r['path'].startswith(prefix)] + values={(r['disk'],r['path']):r for r in rows} + require(len(values)==len(rows),'G05 snapshot contains duplicate physical backup identities') + return values + previous,current=index(before),index(after) + primary={} + for scene in (previous,current): + meta=[r for (disk,path),r in scene.items() if path==prefix+'xl.meta'] + require(len(meta)==4 and all(type(r['disk']) is int for r in meta) + and {r['disk'] for r in meta}==set(range(4)), 'G05 backup scope lacks every original primary member') + require(all(r.get('metadata',{}).get('bucket')==bucket and r['metadata'].get('key')==key + and r['metadata'].get('physicalBitrotVerified') is False for r in meta), + 'G05 primary decoder scope or proof role differs') + for disk in range(4): + old=version_map(previous[disk,prefix+'xl.meta']['metadata']) + new=version_map(current[disk,prefix+'xl.meta']['metadata']) + require(set(old)==set(old_versions) and set(new)==set(old_versions+new_versions) + and old[old_versions[-1]]['isLatest'] and new[new_versions[-1]]['isLatest'], + 'G05 primary versions differ from original and Complete ACK order') + primary[disk]=(old,new) + candidates={k:v for k,v in current.items() if k[1].endswith('/xl.meta.bkp')} + rows=evidence['rows'];require(isinstance(rows,list) and len(rows)<=4*len(old_versions),'G05 backup decode rows exceed original generations') + decoded={(r['disk'],r['path']):r for r in rows} + require(len(decoded)==len(rows) and set(decoded)==set(candidates),'G05 backup decode population differs from snapshot') + for (disk,path),row in candidates.items(): + require(type(disk) is int and disk in range(4),'G05 backup has a foreign local drive') + final=current[disk,prefix+'xl.meta'] + old,new=primary[disk] + require(path in {prefix+old[vid]['dataDir']+'/xl.meta.bkp' for vid in old_versions}, + 'G05 backup directory is not bound to the original committed generation') + witness=decoded[disk,path] + require(all(witness[k]==row[k] for k in ('disk','path','size','sha256')), + 'G05 backup decoder does not describe its exact snapshotted bytes') + value=witness['metadata'];versions=version_map(value) + require({k:v for k,v in value.items() if k!='versions'}== + {k:v for k,v in final['metadata'].items() if k!='versions'}, + 'G05 backup decoder schema/object identity differs') + count=len(versions)-len(old_versions) + require(0<=count<=len(new_versions) and set(versions)==set(old_versions+new_versions[:count]), + 'G05 backup is not a contiguous prefix of acknowledged Complete versions') + latest=old_versions[-1] if count==0 else new_versions[count-1] + for vid,version in versions.items(): + reference=old[vid] if vid in old else new[vid] + require(version['isLatest'] is (vid==latest) + and {k:v for k,v in version.items() if k!='isLatest'}== + {k:v for k,v in reference.items() if k!='isLatest'}, + 'G05 backup version descriptor changed outside the acknowledged prefix') + proofs.append({'node':node,'disk':disk,'path':path,'sha256':row['sha256'],'size':row['size'], + 'acknowledgedCompletePrefix':count,'decoderFieldsVerified':True, + 'unexposedMetadataVerified':False,'liveAuthorityClaimed':False}) + return {'proofs':proofs,'allSnapshottedBackupsVerified':True,'liveAuthorityClaimed':False, + 'physicalRepairProved':False,'canonicalCasePassClaimed':False} diff --git a/scripts/validation_heal/mpu_race_schedule.py b/scripts/validation_heal/mpu_race_schedule.py new file mode 100644 index 00000000..f2533fb8 --- /dev/null +++ b/scripts/validation_heal/mpu_race_schedule.py @@ -0,0 +1,61 @@ +"""Frozen G05 preparation and ordered SDK/checkpoint schedule, without I/O cap.""" +try: + from .heal_probe import canonical, require +except ImportError: + from heal_probe import canonical, require + +SELECTED = {'upload-continue':'upload_part', 'complete':'complete_multipart_upload', 'abort':'abort_multipart_upload'} + + +def prepare(fixture, branch): + with fixture.lock: + require(branch in SELECTED and not fixture.seed_started and not fixture.items and fixture.poisoned is None, + 'G05 branch preparation requires an original fresh model') + fixture.seed() + if branch != 'upload-continue': + fixture.upload('a', 3) + return {'branch':branch, 'selectedOperation':SELECTED[branch], 'manifest':fixture.manifest()} + + +def execute(fixture, branch, prepared, checkpoint): + """Worker callback writes named physical checkpoints outside Fleet.sequence. + + Only the selected first SDK call needs the public running-token witness. + Every companion still needs its original ACK and immediate physical check. + """ + with fixture.lock: + require(branch in SELECTED and callable(checkpoint) and isinstance(prepared,dict) + and set(prepared)=={'branch','selectedOperation','manifest'} and prepared['branch']==branch + and prepared['selectedOperation']==SELECTED[branch] and fixture.poisoned is None + and canonical(prepared['manifest'])==canonical(fixture.manifest()), + 'G05 foreground differs from its original prepared branch/model') + expected_a = 2 if branch=='upload-continue' else 3 + require(set(fixture.items)=={'committed','a','b'} and fixture.items['committed']['state']=='complete' + and fixture.items['a']['state']==fixture.items['b']['state']=='pending' + and len(fixture.items['committed']['parts'])==len(fixture.items['b']['parts'])==3 + and len(fixture.items['a']['parts'])==expected_a, + 'G05 requires the exact original committed/A/B population before Heal') + steps = ([('continued',lambda:fixture.upload('a',3)),('completed',lambda:fixture.complete('a')), + ('aborted',fixture.abort_b)] if branch=='upload-continue' else + [('completed',lambda:fixture.complete('a')),('aborted',fixture.abort_b)] if branch=='complete' else + [('aborted',fixture.abort_b),('completed',lambda:fixture.complete('a'))]) + records=[] + try: + for label,call in steps: + response=call() + sequence=fixture.recorder.sequence + manifest=fixture.manifest() + witness=checkpoint(label,manifest) + require(canonical(manifest)==canonical(fixture.manifest()),'G05 physical checkpoint mutated its ACK model') + records.append({'phase':label,'ackSequence':sequence,'response':response,'checkpoint':witness}) + except BaseException: + # Every received ACK already has the original durable Recorder/model + # files. A missing physical witness cannot authorize another mutation. + if fixture.poisoned is None: + fixture.poisoned='G05 physical checkpoint failed; retain original ACK state and halt companion mutations' + raise + require(fixture.items['a']['state']=='complete' and fixture.items['b']['state']=='aborted', + 'G05 foreground did not retire both original uploads') + return {'branch':branch,'selectedOperation':SELECTED[branch],'selectedAckSequence':records[0]['ackSequence'], + 'steps':records,'manifest':fixture.manifest(),'ioCapRequested':False,'physicalRepairProved':False, + 'publicOverlapProved':False,'canonicalCasePassClaimed':False} diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index f3f565fe..0acf5b62 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -15,6 +15,7 @@ import secrets import signal import sys +import threading import time from urllib.parse import quote import uuid @@ -31,6 +32,7 @@ from .versions import public_version_id from .race_native_evidence import verify as verify_native from .race_io import set_fleet as set_race_io_cap + from .mpu_race_backups import verify_versions as verify_version_backups except ImportError: from heal_probe import Probe, canonical, compare_inventories, payload, require, save from heal_workload import Recorder, seed as seed_workload, verify_seed @@ -43,6 +45,7 @@ from versions import public_version_id from race_native_evidence import verify as verify_native from race_io import set_fleet as set_race_io_cap + from mpu_race_backups import verify_versions as verify_version_backups CASE_IDS = ("heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-markers-and-batch-delete", "heal.race-multipart-complete-abort", "heal.race-copy-and-metadata-update") @@ -328,8 +331,8 @@ def operation_plan(bucket, branch, expected): return [{"operation": "put_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, "ContentType": "application/octet-stream", "Metadata": {"race": "versioned-put", "nonce": str(index)}, "Tagging": "race=versioned-put"}, - "nonce": f"{bucket}/versioned-put/{index}", "readVersionId": target[0]["versionId"], - "readPayload": target[0].get("payload")} + "nonce": f"{bucket}/objects/{index:04d}.bin/0", "referenceKey": f"objects/{index:04d}.bin", + } for index in range(3)] if branch in G04_BRANCHES: target = [row for row in expected if row["key"] == TARGET_KEY] @@ -405,6 +408,35 @@ def operation_plan(bucket, branch, expected): "VersionId": (target[0] if branch == G03_BRANCHES[0] else target[-1])["versionId"]}}] +def version_put_entries(baseline, plan, responses): + """Build only received serial PUT ACKs; a missing response stays unknown.""" + require(len(plan) == 3 and 1 <= len(responses) <= 3, 'G02 ACK prefix is incomplete or excessive') + added = [] + original = {row['versionId'] for row in baseline['entries'] if row['key'] == TARGET_KEY} + require(len(original) == 3, 'G02 lacks its three original acknowledged versions') + seen = set(original) + for index, (step, ack) in enumerate(zip(plan[:len(responses)], responses, strict=True)): + version = ack.get('VersionId') + require(isinstance(version, str) and re.fullmatch(r'[0-9a-f-]{36}', version) + and str(uuid.UUID(version)) == version and uuid.UUID(version).int != 0 and version not in seen + and isinstance(ack.get('ETag'), str) and ack['ETag'], 'G02 PUT lacks a distinct real version ACK') + seen.add(version) + key = f'objects/{index:04d}.bin' + reference = [row for row in baseline['entries'] if row['key'] == key and row.get('isLatest') is True + and row.get('deleteMarker') is False] + body = payload(step['nonce'], SIZE) + contents = {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()} + require(step.get('referenceKey') == key and step['nonce'] == f'{step["params"]["Bucket"]}/{key}/0' + and len(reference) == 1 and contents == reference[0]['payload'], + 'G02 new PUT bytes differ from the original independent control') + added.append({'key': TARGET_KEY, 'versionId': version, 'isLatest': index == len(responses) - 1, + 'deleteMarker': False, 'payload': contents, + 'properties': {'ETag': ack['ETag'], 'ContentLength': len(body), + 'ContentType': step['params']['ContentType'], 'Metadata': step['params']['Metadata']}, + 'tags': [{'Key': 'race', 'Value': 'versioned-put'}]}) + return added + + def final_inventory(baseline, branch, plan, responses): expected_length = (3 if branch in (*G01_BRANCHES, *G02_BRANCHES) else len(plan) if branch in G05_BRANCHES @@ -413,26 +445,23 @@ def final_inventory(baseline, branch, plan, responses): "G01/G03/G05 foreground plan/ACK sequence is incomplete") result = copy.deepcopy(baseline) current_target, current_reference = target_key(branch), reference_key(branch) - if branch in (*G01_BRANCHES, *G02_BRANCHES): + if branch in G02_BRANCHES: + added = version_put_entries(baseline, plan, responses) + for row in result['entries']: + if row['key'] == current_target: row['isLatest'] = False + result['entries'].extend(added) + result['acknowledgedVersionAppendOrder'] = [row['versionId'] for row in added] + result['entries'].sort(key=lambda row: (row['key'], row['versionId'])) + elif branch in G01_BRANCHES: entry = next(row for row in result["entries"] if row["key"] == current_target) step, ack = plan[-1], responses[-1] body = payload(step["nonce"], SIZE) require(isinstance(ack.get("ETag"), str) and ack["ETag"] - and (branch in G02_BRANCHES or ack.get("VersionId") in {None, "null"}), "G01/G02/G03 last PUT ACK lost its identity") - if branch in G01_BRANCHES: - entry.update(payload={"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}, - tags=[{"Key": "race", "Value": "overwrite"}]) - entry["properties"].update(ETag=ack["ETag"], ContentLength=len(body), ContentType="application/octet-stream", - Metadata=step["params"]["Metadata"]) - else: - for row in result["entries"]: - if row["key"] == current_target: row["isLatest"] = False - result["entries"].append({"key": current_target, "versionId": public_version_id(ack.get("VersionId")), - "isLatest": True, "deleteMarker": False, - "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}, - "properties": {"ETag": ack["ETag"], "ContentLength": len(body), - "ContentType": "application/octet-stream", "Metadata": step["params"]["Metadata"]}, - "tags": [{"Key": "race", "Value": "versioned-put"}]}) + and ack.get("VersionId") in {None, "null"}, "G01/G03 last PUT ACK lost its identity") + entry.update(payload={"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}, + tags=[{"Key": "race", "Value": "overwrite"}]) + entry["properties"].update(ETag=ack["ETag"], ContentLength=len(body), ContentType="application/octet-stream", + Metadata=step["params"]["Metadata"]) elif branch in G04_BRANCHES: require(len(responses) == 1, "G04 foreground plan/ACK sequence is incomplete") target_rows = [row for row in result["entries"] if row["key"] == TARGET_KEY] @@ -561,11 +590,47 @@ def physical_transition(before, after, bucket, mapped, final, branch): "G02 version race lost an acknowledged historical or new version") old_dirs = {target["dataDir"] for target in original_target} new_dirs = {target["dataDir"] for target in current_target_values} - old_dirs - require(len(new_dirs) == 3 and old_dirs.isdisjoint(new_dirs), + require(len(old_dirs) == len(new_dirs) == 3 and old_dirs.isdisjoint(new_dirs), "G02 versions share or resurrect a physical data directory") - require(all(any(part["size"] == SIZE for part in target["cohort"]) - for target in current_target_values if target["dataDir"] in new_dirs), - "G02 new version physical parts are incomplete") + old_versions = {target['versionId'] for target in original_target} + entries = [row for row in final['entries'] if row['key'] == current_target] + added = {row['versionId']: row for row in entries if row['versionId'] not in old_versions} + order = final.get('acknowledgedVersionAppendOrder', []) + require(len(order) == len(set(order)) == 3 and set(order) == set(added), + 'G02 physical transition lost the explicit serial ACK append order') + appended = [added[version] for version in order] + require(len(appended) == 3 and len({row['versionId'] for row in entries}) == len(entries) == 6, + 'G02 physical transition lacks all three serial new version ACKs') + def coding(target): + return {part['index']: (part['sha256'], part['size']) for part in target['cohort']} + for target in current_target_values: + if target['versionId'] in old_versions: + original = mapped[current_target, target['versionId']] + require({key: value for key, value in target['logical'].items() if key != 'isLatest'} + == {key: value for key, value in original['logical'].items() if key != 'isLatest'} + and coding(target) == coding(original), 'G02 append changed a retained historical version') + else: + index = next(i for i, row in enumerate(appended) if row['versionId'] == target['versionId']) + control_key = f'objects/{index:04d}.bin' + controls = [row for row in final['entries'] if row['key'] == control_key and row.get('isLatest') is True] + references = [value for (key, _), value in mapped.items() if key == control_key] + require(len(controls) == len(references) == 1 and appended[index]['payload'] == controls[0]['payload'], + 'G02 new ACK bytes lack their independent pre-fault control') + reference = references[0] + require(target['logical']['size'] == SIZE and len(target['logical']['parts']) == 1 + and target['logical']['parts'][0]['size'] == target['logical']['parts'][0]['actualSize'] == SIZE + and coding(target) == coding(reference) + and all(target['logical']['erasure'][key] == reference['logical']['erasure'][key] + for key in ('data_blocks', 'parity_blocks', 'block_size', 'algorithm')), + 'G02 new version logical size or coding-index bytes differ from its immutable reference') + expected_versions = sorted((row['versionId'], row['isLatest'], row['deleteMarker']) for row in entries) + for value in after.values(): + for row in value['files']: + if row['path'] != f'{bucket}/{current_target}/xl.meta': continue + versions = row['metadata']['versions'] + require(all(type(v.get('isLatest')) is bool and type(v.get('deleted')) is bool for v in versions) + and sorted((public_version_id(v.get('versionId')), v['isLatest'], v['deleted']) for v in versions) + == expected_versions, 'G02 physical version population differs from all original and new ACKs') elif branch in G04_BRANCHES: require(len(original_target) == (1 if branch == "unversioned-delete" else 3), "G04 baseline has the wrong acknowledged data-version count") @@ -648,6 +713,9 @@ def physical_transition(before, after, bucket, mapped, final, branch): suffix = row["path"][len(prefix):] require(suffix == "xl.meta" or suffix.split("/")[0] in current_dirs, "G01/G03 resurrected an old/deleted data directory or left an unbound target leaf") + if branch in G02_BRANCHES and suffix != 'xl.meta': + require(re.fullmatch(r'[0-9a-f-]{36}/(?:part\.1|part\.1\.integrity\.[0-9a-f-]{36}|xl\.meta\.bkp)', suffix), + 'G02 retained an unacknowledged physical target leaf') return survivors, {"allControlShardsRestored": True, "lastACKGenerationVerified": True, "physicalProofBeforeGETOrRescue": True, "oldVersionResurrected": False} @@ -715,7 +783,35 @@ def wait_terminal(probe, fleet, path, token, reference=REFERENCE): time.sleep(.2) -def foreground(recorder, plan): +def wait_version_window(probe, fleet, path, token, reference, before, targets, bucket): + """Frozen G02 window: repaired controls and at least 32 physical targets pending.""" + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + observed = observe(probe, fleet, path, token, reference) + snapshot = fleet.snapshot('G02-live-partial-physical-window') + complete, pending = [], [] + for target in targets: + try: + validate_repaired_physical(object_scene(before, bucket, target['key']), + object_scene(snapshot, bucket, target['key']), target) + except ValueError: + pending.append(target['key']) + else: + complete.append(target['key']) + fleet.record('G02-partial-window', {'status': observed, 'completeKeys': complete, 'pendingKeys': pending}) + value = observed['response']; processed = (value.get('outcome') or {}).get('counters', {}).get('processed') + if value['summary'] == 'running' and type(processed) is int and processed > 0 and complete and len(pending) >= 32: + if TARGET_KEY not in pending: + raise WindowNotObserved('G02 selected historical target repaired before the source partial window') + return snapshot + if not pending: + raise WindowNotObserved('G02 all physical targets repaired before the source partial window') + require(value['summary'] == 'running', 'G02 original root stopped before physical partial repair') + time.sleep(.25) + raise WindowNotObserved('G02 nonzero physical partial window was not observed within its source 120-second budget') + + +def foreground(recorder, plan, *, committed=None): responses = [] upload_ids = {} part_etags = {} @@ -736,6 +832,8 @@ def foreground(recorder, plan): body = payload(step["nonce"], step["size"]) response = recorder.call(step["operation"], params, body) responses.append(response) + if committed is not None: + committed(responses) if step.get("uploadName") in {"continue", "complete", "abort"}: require(isinstance(response.get("UploadId"), str) and response["UploadId"], "G05 CreateMultipartUpload ACK lacks UploadId") @@ -748,38 +846,81 @@ def foreground(recorder, plan): return responses, {"uploadIds": upload_ids, "partEtags": part_etags} -def race(recorder, plan, probe, fleet, path, token, reference=REFERENCE): +def read_versions(recorder, ledger, lock, done): + """Read each ACK-known version, including a final round after the writer drains.""" + proofs = [] + deadline = time.monotonic() + 60 + for cycle in range(16): + final_round = done.is_set() + with lock: + entries = copy.deepcopy(ledger['entries']) + for entry in entries: + if time.monotonic() >= deadline: + return proofs + params = {'Bucket': ledger['bucket'], 'Key': entry['key'], 'VersionId': entry['versionId']} + recorder.call('get_object', params, identity={'readCycle': cycle}) + ack = json.loads((recorder.output / f'{recorder.sequence:06d}-ack.json').read_text()) + observed = ack['observation']; response = observed.get('response') or {} + require(observed.get('acknowledged') is True and observed.get('resultUnknown') is False + and not observed.get('clientCleanupError') + and response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 200 + and observed.get('params') == params and response.get('VersionId') == entry['versionId'] + and observed.get('payload') == entry['payload'] + and {key: response[key] for key in entry['properties'] if key in response} == entry['properties'], + 'G02 historical GET did not preserve its exact acknowledged version bytes/properties') + proofs.append(ack) + if final_round: + break + return proofs + + +def race(recorder, plan, probe, fleet, path, token, reference=REFERENCE, *, baseline=None): samples = [] # Only the worker mutates Recorder state. All Fleet.record writes run on # the controller, so evidence sequence allocation never races. readers = [] read_branch = bool(plan and plan[0].get("branch") in {"overwrite-vs-read-repair", "read-all-versions-during-heal"}) + version_reads = bool(plan and plan[0].get('branch') == G02_BRANCHES[1]) + version_race = bool(plan and plan[0].get('branch') in G02_BRANCHES) + done = threading.Event(); lock = threading.Lock() + ledger = {'bucket': plan[0]['params']['Bucket'], 'entries': []} + if version_reads: + require(baseline is not None, 'G02 historical reads need their complete original S3 baseline') + ledger['entries'] = [copy.deepcopy(row) for row in baseline['entries'] if row['key'] == TARGET_KEY] + require(len(ledger['entries']) == 3, 'G02 historical reader lacks three original ACK-known versions') + def committed(responses): + added = version_put_entries(baseline, plan, responses) + with lock: + originals = [copy.deepcopy(row) for row in baseline['entries'] if row['key'] == TARGET_KEY] + for entry in originals: entry['isLatest'] = False + ledger['entries'] = originals + added + def writing(): + try: + return foreground(recorder, plan, committed=committed if version_reads else None) + finally: + done.set() with ThreadPoolExecutor(max_workers=2 if read_branch else 1) as pool: - worker = pool.submit(foreground, recorder, plan) - if read_branch: + worker = pool.submit(writing) + if version_reads: + read_recorder = Recorder(probe, recorder.output.parent / 'historical-read-ACKs') + read_future = pool.submit(read_versions, read_recorder, ledger, lock, done) + elif read_branch: def read_repair(): deadline = time.monotonic() + 90 while not worker.done() and time.monotonic() < deadline: params = {"Bucket": plan[0]["params"]["Bucket"], "Key": plan[0]["params"]["Key"]} if plan[0].get("readVersionId"): params["VersionId"] = plan[0]["readVersionId"] observed = probe.s3("get_object", params) - if plan[0].get("branch") == "read-all-versions-during-heal": - response = observed.get("response") or {} - require(observed.get("acknowledged") is True and not observed.get("resultUnknown") - and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 200 - and response.get("VersionId") == params["VersionId"] - and observed.get("payload") == plan[0]["readPayload"], - "G02 exact historical read was not acknowledged with original bytes") readers.append(observed) time.sleep(.01) read_future = pool.submit(read_repair) else: read_future = None - deadline = time.monotonic() + 120 + deadline = time.monotonic() + (90 if version_race else 120) try: - while not worker.done() and time.monotonic() < deadline: + while (not worker.done() or version_reads and read_future is not None and not read_future.done()) and time.monotonic() < deadline: samples.append(observe(probe, fleet, path, token, reference)) - time.sleep(.01) + time.sleep(.025 if version_race else .01) except BaseException: recorder.mutations.poison("status observation failed; drain in-flight mutations only") raise @@ -788,17 +929,29 @@ def read_repair(): if plan and runtime: plan[0]["runtime"] = runtime if read_future is not None: - read_future.result(timeout=60) + value = read_future.result(timeout=60) + if version_reads: + readers = value acks = [json.loads(path.read_text()) for path in sorted(recorder.output.glob("*-ack.json"))] require(len(acks) == len(plan) and all(row.get("operationCheck") == "PASS" for row in acks), "G01/G03 lacks every durable foreground ACK") overlaps = [any(sample_overlap(ack, sample, path, token, reference) for sample in samples) for ack in acks] + read_complete = True + reader_overlap = True + if version_reads: + required = {row['versionId'] for row in ledger['entries']} + actual = {row['observation']['params']['VersionId'] for row in readers} + read_complete = len(required) == 6 and actual == required + reader_overlap = any(sample_overlap(ack, sample, path, token, reference) for ack in readers for sample in samples) + fleet.record('G02-historical-read-coverage', {'requiredVersionIds': sorted(required), + 'observedVersionIds': sorted(actual), 'complete': read_complete, 'readerCallOverlapObserved': reader_overlap, + 'allReadsBoundToDurableACKs': True, 'nativeFenceProved': False, 'canonicalCasePassClaimed': False}) fleet.record("read-repair-overlap-proof", {"branch": plan[0].get("branch") if plan else None, "readCalls": len(readers), "allReadsObserved": bool(readers) if read_branch else True, "nativeFenceProved": False}) fleet.record("public-call-overlap-proof", {"everyMutationOverlapped": all(overlaps), "mutations": overlaps, "nativeFenceProved": False, "serverCommitOverlapProved": False}) - return responses, all(overlaps) + return responses, all(overlaps) and read_complete and reader_overlap def verify_multipart_state(probe, fleet, bucket, branch, runtime, expected_parts): @@ -840,7 +993,7 @@ def run_branch(args, topology, nodes, branch): fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="manual") candidate = json.loads(args.candidate.read_text()) reference = source_contract(candidate, args.case_id) - fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) + inspector = fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) @@ -881,7 +1034,7 @@ def run_branch(args, topology, nodes, branch): mapped = map_fixture(before, bucket, expected) verify_part_manifest(fleet, list(mapped.values()), "baseline-streaming", count=len(mapped)) target_version = (next(entry for entry in expected if entry["key"] == current_target_key and entry["isLatest"]) - if branch in (G02_BRANCHES[1], G03_BRANCHES[1]) + if branch == G03_BRANCHES[1] else next(entry for entry in expected if entry["key"] == current_target_key)) targets = [mapped[row["key"], row["versionId"]] for row in expected if row["key"] != current_target_key or row["versionId"] == target_version["versionId"]] @@ -908,8 +1061,11 @@ def run_branch(args, topology, nodes, branch): token = start.get("response", {}).get("clientToken") if isinstance(start.get("response"), dict) else None require(start.get("httpStatus") == 200 and isinstance(token, str) and token, "G01/G03 admission lacks an original token") active.append((path, token)) - wait_running(probe, fleet, path, token, len(expected), reference) - witness = fleet.snapshot("target-missing-before-foreground") + if branch in G02_BRANCHES: + witness = wait_version_window(probe, fleet, path, token, reference, before, targets, bucket) + else: + wait_running(probe, fleet, path, token, len(expected), reference) + witness = fleet.snapshot("target-missing-before-foreground") target = next(target for target in targets if target["key"] == current_target_key) if (target["node"], target["disk"], target["relativePath"]) in files(witness): raise WindowNotObserved("G01/G03 old target already repaired before foreground mutations") @@ -919,7 +1075,10 @@ def run_branch(args, topology, nodes, branch): writer.seen.update((row["key"], row["versionId"]) for row in expected) if branch == "batch-version-delete": writer.batch_expected_ids = {item["VersionId"] for item in plan[0]["params"]["Delete"]["Objects"]} - responses, overlapped = race(writer, plan, probe, fleet, path, token, reference) + if branch in G02_BRANCHES: + responses, overlapped = race(writer, plan, probe, fleet, path, token, reference, baseline=baseline) + else: + responses, overlapped = race(writer, plan, probe, fleet, path, token, reference) if branch not in G05_BRANCHES: set_race_io_cap(fleet, args.case_id, branch, False, nodes=sorted({target['node'] for target in targets})) mpu_runtime = plan[0].get("runtime") if branch in G05_BRANCHES else None @@ -938,6 +1097,14 @@ def run_branch(args, topology, nodes, branch): fleet.stop() after = fleet.snapshot("product-physical-before-GET-or-rescue", quiescent=True) survivors, physical = physical_transition(before, after, bucket, mapped, final, branch) + if branch in G02_BRANCHES: + backups = {node: fleet.remote(node, 'version-race-backups', bucket=bucket, + expectedBinarySha256=args.binary_sha, expectedInspectorSha256=inspector['binarySha256']) for node in nodes} + fleet.record('G02-stopped-backup-decoding', backups) + original_order = [row['versionId'] for row in expected if row['key'] == current_target_key] + fleet.record('G02-serial-ACK-backup-prefix-proof', verify_version_backups( + before, after, backups, nodes, bucket, current_target_key, original_order, + final['acknowledgedVersionAppendOrder'], args.binary_sha, inspector['binarySha256'])) fleet.record("last-ACK-physical-proof", physical) verify_part_manifest(fleet, list(survivors.values()), "final-streaming", count=len(survivors)) target_authority = True diff --git a/scripts/validation_heal/race_retention.py b/scripts/validation_heal/race_retention.py index 7d1ed886..3a8c3c6f 100644 --- a/scripts/validation_heal/race_retention.py +++ b/scripts/validation_heal/race_retention.py @@ -1,4 +1,4 @@ -"""Isolate a stopped G01 scene without replaying writes or restoring old shards.""" +"""Isolate stopped G01/G02 scenes without replaying writes or restoring old shards.""" import hashlib import json @@ -13,6 +13,8 @@ CASE = 'heal.race-unversioned-overwrite' +CASES = {CASE: ('unresolved-g01-race', {'overwrite-vs-deep', 'overwrite-vs-read-repair'}), + 'heal.race-versioned-put': ('unresolved-g02-race', {'new-version-vs-historical-heal', 'read-all-versions-during-heal'})} def digest(value): @@ -22,7 +24,7 @@ def digest(value): def validate(plan): require(set(plan) == {'kind', 'runId', 'nodes', 'bucket', 'owners', 'businessSceneSha256', 'candidateCommit', 'harnessCommit', 'binarySha256', 'ackFilesSha256'} - and plan['kind'] == 'unresolved-g01-race' + and plan['kind'] in {'unresolved-g01-race', 'unresolved-g02-race'} and re.fullmatch(r'gh-[1-9][0-9]*-[1-9][0-9]*-h[12]', plan['runId']) and len(plan['nodes']) == len(set(plan['nodes'])) == (1 if plan['runId'].endswith('-h1') else 4) and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in plan['nodes']) @@ -30,7 +32,7 @@ def validate(plan): and all(re.fullmatch(r'[0-9a-f]{40}', plan[field]) for field in ('candidateCommit', 'harnessCommit')) and all(re.fullmatch(r'[0-9a-f]{64}', plan[field]) for field in ('binarySha256', 'ackFilesSha256')) and set(plan['owners']) == set(plan['businessSceneSha256']) == set(plan['nodes']), - 'G01 isolation plan identity, scope or evidence digest differs') + 'G01/G02 isolation plan identity, scope or evidence digest differs') for node in plan['nodes']: owner = plan['owners'][node] require(owner.get('runId') == plan['runId'] and owner.get('bucket') == plan['bucket'] @@ -38,35 +40,37 @@ def validate(plan): and owner.get('unit') == 'rustfs-validation-heal-' + plan['runId'] + '.service' and len(owner.get('mounts', [])) == 4 and re.fullmatch(r'[0-9a-f]{64}', plan['businessSceneSha256'][node]), - 'G01 isolation lost its exact reserved owner or stopped physical scene') + 'G01/G02 isolation lost its exact reserved owner or stopped physical scene') require(len({owner['hostname'] for owner in plan['owners'].values()}) == len(plan['nodes']), - 'G01 isolation host aliases overlap') + 'G01/G02 isolation host aliases overlap') def archived_plan(artifact, failed_run, candidate): - require(type(failed_run) is int and failed_run > 0, 'G01 isolation needs one exact failed run') + require(type(failed_run) is int and failed_run > 0, 'G01/G02 isolation needs one exact failed run') paths = list(artifact.glob('*/run/*/cases/*.json')) - require(len(paths) == 1 and paths[0].name == CASE + '.json', 'G01 isolation cannot omit another case') + require(len(paths) == 1 and paths[0].stem in CASES, 'G01/G02 isolation cannot omit another case') + case = paths[0].stem + kind, branches = CASES[case] result = json.loads(paths[0].read_text()) provenance = result.get('provenance') or {} - require(result.get('case_id') == CASE and result.get('verdict') == result.get('cleanup_status') == 'FAIL' + require(result.get('case_id') == case and result.get('verdict') == result.get('cleanup_status') == 'FAIL' and provenance.get('source_commit') == candidate['source_sha'] and provenance.get('candidate_sha256') == candidate['package_sha256'], - 'G01 isolation original failed result or candidate differs') - evidence = paths[0].parents[1] / 'evidence' / CASE + 'G01/G02 isolation original failed result or candidate differs') + evidence = paths[0].parents[1] / 'evidence' / case summary = json.loads((evidence / 'healthy-summary.json').read_text()) rows = summary.get('results', []) retained = [row for row in rows if row.get('cleanup') != 'PASS'] - require(summary.get('caseId') == CASE and summary.get('cleanup') == 'FAIL' and len(retained) == 1 + require(summary.get('caseId') == case and summary.get('cleanup') == 'FAIL' and len(retained) == 1 and retained[0].get('verdict') == 'FAIL' and retained[0].get('topology') in {'H1', 'H2'} - and retained[0].get('branch') in {'overwrite-vs-deep', 'overwrite-vs-read-repair'} - and all(row.get('caseId') == CASE for row in rows), - 'G01 isolation requires one unmixed retained public branch') + and retained[0].get('branch') in branches + and all(row.get('caseId') == case for row in rows), + 'G01/G02 isolation requires one unmixed retained public branch') row = retained[0] folder = evidence / row['topology'] / row['branch'] def one(label): matches = list(folder.glob('*-' + label + '.json')) - require(len(matches) == 1, 'G01 isolation lacks one original ' + label) + require(len(matches) == 1, 'G01/G02 isolation lacks one original ' + label) return json.loads(matches[0].read_text()) binding, scene, dirty = one('binding'), one('retained-failure-scene'), one('last-ACK-rescue-required') require(re.fullmatch(rf'gh-{failed_run}-[1-9][0-9]*-{row["topology"].lower()}', binding.get('runId', '')) @@ -77,27 +81,33 @@ def one(label): and dirty.get('state') == 'DIRTY' and dirty.get('namespacesRetained') is True and dirty.get('lastACKPhysicalAuthorityVerified') is False and dirty.get('originalGenerationBlindRestorationProhibited') is True, - 'G01 isolation run/provenance or last-ACK refusal differs') + 'G01/G02 isolation run/provenance or last-ACK refusal differs') reservations = [json.loads(path.read_text()) for path in folder.glob('*-reserved.json')] owners = {item['node']: item['owner'] for item in reservations} require(len(owners) == len(reservations) == len(scene) and set(owners) == set(scene) - and all(item.get('reserved') is True for item in reservations), 'G01 isolation lacks all reserved hosts') + and all(item.get('reserved') is True for item in reservations), 'G01/G02 isolation lacks all reserved hosts') ack_files = {} - for name in ('seed-ACKs', 'foreground-ACKs'): + for name in ('seed-ACKs', 'foreground-ACKs', 'historical-read-ACKs'): journal = folder / name entries = sorted(journal.glob('*.json')) - require(0 < len(entries) <= 1024, 'G01 isolation lacks its bounded original request/ACK journal') + if name == 'historical-read-ACKs': + require(len(entries) <= 1024, 'G02 historical journal exceeds its bound') + if not entries: continue + require(0 < len(entries) <= 1024, 'G01/G02 isolation lacks its bounded original request/ACK journal') for path in entries: require(re.fullmatch(r'[0-9]{6}-(request|ack)\.json', path.name) and path.stat().st_size <= 1024**2, - 'G01 isolation journal contains an unexpected member') + 'G01/G02 isolation journal contains an unexpected member') item = json.loads(path.read_text()) - require(item.get('runId') == binding['runId'], 'G01 isolation journal belongs to another run') + require(item.get('runId') == binding['runId'], 'G01/G02 isolation journal belongs to another run') # A missing/unknown ACK remains unknown. Isolation never infers its # effect or sends another request, including for explicit refusals. operation = item.get('operation') if path.name.endswith('-request.json') else (item.get('observation') or {}).get('operation') - require(operation in {'create_bucket', 'put_object'}, 'G01 isolation cannot omit a multipart mutation') + allowed = ({'get_object'} if name == 'historical-read-ACKs' else + {'create_bucket', 'put_bucket_versioning', 'put_object'} if kind == 'unresolved-g02-race' else + {'create_bucket', 'put_object'}) + require(operation in allowed, 'G01/G02 isolation cannot omit a multipart mutation') ack_files[name + '/' + path.name] = hashlib.sha256(path.read_bytes()).hexdigest() - plan = {'kind': 'unresolved-g01-race', 'runId': binding['runId'], 'nodes': sorted(owners), + plan = {'kind': kind, 'runId': binding['runId'], 'nodes': sorted(owners), 'bucket': binding['bucketPrefix'], 'owners': owners, 'candidateCommit': candidate['source_sha'], 'harnessCommit': binding['harnessCommit'], 'binarySha256': binding['candidateBinarySha256'], 'businessSceneSha256': {node: digest(value) for node, value in physical_content(scene).items()}, @@ -109,13 +119,13 @@ def one(label): and value.get('binarySha256') == plan['binarySha256'] and len(value['formats']) == 4 and all(type(part.get('disk')) is int for part in value['formats']) and {part['disk'] for part in value['formats']} == set(range(4)), - 'G01 isolation lacks a stopped four-drive original failure scene') + 'G01/G02 isolation lacks a stopped four-drive original failure scene') return [plan] def isolate(fleet, plan): validate(plan) - require(fleet.run_id == plan['runId'] and list(fleet.nodes) == plan['nodes'], 'G01 isolation changed its fleet') + require(fleet.run_id == plan['runId'] and list(fleet.nodes) == plan['nodes'], 'G01/G02 isolation changed its fleet') def params(node): return {'expectedOwner': plan['owners'][node], 'expectedOriginalSceneSha256': plan['businessSceneSha256'][node]} audits = {} @@ -123,14 +133,14 @@ def params(node): audit = audits[node] = fleet.record('race-retention-preflight', {'node': node, **fleet.remote(node, 'retained-scene-audit', **params(node))}) require(audit.get('inspectionOnly') is True and audit.get('mutated') is False - and audit.get('credentialContentsRead') is False, 'G01 isolation preflight did not preserve its read boundary') + and audit.get('credentialContentsRead') is False, 'G01/G02 isolation preflight did not preserve its read boundary') for node in fleet.nodes: fleet.record('race-retained-service-logs', {'node': node, **fleet.remote(node, 'logs')}) proof = fleet.record('race-retention-isolation', {'node': node, **fleet.remote(node, 'retained-scene-isolate', expectedBaseline=audits[node]['baseline'], **params(node))}) require(proof.get('activeNamespaceIsolated') is True and proof.get('baseline') == audits[node]['baseline'] and all(proof.get(field) is False for field in ('originalSceneRestored', 'backupRestorationUsed', - 'canonicalPassClaimed', 'credentialContentsRead')), 'G01 isolation did not preserve its original failure') + 'canonicalPassClaimed', 'credentialContentsRead')), 'G01/G02 isolation did not preserve its original failure') for node in fleet.nodes: proof = fleet.record('race-retention-post-audit', {'node': node, **fleet.remote(node, 'retained-scene-audit', **params(node))}) @@ -138,7 +148,7 @@ def params(node): and proof.get('activeNamespaceIsolated') is True and proof.get('baseline') == audits[node]['baseline'] and len(proof.get('privateArchiveParents', [])) == 5 and all(row.get('exists') is True and row.get('uid') == 0 and row.get('mode') == 0o700 - for row in proof['privateArchiveParents']), 'G01 isolation post-audit drifted') + for row in proof['privateArchiveParents']), 'G01/G02 isolation post-audit drifted') return {'runId': plan['runId'], 'cleanup': 'ISOLATED', 'activeNamespaceIsolated': True, 'originalSceneRestored': False, 'backupRestorationUsed': False, 'originalVerdictRewritten': False, 'canonicalPassClaimed': False, 'namespaceDeletionPerformed': False, 'intentSha256': digest(plan)} diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index 8c106443..fed5c045 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -149,7 +149,8 @@ def network_recovery_plan(artifact, failed_run, candidate): def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") - if list(artifact.glob("*/run/heal.race-unversioned-overwrite/cases/*.json")): + if (list(artifact.glob("*/run/heal.race-unversioned-overwrite/cases/*.json")) + or list(artifact.glob("*/run/heal.race-versioned-put/cases/*.json"))): return race_retention_plan(artifact, failed_run, candidate) if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/cases/*.json")): if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/evidence/heal.outcome-counters-and-error-truth/DB/H1/unknown-negative/*-one-way-legacy-write-phase.json")): @@ -647,7 +648,7 @@ def main(): if plan.get("buckets"): fleet.snapshot_buckets = plan["buckets"] fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) - if plan.get("kind") == "unresolved-g01-race": + if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race"}: complete.append({"failedRunId": args.failed_run_id, **isolate_race_scene(fleet, plan)}) continue if plan.get("kind") == "unresolved-network-load": diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py index 778a9c23..3bc974c4 100644 --- a/scripts/validation_heal/retained_scene.py +++ b/scripts/validation_heal/retained_scene.py @@ -138,7 +138,7 @@ def visit(path): and re.fullmatch(r"J02-[0-9a-f]{32}\.(original|intent\.json|applied\.json|failed-[0-9a-f]{64})", parts[1])) race_backup = (not state and owner.get("runtimeProfile") == "manual" and len(parts) == 2 and parts[0] == "quarantine" - and re.fullmatch(r"G01-[a-z0-9-]{1,48}\.(original|intent\.json|applied\.json)", parts[1])) + and re.fullmatch(r"G0[12]-[a-z0-9-]{1,48}\.(original|intent\.json|applied\.json)", parts[1])) if (state and relative == "rustfs") or fixture or mrf or backup or opaque_backup or network_mpu or network_backup or race_backup: fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) try: diff --git a/test_validation_heal_g02_source.py b/test_validation_heal_g02_source.py new file mode 100644 index 00000000..3f4f0085 --- /dev/null +++ b/test_validation_heal_g02_source.py @@ -0,0 +1,387 @@ +"""Frozen G02 ACK/reference and decoded physical contracts; no live proof.""" +import copy +import hashlib +import json +import os +from pathlib import Path +import tempfile +import threading +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch +import uuid + +from scripts.validation_heal import race, mpu_race_backups as backups, heal_fault, cursor +from scripts.validation_heal.heal_workload import Recorder +import test_validation_heal_mpu_race_backups as backup_fixtures + + +class VersionPutSourceTests(unittest.TestCase): + bucket = 'rv-heal-0123456789abcdef' + + def baseline(self): + entries = [] + for number, key in enumerate([f'objects/{i:04d}.bin' for i in range(3)] + [race.TARGET_KEY]): + for generation in range(3 if key == race.TARGET_KEY else 1): + nonce = f'{self.bucket}/{key}/{generation}' + body = race.payload(nonce, race.SIZE) + entries.append({'key': key, 'versionId': str(uuid.UUID(int=1 + number * 10 + generation)), + 'isLatest': key != race.TARGET_KEY or generation == 2, 'deleteMarker': False, + 'payload': {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}, + 'properties': {'ETag': '"seed"', 'ContentLength': race.SIZE, + 'ContentType': 'application/octet-stream', 'Metadata': {'nonce': nonce}}, 'tags': []}) + return {'entries': entries} + + def test_original_three_puts_copy_distinct_immutable_control_payloads(self): + for branch in race.G02_BRANCHES: + baseline = self.baseline(); plan = race.operation_plan(self.bucket, branch, baseline['entries']) + self.assertEqual(len(plan), 3) + for i, step in enumerate(plan): + control = baseline['entries'][i] + self.assertEqual(step['referenceKey'], control['key']) + body = race.payload(step['nonce'], race.SIZE) + self.assertEqual({'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}, control['payload']) + + def test_final_manifest_retains_every_original_and_serial_put_ack(self): + for branch in race.G02_BRANCHES: + baseline = self.baseline(); plan = race.operation_plan(self.bucket, branch, baseline['entries']) + acks = [{'VersionId': str(uuid.UUID(int=100 + i)), 'ETag': f'"new-{i}"'} for i in range(3)] + final = race.final_inventory(baseline, branch, plan, acks) + target = [r for r in final['entries'] if r['key'] == race.TARGET_KEY] + self.assertEqual([r['versionId'] for r in target], + [r['versionId'] for r in baseline['entries'] if r['key'] == race.TARGET_KEY] + [a['VersionId'] for a in acks]) + self.assertEqual([r['isLatest'] for r in target], [False] * 5 + [True]) + self.assertEqual([r['payload'] for r in target[-3:]], [r['payload'] for r in baseline['entries'][:3]]) + self.assertEqual(baseline, self.baseline()) + + def test_final_manifest_refuses_duplicate_original_nil_or_missing_new_version_acks(self): + for kind in ('duplicate', 'original', 'nil', 'missing', 'reference-drift'): + with self.subTest(kind=kind): + baseline = self.baseline(); branch = race.G02_BRANCHES[0] + plan = race.operation_plan(self.bucket, branch, baseline['entries']) + acks = [{'VersionId': str(uuid.UUID(int=100 + i)), 'ETag': f'"new-{i}"'} for i in range(3)] + if kind == 'duplicate': acks[1]['VersionId'] = acks[0]['VersionId'] + elif kind == 'original': acks[1]['VersionId'] = baseline['entries'][-1]['versionId'] + elif kind == 'nil': acks[1]['VersionId'] = str(uuid.UUID(int=0)) + elif kind == 'missing': del acks[1]['VersionId'] + else: plan[1]['nonce'] = 'foreign payload' + with self.assertRaises(ValueError): race.final_inventory(baseline, branch, plan, acks) + + def test_public_manifest_sorts_uuid_identity_without_reordering_the_serial_ack_or_latest(self): + baseline = self.baseline(); branch = race.G02_BRANCHES[0] + plan = race.operation_plan(self.bucket, branch, baseline['entries']) + acks = [{'VersionId': str(uuid.UUID(int=i)), 'ETag': f'"new-{i}"'} for i in (900, 700, 800)] + final = race.final_inventory(baseline, branch, plan, acks) + self.assertEqual(final['entries'], sorted(final['entries'], key=lambda r: (r['key'], r['versionId']))) + self.assertEqual(final['acknowledgedVersionAppendOrder'], [a['VersionId'] for a in acks]) + latest = [r for r in final['entries'] if r['key'] == race.TARGET_KEY and r['isLatest']] + self.assertEqual([r['versionId'] for r in latest], [acks[-1]['VersionId']]) + + def scene(self, topology): + baseline = self.baseline(); old = [r for r in baseline['entries'] if r['key'] == race.TARGET_KEY] + new = [{**copy.deepcopy(baseline['entries'][i]), 'key': race.TARGET_KEY, + 'versionId': str(uuid.UUID(int=100 + i)), 'isLatest': i == 2} for i in range(3)] + final = {'entries': [copy.deepcopy(r) for r in baseline['entries']] + new, + 'acknowledgedVersionAppendOrder': [r['versionId'] for r in new]} + for r in final['entries']: + if r['key'] == race.TARGET_KEY: r['isLatest'] = r['versionId'] == new[-1]['versionId'] + nodes = ['vm000'] if topology == 'H1' else [f'vm00{i}' for i in range(4)] + count = 4 * len(nodes); data = 2 if topology == 'H1' else 12 + disk_ids = [str(uuid.UUID(int=1000 + i)) for i in range(count)] + distribution = list(range(count, 0, -1)) + def build(entries): + result = {} + for node_number, node in enumerate(nodes): + value = result[node] = {'files': [], 'formats': []} + for disk in range(4): + slot = node_number * 4 + disk; index = distribution[slot] + value['formats'].append({'disk': disk, 'value': {'format': 'xl', 'id': 'pool', + 'xl': {'sets': [disk_ids], 'this': disk_ids[slot]}}}) + for key in sorted({r['key'] for r in entries}): + versions = [] + for row in (r for r in entries if r['key'] == key): + generation = str(uuid.UUID(int=10000 + uuid.UUID(row['versionId']).int)) + version = {'versionId': row['versionId'], 'isLatest': row['isLatest'], 'deleted': False, + 'dataDir': generation, 'size': race.SIZE, 'inline': None, 'usesLegacyChecksum': False, + 'role': 'data' if index <= data else 'parity', + 'parts': [{'number': 1, 'size': race.SIZE, 'actualSize': race.SIZE}], + 'erasure': {'index': index, 'data_blocks': data, 'parity_blocks': count - data, + 'block_size': 1048576, 'distribution': distribution, 'algorithm': 'rs-vandermonde'}} + versions.append(version) + digest = hashlib.sha256((row['payload']['sha256'] + '/index/' + str(index)).encode()).hexdigest() + value['files'].append({'disk': disk, 'path': f'{self.bucket}/{key}/{generation}/part.1', + 'size': (race.SIZE + data - 1) // data + 32, 'sha256': digest}) + metadata = {'bucket': self.bucket, 'key': key, 'versions': versions} + value['files'].append({'disk': disk, 'path': f'{self.bucket}/{key}/xl.meta', 'size': 1000, + 'sha256': hashlib.sha256(json.dumps(metadata, sort_keys=True).encode()).hexdigest(), 'metadata': metadata}) + return result + before, after = build(baseline['entries']), build(final['entries']) + return before, after, race.map_fixture(before, self.bucket, baseline['entries']), final + + def test_actual_transition_accepts_encoded_lengths_and_all_six_versions_in_h1_and_h2(self): + for topology in ('H1', 'H2'): + for branch in race.G02_BRANCHES: + with self.subTest(topology=topology, branch=branch): + before, after, mapped, final = self.scene(topology) + survivors, proof = race.physical_transition(before, after, self.bucket, mapped, final, branch) + target = [v for (key, _), v in survivors.items() if key == race.TARGET_KEY] + self.assertEqual(len(target), 6) + self.assertTrue(all(p['size'] != race.SIZE for v in target for p in v['cohort'])) + self.assertTrue(proof['physicalProofBeforeGETOrRescue']) + + def test_transition_rejects_wrong_new_or_retained_shards_metadata_and_unacknowledged_versions(self): + for topology in ('H1', 'H2'): + for fault in ('new-size', 'new-sha', 'old-sha', 'metadata-size', 'extra-version', 'missing-part', 'old-generation', 'ack-payload'): + with self.subTest(topology=topology, fault=fault): + before, after, mapped, final = self.scene(topology) + files = after['vm000']['files'] + meta = next(r for r in files if r['path'] == f'{self.bucket}/{race.TARGET_KEY}/xl.meta') + old_dir = meta['metadata']['versions'][0]['dataDir']; new_dir = meta['metadata']['versions'][-1]['dataDir'] + part = next(r for r in files if r['path'] == f'{self.bucket}/{race.TARGET_KEY}/{new_dir}/part.1') + if fault == 'new-size': part['size'] = race.SIZE + elif fault == 'new-sha': part['sha256'] = 'f' * 64 + elif fault == 'old-sha': next(r for r in files if r['path'] == f'{self.bucket}/{race.TARGET_KEY}/{old_dir}/part.1')['sha256'] = 'e' * 64 + elif fault == 'metadata-size': + for scene in after.values(): + m = next(r for r in scene['files'] if r['path'] == f'{self.bucket}/{race.TARGET_KEY}/xl.meta') + m['metadata']['versions'][-1]['size'] += 1 + elif fault == 'extra-version': meta['metadata']['versions'].append({**copy.deepcopy(meta['metadata']['versions'][0]), 'versionId': str(uuid.UUID(int=999))}) + elif fault == 'missing-part': files.remove(part) + elif fault == 'old-generation': meta['metadata']['versions'][-1]['dataDir'] = old_dir + else: final['entries'][-1]['payload']['sha256'] = 'd' * 64 + with self.assertRaises(ValueError): race.physical_transition(before, after, self.bucket, mapped, final, race.G02_BRANCHES[0]) + + def backup_scene(self, topology, prefix): + before, after, mapped, final = self.scene(topology) + old = [v['versionId'] for (key, _), v in mapped.items() if key == race.TARGET_KEY] + new = final['acknowledgedVersionAppendOrder']; observations = {} + for node in before: + common = {'hostname': 'host-' + node, 'binarySha256': 'a' * 64, 'quiescent': True} + before[node].update(common); after[node].update(common) + observations[node] = {**common, 'inspectorSha256': 'b' * 64, 'rows': [], 'readOnly': True, + 'sameHeldFileHashedAndDecoded': True, 'rawContentsExported': False, 'liveAuthorityClaimed': False} + for value in (before[node], after[node]): + for row in value['files']: + if 'metadata' in row: row['metadata']['physicalBitrotVerified'] = False + for disk in range(4): + path = f'{self.bucket}/{race.TARGET_KEY}/xl.meta' + primary = next(r['metadata'] for r in after[node]['files'] if r['disk'] == disk and r['path'] == path) + versions = {v['versionId']: v for v in primary['versions']} + ids = old + new[:prefix] + decoded = {**primary, 'versions': [{**copy.deepcopy(versions[vid]), 'isLatest': vid == ids[-1]} for vid in ids]} + for vid in old: + row = {'disk': disk, 'path': f'{self.bucket}/{race.TARGET_KEY}/{versions[vid]["dataDir"]}/xl.meta.bkp', + 'size': 100 + prefix, 'sha256': hashlib.sha256(f'{disk}/{vid}/{prefix}'.encode()).hexdigest()} + after[node]['files'].append(copy.deepcopy(row)) + observations[node]['rows'].append({**row, 'metadata': copy.deepcopy(decoded)}) + return before, after, observations, old, new + + def test_all_three_original_generation_backups_require_a_contiguous_three_ack_prefix(self): + for topology in ('H1', 'H2'): + for count in range(4): + with self.subTest(topology=topology, prefix=count): + before, after, observed, old, new = self.backup_scene(topology, count) + result = backups.verify_versions(before, after, observed, list(before), self.bucket, race.TARGET_KEY, + old, new, 'a' * 64, 'b' * 64) + self.assertEqual(len(result['proofs']), 12 * len(before)) + self.assertEqual({r['acknowledgedCompletePrefix'] for r in result['proofs']}, {count}) + self.assertFalse(result['liveAuthorityClaimed'] or result['physicalRepairProved'] or result['canonicalCasePassClaimed']) + for fault in ('nonprefix', 'missing-old', 'descriptor', 'latest', 'sha', 'new-directory', 'omitted', 'foreign-host'): + with self.subTest(fault=fault): + before, after, observed, old, new = self.backup_scene('H1', 1) + row = observed['vm000']['rows'][0] + if fault == 'nonprefix': row['metadata']['versions'][-1]['versionId'] = new[1] + elif fault == 'missing-old': row['metadata']['versions'].pop(0) + elif fault == 'descriptor': row['metadata']['versions'][0]['parts'][0]['size'] += 1 + elif fault == 'latest': row['metadata']['versions'][0]['isLatest'] = True + elif fault == 'sha': row['sha256'] = 'e' * 64 + elif fault == 'new-directory': + old_path = row['path']; row['path'] = row['path'].replace(str(uuid.UUID(int=10031)), str(uuid.UUID(int=10100))) + next(r for r in after['vm000']['files'] if r['path'] == old_path and r['disk'] == row['disk'])['path'] = row['path'] + elif fault == 'omitted': observed['vm000']['rows'].pop() + else: observed['vm000']['hostname'] = 'foreign' + with self.assertRaises(ValueError): + backups.verify_versions(before, after, observed, list(before), self.bucket, race.TARGET_KEY, old, new, 'a' * 64, 'b' * 64) + + def test_nested_key_collector_binds_actual_open_fd_and_all_six_bounded_generations(self): + with tempfile.TemporaryDirectory() as directory: + helper = backup_fixtures.MultipartBackupTests() + request, node, files, _ = helper.fixture(directory) + for mount in node['MOUNTS']: + parent = mount / 'heal-validation' / request['runId'] / 'data' / self.bucket + (parent / 'objects').mkdir(); (parent / backups.KEY).rename(parent / race.TARGET_KEY) + for i in range(101, 106): (parent / race.TARGET_KEY / str(uuid.UUID(int=i))).mkdir() + request['action'] = 'version-race-backups' + def command(tool, path, bucket, key, **kwargs): + self.assertEqual((bucket, key), (self.bucket, race.TARGET_KEY)) + fd = int(path.rsplit('/', 1)[1]); raw = os.pread(fd, backups.LIMIT + 1, 0) + return SimpleNamespace(returncode=0, stdout=json.dumps({'bucket': bucket, 'key': key, + 'physicalBitrotVerified': False, 'heldFileSha256': hashlib.sha256(raw).hexdigest()})) + node['command'] = command + with patch.object(heal_fault, 'mount_id', side_effect=lambda fd: os.fstat(fd).st_dev): + value = backups.run(request, node) + self.assertEqual(len(value['rows']), 4) + self.assertTrue(all(r['metadata']['heldFileSha256'] == r['sha256'] for r in value['rows'])) + self.assertFalse(value['liveAuthorityClaimed'] or value['rawContentsExported']) + + def test_actual_reader_keeps_durable_acks_and_final_round_for_all_old_and_new_versions(self): + for fault in (None, 'version', 'payload', 'property', 'unknown'): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as directory: + baseline = self.baseline(); original = [r for r in baseline['entries'] if r['key'] == race.TARGET_KEY] + plan = race.operation_plan(self.bucket, race.G02_BRANCHES[1], baseline['entries']) + acks = [{'VersionId': str(uuid.UUID(int=100 + i)), 'ETag': f'"new-{i}"'} for i in range(3)] + added = race.version_put_entries(baseline, plan, acks) + all_entries = {r['versionId']: r for r in original + added} + lock = threading.Lock(); done = threading.Event() + ledger = {'bucket': self.bucket, 'entries': copy.deepcopy(original)}; calls = [] + def invoke(operation, params, body): + self.assertEqual(operation, 'get_object'); calls.append(params['VersionId']) + entry = all_entries[params['VersionId']] + value = {'operation': operation, 'params': copy.deepcopy(params), 'acknowledged': True, + 'resultUnknown': False, 'response': {'ResponseMetadata': {'HTTPStatusCode': 200}, + 'VersionId': entry['versionId'], **copy.deepcopy(entry['properties'])}, 'payload': copy.deepcopy(entry['payload'])} + if fault == 'version': value['response']['VersionId'] = str(uuid.UUID(int=999)) + elif fault == 'payload': value['payload']['sha256'] = 'e' * 64 + elif fault == 'property': del value['response']['ContentLength'] + elif fault == 'unknown': value.update(acknowledged=False, resultUnknown=True) + if len(calls) == 3: + with lock: ledger['entries'] = copy.deepcopy(original + added) + done.set() + return value + probe = SimpleNamespace(binding={'runId': 'gh-12345-1-h1', 'bucketPrefix': self.bucket, + 'maxInventoryBytes': 512 * 1024**2}, s3=invoke) + recorder = Recorder(probe, Path(directory) / 'reads') + if fault is None: + proofs = race.read_versions(recorder, ledger, lock, done) + self.assertEqual(len(proofs), 9) + self.assertEqual(set(calls), set(all_entries)) + self.assertEqual(len(list(recorder.output.glob('*-ack.json'))), 9) + else: + with self.assertRaises(ValueError): race.read_versions(recorder, ledger, lock, done) + self.assertEqual(len(calls), 1) + self.assertEqual(len(list(recorder.output.glob('*-ack.json'))), 1) + + def test_existing_part_verifier_streams_all_259_seed_or_262_final_versions_in_h1_h2(self): + for nodes in (['vm000'], [f'vm00{i}' for i in range(4)]): + for count in (259, 262): + with self.subTest(nodes=nodes, count=count): + cohorts = []; indexed = {}; calls = [] + for number in range(count): + members = [] + for node in nodes: + for disk in range(4): + row = {'node': node, 'disk': disk, 'relativePath': f'bucket/key/{number}/part.1', + 'versionId': str(uuid.UUID(int=number + 1)), 'partNumber': 1, + 'sha256': hashlib.sha256(f'{node}/{disk}/{number}'.encode()).hexdigest(), 'size': 131678} + members.append(row); indexed[node, disk, row['relativePath']] = row + cohorts.append({'cohort': members}) + def remote(node, action, **params): + self.assertEqual(action, 'verify-parts'); calls.extend((node, r['disk'], r['relativePath']) for r in params['parts']) + return {'complete': True, 'parts': [{**indexed[node, r['disk'], r['relativePath']], + 'physicalSize': indexed[node, r['disk'], r['relativePath']]['size'], 'physicalBitrotVerified': True} for r in params['parts']]} + fleet = SimpleNamespace(nodes=nodes, remote=remote, record=lambda label, value: value) + result = cursor.verify_part_manifest(fleet, cohorts, 'bounded-version-manifest', count=count) + self.assertEqual(result['allPartsStreamingVerified'], count * 4 * len(nodes)) + self.assertEqual(len(calls), len(set(calls))) + for wrong_count in (False, -1, cursor.OBJECTS + 1, count - 1): + with self.assertRaises(ValueError): cursor.verify_part_manifest(fleet, cohorts, 'refused', count=wrong_count) + + def test_real_g02_caller_faults_the_original_oldest_version_in_both_public_branches(self): + class FaultBoundaryReached(BaseException): + pass + for branch in race.G02_BRANCHES: + with self.subTest(branch=branch), tempfile.TemporaryDirectory() as directory: + root = Path(directory); before, _, mapped, _ = self.scene('H1'); baseline = self.baseline() + candidate = root / 'candidate.json'; candidate.write_text(json.dumps({'source_sha': race.REFERENCE['candidateCommit'], 'package_sha256': 'b' * 64})) + rc = root / 'rc'; rc.write_bytes(b'fixture-rc') + fleet = Mock(); fleet.faults = []; fleet.cleanup.return_value = True + fleet.record.side_effect = lambda label, value: value + fleet.snapshot.return_value = before + chosen = mapped[race.TARGET_KEY, next(r['versionId'] for r in baseline['entries'] if r['key'] == race.TARGET_KEY)] + def remote(node, action, **params): + self.assertEqual(action, 'apply-fault') + if params['relativePath'].startswith(f'{self.bucket}/{race.TARGET_KEY}/'): + self.assertEqual(params['relativePath'], chosen['relativePath']) + self.assertEqual(params['disk'], 3) + raise FaultBoundaryReached() + return {'kind': 'missing'} + fleet.remote.side_effect = remote + probe = Mock(); probe.inventory.return_value = baseline + args = SimpleNamespace(evidence=root / 'evidence', candidate=candidate, rc_bin=rc, + tool_directory=root, testing_commit='c' * 40, binary_sha='a' * 64, case_id='heal.race-versioned-put') + with patch.object(race, 'Fleet', return_value=fleet), patch.object(race, 'verify_inspector', return_value={}), \ + patch.object(race, 'Probe', return_value=probe), patch.object(race, 'validate_topology'), \ + patch.object(race, 'seed_fixture', return_value=baseline['entries']), patch.object(race, 'verify_seed'), \ + patch.object(race, 'map_fixture', return_value=mapped), patch.object(race, 'OBJECTS', 4), \ + patch.object(race, 'verify_part_manifest') as verify, \ + patch.dict(race.os.environ, {'GITHUB_RUN_ID': '12345', 'GITHUB_RUN_ATTEMPT': '1'}): + # The actual binding selects a fresh bucket; its read-only + # physical plan here remains the known synthetic baseline. + actual_plan = race.operation_plan(self.bucket, branch, baseline['entries']) + with patch.object(race, 'operation_plan', return_value=actual_plan), self.assertRaises(FaultBoundaryReached): + race.run_branch(args, 'H1', ['vm000'], branch) + self.assertEqual(verify.call_args.kwargs['count'], len(mapped)) + + def test_controller_requires_both_complete_six_version_reads_and_original_token_reader_overlap(self): + for complete, reader_overlap in ((True, True), (True, False), (False, True)): + with self.subTest(complete=complete, reader_overlap=reader_overlap), tempfile.TemporaryDirectory() as directory: + baseline = self.baseline(); branch = race.G02_BRANCHES[1] + plan = race.operation_plan(self.bucket, branch, baseline['entries']) + for step in plan: step['branch'] = branch + output = Path(directory) / 'foreground'; output.mkdir() + writer = SimpleNamespace(output=output, mutations=SimpleNamespace(poison=lambda message: None)) + sequence = 0 + def call(operation, params, body): + nonlocal sequence + sequence += 1 + ack = {'VersionId': str(uuid.UUID(int=100 + sequence)), 'ETag': f'"new-{sequence}"'} + (output / f'{sequence:06d}-ack.json').write_text(json.dumps({'operationCheck': 'PASS', + 'observation': {'operation': operation, 'params': params}})) + return ack + writer.call = call + sampled = threading.Event(); records = {} + def observe(*args): + sampled.set(); return {'originalTokenSample': True} + def read(recorder, ledger, lock, done): + self.assertTrue(done.wait(2)); self.assertTrue(sampled.wait(2)) + with lock: entries = copy.deepcopy(ledger['entries']) + return [{'observation': {'operation': 'get_object', 'params': {'VersionId': row['versionId']}}} + for row in (entries if complete else entries[:3])] + fleet = SimpleNamespace(record=lambda label, value: records.setdefault(label, value)) + probe = SimpleNamespace(binding={'runId': 'gh-12345-1-h1', 'bucketPrefix': self.bucket, + 'maxInventoryBytes': 512 * 1024**2}) + def overlap(ack, *args): + return ack['observation']['operation'] == 'put_object' or reader_overlap + with patch.object(race, 'observe', side_effect=observe), patch.object(race, 'read_versions', side_effect=read), \ + patch.object(race, 'sample_overlap', side_effect=overlap): + responses, accepted = race.race(writer, plan, probe, fleet, '/original/path', 'original-token', baseline=baseline) + self.assertEqual(len(responses), 3) + self.assertEqual(accepted, complete and reader_overlap) + self.assertEqual(records['G02-historical-read-coverage']['complete'], complete) + self.assertEqual(records['G02-historical-read-coverage']['readerCallOverlapObserved'], reader_overlap) + + def test_frozen_partial_window_requires_repaired_controls_32_pending_and_old_target_still_missing(self): + keys = [f'objects/{i:04d}.bin' for i in range(32)] + [race.TARGET_KEY] + targets = [{'key': key} for key in keys] + scene = {'vm000': {'files': [], 'formats': []}} + for kind in ('valid', '31-pending', 'no-repaired', 'target-repaired', 'all-repaired', 'root-finished', 'bool-progress'): + with self.subTest(kind=kind): + repaired = ({keys[0]} if kind not in {'31-pending', 'no-repaired', 'target-repaired', 'all-repaired'} else + set(keys[:2]) if kind == '31-pending' else set() if kind == 'no-repaired' else + {race.TARGET_KEY} if kind == 'target-repaired' else set(keys)) + def validate(before, after, target): + if target['key'] not in repaired: raise ValueError('original missing part remains absent') + response = {'summary': 'finished' if kind == 'root-finished' else 'running', + 'outcome': {'counters': {'processed': True if kind == 'bool-progress' else 1}}} + fleet = SimpleNamespace(snapshot=lambda label: scene, record=lambda label, value: value) + with patch.object(race, 'observe', return_value={'response': response}), \ + patch.object(race, 'validate_repaired_physical', side_effect=validate), \ + patch.object(race.time, 'monotonic', side_effect=[0, 0, 121]), patch.object(race.time, 'sleep'): + if kind == 'valid': + self.assertEqual(race.wait_version_window(None, fleet, '/original', 'token', {}, scene, targets, self.bucket), scene) + else: + with self.assertRaises(ValueError): race.wait_version_window(None, fleet, '/original', 'token', {}, scene, targets, self.bucket) + + +if __name__ == '__main__': + unittest.main() diff --git a/test_validation_heal_mpu_race_backups.py b/test_validation_heal_mpu_race_backups.py new file mode 100644 index 00000000..7e7d056a --- /dev/null +++ b/test_validation_heal_mpu_race_backups.py @@ -0,0 +1,151 @@ +"""Actual file/descriptor checks with a mock decoder; no live metadata proof.""" +import copy +import hashlib +import json +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import Mock,patch +import uuid + +from scripts.validation_heal import fleet_node, heal_fault, mpu_race_backups as backup +from scripts.validation_heal.healthy import Fleet + + +class MultipartBackupTests(unittest.TestCase): + def frames(self,nodes,prefix_count): + bucket='rv-heal-0123456789abcdef';old,new=str(uuid.UUID(int=1)),str(uuid.UUID(int=2));binary='a'*64;inspector='b'*64 + before,after,evidence={},{},{} + for host_number,host in enumerate(nodes): + common={'hostname':'host-'+host,'binarySha256':binary,'quiescent':True} + before[host]={**common,'files':[]};after[host]={**common,'files':[]} + evidence[host]={**common,'readOnly':True,'sameHeldFileHashedAndDecoded':True,'rawContentsExported':False, + 'liveAuthorityClaimed':False,'inspectorSha256':inspector,'rows':[]} + for disk in range(4): + v0={'versionId':old,'dataDir':str(uuid.UUID(int=100)),'isLatest':True,'deleted':False, + 'codingIndex':host_number*4+disk+1,'parts':[{'number':n,'digest':'old-'+str(n)} for n in (1,2,3)]} + v1={**v0,'versionId':new,'dataDir':str(uuid.UUID(int=200)),'parts':[{'number':n,'digest':'new-'+str(n)} for n in (1,2,3)]} + m0={'bucket':bucket,'key':backup.KEY,'physicalBitrotVerified':False,'versions':[copy.deepcopy(v0)]} + m1={**m0,'versions':[{**v0,'isLatest':False},v1]} + before[host]['files'].append({'disk':disk,'path':f'{bucket}/{backup.KEY}/xl.meta','size':100,'sha256':'0'*64,'metadata':m0}) + after[host]['files'].append({'disk':disk,'path':f'{bucket}/{backup.KEY}/xl.meta','size':200,'sha256':'1'*64,'metadata':m1}) + row={'disk':disk,'path':f'{bucket}/{backup.KEY}/{v0["dataDir"]}/xl.meta.bkp','size':100+prefix_count, + 'sha256':str(prefix_count+2)*64} + after[host]['files'].append(copy.deepcopy(row)) + evidence[host]['rows'].append({**row,'metadata':copy.deepcopy(m0 if prefix_count==0 else m1)}) + return before,after,evidence,bucket,old,new,binary,inspector + + def test_frozen_backup_oracle_accepts_only_original_or_complete_ack_prefix_in_h1_h2(self): + for nodes in (['vm000'],['vm000','vm001','vm002','vm003']): + for count in (0,1): + with self.subTest(nodes=nodes,prefix=count): + before,after,evidence,bucket,old,new,binary,inspector=self.frames(nodes,count) + proof=backup.verify(before,after,evidence,nodes,bucket,old,new,binary,inspector) + self.assertEqual(len(proof['proofs']),4*len(nodes)) + self.assertEqual({r['acknowledgedCompletePrefix'] for r in proof['proofs']},{count}) + self.assertFalse(proof['liveAuthorityClaimed'] or proof['physicalRepairProved'] or proof['canonicalCasePassClaimed']) + self.assertTrue(all(not r['unexposedMetadataVerified'] for r in proof['proofs'])) + + def test_backup_oracle_rejects_drift_foreign_generation_nonprefix_and_incomplete_scope(self): + for fault in ('sha','generation','version','latest','descriptor','omitted','duplicate','hostname','primary','bool-disk','binary'): + with self.subTest(fault=fault): + nodes=['vm000'];before,after,evidence,bucket,old,new,binary,inspector=self.frames(nodes,1) + row=evidence['vm000']['rows'][0] + if fault=='sha':row['sha256']='f'*64 + elif fault=='generation': + row['path']=row['path'].replace(str(uuid.UUID(int=100)),str(uuid.UUID(int=200))) + after['vm000']['files'][1]['path']=row['path'] + elif fault=='version':row['metadata']['versions'][1]['versionId']=str(uuid.UUID(int=3)) + elif fault=='latest':row['metadata']['versions'][0]['isLatest']=True + elif fault=='descriptor':row['metadata']['versions'][0]['parts'][0]['digest']='wrong' + elif fault=='omitted':evidence['vm000']['rows'].pop() + elif fault=='duplicate':evidence['vm000']['rows'].append(copy.deepcopy(row)) + elif fault=='hostname':evidence['vm000']['hostname']='foreign' + elif fault=='primary':after['vm000']['files']=[r for r in after['vm000']['files'] if not (r['disk']==2 and r['path'].endswith('/xl.meta'))] + elif fault=='bool-disk':before['vm000']['files'][0]['disk']=False + else:binary=None + with self.assertRaises(ValueError):backup.verify(before,after,evidence,nodes,bucket,old,new,binary,inspector) + + def fixture(self,folder): + folder=Path(folder).resolve();state=folder/'state';state.mkdir() + (state/'rustfs').write_bytes(b'owned candidate');(state/'heal-meta-inspect').write_bytes(b'owned inspector') + (state/'credentials.env').write_bytes(b'private mock credential; never opened by collector') + mounts=tuple(folder/f'mount-{i}' for i in range(4));run='gh-12345-1-h1';bucket='rv-heal-0123456789abcdef' + roots=tuple(m/'heal-validation'/run for m in mounts);files=[] + for i,root in enumerate(roots): + key=root/'data'/bucket/backup.KEY;key.mkdir(parents=True);(key/'xl.meta').write_bytes(b'live current metadata not read here') + generation=key/str(uuid.UUID(int=100));generation.mkdir() + path=generation/'xl.meta.bkp';path.write_bytes(f'actual original backup on disk{i}'.encode());files.append(path) + (generation/'part.1').write_bytes(b'payload part not read by collector') + owner={'runtimeProfile':'manual','bucket':bucket,'hostname':'owned-test-host', + 'binarySha256':fleet_node.digest(state/'rustfs'),'inspectorSha256':fleet_node.digest(state/'heal-meta-inspect')} + stopped=Mock();decoded=[] + def digest(p): + self.assertIn(p,(state/'rustfs',state/'heal-meta-inspect')) + return fleet_node.digest(p) + def command(*args,**kwargs): + self.assertEqual(args[0],str(state/'heal-meta-inspect')) + self.assertTrue(args[1].startswith(f'/proc/{os.getpid()}/fd/')) + fd=int(args[1].rsplit('/',1)[1]);raw=os.pread(fd,backup.LIMIT+1,0) + self.assertIn(raw,[p.read_bytes() for p in files]) + decoded.append(hashlib.sha256(raw).hexdigest()) + return SimpleNamespace(returncode=0,stdout=json.dumps({'bucket':bucket,'key':backup.KEY, + 'physicalBitrotVerified':False,'mockInputSha256':decoded[-1]}),stderr='') + node={'owned':lambda run:(state,roots,copy.deepcopy(owner)),'MOUNTS':mounts,'checked':fleet_node.checked, + 'digest':digest,'stopped':stopped,'fault':heal_fault,'command':command} + request={'runId':run,'action':'mpu-race-backups','bucket':bucket, + 'expectedBinarySha256':owner['binarySha256'],'expectedInspectorSha256':owner['inspectorSha256']} + return request,node,files,decoded + + def test_stopped_census_hash_and_decoder_share_actual_opened_inode_on_every_drive(self): + with tempfile.TemporaryDirectory() as tmp: + request,node,files,decoded=self.fixture(tmp);original=[p.read_bytes() for p in files] + with patch.object(heal_fault,'mount_id',side_effect=lambda fd:os.fstat(fd).st_dev):value=backup.run(request,node) + self.assertEqual(len(value['rows']),4);self.assertEqual(len(decoded),4) + for i,row in enumerate(value['rows']): + self.assertEqual(row['disk'],i);self.assertEqual(row['size'],len(original[i])) + self.assertEqual(row['sha256'],hashlib.sha256(original[i]).hexdigest()) + self.assertEqual(row['metadata']['mockInputSha256'],row['sha256']) + self.assertTrue(value['readOnly'] and value['quiescent'] and value['sameHeldFileHashedAndDecoded']) + self.assertFalse(value['rawContentsExported'] or value['liveAuthorityClaimed'] or value['canonicalCasePassClaimed']) + self.assertEqual([p.read_bytes() for p in files],original) + self.assertEqual(node['stopped'].call_count,2) + + def test_hardlinks_symlinks_fifo_oversize_and_unknown_generation_refuse_without_decoder(self): + for fault in ('hardlink','symlink','fifo','oversize','generation'): + with self.subTest(fault=fault),tempfile.TemporaryDirectory() as tmp: + request,node,files,decoded=self.fixture(tmp);p=files[0] + if fault=='hardlink':os.link(p,p.with_name('extra-link')) + elif fault=='symlink':p.unlink();p.symlink_to(Path(tmp)/'state/credentials.env') + elif fault=='fifo':p.unlink();os.mkfifo(p) + elif fault=='oversize':p.write_bytes(b'x'*(backup.LIMIT+1)) + else:p.parent.rename(p.parent.with_name('unregistered-directory')) + with patch.object(heal_fault,'mount_id',side_effect=lambda fd:os.fstat(fd).st_dev),self.assertRaises((OSError,ValueError)): + backup.run(request,node) + self.assertFalse(decoded) + + def test_injected_file_mount_escape_and_decoder_identity_or_inode_drift_refuse(self): + for fault in ('mount','decoder','inode'): + with self.subTest(fault=fault),tempfile.TemporaryDirectory() as tmp: + request,node,files,decoded=self.fixture(tmp);p=files[0];inode=p.stat().st_ino + original=node['command'] + def command(*args,**kwargs): + value=original(*args,**kwargs) + if fault=='decoder':value.stdout=json.dumps({'bucket':'foreign','key':backup.KEY,'physicalBitrotVerified':False}) + if fault=='inode':p.unlink();p.write_bytes(b'replaced after held-file decode') + return value + node['command']=command + def mount_id(fd):return os.fstat(fd).st_dev+(1 if fault=='mount' and os.fstat(fd).st_ino==inode else 0) + with patch.object(heal_fault,'mount_id',side_effect=mount_id),self.assertRaises(ValueError):backup.run(request,node) + + def test_real_remote_adapter_embeds_backend_and_existing_descriptor_reader(self): + def execute(*args,**kwargs): + source=kwargs['input'].rsplit('\nmain(',1)[0];scope={'__name__':'__mpu_backups_test__'} + exec(compile(source,'g05-backups-node','exec'),scope) + self.assertTrue(callable(scope['mpu_race_backups_backend'].run)) + self.assertTrue(callable(scope['fault'].parent)) + return SimpleNamespace(returncode=0,stdout='{}',stderr='') + with tempfile.TemporaryDirectory() as tmp,patch('scripts.validation_heal.healthy.subprocess.run',side_effect=execute): + Fleet(['vm000'],Path(tmp),'gh-12345-1-h1').remote('vm000','mpu-race-backups') diff --git a/test_validation_heal_mpu_race_schedule.py b/test_validation_heal_mpu_race_schedule.py new file mode 100644 index 00000000..67df9c3f --- /dev/null +++ b/test_validation_heal_mpu_race_schedule.py @@ -0,0 +1,66 @@ +"""Original G05 operation order and halt boundaries through the migrated model.""" +import copy +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import test_validation_heal_mpu_race_fixture as model_fixtures +from scripts.validation_heal import mpu_race_fixture as model, mpu_race_schedule as schedule + + +class MultipartScheduleTests(unittest.TestCase): + def test_each_branch_prepares_the_original_parts_before_foreground_and_keeps_both_final_versions(self): + for branch in schedule.SELECTED: + with self.subTest(branch=branch),tempfile.TemporaryDirectory() as tmp,patch.object(model,'PART_SIZES',(12,13,14)): + recorder=model_fixtures.Recorder(Path(tmp)/'seed');recorder.output.mkdir() + fixture=model.MpuRaceFixture(recorder,'rv-g05-owned');prepared=schedule.prepare(fixture,branch) + self.assertEqual([len(prepared['manifest']['items'][r]['parts']) for r in model.ROLES], + [3,2 if branch=='upload-continue' else 3,3]) + fixture.verify_s3(recorder) + # Preparation and foreground use separate actual request sequences. + recorder.sequence=0;recorder.output=Path(tmp)/'foreground';recorder.output.mkdir();recorder.calls=[] + seen=[] + def checkpoint(label,manifest): + seen.append(label) + self.assertEqual(manifest,fixture.manifest()) + return {'checkpointLabel':label,'readOnlyMock':True} + result=schedule.execute(fixture,branch,prepared,checkpoint) + wanted=(['upload_part','complete_multipart_upload','abort_multipart_upload'] if branch=='upload-continue' + else ['complete_multipart_upload','abort_multipart_upload'] if branch=='complete' + else ['abort_multipart_upload','complete_multipart_upload']) + self.assertEqual([c[0] for c in recorder.calls],wanted) + self.assertEqual(seen,['continued','completed','aborted'] if branch=='upload-continue' + else ['completed','aborted'] if branch=='complete' else ['aborted','completed']) + self.assertEqual(result['selectedAckSequence'],1) + self.assertEqual(result['selectedOperation'],wanted[0]) + self.assertFalse(result['ioCapRequested'] or result['physicalRepairProved'] or result['publicOverlapProved'] + or result['canonicalCasePassClaimed']) + proof=fixture.verify_s3(recorder) + self.assertEqual((proof['committedVersions'],proof['pendingUploads']),(2,0)) + + def test_missing_or_unknown_mutation_checkpoint_stops_every_later_companion(self): + for branch in schedule.SELECTED: + for failure in ('checkpoint','sdk'): + with self.subTest(branch=branch,failure=failure),tempfile.TemporaryDirectory() as tmp,patch.object(model,'PART_SIZES',(12,13,14)): + recorder=model_fixtures.Recorder(tmp);fixture=model.MpuRaceFixture(recorder,'rv-g05-owned') + prepared=schedule.prepare(fixture,branch);count=len(recorder.calls) + if failure=='sdk':recorder.fail=schedule.SELECTED[branch] + def failed_checkpoint(*args):raise ValueError('original physical witness unavailable') + with self.assertRaises(ValueError):schedule.execute(fixture,branch,prepared,failed_checkpoint) + self.assertEqual(len(recorder.calls),count+1) + self.assertIsNotNone(fixture.poisoned) + with self.assertRaises(ValueError):fixture.abort_b() + self.assertEqual(len(recorder.calls),count+1) + + def test_foreign_preparation_branch_or_changed_model_is_refused_before_foreground_sdk(self): + for fault in ('branch','manifest','operation','callback'): + with self.subTest(fault=fault),tempfile.TemporaryDirectory() as tmp,patch.object(model,'PART_SIZES',(12,13,14)): + recorder=model_fixtures.Recorder(tmp);fixture=model.MpuRaceFixture(recorder,'rv-g05-owned') + prepared=schedule.prepare(fixture,'complete');count=len(recorder.calls);callback=lambda *args:{} + if fault=='branch':prepared['branch']='abort' + elif fault=='manifest':prepared=copy.deepcopy(prepared);prepared['manifest']['items']['a']['uploadId']='foreign' + elif fault=='operation':prepared['selectedOperation']='abort_multipart_upload' + else:callback=None + with self.assertRaises(ValueError):schedule.execute(fixture,'complete',prepared,callback) + self.assertEqual(len(recorder.calls),count) diff --git a/test_validation_heal_race.py b/test_validation_heal_race.py index 57b8d89b..b42173b2 100644 --- a/test_validation_heal_race.py +++ b/test_validation_heal_race.py @@ -33,7 +33,8 @@ def test_g02_plan_creates_three_acknowledged_versions(self): for value in ("v0", "v1", "v2")] plan = race.operation_plan("rv-heal-0123456789abcdef", "read-all-versions-during-heal", expected) self.assertEqual(len(plan), 3) - self.assertEqual({step["readVersionId"] for step in plan}, {"v0"}) + self.assertEqual([step['referenceKey'] for step in plan], [f'objects/{i:04d}.bin' for i in range(3)]) + self.assertTrue(all('readVersionId' not in step for step in plan)) self.assertEqual([step["params"]["Metadata"]["nonce"] for step in plan], ["0", "1", "2"]) def test_g06_keeps_copy_metadata_and_tag_operations_distinct(self): diff --git a/test_validation_heal_race_retention.py b/test_validation_heal_race_retention.py index 35e91ba4..fcad9f33 100644 --- a/test_validation_heal_race_retention.py +++ b/test_validation_heal_race_retention.py @@ -198,6 +198,49 @@ def test_recovery_routes_isolated_scene_before_absent_namespace_cleanup(self): summary = json.loads((evidence / 'recovery-summary.json').read_text()) self.assertIn('ISOLATED', json.dumps(summary)) + def test_version_append_failure_keeps_unknown_put_and_complete_historical_journal_for_isolation(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, folder = self.archive(directory) + original = next(root.glob('*/run/*')) + case = 'heal.race-versioned-put' + run = original.with_name(case); original.rename(run) + old_case = run / 'cases' / (controller.CASE + '.json') + value = json.loads(old_case.read_text()); value['case_id'] = case + old_case.rename(old_case.with_name(case + '.json')) + (run / 'cases' / (case + '.json')).write_text(json.dumps(value)) + evidence = run / 'evidence' / controller.CASE + evidence.rename(evidence.with_name(case)); evidence = evidence.with_name(case) + summary = json.loads((evidence / 'healthy-summary.json').read_text()); summary['caseId'] = case + summary['results'][0].update(caseId=case, branch='read-all-versions-during-heal') + (evidence / 'healthy-summary.json').write_text(json.dumps(summary)) + branch = evidence / 'H2/overwrite-vs-deep'; branch.rename(branch.with_name('read-all-versions-during-heal')) + branch = branch.with_name('read-all-versions-during-heal') + reads = branch / 'historical-read-ACKs'; reads.mkdir() + path = reads / '000001-request.json' + path.write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'get_object'})) + plan = recovery.recovery_plan(root, 12345, candidate)[0] + self.assertEqual(plan['kind'], 'unresolved-g02-race') + before = plan['ackFilesSha256'] + path.write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'get_object', 'identity': {'uncertain': True}})) + self.assertNotEqual(recovery.recovery_plan(root, 12345, candidate)[0]['ackFilesSha256'], before) + path.write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'abort_multipart_upload'})) + with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) + + def test_g02_original_quarantine_contents_are_hashed_without_reading_credentials(self): + with tempfile.TemporaryDirectory() as directory: + helper, state, roots, node, request, _ = self.scene(directory) + for root in roots: + for path in (root / 'quarantine').glob('G01-*'): + path.rename(path.with_name(path.name.replace('G01-', 'G02-', 1))) + opened, original_open = [], os.open + def tracked(path, *args, **kwargs): + opened.append(str(path)); return original_open(path, *args, **kwargs) + with helper.root_stats(), patch.object(os, 'open', side_effect=tracked): + result = retained.run(request, node) + self.assertEqual(sum(path.endswith('G02-object-0256.original') for path in opened), 4) + self.assertFalse(any(path.endswith('service.env') or path.endswith('/credential') for path in opened)) + self.assertFalse(result['canonicalPassClaimed'] or result['backupRestorationUsed']) + if __name__ == '__main__': unittest.main() From caff7c2f93cca0c90f4c1c88d84ee42032687930 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 13:37:34 +0800 Subject: [PATCH 267/343] fix(validation): retain uncertain default-owner race scenes --- scripts/validation_heal/race_retention.py | 10 ++-- .../validation_heal/recover_environment.py | 5 +- scripts/validation_heal/retained_scene.py | 2 +- test_validation_heal_race_retention.py | 47 +++++++++++++++++++ 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/scripts/validation_heal/race_retention.py b/scripts/validation_heal/race_retention.py index 3a8c3c6f..a5f93cf3 100644 --- a/scripts/validation_heal/race_retention.py +++ b/scripts/validation_heal/race_retention.py @@ -1,4 +1,4 @@ -"""Isolate stopped G01/G02 scenes without replaying writes or restoring old shards.""" +"""Isolate stopped ACK-bearing races without replaying writes or restoring old shards.""" import hashlib import json @@ -14,7 +14,8 @@ CASE = 'heal.race-unversioned-overwrite' CASES = {CASE: ('unresolved-g01-race', {'overwrite-vs-deep', 'overwrite-vs-read-repair'}), - 'heal.race-versioned-put': ('unresolved-g02-race', {'new-version-vs-historical-heal', 'read-all-versions-during-heal'})} + 'heal.race-versioned-put': ('unresolved-g02-race', {'new-version-vs-historical-heal', 'read-all-versions-during-heal'}), + 'heal.admin-nolock-safety': ('unresolved-a09-race', {'overwrite-nolock', 'delete-nolock'})} def digest(value): @@ -24,7 +25,7 @@ def digest(value): def validate(plan): require(set(plan) == {'kind', 'runId', 'nodes', 'bucket', 'owners', 'businessSceneSha256', 'candidateCommit', 'harnessCommit', 'binarySha256', 'ackFilesSha256'} - and plan['kind'] in {'unresolved-g01-race', 'unresolved-g02-race'} + and plan['kind'] in {'unresolved-g01-race', 'unresolved-g02-race', 'unresolved-a09-race'} and re.fullmatch(r'gh-[1-9][0-9]*-[1-9][0-9]*-h[12]', plan['runId']) and len(plan['nodes']) == len(set(plan['nodes'])) == (1 if plan['runId'].endswith('-h1') else 4) and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in plan['nodes']) @@ -64,7 +65,7 @@ def archived_plan(artifact, failed_run, candidate): require(summary.get('caseId') == case and summary.get('cleanup') == 'FAIL' and len(retained) == 1 and retained[0].get('verdict') == 'FAIL' and retained[0].get('topology') in {'H1', 'H2'} and retained[0].get('branch') in branches - and all(row.get('caseId') == case for row in rows), + and all(row.get('caseId', case if kind == 'unresolved-a09-race' else None) == case for row in rows), 'G01/G02 isolation requires one unmixed retained public branch') row = retained[0] folder = evidence / row['topology'] / row['branch'] @@ -103,6 +104,7 @@ def one(label): # effect or sends another request, including for explicit refusals. operation = item.get('operation') if path.name.endswith('-request.json') else (item.get('observation') or {}).get('operation') allowed = ({'get_object'} if name == 'historical-read-ACKs' else + {'create_bucket', 'put_bucket_versioning', 'put_object', 'delete_object'} if kind == 'unresolved-a09-race' else {'create_bucket', 'put_bucket_versioning', 'put_object'} if kind == 'unresolved-g02-race' else {'create_bucket', 'put_object'}) require(operation in allowed, 'G01/G02 isolation cannot omit a multipart mutation') diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index fed5c045..3af8aa74 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -150,7 +150,8 @@ def network_recovery_plan(artifact, failed_run, candidate): def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") if (list(artifact.glob("*/run/heal.race-unversioned-overwrite/cases/*.json")) - or list(artifact.glob("*/run/heal.race-versioned-put/cases/*.json"))): + or list(artifact.glob("*/run/heal.race-versioned-put/cases/*.json")) + or list(artifact.glob("*/run/heal.admin-nolock-safety/cases/*.json"))): return race_retention_plan(artifact, failed_run, candidate) if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/cases/*.json")): if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/evidence/heal.outcome-counters-and-error-truth/DB/H1/unknown-negative/*-one-way-legacy-write-phase.json")): @@ -648,7 +649,7 @@ def main(): if plan.get("buckets"): fleet.snapshot_buckets = plan["buckets"] fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) - if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race"}: + if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race", "unresolved-a09-race"}: complete.append({"failedRunId": args.failed_run_id, **isolate_race_scene(fleet, plan)}) continue if plan.get("kind") == "unresolved-network-load": diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py index 3bc974c4..32132862 100644 --- a/scripts/validation_heal/retained_scene.py +++ b/scripts/validation_heal/retained_scene.py @@ -138,7 +138,7 @@ def visit(path): and re.fullmatch(r"J02-[0-9a-f]{32}\.(original|intent\.json|applied\.json|failed-[0-9a-f]{64})", parts[1])) race_backup = (not state and owner.get("runtimeProfile") == "manual" and len(parts) == 2 and parts[0] == "quarantine" - and re.fullmatch(r"G0[12]-[a-z0-9-]{1,48}\.(original|intent\.json|applied\.json)", parts[1])) + and re.fullmatch(r"(?:G0[12]|A09)-[a-z0-9-]{1,48}\.(original|intent\.json|applied\.json)", parts[1])) if (state and relative == "rustfs") or fixture or mrf or backup or opaque_backup or network_mpu or network_backup or race_backup: fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) try: diff --git a/test_validation_heal_race_retention.py b/test_validation_heal_race_retention.py index fcad9f33..9ab5d62d 100644 --- a/test_validation_heal_race_retention.py +++ b/test_validation_heal_race_retention.py @@ -226,6 +226,38 @@ def test_version_append_failure_keeps_unknown_put_and_complete_historical_journa path.write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'abort_multipart_upload'})) with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) + def test_a09_source_summary_without_public_case_field_keeps_uncertain_delete_for_isolation(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, folder = self.archive(directory) + original = next(root.glob('*/run/*')) + case = 'heal.admin-nolock-safety' + run = original.with_name(case); original.rename(run) + old_case = run / 'cases' / (controller.CASE + '.json') + value = json.loads(old_case.read_text()); value['case_id'] = case + old_case.rename(old_case.with_name(case + '.json')) + (run / 'cases' / (case + '.json')).write_text(json.dumps(value)) + evidence = run / 'evidence' / controller.CASE + evidence.rename(evidence.with_name(case)); evidence = evidence.with_name(case) + summary = json.loads((evidence / 'healthy-summary.json').read_text()); summary['caseId'] = case + summary['results'][0].update(branch='delete-nolock'); del summary['results'][0]['caseId'] + (evidence / 'healthy-summary.json').write_text(json.dumps(summary)) + branch = evidence / 'H2/overwrite-vs-deep'; branch.rename(branch.with_name('delete-nolock')) + branch = branch.with_name('delete-nolock') + path = branch / 'foreground-ACKs/000001-request.json' + path.write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'delete_object'})) + plan = recovery.recovery_plan(root, 12345, candidate)[0] + self.assertEqual(plan['kind'], 'unresolved-a09-race') + before = plan['ackFilesSha256'] + path.write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'delete_object', 'identity': {'uncertain': True}})) + self.assertNotEqual(recovery.recovery_plan(root, 12345, candidate)[0]['ackFilesSha256'], before) + summary['results'][0]['caseId'] = 'foreign-case' + (evidence / 'healthy-summary.json').write_text(json.dumps(summary)) + with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) + del summary['results'][0]['caseId'] + (evidence / 'healthy-summary.json').write_text(json.dumps(summary)) + path.write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'abort_multipart_upload'})) + with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) + def test_g02_original_quarantine_contents_are_hashed_without_reading_credentials(self): with tempfile.TemporaryDirectory() as directory: helper, state, roots, node, request, _ = self.scene(directory) @@ -241,6 +273,21 @@ def tracked(path, *args, **kwargs): self.assertFalse(any(path.endswith('service.env') or path.endswith('/credential') for path in opened)) self.assertFalse(result['canonicalPassClaimed'] or result['backupRestorationUsed']) + def test_a09_original_quarantine_contents_are_hashed_without_reading_credentials(self): + with tempfile.TemporaryDirectory() as directory: + helper, state, roots, node, request, _ = self.scene(directory) + for root in roots: + for path in (root / 'quarantine').glob('G01-*'): + path.rename(path.with_name(path.name.replace('G01-', 'A09-', 1))) + opened, original_open = [], os.open + def tracked(path, *args, **kwargs): + opened.append(str(path)); return original_open(path, *args, **kwargs) + with helper.root_stats(), patch.object(os, 'open', side_effect=tracked): + result = retained.run(request, node) + self.assertEqual(sum(path.endswith('A09-object-0256.original') for path in opened), 4) + self.assertFalse(any(path.endswith('service.env') or path.endswith('/credential') for path in opened)) + self.assertFalse(result['canonicalPassClaimed'] or result['backupRestorationUsed']) + if __name__ == '__main__': unittest.main() From 1ab2e437819ee029887dfcea51b0e65c01dd9f95 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 14:12:30 +0800 Subject: [PATCH 268/343] fix(validation): add multipart proof and preserve copy race scenes --- scripts/validation_heal/mpu_race_committed.py | 155 ++++++++++++++++++ scripts/validation_heal/race_retention.py | 6 +- .../validation_heal/recover_environment.py | 5 +- scripts/validation_heal/retained_scene.py | 2 +- test_validation_heal_mpu_race_committed.py | 127 ++++++++++++++ test_validation_heal_race_retention.py | 50 ++++++ 6 files changed, 340 insertions(+), 5 deletions(-) create mode 100644 scripts/validation_heal/mpu_race_committed.py create mode 100644 test_validation_heal_mpu_race_committed.py diff --git a/scripts/validation_heal/mpu_race_committed.py b/scripts/validation_heal/mpu_race_committed.py new file mode 100644 index 00000000..368b25bb --- /dev/null +++ b/scripts/validation_heal/mpu_race_committed.py @@ -0,0 +1,155 @@ +"""Frozen G05 committed multipart geometry and bytes; caller owns staging/S3 proof.""" +import copy +import hashlib +import re +import uuid + +try: + from . import mpu_race_fixture as model + from .heal_probe import require + from .shards import files, map_target + from .mpu_race_backups import verify as verify_backups +except ImportError: + import mpu_race_fixture as model + from heal_probe import require + from shards import files, map_target + from mpu_race_backups import verify as verify_backups + + +def entries(manifest): + require(manifest.get('key') == model.KEY and re.fullmatch(r'rv-heal-[0-9a-f]{16}', manifest.get('bucket', '')) + and manifest.get('poisoned') is None and set(manifest.get('items', {})) == set(model.ROLES), + 'G05 committed proof requires the original known same-key model') + completed = [manifest['items'][role] for role in model.ROLES if manifest['items'][role]['state'] == 'complete'] + require([item['role'] for item in completed] in (['committed'], ['committed', 'a']), + 'G05 committed population differs from the original and A Complete ACKs') + result, seen = [], set() + for index, item in enumerate(completed): + version = item['versionId'] + require(isinstance(version, str) and str(uuid.UUID(version)) == version and uuid.UUID(version).int != 0 + and version not in seen and len(item['parts']) == 3, 'G05 Complete lacks one distinct real version') + seen.add(version); digest = hashlib.sha256() + for number, part in enumerate(item['parts'], 1): + body = model.payload(manifest['bucket'], item['role'], number) + require(type(part.get('number')) is int and part['number'] == number + and type(part.get('size')) is int and part['size'] == len(body) + and part.get('etag') == model.etag(body) + and part.get('sha256') == hashlib.sha256(body).hexdigest(), + 'G05 committed part differs from its exact original role payload/ACK') + digest.update(body) + require(item['etag'] == model.multipart_etag(item['parts']) + and type(item['payload'].get('size')) is int + and item['payload'] == {'size': sum(model.PART_SIZES), 'sha256': digest.hexdigest()}, + 'G05 Complete payload or ordered multipart ETag differs') + result.append({'key': model.KEY, 'versionId': version, 'isLatest': index == len(completed) - 1, + 'deleteMarker': False, 'payload': copy.deepcopy(item['payload']), + 'multipartParts': [{'number': part['number'], 'size': part['size']} for part in item['parts']]}) + return result + + +def mapped(snapshot, bucket, expected, *, backup_directories=()): + """Bind all three parts on every original coding index; reject unknown leaves.""" + require(len(snapshot) in (1, 4), 'G05 committed topology needs one or four original hosts') + prefix = f'{bucket}/{model.KEY}/' + scope = {node: {**value, 'files': [row for row in value['files'] if row['path'].startswith(prefix)]} + for node, value in snapshot.items()} + require(all(type(row.get('disk')) is int and row['disk'] in range(4) + and type(row.get('size')) is int and row['size'] >= 0 + and isinstance(row.get('sha256'), str) and re.fullmatch(r'[0-9a-f]{64}', row['sha256']) + for value in scope.values() for row in value['files']), + 'G05 committed file identity or digest/size is invalid') + indexed = files(scope); cohorts = {}; used = set(); auxiliary = {} + expected_versions = sorted((row['versionId'], row['isLatest'], False) for row in expected) + for node, value in scope.items(): + require(len(value['formats']) == 4 and all(type(row.get('disk')) is int for row in value['formats']) + and {row['disk'] for row in value['formats']} == set(range(4)), 'G05 committed scope lacks four distinct original drives') + for drive in value['formats']: + identity = (node, drive['disk'], prefix + 'xl.meta') + metadata = indexed.get(identity, {}).get('metadata', {}) + versions = metadata.get('versions', []) + require(metadata.get('bucket') == bucket and metadata.get('key') == model.KEY + and all(type(v.get('isLatest')) is bool and v.get('deleted') is False for v in versions) + and sorted((v['versionId'], v['isLatest'], v['deleted']) for v in versions) == expected_versions, + 'G05 primary version/latest population differs from committed ACKs') + used.add(identity) + directories = set() + for entry in expected: + for number in (1, 2, 3): + target = map_target(scope, bucket, model.KEY, 'data', number, entry['versionId'], latest_required=False) + logical = target['logical'] + require(type(logical['size']) is int and logical['size'] == entry['payload']['size'] and uuid.UUID(target['dataDir']).int != 0 + and all(type(p['number']) is int and type(p['size']) is int and type(p['actualSize']) is int for p in logical['parts']) + and [{'number': p['number'], 'size': p['size']} for p in logical['parts']] == entry['multipartParts'] + and type(logical['erasure']['block_size']) is int and logical['erasure']['block_size'] > 0 + and all(type(index) is int for index in logical['erasure']['distribution']) + and all(type(part['index']) is int for part in target['cohort']) + and logical['erasure']['data_blocks'] == (2 if len(snapshot) == 1 else 12) + and logical['erasure']['parity_blocks'] == (2 if len(snapshot) == 1 else 4), + 'G05 committed total/part sizes or original EC geometry differs') + cohorts[entry['versionId'], number] = target + for part in target['cohort']: + identity = (part['node'], part['disk'], part['relativePath']); used.add(identity) + sidecar = (part['node'], part['disk'], part['relativePath'] + '.meta') + if sidecar in indexed: + require(indexed[sidecar]['size'] > 0, 'G05 upload auxiliary sidecar is empty') + used.add(sidecar); auxiliary[sidecar] = indexed[sidecar] + generation = {cohorts[entry['versionId'], n]['dataDir'] for n in (1, 2, 3)} + require(len(generation) == 1 and not directories.intersection(generation), + 'G05 parts or distinct versions share the wrong committed generation') + directories.update(generation) + for identity, row in indexed.items(): + if identity in used: + continue + suffix = row['path'][len(prefix):] + require(any(suffix == directory + '/xl.meta.bkp' for directory in backup_directories), + 'G05 committed key contains an unknown or unacknowledged leaf') + return cohorts, auxiliary + + +def transition(before, after, original, final, backup_observations, binary_sha, inspector_sha): + """Port physical_final; staging continuity and full S3 still close new payloads.""" + require(set(before) == set(after) and len(before) in (1, 4) + and re.fullmatch(r'[0-9a-f]{64}', binary_sha) + and original['bucket'] == final['bucket'] and original['key'] == final['key'] == model.KEY, + 'G05 committed transition changed original hosts, candidate or bucket/key') + for node in before: + require(before[node]['formats'] == after[node]['formats'] + and before[node].get('quiescent') is True and after[node].get('quiescent') is True + and type(before[node].get('pid')) is int and type(after[node].get('pid')) is int + and before[node].get('pid') == after[node].get('pid') == 0 + and before[node].get('binarySha256') == after[node].get('binarySha256') == binary_sha + and before[node].get('hostname') == after[node].get('hostname'), + 'G05 committed proof lost stopped process, original formats or candidate') + previous, current = entries(original), entries(final) + require(len(previous) == 1 and len(current) == 2 and previous[0]['versionId'] == current[0]['versionId'] + and original['items']['committed'] == final['items']['committed'] + and original['items']['a']['state'] == original['items']['b']['state'] == 'pending' + and len(original['items']['a']['parts']) in (2, 3) and len(original['items']['b']['parts']) == 3 + and original['items']['a']['parts'] == final['items']['a']['parts'][:len(original['items']['a']['parts'])] + and original['items']['a']['metadata'] == final['items']['a']['metadata'] + and {k: v for k, v in original['items']['b'].items() if k != 'state'} + == {k: v for k, v in final['items']['b'].items() if k != 'state'} + and final['items']['a']['state'] == 'complete' and final['items']['b']['state'] == 'aborted' + and all(original['items'][role]['uploadId'] == final['items'][role]['uploadId'] for role in model.ROLES), + 'G05 Complete replaced the old version or original sibling upload identities') + old, old_aux = mapped(before, original['bucket'], previous) + old_version = previous[0]['versionId']; new_version = current[1]['versionId'] + old_dir = old[old_version, 1]['dataDir'] + new, new_aux = mapped(after, final['bucket'], current, backup_directories=[old_dir]) + for number in (1, 2, 3): + left, right = old[old_version, number], new[old_version, number] + require({k: v for k, v in left['logical'].items() if k != 'isLatest'} + == {k: v for k, v in right['logical'].items() if k != 'isLatest'} + and {p['index']: (p['sha256'], p['size']) for p in left['cohort']} + == {p['index']: (p['sha256'], p['size']) for p in right['cohort']}, + 'G05 Complete changed an original committed multipart descriptor or shard') + require(new[new_version, 1]['dataDir'] != old_dir, 'G05 A Complete reused the old generation') + for identity, row in old_aux.items(): + require(identity not in new_aux or new_aux[identity] == row, 'G05 retained upload sidecar bytes changed') + backups = verify_backups(before, after, backup_observations, list(before), original['bucket'], + old_version, new_version, binary_sha, inspector_sha) + return new, {'oldVersionRetained': True, 'newVersionExactlyOnce': True, + 'allPartsCodingIndicesVerified': True, 'backupProof': backups, + 'reclaimedOldUploadSidecars': [list(identity) for identity in old_aux if identity not in new_aux], + 'uploadSidecarsAreCommittedAuthority': False, 'stagingContinuityStillRequired': True, + 'fullPayloadS3StillRequired': True, 'physicalRepairProved': False, 'canonicalCasePassClaimed': False} diff --git a/scripts/validation_heal/race_retention.py b/scripts/validation_heal/race_retention.py index a5f93cf3..8b9591ad 100644 --- a/scripts/validation_heal/race_retention.py +++ b/scripts/validation_heal/race_retention.py @@ -15,7 +15,8 @@ CASE = 'heal.race-unversioned-overwrite' CASES = {CASE: ('unresolved-g01-race', {'overwrite-vs-deep', 'overwrite-vs-read-repair'}), 'heal.race-versioned-put': ('unresolved-g02-race', {'new-version-vs-historical-heal', 'read-all-versions-during-heal'}), - 'heal.admin-nolock-safety': ('unresolved-a09-race', {'overwrite-nolock', 'delete-nolock'})} + 'heal.admin-nolock-safety': ('unresolved-a09-race', {'overwrite-nolock', 'delete-nolock'}), + 'heal.race-copy-and-metadata-update': ('unresolved-g06-race', {'copy-source', 'copy-destination', 'replace-metadata', 'update-tags'})} def digest(value): @@ -25,7 +26,7 @@ def digest(value): def validate(plan): require(set(plan) == {'kind', 'runId', 'nodes', 'bucket', 'owners', 'businessSceneSha256', 'candidateCommit', 'harnessCommit', 'binarySha256', 'ackFilesSha256'} - and plan['kind'] in {'unresolved-g01-race', 'unresolved-g02-race', 'unresolved-a09-race'} + and plan['kind'] in {'unresolved-g01-race', 'unresolved-g02-race', 'unresolved-a09-race', 'unresolved-g06-race'} and re.fullmatch(r'gh-[1-9][0-9]*-[1-9][0-9]*-h[12]', plan['runId']) and len(plan['nodes']) == len(set(plan['nodes'])) == (1 if plan['runId'].endswith('-h1') else 4) and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in plan['nodes']) @@ -104,6 +105,7 @@ def one(label): # effect or sends another request, including for explicit refusals. operation = item.get('operation') if path.name.endswith('-request.json') else (item.get('observation') or {}).get('operation') allowed = ({'get_object'} if name == 'historical-read-ACKs' else + {'create_bucket', 'put_object', 'copy_object', 'put_object_tagging'} if kind == 'unresolved-g06-race' else {'create_bucket', 'put_bucket_versioning', 'put_object', 'delete_object'} if kind == 'unresolved-a09-race' else {'create_bucket', 'put_bucket_versioning', 'put_object'} if kind == 'unresolved-g02-race' else {'create_bucket', 'put_object'}) diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index 3af8aa74..94821613 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -151,7 +151,8 @@ def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") if (list(artifact.glob("*/run/heal.race-unversioned-overwrite/cases/*.json")) or list(artifact.glob("*/run/heal.race-versioned-put/cases/*.json")) - or list(artifact.glob("*/run/heal.admin-nolock-safety/cases/*.json"))): + or list(artifact.glob("*/run/heal.admin-nolock-safety/cases/*.json")) + or list(artifact.glob("*/run/heal.race-copy-and-metadata-update/cases/*.json"))): return race_retention_plan(artifact, failed_run, candidate) if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/cases/*.json")): if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/evidence/heal.outcome-counters-and-error-truth/DB/H1/unknown-negative/*-one-way-legacy-write-phase.json")): @@ -649,7 +650,7 @@ def main(): if plan.get("buckets"): fleet.snapshot_buckets = plan["buckets"] fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) - if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race", "unresolved-a09-race"}: + if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race", "unresolved-a09-race", "unresolved-g06-race"}: complete.append({"failedRunId": args.failed_run_id, **isolate_race_scene(fleet, plan)}) continue if plan.get("kind") == "unresolved-network-load": diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py index 32132862..3f09eda7 100644 --- a/scripts/validation_heal/retained_scene.py +++ b/scripts/validation_heal/retained_scene.py @@ -138,7 +138,7 @@ def visit(path): and re.fullmatch(r"J02-[0-9a-f]{32}\.(original|intent\.json|applied\.json|failed-[0-9a-f]{64})", parts[1])) race_backup = (not state and owner.get("runtimeProfile") == "manual" and len(parts) == 2 and parts[0] == "quarantine" - and re.fullmatch(r"(?:G0[12]|A09)-[a-z0-9-]{1,48}\.(original|intent\.json|applied\.json)", parts[1])) + and re.fullmatch(r"(?:G0[126]|A09)-[a-z0-9-]{1,48}\.(original|intent\.json|applied\.json)", parts[1])) if (state and relative == "rustfs") or fixture or mrf or backup or opaque_backup or network_mpu or network_backup or race_backup: fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) try: diff --git a/test_validation_heal_mpu_race_committed.py b/test_validation_heal_mpu_race_committed.py new file mode 100644 index 00000000..020a3331 --- /dev/null +++ b/test_validation_heal_mpu_race_committed.py @@ -0,0 +1,127 @@ +"""Decoded G05 business generations across H1/H2; no staging/live PASS claim.""" +import copy +import hashlib +import tempfile +import unittest +from unittest.mock import patch +import uuid + +from scripts.validation_heal import mpu_race_committed as committed, mpu_race_fixture as model +import test_validation_heal_mpu_race_fixture as model_fixtures + + +class MultipartCommittedTests(unittest.TestCase): + bucket = 'rv-heal-0123456789abcdef' + + def scene(self, directory, topology): + recorder = model_fixtures.Recorder(directory) + fixture = model.MpuRaceFixture(recorder, self.bucket); fixture.seed() + before_model = fixture.manifest() + fixture.upload('a', 3); fixture.complete('a'); fixture.abort_b() + after_model = fixture.manifest() + nodes = ['vm000'] if topology == 'H1' else [f'vm00{i}' for i in range(4)] + width = 4 * len(nodes); data = 2 if topology == 'H1' else 12 + ids = [str(uuid.UUID(int=1000 + i)) for i in range(width)]; distribution = list(range(width, 0, -1)) + def snapshot(manifest): + expected = committed.entries(manifest); result = {} + for node_number, node in enumerate(nodes): + value = result[node] = {'hostname': 'host-' + node, 'binarySha256': 'a' * 64, + 'quiescent': True, 'pid': 0, 'formats': [], 'files': []} + for disk in range(4): + slot = node_number * 4 + disk; index = distribution[slot] + value['formats'].append({'disk': disk, 'value': {'format': 'xl', 'id': 'pool', + 'xl': {'sets': [ids], 'this': ids[slot]}}}) + versions = [] + for entry in expected: + role = next(r for r in ('committed', 'a') if manifest['items'][r].get('versionId') == entry['versionId']) + directory_id = str(uuid.UUID(int=100 if role == 'committed' else 200)) + versions.append({'versionId': entry['versionId'], 'isLatest': entry['isLatest'], 'deleted': False, + 'dataDir': directory_id, 'size': entry['payload']['size'], 'inline': None, 'usesLegacyChecksum': True, + 'role': 'data' if index <= data else 'parity', + 'parts': [{'number': p['number'], 'size': p['size'], 'actualSize': p['size']} for p in entry['multipartParts']], + 'erasure': {'index': index, 'data_blocks': data, 'parity_blocks': width - data, + 'block_size': 1048576, 'distribution': distribution, 'algorithm': 'rs-vandermonde'}}) + for part in entry['multipartParts']: + path = f'{self.bucket}/{model.KEY}/{directory_id}/part.{part["number"]}' + value['files'].append({'disk': disk, 'path': path, 'size': (part['size'] + data - 1) // data + 32, + 'sha256': hashlib.sha256(f'{role}/{index}/{part["number"]}'.encode()).hexdigest()}) + value['files'].append({'disk': disk, 'path': path + '.meta', 'size': 29, + 'sha256': hashlib.sha256(f'upload/{role}/{index}/{part["number"]}'.encode()).hexdigest()}) + value['files'].append({'disk': disk, 'path': f'{self.bucket}/{model.KEY}/xl.meta', 'size': 1000, + 'sha256': 'd' * 64, 'metadata': {'bucket': self.bucket, 'key': model.KEY, + 'physicalBitrotVerified': False, 'versions': versions}}) + return result + before, after = snapshot(before_model), snapshot(after_model) + evidence = {node: {'hostname': 'host-' + node, 'binarySha256': 'a' * 64, 'inspectorSha256': 'b' * 64, + 'quiescent': True, 'readOnly': True, 'sameHeldFileHashedAndDecoded': True, + 'rawContentsExported': False, 'liveAuthorityClaimed': False, 'rows': []} for node in nodes} + return before, after, before_model, after_model, evidence + + def test_three_part_full_cohorts_and_old_version_survive_complete_in_both_topologies(self): + for topology in ('H1', 'H2'): + with self.subTest(topology=topology), tempfile.TemporaryDirectory() as directory, patch.object(model, 'PART_SIZES', (101, 111, 131)): + before, after, original, final, evidence = self.scene(directory, topology) + cohorts, proof = committed.transition(before, after, original, final, evidence, 'a' * 64, 'b' * 64) + self.assertEqual(len(cohorts), 6) + self.assertTrue(all(len(t['cohort']) == 4 * len(before) for t in cohorts.values())) + self.assertTrue(proof['oldVersionRetained'] and proof['newVersionExactlyOnce'] and proof['allPartsCodingIndicesVerified']) + self.assertTrue(proof['stagingContinuityStillRequired'] and proof['fullPayloadS3StillRequired']) + self.assertFalse(proof['canonicalCasePassClaimed'] or proof['physicalRepairProved']) + + def test_reclaimed_upload_sidecar_is_reported_and_never_used_as_committed_authority(self): + with tempfile.TemporaryDirectory() as directory, patch.object(model, 'PART_SIZES', (101, 111, 131)): + before, after, original, final, evidence = self.scene(directory, 'H1') + old_path = f'{self.bucket}/{model.KEY}/{uuid.UUID(int=100)}/part.2.meta' + after['vm000']['files'] = [r for r in after['vm000']['files'] if not (r['disk'] == 2 and r['path'] == old_path)] + _, proof = committed.transition(before, after, original, final, evidence, 'a' * 64, 'b' * 64) + self.assertEqual(proof['reclaimedOldUploadSidecars'], [['vm000', 2, old_path]]) + self.assertFalse(proof['uploadSidecarsAreCommittedAuthority']) + + def test_backup_prefix_decoder_is_required_by_the_actual_business_transition(self): + for count in (0, 1): + with self.subTest(prefix=count), tempfile.TemporaryDirectory() as directory, patch.object(model, 'PART_SIZES', (101, 111, 131)): + before, after, original, final, evidence = self.scene(directory, 'H1') + for disk in range(4): + metadata = next(r['metadata'] for r in (before if count == 0 else after)['vm000']['files'] + if r['disk'] == disk and r['path'].endswith('/xl.meta')) + row = {'disk': disk, 'path': f'{self.bucket}/{model.KEY}/{uuid.UUID(int=100)}/xl.meta.bkp', + 'size': 101 + count, 'sha256': hashlib.sha256(f'backup/{disk}/{count}'.encode()).hexdigest()} + after['vm000']['files'].append(copy.deepcopy(row)) + evidence['vm000']['rows'].append({**row, 'metadata': copy.deepcopy(metadata)}) + _, proof = committed.transition(before, after, original, final, evidence, 'a' * 64, 'b' * 64) + self.assertEqual({p['acknowledgedCompletePrefix'] for p in proof['backupProof']['proofs']}, {count}) + evidence['vm000']['rows'][0]['metadata']['versions'][0]['parts'][0]['size'] += 1 + with self.assertRaises(ValueError): committed.transition(before, after, original, final, evidence, 'a' * 64, 'b' * 64) + + def test_drift_missing_parts_wrong_geometry_and_unknown_leaves_are_rejected(self): + for topology in ('H1', 'H2'): + for fault in ('old-shard', 'missing-new-part', 'old-sidecar', 'generation', 'extra-version', 'logical-size', + 'unknown-leaf', 'format', 'live', 'bool-pid', 'bool-disk', 'sibling-id', 'model-payload', 'multipart-etag', + 'a-before-state', 'a-metadata', 'b-parts'): + with self.subTest(topology=topology, fault=fault), tempfile.TemporaryDirectory() as directory, patch.object(model, 'PART_SIZES', (101, 111, 131)): + before, after, original, final, evidence = self.scene(directory, topology) + rows = after['vm000']['files']; prefix = f'{self.bucket}/{model.KEY}/' + old_part = next(r for r in rows if r['path'] == prefix + str(uuid.UUID(int=100)) + '/part.2') + meta = next(r['metadata'] for r in rows if r['path'] == prefix + 'xl.meta') + if fault == 'old-shard': old_part['sha256'] = 'e' * 64 + elif fault == 'missing-new-part': rows.remove(next(r for r in rows if r['path'] == prefix + str(uuid.UUID(int=200)) + '/part.3')) + elif fault == 'old-sidecar': next(r for r in rows if r['path'] == old_part['path'] + '.meta')['sha256'] = 'e' * 64 + elif fault == 'generation': meta['versions'][-1]['dataDir'] = str(uuid.UUID(int=100)) + elif fault == 'extra-version': meta['versions'].append({**copy.deepcopy(meta['versions'][0]), 'versionId': str(uuid.UUID(int=999))}) + elif fault == 'logical-size': meta['versions'][-1]['parts'][1]['size'] += 1 + elif fault == 'unknown-leaf': rows.append({**old_part, 'path': old_part['path'] + '.unknown'}) + elif fault == 'format': after['vm000']['formats'][0]['value']['id'] = 'foreign' + elif fault == 'live': after['vm000']['quiescent'] = False + elif fault == 'bool-pid': after['vm000']['pid'] = False + elif fault == 'bool-disk': old_part['disk'] = False + elif fault == 'sibling-id': final['items']['b']['uploadId'] += '-different' + elif fault == 'model-payload': final['items']['a']['payload']['sha256'] = 'e' * 64 + elif fault == 'multipart-etag': final['items']['a']['etag'] = '"wrong-3"' + elif fault == 'a-before-state': original['items']['a']['state'] = 'unknown' + elif fault == 'a-metadata': final['items']['a']['metadata']['nonce'] += '-changed' + else: final['items']['b']['parts'].pop() + with self.assertRaises(ValueError): committed.transition(before, after, original, final, evidence, 'a' * 64, 'b' * 64) + + +if __name__ == '__main__': + unittest.main() diff --git a/test_validation_heal_race_retention.py b/test_validation_heal_race_retention.py index 9ab5d62d..3db396a0 100644 --- a/test_validation_heal_race_retention.py +++ b/test_validation_heal_race_retention.py @@ -258,6 +258,41 @@ def test_a09_source_summary_without_public_case_field_keeps_uncertain_delete_for path.write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'abort_multipart_upload'})) with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) + def test_g06_original_copy_and_metadata_journals_require_explicit_source_case_identity(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, folder = self.archive(directory) + original = next(root.glob('*/run/*')) + case = 'heal.race-copy-and-metadata-update' + run = original.with_name(case); original.rename(run) + old_case = run / 'cases' / (controller.CASE + '.json') + value = json.loads(old_case.read_text()); value['case_id'] = case + old_case.rename(old_case.with_name(case + '.json')) + (run / 'cases' / (case + '.json')).write_text(json.dumps(value)) + evidence = run / 'evidence' / controller.CASE + evidence.rename(evidence.with_name(case)); evidence = evidence.with_name(case) + summary = json.loads((evidence / 'healthy-summary.json').read_text()); summary['caseId'] = case + summary['results'][0].update(caseId=case, branch='copy-source') + (evidence / 'healthy-summary.json').write_text(json.dumps(summary)) + branch = evidence / 'H2/overwrite-vs-deep'; branch.rename(branch.with_name('copy-source')) + branch = branch.with_name('copy-source') + path = branch / 'foreground-ACKs/000001-request.json' + path.write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'copy_object'})) + plan = recovery.recovery_plan(root, 12345, candidate)[0] + self.assertEqual(plan['kind'], 'unresolved-g06-race') + before = plan['ackFilesSha256'] + path.write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'copy_object', 'identity': {'uncertain': True}})) + self.assertNotEqual(recovery.recovery_plan(root, 12345, candidate)[0]['ackFilesSha256'], before) + summary['results'][0]['caseId'] = 'foreign-case' + (evidence / 'healthy-summary.json').write_text(json.dumps(summary)) + with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) + del summary['results'][0]['caseId'] + (evidence / 'healthy-summary.json').write_text(json.dumps(summary)) + with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) + summary['results'][0]['caseId'] = case + (evidence / 'healthy-summary.json').write_text(json.dumps(summary)) + path.write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'abort_multipart_upload'})) + with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) + def test_g02_original_quarantine_contents_are_hashed_without_reading_credentials(self): with tempfile.TemporaryDirectory() as directory: helper, state, roots, node, request, _ = self.scene(directory) @@ -288,6 +323,21 @@ def tracked(path, *args, **kwargs): self.assertFalse(any(path.endswith('service.env') or path.endswith('/credential') for path in opened)) self.assertFalse(result['canonicalPassClaimed'] or result['backupRestorationUsed']) + def test_g06_original_quarantine_contents_are_hashed_without_reading_credentials(self): + with tempfile.TemporaryDirectory() as directory: + helper, state, roots, node, request, _ = self.scene(directory) + for root in roots: + for path in (root / 'quarantine').glob('G01-*'): + path.rename(path.with_name(path.name.replace('G01-', 'G06-', 1))) + opened, original_open = [], os.open + def tracked(path, *args, **kwargs): + opened.append(str(path)); return original_open(path, *args, **kwargs) + with helper.root_stats(), patch.object(os, 'open', side_effect=tracked): + result = retained.run(request, node) + self.assertEqual(sum(path.endswith('G06-object-0256.original') for path in opened), 4) + self.assertFalse(any(path.endswith('service.env') or path.endswith('/credential') for path in opened)) + self.assertFalse(result['canonicalPassClaimed'] or result['backupRestorationUsed']) + if __name__ == '__main__': unittest.main() From f77b0496ba9e735c54ce20a8d945743a6686b994 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 14:36:21 +0800 Subject: [PATCH 269/343] fix(validation): preserve source metadata race generation contract --- scripts/validation_heal/race.py | 9 +++--- test_validation_heal_g06_source.py | 46 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index 0acf5b62..b2ca2516 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -687,10 +687,11 @@ def coding(target): == {part["index"]: (part["sha256"], part["size"]) for part in source["cohort"]}, "G06 copy did not publish a distinct target generation with source shard bytes") else: - require(target_after["dataDir"] == target_before["dataDir"] - and [(part["index"], part["sha256"], part["size"]) for part in target_after["cohort"]] - == [(part["index"], part["sha256"], part["size"]) for part in target_before["cohort"]], - "G06 metadata/tag update changed the data generation") + # The frozen metadata/tag oracle permits a new current directory; + # every coding index must still contain the original payload bytes. + require({part["index"]: (part["sha256"], part["size"]) for part in target_after["cohort"]} + == {part["index"]: (part["sha256"], part["size"]) for part in target_before["cohort"]}, + "G06 metadata/tag update changed the original coding bytes") else: remaining = {(row["key"], row["versionId"]) for row in final["entries"]} removed = [value for pair, value in mapped.items() if pair not in remaining] diff --git a/test_validation_heal_g06_source.py b/test_validation_heal_g06_source.py index 9974801d..15421fa0 100644 --- a/test_validation_heal_g06_source.py +++ b/test_validation_heal_g06_source.py @@ -122,6 +122,52 @@ def test_copy_rejects_wrong_source_shards_and_resurrected_destination_generation with self.assertRaises(ValueError): race.physical_transition(before, after, self.bucket, mapped, final, branch) + def metadata_scene(self): + before, after, mapped, final, _ = self.scene('copy-destination') + original = {(f['disk'], f['path']): f for f in before['node1']['files']} + for f in after['node1']['files']: + if f['path'].startswith(f'{self.bucket}/{race.G06_TARGET_KEY}/') and f['path'].endswith('/part.1'): + old_path = f'{self.bucket}/{race.G06_TARGET_KEY}/{uuid.UUID(int=100)}/part.1' + f['sha256'] = original[f['disk'], old_path]['sha256'] + return before, after, mapped, final + + def test_metadata_and_tags_allow_one_current_directory_with_original_coding_bytes(self): + # The frozen generation_proof requires replacement only for COPY and + # overwrite. Metadata/tag mutations still require the complete coding set. + for branch in ('replace-metadata', 'update-tags'): + for replace in (False, True): + with self.subTest(branch=branch, replace=replace): + before, after, mapped, final = self.metadata_scene() + if not replace: after = copy.deepcopy(before) + _, proof = race.physical_transition(before, after, self.bucket, mapped, final, branch) + self.assertTrue(proof['physicalProofBeforeGETOrRescue']) + + def test_metadata_and_tags_reject_incomplete_mixed_or_wrong_coding_generations(self): + for branch in ('replace-metadata', 'update-tags'): + for fault in ('wrong-shard', 'wrong-index', 'missing-part', 'mixed-generation', 'old-leaf', 'control'): + with self.subTest(branch=branch, fault=fault): + before, after, mapped, final = self.metadata_scene() + rows = after['node1']['files'] + parts = [f for f in rows if f['path'].startswith(f'{self.bucket}/{race.G06_TARGET_KEY}/') + and f['path'].endswith('/part.1')] + if fault == 'wrong-shard': parts[0]['sha256'] = 'wrong' + elif fault == 'wrong-index': + parts[0]['sha256'], parts[1]['sha256'] = parts[1]['sha256'], parts[0]['sha256'] + elif fault == 'missing-part': rows.remove(parts[0]) + elif fault == 'mixed-generation': + meta = next(f for f in rows if f['disk'] == 0 + and f['path'] == f'{self.bucket}/{race.G06_TARGET_KEY}/xl.meta') + meta['metadata']['versions'][0]['dataDir'] = str(uuid.UUID(int=100)) + elif fault == 'old-leaf': + rows.append(copy.deepcopy(next(f for f in before['node1']['files'] + if f['path'].startswith(f'{self.bucket}/{race.G06_TARGET_KEY}/') + and f['path'].endswith('/part.1')))) + else: + next(f for f in rows if f['path'].startswith(f'{self.bucket}/{race.G06_REFERENCE_KEY}/') + and f['path'].endswith('/part.1'))['sha256'] = 'changed-control' + with self.assertRaises(ValueError): + race.physical_transition(before, after, self.bucket, mapped, final, branch) + if __name__ == '__main__': unittest.main() From 19a47ddf6a057e1f6cb0d58c1cb400debcdab4f4 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 14:47:50 +0800 Subject: [PATCH 270/343] feat(validation): preserve uncertain multipart race scenes --- scripts/validation_heal/fleet_node.py | 1 + scripts/validation_heal/mpu_scene.py | 33 ++++++++--- scripts/validation_heal/race_io.py | 43 ++++++++++++-- scripts/validation_heal/retained_scene.py | 32 +++++++---- test_validation_heal_mpu_scene.py | 24 ++++++++ test_validation_heal_race_io.py | 67 ++++++++++++++++++++++ test_validation_heal_retained_scene.py | 68 ++++++++++++++++++++++- 7 files changed, 242 insertions(+), 26 deletions(-) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 6acdb69c..71c829f8 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1819,6 +1819,7 @@ def main(request): actions["mpu-race-scene"] = lambda value: globals()["mpu_race_backend"].run(value, globals()) actions["network-delete-records"] = lambda value: globals()["network_delete_records_backend"].run(value, globals()) actions["race-io-cap"] = lambda value: globals()["race_io_backend"].run(value, globals()) + actions["race-io-observe"] = lambda value: globals()["race_io_backend"].run(value, globals()) actions["mpu-race-backups"] = lambda value: globals()["mpu_race_backups_backend"].run(value, globals()) actions["version-race-backups"] = lambda value: globals()["mpu_race_backups_backend"].run(value, globals()) actions["process-clock"] = process_clock diff --git a/scripts/validation_heal/mpu_scene.py b/scripts/validation_heal/mpu_scene.py index ba7d2b97..e735f6d7 100644 --- a/scripts/validation_heal/mpu_scene.py +++ b/scripts/validation_heal/mpu_scene.py @@ -1,4 +1,4 @@ -"""Read-only stopped I03 multipart census, scoped by original bucket/key SHA-256.""" +"""Read-only stopped I03/G05 multipart census, scoped by original bucket/key SHA-256.""" import hashlib import json @@ -9,6 +9,7 @@ NAMES = ('abort', 'complete', 'create', 'upload') +G05_KEY = 'objects-0256.bin' MAX_FILES = 4096 MAX_BYTES = 256 * 1024**2 @@ -24,6 +25,19 @@ def keys_for(endpoints): return sorted(f'dl/{endpoint}/{name}.bin' for endpoint in endpoints for name in NAMES) +def validate_scope(owner, keys): + if owner.get('runtimeProfile') == 'manual': + require(re.fullmatch(r'rv-heal-[0-9a-f]{16}', owner.get('bucket', '')) + and keys == [G05_KEY], 'G05 MPU retention requires its one frozen same-key prefix') + return 'the original G05 bucket/key SHA-256 prefix, including unknown upload IDs; not a bitrot/EC proof' + require(owner.get('runtimeProfile') == 'lock-quorum' and isinstance(keys, list) and len(keys) == 8 + and all(isinstance(key, str) and re.fullmatch(r'dl/(node1|node3|node4)/(abort|complete|create|upload)\.bin', key) + for key in keys), 'MPU scene keys escaped the exact DL matrix') + endpoints = sorted({key.split('/')[1] for key in keys}) + require(keys == keys_for(endpoints), 'MPU scene omitted or duplicated an original request key') + return 'all files under the eight original DL bucket/key SHA-256 prefixes; not a bitrot/EC proof' + + def stable(info): return (info.st_mode, info.st_uid, info.st_gid, info.st_dev, info.st_ino, info.st_nlink, info.st_size, info.st_mtime_ns, info.st_ctime_ns) @@ -33,7 +47,7 @@ def run(request, node): require(set(request) == {'runId', 'action', 'bucket', 'keys', 'expectedBinarySha256'} and request['action'] == 'mpu-scene', 'MPU scene accepts only an exact owned read-only scope') state, roots, owner = node['owned'](request['runId']) - require(owner.get('runtimeProfile') == 'lock-quorum' and owner.get('bucket') == request['bucket'] + require(owner.get('runtimeProfile') in {'lock-quorum', 'manual'} and owner.get('bucket') == request['bucket'] and owner.get('binarySha256') == request['expectedBinarySha256'] and re.fullmatch(r'[0-9a-f]{64}', request['expectedBinarySha256']) is not None and len(roots) == 4, 'MPU scene candidate/runtime/primary bucket differs from the owned H2 fixture') @@ -41,11 +55,7 @@ def run(request, node): require(node['digest'](node['checked'](state / 'rustfs', state)) == owner['binarySha256'], 'MPU scene candidate binary changed') keys = request['keys'] - require(isinstance(keys, list) and len(keys) == 8 and all(isinstance(key, str) - and re.fullmatch(r'dl/(node1|node3|node4)/(abort|complete|create|upload)\.bin', key) for key in keys), - 'MPU scene keys escaped the exact DL matrix') - endpoints = sorted({key.split('/')[1] for key in keys}) - require(keys == keys_for(endpoints), 'MPU scene omitted or duplicated an original request key') + scope = validate_scope(owner, keys) files, directories, scopes, total = [], [], [], 0 for disk, root in enumerate(roots): data = node['checked'](root / 'data', root) @@ -67,7 +77,12 @@ def visit(path, depth=0): if stat.S_ISDIR(before.st_mode): require(len(directories) < MAX_FILES, 'MPU scene directory budget exceeded') directories.append({'disk': disk, 'path': relative, 'key': key}) - for child in sorted(path.iterdir()): + children = [] + for child in path.iterdir(): + require(len(children) < (3 if depth == 0 and owner['runtimeProfile'] == 'manual' else MAX_FILES), + 'MPU scene directory population exceeded its bound') + children.append(child) + for child in sorted(children): visit(child, depth + 1) else: require(len(files) < MAX_FILES and before.st_size <= 64 * 1024**2 @@ -98,4 +113,4 @@ def visit(path, depth=0): 'quiescent': True, 'originalPhysicalDrives': 4, 'sceneSha256': hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest(), 'readOnly': True, 'rawContentsExported': False, 'payloadIntegrityClaimed': False, - 'scope': 'all files under the eight original DL bucket/key SHA-256 prefixes; not a bitrot/EC proof'} + 'scope': scope} diff --git a/scripts/validation_heal/race_io.py b/scripts/validation_heal/race_io.py index ea7ff823..9d26b521 100644 --- a/scripts/validation_heal/race_io.py +++ b/scripts/validation_heal/race_io.py @@ -37,9 +37,14 @@ def limits(raw): def run(request, node): - require(set(request) == {'runId', 'action', 'caseId', 'branch', 'enabled', 'expectedProcess'} - and request['action'] == 'race-io-cap' and type(request['enabled']) is bool - and request['branch'] in CAP_BRANCHES.get(request['caseId'], ()), + observing = request.get('action') == 'race-io-observe' + fields = {'runId', 'action', 'caseId', 'branch', 'expectedProcess'} + require((observing and set(request) == fields + and request['caseId'] == 'heal.race-multipart-complete-abort' + and request['branch'] in ('upload-continue', 'complete', 'abort')) + or (not observing and set(request) == fields | {'enabled'} + and request['action'] == 'race-io-cap' and type(request['enabled']) is bool + and request['branch'] in CAP_BRANCHES.get(request['caseId'], ())), 'write cap requires a registered frozen public race; G05 forbids this cap') state, roots, owner = node['owned'](request['runId']) require(owner.get('runtimeProfile') == 'manual' and len(roots) == len(node['MOUNTS']) == 4, @@ -70,16 +75,42 @@ def scope(): 'write cap process or unit moved to another cgroup') return {'systemdControlGroup': systemd_group, 'processCgroup': membership} scope() - def evidence(): - value = node['command']('cat', str(cgroup / 'io.max'), check=False) + def evidence(directory=cgroup): + value = node['command']('cat', str(directory / 'io.max'), check=False) if value.returncode: # A fresh unthrottled unit need not have its IO controller enabled. # Verify the group still exists; any other cat error remains fatal. require(value.returncode == 1 and 'No such file or directory' in value.stderr - and node['command']('test', '-d', str(cgroup), check=False).returncode == 0, + and node['command']('test', '-d', str(directory), check=False).returncode == 0, 'owned cgroup IO limits could not be read') return '', {} return value.stdout, limits(value.stdout) + if observing: + def sample(): + result = [] + # io.max is hierarchical; an empty owned unit does not rule out an + # inherited cap on system.slice. These paths follow the fixed group. + for directory in (cgroup, cgroup.parent, cgroup.parent.parent): + raw, parsed = evidence(directory) + require(all(all(value == 'max' for value in device.values()) for device in parsed.values()), + 'G05 functional race has an owned or inherited kernel IO cap') + result.append({'cgroup': str(directory), 'ioMax': raw}) + return result + before_scope = scope() + before_sample = sample() + process() + after_scope = scope() + after_sample = sample() + process() + require(scope() == after_scope == before_scope, + 'G05 IO observation process or unit moved during sampling') + require(node['owned'](request['runId']) == (state, roots, owner) and node['mounts']() == owner['mounts'], + 'G05 IO observation owner or mount changed during sampling') + return {'process': original, 'controlGroup': group, 'beforeScope': before_scope, 'afterScope': after_scope, + 'beforeIoScopes': before_sample, 'afterIoScopes': after_sample, + 'actualKernelNoLimitVerifiedAtSamples': True, 'continuousNoLimitProved': False, + 'inspectionOnly': True, 'mutated': False, 'ioCapRequested': False, + 'publicOverlapProved': False, 'canonicalCasePassClaimed': False} raw_before, before = evidence() device = target['maj:min'] defaults = dict.fromkeys(('rbps','wbps','riops','wiops'), 'max') diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py index 3f09eda7..96015bbd 100644 --- a/scripts/validation_heal/retained_scene.py +++ b/scripts/validation_heal/retained_scene.py @@ -1,6 +1,6 @@ """Retain incomplete stopped fixtures by same-filesystem, non-overwriting rename. -Only owner/binary and fixture/MRF bytes are read. Other entries, including +Only owner/binary and scoped fixture/MRF/multipart bytes are read. Other entries, including credentials, are retained by inode and metadata without reading their contents. """ @@ -88,9 +88,9 @@ def parent_ready(parent, boundary, node, *, create=False): and info.st_dev == boundary.stat().st_dev, "retained archive parent is not private on the original filesystem") -def census(folder, owner, *, state, node): +def census(folder, owner, *, state, node, mpu_keys=()): """Hash the complete inode inventory; read only explicitly scoped content.""" - rows, content_count = [], 0 + rows, content_count, mpu_bytes_hashed = [], 0, 0 boundary_device = folder.lstat().st_dev network = owner.get("runtimeProfile") == "lock-quorum" fixture_buckets = {owner["bucket"]} @@ -99,8 +99,11 @@ def census(folder, owner, *, state, node): multipart_keys = {hashlib.sha256(f"{owner['bucket']}/dl/{endpoint}/{name}.bin".encode()).hexdigest() for endpoint in ("node1", "node3", "node4") for name in ("abort", "complete", "create", "upload")} if network else set() + if mpu_keys and owner.get('runtimeProfile') == 'manual': + node['mpu_scene_backend'].validate_scope(owner, mpu_keys) + multipart_keys = {hashlib.sha256(f"{owner['bucket']}/{key}".encode()).hexdigest() for key in mpu_keys} def visit(path): - nonlocal content_count + nonlocal content_count, mpu_bytes_hashed node["checked"](path, folder) info = path.lstat() require(info.st_dev == boundary_device and (stat.S_ISDIR(info.st_mode) @@ -123,7 +126,7 @@ def visit(path): else: parts = Path(relative).parts fixture = not state and len(parts) > 2 and parts[0] == "data" and parts[1] in fixture_buckets - network_mpu = (not state and network and len(parts) > 4 + network_mpu = (not state and (network or mpu_keys) and len(parts) > 4 and parts[:3] == ("data", ".rustfs.sys", "multipart") and parts[3] in multipart_keys) network_backup = (not state and network and len(parts) == 2 and parts[0] == "quarantine" and re.fullmatch(r"I03-missing-target\.(original|intent\.json|applied\.json)", parts[1])) @@ -138,7 +141,12 @@ def visit(path): and re.fullmatch(r"J02-[0-9a-f]{32}\.(original|intent\.json|applied\.json|failed-[0-9a-f]{64})", parts[1])) race_backup = (not state and owner.get("runtimeProfile") == "manual" and len(parts) == 2 and parts[0] == "quarantine" - and re.fullmatch(r"(?:G0[126]|A09)-[a-z0-9-]{1,48}\.(original|intent\.json|applied\.json)", parts[1])) + and re.fullmatch(r"(?:G0[1256]|A09)-[a-z0-9-]{1,48}\.(original|intent\.json|applied\.json)", parts[1])) + if network_mpu and not network: + require(info.st_size <= 64 * 1024**2 + and mpu_bytes_hashed + info.st_size <= node['mpu_scene_backend'].MAX_BYTES, + 'G05 retained multipart content exceeds its byte budget') + mpu_bytes_hashed += info.st_size if (state and relative == "rustfs") or fixture or mrf or backup or opaque_backup or network_mpu or network_backup or race_backup: fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) try: @@ -218,7 +226,7 @@ def inspect(request, node): require(not any((folder / name).exists() for name in ( "drive-gate.json", "write-quorum-gate.json", "network-gate.json", "disk-replacement.json")), "retained scene still has a gate or replacement intent") - witnesses.append(census(folder, expected, state=index == 4, node=node)) + witnesses.append(census(folder, expected, state=index == 4, node=node, mpu_keys=request.get('mpuKeys', ()))) baseline = {"runId": request["runId"], "ownerSha256": sha(expected), "roots": witnesses[:4], "state": witnesses[4]} if per_run: baseline["archiveLayout"] = "per-run-parent" @@ -230,7 +238,9 @@ def inspect(request, node): require(re.fullmatch(r"[0-9a-f]{64}", original_sha) is not None, "restored fixture requires its original physical scene digest") baseline["originalSceneSha256"] = original_sha - if network: + multipart = network or expected['runtimeProfile'] == 'manual' and 'mpuKeys' in request + if multipart: + node['mpu_scene_backend'].validate_scope(expected, request['mpuKeys']) require(re.fullmatch(r"[0-9a-f]{64}", request.get("expectedMpuSceneSha256", "")) is not None, "network retention requires its stopped original MPU scene digest") baseline["originalMpuSceneSha256"] = request["expectedMpuSceneSha256"] @@ -244,7 +254,7 @@ def inspect(request, node): and scene.get("binarySha256") == expected["binarySha256"] and sha({key: scene[key] for key in ("files", "formats")}) == original_sha, "restored fixture differs from the original business scene") - if network: + if multipart: mpu = node["mpu_scene_backend"].run({"runId": request["runId"], "action": "mpu-scene", "bucket": expected["bucket"], "keys": request["mpuKeys"], "expectedBinarySha256": expected["binarySha256"]}, node) @@ -261,7 +271,9 @@ def run(request, node): fields = {"runId", "action", "expectedOwner"} if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual"}: fields.add("expectedOriginalSceneSha256") - if request.get("expectedOwner", {}).get("runtimeProfile") == "lock-quorum": + if request.get("expectedOwner", {}).get("runtimeProfile") == "lock-quorum" or ( + request.get('expectedOwner', {}).get('runtimeProfile') == 'manual' + and ('mpuKeys' in request or 'expectedMpuSceneSha256' in request)): fields.update({"expectedMpuSceneSha256", "mpuKeys"}) if request["action"] == "retained-scene-isolate": fields.add("expectedBaseline") diff --git a/test_validation_heal_mpu_scene.py b/test_validation_heal_mpu_scene.py index fa5bf1e6..b173cfe9 100644 --- a/test_validation_heal_mpu_scene.py +++ b/test_validation_heal_mpu_scene.py @@ -113,6 +113,30 @@ def changing(run_id): with self.assertRaisesRegex(ValueError, 'owner or original roots changed'): scene.run(request, node) self.assertEqual(len(calls), 2) + def test_g05_partial_seed_captures_unknown_upload_ids_and_empty_drives(self): + with tempfile.TemporaryDirectory() as d: + request, node, roots, _ = self.fixture(Path(d)) + owner = node['owned'](request['runId'])[2] + owner.update(runtimeProfile='manual', bucket='rv-heal-0123456789abcdef') + request.update(bucket=owner['bucket'], keys=[scene.G05_KEY]) + key_hash = hashlib.sha256(f"{owner['bucket']}/{scene.G05_KEY}".encode()).hexdigest() + expected = {} + for disk, root in enumerate(roots[:3]): + upload = root/'data/.rustfs.sys/multipart'/key_hash/f'unknown-create-{disk}' + upload.mkdir(parents=True) + file = upload/'partial-part';file.write_bytes(b'uncertain original bytes'+bytes([disk])) + expected[disk, file.relative_to(root/'data').as_posix()] = hashlib.sha256(file.read_bytes()).hexdigest() + result = scene.run(request, node) + self.assertEqual({(r['disk'],r['path']):r['sha256'] for r in result['files']}, expected) + self.assertEqual([r['present'] for r in result['scopes']], [True,True,True,False]) + self.assertFalse(result['payloadIntegrityClaimed'] or result['rawContentsExported']) + for bad in ([], [scene.G05_KEY, scene.G05_KEY], ['objects-0255.bin']): + with self.subTest(keys=bad), self.assertRaises(ValueError): + scene.run({**request, 'keys':bad}, node) + for index in range(3): (roots[0]/'data/.rustfs.sys/multipart'/key_hash/f'extra-{index}').mkdir() + with self.assertRaisesRegex(ValueError, 'directory population'): + scene.run(request, node) + if __name__ == '__main__': unittest.main() diff --git a/test_validation_heal_race_io.py b/test_validation_heal_race_io.py index 6dae96b1..713ae0ab 100644 --- a/test_validation_heal_race_io.py +++ b/test_validation_heal_race_io.py @@ -120,6 +120,73 @@ def test_kernel_parser_rejects_incomplete_duplicate_or_unbounded_fields(self): '8:4 rbps=max wbps=max riops=max wiops=max\n'*2, 'x'*65537): with self.subTest(raw=raw[:80]), self.assertRaises(ValueError): race_io.limits(raw) + def observation(self): + request, node, kernel, process, owner = self.fixture() + request.pop('enabled') + request.update(action='race-io-observe', caseId=race.CASE_IDS[4], branch='complete') + original = node['command'] + kernel['ancestors'] = {'/sys/fs/cgroup/system.slice/io.max': '', '/sys/fs/cgroup/io.max': ''} + kernel['reads'] = [] + def command(*args, **kwargs): + if args[0] == 'cat' and args[1].endswith('/io.max'): + kernel['reads'].append(args[1]) + if args[1] in kernel['ancestors']: + return SimpleNamespace(stdout=kernel['ancestors'][args[1]], stderr='', returncode=0) + return original(*args, **kwargs) + node['command'] = command + return request, node, kernel, process, owner + + def test_g05_observes_owned_and_inherited_io_limits_without_writes(self): + request, node, kernel, _, _ = self.observation() + for branch in ('upload-continue', 'complete', 'abort'): + result = race_io.run({**request, 'branch': branch}, node) + self.assertTrue(result['inspectionOnly'] and result['actualKernelNoLimitVerifiedAtSamples']) + self.assertFalse(result['mutated'] or result['continuousNoLimitProved'] or result['ioCapRequested']) + self.assertEqual(len(result['beforeIoScopes']), 3) + self.assertEqual(result['beforeIoScopes'], result['afterIoScopes']) + self.assertFalse(kernel['writes']) + self.assertEqual(len(kernel['reads']), 18) + + def test_g05_refuses_each_ancestor_cap_and_cannot_release_a_foreign_limit(self): + for where in ('owned', '/sys/fs/cgroup/system.slice/io.max', '/sys/fs/cgroup/io.max'): + for limit in ('rbps', 'wbps', 'riops', 'wiops'): + with self.subTest(where=where, limit=limit): + request, node, kernel, _, _ = self.observation() + raw = '8:4 ' + ' '.join(f'{k}={65536 if k == limit else "max"}' + for k in ('rbps', 'wbps', 'riops', 'wiops')) + '\n' + if where == 'owned': kernel['raw'] = raw + else: kernel['ancestors'][where] = raw + with self.assertRaisesRegex(ValueError, 'inherited kernel IO cap'): + race_io.run(request, node) + self.assertFalse(kernel['writes']) + + def test_g05_observation_rejects_scope_changes_and_mutation_parameters(self): + for fault in ('case', 'branch', 'enabled', 'process', 'group', 'runtime', 'mount', 'late-cap', 'late-birth', 'late-group'): + with self.subTest(fault=fault): + request, node, kernel, process, owner = self.observation() + if fault == 'case': request['caseId'] = race.CASE_IDS[0] + elif fault == 'branch': request['branch'] = 'copy-source' + elif fault == 'enabled': request['enabled'] = False + elif fault == 'process': process['startTicks'] += 1 + elif fault == 'group': kernel['membership'] = '0::/system.slice/other.service\n' + elif fault == 'runtime': owner['runtimeProfile'] = 'lock-quorum' + elif fault == 'mount': + mounts = copy.deepcopy(owner['mounts']) + mounts[0]['maj:min'] = '8:99' + node['mounts'] = lambda: mounts + else: + original = node['command'] + def change(*args, **kwargs): + result = original(*args, **kwargs) + if len(kernel['reads']) == 3: + if fault == 'late-cap': kernel['ancestors']['/sys/fs/cgroup/io.max'] = '8:4 rbps=max wbps=7 riops=max wiops=max\n' + elif fault == 'late-birth': process['startTicks'] += 1 + else: kernel['membership'] = '0::/system.slice/foreign.service\n' + return result + node['command'] = change + with self.assertRaises(ValueError): race_io.run(request, node) + self.assertFalse(kernel['writes']) + def test_missing_unenabled_controller_is_accepted_only_for_an_existing_owned_group(self): for group_exists in (True, False): with self.subTest(group_exists=group_exists): diff --git a/test_validation_heal_retained_scene.py b/test_validation_heal_retained_scene.py index 8e1e2b53..044ab682 100644 --- a/test_validation_heal_retained_scene.py +++ b/test_validation_heal_retained_scene.py @@ -13,7 +13,7 @@ import unittest from unittest.mock import patch -from scripts.validation_heal import fleet_node, retained_scene as retained, recover_environment as recovery +from scripts.validation_heal import fleet_node, retained_scene as retained, recover_environment as recovery, mpu_scene from scripts.validation_heal.healthy import Fleet @@ -178,6 +178,72 @@ def test_linux_rename_explicitly_uses_noreplace_and_syncs_both_parents(self): call.assert_called_once_with(-100, b"/original/run", -100, b"/retained/run", 1) self.assertEqual(sync.call_args_list, [unittest.mock.call(Path("/original")), unittest.mock.call(Path("/retained"))]) + def mpu_partial_scene(self, directory): + state, roots, node, request = self.scene(directory) + shutil.copytree(roots[1], roots[0]) + owner = request['expectedOwner'];owner['runtimeProfile'] = 'manual' + for folder in (*roots, state): (folder/'owner.json').write_text(json.dumps(owner)) + snapshot = {'files':[], 'formats':[], 'pid':0, 'quiescent':True, 'binarySha256':owner['binarySha256']} + node.update(owned=lambda run:(state, roots, copy.deepcopy(owner)), + snapshot=lambda value:copy.deepcopy(snapshot), mpu_scene_backend=mpu_scene) + key_hash = hashlib.sha256(f"{owner['bucket']}/{mpu_scene.G05_KEY}".encode()).hexdigest() + for root in roots: + folder = root/'data/.rustfs.sys/multipart'/key_hash/'unknown-create-before-ACK' + folder.mkdir(parents=True) + (folder/'part.1').write_bytes(b'unknown MPU original bytes') + (root/'quarantine/G05-partial.original').write_bytes(b'original missing-shard backup') + raw = mpu_scene.run({'runId':request['runId'], 'action':'mpu-scene', 'bucket':owner['bucket'], + 'keys':[mpu_scene.G05_KEY], 'expectedBinarySha256':owner['binarySha256']}, node) + request.update(expectedOriginalSceneSha256=retained.sha({k:snapshot[k] for k in ('files','formats')}), + mpuKeys=[mpu_scene.G05_KEY], expectedMpuSceneSha256=raw['sceneSha256']) + return state, roots, node, request + + def test_g05_unknown_partial_seed_isolates_original_mpu_inodes_without_reading_credentials(self): + with tempfile.TemporaryDirectory() as directory: + state, roots, node, request = self.mpu_partial_scene(directory) + original = {p:(p.stat().st_ino,p.read_bytes()) for root in roots + for p in root.rglob('*') if p.is_file()} + opened, real_open = [], os.open + def record(path, *args, **kwargs): + opened.append(str(path));return real_open(path, *args, **kwargs) + with self.root_stats(), patch.object(os,'open',side_effect=record), \ + patch.object(retained,'rename_new',side_effect=self.local_rename): + before = retained.run(request,node) + self.assertTrue(all(row['contentFilesHashed']==3 for row in before['baseline']['roots'])) + change = {**request,'action':'retained-scene-isolate','expectedBaseline':before['baseline']} + def interrupted(source, target): + self.local_rename(source, target) + if source == roots[0]: raise RuntimeError('lost response after first MPU root move') + with patch.object(retained,'rename_new',side_effect=interrupted),self.assertRaises(RuntimeError): + retained.run(change,node) + result = retained.run(change,node) + self.assertEqual(result['baseline'],before['baseline']) + self.assertEqual(result['locations'],['retained']*5) + self.assertTrue(result['activeNamespaceIsolated']) + self.assertFalse(result['originalSceneRestored'] or result['canonicalPassClaimed']) + _, targets = retained.destinations(state,roots,per_run_parent=True) + for old,(inode,raw) in original.items(): + index = next(i for i,root in enumerate(roots) if old.is_relative_to(root)) + new = targets[index]/old.relative_to(roots[index]) + self.assertEqual((new.stat().st_ino,new.read_bytes()),(inode,raw)) + self.assertFalse(any(p.endswith(('service.env','credential')) for p in opened)) + + def test_g05_unknown_mpu_scope_or_digest_drift_refuses_before_any_move(self): + for fault in ('scope','digest','missing-digest','changed-content','byte-cap'): + with self.subTest(fault=fault),tempfile.TemporaryDirectory() as directory: + state,roots,node,request = self.mpu_partial_scene(directory) + with self.root_stats(): + before = retained.run(request,node) + if fault == 'scope': request['mpuKeys'] = ['objects-0255.bin'] + elif fault == 'digest': request['expectedMpuSceneSha256'] = 'f'*64 + elif fault == 'missing-digest': request.pop('expectedMpuSceneSha256') + elif fault == 'changed-content': next((roots[0]/'data/.rustfs.sys/multipart').rglob('part.1')).write_bytes(b'changed MPU') + with patch.object(retained,'rename_new') as move, \ + patch.object(mpu_scene,'MAX_BYTES',1 if fault == 'byte-cap' else mpu_scene.MAX_BYTES), \ + self.assertRaises(ValueError): + retained.run({**request,'action':'retained-scene-isolate','expectedBaseline':before['baseline']},node) + move.assert_not_called() + class IncompleteFleetIsolationTests(unittest.TestCase): def test_main_does_not_treat_an_archived_namespace_as_restored_or_already_clean(self): From 9e40dc376c6d9185a12c16aa2f02cb6b558b04e0 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 20 Oct 2026 14:54:46 +0800 Subject: [PATCH 271/343] feat(validation): recover exact uncertain multipart race archives --- scripts/validation_heal/race_retention.py | 71 ++++++++++++-- .../validation_heal/recover_environment.py | 5 +- test_validation_heal_race_retention.py | 95 +++++++++++++++++++ 3 files changed, 162 insertions(+), 9 deletions(-) diff --git a/scripts/validation_heal/race_retention.py b/scripts/validation_heal/race_retention.py index 8b9591ad..f89adbeb 100644 --- a/scripts/validation_heal/race_retention.py +++ b/scripts/validation_heal/race_retention.py @@ -16,6 +16,7 @@ CASES = {CASE: ('unresolved-g01-race', {'overwrite-vs-deep', 'overwrite-vs-read-repair'}), 'heal.race-versioned-put': ('unresolved-g02-race', {'new-version-vs-historical-heal', 'read-all-versions-during-heal'}), 'heal.admin-nolock-safety': ('unresolved-a09-race', {'overwrite-nolock', 'delete-nolock'}), + 'heal.race-multipart-complete-abort': ('unresolved-g05-race', {'upload-continue', 'complete', 'abort'}), 'heal.race-copy-and-metadata-update': ('unresolved-g06-race', {'copy-source', 'copy-destination', 'replace-metadata', 'update-tags'})} @@ -24,9 +25,11 @@ def digest(value): def validate(plan): - require(set(plan) == {'kind', 'runId', 'nodes', 'bucket', 'owners', 'businessSceneSha256', - 'candidateCommit', 'harnessCommit', 'binarySha256', 'ackFilesSha256'} - and plan['kind'] in {'unresolved-g01-race', 'unresolved-g02-race', 'unresolved-a09-race', 'unresolved-g06-race'} + multipart = plan.get('kind') == 'unresolved-g05-race' + fields = {'kind', 'runId', 'nodes', 'bucket', 'owners', 'businessSceneSha256', + 'candidateCommit', 'harnessCommit', 'binarySha256', 'ackFilesSha256'} + require(set(plan) == fields | ({'mpuKeys', 'mpuSceneSha256'} if multipart else set()) + and plan['kind'] in {kind for kind, _ in CASES.values()} and re.fullmatch(r'gh-[1-9][0-9]*-[1-9][0-9]*-h[12]', plan['runId']) and len(plan['nodes']) == len(set(plan['nodes'])) == (1 if plan['runId'].endswith('-h1') else 4) and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in plan['nodes']) @@ -45,6 +48,10 @@ def validate(plan): 'G01/G02 isolation lost its exact reserved owner or stopped physical scene') require(len({owner['hostname'] for owner in plan['owners'].values()}) == len(plan['nodes']), 'G01/G02 isolation host aliases overlap') + if multipart: + require(plan['mpuKeys'] == ['objects-0256.bin'] and set(plan['mpuSceneSha256']) == set(plan['nodes']) + and all(re.fullmatch(r'[0-9a-f]{64}', value) for value in plan['mpuSceneSha256'].values()), + 'G05 isolation lost its exact same-key stopped multipart scene') def archived_plan(artifact, failed_run, candidate): @@ -89,14 +96,26 @@ def one(label): require(len(owners) == len(reservations) == len(scene) and set(owners) == set(scene) and all(item.get('reserved') is True for item in reservations), 'G01/G02 isolation lacks all reserved hosts') ack_files = {} - for name in ('seed-ACKs', 'foreground-ACKs', 'historical-read-ACKs'): + multipart = kind == 'unresolved-g05-race' + for name in ('seed-ACKs', 'foreground-ACKs', 'historical-read-ACKs', *(['mpu-read-ACKs'] if multipart else [])): journal = folder / name entries = sorted(journal.glob('*.json')) - if name == 'historical-read-ACKs': + if name in {'historical-read-ACKs', 'mpu-read-ACKs'} or multipart and name == 'foreground-ACKs': require(len(entries) <= 1024, 'G02 historical journal exceeds its bound') if not entries: continue require(0 < len(entries) <= 1024, 'G01/G02 isolation lacks its bounded original request/ACK journal') + if multipart and name == 'seed-ACKs': + require(any(re.fullmatch(r'[0-9]{6}-request\.json', path.name) for path in entries), + 'G05 isolation requires an original SDK request, not only a model manifest') for path in entries: + if multipart and re.fullmatch(r'mpu-manifest-[0-9]{4}\.json', path.name): + require(path.stat().st_size <= 1024**2, 'G05 isolation multipart manifest exceeds its bound') + item = json.loads(path.read_text()) + require(item.get('bucket') == binding['bucketPrefix'] + and item.get('key') == 'objects-0256.bin' and item.get('fixture') == 'DM', + 'G05 isolation has a foreign multipart manifest') + ack_files[name + '/' + path.name] = hashlib.sha256(path.read_bytes()).hexdigest() + continue require(re.fullmatch(r'[0-9]{6}-(request|ack)\.json', path.name) and path.stat().st_size <= 1024**2, 'G01/G02 isolation journal contains an unexpected member') item = json.loads(path.read_text()) @@ -104,7 +123,32 @@ def one(label): # A missing/unknown ACK remains unknown. Isolation never infers its # effect or sends another request, including for explicit refusals. operation = item.get('operation') if path.name.endswith('-request.json') else (item.get('observation') or {}).get('operation') - allowed = ({'get_object'} if name == 'historical-read-ACKs' else + if multipart: + request_item = item + if path.name.endswith('-ack.json'): + request_name = item.get('requestFile', '') + require(re.fullmatch(r'[0-9]{6}-request\.json', request_name) + and request_name == path.name.replace('-ack.json', '-request.json'), + 'G05 isolation receipt lacks its exact original request') + request_path = journal / request_name + require(request_path.is_file() and request_path.stat().st_size <= 1024**2 + and hashlib.sha256(request_path.read_bytes()).hexdigest() == item.get('requestSha256'), + 'G05 isolation receipt changed its original request bytes') + request_item = json.loads(request_path.read_text()) + require(operation is None or operation == request_item.get('operation'), + 'G05 isolation receipt reports a different SDK operation') + operation = request_item.get('operation') + params = request_item.get('params') or {} + require(request_item.get('runId') == binding['runId'] and params.get('Bucket') == binding['bucketPrefix'], + 'G05 isolation SDK request belongs to another reserved bucket') + if operation in {'create_multipart_upload', 'upload_part', 'complete_multipart_upload', + 'abort_multipart_upload', 'list_parts', 'get_object', 'head_object', 'get_object_tagging'}: + require(params.get('Key') == 'objects-0256.bin', 'G05 isolation SDK request changed its frozen multipart key') + allowed = ({'list_multipart_uploads', 'list_parts', 'list_object_versions', 'get_object', 'head_object', + 'get_object_tagging'} if multipart and name == 'mpu-read-ACKs' else + {'get_object'} if name == 'historical-read-ACKs' else + {'create_bucket', 'put_bucket_versioning', 'put_object', 'create_multipart_upload', + 'upload_part', 'complete_multipart_upload', 'abort_multipart_upload'} if multipart else {'create_bucket', 'put_object', 'copy_object', 'put_object_tagging'} if kind == 'unresolved-g06-race' else {'create_bucket', 'put_bucket_versioning', 'put_object', 'delete_object'} if kind == 'unresolved-a09-race' else {'create_bucket', 'put_bucket_versioning', 'put_object'} if kind == 'unresolved-g02-race' else @@ -116,6 +160,17 @@ def one(label): 'harnessCommit': binding['harnessCommit'], 'binarySha256': binding['candidateBinarySha256'], 'businessSceneSha256': {node: digest(value) for node, value in physical_content(scene).items()}, 'ackFilesSha256': digest(ack_files)} + if multipart: + mpu = one('raw-mpu-failure-scene') + require(set(mpu) == set(owners), 'G05 isolation omitted an original stopped MPU peer') + for value in mpu.values(): + require(value.get('runId') == binding['runId'] and value.get('bucket') == binding['bucketPrefix'] + and value.get('keys') == ['objects-0256.bin'] and value.get('quiescent') is True + and value.get('readOnly') is True and value.get('originalPhysicalDrives') == 4 + and value.get('binarySha256') == binding['candidateBinarySha256'] + and digest({key: value[key] for key in ('bucket', 'keys', 'scopes', 'directories', 'files', 'bytesHashed')}) + == value.get('sceneSha256'), 'G05 isolation original raw MPU capture differs') + plan.update(mpuKeys=['objects-0256.bin'], mpuSceneSha256={node: value['sceneSha256'] for node, value in mpu.items()}) validate(plan) for node, value in scene.items(): require(value.get('quiescent') is True and type(value.get('pid')) is int and value['pid'] == 0 @@ -131,7 +186,9 @@ def isolate(fleet, plan): validate(plan) require(fleet.run_id == plan['runId'] and list(fleet.nodes) == plan['nodes'], 'G01/G02 isolation changed its fleet') def params(node): - return {'expectedOwner': plan['owners'][node], 'expectedOriginalSceneSha256': plan['businessSceneSha256'][node]} + return {'expectedOwner': plan['owners'][node], 'expectedOriginalSceneSha256': plan['businessSceneSha256'][node], + **({'mpuKeys': plan['mpuKeys'], 'expectedMpuSceneSha256': plan['mpuSceneSha256'][node]} + if plan['kind'] == 'unresolved-g05-race' else {})} audits = {} for node in fleet.nodes: audit = audits[node] = fleet.record('race-retention-preflight', {'node': node, diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index 94821613..fe15410c 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -152,7 +152,8 @@ def recovery_plan(artifact, failed_run, candidate): if (list(artifact.glob("*/run/heal.race-unversioned-overwrite/cases/*.json")) or list(artifact.glob("*/run/heal.race-versioned-put/cases/*.json")) or list(artifact.glob("*/run/heal.admin-nolock-safety/cases/*.json")) - or list(artifact.glob("*/run/heal.race-copy-and-metadata-update/cases/*.json"))): + or list(artifact.glob("*/run/heal.race-copy-and-metadata-update/cases/*.json")) + or list(artifact.glob("*/run/heal.race-multipart-complete-abort/cases/*.json"))): return race_retention_plan(artifact, failed_run, candidate) if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/cases/*.json")): if list(artifact.glob("*/run/heal.outcome-counters-and-error-truth/evidence/heal.outcome-counters-and-error-truth/DB/H1/unknown-negative/*-one-way-legacy-write-phase.json")): @@ -650,7 +651,7 @@ def main(): if plan.get("buckets"): fleet.snapshot_buckets = plan["buckets"] fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) - if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race", "unresolved-a09-race", "unresolved-g06-race"}: + if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race", "unresolved-a09-race", "unresolved-g05-race", "unresolved-g06-race"}: complete.append({"failedRunId": args.failed_run_id, **isolate_race_scene(fleet, plan)}) continue if plan.get("kind") == "unresolved-network-load": diff --git a/test_validation_heal_race_retention.py b/test_validation_heal_race_retention.py index 3db396a0..c58f8e86 100644 --- a/test_validation_heal_race_retention.py +++ b/test_validation_heal_race_retention.py @@ -285,6 +285,7 @@ def test_g06_original_copy_and_metadata_journals_require_explicit_source_case_id summary['results'][0]['caseId'] = 'foreign-case' (evidence / 'healthy-summary.json').write_text(json.dumps(summary)) with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) + del summary['results'][0]['caseId'] (evidence / 'healthy-summary.json').write_text(json.dumps(summary)) with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) @@ -293,6 +294,100 @@ def test_g06_original_copy_and_metadata_journals_require_explicit_source_case_id path.write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'abort_multipart_upload'})) with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) + def mpu_archive(self, directory, *, foreground=False): + root,candidate,folder = self.archive(directory) + original = next(root.glob('*/run/*'));case = 'heal.race-multipart-complete-abort' + run = original.with_name(case);original.rename(run) + old = run/'cases'/(controller.CASE+'.json');value = json.loads(old.read_text());value['case_id'] = case + old.rename(old.with_name(case+'.json'));(run/'cases'/(case+'.json')).write_text(json.dumps(value)) + old = run/'evidence'/controller.CASE;evidence = old.with_name(case);old.rename(evidence) + old = evidence/'H2/overwrite-vs-deep';folder = old.with_name('upload-continue');old.rename(folder) + summary = {'caseId':case,'cleanup':'FAIL','results':[{'caseId':case,'cleanup':'FAIL','verdict':'FAIL', + 'topology':'H2','branch':'upload-continue'}]} + (evidence/'healthy-summary.json').write_text(json.dumps(summary)) + binding = json.loads((folder/'00001-binding.json').read_text()) + request = {'runId':binding['runId'],'operation':'create_multipart_upload', + 'params':{'Bucket':binding['bucketPrefix'],'Key':'objects-0256.bin'}} + path = folder/'seed-ACKs/000001-request.json';path.write_text(json.dumps(request)) + receipt = {'runId':binding['runId'],'requestFile':path.name, + 'requestSha256':hashlib.sha256(path.read_bytes()).hexdigest(), + 'observation':{'acknowledged':False,'resultUnknown':True}} + (path.parent/'000001-ack.json').write_text(json.dumps(receipt)) + (path.parent/'mpu-manifest-0001.json').write_text(json.dumps({'bucket':binding['bucketPrefix'], + 'key':'objects-0256.bin','fixture':'DM','poisoned':'unknown Create response','items':{}})) + fg = folder/'foreground-ACKs/000001-request.json' + if foreground: fg.write_text(json.dumps({**request,'operation':'complete_multipart_upload'})) + else: fg.unlink() + raw = {} + for node in ('vm000','vm001','vm002','vm003'): + value = {'bucket':binding['bucketPrefix'],'keys':['objects-0256.bin'], 'scopes':[], + 'directories':[], 'files':[], 'bytesHashed':0} + raw[node] = {**value,'sceneSha256':controller.digest(value),'runId':binding['runId'], + 'binarySha256':binding['candidateBinarySha256'],'quiescent':True,'readOnly':True, + 'originalPhysicalDrives':4,'payloadIntegrityClaimed':False} + (folder/'00007-raw-mpu-failure-scene.json').write_text(json.dumps(raw)) + return root,candidate,folder + + def test_g05_unknown_create_or_complete_retains_raw_scene_without_inventing_ack(self): + for foreground in (False,True): + with self.subTest(foreground=foreground),tempfile.TemporaryDirectory() as d: + root,candidate,folder = self.mpu_archive(d,foreground=foreground) + plan = recovery.recovery_plan(root,12345,candidate)[0] + self.assertEqual(plan['kind'],'unresolved-g05-race') + self.assertEqual(plan['mpuKeys'],['objects-0256.bin']) + self.assertEqual(set(plan['mpuSceneSha256']),set(plan['nodes'])) + before = plan['ackFilesSha256'];path = folder/'seed-ACKs/mpu-manifest-0001.json' + value = json.loads(path.read_text());value['poisoned'] = 'original uncertainty, no retry' + path.write_text(json.dumps(value)) + self.assertNotEqual(recovery.recovery_plan(root,12345,candidate)[0]['ackFilesSha256'],before) + + def test_g05_archive_rejects_missing_foreign_or_unbound_multipart_witnesses(self): + for fault in ('missing-peer','run','binary','live','digest','key','sdk-key','sdk-bucket','receipt', + 'model-only','foreign-manifest','forbidden-operation'): + with self.subTest(fault=fault),tempfile.TemporaryDirectory() as d: + root,candidate,folder = self.mpu_archive(d) + raw_path = folder/'00007-raw-mpu-failure-scene.json';raw = json.loads(raw_path.read_text()) + if fault == 'missing-peer': raw.pop('vm003') + elif fault in ('run','binary','live','digest','key'): + field = {'run':'runId','binary':'binarySha256','live':'quiescent','digest':'sceneSha256','key':'keys'}[fault] + raw['vm003'][field] = False if fault == 'live' else ['foreign-key'] if fault == 'key' else 'foreign' + elif fault in ('sdk-key','sdk-bucket','forbidden-operation'): + path = folder/'seed-ACKs/000001-request.json';value = json.loads(path.read_text()) + if fault == 'forbidden-operation': value['operation'] = 'delete_bucket' + else: value['params']['Key' if fault == 'sdk-key' else 'Bucket'] = 'foreign' + path.write_text(json.dumps(value)) + elif fault == 'receipt': + path = folder/'seed-ACKs/000001-ack.json';value = json.loads(path.read_text());value['requestSha256']='f'*64 + path.write_text(json.dumps(value)) + elif fault == 'model-only': + (folder/'seed-ACKs/000001-request.json').unlink();(folder/'seed-ACKs/000001-ack.json').unlink() + else: + path = folder/'seed-ACKs/mpu-manifest-0001.json';value = json.loads(path.read_text());value['key']='foreign' + path.write_text(json.dumps(value)) + raw_path.write_text(json.dumps(raw)) + with self.assertRaises(ValueError): recovery.recovery_plan(root,12345,candidate) + + def test_g05_controller_preflights_all_original_mpu_peers_before_any_move(self): + with tempfile.TemporaryDirectory() as d: + root,candidate,_ = self.mpu_archive(d);plan = recovery.recovery_plan(root,12345,candidate)[0] + audits = {};calls = [] + def remote(node,action,**params): + calls.append((node,action)) + if action == 'logs': return {} + self.assertEqual(params['mpuKeys'],['objects-0256.bin']) + self.assertEqual(params['expectedMpuSceneSha256'],plan['mpuSceneSha256'][node]) + if action == 'retained-scene-isolate': + self.assertEqual(set(audits),set(plan['nodes'])) + return {'baseline':audits[node],'activeNamespaceIsolated':True,'originalSceneRestored':False, + 'backupRestorationUsed':False,'canonicalPassClaimed':False,'credentialContentsRead':False} + post = node in audits;audits[node] = {'node':node} + return {'baseline':audits[node],'inspectionOnly':True,'mutated':False,'credentialContentsRead':False, + 'activeNamespaceIsolated':post,'privateArchiveParents':[{'exists':True,'uid':0,'mode':0o700}]*5} + fleet = SimpleNamespace(run_id=plan['runId'],nodes=plan['nodes'],remote=remote,record=lambda label,value:value) + result = controller.isolate(fleet,plan) + self.assertEqual(result['cleanup'],'ISOLATED');self.assertFalse(result['namespaceDeletionPerformed']) + self.assertTrue(all(action == 'retained-scene-audit' for _,action in calls[:4])) + def test_g02_original_quarantine_contents_are_hashed_without_reading_credentials(self): with tempfile.TemporaryDirectory() as directory: helper, state, roots, node, request, _ = self.scene(directory) From d834a7e963d2fdc3b387f35467f0b5c587fbd498 Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 21 Oct 2026 10:09:01 +0800 Subject: [PATCH 272/343] feat(validation): run frozen same-key multipart heal races --- .../workflows/validation-catalog-tests.yml | 3 + docs/validation-heal-operations.md | 10 +- scripts/validation_heal/healthy.py | 2 +- scripts/validation_heal/mpu_race_runner.py | 384 ++++++++++++++++++ scripts/validation_heal/race.py | 206 ++-------- scripts/validation_heal/race_retention.py | 11 +- scripts/validation_heal/status_keepalive.py | 155 +++++++ test_validation_heal_mpu_race_runner.py | 309 ++++++++++++++ test_validation_heal_race.py | 75 ++-- test_validation_heal_race_io.py | 11 +- test_validation_heal_race_native.py | 15 +- test_validation_heal_race_physical.py | 4 + test_validation_heal_race_retention.py | 22 + test_validation_heal_status_keepalive.py | 215 ++++++++++ 14 files changed, 1179 insertions(+), 243 deletions(-) create mode 100644 scripts/validation_heal/mpu_race_runner.py create mode 100644 scripts/validation_heal/status_keepalive.py create mode 100644 test_validation_heal_mpu_race_runner.py create mode 100644 test_validation_heal_status_keepalive.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index e894c384..fffd5190 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -16,6 +16,7 @@ on: - test_validation_replication.py - scripts/validation_replication/** - test_validation_heal.py + - test_validation_heal*.py - test_validation_heal_recovery.py - test_validation_heal_authorization.py - test_validation_heal_full_restart.py @@ -82,6 +83,7 @@ on: - test_validation_replication.py - scripts/validation_replication/** - test_validation_heal.py + - test_validation_heal*.py - test_validation_heal_recovery.py - test_validation_heal_authorization.py - test_validation_heal_full_restart.py @@ -157,6 +159,7 @@ jobs: sudo apt-get install --no-install-recommends -y nftables fi - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py + - run: python3 -B -m unittest -v test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 738039ec..ac14ccce 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -668,7 +668,7 @@ Source `8380466f3a27c3c3ef4115084178a0ab99a78f32` additionally requires the comp `race_native_evidence.py` admits the downloaded artifact without executing it. It checks the successful hosted workflow attempt, exact candidate/driver/helper identity, selected source contract, native executable, complete unique regression set and every original log. Each ordered-barrier regression must emit exactly the complete source-registered milestone sequence. Regressions marked as in-test assertions retain that narrower witness type; assertion-only evidence cannot invent emitted barrier events. The result states `requiredNativeSubsetPassed`; whole-case, fleet and canonical PASS remain false. Missing source groups, old-source artifacts, ignored or zero-test output and partial evidence are rejected. Updating another candidate's registry entry does not invalidate an unchanged selected contract. Changing the producer or the selected contract requires a fresh tools artifact. -Runtime sources opt in to specific cases separately from the native registry. The `6af` source enables only G01; `bd0671b` enables G01/G02/G03; `8380466` additionally enables G04 with its complete, distinct native group. Source `757b751` enables only A09 and G06 on their respective `nolock.py` and `race.py` runtime paths; it does not enable G05 or substitute a different source into G01-G04. The workflow verifies the bundle and invokes the pinned helper's read-only runtime preflight before exporting it to canonical execution. A collector-only helper without the corresponding runtime adapter fails before fleet mutation. The executor repeats admission before package installation or service changes. +Runtime sources opt in to specific cases separately from the native registry. The `6af` source enables only G01; `bd0671b` enables G01/G02/G03; `8380466` additionally enables G04 with its complete, distinct native group. Source `757b751` enables A09 through `nolock.py`, G06 through `race.py`, and G05 through the dedicated `mpu_race_runner.py` controller. It does not substitute a different source into G01-G04. The workflow verifies the bundle and invokes the pinned helper's read-only runtime preflight before exporting it to canonical execution. A collector-only helper without the corresponding runtime adapter fails before fleet mutation. The executor repeats admission before package installation or service changes. All original H1/H2 black-box branches, SDK ACKs, manifests, overlap witnesses, exact Admin settings and physical/S3 checks remain required. Native logical disk shapes do not certify distributed VM execution. The original `072b57b37559fbe626aeda8abf61617fdcbdd926` route retains its blocked native rows. A09, G05/G06 and I05 still require their own complete native groups and executor evidence. G04 also remains unaccepted until its same-source hosted evidence and every original live branch pass independent audit. No source or case gains canonical acceptance from a registry entry or local native test; a successful exact-source hosted bundle, every live branch and independent original-evidence audit are required. @@ -676,7 +676,7 @@ The A09/G01-G04/G06 public race adapters retain the frozen fourth-physical-drive ### A09/G06 native groups and frozen G06 public fixtures -Source `757b751482bb970bc321e92059690a7698554358` adds two collector-only groups. `default-heal-exclusive-owner` observes actual PUT and exact historical DELETE lock futures becoming pending while default Deep Heal exclusively owns a real staged external repair. Physical reconstruction precedes foreground publication; all acknowledged successor or retained-current shards and full bodies are checked in 4/2 and 16/4. Its sixteen events prove the storage fence, while Admin `nolock=true` normalization and public request timing still require separate evidence. `copy-and-metadata-final-rename` covers real streamed cross-key COPY source/destination, self metadata replacement and tags. Twenty-four events bind actual ACK and physical-tail completion before releasing shared reconstruction, exact stale refusal or unchanged-source repair, final source/target/control payloads, metadata and tags. The registry binds all five storage fixture/barrier source files. This candidate admits A09 and G06 only with their exact case-specific native group and matching hosted helper/source artifact. Both workflow and executor run read-only preflight before bootstrap. A09 always sends wire `nolock=true` and requires effective `nolock=false`, plus both original H1/H2 public branches and the separate storage-native proof. G05 remains outside runtime admission. +Source `757b751482bb970bc321e92059690a7698554358` adds two collector-only groups. `default-heal-exclusive-owner` observes actual PUT and exact historical DELETE lock futures becoming pending while default Deep Heal exclusively owns a real staged external repair. Physical reconstruction precedes foreground publication; all acknowledged successor or retained-current shards and full bodies are checked in 4/2 and 16/4. Its sixteen events prove the storage fence, while Admin `nolock=true` normalization and public request timing still require separate evidence. `copy-and-metadata-final-rename` covers real streamed cross-key COPY source/destination, self metadata replacement and tags. Twenty-four events bind actual ACK and physical-tail completion before releasing shared reconstruction, exact stale refusal or unchanged-source repair, final source/target/control payloads, metadata and tags. The registry binds all five storage fixture/barrier source files. This candidate admits A09, G05 and G06 only with their distinct complete case-specific native group and matching hosted helper/source artifact. Both workflow and executor run read-only preflight before bootstrap. A09 always sends wire `nolock=true` and requires effective `nolock=false`, plus both original H1/H2 public branches and the separate storage-native proof. G05 uses the separate same-key runtime described below. The G06 public adapter uses the existing DS workload: all 257 original keys, including Chinese, spaces, literal `%2F`, actual slashes and parent-like prefixes. The observed missing key is `中文/文件/0252.bin`; its companion is `literal%2Fslash/0254.bin`. `copy-source` copies from the observed key into the companion; `copy-destination` copies in the reverse direction. Default COPY expectations inherit the source payload, content properties, metadata and tags. Both directions verify the unchanged source, exact destination coding-index shard bytes and a distinct destination generation before GET, rejecting retained old destination directories. Registry, fixture and local transition tests do not establish public overlap or complete canonical PASS. @@ -688,7 +688,11 @@ The caller supplies a fresh versioned owned bucket. One completed three-part obj Immutable native source `09fb813a003e7964bd1b893efb7446a3c41bd3bf` adds `multipart-continue-complete-abort-final-rename`: ten ordered milestones across 4/2 and 16/4 logical geometries, using real three-part commits, same-key A/B pending parts and a paused shared Deep Heal. Completion tails drain before stale Heal release; exact retry disposition, old/sibling preservation, new committed bytes and upload retirement are asserted. The native test's unversioned logical scope is supplemental to the full versioned SDK/fleet case. -Its native registry entry permits collecting this evidence only. G05 runtime admission remains closed until the executor integrates the complete same-key model, all pending/committed physical part witnesses, real public overlap and both original H1/H2 instances. Component tests and native bundles do not add canonical PASS. +The historical `09fb813` entry remains collector-only. Runtime admission binds source `757b751482bb970bc321e92059690a7698554358`, its complete multipart native group and the exact hosted helper artifact. The old `072b57b` source is rejected before bootstrap. Component tests and native bundles do not add canonical PASS. + +`mpu_race_runner.py` replaces the generic unversioned MPU path with the frozen same-key fixture and schedule for all three public branches in H1/H2. It records 256 versioned flat-key controls, one committed three-part version and A/B pending uploads through the existing durable `Recorder`. The 257 faults target the fourth physical drive on the first host. All 259 original and 262 final part cohorts receive full streaming verification. Every companion ACK has an immediate independent staging checkpoint; only the selected first SDK operation requires a contained original running-token sample. The status transport preserves the frozen warmup, 1024-sample/90-second budget and 5ms poll pause, restricts requests to the owned bucket and original token, and records request/response persistence phases. Kernel I/O sampling reads the owned cgroup and its ancestors without setting or releasing a cap; it makes no continuous no-limit claim. + +Final committed/staging/control proofs precede all-peer S3 verification. Assertion failures retain the frozen source's remaining independent diagnostics while evidence I/O errors stop execution. Repeated terminal responsibility, ACK bounds, post-GET physical stability and exact original-root retirement must pass before cleanup authority exists. Any issued seed request without that authority retains the stopped raw MPU prefix, committed scene and original request/ACK files as DIRTY, with no retry, abort, restore or namespace cleanup. A missing overlap remains `NOT_OBSERVED` even after safe cleanup. Full hosted native acceptance and all six public live instances still require independent evidence before G05 can count as canonical PASS. The additive `mpu-race-scene` node action in `mpu_race_physical.py` adapts the frozen `heal_h1_mpu_physical.py` upload-ID and byte-continuity oracle to the current owned four-drive hosts. Each request binds the manual runtime, bucket, candidate binary and all three ACK-selected uploads. It inventories the complete fixed key prefix, hashes only selected staging files under bounded entry/depth/byte limits, rejects unknown uploads and links, and preserves original process identity during a live capture. Stopped captures additionally verify the owned service remains stopped. Across H1/H2, pending part/sidecar populations and existing sibling bytes remain exact; a completed upload must lose its staging directory and retain all three previously witnessed encoded parts in committed storage. These are staging/continuity witnesses; independent inspector coding/bitrot checks, product Heal proof, real overlap and full SDK inventories remain required. diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index d14ccc1b..36115c92 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -188,7 +188,7 @@ def remote(self, node, action, **params): if action in {"mpu-race-backups", "version-race-backups"}: source += "\nimport types\nmpu_race_backups_backend = types.ModuleType('mpu_race_backups')\nexec(" + repr( NODE_SOURCE.with_name("mpu_race_backups.py").read_text()) + ", mpu_race_backups_backend.__dict__)\n" - if action == "race-io-cap": + if action in {"race-io-cap", "race-io-observe"}: source += "\nimport types\nrace_io_backend = types.ModuleType('race_io')\nexec(" + repr( NODE_SOURCE.with_name("race_io.py").read_text()) + ", race_io_backend.__dict__)\n" if action == "mpu-race-scene": diff --git a/scripts/validation_heal/mpu_race_runner.py b/scripts/validation_heal/mpu_race_runner.py new file mode 100644 index 00000000..6d41b459 --- /dev/null +++ b/scripts/validation_heal/mpu_race_runner.py @@ -0,0 +1,384 @@ +"""Frozen G05 same-key multipart execution; runtime admission is owned by race.py.""" +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +import secrets +import time +from urllib.parse import quote +import uuid + +try: + from . import race, mpu_race_fixture as model, mpu_race_schedule as schedule + from . import mpu_race_physical as staging, mpu_race_committed as business + from .heal_probe import Probe, canonical, compare_inventories, observation_phase, payload, require, save + from .heal_workload import Recorder, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .cursor import validate_missing_batch, verify_part_manifest + from .shards import files, validate_repaired_physical + from .heal_fault import bounded_fault_id + from .status_keepalive import KeepAliveStatusProbe + from .checkpoint import records as journal_records + from .persistence import exact_records +except ImportError: + import race, mpu_race_fixture as model, mpu_race_schedule as schedule + import mpu_race_physical as staging, mpu_race_committed as business + from heal_probe import Probe, canonical, compare_inventories, observation_phase, payload, require, save + from heal_workload import Recorder, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from cursor import validate_missing_batch, verify_part_manifest + from shards import files, validate_repaired_physical + from heal_fault import bounded_fault_id + from status_keepalive import KeepAliveStatusProbe + from checkpoint import records as journal_records + from persistence import exact_records + + +CASE = 'heal.race-multipart-complete-abort' + + +def observe_status(probe, fleet, path, token, reference): + with probe.status_observation() as measured: + require(measured is not None, 'G05 warm/race status requires its original keepalive session') + request_value = {'path': path, 'query': [('clientToken', token)], 'method': 'POST', 'body': '', + 'statusObservation': {'sessionId': measured['sessionId'], 'sequence': measured['sequence']}} + with observation_phase(measured, 'requestPersistence'): + fleet.record('G05-status-request', request_value) + request_file = f'{fleet.sequence:05d}-G05-status-request.json' + with observation_phase(measured, 'requestEvidenceHash'): + request_sha = hashlib.sha256((fleet.evidence / request_file).read_bytes()).hexdigest() + value = probe.admin(path, [('clientToken', token)], b'', observation=measured) + value['requestSha256'] = request_sha + with observation_phase(measured, 'responsePersistence'): + fleet.record('G05-status-response', value) + response_file = f'{fleet.sequence:05d}-G05-status-response.json' + measured.update(requestFile=request_file, responseFile=response_file, responsePersisted=True) + race.settings(value, reference) + return value + + +def ack_records(recorder, expected=None): + requests = sorted(recorder.output.glob('*-request.json')) + acks = sorted(recorder.output.glob('*-ack.json')) + count = recorder.sequence if expected is None else expected + require(len(requests) == len(acks) == recorder.sequence == count, + 'G05 SDK journal has an orphan, missing request/ACK or wrong population') + result = [] + for sequence, (request_path, ack_path) in enumerate(zip(requests, acks, strict=True), 1): + request, ack = json.loads(request_path.read_text()), json.loads(ack_path.read_text()) + observation = ack.get('observation') or {} + require(request_path.name == f'{sequence:06d}-request.json' and ack_path.name == f'{sequence:06d}-ack.json' + and request.get('runId') == ack.get('runId') == recorder.probe.binding['runId'] + and request.get('sequence') == ack.get('sequence') == sequence + and ack.get('requestFile') == request_path.name + and ack.get('requestSha256') == hashlib.sha256(request_path.read_bytes()).hexdigest() + and ack.get('operationCheck') == 'PASS' + and observation.get('operation') == request.get('operation') + and observation.get('params') == request.get('params') and observation.get('resultUnknown') is False, + 'G05 original SDK request/ACK pairing, identity or result differs') + result.append(ack) + return result + + +def seed(recorder, bucket, branch): + require(branch in schedule.SELECTED, 'unregistered G05 branch') + recorder.call('create_bucket', {'Bucket': bucket}, expect_http=200) + recorder.call('put_bucket_versioning', {'Bucket': bucket, 'VersioningConfiguration': {'Status': 'Enabled'}}, expect_http=200) + controls = [] + for index in range(256): + key = f'objects-{index:04d}.bin'; body = payload(bucket + '/' + key, race.SIZE) + response = recorder.call('put_object', {'Bucket': bucket, 'Key': key}, body, expect_http=200) + version = response.get('VersionId') + require(isinstance(version, str) and str(uuid.UUID(version)) == version and uuid.UUID(version).int != 0 + and response.get('ETag') == model.etag(body), 'G05 control lacks its real version and exact payload ETag') + controls.append({'key': key, 'versionId': version, 'isLatest': True, 'deleteMarker': False, + 'payload': {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}}) + fixture = model.MpuRaceFixture(recorder, bucket) + prepared = schedule.prepare(fixture, branch) + ack_records(recorder, 270 + int(branch != 'upload-continue')) + return controls, fixture, prepared + + +def identities(manifest): + return [{'role': role, 'uploadId': item['uploadId'], 'bucket': manifest['bucket'], 'key': manifest['key']} + for role, item in manifest['items'].items()] + + +def capture_staging(fleet, manifest, binary_sha, *, quiescent): + return {node: fleet.remote(node, 'mpu-race-scene', bucket=manifest['bucket'], items=identities(manifest), + expectedBinarySha256=binary_sha, quiescent=quiescent) for node in fleet.nodes} + + +def observe_io(fleet, branch): + for node in fleet.nodes: + process = fleet.record('G05-io-process', {'node': node, **fleet.remote(node, 'process')}) + value = fleet.record('G05-kernel-no-cap-sample', {'node': node, **fleet.remote(node, 'race-io-observe', + caseId=CASE, branch=branch, expectedProcess={k: v for k, v in process.items() if k != 'node'})}) + require(value.get('inspectionOnly') is True and value.get('mutated') is False + and value.get('actualKernelNoLimitVerifiedAtSamples') is True + and value.get('continuousNoLimitProved') is False, 'G05 kernel IO observation is incomplete') + + +def foreground(fixture, prepared, probe, fleet, path, token, reference, binary_sha): + """Only the selected first SDK call needs a contained original-token sample.""" + samples = [] + def checkpoint(label, manifest): + physical = capture_staging(fleet, manifest, binary_sha, quiescent=False) + value = {'phase': label, 'manifest': manifest, 'physical': physical} + # Worker uses independent names; Fleet.record's sequence stays on the controller. + save(fleet.evidence / 'mpu-checkpoints' / (label + '.json'), value) + return value + with ThreadPoolExecutor(max_workers=1) as executor: + worker = executor.submit(schedule.execute, fixture, prepared['branch'], prepared, checkpoint) + deadline = time.monotonic() + 90 + try: + while not worker.done() and time.monotonic() < deadline and len(samples) < 1024: + sample = observe_status(probe, fleet, path, token, reference); samples.append(sample) + with observation_phase(probe.observation_for(sample), 'pollPause'): + time.sleep(.005) + if not worker.done() and time.monotonic() >= deadline: + fixture.recorder.mutations.poison('G05 original observation deadline exceeded; drain only') + except BaseException: + fixture.recorder.mutations.poison('G05 status observation failed; drain only') + raise + result = worker.result(timeout=90) + records = ack_records(fixture.recorder, 3 if prepared['branch'] == 'upload-continue' else 2) + selected = records[0] + require(selected['sequence'] == result['selectedAckSequence'] + and selected['observation']['operation'] == prepared['selectedOperation'] + and fixture.poisoned is None, 'G05 selected first mutation differs from original schedule') + matches = [sample['statusObservation'] for sample in samples + if race.sample_overlap(selected, sample, path, token, reference)] + proof = {'selectedOperation': prepared['selectedOperation'], 'selectedAckSequence': selected['sequence'], + 'matchingStatusSamples': matches, 'sampleCount': len(samples), 'sampleBudget': 1024, + 'pollPauseSeconds': .005, 'allCompanionACKsVerified': True, 'publicCallOverlapObserved': bool(matches), + 'nativeFenceProved': False, 'serverCommitOverlapProved': False, 'canonicalCasePassClaimed': False} + fleet.record('G05-selected-public-overlap', proof) + return result, bool(matches) + + +def wait_terminal(probe, fleet, path, token, reference): + deadline = time.monotonic() + 180 + while True: + value = observe_terminal(probe, fleet, path, token, reference) + if value['response']['summary'] != 'running': return value['response'] + require(time.monotonic() < deadline, 'G05 original root completion exceeded180seconds') + time.sleep(.1) + + +def observe_terminal(probe, fleet, path, token, reference): + value = fleet.record('G05-original-terminal-status', probe.admin(path, [('clientToken', token)], b'')) + require(value.get('httpStatus') == 200 and isinstance(value.get('response'), dict) + and value['response'].get('summary') in {'running', 'finished', 'stopped'} + and canonical(value['response'].get('settings')) == canonical(reference['effective']), + 'G05 original terminal status identity/settings unavailable') + # A stopped root remains a failure in terminal_proof. Preserve it so the + # frozen source's independent final diagnostics still run. + return value + + +def simple_entries(controls, manifest): + return controls + [{key: value for key, value in row.items() if key != 'multipartParts'} + for row in business.entries(manifest)] + + +def physical_population(snapshot, bucket, controls): + keys = {row['key'] for row in controls} | {model.KEY} + require(all(row['path'].startswith(bucket + '/') and row['path'].split('/', 2)[1] in keys + for value in snapshot.values() for row in value['files']), + 'G05 physical population contains an unacknowledged object') + + +def final_checks(probe, fleet, binding, controls, fixture, original, executed, before, before_staging, + mapped_controls, baseline, terminal, path, token, reference, binary_sha, inspector_sha): + """Keep independent frozen-source diagnostics after assertion failures. + + Evidence I/O errors remain fatal. No failed assertion can authorize cleanup, + blind restoration or a later success claim. + """ + violations = [] + def check(label, operation): + try: + proof = operation() + except ValueError as error: + proof = {'state': 'FAIL', 'error': str(error), 'passClaimed': False} + violations.append({'stage': label, 'error': str(error)}) + fleet.record(label, proof) + return proof + + def repeat(): + for _ in range(2): + repeated = observe_terminal(probe, fleet, path, token, reference)['response'] + require(repeated['summary'] == terminal['summary'] + and canonical(repeated.get('outcome')) == canonical(terminal.get('outcome')), + 'G05 original terminal responsibility changed on repetition') + return {'terminalStable': True} + check('G05-terminal-repeat', repeat) + fleet.stop(); after = fleet.snapshot('product-physical-before-GET-or-rescue', quiescent=True) + check('G05-final-physical-population', lambda: physical_population(after, fixture.bucket, controls)) + def verify_controls(): + for target in mapped_controls.values(): + validate_repaired_physical(race.object_scene(before, fixture.bucket, target['key']), + race.object_scene(after, fixture.bucket, target['key']), target) + return {'physicalControlsReconstructed': 256} + check('G05-control-reconstruction', verify_controls) + backups = fleet.record('G05-original-backup-decoding', {node: fleet.remote(node, 'mpu-race-backups', bucket=fixture.bucket, + expectedBinarySha256=binary_sha, expectedInspectorSha256=inspector_sha) for node in fleet.nodes}) + final = executed['manifest'] + def verify_committed(): + _, proof = business.transition(before, after, original, final, backups, binary_sha, inspector_sha) + return proof + physical = check('G05-committed-transition', verify_committed) + previous = before_staging + for step in executed['steps']: + checkpoint = step['checkpoint'] + check('G05-staging-' + step['phase'], lambda: staging.verify(checkpoint['manifest'], checkpoint['physical'], + fleet.nodes, binary_sha, previous=previous, committed=after)) + previous = checkpoint['physical'] + def full_streams(): + final_controls = race.map_fixture(after, fixture.bucket, controls) + # Independently map even when another committed-transition assertion failed. + old_dir = business.mapped(before, fixture.bucket, business.entries(original))[0][ + original['items']['committed']['versionId'], 1]['dataDir'] + final_mpu = business.mapped(after, fixture.bucket, business.entries(final), backup_directories=[old_dir])[0] + return verify_part_manifest(fleet, list(final_controls.values()) + list(final_mpu.values()), 'final-streaming', count=262) + check('G05-final-full-streams', full_streams) + fleet.start(); inventory = None + for endpoint in binding['endpoints']: + peer = Probe(binding, endpoint) + reader = Recorder(peer, fleet.evidence / ('final-read-ACKs-' + endpoint)) + check('G05-all-peer-multipart-S3-' + endpoint, lambda: fixture.verify_s3(reader)) + current = fleet.record('all-peer-final-S3', {'endpoint': endpoint, **peer.inventory(fixture.bucket)}) + check('G05-final-payload-' + endpoint, lambda: verify_seed(simple_entries(controls, final), current)) + check('G05-control-properties-' + endpoint, lambda: compare_inventories( + {'bucket': fixture.bucket, 'entries': [row for row in baseline['entries'] if row['key'] != model.KEY]}, + {'bucket': fixture.bucket, 'entries': [row for row in current['entries'] if row['key'] != model.KEY]})) + if inventory is not None: + check('G05-peer-inventory-' + endpoint, lambda: compare_inventories(inventory, current)) + inventory = current + check('G05-final-read-ledger-' + endpoint, lambda: ack_records(reader)) + fleet.stop() + def unchanged(): + require(physical_content(after) == physical_content(fleet.snapshot('physical-after-S3', quiescent=True)), + 'G05 final full GET changed the already-proven physical result') + return {'postGetPhysicalUnchanged': True} + check('G05-post-GET-physical', unchanged) + terminal_proof = check('G05-original-terminal-proof', lambda: race.terminal_proof(terminal, fixture.bucket, baseline, inventory)) + def retired(): + roots = exact_records(journal_records(fleet, token, 'G05-original-root-retirement', quiescent=True, family='root'), token, 'root') + require(not roots, + 'G05 original runnable root responsibility was not retired') + return {'originalRootRetired': True} + check('G05-root-retirement-proof', retired) + fleet.record('G05-contract-violations', violations) + require(not violations, 'G05 final contracts failed; independent diagnostics retained') + return physical, terminal_proof + + +def run_branch(args, topology, nodes, branch): + require(args.case_id == CASE and branch in schedule.SELECTED and len(nodes) == (1 if topology == 'H1' else 4), + 'G05 runner requires its original case/branch and H1/H2 hosts') + candidate = json.loads(args.candidate.read_text()) + require(candidate['source_sha'] == race.COPY_CANDIDATE, 'G05 same-key runner requires its reviewed native candidate') + reference = race.COPY_REFERENCE + directory = args.evidence / topology / branch; directory.mkdir(parents=True, exist_ok=False) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile='manual') + inspector = fleet.record('inspector-build', verify_inspector(args.tool_directory, candidate)) + bucket = 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] + access, secret = secrets.token_hex(16), secrets.token_hex(24) + os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) + binding = {'schemaVersion': 1, 'runId': run_id, 'jumpHostOnly': True, 'exclusiveHealService': True, + 'leaseExpiresAt': (datetime.now(timezone.utc) + timedelta(minutes=45)).isoformat(), 'bucketPrefix': bucket, + 'endpoints': {f'node{i}': f'http://rustfs-node{i}:9000' for i in range(1, len(nodes) + 1)}, + 'defaultEndpoint': 'node1', 'candidateCommit': candidate['source_sha'], 'harnessCommit': args.testing_commit, + 'candidatePackageSha256': candidate['package_sha256'], 'candidateBinarySha256': args.binary_sha, + 'rcBinarySha256': hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + 'maxInventoryObjects': 512, 'maxInventoryBytes': 512 * 1024**2} + fleet.record('binding', binding) + fleet.record('source-contract', {'caseId': CASE, 'branch': branch, 'objects': 257, 'fixture': 'DM', 'fault': 'F1', + 'sameKey': model.KEY, 'requested': reference['requested'], 'effective': reference['effective'], + 'processIoCapRequested': False, 'nativeFenceProved': False, 'canonicalCasePassClaimed': False}) + probe = KeepAliveStatusProbe(binding) + result = {'caseId': CASE, 'topology': topology, 'branch': branch, 'fixture': 'DM', 'fault': 'F1', + 'verdict': 'RUNNING', 'cleanup': 'PENDING', 'canonicalCasePassClaimed': False} + seed_recorder, target_authority = None, False + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record('topology-proof', validate_topology(fleet.snapshot('deployed-topology'), topology)) + seed_recorder = Recorder(probe, directory / 'seed-ACKs') + controls, fixture, prepared = seed(seed_recorder, bucket, branch) + original = prepared['manifest'] + reader = Recorder(probe, directory / 'mpu-read-ACKs') + fleet.record('G05-original-multipart-S3', fixture.verify_s3(reader)) + fleet.record('G05-baseline-read-ledger', ack_records(reader)) + baseline = fleet.record('complete-baseline-S3', probe.inventory(bucket)) + verify_seed(simple_entries(controls, original), baseline) + before_staging = fleet.record('G05-original-staging', capture_staging(fleet, original, args.binary_sha, quiescent=False)) + staging.verify(original, before_staging, nodes, args.binary_sha) + fleet.stop(); before = fleet.snapshot('complete-baseline-physical', quiescent=True) + physical_population(before, bucket, controls) + mapped_controls = race.map_fixture(before, bucket, controls) + mapped_mpu, _ = business.mapped(before, bucket, business.entries(original)) + baseline_parts = list(mapped_controls.values()) + list(mapped_mpu.values()) + fleet.record('G05-baseline-full-streams', verify_part_manifest(fleet, baseline_parts, 'baseline-streaming', count=259)) + committed = mapped_mpu[original['items']['committed']['versionId'], 1] + selected = next(part for part in committed['cohort'] if part['node'] == nodes[0] and part['disk'] == 3) + targets = list(mapped_controls.values()) + [{**committed, **selected}] + require(len(targets) == 257 and all(target['node'] == nodes[0] and target['disk'] == 3 for target in targets), + 'G05 faults must cover all 257 original objects on the fourth drive of the first host') + receipts = [] + for index, target in enumerate(targets): + params = {'disk': target['disk'], 'faultId': bounded_fault_id('G05', branch, index)} + fleet.faults.append((target['node'], params)) + receipts.append(fleet.record('exact-missing-part', {'node': target['node'], **fleet.remote(target['node'], + 'apply-fault', **params, relativePath=target['relativePath'], expectedSha256=target['sha256'], kind='missing')})) + damaged = fleet.snapshot('complete-fault-physical', quiescent=True) + fleet.record('fault-batch-proof', validate_missing_batch(before, damaged, targets, receipts)) + fleet.start(); observe_io(fleet, branch) + require(physical_content(damaged) == physical_content(fleet.snapshot('attribution-before-START')), + 'G05 missing shards changed before original manual admission') + path = '/rustfs/admin/v3/heal/' + quote(bucket, safe='') + start = fleet.record('original-Admin-start', probe.admin(path, [], json.dumps(reference['requested']).encode())) + token = start.get('response', {}).get('clientToken') if isinstance(start.get('response'), dict) else None + require(start.get('httpStatus') == 200 and isinstance(token, str) and str(uuid.UUID(token)) == token, + 'G05 admission lacks one original canonical token') + with probe.status_session(token, path) as transport: + try: + observe_status(probe, fleet, path, token, reference) + witness = fleet.snapshot('G05-target-window') + missing = (selected['node'], selected['disk'], selected['relativePath']) not in files(witness) + fixture.recorder = Recorder(probe, directory / 'foreground-ACKs') + executed, overlapped = foreground(fixture, prepared, probe, fleet, path, token, reference, args.binary_sha) + finally: + fleet.record('G05-status-transport', transport) + observe_io(fleet, branch) + terminal = wait_terminal(probe, fleet, path, token, reference) + physical, result['terminal'] = final_checks(probe, fleet, binding, controls, fixture, original, executed, before, + before_staging, mapped_controls, baseline, terminal, path, token, reference, args.binary_sha, inspector['binarySha256']) + target_authority = True; fleet.faults.clear() + fleet.record('last-ACK-physical-proof', physical) + if not (missing and overlapped): raise race.WindowNotObserved('G05 selected missing part and SDK running-token overlap not both observed') + result.update(verdict='PASS', publicCallOverlapObserved=True, nativeFenceProved=False, physical=physical) + except (Exception, KeyboardInterrupt) as error: + result.update(verdict='NOT_OBSERVED' if isinstance(error, race.WindowNotObserved) else 'FAIL', + failure={'type': type(error).__name__, 'message': str(error)}) + finally: + issued = seed_recorder is not None and seed_recorder.sequence > 0 + if issued and not target_authority: + try: + fleet.stop(); fleet.snapshot('retained-failure-scene', quiescent=True) + fleet.record('raw-mpu-failure-scene', {node: fleet.remote(node, 'mpu-scene', bucket=bucket, + keys=[model.KEY], expectedBinarySha256=args.binary_sha) for node in nodes}) + except Exception as error: + fleet.record('failure-scene-error', {'type': type(error).__name__, 'message': str(error)}) + fleet.record('last-ACK-rescue-required', {'state': 'DIRTY', 'namespacesRetained': True, + 'lastACKPhysicalAuthorityVerified': False, 'originalGenerationBlindRestorationProhibited': True}) + result['cleanup'] = 'FAIL' + else: + result['cleanup'] = 'PASS' if fleet.cleanup() else 'FAIL' + fleet.record('branch-result', result) + return result diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index b2ca2516..eb10f1b2 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -69,8 +69,6 @@ REFERENCE_KEY = "objects/0000.bin" G06_TARGET_KEY = "中文/文件/0252.bin" G06_REFERENCE_KEY = "literal%2Fslash/0254.bin" -G05_TARGET_KEY = "multipart/target.bin" -G05_PART_SIZES = (5 * 1024 * 1024 + 17, 5 * 1024 * 1024 + 93, 1024 * 1024 + 47) REFERENCE = { "candidateCommit": "072b57b37559fbe626aeda8abf61617fdcbdd926", "sourceFiles": { @@ -106,7 +104,8 @@ MARKER_CANDIDATE: {'reference': MARKER_REFERENCE, 'cases': ('heal.race-unversioned-overwrite', 'heal.race-versioned-put', 'heal.race-version-delete', 'heal.race-markers-and-batch-delete')}, - COPY_CANDIDATE: {'reference': COPY_REFERENCE, 'cases': ('heal.race-copy-and-metadata-update',)}, + COPY_CANDIDATE: {'reference': COPY_REFERENCE, + 'cases': ('heal.race-copy-and-metadata-update', 'heal.race-multipart-complete-abort')}, } # Registry subsets do not enable a fleet case. Its reviewed runtime source and # complete case-specific native group must both be present before bootstrap. @@ -116,12 +115,14 @@ 'heal.race-version-delete': ['explicit-version-delete-final-rename'], 'heal.race-markers-and-batch-delete': ['marker-and-batch-delete-final-rename'], 'heal.race-copy-and-metadata-update': ['copy-and-metadata-final-rename'], + 'heal.race-multipart-complete-abort': ['multipart-continue-complete-abort-final-rename'], } def source_contract(candidate, case_id): require(case_id in CASE_IDS, 'unregistered race case') if candidate.get('source_sha') == REFERENCE['candidateCommit']: + require(case_id != 'heal.race-multipart-complete-abort', 'G05 requires its exact-source multipart native group') return REFERENCE entry = NATIVE_RUNTIME.get(candidate.get('source_sha')) require(entry is not None and case_id in entry['cases'] and case_id in FULL_NATIVE_GROUPS @@ -147,7 +148,6 @@ class WindowNotObserved(ValueError): def target_key(branch): - if branch in G05_BRANCHES: return G05_TARGET_KEY return G06_TARGET_KEY if branch in G06_BRANCHES else TARGET_KEY @@ -233,7 +233,7 @@ def sample_overlap(ack, observed, path, token, reference=REFERENCE): def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): - require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES, *G05_BRANCHES, *G06_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G02/G03/G04/G05/G06 unregistered fixture") + require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES, *G06_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G02/G03/G04/G05/G06 unregistered fixture") if branch in G06_BRANCHES: require(count == OBJECTS, "G06 requires the frozen complete 257-key DS population") return seed_workload(recorder, bucket, "DS", count, [SIZE]) @@ -270,31 +270,6 @@ def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): if row["key"] == TARGET_KEY: row["isLatest"] = False entries.append({"key": TARGET_KEY, "versionId": marker_id, "isLatest": True, "deleteMarker": True}) return entries - if branch in G05_BRANCHES: - entries = [] - for index in range(count - 1): - key = f"objects/{index:04d}.bin" - body = payload(f"{bucket}/{key}/0", SIZE) - response = recorder.call("put_object", {"Bucket": bucket, "Key": key, - "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": "DM", "nonce": str(index)}, - "Tagging": "suite=heal&fixture=DM"}, body) - entries.append({"key": key, "versionId": public_version_id(response.get("VersionId")), - "isLatest": True, "deleteMarker": False, - "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) - upload = recorder.call("create_multipart_upload", {"Bucket": bucket, "Key": G05_TARGET_KEY, - "Metadata": {"heal-fixture": "DM", "multipart": "target"}, "ContentType": "application/octet-stream"}) - parts, digest = [], hashlib.sha256() - for number, size in enumerate(G05_PART_SIZES, 1): - body = payload(f"{bucket}/{G05_TARGET_KEY}/{number}", size); digest.update(body) - response = recorder.call("upload_part", {"Bucket": bucket, "Key": G05_TARGET_KEY, - "UploadId": upload["UploadId"], "PartNumber": number}, body) - parts.append({"PartNumber": number, "ETag": response["ETag"]}) - completed = recorder.call("complete_multipart_upload", {"Bucket": bucket, "Key": G05_TARGET_KEY, - "UploadId": upload["UploadId"], "MultipartUpload": {"Parts": parts}}) - entries.append({"key": G05_TARGET_KEY, "versionId": public_version_id(completed.get("VersionId")), - "isLatest": True, "deleteMarker": False, - "payload": {"size": sum(G05_PART_SIZES), "sha256": digest.hexdigest()}}) - return entries if branch in (*G02_BRANCHES, *G03_BRANCHES): recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) entries = [] @@ -315,7 +290,7 @@ def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): def operation_plan(bucket, branch, expected): - require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES, *G05_BRANCHES, *G06_BRANCHES), "G01/G02/G03/G04/G05/G06 unregistered foreground branch") + require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES, *G06_BRANCHES), "G01/G02/G03/G04/G05/G06 unregistered foreground branch") if branch in G01_BRANCHES: # Three distinct serial commits; the final independent reference was # written before injection, allowing a coding-index byte comparison. @@ -355,38 +330,6 @@ def operation_plan(bucket, branch, expected): "Quiet": False}}, "nonce": "batch-version-delete"}] return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": TARGET_KEY}, "nonce": "unversioned-delete"}] - if branch in G05_BRANCHES: - # Each branch uses a fresh UploadId for the same already committed - # object. The foreground resolver binds every later request to the - # exact create ACK and never guesses an UploadId from LIST output. - if branch == "upload-continue": - return [ - {"operation": "create_multipart_upload", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY}, - "uploadName": "continue"}, - {"operation": "upload_part", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY, - "UploadIdRef": "continue", "PartNumber": 1}, "nonce": f"{bucket}/{G05_TARGET_KEY}/continue/1", - "size": G05_PART_SIZES[0]}, - ] - if branch == "complete": - return [ - {"operation": "create_multipart_upload", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY}, - "uploadName": "complete"}, - *[{"operation": "upload_part", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY, - "UploadIdRef": "complete", "PartNumber": number}, - "nonce": f"{bucket}/{G05_TARGET_KEY}/complete/{number}", "size": size} - for number, size in enumerate(G05_PART_SIZES, 1)], - {"operation": "complete_multipart_upload", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY, - "UploadIdRef": "complete", "MultipartUpload": {"Parts": []}}, "uploadName": "completed"}, - ] - return [ - {"operation": "create_multipart_upload", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY}, - "uploadName": "abort"}, - {"operation": "upload_part", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY, - "UploadIdRef": "abort", "PartNumber": 1}, "nonce": f"{bucket}/{G05_TARGET_KEY}/abort/1", - "size": G05_PART_SIZES[0]}, - {"operation": "abort_multipart_upload", "params": {"Bucket": bucket, "Key": G05_TARGET_KEY, - "UploadIdRef": "abort"}, "uploadName": "aborted"}, - ] if branch in G06_BRANCHES: if branch == "copy-source": return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": current_reference, @@ -439,7 +382,6 @@ def version_put_entries(baseline, plan, responses): def final_inventory(baseline, branch, plan, responses): expected_length = (3 if branch in (*G01_BRANCHES, *G02_BRANCHES) - else len(plan) if branch in G05_BRANCHES else 16 if branch == "update-tags" else 1) require(len(plan) == len(responses) == expected_length, "G01/G03/G05 foreground plan/ACK sequence is incomplete") @@ -491,27 +433,6 @@ def final_inventory(baseline, branch, plan, responses): require(responses[0].get("VersionId") in {None, "null"} and responses[0].get("DeleteMarker") in {None, False}, "G04 unversioned delete ACK changed object identity") result["entries"] = [row for row in result["entries"] if row["key"] != TARGET_KEY] - elif branch in G05_BRANCHES: - runtime = plan[0].get("runtime") or {} - upload_id = runtime.get("uploadIds", {}).get(branch.split("-", 1)[-1]) - require(isinstance(upload_id, str) and upload_id, "G05 foreground lost its exact UploadId") - if branch == "complete": - entry = next(row for row in result["entries"] if row["key"] == current_target) - digest = hashlib.sha256() - total = 0 - for number, size in enumerate(G05_PART_SIZES, 1): - body = payload(f"{plan[0]['params']['Bucket']}/{G05_TARGET_KEY}/complete/{number}", size) - digest.update(body); total += len(body) - ack = responses[-1] - require(isinstance(ack.get("ETag"), str) and ack["ETag"], - "G05 Complete ACK lacks its committed ETag") - entry.update(payload={"size": total, "sha256": digest.hexdigest()}, - versionId=public_version_id(ack.get("VersionId")), isLatest=True, - deleteMarker=False) - entry.setdefault("properties", {}).update(ETag=ack["ETag"], ContentLength=total, - ContentType="application/octet-stream") - else: - require(branch in {"upload-continue", "abort"}, "G05 unknown MPU branch") elif branch in G06_BRANCHES: entry = next(row for row in result["entries"] if row["key"] == current_target) copy_result = responses[-1].get("CopyObjectResult") or {} @@ -661,21 +582,6 @@ def coding(target): version["isLatest"], version["deleted"]) for version in versions) require(actual_versions == expected_versions, "G04 physical data/marker version set differs from exact foreground ACKs") - elif branch in G05_BRANCHES: - require(len(original_target) == len(current_target_values) == 1, - "G05 multipart transition changed the target identity count") - if branch == "complete": - require(original_target[0]["dataDir"] != current_target_values[0]["dataDir"], - "G05 Complete retained the old physical generation") - require(len(current_target_values[0]["cohort"]) == len(original_target[0]["cohort"]) - and all(part["size"] > 0 for part in current_target_values[0]["cohort"]), - "G05 Complete physical parts are incomplete") - else: - require(original_target[0]["dataDir"] == current_target_values[0]["dataDir"], - "G05 pending/aborted MPU changed the committed object generation") - require([(part["index"], part["sha256"], part["size"]) for part in original_target[0]["cohort"]] - == [(part["index"], part["sha256"], part["size"]) for part in current_target_values[0]["cohort"]], - "G05 pending/aborted MPU changed committed object bytes") elif branch in G06_BRANCHES: target_after = next(value for (key, _), value in survivors.items() if key == changed_key) target_before = next(value for (key, _), value in mapped.items() if key == changed_key) @@ -814,37 +720,15 @@ def wait_version_window(probe, fleet, path, token, reference, before, targets, b def foreground(recorder, plan, *, committed=None): responses = [] - upload_ids = {} - part_etags = {} for step in plan: - params = copy.deepcopy(step["params"]) - upload_ref = params.pop("UploadIdRef", None) - if upload_ref is not None: - require(upload_ref in upload_ids, "G05 foreground referenced an unacknowledged UploadId") - params["UploadId"] = upload_ids[upload_ref] - if step["operation"] == "complete_multipart_upload": - parts = [{"PartNumber": number, "ETag": part_etags[(upload_ref, number)]} - for number, _ in enumerate(G05_PART_SIZES, 1)] - params["MultipartUpload"] = {"Parts": parts} - body = None - if step["operation"] == "put_object": - body = payload(step["nonce"], SIZE) - elif step["operation"] == "upload_part": - body = payload(step["nonce"], step["size"]) - response = recorder.call(step["operation"], params, body) + require(step['operation'] not in {'create_multipart_upload', 'upload_part', 'complete_multipart_upload', + 'abort_multipart_upload'}, 'G05 requires its same-key ACK model and schedule') + body = payload(step['nonce'], SIZE) if step['operation'] == 'put_object' else None + response = recorder.call(step['operation'], copy.deepcopy(step['params']), body) responses.append(response) if committed is not None: committed(responses) - if step.get("uploadName") in {"continue", "complete", "abort"}: - require(isinstance(response.get("UploadId"), str) and response["UploadId"], - "G05 CreateMultipartUpload ACK lacks UploadId") - upload_ids[step["uploadName"]] = response["UploadId"] - if step["operation"] == "upload_part": - require(isinstance(response.get("ETag"), str) and response["ETag"], - "G05 UploadPart ACK lacks ETag") - part_etags[(upload_ref, params["PartNumber"])] = response["ETag"] - plan[0]["runtime"] = {"uploadIds": upload_ids, "partEtags": part_etags} - return responses, {"uploadIds": upload_ids, "partEtags": part_etags} + return responses, {} def read_versions(recorder, ledger, lock, done): @@ -955,39 +839,15 @@ def read_repair(): return responses, all(overlaps) and read_complete and reader_overlap -def verify_multipart_state(probe, fleet, bucket, branch, runtime, expected_parts): - """Bind MPU control evidence to the foreground UploadId/part ACKs.""" - require(branch in G05_BRANCHES and isinstance(runtime, dict), "G05 runtime receipt is missing") - name = branch.split("-", 1)[-1] - upload_id = (runtime.get("uploadIds") or {}).get(name) - require(isinstance(upload_id, str) and upload_id, "G05 runtime UploadId is missing") - listed = fleet.record("G05-list-multipart-uploads", probe.s3("list_multipart_uploads", {"Bucket": bucket})) - uploads = (listed.get("response") or {}).get("Uploads", []) - matching = [row for row in uploads if row.get("UploadId") == upload_id] - if branch == "upload-continue": - require(len(matching) == 1, "G05 continued UploadId disappeared from ListMultipartUploads") - parts = fleet.record("G05-list-parts", probe.s3("list_parts", {"Bucket": bucket, "Key": G05_TARGET_KEY, - "UploadId": upload_id})) - rows = (parts.get("response") or {}).get("Parts", []) - require(len(rows) == len(expected_parts) and all( - row.get("PartNumber") == number and row.get("ETag") == runtime["partEtags"][(name, number)] - and row.get("Size") == size for row, (number, size) in zip(rows, expected_parts)), - "G05 continued UploadId lost an acknowledged part") - else: - require(not matching, f"G05 {branch} left a completed/aborted UploadId visible") - if branch == "abort": - parts = fleet.record("G05-list-aborted-parts", probe.s3("list_parts", {"Bucket": bucket, - "Key": G05_TARGET_KEY, - "UploadId": upload_id})) - response = parts.get("response") or {} - require(parts.get("acknowledged") is False and parts.get("resultUnknown") is False - and response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404, - "G05 aborted UploadId still exposes acknowledged parts") - return {"uploadId": upload_id, "listedUploads": len(uploads), "branch": branch, - "acknowledgedParts": len(expected_parts) if branch == "upload-continue" else 0} - - def run_branch(args, topology, nodes, branch): + if args.case_id == 'heal.race-multipart-complete-abort': + # Lazy import keeps the generic controller and G05's model acyclic. + try: + from .mpu_race_runner import run_branch as run_mpu_branch + except ImportError: + from mpu_race_runner import run_branch as run_mpu_branch + source_contract(json.loads(args.candidate.read_text()), args.case_id) + return run_mpu_branch(args, topology, nodes, branch) directory = args.evidence / topology / branch directory.mkdir(parents=True, exist_ok=False) run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" @@ -1012,10 +872,9 @@ def run_branch(args, topology, nodes, branch): probe, active, recorder, writer = Probe(binding), [], None, None result = {"topology": topology, "branch": branch, "fixture": "DL" if branch in (*G01_BRANCHES, "unversioned-delete") else "DV" if branch in G04_BRANCHES - else "DM" if branch in G05_BRANCHES else "DS" if branch in G06_BRANCHES else "DV", "verdict": "RUNNING", "cleanup": "PENDING", "canonicalCasePassClaimed": False} - changed_target_registered, target_authority, mpu_runtime = None, False, None + changed_target_registered, target_authority = None, False try: fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) @@ -1024,7 +883,7 @@ def run_branch(args, topology, nodes, branch): recorder.marker_branch = branch if branch in G04_BRANCHES else None expected = seed_fixture(recorder, bucket, branch) baseline = fleet.record("complete-baseline-S3", probe.inventory( - bucket, unversioned_current_tags=branch in (*G01_BRANCHES, "unversioned-delete", *G05_BRANCHES, *G06_BRANCHES))) + bucket, unversioned_current_tags=branch in (*G01_BRANCHES, "unversioned-delete", *G06_BRANCHES))) verify_seed(expected, baseline) plan = operation_plan(bucket, branch, expected) for step in plan: @@ -1052,8 +911,7 @@ def run_branch(args, topology, nodes, branch): faulted = fleet.snapshot("complete-fault-physical", quiescent=True) fleet.record("fault-batch-proof", validate_missing_batch(before, faulted, targets, receipts)) fleet.start() - if branch not in G05_BRANCHES: - set_race_io_cap(fleet, args.case_id, branch, True, nodes=sorted({target['node'] for target in targets})) + set_race_io_cap(fleet, args.case_id, branch, True, nodes=sorted({target['node'] for target in targets})) require(physical_content(faulted) == physical_content(fleet.snapshot("attribution-before-START")), "NOT_OBSERVED: G01/G03 missing shards changed before manual admission") path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") @@ -1080,12 +938,7 @@ def run_branch(args, topology, nodes, branch): responses, overlapped = race(writer, plan, probe, fleet, path, token, reference, baseline=baseline) else: responses, overlapped = race(writer, plan, probe, fleet, path, token, reference) - if branch not in G05_BRANCHES: - set_race_io_cap(fleet, args.case_id, branch, False, nodes=sorted({target['node'] for target in targets})) - mpu_runtime = plan[0].get("runtime") if branch in G05_BRANCHES else None - if branch in G05_BRANCHES: - verify_multipart_state(probe, fleet, bucket, branch, mpu_runtime, - [(1, G05_PART_SIZES[0])] if branch != "complete" else list(enumerate(G05_PART_SIZES, 1))) + set_race_io_cap(fleet, args.case_id, branch, False, nodes=sorted({target['node'] for target in targets})) final = fleet.record("ACK-derived-final-manifest", final_inventory(baseline, branch, plan, responses)) terminal = wait_terminal(probe, fleet, path, token, reference) active.remove((path, token)) @@ -1119,7 +972,7 @@ def run_branch(args, topology, nodes, branch): fleet.start() for endpoint in binding["endpoints"]: current = fleet.record("all-peer-final-S3", {"endpoint": endpoint, **Probe(binding, endpoint).inventory( - bucket, unversioned_current_tags=branch in (*G01_BRANCHES, "unversioned-delete", *G05_BRANCHES, *G06_BRANCHES))}) + bucket, unversioned_current_tags=branch in (*G01_BRANCHES, "unversioned-delete", *G06_BRANCHES))}) compare_inventories(final, current) if branch in G03_BRANCHES: negative = fleet.record("all-peer-deleted-version-GET", Probe(binding, endpoint).s3("get_object", plan[0]["params"])) @@ -1146,17 +999,6 @@ def run_branch(args, topology, nodes, branch): for path, token in active: try: fleet.record("cancel-original-token", probe.admin(path, [("clientToken", token), ("forceStop", "true")], b"")) except Exception as exc: fleet.record("cancel-error", {"error": str(exc)}) - if branch == "upload-continue" and mpu_runtime: - try: - fleet.start() - upload_id = mpu_runtime["uploadIds"]["continue"] - fleet.record("G05-cleanup-abort", probe.s3("abort_multipart_upload", { - "Bucket": bucket, "Key": G05_TARGET_KEY, "UploadId": upload_id})) - fleet.record("G05-cleanup-abort-proof", verify_multipart_state( - probe, fleet, bucket, "abort", {"uploadIds": {"abort": upload_id}, "partEtags": {}}, [])) - except Exception as exc: - fleet.record("G05-cleanup-abort-error", {"error": str(exc), "uploadId": mpu_runtime.get("uploadIds", {}).get("continue")}) - result["cleanup"] = "FAIL" unresolved = any(item is not None and item.pending is not None for item in (recorder, writer)) mutated = writer is not None and writer.sequence > 0 if unresolved or mutated and not target_authority: diff --git a/scripts/validation_heal/race_retention.py b/scripts/validation_heal/race_retention.py index f89adbeb..79ee6788 100644 --- a/scripts/validation_heal/race_retention.py +++ b/scripts/validation_heal/race_retention.py @@ -97,10 +97,15 @@ def one(label): and all(item.get('reserved') is True for item in reservations), 'G01/G02 isolation lacks all reserved hosts') ack_files = {} multipart = kind == 'unresolved-g05-race' - for name in ('seed-ACKs', 'foreground-ACKs', 'historical-read-ACKs', *(['mpu-read-ACKs'] if multipart else [])): + read_journals = (['mpu-read-ACKs', *['final-read-ACKs-node' + str(i) for i in range(1, len(owners) + 1)]] + if multipart else []) + if multipart: + require({path.name for path in folder.glob('final-read-ACKs-*')} <= set(read_journals), + 'G05 isolation has an unbound final peer read journal') + for name in ('seed-ACKs', 'foreground-ACKs', 'historical-read-ACKs', *read_journals): journal = folder / name entries = sorted(journal.glob('*.json')) - if name in {'historical-read-ACKs', 'mpu-read-ACKs'} or multipart and name == 'foreground-ACKs': + if name == 'historical-read-ACKs' or name in read_journals or multipart and name == 'foreground-ACKs': require(len(entries) <= 1024, 'G02 historical journal exceeds its bound') if not entries: continue require(0 < len(entries) <= 1024, 'G01/G02 isolation lacks its bounded original request/ACK journal') @@ -145,7 +150,7 @@ def one(label): 'abort_multipart_upload', 'list_parts', 'get_object', 'head_object', 'get_object_tagging'}: require(params.get('Key') == 'objects-0256.bin', 'G05 isolation SDK request changed its frozen multipart key') allowed = ({'list_multipart_uploads', 'list_parts', 'list_object_versions', 'get_object', 'head_object', - 'get_object_tagging'} if multipart and name == 'mpu-read-ACKs' else + 'get_object_tagging'} if multipart and name in read_journals else {'get_object'} if name == 'historical-read-ACKs' else {'create_bucket', 'put_bucket_versioning', 'put_object', 'create_multipart_upload', 'upload_part', 'complete_multipart_upload', 'abort_multipart_upload'} if multipart else diff --git a/scripts/validation_heal/status_keepalive.py b/scripts/validation_heal/status_keepalive.py new file mode 100644 index 00000000..30f714dd --- /dev/null +++ b/scripts/validation_heal/status_keepalive.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Opt-in same-token status transport; no redirects, proxy or request retries.""" +from contextlib import contextmanager +import http.client +import threading +from urllib.parse import parse_qsl, urlsplit, quote +import uuid + +try: + from .heal_probe import CLOCK_DOMAIN, Probe, observation_phase, require +except ImportError: + from heal_probe import CLOCK_DOMAIN, Probe, observation_phase, require + + +MAX_STATUS_OBSERVATIONS = 1025 # One warmup plus the existing 1024-sample race budget. + + +class KeepAliveStatusProbe(Probe): + @contextmanager + def status_session(self, token, path): + require(isinstance(token, str) and str(uuid.UUID(token)) == token, + 'canonical status token required') + require(path == '/rustfs/admin/v3/heal/' + quote(self.binding['bucketPrefix'], safe=''), + 'status session requires the original owned bucket path') + require(getattr(self, '_status', None) is None, 'nested status session prohibited') + endpoint = urlsplit(self.endpoint) + connection_type = http.client.HTTPSConnection if endpoint.scheme == 'https' else http.client.HTTPConnection + state = dict(token=token, path=path, owner=threading.get_ident(), poisoned=False, requests=0, + transport='direct-http-keepalive', implicitRetriesAllowed=False, + foregroundDelayInjected=False, observationSchemaVersion=1, + sessionId=str(uuid.uuid4()), clockDomain=CLOCK_DOMAIN, + observationBudget=MAX_STATUS_OBSERVATIONS, observations=[], + connectionAttempts=0, connectionsOpened=0) + probe = self + + class ObservedConnection(connection_type): + def connect(connection_self): + state['connectionAttempts'] += 1 + with observation_phase(probe._status_observation, 'connectionEstablishment'): + super().connect() + state['connectionsOpened'] += 1 + connection_self.observed_socket = connection_self.sock + connection_self.observed_id = f"{state['sessionId']}:{state['connectionsOpened']}" + + connection = ObservedConnection(endpoint.hostname, endpoint.port, timeout=30) + self._status_observation = None + self._status = (connection, state) + try: + yield state + finally: + connection.close() + self._status = None + self._status_observation = None + + @contextmanager + def status_observation(self): + session = getattr(self, '_status', None) + if session is None: + yield None + return + connection, state = session + observation = None + owns_observation = False + try: + require(threading.get_ident() == state['owner'], 'status session thread changed') + require(not state['poisoned'], 'status session poisoned; no implicit retry') + observation = self._status_observation + if observation is None: + require(len(state['observations']) < state['observationBudget'], 'status observation budget exhausted') + observation = dict(sessionId=state['sessionId'], sequence=len(state['observations']) + 1, + clockDomain=CLOCK_DOMAIN, phases={}, requestWriteCompleted=False, + responseObserved=False, responsePersisted=False) + state['observations'].append(observation) + self._status_observation = observation + owns_observation = True + yield observation + except BaseException as exc: + if observation is not None: + observation.setdefault('failureType', type(exc).__name__) + state['poisoned'] = True + state.setdefault('failureType', type(exc).__name__) + connection.close() + raise + finally: + if owns_observation: + self._status_observation = None + + def observation_for(self, sample): + session = getattr(self, '_status', None) + require(session is not None, 'status observation session required') + _, state = session + require(threading.get_ident() == state['owner'] and not state['poisoned'], 'status observer unavailable') + require(sample.get('clockDomain') == state['clockDomain'], 'status observation clock changed') + require(sample.get('query') == [('clientToken', state['token'])], 'status observation token changed') + reference = sample.get('statusObservation') or {} + require(type(reference.get('sequence')) is int + and reference == dict(sessionId=state['sessionId'], sequence=len(state['observations'])) + and bool(state['observations']), 'status observation reference changed') + return state['observations'][-1] + + def _open_admin(self, outgoing): + session = getattr(self, '_status', None) + if session is None: + return super()._open_admin(outgoing) + connection, state = session + target = urlsplit(outgoing.full_url) + endpoint = urlsplit(self.endpoint) + require(threading.get_ident() == state['owner'], 'status session thread changed') + require(not state['poisoned'], 'status session poisoned; no implicit retry') + require((target.scheme, target.netloc) == (endpoint.scheme, endpoint.netloc) + and target.path == state['path'] and not target.fragment + and parse_qsl(target.query, keep_blank_values=True) == [('clientToken', state['token'])] + and outgoing.get_method() == 'POST' and outgoing.data == b'', + 'keepalive permits only the original token status; no control mutation') + state['requests'] += 1 + observation = self._status_observation + try: + opened_before = state['connectionsOpened'] + with observation_phase(observation, 'requestWrite'): + connection.request('POST', target.path + '?' + target.query, body=b'', + headers=dict(outgoing.header_items())) + if observation is not None: + observation['requestWriteCompleted'] = True + sock = connection.sock + require(sock is not None and sock is connection.observed_socket, + 'status socket identity unavailable after request write') + observation.update(transport=dict( + requestSequence=state['requests'], connectionId=connection.observed_id, + newConnection=state['connectionsOpened'] > opened_before, + socketDescriptor=sock.fileno(), localAddress=list(sock.getsockname()), + peerAddress=list(sock.getpeername()))) + with observation_phase(observation, 'responseHeaders'): + response = connection.getresponse() + if observation is not None: + observation['transport']['serverWillClose'] = response.will_close + # Probe owns the bounded full-body read and records the same timing, + # headers and response as its default urllib path. Closing a fully + # consumed HTTPResponse retains a server-supported keepalive socket. + response.code = response.status + if 300 <= response.status < 400: + response.close() + raise ValueError('Admin redirects are prohibited') + return response + except BaseException: + state['poisoned'] = True + connection.close() + raise + + def admin(self, *args, **kwargs): + with self.status_observation() as observation: + provided = kwargs.pop('observation', None) + require(provided is None or provided is observation, 'status observation owner changed') + require(observation is None or 'callStartedMonotonicNs' not in observation, + 'status observation already invoked') + return super().admin(*args, **kwargs, observation=observation) diff --git a/test_validation_heal_mpu_race_runner.py b/test_validation_heal_mpu_race_runner.py new file mode 100644 index 00000000..1fa8a086 --- /dev/null +++ b/test_validation_heal_mpu_race_runner.py @@ -0,0 +1,309 @@ +"""Actual controller/Recorder boundaries; synthetic SDK/physical data is not live proof.""" +import copy +from contextlib import ExitStack, nullcontext +import hashlib +import json +import os +from pathlib import Path +import tempfile +import threading +import time +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch +import uuid + +import test_validation_heal_mpu_race_fixture as model_fixtures +import test_validation_heal_status_keepalive as transport_fixtures +from scripts.validation_heal import mpu_race_runner as runner, mpu_race_fixture as model, race +from scripts.validation_heal.heal_probe import CLOCK_DOMAIN, save +from scripts.validation_heal.heal_workload import Recorder +from scripts.validation_heal.healthy import Fleet + + +class SDKProbe: + def __init__(self, root): + self.binding = transport_fixtures.binding() + self.sdk = model_fixtures.Recorder(root);Path(root).mkdir() + self.calls = [];self.gate = None;self.fail = None;self.control = 500;self.controls = {} + + def s3(self, operation, params, body=None): + started = time.monotonic_ns();self.calls.append((operation,copy.deepcopy(params))) + if self.gate and self.gate[0] == operation: + _,entered,release = self.gate;self.gate = None;entered.set() + if not release.wait(5): raise TimeoutError('component gate not released') + if self.fail == operation: raise OSError('lost original response; do not retry') + if operation in {'create_bucket','put_bucket_versioning'}: + response = {'ResponseMetadata':{'HTTPStatusCode':200}} + elif operation == 'put_object': + self.control += 1 + response = {'ResponseMetadata':{'HTTPStatusCode':200},'ETag':model.etag(body), + 'VersionId':str(uuid.UUID(int=self.control))} + self.controls[params['Key']] = {'key':params['Key'],'versionId':response['VersionId'], + 'isLatest':True,'deleteMarker':False,'payload':{'size':len(body),'sha256':hashlib.sha256(body).hexdigest()}} + else: + response = self.sdk.call(operation,params,body) + result = {'operation':operation,'params':copy.deepcopy(params),'response':response, + 'acknowledged':200 <= response['ResponseMetadata']['HTTPStatusCode'] < 300,'resultUnknown':False, + 'clockDomain':CLOCK_DOMAIN,'callStartedMonotonicNs':started,'callFinishedMonotonicNs':time.monotonic_ns(), + 'clientCallInvoked':True} + if operation == 'get_object': + version = next(v for v in self.sdk.versions if v['VersionId']==params['VersionId']) if 'VersionId' in params else self.sdk.versions[-1] + result['payload'] = {'size':len(version['body']),'sha256':hashlib.sha256(version['body']).hexdigest()} + return result + + def observation_for(self, sample): return {'phases':{}} + + def inventory(self, bucket): + entries = copy.deepcopy(list(self.controls.values())) + for version in self.sdk.versions: + entries.append({'key':model.KEY,'versionId':version['VersionId'],'isLatest':version is self.sdk.versions[-1], + 'deleteMarker':False,'payload':{'size':len(version['body']),'sha256':hashlib.sha256(version['body']).hexdigest()}}) + return {'bucket':bucket,'entries':entries} + + def status_session(self, token, path): return nullcontext({'componentOnly':True}) + + +class MpuRunnerTests(unittest.TestCase): + def controller(self, directory, *, fail_stage=None, overlapped=True): + """Exercise the actual controller, durable SDK model and final diagnostics. + + Physical/terminal oracles are mocked here; their full-byte tests remain + independent. This fixture proves ordering and cleanup authority only. + """ + root = Path(directory);candidate=root/'candidate.json';rc=root/'rc' + candidate.write_text(json.dumps({'source_sha':race.COPY_CANDIDATE,'package_sha256':'a'*64}));rc.write_bytes(b'client') + probe=SDKProbe(root/'sdk');trace=[];owned=[] + args=SimpleNamespace(case_id=runner.CASE,candidate=candidate,evidence=root/'evidence',tool_directory=root, + testing_commit='b'*40,binary_sha='c'*64,rc_bin=rc) + scene={'vm000':{'files':[],'formats':[]}} + def fleet_factory(*positional,**kwargs): + fleet=Fleet(*positional,**kwargs);owned.append(fleet) + fleet.deploy=Mock(side_effect=lambda *a:trace.append('deploy')) + fleet.stop=Mock(side_effect=lambda:trace.append('stop')) + fleet.start=Mock(side_effect=lambda:trace.append('start')) + fleet.snapshot=Mock(side_effect=lambda label,**kw:(trace.append(label) or scene)) + def remote(node,action,**kw): + if action == 'apply-fault': + self.assertEqual(node,'vm000');self.assertEqual(kw['disk'],3) + self.assertRegex(kw['faultId'],r'^G05-[0-9a-f]{32}$') + trace.append(action) + return {} + fleet.remote=Mock(side_effect=remote) + fleet.cleanup=Mock(return_value=True) + return fleet + def probe_factory(binding,endpoint=None): + probe.binding=binding;return probe + probe.admin=Mock(return_value={'httpStatus':200,'response':{'clientToken':transport_fixtures.TOKEN}}) + real_inventory=probe.inventory + def inventory(bucket): + trace.append('inventory');return real_inventory(bucket) + probe.inventory=inventory + real_verify=runner.model.MpuRaceFixture.verify_s3 + def verify_s3(fixture,reader): + trace.append('model-S3');return real_verify(fixture,reader) + def mapped_controls(snapshot,bucket,controls): + return {(row['key'],row['versionId']):{**row,'node':'vm000','disk':3,'relativePath':row['key']+'/part.1','sha256':'d'*64} + for row in controls} + def mapped_mpu(snapshot,bucket,entries,**kw): + return {(row['versionId'],part):{**row,'dataDir':'old-dir','cohort':[{'node':'vm000','disk':3, + 'relativePath':model.KEY+'/part.'+str(part),'sha256':'d'*64}]} + for row in entries for part in (1,2,3)},{} + def committed(*a): + trace.append('committed-physical') + if fail_stage=='committed':raise ValueError('committed proof refused') + return {},{'physicalComponentOnly':True} + def streams(fleet,parts,label,**kw): + trace.append(label);self.assertEqual(kw['count'],259 if label=='baseline-streaming' else 262) + if fail_stage=='streams-io' and label=='final-streaming':raise OSError('evidence fsync failed') + return {'count':kw['count']} + def execute(fixture,prepared,*unused): + trace.append('foreground') + result=runner.schedule.execute(fixture,prepared['branch'],prepared, + lambda label,manifest:{'manifest':manifest,'physical':{'componentOnly':True}}) + return result,overlapped + def terminal_proof(*a):trace.append('terminal-proof');return {'componentOnly':True} + def roots(*a,**kw): + trace.append('root-retirement') + return {'.rustfs.sys/root-heal-'+transport_fixtures.TOKEN+'.json':{}} if fail_stage=='root' else {} + with ExitStack() as stack: + stack.enter_context(patch.dict(os.environ,{'GITHUB_RUN_ID':'12345','GITHUB_RUN_ATTEMPT':'1'})) + patches=[(runner,'Fleet',fleet_factory),(runner,'KeepAliveStatusProbe',probe_factory),(runner,'Probe',probe_factory), + (runner,'verify_inspector',lambda *a:{'binarySha256':'e'*64}),(runner,'validate_topology',Mock()), + (runner,'physical_population',Mock()),(runner.race,'map_fixture',mapped_controls),(runner.business,'mapped',mapped_mpu), + (runner,'verify_part_manifest',streams),(runner.staging,'verify',Mock(return_value={})), + (runner,'validate_missing_batch',Mock()),(runner,'observe_io',lambda *a:trace.append('io-observe')), + (runner,'observe_status',Mock()),(runner,'foreground',execute),(runner,'wait_terminal',lambda *a:{'summary':'finished'}), + (runner,'observe_terminal',lambda *a:{'response':{'summary':'finished'}}), + (runner,'validate_repaired_physical',Mock()),(runner.business,'transition',committed), + (runner.race,'terminal_proof',terminal_proof),(runner,'journal_records',Mock()),(runner,'exact_records',roots), + (runner.model.MpuRaceFixture,'verify_s3',verify_s3),(model,'PART_SIZES',(12,13,14)),(race,'SIZE',31)] + for target,name,value in patches:stack.enter_context(patch.object(target,name,value)) + result=runner.run_branch(args,'H1',['vm000'],'upload-continue') + return result,owned[0],trace,probe + + def test_complete_controller_proves_physical_before_final_get_and_only_then_cleans(self): + with tempfile.TemporaryDirectory() as d: + result,fleet,trace,probe=self.controller(d) + self.assertEqual((result['verdict'],result['cleanup']),('PASS','PASS')) + self.assertFalse(result['canonicalCasePassClaimed']);fleet.cleanup.assert_called_once() + self.assertEqual(trace.count('apply-fault'),257) + self.assertEqual(trace.count('io-observe'),2) + self.assertLess(trace.index('committed-physical'),trace.index('final-streaming')) + self.assertLess(trace.index('final-streaming'),trace.index('model-S3',trace.index('foreground'))) + self.assertLess(trace.index('physical-after-S3'),trace.index('terminal-proof')) + self.assertLess(trace.index('terminal-proof'),trace.index('root-retirement')) + self.assertFalse(fleet.faults) + + def test_assertion_failure_collects_independent_final_diagnostics_and_retains_all_acks(self): + for stage in ('committed','root'): + with self.subTest(stage=stage),tempfile.TemporaryDirectory() as d: + result,fleet,trace,probe=self.controller(d,fail_stage=stage) + self.assertEqual((result['verdict'],result['cleanup']),('FAIL','FAIL')) + fleet.cleanup.assert_not_called();self.assertEqual(len(fleet.faults),257) + self.assertIn('final-streaming',trace);self.assertIn('terminal-proof',trace);self.assertIn('root-retirement',trace) + self.assertEqual(trace[-1],'mpu-scene') + acks=sorted((fleet.evidence/'foreground-ACKs').glob('*-ack.json')) + self.assertEqual(len(acks),3) + self.assertEqual(json.loads(acks[-1].read_text())['observation']['operation'],'abort_multipart_upload') + + def test_evidence_io_failure_stops_final_diagnostics_and_never_authorizes_cleanup(self): + with tempfile.TemporaryDirectory() as d: + result,fleet,trace,_=self.controller(d,fail_stage='streams-io') + self.assertEqual((result['verdict'],result['cleanup']),('FAIL','FAIL')) + self.assertEqual(result['failure']['type'],'OSError');fleet.cleanup.assert_not_called() + self.assertNotIn('terminal-proof',trace);self.assertIn('mpu-scene',trace) + + def test_missing_overlap_can_clean_only_after_all_physical_s3_and_root_proofs(self): + with tempfile.TemporaryDirectory() as d: + result,fleet,trace,_=self.controller(d,overlapped=False) + self.assertEqual((result['verdict'],result['cleanup']),('NOT_OBSERVED','PASS')) + self.assertIn('root-retirement',trace);fleet.cleanup.assert_called_once();self.assertFalse(fleet.faults) + + def test_real_seed_and_durable_pairing_preserve_all_source_controls_and_same_key_uploads(self): + for branch in runner.schedule.SELECTED: + with self.subTest(branch=branch),tempfile.TemporaryDirectory() as d, \ + patch.object(model,'PART_SIZES',(12,13,14)),patch.object(race,'SIZE',31): + probe = SDKProbe(Path(d)/'sdk');recorder = Recorder(probe,Path(d)/'seed') + controls,fixture,prepared = runner.seed(recorder,probe.binding['bucketPrefix'],branch) + self.assertEqual([row['key'] for row in controls],[f'objects-{i:04d}.bin' for i in range(256)]) + self.assertEqual(len(runner.ack_records(recorder)),270+int(branch!='upload-continue')) + self.assertEqual([len(prepared['manifest']['items'][r]['parts']) for r in model.ROLES], + [3,2 if branch=='upload-continue' else 3,3]) + self.assertEqual(len({item['uploadId'] for item in fixture.items.values()}),3) + self.assertTrue(all(params.get('Key') == model.KEY for op,params in probe.calls if 'multipart' in op or op=='upload_part')) + + def test_original_sdk_pairing_rejects_orphans_changed_requests_or_unknown_results(self): + for fault in ('request','ack','sha','sequence','operation','params','unknown'): + with self.subTest(fault=fault),tempfile.TemporaryDirectory() as d: + probe=SDKProbe(Path(d)/'sdk');recorder=Recorder(probe,Path(d)/'seed') + recorder.call('create_bucket',{'Bucket':probe.binding['bucketPrefix']}) + req=recorder.output/'000001-request.json';ack=recorder.output/'000001-ack.json' + if fault=='request':req.unlink() + elif fault=='ack':ack.unlink() + else: + value=json.loads(ack.read_text()) + if fault=='sha':value['requestSha256']='f'*64 + elif fault=='sequence':value['sequence']=2 + else:value['observation'][{'operation':'operation','params':'params','unknown':'resultUnknown'}[fault]]=True if fault=='unknown' else 'foreign' + ack.write_text(json.dumps(value)) + with self.assertRaises(ValueError):runner.ack_records(recorder) + + def test_unknown_selected_foreground_result_halts_all_companions_without_retry(self): + for branch,operation in runner.schedule.SELECTED.items(): + with self.subTest(branch=branch),tempfile.TemporaryDirectory() as d, \ + patch.object(model,'PART_SIZES',(12,13,14)),patch.object(race,'SIZE',31): + root=Path(d);probe=SDKProbe(root/'sdk');seed=Recorder(probe,root/'seed') + _,fixture,prepared=runner.seed(seed,probe.binding['bucketPrefix'],branch) + fixture.recorder=Recorder(probe,root/'foreground');probe.fail=operation + original_calls=len(probe.calls) + fleet=SimpleNamespace(nodes=['vm000'],evidence=root,record=Mock()) + with patch.object(runner,'observe_status',return_value={}),patch.object(runner,'capture_staging') as capture: + with self.assertRaises(ValueError): + runner.foreground(fixture,prepared,probe,fleet,'path','token',race.COPY_REFERENCE,'a'*64) + self.assertEqual([op for op,_ in probe.calls[original_calls:]],[operation]);capture.assert_not_called() + self.assertIsNotNone(fixture.poisoned) + ack=json.loads((root/'foreground/000001-ack.json').read_text()) + self.assertTrue(ack['observation']['resultUnknown']);self.assertEqual(ack['operationCheck'],'FAIL') + + def test_stopped_original_root_reaches_final_diagnostics_without_becoming_success(self): + probe=Mock();fleet=Mock();fleet.record.side_effect=lambda label,value:value + probe.admin.return_value={'httpStatus':200,'response':{'summary':'stopped','settings':race.COPY_REFERENCE['effective']}} + with patch.object(runner.time,'sleep',side_effect=AssertionError('stopped root must drain immediately')): + terminal=runner.wait_terminal(probe,fleet,'path','token',race.COPY_REFERENCE) + self.assertEqual(terminal['summary'],'stopped') + with self.assertRaisesRegex(ValueError,'terminal coverage'): + race.terminal_proof(terminal,'bucket',{'entries':[]},{'entries':[]}) + + def test_selected_first_sdk_overlap_does_not_require_later_companions_to_overlap(self): + for branch in runner.schedule.SELECTED: + with self.subTest(branch=branch),tempfile.TemporaryDirectory() as d, \ + patch.object(model,'PART_SIZES',(12,13,14)),patch.object(race,'SIZE',31): + root=Path(d);probe=SDKProbe(root/'sdk');seed=Recorder(probe,root/'seed') + _,fixture,prepared=runner.seed(seed,probe.binding['bucketPrefix'],branch) + fixture.recorder=Recorder(probe,root/'foreground') + entered,release=threading.Event(),threading.Event();probe.gate=(prepared['selectedOperation'],entered,release) + fleet=SimpleNamespace(nodes=['vm000'],evidence=root,record=Mock(side_effect=lambda label,value:value)) + path='/rustfs/admin/v3/heal/'+probe.binding['bucketPrefix'];token=str(uuid.UUID(int=1)) + def observed(*args): + self.assertTrue(entered.wait(5));started=time.monotonic_ns();finished=time.monotonic_ns() + release.set() + return {'path':path,'query':[('clientToken',token)],'httpStatus':200,'clockDomain':CLOCK_DOMAIN, + 'callStartedMonotonicNs':started,'callFinishedMonotonicNs':finished, + 'statusObservation':{'sessionId':'component','sequence':1}, + 'response':{'summary':'running','settings':race.COPY_REFERENCE['effective'], + 'outcome':{'counters':{'processed':1}}}} + with patch.object(runner,'observe_status',side_effect=observed),patch.object(runner,'capture_staging',return_value={'mock':True}): + result,overlapped=runner.foreground(fixture,prepared,probe,fleet,path,token,race.COPY_REFERENCE,'a'*64) + self.assertTrue(overlapped) + acks=runner.ack_records(fixture.recorder) + self.assertEqual(result['selectedAckSequence'],1) + proof=fleet.record.call_args.args[1] + self.assertEqual(proof['selectedOperation'],prepared['selectedOperation']) + self.assertEqual(len(acks),3 if branch=='upload-continue' else 2) + self.assertTrue(proof['allCompanionACKsVerified']);self.assertFalse(proof['nativeFenceProved']) + self.assertEqual((fixture.items['a']['state'],fixture.items['b']['state']),('complete','aborted')) + + def test_seed_unknown_response_preserves_stopped_raw_scene_without_abort_restore_or_cleanup(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d);candidate=root/'candidate.json';candidate.write_text(json.dumps({'source_sha':race.COPY_CANDIDATE,'package_sha256':'a'*64})) + rc=root/'rc';rc.write_bytes(b'owned client');probe=SDKProbe(root/'sdk');probe.fail='create_multipart_upload' + fleet=Mock();fleet.nodes=['vm000'];fleet.record.side_effect=lambda label,value:value + fleet.remote.return_value={'readOnly':True};fleet.snapshot.return_value={'vm000':{'files':[],'formats':[]}} + args=SimpleNamespace(case_id=runner.CASE,candidate=candidate,evidence=root/'evidence',tool_directory=root, + testing_commit='b'*40,binary_sha='c'*64,rc_bin=rc) + def failed_seed(recorder,bucket,branch): + recorder.probe.binding['bucketPrefix']=bucket + recorder.call('create_multipart_upload',{'Bucket':bucket,'Key':model.KEY}) + with patch.dict(os.environ,{'GITHUB_RUN_ID':'12345','GITHUB_RUN_ATTEMPT':'1'}), \ + patch.object(runner,'Fleet',return_value=fleet),patch.object(runner,'KeepAliveStatusProbe',return_value=probe), \ + patch.object(runner,'verify_inspector',return_value={}),patch.object(runner,'validate_topology'), \ + patch.object(runner,'seed',side_effect=failed_seed): + result=runner.run_branch(args,'H1',['vm000'],'upload-continue') + self.assertEqual((result['verdict'],result['cleanup']),('FAIL','FAIL')) + fleet.cleanup.assert_not_called();fleet.start.assert_not_called() + self.assertEqual([op for op,_ in probe.calls],['create_multipart_upload']) + self.assertTrue(any(call.args[:2] == ('vm000','mpu-scene') for call in fleet.remote.call_args_list)) + self.assertTrue(any(call.args[0]=='last-ACK-rescue-required' for call in fleet.record.call_args_list)) + + +class DurableStatusTests(unittest.TestCase): + # Reuse the actual socket fixture; only these new integration checks run here. + setUp = transport_fixtures.KeepaliveTests.setUp + tearDown = transport_fixtures.KeepaliveTests.tearDown + + def test_g05_status_persists_original_request_before_send_and_response_after_receive(self): + with tempfile.TemporaryDirectory() as d,patch.object(race,'settings'), \ + patch('scripts.validation_heal.heal_probe.os.fsync',wraps=os.fsync) as sync, \ + patch.dict(os.environ,{'HEAL_ACCESS_KEY':'local-access','HEAL_SECRET_KEY':'local-secret'}): + fleet=Fleet(['vm000'],Path(d),'gh-12345-1-h1');path='/rustfs/admin/v3/heal/'+self.probe.binding['bucketPrefix'] + with self.probe.status_session(transport_fixtures.TOKEN,path) as state: + value=runner.observe_status(self.probe,fleet,path,transport_fixtures.TOKEN,race.COPY_REFERENCE) + measured=state['observations'][0] + self.assertEqual(sync.call_count,4);self.assertTrue(measured['responsePersisted']) + self.assertLessEqual(measured['phases']['requestPersistence']['finishedMonotonicNs'],value['callStartedMonotonicNs']) + self.assertLessEqual(value['callFinishedMonotonicNs'],measured['phases']['responsePersistence']['startedMonotonicNs']) + request=Path(d)/measured['requestFile'];response=Path(d)/measured['responseFile'] + self.assertEqual(hashlib.sha256(request.read_bytes()).hexdigest(),value['requestSha256']) + self.assertEqual(json.loads(response.read_text())['requestSha256'],value['requestSha256']) + self.assertNotIn('local-secret',json.dumps(state)) diff --git a/test_validation_heal_race.py b/test_validation_heal_race.py index b42173b2..e4da0f03 100644 --- a/test_validation_heal_race.py +++ b/test_validation_heal_race.py @@ -1,6 +1,9 @@ import json from pathlib import Path import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch +import tempfile from scripts.validation_heal import race from scripts.validation_heal.matrix import AUTOMATIC_CASES, select @@ -48,50 +51,34 @@ def test_g06_keeps_copy_metadata_and_tag_operations_distinct(self): self.assertEqual(race.target_key("copy-source"), race.G06_TARGET_KEY) self.assertEqual(race.reference_key("copy-source"), race.G06_REFERENCE_KEY) - def test_g05_binds_multipart_operations_to_acknowledged_upload(self): - # The frozen DM fixture requires two nonfinal parts above 5 MiB. - self.assertEqual(race.G05_PART_SIZES, - (5 * 1024 * 1024 + 17, 5 * 1024 * 1024 + 93, 1024 * 1024 + 47)) - expected = [{"key": race.G05_TARGET_KEY, "versionId": "null", "isLatest": True}] - continued = race.operation_plan("rv-heal-0123456789abcdef", "upload-continue", expected) - completed = race.operation_plan("rv-heal-0123456789abcdef", "complete", expected) - aborted = race.operation_plan("rv-heal-0123456789abcdef", "abort", expected) - self.assertEqual([step["operation"] for step in continued], ["create_multipart_upload", "upload_part"]) - self.assertEqual([step["operation"] for step in completed], - ["create_multipart_upload", "upload_part", "upload_part", "upload_part", - "complete_multipart_upload"]) - self.assertEqual([step["operation"] for step in aborted], - ["create_multipart_upload", "upload_part", "abort_multipart_upload"]) - self.assertEqual([step["size"] for step in completed[1:4]], list(race.G05_PART_SIZES)) - self.assertEqual(completed[-1]["params"]["MultipartUpload"]["Parts"], []) - self.assertEqual(race.target_key("complete"), race.G05_TARGET_KEY) - - def test_g05_foreground_resolves_upload_id_and_part_etags_from_ack(self): - class FakeRecorder: - def __init__(self): - self.calls = [] - - def call(self, operation, params, body=None): - self.calls.append((operation, params, body)) - if operation == "create_multipart_upload": - return {"UploadId": "upload-ack"} - if operation == "upload_part": - return {"ETag": f"etag-{params['PartNumber']}"} - return {"ETag": "complete-etag", "VersionId": "null"} - - recorder = FakeRecorder() - plan = race.operation_plan("rv-heal-0123456789abcdef", "complete", []) - responses, runtime = race.foreground(recorder, plan) - self.assertEqual(runtime["uploadIds"]["complete"], "upload-ack") - self.assertEqual(recorder.calls[1][1]["UploadId"], "upload-ack") - self.assertEqual(recorder.calls[-1][1]["UploadId"], "upload-ack") - self.assertEqual(recorder.calls[-1][1]["MultipartUpload"], { - "Parts": [{"PartNumber": 1, "ETag": "etag-1"}, - {"PartNumber": 2, "ETag": "etag-2"}, - {"PartNumber": 3, "ETag": "etag-3"}]}) - self.assertEqual([len(call[2]) if call[2] is not None else None for call in recorder.calls], - [None, *race.G05_PART_SIZES, None]) - self.assertEqual(len(responses), 5) + def test_g05_cannot_use_the_superseded_unversioned_fixture_or_generic_plan(self): + recorder = Mock() + for branch in race.G05_BRANCHES: + with self.subTest(branch=branch), self.assertRaises(ValueError): + race.seed_fixture(recorder, 'rv-heal-0123456789abcdef', branch) + with self.subTest(branch=branch), self.assertRaises(ValueError): + race.operation_plan('rv-heal-0123456789abcdef', branch, []) + recorder.call.assert_not_called() + for operation in ('create_multipart_upload', 'upload_part', 'complete_multipart_upload', 'abort_multipart_upload'): + with self.subTest(operation=operation), self.assertRaises(ValueError): + race.foreground(recorder, [{'operation': operation}]) + recorder.call.assert_not_called() + + def test_g05_controller_delegates_all_public_branches_and_refuses_old_source_before_bootstrap(self): + case = race.CASE_IDS[4] + with tempfile.TemporaryDirectory() as d: + candidate = Path(d) / 'candidate.json' + args = SimpleNamespace(case_id=case, candidate=candidate) + with patch('scripts.validation_heal.mpu_race_runner.run_branch', return_value={'owned': True}) as execute, \ + patch.object(race, 'Fleet', side_effect=AssertionError('generic fleet must not run')): + candidate.write_text(json.dumps({'source_sha': race.REFERENCE['candidateCommit']})) + with self.assertRaisesRegex(ValueError, 'exact-source multipart'): + race.run_branch(args, 'H1', ['vm000'], 'upload-continue') + execute.assert_not_called() + candidate.write_text(json.dumps({'source_sha': race.COPY_CANDIDATE})) + for branch in race.G05_BRANCHES: + self.assertEqual(race.run_branch(args, 'H1', ['vm000'], branch), {'owned': True}) + execute.assert_called_with(args, 'H1', ['vm000'], branch) def test_g04_preserves_marker_and_batch_delete_semantics(self): expected = [{"key": race.TARGET_KEY, "versionId": value, "isLatest": value == "v2", diff --git a/test_validation_heal_race_io.py b/test_validation_heal_race_io.py index 713ae0ab..9061f837 100644 --- a/test_validation_heal_race_io.py +++ b/test_validation_heal_race_io.py @@ -215,8 +215,10 @@ def execute(*args, **kwargs): self.assertEqual(scope['race_io_backend'].RATE,'65536') self.assertTrue(callable(scope['race_io_backend'].run)) return SimpleNamespace(returncode=0,stdout='{}',stderr='') - with tempfile.TemporaryDirectory() as tmp, patch('scripts.validation_heal.healthy.subprocess.run',side_effect=execute): - Fleet(['vm000'],Path(tmp),'gh-12345-1-h1').remote('vm000','race-io-cap') + for action in ('race-io-cap','race-io-observe'): + with self.subTest(action=action),tempfile.TemporaryDirectory() as tmp, \ + patch('scripts.validation_heal.healthy.subprocess.run',side_effect=execute): + Fleet(['vm000'],Path(tmp),'gh-12345-1-h1').remote('vm000',action) def test_controller_records_intent_before_single_call_and_never_retries_a_lost_result(self): fleet = Mock(); fleet.nodes = ['vm000','vm001']; trace = [] @@ -242,7 +244,6 @@ def remote(host,action,**params): def test_real_runner_arms_before_admin_and_releases_after_foreground_before_terminal(self): class ReachedTerminalBoundary(BaseException): pass for driver, case_id, branch in [(race,race.CASE_IDS[0],'overwrite-vs-deep'), - (race,race.CASE_IDS[4],'upload-continue'), (nolock,nolock.CASE_ID,'overwrite-nolock')]: with self.subTest(case=case_id), tempfile.TemporaryDirectory() as tmp, ExitStack() as stack: root = Path(tmp); candidate = root/'candidate.json' @@ -269,14 +270,12 @@ def terminal(*args): trace.append('terminal'); raise ReachedTerminalBoundary() 'validate_missing_batch':Mock(),'wait_running':Mock(),'race':Mock(side_effect=foreground), 'final_inventory':Mock(return_value={'entries':expected}), 'wait_terminal':Mock(side_effect=terminal),'set_race_io_cap':Mock(side_effect=cap),'OBJECTS':3} - if driver is race: values['verify_multipart_state'] = Mock() for name,value in values.items(): stack.enter_context(patch.object(driver,name,value)) stack.enter_context(patch.dict(race.os.environ,{'GITHUB_RUN_ID':'12345','GITHUB_RUN_ATTEMPT':'1'})) args = SimpleNamespace(evidence=root/'evidence',candidate=candidate,rc_bin=rc,tool_directory=root, testing_commit='helper',binary_sha='binary',case_id=case_id) with self.assertRaises(ReachedTerminalBoundary): driver.run_branch(args,'H1',['vm000'],branch) - self.assertEqual(trace[:trace.index('terminal')+1],['admin','foreground','terminal'] if case_id == race.CASE_IDS[4] - else ['cap-on','admin','foreground','cap-off','terminal']) + self.assertEqual(trace[:trace.index('terminal')+1],['cap-on','admin','foreground','cap-off','terminal']) if __name__ == '__main__': unittest.main() diff --git a/test_validation_heal_race_native.py b/test_validation_heal_race_native.py index 284eea3e..4abeedde 100644 --- a/test_validation_heal_race_native.py +++ b/test_validation_heal_race_native.py @@ -62,10 +62,16 @@ def test_a_registered_historical_owner_subset_cannot_certify_the_full_version_pu def test_runtime_case_opt_in_does_not_change_the_exact_source_or_enable_other_native_groups(self): wrong_source = copy.deepcopy(race.NATIVE_REFERENCE) wrong_source['candidateCommit'] = 'e' * 40 - for reference, case in [(wrong_source, CASE), (race.NATIVE_REFERENCE, 'heal.race-multipart-complete-abort')]: + for reference, case in [(wrong_source, CASE)]: with self.subTest(case=case), patch.dict(race.NATIVE_RUNTIME, {SOURCE: {'reference': reference, 'cases': (case,)}}): with self.assertRaisesRegex(ValueError, 'race candidate/case contract is not reviewed'): race.native_preflight(CANDIDATE, HELPER, case, '/never-used') + case = 'heal.race-multipart-complete-abort' + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); self.fixture(root) + with patch.dict(race.NATIVE_RUNTIME, {SOURCE: {'reference': race.NATIVE_REFERENCE, 'cases': (case,)}}): + with self.assertRaises(ValueError): + race.native_preflight(CANDIDATE, HELPER, case, root) def test_version_source_admits_only_three_complete_native_groups_without_executing_artifacts(self): source = race.VERSION_CANDIDATE @@ -140,7 +146,7 @@ def test_multipart_native_source_collects_all_ten_events_without_enabling_incomp with self.assertRaisesRegex(ValueError, 'invented or omitted'): consumer.verify(root, candidate, HELPER, case) - def test_default_owner_and_copy_source_keep_native_groups_separate_and_only_g06_race_runtime(self): + def test_default_owner_copy_and_mpu_source_keep_complete_native_groups_separate(self): source = '757b751482bb970bc321e92059690a7698554358' candidate = {**CANDIDATE, 'source_sha': source} contract = suite.contract_for(source) @@ -155,6 +161,7 @@ def test_default_owner_and_copy_source_keep_native_groups_separate_and_only_g06_ for case, label, count in [ ('heal.admin-nolock-safety', 'default-heal-exclusive-owner', 16), ('heal.race-copy-and-metadata-update', 'copy-and-metadata-final-rename', 24), + ('heal.race-multipart-complete-abort', 'multipart-continue-complete-abort-final-rename', 10), ]: with self.subTest(case=case): self.assertEqual(contract['cases'][case], [label]) @@ -166,9 +173,9 @@ def test_default_owner_and_copy_source_keep_native_groups_separate_and_only_g06_ self.assertFalse(proof['canonicalCasePassClaimed']) for case in race.CASE_IDS: with self.subTest(case=case): - if case == 'heal.race-copy-and-metadata-update': + if case in {'heal.race-copy-and-metadata-update', 'heal.race-multipart-complete-abort'}: proof = race.native_preflight(candidate, HELPER, case, root) - self.assertEqual(proof['requiredRegressionLabels'], ['copy-and-metadata-final-rename']) + self.assertEqual(proof['requiredRegressionLabels'], race.FULL_NATIVE_GROUPS[case]) else: with self.assertRaisesRegex(ValueError, 'case contract is not reviewed'): race.native_preflight(candidate, HELPER, case, root) diff --git a/test_validation_heal_race_physical.py b/test_validation_heal_race_physical.py index e24692ae..d09f82c2 100644 --- a/test_validation_heal_race_physical.py +++ b/test_validation_heal_race_physical.py @@ -110,6 +110,10 @@ def test_real_runner_uses_each_registered_case_code_before_first_fault(self): class FaultBoundaryReached(BaseException): pass for case_id, branches in race.CASE_BRANCHES.items(): + # G05's dedicated controller checks all 257 fourth-drive G05 fault + # IDs in MpuRunnerTests.test_complete_controller_proves_physical_before_final_get_and_only_then_cleans. + if case_id == 'heal.race-multipart-complete-abort': + continue with self.subTest(case_id=case_id), tempfile.TemporaryDirectory() as folder, ExitStack() as stack: root = Path(folder) candidate = root / "candidate.json" diff --git a/test_validation_heal_race_retention.py b/test_validation_heal_race_retention.py index c58f8e86..30e9aa09 100644 --- a/test_validation_heal_race_retention.py +++ b/test_validation_heal_race_retention.py @@ -367,6 +367,28 @@ def test_g05_archive_rejects_missing_foreign_or_unbound_multipart_witnesses(self raw_path.write_text(json.dumps(raw)) with self.assertRaises(ValueError): recovery.recovery_plan(root,12345,candidate) + def test_g05_final_peer_read_journals_bind_original_requests_and_cannot_authorize_mutations(self): + for fault in ('changed', 'foreign-peer', 'foreign-key', 'mutation'): + with self.subTest(fault=fault),tempfile.TemporaryDirectory() as d: + root,candidate,folder=self.mpu_archive(d) + binding=json.loads((folder/'00001-binding.json').read_text()) + journal=folder/'final-read-ACKs-node4';journal.mkdir() + path=journal/'000001-request.json' + request={'runId':binding['runId'],'operation':'get_object', + 'params':{'Bucket':binding['bucketPrefix'],'Key':'objects-0256.bin'}} + path.write_text(json.dumps(request)) + before=recovery.recovery_plan(root,12345,candidate)[0]['ackFilesSha256'] + if fault=='foreign-peer':journal.rename(journal.with_name('final-read-ACKs-node5')) + else: + if fault=='changed':request['identity']={'original':'read'} + elif fault=='foreign-key':request['params']['Key']='foreign' + else:request['operation']='abort_multipart_upload' + path.write_text(json.dumps(request)) + if fault=='changed': + self.assertNotEqual(recovery.recovery_plan(root,12345,candidate)[0]['ackFilesSha256'],before) + else: + with self.assertRaises(ValueError):recovery.recovery_plan(root,12345,candidate) + def test_g05_controller_preflights_all_original_mpu_peers_before_any_move(self): with tempfile.TemporaryDirectory() as d: root,candidate,_ = self.mpu_archive(d);plan = recovery.recovery_plan(root,12345,candidate)[0] diff --git a/test_validation_heal_status_keepalive.py b/test_validation_heal_status_keepalive.py new file mode 100644 index 00000000..b9055602 --- /dev/null +++ b/test_validation_heal_status_keepalive.py @@ -0,0 +1,215 @@ +import http.server +import copy +import hashlib +import json +import os +from pathlib import Path +import socket +import sys +import threading +import tempfile +import time +import unittest +from unittest.mock import patch + +from datetime import datetime, timedelta, timezone +from scripts.validation_heal.status_keepalive import KeepAliveStatusProbe +from scripts.validation_heal.heal_probe import observation_phase, ADMIN_RESPONSE_LIMIT + + +def binding(): + return {"schemaVersion":1,"runId":"gh-12345-1-h1","leaseExpiresAt":(datetime.now(timezone.utc)+timedelta(hours=1)).isoformat(), + "jumpHostOnly":True,"exclusiveHealService":True,"candidateCommit":"a"*40,"harnessCommit":"b"*40, + "candidatePackageSha256":"c"*64,"candidateBinarySha256":"d"*64,"rcBinarySha256":"e"*64, + "bucketPrefix":"rv-heal-0123456789abcdef","defaultEndpoint":"node1","endpoints":{"node1":"http://127.0.0.1:9400"}, + "maxInventoryObjects":512,"maxInventoryBytes":512*1024**2} + +TOKEN = '00000000-0000-4000-8000-000000000001' + + +class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def log_message(self, *args): + pass + + def do_POST(self): + self.server.peers.append(self.client_address) + self.server.requests.append((self.path, dict(self.headers))) + self.rfile.read(int(self.headers.get('Content-Length', '0'))) + if self.server.mode == 'disconnect': + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + return + body = json.dumps({'summary': 'running', 'clientToken': TOKEN}).encode() + if self.server.mode == 'oversize': + body = b'x' * (ADMIN_RESPONSE_LIMIT + 1) + status = 302 if self.server.mode == 'redirect' else 503 if self.server.mode == 'error' else 200 + self.send_response(status) + self.send_header('Content-Length', str(len(body) + int(self.server.mode == 'truncated'))) + if self.server.mode in ('truncated', 'close'): + self.send_header('Connection', 'close') + self.send_header('Location', 'http://not-followed.invalid/') + self.end_headers() + self.wfile.write(body) + + +class KeepaliveTests(unittest.TestCase): + def setUp(self): + self.server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Handler) + self.server.peers = []; self.server.requests = []; self.server.mode = 'ok' + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + lab = binding(); lab['endpoints'][lab['defaultEndpoint']] = f'http://127.0.0.1:{self.server.server_port}' + self.probe = KeepAliveStatusProbe(lab) + + def tearDown(self): + self.server.shutdown(); self.server.server_close(); self.thread.join() + + def call(self, query=None, body=b''): + return self.probe.admin('/rustfs/admin/v3/heal/rv-heal-0123456789abcdef', [('clientToken', TOKEN)] if query is None else query, + body, auth='anonymous') + + def test_real_socket_reused_and_full_evidence_retained(self): + with self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef") as state: + a = self.call(); b = self.call() + self.assertEqual(state['requests'], 2) + self.assertEqual(len(set(self.server.peers)), 1) + self.assertLess(a['callFinishedMonotonicNs'], b['callStartedMonotonicNs']) + self.assertEqual(a['clockDomain'], b['clockDomain']) + self.assertEqual(b['response']['summary'], 'running') + self.assertIn('Content-Length', b['responseHeaders']) + self.assertEqual((state['connectionAttempts'], state['connectionsOpened']), (1, 1)) + first, second = state['observations'] + self.assertTrue(first['transport']['newConnection']) + self.assertFalse(second['transport']['newConnection']) + self.assertEqual(first['transport']['connectionId'], second['transport']['connectionId']) + self.assertEqual(first['transport']['localAddress'], list(self.server.peers[0])) + self.assertEqual(first['transport']['peerAddress'], ['127.0.0.1', self.server.server_port]) + self.assertIn('connectionEstablishment', first['phases']) + self.assertNotIn('connectionEstablishment', second['phases']) + for sample, measured in ((a, first), (b, second)): + self.assertEqual(sample['callStartedMonotonicNs'], measured['callStartedMonotonicNs']) + self.assertEqual(sample['callFinishedMonotonicNs'], measured['callFinishedMonotonicNs']) + for name in ('requestWrite', 'responseHeaders', 'responseBody', 'responseParsing'): + phase = measured['phases'][name] + self.assertTrue(phase['completed']) + self.assertLessEqual(sample['callStartedMonotonicNs'], phase['startedMonotonicNs']) + self.assertLessEqual(phase['finishedMonotonicNs'], sample['callFinishedMonotonicNs']) + self.assertIsNone(self.probe._status) + + def test_successful_server_close_records_a_new_connection_for_the_next_request(self): + self.server.mode = 'close' + with self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef") as state: + self.call(); self.call() + self.assertEqual((state['requests'], state['connectionAttempts'], state['connectionsOpened']), (2, 2, 2)) + first, second = [row['transport'] for row in state['observations']] + self.assertNotEqual(first['connectionId'], second['connectionId']) + self.assertTrue(first['newConnection'] and second['newConnection']) + self.assertTrue(first['serverWillClose'] and second['serverWillClose']) + self.assertFalse(state['poisoned']) + + def test_connection_failure_is_retained_without_reconnect_or_false_write_receipt(self): + with self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef") as state, \ + patch('http.client.HTTPConnection.connect', side_effect=OSError('local connect failure')) as connect: + with self.assertRaises(OSError): self.call() + self.assertEqual((state['connectionAttempts'], state['connectionsOpened']), (1, 0)) + measured = state['observations'][0] + self.assertFalse(measured['requestWriteCompleted']) + self.assertFalse(measured['responseObserved']) + self.assertFalse(measured['phases']['connectionEstablishment']['completed']) + self.assertEqual(measured['failureType'], 'OSError') + with self.assertRaisesRegex(ValueError, 'poisoned'): self.call() + connect.assert_called_once() + self.assertEqual(state['failureType'], 'OSError') + self.assertEqual(measured['failureType'], 'OSError') + self.assertEqual(self.server.requests, []) + + def test_observation_budget_fails_before_an_additional_request(self): + with patch('scripts.validation_heal.status_keepalive.MAX_STATUS_OBSERVATIONS', 2), self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef") as state: + self.call(); self.call() + with self.assertRaisesRegex(ValueError, 'budget exhausted'): self.call() + self.assertEqual(len(state['observations']), 2) + self.assertEqual(state['requests'], 2) + self.assertTrue(state['poisoned']) + self.assertEqual(len(self.server.requests), 2) + + def test_one_observation_cannot_hide_two_http_requests(self): + with self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef") as state: + with self.assertRaisesRegex(ValueError, 'already invoked'): + with self.probe.status_observation(): + self.call(); self.call() + self.assertEqual(len(state['observations']), 1) + self.assertEqual(state['requests'], 1) + self.assertTrue(state['poisoned']) + self.assertEqual(len(self.server.requests), 1) + + def test_poll_measurement_rejects_foreign_clock_token_session_or_sequence(self): + with self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef"): + sample = self.call() + for field, value in [('clockDomain', 'foreign'), ('query', [('clientToken', 'foreign')]), + ('statusObservation', {'sessionId': 'foreign', 'sequence': 1}), + ('statusObservation', {**sample['statusObservation'], 'sequence': True}), + ('statusObservation', {**sample['statusObservation'], 'sequence': 2})]: + altered = copy.deepcopy(sample); altered[field] = value + with self.assertRaises(ValueError): self.probe.observation_for(altered) + with self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef"): + self.call() + with self.assertRaises(ValueError): self.probe.observation_for(sample) + + + + def test_signed_headers_preserved(self): + with patch.dict('os.environ', HEAL_ACCESS_KEY='test-access', HEAL_SECRET_KEY='test-secret'): + with self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef"): + self.probe.admin('/rustfs/admin/v3/heal/rv-heal-0123456789abcdef', [('clientToken', TOKEN)], b'') + headers = {k.lower(): v for k, v in self.server.requests[0][1].items()} + self.assertIn('AWS4-HMAC-SHA256', headers['authorization']) + self.assertIn('x-amz-content-sha256', headers) + + def test_control_or_different_token_not_sent(self): + for query, body in [([], b'{}'), ([('clientToken', TOKEN), ('forceStop', 'true')], b''), + ([('clientToken', 'different')], b''), ([('clientToken', TOKEN)] * 2, b'')]: + with self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef") as state: + with self.assertRaises(ValueError): self.call(query, body) + self.assertTrue(state['poisoned']) + self.assertEqual(self.server.requests, []) + + def test_disconnect_redirect_and_oversize_poison_without_retry(self): + for mode in ('disconnect', 'redirect', 'oversize', 'truncated'): + self.server.mode = mode + before = len(self.server.requests) + with self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef") as state: + with self.assertRaises(Exception): self.call() + self.assertTrue(state['poisoned']) + with self.assertRaisesRegex(ValueError, 'poisoned'): self.call() + self.assertEqual(len(self.server.requests), before + 1) + + def test_http_error_kept_as_observation_not_retried(self): + self.server.mode = 'error' + with self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef"): + self.assertEqual(self.call()['httpStatus'], 503) + self.assertEqual(len(self.server.requests), 1) + + def test_other_thread_cannot_issue_a_status_request(self): + with self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef"): + with patch('scripts.validation_heal.status_keepalive.threading.get_ident', return_value=-1): + with self.assertRaisesRegex(ValueError, 'thread changed'): self.call() + self.assertEqual(self.server.requests, []) + + def test_default_transport_unchanged_and_nested_session_rejected(self): + with patch('scripts.validation_heal.status_keepalive.Probe._open_admin', return_value='default') as original: + self.assertEqual(self.probe._open_admin(object()), 'default') + original.assert_called_once() + with self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef"): + with self.assertRaisesRegex(ValueError, 'nested'): + with self.probe.status_session(TOKEN, "/rustfs/admin/v3/heal/rv-heal-0123456789abcdef"): pass + + def test_outside_status_session_has_no_timing_schema_or_sample_budget(self): + sample = self.call() + self.assertNotIn('statusObservation', sample) + self.assertIsNone(getattr(self.probe, '_status', None)) + self.assertEqual(len(self.server.requests), 1) + + +if __name__ == '__main__': unittest.main() From 96f289163172fef3b19419da887e3f482f09d170 Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 21 Oct 2026 10:16:20 +0800 Subject: [PATCH 273/343] fix(validation): preserve serial version order after deletion --- .../workflows/validation-catalog-tests.yml | 2 +- docs/validation-heal-operations.md | 4 + scripts/validation_heal/race.py | 27 +++++- test_validation_heal_g03_source.py | 83 +++++++++++++++++++ 4 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 test_validation_heal_g03_source.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index fffd5190..59dc54ed 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -159,7 +159,7 @@ jobs: sudo apt-get install --no-install-recommends -y nftables fi - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - - run: python3 -B -m unittest -v test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py + - run: python3 -B -m unittest -v test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index ac14ccce..d954645a 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -680,6 +680,10 @@ Source `757b751482bb970bc321e92059690a7698554358` adds two collector-only groups The G06 public adapter uses the existing DS workload: all 257 original keys, including Chinese, spaces, literal `%2F`, actual slashes and parent-like prefixes. The observed missing key is `中文/文件/0252.bin`; its companion is `literal%2Fslash/0254.bin`. `copy-source` copies from the observed key into the companion; `copy-destination` copies in the reverse direction. Default COPY expectations inherit the source payload, content properties, metadata and tags. Both directions verify the unchanged source, exact destination coding-index shard bytes and a distinct destination generation before GET, rejecting retained old destination directories. Registry, fixture and local transition tests do not establish public overlap or complete canonical PASS. +### G03 serial deletion authority + +G03's frozen `VersionLedger.history` determines the new current version after an explicit DELETE. The public S3 inventory is UUID-sorted and cannot supply that order. Each original foreground plan now carries the three seed ACK version IDs in order; the final oracle rejects missing, duplicate, foreign or differently selected identities, removes exactly the acknowledged version and promotes the last retained seed ACK. The physical oracle permits only that `isLatest` change while preserving every other retained descriptor, coding-index byte and data directory. Every original drive must contain exactly the retained version/latest population, with no extra marker or duplicate version. Focused tests use nonmonotonic UUIDs, both delete branches and corrupted physical populations. This source adaptation adds no native or live acceptance. + ### G05 same-key multipart components `mpu_race_fixture.py` migrates the frozen validation repository's `heal_mpu_race_fixture.py` model at `524eb5b32c3236497fdd8a9271df0494fa3d9974`. It uses the existing auto-testing `Recorder` and probe directly. Imports are local; no source validation checkout is needed at runtime. Part sizes follow the frozen common DM recipe: `5MiB+17`, `5MiB+93`, and `1MiB+47`. diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index eb10f1b2..11772993 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -348,7 +348,8 @@ def operation_plan(bucket, branch, expected): require(len(target) == 3 and len({row["versionId"] for row in target}) == 3 and target[-1]["isLatest"], "G01/G03 DV lacks three ACK-ordered versions") return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": current_target, - "VersionId": (target[0] if branch == G03_BRANCHES[0] else target[-1])["versionId"]}}] + "VersionId": (target[0] if branch == G03_BRANCHES[0] else target[-1])["versionId"]}, + "seedVersionOrder": [row['versionId'] for row in target]}] def version_put_entries(baseline, plan, responses): @@ -455,9 +456,21 @@ def final_inventory(baseline, branch, plan, responses): else: version = plan[0]["params"]["VersionId"] require(responses[0].get("VersionId") == version, "G01/G03 DELETE response changed the selected VersionId") + order = plan[0].get('seedVersionOrder') + target_rows = [row for row in result['entries'] if row['key'] == current_target] + require(branch in G03_BRANCHES and isinstance(order, list) and len(order) == len(set(order)) == 3 + and set(order) == {row['versionId'] for row in target_rows} + and all(row['deleteMarker'] is False and row['isLatest'] is (row['versionId'] == order[-1]) + for row in target_rows) + and version == order[0 if branch == G03_BRANCHES[0] else -1], + 'G03 DELETE lost its original serial seed ACK order or selected version') result["entries"] = [row for row in result["entries"] if (row["key"], row["versionId"]) != (current_target, version)] require(len(result["entries"]) + 1 == len(baseline["entries"]), "G01/G03 deleted a foreign/missing version") + latest = next(item for item in reversed(order) if item != version) + for row in result['entries']: + if row['key'] == current_target: + row['isLatest'] = row['versionId'] == latest return result @@ -604,10 +617,20 @@ def coding(target): require(len(removed) == 1, "G01/G03 delete transition lacks one exact deleted version") for target in current_target_values: original = mapped[current_target, target["versionId"]] - require(target["logical"] == original["logical"] + require({name: value for name, value in target['logical'].items() if name != 'isLatest'} + == {name: value for name, value in original['logical'].items() if name != 'isLatest'} and [(row["index"], row["sha256"], row["size"]) for row in target["cohort"]] == [(row["index"], row["sha256"], row["size"]) for row in original["cohort"]], "G01/G03 delete changed another retained ACK generation") + wanted = sorted((row['versionId'], row['isLatest'], row['deleteMarker']) + for row in final['entries'] if row['key'] == current_target) + indexed = files(after) + for node, value in after.items(): + for drive in value['formats']: + versions = indexed[node, drive['disk'], f'{bucket}/{current_target}/xl.meta']['metadata']['versions'] + require(all(type(row.get('isLatest')) is bool and type(row.get('deleted')) is bool for row in versions) + and sorted((row['versionId'], row['isLatest'], row['deleted']) for row in versions) == wanted, + 'G03 physical version/latest population differs from retained serial ACKs') inspected_keys = {current_target, changed_key} if branch in G06_BRANCHES else {current_target} for value in after.values(): for row in value["files"]: diff --git a/test_validation_heal_g03_source.py b/test_validation_heal_g03_source.py new file mode 100644 index 00000000..01b0beb3 --- /dev/null +++ b/test_validation_heal_g03_source.py @@ -0,0 +1,83 @@ +"""Frozen serial-ACK deletion order and physical latest authority; no live proof.""" +import copy +import unittest +import uuid + +from scripts.validation_heal import race +import test_validation_heal_race_physical as physical_fixtures + + +class VersionDeleteSourceTests(unittest.TestCase): + bucket = 'rv-heal-0123456789abcdef' + + def scene(self, branch): + before, _, _, _ = physical_fixtures.MarkerPhysicalTests().scene('batch-version-delete') + order = [str(uuid.UUID(int=i)) for i in (900, 700, 800)] + replacement = {str(uuid.UUID(int=i+1)): version for i, version in enumerate(order)} + for value in before.values(): + for row in value['files']: + for version in row.get('metadata', {}).get('versions', []): + version['versionId'] = replacement[version['versionId']] + seed = [{'key':race.TARGET_KEY,'versionId':version,'isLatest':i==2,'deleteMarker':False} + for i,version in enumerate(order)] + mapped = race.map_fixture(before,self.bucket,seed) + plan = race.operation_plan(self.bucket,branch,seed) + selected = plan[0]['params']['VersionId'] + final = [copy.deepcopy(row) for row in seed if row['versionId']!=selected] + for row in final:row['isLatest'] = row is final[-1] + after = copy.deepcopy(before) + selected_dir = mapped[race.TARGET_KEY,selected]['dataDir'] + for value in after.values(): + value['files'] = [row for row in value['files'] if selected_dir not in row['path']] + for row in value['files']: + metadata=row.get('metadata') + if metadata: + metadata['versions']=[version for version in metadata['versions'] if version['versionId']!=selected] + for version in metadata['versions']: + version['isLatest'] = version['versionId']==final[-1]['versionId'] + baseline={'entries':sorted(copy.deepcopy(seed),key=lambda row:row['versionId'])} + return before,after,mapped,baseline,plan,{'entries':sorted(final,key=lambda row:row['versionId'])} + + def test_deleted_current_promotes_last_retained_seed_ack_even_when_uuid_sort_disagrees(self): + for branch in race.G03_BRANCHES: + with self.subTest(branch=branch): + before,after,mapped,baseline,plan,wanted=self.scene(branch) + original=copy.deepcopy(baseline) + actual=race.final_inventory(baseline,branch,plan,[{'VersionId':plan[0]['params']['VersionId']}]) + self.assertEqual(actual,wanted) + self.assertEqual(baseline,original) + self.assertEqual(plan[0]['seedVersionOrder'],[str(uuid.UUID(int=i)) for i in (900,700,800)]) + + def test_real_physical_oracle_allows_only_ack_defined_latest_promotion(self): + for branch in race.G03_BRANCHES: + with self.subTest(branch=branch): + before,after,mapped,baseline,plan,wanted=self.scene(branch) + survivors,proof=race.physical_transition(before,after,self.bucket,mapped,wanted,branch) + self.assertTrue(proof['physicalProofBeforeGETOrRescue']) + self.assertEqual(len(survivors),2) + + def test_delete_manifest_rejects_missing_duplicate_or_foreign_ack_order(self): + for fault in ('missing','duplicate','foreign','wrong-delete','wrong-ack'): + with self.subTest(fault=fault): + _,_,_,baseline,plan,_=self.scene(race.G03_BRANCHES[1]) + plan[0]['seedVersionOrder']=[str(uuid.UUID(int=i)) for i in (900,700,800)] + response={'VersionId':plan[0]['params']['VersionId']} + if fault=='missing':plan[0].pop('seedVersionOrder') + elif fault=='duplicate':plan[0]['seedVersionOrder'][1]=plan[0]['seedVersionOrder'][0] + elif fault=='foreign':plan[0]['seedVersionOrder'][1]=str(uuid.UUID(int=999)) + elif fault=='wrong-delete':plan[0]['params']['VersionId']=str(uuid.UUID(int=700));response['VersionId']=str(uuid.UUID(int=700)) + else:response['VersionId']=str(uuid.UUID(int=700)) + with self.assertRaises(ValueError):race.final_inventory(baseline,race.G03_BRANCHES[1],plan,[response]) + + def test_retained_generation_payload_unknown_marker_or_latest_drift_is_refused(self): + for fault in ('bytes','directory','latest','marker','duplicate'): + with self.subTest(fault=fault): + before,after,mapped,_,_,wanted=self.scene(race.G03_BRANCHES[1]) + value=after['node1'] + meta=next(row['metadata'] for row in value['files'] if row['path'].endswith('/xl.meta')) + if fault=='bytes':next(row for row in value['files'] if row['path'].endswith('part.1'))['sha256']='changed' + elif fault=='directory':meta['versions'][0]['dataDir']=str(uuid.UUID(int=999)) + elif fault=='latest':meta['versions'][0]['isLatest']=not meta['versions'][0]['isLatest'] + elif fault=='marker':meta['versions'].append({'versionId':str(uuid.UUID(int=999)),'isLatest':False,'deleted':True}) + else:meta['versions'].append(copy.deepcopy(meta['versions'][0])) + with self.assertRaises(ValueError):race.physical_transition(before,after,self.bucket,mapped,wanted,race.G03_BRANCHES[1]) From d13439d927c24f7de7b8ae32b9b6b392bd4cd4d8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 21 Oct 2026 10:28:04 +0800 Subject: [PATCH 274/343] feat(validation): preserve frozen version race fixture oracles --- .../workflows/validation-catalog-tests.yml | 2 +- docs/validation-heal-operations.md | 4 + .../validation_heal/version_race_fixture.py | 277 ++++++++++++++++++ .../validation_heal/version_race_physical.py | 270 +++++++++++++++++ test_validation_heal_version_race_fixture.py | 247 ++++++++++++++++ test_validation_heal_version_race_physical.py | 233 +++++++++++++++ 6 files changed, 1032 insertions(+), 1 deletion(-) create mode 100644 scripts/validation_heal/version_race_fixture.py create mode 100644 scripts/validation_heal/version_race_physical.py create mode 100644 test_validation_heal_version_race_fixture.py create mode 100644 test_validation_heal_version_race_physical.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 59dc54ed..aaf4331a 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -159,7 +159,7 @@ jobs: sudo apt-get install --no-install-recommends -y nftables fi - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - - run: python3 -B -m unittest -v test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py + - run: python3 -B -m unittest -v test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index d954645a..e52d3b18 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -686,6 +686,10 @@ G03's frozen `VersionLedger.history` determines the new current version after an ### G05 same-key multipart components +`version_race_fixture.py` preserves the frozen `heal_version_race_fixture.py` model at `524eb5b32c3236497fdd8a9271df0494fa3d9974` (source SHA256 `755f476ab567dc413e9798bb2bcac15252c3e8308e4b15b15a9eb3083d1df485`), with local import adaptation only. Its eight G02–G04 plans derive versions from serial ACK order. G04 uses flat keys, seeds a latest marker for both marker removal and batch deletion, and deletes exactly the oldest target version, newest target version, marker and control `objects-0002.bin` version. It preserves the middle target generation. A partially acknowledged batch retains each known effect and permanently blocks final acceptance and further mutations. Historical reads address exact versions; negative reads require the original deleted-version 404 and marker 405/current-marker 404 contracts. + +`version_race_physical.py` preserves the frozen H1 physical oracle, replacing the source transport's fixed inspector digest with a mandatory caller-supplied immutable tool digest. Every drive must match the ACK-derived version/marker population and coding indices. Deleted generations' unreferenced bytes remain separately reported; they are neither live redundancy nor a garbage-collection PASS. These components are migration prerequisites: the complete G04 metadata-fault controller and live acceptance remain outstanding, and component checks cannot certify G04 or H2. + `mpu_race_fixture.py` migrates the frozen validation repository's `heal_mpu_race_fixture.py` model at `524eb5b32c3236497fdd8a9271df0494fa3d9974`. It uses the existing auto-testing `Recorder` and probe directly. Imports are local; no source validation checkout is needed at runtime. Part sizes follow the frozen common DM recipe: `5MiB+17`, `5MiB+93`, and `1MiB+47`. The caller supplies a fresh versioned owned bucket. One completed three-part object and two distinct pending uploads share `objects-0256.bin`: A initially has parts 1/2, B has parts 1/2/3. The model binds each mutation to its actual UploadId/part ETags and durable request/ACK files. A continues with part 3 and completes; B aborts. Unknown or malformed mutation results poison the model and prevent later sibling changes. Bounded paginated listings, exact terminal `404/NoSuchUpload`, all acknowledged historical/current versions and complete GET payloads verify sibling isolation. The model does not clean up uploads with uncertain outcomes. diff --git a/scripts/validation_heal/version_race_fixture.py b/scripts/validation_heal/version_race_fixture.py new file mode 100644 index 00000000..ffd097e0 --- /dev/null +++ b/scripts/validation_heal/version_race_fixture.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""ACK-derived G02-G04 fixtures; never infer versions from wall clocks/current GET. + +The fleet owner supplies Recorder/Probe and runs fault/overlap/physical oracles. +This component is not a standalone concurrency or whole-case PASS generator. +""" +# Frozen validation source: 524eb5b32c3236497fdd8a9271df0494fa3d9974 +# scripts/heal_version_race_fixture.py; only local import adaptation. +import copy +import hashlib +import threading + +try: + from .heal_probe import bucket_allowed,payload,require + from .heal_workload_journal import batch_receipts,validate_batch_request + from .heal_workload import RecordedCallError +except ImportError: + from heal_probe import bucket_allowed,payload,require + from heal_workload_journal import batch_receipts,validate_batch_request + from heal_workload import RecordedCallError + +BRANCHES=('new-version-vs-historical-heal','read-all-versions-during-heal', + 'delete-old-version-vs-deep','delete-current-version-vs-deep', + 'create-marker','delete-marker','batch-version-delete','unversioned-delete') +TARGET='objects-0256.bin' +SIZE=257*1024+123 + + +def version_id(value): + require(isinstance(value,str) and 0=0 and + re.fullmatch('[0-9a-f]{64}',row['sha256']) is not None,'invalid file size or digest') + identity=(row['disk'],row['path']) + require(identity not in indexed,'duplicate physical member');indexed[identity]=row + return indexed + + +def key_state(snapshot,metadata,bucket,key,expected,*,inspector_sha,allowed_orphans=(),allowed_backups=None,allow_multipart=False): + """Bind the exact expected versions to 4 xl.meta decodes and coding indices. + + `expected` is an immutable ACK-ledger slice, not a version list discovered + from this same snapshot. `allowed_orphans` are exact old dataDir UUIDs from + the verified baseline, only for versions with successful deletion ACKs. + """ + require(isinstance(inspector_sha,str) and re.fullmatch('[0-9a-f]{64}',inspector_sha) is not None + and metadata.get('inspectorSha256')==inspector_sha,'candidate inspector digest mismatch') + require(isinstance(key,str) and re.fullmatch(r'objects-[0-9]{4}\.bin',key) is not None, + 'registered flat fixture key required') + require(isinstance(bucket,str) and re.fullmatch('[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]',bucket) is not None, + 'invalid fixture bucket') + require(all(v.get('key')==key for v in expected),'foreign key in expected version slice') + wanted={v['versionId']:v for v in expected} + require(len(wanted)==len(expected) and len(wanted)<=8,'duplicate or excessive expected versions') + require(not wanted or sum(v.get('isLatest') is True for v in wanted.values())==1, + 'expected ACK ledger must have exactly one latest entry') + rows=[v for v in metadata['rows'] if v['key']==key] + require(len(rows)==4 and all(type(v['disk']) is int for v in rows) and + {v['disk'] for v in rows}==set(range(4)),'four distinct metadata observations required') + files=file_index(snapshot);prefix=f'{bucket}/{key}/' + members={i:v for i,v in files.items() if i[1].startswith(prefix)} + used=set();versions={};directories=set();multipart_sidecars=[] + for row in rows: + disk=row['disk'];meta_id=(disk,prefix+'xl.meta') + if row.get('absent') is True: + require(not wanted and meta_id not in files and 'decoded' not in row, + 'metadata absence conflicts with live ACK version or snapshot') + continue + require(row.get('absent') in (None,False) and meta_id in files,'metadata file/observation missing') + require(files[meta_id]['sha256']==row['metadataSha256'],'metadata decode not bound to snapshot SHA') + used.add(meta_id);decoded=row['decoded'] + require(decoded['bucket']==bucket and decoded['key']==key,'decoded object identity differs') + decoded_versions=decoded['versions'] + by_id={canonical_version(v['versionId']):v for v in decoded_versions} + require(len(by_id)==len(decoded_versions) and set(by_id)==set(wanted), + 'physical version/marker population differs from ACK ledger') + require(bool(wanted),'empty xl.meta is not authoritative deletion proof') + for vid,v in by_id.items(): + entry=wanted[vid] + require(type(entry.get('deleteMarker')) is bool and type(entry.get('isLatest')) is bool, + 'ACK marker/latest must be booleans') + require(v['deleted'] is entry['deleteMarker'] and v['isLatest'] is entry['isLatest'], + 'physical latest/marker differs from serial ACK order') + item=versions.setdefault(vid,dict(deleteMarker=entry['deleteMarker'],isLatest=entry['isLatest'], + coding={},layouts=[],dataDirs=[],descriptors=[])) + if v['deleted']: + require(v.get('dataDir') in (None,NIL) and not v.get('parts') and v.get('inline') is None, + 'delete marker unexpectedly references data') + continue + directory=v['dataDir'] + require(isinstance(directory,str) and str(uuid.UUID(directory))==directory and directory!=NIL, + 'invalid external data directory') + require(v.get('inline') is None and v['size']==entry['payload']['size'], + 'physical external object size differs from ACK bytes') + parts=v['parts'];wanted_parts=entry.get('multipartParts') if allow_multipart else None + if wanted_parts is not None: + require(len(wanted_parts)==3 and [p['number'] for p in wanted_parts]==[1,2,3] + and sum(p['size'] for p in wanted_parts)==entry['payload']['size'], + 'three ordered ACK multipart descriptors required') + require([{k:p.get(k) for k in ('number','size')} for p in parts]==wanted_parts, + 'multipart metadata differs from ACK part sizes/order') + else: + require(len(parts)==1 and type(parts[0]['number']) is int and parts[0]['number']==1 + and parts[0]['size']==entry['payload']['size'],'single-part external fixture required') + ec=v['erasure'];index=ec['index'] + require(type(index) is int and index in range(1,5) and index not in item['coding'], + 'invalid or duplicate erasure index') + require(type(ec['data_blocks']) is int and ec['data_blocks']==2 and + type(ec['parity_blocks']) is int and ec['parity_blocks']==2 and + type(ec['block_size']) is int and ec['block_size']>0 and + len(ec['distribution'])==4 and all(type(i) is int for i in ec['distribution']) and + set(ec['distribution'])==set(range(1,5)),'invalid H1 EC2+2 geometry') + require(ec['distribution'][disk]==index,'erasure index differs from disk distribution') + coding={} + for descriptor in parts: + part_id=(disk,prefix+directory+'/part.'+str(descriptor['number'])) + require(part_id in files,'live version lacks a physical shard');used.add(part_id) + part=files[part_id];require(part['size']>0,'external shard cannot be empty') + coding[descriptor['number']]={'sha256':part['sha256'],'size':part['size']} + if wanted_parts is not None: + side_id=(disk,part_id[1]+'.meta') + # Committed authority is xl.meta + all part.N shards. + # part.N.meta belongs to the upload protocol and may be + # reclaimed after commit or omitted by Heal reconstruction. + # Keep any remaining auxiliary bytes separate from coding. + if side_id in files: + require(files[side_id]['size']>0,'empty multipart sidecar') + used.add(side_id);multipart_sidecars.append(copy.deepcopy(files[side_id])) + item['coding'][index]=coding if wanted_parts is not None else coding[1] + item['layouts'].append({k:copy.deepcopy(ec[k]) for k in ('data_blocks','parity_blocks','block_size','distribution')}) + item['dataDirs'].append(directory) + item['descriptors'].append({k:copy.deepcopy(v[k]) for k in ('size','parts')}) + for item in versions.values(): + if item['deleteMarker']:continue + require(set(item['coding'])==set(range(1,5)) and len(set(item['dataDirs']))==1 and + all(v==item['layouts'][0] for v in item['layouts']) and + all(v==item['descriptors'][0] for v in item['descriptors']), + 'live version generation/layout/parts differ across drives') + item['dataDir']=item.pop('dataDirs')[0];item['layout']=item.pop('layouts')[0] + item['descriptor']=item.pop('descriptors')[0] + require(item['dataDir'] not in directories,'distinct live versions share a fixture data directory') + directories.add(item['dataDir']) + allowed=set(allowed_orphans) + require(not directories.intersection(allowed),'deleted generation became a live version') + orphans=[];backups=[] + for identity,row in members.items(): + if identity in used:continue + if identity in (allowed_backups or {}): + original=allowed_backups[identity] + require(all(row[k]==original[k] for k in ('size','sha256')),'metadata backup differs from exact pre-fault bytes') + backups.append(copy.deepcopy(row));continue + relative=identity[1][len(prefix):].split('/') + require(len(relative)==2 and relative[0] in allowed and relative[1]=='part.1', + 'unregistered/orphan physical member not bound to a deleted generation') + orphans.append(copy.deepcopy(row)) + return dict(versions=versions,unreferencedDeletedGenerationFiles=orphans,verifiedMetadataBackups=backups,allLiveVersionsFourShards=True, + observedMultipartSidecars=multipart_sidecars, + casePassClaimed=False,deletedGenerationFilesAbsent=not orphans,garbageCollectorResponsibilityProved=False) + + +def additive_backups(after,before_metadata,after_metadata,bucket,key,before_entries,after_entries,order,allowed,observations,*,inspector_sha): + """Validate exposed decoder semantics, not arbitrary bytes or live authority. + + Only serial additive PUTs are admitted. A rollback view may represent any + ACK prefix; every old descriptor comes from the pre-race decode, every new + one from the final decode which transition also binds to independent data. + """ + old={v['versionId']:v for v in before_entries};new={v['versionId']:v for v in after_entries} + require(isinstance(order,list) and 1<=len(order)<=5 and len(order)==len(set(order)) and + set(old) Date: Wed, 21 Oct 2026 14:15:06 +0800 Subject: [PATCH 275/343] feat(validation): complete frozen G04 marker race runner --- .../workflows/validation-catalog-tests.yml | 2 +- docs/validation-heal-operations.md | 11 +- scripts/validation_heal/fleet_node.py | 9 +- scripts/validation_heal/healthy.py | 2 +- scripts/validation_heal/mpu_race_backups.py | 4 +- scripts/validation_heal/mpu_race_physical.py | 2 +- scripts/validation_heal/mpu_race_runner.py | 2 +- scripts/validation_heal/mpu_scene.py | 6 +- scripts/validation_heal/race.py | 176 ++----------- scripts/validation_heal/race_io.py | 7 +- scripts/validation_heal/race_retention.py | 39 ++- .../validation_heal/recover_environment.py | 3 +- scripts/validation_heal/retained_scene.py | 21 +- .../validation_heal/version_race_adapter.py | 105 ++++++++ .../validation_heal/version_race_physical.py | 53 ++-- .../validation_heal/version_race_runner.py | 244 ++++++++++++++++++ test_validation_heal.py | 6 + test_validation_heal_race.py | 25 +- test_validation_heal_race_physical.py | 28 +- test_validation_heal_race_retention.py | 54 ++++ test_validation_heal_version_race_adapter.py | 148 +++++++++++ test_validation_heal_version_race_runner.py | 214 +++++++++++++++ 22 files changed, 909 insertions(+), 252 deletions(-) create mode 100644 scripts/validation_heal/version_race_adapter.py create mode 100644 scripts/validation_heal/version_race_runner.py create mode 100644 test_validation_heal_version_race_adapter.py create mode 100644 test_validation_heal_version_race_runner.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index aaf4331a..3ff8b34a 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -159,7 +159,7 @@ jobs: sudo apt-get install --no-install-recommends -y nftables fi - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - - run: python3 -B -m unittest -v test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py + - run: python3 -B -m unittest -v test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index e52d3b18..a7cf115c 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -684,11 +684,16 @@ The G06 public adapter uses the existing DS workload: all 257 original keys, inc G03's frozen `VersionLedger.history` determines the new current version after an explicit DELETE. The public S3 inventory is UUID-sorted and cannot supply that order. Each original foreground plan now carries the three seed ACK version IDs in order; the final oracle rejects missing, duplicate, foreign or differently selected identities, removes exactly the acknowledged version and promotes the last retained seed ACK. The physical oracle permits only that `isLatest` change while preserving every other retained descriptor, coding-index byte and data directory. Every original drive must contain exactly the retained version/latest population, with no extra marker or duplicate version. Focused tests use nonmonotonic UUIDs, both delete branches and corrupted physical populations. This source adaptation adds no native or live acceptance. -### G05 same-key multipart components +### Frozen version and multipart race controllers + +G04 and G05 use the owned `frozen-race-legacy` runtime profile. It explicitly sets `RUSTFS_SHARD_INTEGRITY_WRITE=false`, `RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED=false`, `RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE=false`, and `RUSTFS_HEAL_MAX_CONCURRENT_HEALS=1`, matching the frozen source's legacy shard population and serial scheduling. The profile retains disabled scanner, automatic Heal and MRF. The regular `manual` profile continues to enable protected shard writes. Original physical file-population assertions remain strict; an unexpected integrity sidecar is a failure. Owned service configuration is attested through the existing runtime checks. G04 alone may use the frozen fourth-drive write cap under this profile; G05 only observes kernel limits. Retention supports both historical `manual` evidence and the new profile, with the same fixed bucket/key scope and stopped original scene binding. An older helper's tools remain bound to that helper; they cannot certify a newer controller. + `version_race_fixture.py` preserves the frozen `heal_version_race_fixture.py` model at `524eb5b32c3236497fdd8a9271df0494fa3d9974` (source SHA256 `755f476ab567dc413e9798bb2bcac15252c3e8308e4b15b15a9eb3083d1df485`), with local import adaptation only. Its eight G02–G04 plans derive versions from serial ACK order. G04 uses flat keys, seeds a latest marker for both marker removal and batch deletion, and deletes exactly the oldest target version, newest target version, marker and control `objects-0002.bin` version. It preserves the middle target generation. A partially acknowledged batch retains each known effect and permanently blocks final acceptance and further mutations. Historical reads address exact versions; negative reads require the original deleted-version 404 and marker 405/current-marker 404 contracts. -`version_race_physical.py` preserves the frozen H1 physical oracle, replacing the source transport's fixed inspector digest with a mandatory caller-supplied immutable tool digest. Every drive must match the ACK-derived version/marker population and coding indices. Deleted generations' unreferenced bytes remain separately reported; they are neither live redundancy nor a garbage-collection PASS. These components are migration prerequisites: the complete G04 metadata-fault controller and live acceptance remain outstanding, and component checks cannot certify G04 or H2. +`version_race_physical.py` preserves the frozen physical oracle, replacing the source transport's fixed inspector digest with a mandatory caller-supplied immutable tool digest. `version_race_adapter.py` maps every owned drive to its real format set slot; H1 retains EC2+2 and H2 explicitly requires the deployed EC12+4 geometry. Every drive must match the ACK-derived version/marker population and coding indices. Deleted generations' unreferenced bytes remain separately reported; they are neither live redundancy nor a garbage-collection PASS. + +`version_race_runner.py` replaces the superseded generic G04 fixture and shard-fault path. Its 257 faults remove the first host's fourth-drive `xl.meta` files. The original 120-second partial window requires nonzero original-token progress, a physically reconstructed control, at least 32 pending objects and the selected target metadata still absent. One original SDK mutation runs with the source 90-second observation budget and 25ms sample pause; the owned fourth-drive 65536-byte/s write cap is released while the original call drains. Final checks prove the 253 other controls, exact version/marker/coding populations, full live part streams, every peer's deleted-version/marker negatives and complete inventory, post-GET physical immutability, ACK-bounded completion and original root retirement. Final assertion failures preserve independent diagnostics; evidence I/O failures stop execution. Seed or foreground uncertainty retains the stopped scene and original journals as DIRTY. Recovery can isolate that exact scene without restoration, retry, deletion or a PASS rewrite. Runtime entry requires source `8380466` and the complete marker/batch native group before bootstrap; the legacy source cannot enter this controller. These implementation/component checks add no canonical PASS; fresh exact-helper native tools and complete H1/H2 live acceptance remain required. `mpu_race_fixture.py` migrates the frozen validation repository's `heal_mpu_race_fixture.py` model at `524eb5b32c3236497fdd8a9271df0494fa3d9974`. It uses the existing auto-testing `Recorder` and probe directly. Imports are local; no source validation checkout is needed at runtime. Part sizes follow the frozen common DM recipe: `5MiB+17`, `5MiB+93`, and `1MiB+47`. @@ -702,7 +707,7 @@ The historical `09fb813` entry remains collector-only. Runtime admission binds s Final committed/staging/control proofs precede all-peer S3 verification. Assertion failures retain the frozen source's remaining independent diagnostics while evidence I/O errors stop execution. Repeated terminal responsibility, ACK bounds, post-GET physical stability and exact original-root retirement must pass before cleanup authority exists. Any issued seed request without that authority retains the stopped raw MPU prefix, committed scene and original request/ACK files as DIRTY, with no retry, abort, restore or namespace cleanup. A missing overlap remains `NOT_OBSERVED` even after safe cleanup. Full hosted native acceptance and all six public live instances still require independent evidence before G05 can count as canonical PASS. -The additive `mpu-race-scene` node action in `mpu_race_physical.py` adapts the frozen `heal_h1_mpu_physical.py` upload-ID and byte-continuity oracle to the current owned four-drive hosts. Each request binds the manual runtime, bucket, candidate binary and all three ACK-selected uploads. It inventories the complete fixed key prefix, hashes only selected staging files under bounded entry/depth/byte limits, rejects unknown uploads and links, and preserves original process identity during a live capture. Stopped captures additionally verify the owned service remains stopped. Across H1/H2, pending part/sidecar populations and existing sibling bytes remain exact; a completed upload must lose its staging directory and retain all three previously witnessed encoded parts in committed storage. These are staging/continuity witnesses; independent inspector coding/bitrot checks, product Heal proof, real overlap and full SDK inventories remain required. +The additive `mpu-race-scene` node action in `mpu_race_physical.py` adapts the frozen `heal_h1_mpu_physical.py` upload-ID and byte-continuity oracle to the current owned four-drive hosts. Each request binds the owned race runtime, bucket, candidate binary and all three ACK-selected uploads. It inventories the complete fixed key prefix, hashes only selected staging files under bounded entry/depth/byte limits, rejects unknown uploads and links, and preserves original process identity during a live capture. Stopped captures additionally verify the owned service remains stopped. Across H1/H2, pending part/sidecar populations and existing sibling bytes remain exact; a completed upload must lose its staging directory and retain all three previously witnessed encoded parts in committed storage. These are staging/continuity witnesses; independent inspector coding/bitrot checks, product Heal proof, real overlap and full SDK inventories remain required. ### J02 original opaque metadata probe diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 71c829f8..dbffc396 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -33,6 +33,13 @@ def runtime_config(profile): + if profile == "frozen-race-legacy": + # The frozen G04/G05 fixture predates protected shard sidecars. Keep + # its exact file population and serial Heal scheduling explicit. + return {**BACKGROUND_CONFIG, "RUSTFS_SHARD_INTEGRITY_WRITE": "false", + "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED": "false", + "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE": "false", + "RUSTFS_HEAL_MAX_CONCURRENT_HEALS": "1", "RUST_LOG": "info"} if profile in {"incarnation-checkpoint", "incarnation-mrf"}: logging_filter = "info,rustfs::heal::manager=debug,rustfs::heal::task=debug,rustfs::heal::mrf=debug" return {**BACKGROUND_CONFIG, "RUSTFS_HEAL_MRF_ENABLE": "true" if profile == "incarnation-mrf" else "false", @@ -544,7 +551,7 @@ def start(request): "J02 phase commit is incomplete; service cannot start") if owner.get("outcomePhase") == "legacy": outcome_legacy_phase({"runId": owner["runId"], "action": "outcome-legacy-phase"}) - if owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf", "outcome-legacy"}: + if owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf", "outcome-legacy", "frozen-race-legacy"}: runtime_configuration(state, owner) command("systemd-run", "--unit=" + owner["unit"], "--property=Type=exec", "--property=KillMode=control-group", "--property=TimeoutStopSec=30", diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 36115c92..2a2e6e3f 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"outcome-legacy", "outcome-grace", "queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation", "incarnation-checkpoint", "incarnation-mrf"}, "unsupported Heal runtime profile") + require(runtime_profile in {"frozen-race-legacy", "outcome-legacy", "outcome-grace", "queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation", "incarnation-checkpoint", "incarnation-mrf"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory diff --git a/scripts/validation_heal/mpu_race_backups.py b/scripts/validation_heal/mpu_race_backups.py index cb31ebe3..ecd1d2b4 100644 --- a/scripts/validation_heal/mpu_race_backups.py +++ b/scripts/validation_heal/mpu_race_backups.py @@ -28,7 +28,9 @@ def run(request,node): key = KEY if request['action']=='mpu-race-backups' else 'objects/0256.bin' generations = 3 if request['action']=='mpu-race-backups' else 6 state,roots,owner=node['owned'](request['runId']) - require(owner.get('runtimeProfile')=='manual' and owner.get('bucket')==request['bucket'] + require((owner.get('runtimeProfile')=='manual' or + owner.get('runtimeProfile')=='frozen-race-legacy' and request['action']=='mpu-race-backups') + and owner.get('bucket')==request['bucket'] and owner.get('binarySha256')==request['expectedBinarySha256'] and owner.get('inspectorSha256')==request['expectedInspectorSha256'] and all(re.fullmatch(r'[0-9a-f]{64}',request[k]) for k in ('expectedBinarySha256','expectedInspectorSha256')) diff --git a/scripts/validation_heal/mpu_race_physical.py b/scripts/validation_heal/mpu_race_physical.py index e7a45725..83d6a8fd 100644 --- a/scripts/validation_heal/mpu_race_physical.py +++ b/scripts/validation_heal/mpu_race_physical.py @@ -54,7 +54,7 @@ def run(request, node): and request['action'] == 'mpu-race-scene' and type(request['quiescent']) is bool, 'G05 census accepts an exact owned read-only upload scope') state, roots, owner = node['owned'](request['runId']) - require(owner.get('runtimeProfile') == 'manual' and owner.get('bucket') == request['bucket'] + require(owner.get('runtimeProfile') in {'manual', 'frozen-race-legacy'} and owner.get('bucket') == request['bucket'] and owner.get('binarySha256') == request['expectedBinarySha256'] and re.fullmatch(r'[0-9a-f]{64}', request['expectedBinarySha256']) and len(roots) == 4, 'G05 census owner/runtime/binary differs') diff --git a/scripts/validation_heal/mpu_race_runner.py b/scripts/validation_heal/mpu_race_runner.py index 6d41b459..1ff8c47a 100644 --- a/scripts/validation_heal/mpu_race_runner.py +++ b/scripts/validation_heal/mpu_race_runner.py @@ -286,7 +286,7 @@ def run_branch(args, topology, nodes, branch): reference = race.COPY_REFERENCE directory = args.evidence / topology / branch; directory.mkdir(parents=True, exist_ok=False) run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" - fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile='manual') + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile='frozen-race-legacy') inspector = fleet.record('inspector-build', verify_inspector(args.tool_directory, candidate)) bucket = 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) diff --git a/scripts/validation_heal/mpu_scene.py b/scripts/validation_heal/mpu_scene.py index e735f6d7..b594cce1 100644 --- a/scripts/validation_heal/mpu_scene.py +++ b/scripts/validation_heal/mpu_scene.py @@ -26,7 +26,7 @@ def keys_for(endpoints): def validate_scope(owner, keys): - if owner.get('runtimeProfile') == 'manual': + if owner.get('runtimeProfile') in {'manual', 'frozen-race-legacy'}: require(re.fullmatch(r'rv-heal-[0-9a-f]{16}', owner.get('bucket', '')) and keys == [G05_KEY], 'G05 MPU retention requires its one frozen same-key prefix') return 'the original G05 bucket/key SHA-256 prefix, including unknown upload IDs; not a bitrot/EC proof' @@ -47,7 +47,7 @@ def run(request, node): require(set(request) == {'runId', 'action', 'bucket', 'keys', 'expectedBinarySha256'} and request['action'] == 'mpu-scene', 'MPU scene accepts only an exact owned read-only scope') state, roots, owner = node['owned'](request['runId']) - require(owner.get('runtimeProfile') in {'lock-quorum', 'manual'} and owner.get('bucket') == request['bucket'] + require(owner.get('runtimeProfile') in {'lock-quorum', 'manual', 'frozen-race-legacy'} and owner.get('bucket') == request['bucket'] and owner.get('binarySha256') == request['expectedBinarySha256'] and re.fullmatch(r'[0-9a-f]{64}', request['expectedBinarySha256']) is not None and len(roots) == 4, 'MPU scene candidate/runtime/primary bucket differs from the owned H2 fixture') @@ -79,7 +79,7 @@ def visit(path, depth=0): directories.append({'disk': disk, 'path': relative, 'key': key}) children = [] for child in path.iterdir(): - require(len(children) < (3 if depth == 0 and owner['runtimeProfile'] == 'manual' else MAX_FILES), + require(len(children) < (3 if depth == 0 and owner['runtimeProfile'] in {'manual', 'frozen-race-legacy'} else MAX_FILES), 'MPU scene directory population exceeded its bound') children.append(child) for child in sorted(children): diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index 11772993..2a1a9f81 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -122,7 +122,8 @@ def source_contract(candidate, case_id): require(case_id in CASE_IDS, 'unregistered race case') if candidate.get('source_sha') == REFERENCE['candidateCommit']: - require(case_id != 'heal.race-multipart-complete-abort', 'G05 requires its exact-source multipart native group') + require(case_id not in ('heal.race-multipart-complete-abort','heal.race-markers-and-batch-delete'), + 'G04/G05 require their exact-source multipart or marker/batch native group') return REFERENCE entry = NATIVE_RUNTIME.get(candidate.get('source_sha')) require(entry is not None and case_id in entry['cases'] and case_id in FULL_NATIVE_GROUPS @@ -160,8 +161,6 @@ class RaceRecorder(Recorder): def __init__(self, *args, versioned, **kwargs): super().__init__(*args, **kwargs) self.versioned, self.pending, self.seen = versioned, None, set() - self.marker_branch = None - self.batch_expected_ids = set() def call(self, operation, params, body=None, **kwargs): self.pending = {"operation": operation, "params": copy.deepcopy(params)} @@ -177,32 +176,10 @@ def call(self, operation, params, body=None, **kwargs): else: require(version in {None, "null"}, "G01/G03 never-versioned PUT created a real version") elif operation == "delete_object": - if self.marker_branch in G04_BRANCHES: - known = {value for key, value in self.seen if key == params["Key"]} - if params.get("VersionId") is None: - if self.versioned: - require(result.get("DeleteMarker") is True and isinstance(version, str) - and version not in known, "G04 marker creation ACK lacks a new marker identity") - self.seen.add((params["Key"], version)) - else: - require(version in {None, "null"} and result.get("DeleteMarker") in {None, False}, - "G04 unversioned DELETE ACK created an unexpected marker") - else: - require(self.versioned and params["VersionId"] in known and version == params["VersionId"] - and result.get("DeleteMarker") is True, - "G04 marker removal ACK changed the selected marker identity") - else: - require(self.versioned and params.get("VersionId") in { - value for key, value in self.seen if key == params["Key"]} - and version == params["VersionId"] and result.get("DeleteMarker") in {None, False}, - "G01/G03 DELETE ACK changed the exact known data-version identity") - elif operation == "delete_objects" and self.marker_branch == "batch-version-delete": - deleted = result.get("Deleted") or [] - errors = result.get("Errors") or [] - deleted_ids = {row.get("VersionId") for row in deleted} - require(not errors and deleted_ids == self.batch_expected_ids - and all(isinstance(value, str) for value in deleted_ids), - "G04 batch DELETE ACK has errors or incomplete per-item receipts") + require(self.versioned and params.get("VersionId") in { + value for key, value in self.seen if key == params["Key"]} + and version == params["VersionId"] and result.get("DeleteMarker") in {None, False}, + "G01/G03 DELETE ACK changed the exact known data-version identity") self.pending = None return result @@ -233,43 +210,11 @@ def sample_overlap(ack, observed, path, token, reference=REFERENCE): def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): - require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES, *G06_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G02/G03/G04/G05/G06 unregistered fixture") + require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G06_BRANCHES) and type(count) is int and 3 <= count <= OBJECTS, "G01/G02/G03/G04/G05/G06 unregistered fixture") if branch in G06_BRANCHES: require(count == OBJECTS, "G06 requires the frozen complete 257-key DS population") return seed_workload(recorder, bucket, "DS", count, [SIZE]) recorder.call("create_bucket", {"Bucket": bucket}) - if branch in G04_BRANCHES: - versioned = branch != "unversioned-delete" - if versioned: - recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) - entries = [] - for index in range(count - 1): - key = f"objects/{index:04d}.bin" - body = payload(f"{bucket}/{key}/0", SIZE) - response = recorder.call("put_object", {"Bucket": bucket, "Key": key, - "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": "G04", "nonce": str(index)}, - "Tagging": "suite=heal&fixture=G04"}, body) - entries.append({"key": key, "versionId": public_version_id(response.get("VersionId")), - "isLatest": True, "deleteMarker": False, - "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) - target_versions = 1 if not versioned else 3 - for generation in range(target_versions): - body = payload(f"{bucket}/{TARGET_KEY}/{generation}", SIZE) - response = recorder.call("put_object", {"Bucket": bucket, "Key": TARGET_KEY, - "ContentType": "application/octet-stream", "Metadata": {"heal-fixture": "G04", "generation": str(generation)}, - "Tagging": "suite=heal&fixture=G04"}, body) - version = public_version_id(response.get("VersionId")) - for row in entries: - if row["key"] == TARGET_KEY: row["isLatest"] = False - entries.append({"key": TARGET_KEY, "versionId": version, "isLatest": generation == target_versions - 1, - "deleteMarker": False, "payload": {"size": len(body), "sha256": hashlib.sha256(body).hexdigest()}}) - if branch == "delete-marker": - marker = recorder.call("delete_object", {"Bucket": bucket, "Key": TARGET_KEY}) - marker_id = marker.get("VersionId") - for row in entries: - if row["key"] == TARGET_KEY: row["isLatest"] = False - entries.append({"key": TARGET_KEY, "versionId": marker_id, "isLatest": True, "deleteMarker": True}) - return entries if branch in (*G02_BRANCHES, *G03_BRANCHES): recorder.call("put_bucket_versioning", {"Bucket": bucket, "VersioningConfiguration": {"Status": "Enabled"}}) entries = [] @@ -290,7 +235,7 @@ def seed_fixture(recorder, bucket, branch, *, count=OBJECTS): def operation_plan(bucket, branch, expected): - require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES, *G06_BRANCHES), "G01/G02/G03/G04/G05/G06 unregistered foreground branch") + require(branch in (*G01_BRANCHES, *G02_BRANCHES, *G03_BRANCHES, *G06_BRANCHES), "G01/G02/G03/G04/G05/G06 unregistered foreground branch") if branch in G01_BRANCHES: # Three distinct serial commits; the final independent reference was # written before injection, allowing a coding-index byte comparison. @@ -309,27 +254,6 @@ def operation_plan(bucket, branch, expected): "nonce": f"{bucket}/objects/{index:04d}.bin/0", "referenceKey": f"objects/{index:04d}.bin", } for index in range(3)] - if branch in G04_BRANCHES: - target = [row for row in expected if row["key"] == TARGET_KEY] - if branch == "create-marker": - require(len(target) == 3 and target[-1]["isLatest"], "G04 create-marker fixture lacks current data") - return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": TARGET_KEY}, - "nonce": "create-marker"}] - if branch == "delete-marker": - marker = next((row for row in target if row["deleteMarker"] and row["isLatest"]), None) - require(marker is not None, "G04 delete-marker fixture lacks an acknowledged current marker") - return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": TARGET_KEY, - "VersionId": marker["versionId"]}, - "nonce": "delete-marker"}] - if branch == "batch-version-delete": - data = [row for row in target if not row["deleteMarker"]] - require(len(data) == 3, "G04 batch fixture lacks three data versions") - return [{"operation": "delete_objects", "params": {"Bucket": bucket, - "Delete": {"Objects": [{"Key": TARGET_KEY, "VersionId": data[0]["versionId"]}, - {"Key": TARGET_KEY, "VersionId": data[1]["versionId"]}], - "Quiet": False}}, "nonce": "batch-version-delete"}] - return [{"operation": "delete_object", "params": {"Bucket": bucket, "Key": TARGET_KEY}, - "nonce": "unversioned-delete"}] if branch in G06_BRANCHES: if branch == "copy-source": return [{"operation": "copy_object", "params": {"Bucket": bucket, "Key": current_reference, @@ -405,35 +329,6 @@ def final_inventory(baseline, branch, plan, responses): tags=[{"Key": "race", "Value": "overwrite"}]) entry["properties"].update(ETag=ack["ETag"], ContentLength=len(body), ContentType="application/octet-stream", Metadata=step["params"]["Metadata"]) - elif branch in G04_BRANCHES: - require(len(responses) == 1, "G04 foreground plan/ACK sequence is incomplete") - target_rows = [row for row in result["entries"] if row["key"] == TARGET_KEY] - if branch == "create-marker": - marker = responses[0].get("VersionId") - require(responses[0].get("DeleteMarker") is True and isinstance(marker, str), - "G04 marker creation ACK lacks its exact marker identity") - for row in target_rows: row["isLatest"] = False - result["entries"].append({"key": TARGET_KEY, "versionId": marker, - "isLatest": True, "deleteMarker": True}) - elif branch == "delete-marker": - marker = next(row for row in target_rows if row["deleteMarker"] and row["isLatest"]) - require(responses[0].get("DeleteMarker") is True and responses[0].get("VersionId") == marker["versionId"], - "G04 marker delete ACK changed the selected marker identity") - result["entries"] = [row for row in result["entries"] if row is not marker] - data = [row for row in result["entries"] if row["key"] == TARGET_KEY and not row["deleteMarker"]] - require(data, "G04 marker delete removed every retained data version") - for row in data: row["isLatest"] = row is data[-1] - elif branch == "batch-version-delete": - deleted = responses[0].get("Deleted") or [] - deleted_ids = {row.get("VersionId") for row in deleted} - require(len(deleted_ids) == 2 and not responses[0].get("Errors"), - "G04 batch delete ACK is not a complete per-item success") - result["entries"] = [row for row in result["entries"] - if not (row["key"] == TARGET_KEY and row["versionId"] in deleted_ids)] - else: - require(responses[0].get("VersionId") in {None, "null"} and responses[0].get("DeleteMarker") in {None, False}, - "G04 unversioned delete ACK changed object identity") - result["entries"] = [row for row in result["entries"] if row["key"] != TARGET_KEY] elif branch in G06_BRANCHES: entry = next(row for row in result["entries"] if row["key"] == current_target) copy_result = responses[-1].get("CopyObjectResult") or {} @@ -565,36 +460,6 @@ def coding(target): require(all(type(v.get('isLatest')) is bool and type(v.get('deleted')) is bool for v in versions) and sorted((public_version_id(v.get('versionId')), v['isLatest'], v['deleted']) for v in versions) == expected_versions, 'G02 physical version population differs from all original and new ACKs') - elif branch in G04_BRANCHES: - require(len(original_target) == (1 if branch == "unversioned-delete" else 3), - "G04 baseline has the wrong acknowledged data-version count") - old_versions = {target["versionId"] for target in original_target} - remaining = {target["versionId"] for target in current_target_values} - expected_remaining = (set() if branch == "unversioned-delete" else - {original_target[-1]["versionId"]} if branch == "batch-version-delete" else old_versions) - require(remaining == expected_remaining, - "G04 operation lost, replaced or retained the wrong acknowledged data versions") - for target in current_target_values: - original = mapped[current_target, target["versionId"]] - require({name: value for name, value in target["logical"].items() if name != "isLatest"} - == {name: value for name, value in original["logical"].items() if name != "isLatest"} - and [(part["index"], part["sha256"], part["size"]) for part in target["cohort"]] - == [(part["index"], part["sha256"], part["size"]) for part in original["cohort"]], - "G04 marker/delete changed another retained data generation or shard") - if current_target_values: - expected_versions = sorted((row["versionId"], row["isLatest"], row["deleteMarker"]) - for row in final["entries"] if row["key"] == current_target) - indexed = files(after) - for node, value in after.items(): - for drive in value["formats"]: - metadata = indexed[node, drive["disk"], f"{bucket}/{current_target}/xl.meta"]["metadata"] - versions = metadata.get("versions", []) - require(all(type(version.get("isLatest")) is bool and type(version.get("deleted")) is bool - for version in versions), "G04 physical version flags are missing or untyped") - actual_versions = sorted((public_version_id(version.get("versionId")), - version["isLatest"], version["deleted"]) for version in versions) - require(actual_versions == expected_versions, - "G04 physical data/marker version set differs from exact foreground ACKs") elif branch in G06_BRANCHES: target_after = next(value for (key, _), value in survivors.items() if key == changed_key) target_before = next(value for (key, _), value in mapped.items() if key == changed_key) @@ -638,8 +503,6 @@ def coding(target): if key is None: continue prefix = f"{bucket}/{key}/" current_dirs = {target["dataDir"] for (name, _), target in survivors.items() if name == key} - if branch == "unversioned-delete": - raise ValueError("G04 unversioned DELETE left a physical target leaf") suffix = row["path"][len(prefix):] require(suffix == "xl.meta" or suffix.split("/")[0] in current_dirs, "G01/G03 resurrected an old/deleted data directory or left an unbound target leaf") @@ -863,6 +726,14 @@ def read_repair(): def run_branch(args, topology, nodes, branch): + if args.case_id == 'heal.race-markers-and-batch-delete': + try: + from .version_race_runner import run_branch as run_marker_branch + except ImportError: + from version_race_runner import run_branch as run_marker_branch + source_contract(json.loads(args.candidate.read_text()), args.case_id) + return run_marker_branch(args, topology, nodes, branch) + if args.case_id == 'heal.race-multipart-complete-abort': # Lazy import keeps the generic controller and G05's model acyclic. try: @@ -894,19 +765,17 @@ def run_branch(args, topology, nodes, branch): "requiredPUTs": 3 if branch in (*G01_BRANCHES, *G02_BRANCHES) else 0, "serverCommitOverlapProved": False}) probe, active, recorder, writer = Probe(binding), [], None, None result = {"topology": topology, "branch": branch, - "fixture": "DL" if branch in (*G01_BRANCHES, "unversioned-delete") else "DV" if branch in G04_BRANCHES - else "DS" if branch in G06_BRANCHES else "DV", + "fixture": "DL" if branch in G01_BRANCHES else "DS" if branch in G06_BRANCHES else "DV", "verdict": "RUNNING", "cleanup": "PENDING", "canonicalCasePassClaimed": False} changed_target_registered, target_authority = None, False try: fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) recorder = RaceRecorder(probe, directory / "seed-ACKs", - versioned=branch in (*G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES[:3])) - recorder.marker_branch = branch if branch in G04_BRANCHES else None + versioned=branch in (*G02_BRANCHES, *G03_BRANCHES)) expected = seed_fixture(recorder, bucket, branch) baseline = fleet.record("complete-baseline-S3", probe.inventory( - bucket, unversioned_current_tags=branch in (*G01_BRANCHES, "unversioned-delete", *G06_BRANCHES))) + bucket, unversioned_current_tags=branch in (*G01_BRANCHES, *G06_BRANCHES))) verify_seed(expected, baseline) plan = operation_plan(bucket, branch, expected) for step in plan: @@ -952,11 +821,8 @@ def run_branch(args, topology, nodes, branch): if (target["node"], target["disk"], target["relativePath"]) in files(witness): raise WindowNotObserved("G01/G03 old target already repaired before foreground mutations") writer = RaceRecorder(probe, directory / "foreground-ACKs", - versioned=branch in (*G02_BRANCHES, *G03_BRANCHES, *G04_BRANCHES[:3])) - writer.marker_branch = branch if branch in G04_BRANCHES else None + versioned=branch in (*G02_BRANCHES, *G03_BRANCHES)) writer.seen.update((row["key"], row["versionId"]) for row in expected) - if branch == "batch-version-delete": - writer.batch_expected_ids = {item["VersionId"] for item in plan[0]["params"]["Delete"]["Objects"]} if branch in G02_BRANCHES: responses, overlapped = race(writer, plan, probe, fleet, path, token, reference, baseline=baseline) else: @@ -995,7 +861,7 @@ def run_branch(args, topology, nodes, branch): fleet.start() for endpoint in binding["endpoints"]: current = fleet.record("all-peer-final-S3", {"endpoint": endpoint, **Probe(binding, endpoint).inventory( - bucket, unversioned_current_tags=branch in (*G01_BRANCHES, "unversioned-delete", *G06_BRANCHES))}) + bucket, unversioned_current_tags=branch in (*G01_BRANCHES, *G06_BRANCHES))}) compare_inventories(final, current) if branch in G03_BRANCHES: negative = fleet.record("all-peer-deleted-version-GET", Probe(binding, endpoint).s3("get_object", plan[0]["params"])) diff --git a/scripts/validation_heal/race_io.py b/scripts/validation_heal/race_io.py index 9d26b521..2bba553d 100644 --- a/scripts/validation_heal/race_io.py +++ b/scripts/validation_heal/race_io.py @@ -47,8 +47,11 @@ def run(request, node): and request['branch'] in CAP_BRANCHES.get(request['caseId'], ())), 'write cap requires a registered frozen public race; G05 forbids this cap') state, roots, owner = node['owned'](request['runId']) - require(owner.get('runtimeProfile') == 'manual' and len(roots) == len(node['MOUNTS']) == 4, - 'write cap requires the original four-drive manual owner') + legacy_case = (observing or request['caseId'] == 'heal.race-markers-and-batch-delete') + require((owner.get('runtimeProfile') == 'manual' or + owner.get('runtimeProfile') == 'frozen-race-legacy' and legacy_case) + and len(roots) == len(node['MOUNTS']) == 4, + 'write cap requires the original four-drive race owner') expected = request['expectedProcess'] def process(): actual = node['process'](request) diff --git a/scripts/validation_heal/race_retention.py b/scripts/validation_heal/race_retention.py index 79ee6788..f88698b6 100644 --- a/scripts/validation_heal/race_retention.py +++ b/scripts/validation_heal/race_retention.py @@ -15,6 +15,7 @@ CASE = 'heal.race-unversioned-overwrite' CASES = {CASE: ('unresolved-g01-race', {'overwrite-vs-deep', 'overwrite-vs-read-repair'}), 'heal.race-versioned-put': ('unresolved-g02-race', {'new-version-vs-historical-heal', 'read-all-versions-during-heal'}), + 'heal.race-markers-and-batch-delete': ('unresolved-g04-race', {'create-marker','delete-marker','batch-version-delete','unversioned-delete'}), 'heal.admin-nolock-safety': ('unresolved-a09-race', {'overwrite-nolock', 'delete-nolock'}), 'heal.race-multipart-complete-abort': ('unresolved-g05-race', {'upload-continue', 'complete', 'abort'}), 'heal.race-copy-and-metadata-update': ('unresolved-g06-race', {'copy-source', 'copy-destination', 'replace-metadata', 'update-tags'})} @@ -41,7 +42,10 @@ def validate(plan): for node in plan['nodes']: owner = plan['owners'][node] require(owner.get('runId') == plan['runId'] and owner.get('bucket') == plan['bucket'] - and owner.get('runtimeProfile') == 'manual' and owner.get('binarySha256') == plan['binarySha256'] + and (owner.get('runtimeProfile') == 'manual' or + owner.get('runtimeProfile') == 'frozen-race-legacy' + and plan['kind'] in {'unresolved-g04-race', 'unresolved-g05-race'}) + and owner.get('binarySha256') == plan['binarySha256'] and owner.get('unit') == 'rustfs-validation-heal-' + plan['runId'] + '.service' and len(owner.get('mounts', [])) == 4 and re.fullmatch(r'[0-9a-f]{64}', plan['businessSceneSha256'][node]), @@ -71,7 +75,8 @@ def archived_plan(artifact, failed_run, candidate): rows = summary.get('results', []) retained = [row for row in rows if row.get('cleanup') != 'PASS'] require(summary.get('caseId') == case and summary.get('cleanup') == 'FAIL' and len(retained) == 1 - and retained[0].get('verdict') == 'FAIL' and retained[0].get('topology') in {'H1', 'H2'} + and retained[0].get('verdict') in ({'FAIL','NOT_OBSERVED'} if kind=='unresolved-g04-race' else {'FAIL'}) + and retained[0].get('topology') in {'H1', 'H2'} and retained[0].get('branch') in branches and all(row.get('caseId', case if kind == 'unresolved-a09-race' else None) == case for row in rows), 'G01/G02 isolation requires one unmixed retained public branch') @@ -97,15 +102,19 @@ def one(label): and all(item.get('reserved') is True for item in reservations), 'G01/G02 isolation lacks all reserved hosts') ack_files = {} multipart = kind == 'unresolved-g05-race' + markers = kind == 'unresolved-g04-race' read_journals = (['mpu-read-ACKs', *['final-read-ACKs-node' + str(i) for i in range(1, len(owners) + 1)]] - if multipart else []) + if multipart else ['negative-read-ACKs-node'+str(i) for i in range(1,len(owners)+1)] if markers else []) if multipart: require({path.name for path in folder.glob('final-read-ACKs-*')} <= set(read_journals), 'G05 isolation has an unbound final peer read journal') + if markers: + require({path.name for path in folder.glob('negative-read-ACKs-*')} <= set(read_journals), + 'G04 isolation has an unbound negative peer read journal') for name in ('seed-ACKs', 'foreground-ACKs', 'historical-read-ACKs', *read_journals): journal = folder / name entries = sorted(journal.glob('*.json')) - if name == 'historical-read-ACKs' or name in read_journals or multipart and name == 'foreground-ACKs': + if name == 'historical-read-ACKs' or name in read_journals or (multipart or markers) and name == 'foreground-ACKs': require(len(entries) <= 1024, 'G02 historical journal exceeds its bound') if not entries: continue require(0 < len(entries) <= 1024, 'G01/G02 isolation lacks its bounded original request/ACK journal') @@ -113,6 +122,15 @@ def one(label): require(any(re.fullmatch(r'[0-9]{6}-request\.json', path.name) for path in entries), 'G05 isolation requires an original SDK request, not only a model manifest') for path in entries: + if markers and name in read_journals and re.fullmatch(r'[0-9]{6}-negative-proof\.json',path.name): + require(path.stat().st_size<=1024**2,'G04 negative proof exceeds its evidence bound') + item=json.loads(path.read_text());expected=item.get('expected') or {};params=expected.get('params') or {} + require(params.get('Bucket')==binding['bucketPrefix'] + and params.get('Key') in {f'objects-{i:04d}.bin' for i in range(257)} + and expected.get('httpStatus') in (404,405) and item.get('state') in ('PASS','FAIL') + and item.get('canonicalCasePassClaimed') is False, + 'G04 negative proof belongs to another object or contract') + ack_files[name+'/'+path.name]=hashlib.sha256(path.read_bytes()).hexdigest();continue if multipart and re.fullmatch(r'mpu-manifest-[0-9]{4}\.json', path.name): require(path.stat().st_size <= 1024**2, 'G05 isolation multipart manifest exceeds its bound') item = json.loads(path.read_text()) @@ -128,7 +146,7 @@ def one(label): # A missing/unknown ACK remains unknown. Isolation never infers its # effect or sends another request, including for explicit refusals. operation = item.get('operation') if path.name.endswith('-request.json') else (item.get('observation') or {}).get('operation') - if multipart: + if multipart or markers: request_item = item if path.name.endswith('-ack.json'): request_name = item.get('requestFile', '') @@ -148,10 +166,17 @@ def one(label): 'G05 isolation SDK request belongs to another reserved bucket') if operation in {'create_multipart_upload', 'upload_part', 'complete_multipart_upload', 'abort_multipart_upload', 'list_parts', 'get_object', 'head_object', 'get_object_tagging'}: - require(params.get('Key') == 'objects-0256.bin', 'G05 isolation SDK request changed its frozen multipart key') + if multipart:require(params.get('Key') == 'objects-0256.bin', 'G05 isolation SDK request changed its frozen multipart key') + if markers: + keys=([v.get('Key') for v in params.get('Delete',{}).get('Objects',[])] if operation=='delete_objects' + else [params.get('Key')] if 'Key' in params else []) + require(all(isinstance(key,str) and key in {f'objects-{i:04d}.bin' for i in range(257)} for key in keys) + and (operation!='delete_objects' or 0 0, "exact failed workflow run required") if (list(artifact.glob("*/run/heal.race-unversioned-overwrite/cases/*.json")) or list(artifact.glob("*/run/heal.race-versioned-put/cases/*.json")) + or list(artifact.glob("*/run/heal.race-markers-and-batch-delete/cases/*.json")) or list(artifact.glob("*/run/heal.admin-nolock-safety/cases/*.json")) or list(artifact.glob("*/run/heal.race-copy-and-metadata-update/cases/*.json")) or list(artifact.glob("*/run/heal.race-multipart-complete-abort/cases/*.json"))): @@ -651,7 +652,7 @@ def main(): if plan.get("buckets"): fleet.snapshot_buckets = plan["buckets"] fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) - if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race", "unresolved-a09-race", "unresolved-g05-race", "unresolved-g06-race"}: + if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race", "unresolved-a09-race", "unresolved-g04-race", "unresolved-g05-race", "unresolved-g06-race"}: complete.append({"failedRunId": args.failed_run_id, **isolate_race_scene(fleet, plan)}) continue if plan.get("kind") == "unresolved-network-load": diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py index 96015bbd..0898f138 100644 --- a/scripts/validation_heal/retained_scene.py +++ b/scripts/validation_heal/retained_scene.py @@ -99,7 +99,7 @@ def census(folder, owner, *, state, node, mpu_keys=()): multipart_keys = {hashlib.sha256(f"{owner['bucket']}/dl/{endpoint}/{name}.bin".encode()).hexdigest() for endpoint in ("node1", "node3", "node4") for name in ("abort", "complete", "create", "upload")} if network else set() - if mpu_keys and owner.get('runtimeProfile') == 'manual': + if mpu_keys and owner.get('runtimeProfile') in {'manual', 'frozen-race-legacy'}: node['mpu_scene_backend'].validate_scope(owner, mpu_keys) multipart_keys = {hashlib.sha256(f"{owner['bucket']}/{key}".encode()).hexdigest() for key in mpu_keys} def visit(path): @@ -139,9 +139,10 @@ def visit(path): opaque_backup = (not state and owner.get("runtimeProfile") == "outcome-legacy" and len(parts) == 2 and parts[0] == "quarantine" and re.fullmatch(r"J02-[0-9a-f]{32}\.(original|intent\.json|applied\.json|failed-[0-9a-f]{64})", parts[1])) - race_backup = (not state and owner.get("runtimeProfile") == "manual" + race_backup = (not state and owner.get("runtimeProfile") in {"manual", "frozen-race-legacy"} and len(parts) == 2 and parts[0] == "quarantine" - and re.fullmatch(r"(?:G0[1256]|A09)-[a-z0-9-]{1,48}\.(original|intent\.json|applied\.json)", parts[1])) + and re.fullmatch((r"G0[45]" if owner.get("runtimeProfile") == "frozen-race-legacy" else r"(?:G0[12456]|A09)") + + r"-[a-z0-9-]{1,48}\.(original|intent\.json|applied\.json)", parts[1])) if network_mpu and not network: require(info.st_size <= 64 * 1024**2 and mpu_bytes_hashed + info.st_size <= node['mpu_scene_backend'].MAX_BYTES, @@ -179,7 +180,7 @@ def mounted_paths(): def inspect(request, node): state, roots = node["paths"](request["runId"]) expected = request["expectedOwner"] - per_run = expected.get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual"} + per_run = expected.get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy"} retained_state, retained_roots = destinations(state, roots, per_run_parent=per_run) if per_run: old_state, old_roots = destinations(state, roots) @@ -189,7 +190,7 @@ def inspect(request, node): require(isinstance(expected, dict) and expected.get("runId") == request["runId"] and expected.get("unit") == node["unit"](request["runId"]) and expected.get("hostname") == node["socket"].gethostname() - and expected.get("runtimeProfile") in {"mrf-crash", "root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual"}, "retained scene archived owner differs") + and expected.get("runtimeProfile") in {"mrf-crash", "root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy"}, "retained scene archived owner differs") current = node["mounts"]() require(len(current) == len(expected.get("mounts", [])) == 4 and all(all(actual[key] == old[key] for key in ("target", "source", "fstype", "uuid", "maj:min")) @@ -233,12 +234,12 @@ def inspect(request, node): intent = retained_state.parent / (request["runId"] + ".intent.json") node["checked"](intent, state.parent) network = expected["runtimeProfile"] == "lock-quorum" - if expected["runtimeProfile"] in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual"}: + if expected["runtimeProfile"] in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy"}: original_sha = request.get("expectedOriginalSceneSha256", "") require(re.fullmatch(r"[0-9a-f]{64}", original_sha) is not None, "restored fixture requires its original physical scene digest") baseline["originalSceneSha256"] = original_sha - multipart = network or expected['runtimeProfile'] == 'manual' and 'mpuKeys' in request + multipart = network or expected['runtimeProfile'] in {'manual', 'frozen-race-legacy'} and 'mpuKeys' in request if multipart: node['mpu_scene_backend'].validate_scope(expected, request['mpuKeys']) require(re.fullmatch(r"[0-9a-f]{64}", request.get("expectedMpuSceneSha256", "")) is not None, @@ -269,10 +270,10 @@ def inspect(request, node): def run(request, node): fields = {"runId", "action", "expectedOwner"} - if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual"}: + if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy"}: fields.add("expectedOriginalSceneSha256") if request.get("expectedOwner", {}).get("runtimeProfile") == "lock-quorum" or ( - request.get('expectedOwner', {}).get('runtimeProfile') == 'manual' + request.get('expectedOwner', {}).get('runtimeProfile') in {'manual', 'frozen-race-legacy'} and ('mpuKeys' in request or 'expectedMpuSceneSha256' in request)): fields.update({"expectedMpuSceneSha256", "mpuKeys"}) if request["action"] == "retained-scene-isolate": @@ -308,7 +309,7 @@ def run(request, node): "originalSceneRestored": False, "canonicalPassClaimed": False, "backupRestorationUsed": False, "credentialContentsRead": False, "inspectionOnly": request["action"] == "retained-scene-audit", "mutated": mutated, "activeNamespaceIsolated": "active" not in locations} - if request["expectedOwner"].get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual"}: + if request["expectedOwner"].get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy"}: result["privateArchiveParents"] = [] for target in (*retained_roots, retained_state): info = target.parent.lstat() if target.parent.exists() else None diff --git a/scripts/validation_heal/version_race_adapter.py b/scripts/validation_heal/version_race_adapter.py new file mode 100644 index 00000000..b8a0e73e --- /dev/null +++ b/scripts/validation_heal/version_race_adapter.py @@ -0,0 +1,105 @@ +"""Adapt owned H1/H2 fleet snapshots to the frozen version physical oracle.""" +import copy + +try: + from . import version_race_fixture as model, version_race_physical as physical + from .heal_probe import require + from .healthy import validate_topology + from .shards import files +except ImportError: + import version_race_fixture as model, version_race_physical as physical + from heal_probe import require + from healthy import validate_topology + from shards import files + + +SELECTED=('objects-0000.bin','objects-0001.bin','objects-0002.bin',model.TARGET) + + +def view(snapshot,bucket,keys,inspector_sha): + """Use format set slots, never node/disk sorting, as decoder drive indices.""" + topology='H1' if len(snapshot)==1 else 'H2' + proof=validate_topology(snapshot,topology) + require(len(keys)==len(set(keys)) and set(keys)<={f'objects-{i:04d}.bin' for i in range(257)}, + 'version snapshot keys differ from the frozen flat namespace') + formats={};rows=[];metadata=[];identities={};indexed=files(snapshot) + for node,value in snapshot.items(): + require(len(value['formats'])==4 and {v['disk'] for v in value['formats']}==set(range(4)), + 'version snapshot lacks four distinct local drives') + for drive in value['formats']: + slot=proof['diskUUIDs'].index(drive['value']['xl']['this']) + identities[node,drive['disk']]=slot;formats[slot]=copy.deepcopy(drive) + for key in keys: + identity=node,drive['disk'],f'{bucket}/{key}/xl.meta' + row=indexed.get(identity) + if row is None:metadata.append({'disk':slot,'key':key,'absent':True}) + else: + require(isinstance(row.get('metadata'),dict) and not row.get('metadataError'), + 'metadata absence cannot substitute for a decode error') + metadata.append({'disk':slot,'key':key,'metadataSha256':row['sha256'], + 'decoded':copy.deepcopy(row['metadata'])}) + for row in value['files']: + require(row['path'].startswith(bucket+'/'),'version snapshot contains another bucket') + key=row['path'][len(bucket)+1:].split('/')[0] + require(key in keys,'version snapshot contains an unregistered key') + rows.append({k:copy.deepcopy(v) for k,v in {**row,'disk':identities[node,row['disk']]}.items() + if k!='metadata'}) + require(len(set(identities.values()))==len(identities)==len(proof['diskUUIDs']), + 'version snapshot duplicates a global set slot') + normalized={'formats':formats,'files':rows} + physical.file_index(normalized,disk_count=len(identities)) + return normalized,{'inspectorSha256':inspector_sha,'rows':metadata} + + +def slice_entries(entries,key):return [copy.deepcopy(v) for v in entries if v['key']==key] + + +def metadata_targets(snapshot,bucket,ledger,branch,inspector_sha): + require(branch in model.BRANCHES[4:] and ledger.bucket==bucket and ledger.bound and not ledger.poisoned + and ledger.keys=={f'objects-{i:04d}.bin' for i in range(257)}, + 'G04 requires the complete bound original 257-key ledger') + entries=ledger.entries();keys=sorted(ledger.keys) + normalized,decoded=view(snapshot,bucket,keys,inspector_sha) + for key in SELECTED: + physical.key_state(normalized,decoded,bucket,key,slice_entries(entries,key), + inspector_sha=inspector_sha,disk_count=len(snapshot)*4) + indexed=files(snapshot);node=next(iter(snapshot));targets=[] + for key in keys: + data=[v for v in entries if v['key']==key and not v['deleteMarker']] + members={identity:row for identity,row in indexed.items() if identity[2].startswith(f'{bucket}/{key}/')} + require(len(members)==len(snapshot)*4*(1+len(data)) + and sum(path.endswith('/part.1') for _,_,path in members)==len(snapshot)*4*len(data), + 'G04 baseline has unregistered files or incomplete original version shards') + relative=f'{bucket}/{key}/xl.meta';row=indexed[node,3,relative] + targets.append({'node':node,'disk':3,'relativePath':relative,'key':key, + 'sha256':row['sha256'],'size':row['size']}) + require(targets[-1]['key']==model.TARGET,'G04 selected target is not the final flat key') + return targets + + +def reconstruction(before,after,bucket,target): + """Only the selected metadata may change; every data byte stays original.""" + identity=target['node'],target['disk'],target['relativePath'] + prefix=f"{bucket}/{target['key']}/" + left={i:v for i,v in files(before).items() if i[2].startswith(prefix)} + right={i:v for i,v in files(after).items() if i[2].startswith(prefix)} + require(set(left)==set(right) and identity in left + and all(before[node]['formats']==after[node]['formats'] for node in before), + 'G04 metadata repair changed file population or formats') + require(all(value==right[i] for i,value in left.items() if i!=identity), + 'G04 metadata repair changed another metadata member or data shard') + require(0=0 and @@ -41,8 +42,8 @@ def file_index(snapshot): return indexed -def key_state(snapshot,metadata,bucket,key,expected,*,inspector_sha,allowed_orphans=(),allowed_backups=None,allow_multipart=False): - """Bind the exact expected versions to 4 xl.meta decodes and coding indices. +def key_state(snapshot,metadata,bucket,key,expected,*,inspector_sha,disk_count=4,allowed_orphans=(),allowed_backups=None,allow_multipart=False): + """Bind exact expected versions to every set member and coding index. `expected` is an immutable ACK-ledger slice, not a version list discovered from this same snapshot. `allowed_orphans` are exact old dataDir UUIDs from @@ -60,9 +61,9 @@ def key_state(snapshot,metadata,bucket,key,expected,*,inspector_sha,allowed_orph require(not wanted or sum(v.get('isLatest') is True for v in wanted.values())==1, 'expected ACK ledger must have exactly one latest entry') rows=[v for v in metadata['rows'] if v['key']==key] - require(len(rows)==4 and all(type(v['disk']) is int for v in rows) and - {v['disk'] for v in rows}==set(range(4)),'four distinct metadata observations required') - files=file_index(snapshot);prefix=f'{bucket}/{key}/' + require(len(rows)==disk_count and all(type(v['disk']) is int for v in rows) and + {v['disk'] for v in rows}==set(range(disk_count)),'complete distinct set metadata observations required') + files=file_index(snapshot,disk_count=disk_count);prefix=f'{bucket}/{key}/' members={i:v for i,v in files.items() if i[1].startswith(prefix)} used=set();versions={};directories=set();multipart_sidecars=[] for row in rows: @@ -108,13 +109,13 @@ def key_state(snapshot,metadata,bucket,key,expected,*,inspector_sha,allowed_orph require(len(parts)==1 and type(parts[0]['number']) is int and parts[0]['number']==1 and parts[0]['size']==entry['payload']['size'],'single-part external fixture required') ec=v['erasure'];index=ec['index'] - require(type(index) is int and index in range(1,5) and index not in item['coding'], + require(type(index) is int and index in range(1,disk_count+1) and index not in item['coding'], 'invalid or duplicate erasure index') - require(type(ec['data_blocks']) is int and ec['data_blocks']==2 and - type(ec['parity_blocks']) is int and ec['parity_blocks']==2 and + require(type(ec['data_blocks']) is int and ec['data_blocks']==(2 if disk_count==4 else 12) and + type(ec['parity_blocks']) is int and ec['parity_blocks']==(2 if disk_count==4 else 4) and type(ec['block_size']) is int and ec['block_size']>0 and - len(ec['distribution'])==4 and all(type(i) is int for i in ec['distribution']) and - set(ec['distribution'])==set(range(1,5)),'invalid H1 EC2+2 geometry') + len(ec['distribution'])==disk_count and all(type(i) is int for i in ec['distribution']) and + set(ec['distribution'])==set(range(1,disk_count+1)),'invalid registered H1/H2 EC geometry') require(ec['distribution'][disk]==index,'erasure index differs from disk distribution') coding={} for descriptor in parts: @@ -137,7 +138,7 @@ def key_state(snapshot,metadata,bucket,key,expected,*,inspector_sha,allowed_orph item['descriptors'].append({k:copy.deepcopy(v[k]) for k in ('size','parts')}) for item in versions.values(): if item['deleteMarker']:continue - require(set(item['coding'])==set(range(1,5)) and len(set(item['dataDirs']))==1 and + require(set(item['coding'])==set(range(1,disk_count+1)) and len(set(item['dataDirs']))==1 and all(v==item['layouts'][0] for v in item['layouts']) and all(v==item['descriptors'][0] for v in item['descriptors']), 'live version generation/layout/parts differ across drives') @@ -158,12 +159,12 @@ def key_state(snapshot,metadata,bucket,key,expected,*,inspector_sha,allowed_orph require(len(relative)==2 and relative[0] in allowed and relative[1]=='part.1', 'unregistered/orphan physical member not bound to a deleted generation') orphans.append(copy.deepcopy(row)) - return dict(versions=versions,unreferencedDeletedGenerationFiles=orphans,verifiedMetadataBackups=backups,allLiveVersionsFourShards=True, + return dict(versions=versions,unreferencedDeletedGenerationFiles=orphans,verifiedMetadataBackups=backups,allLiveVersionsFourShards=disk_count==4,allLiveVersionsFullSet=True,shardsPerVersion=disk_count, observedMultipartSidecars=multipart_sidecars, casePassClaimed=False,deletedGenerationFilesAbsent=not orphans,garbageCollectorResponsibilityProved=False) -def additive_backups(after,before_metadata,after_metadata,bucket,key,before_entries,after_entries,order,allowed,observations,*,inspector_sha): +def additive_backups(after,before_metadata,after_metadata,bucket,key,before_entries,after_entries,order,allowed,observations,*,inspector_sha,disk_count=4): """Validate exposed decoder semantics, not arbitrary bytes or live authority. Only serial additive PUTs are admitted. A rollback view may represent any @@ -177,7 +178,7 @@ def additive_backups(after,before_metadata,after_metadata,bucket,key,before_entr require(all({k:v for k,v in old[vid].items() if k!='isLatest'}== {k:v for k,v in new[vid].items() if k!='isLatest'} for vid in old) and [v['versionId'] for v in after_entries if v['isLatest']]==[order[-1]],'additive ACK population/order differs') - files=file_index(after);prefix=f'{bucket}/{key}/' + files=file_index(after,disk_count=disk_count);prefix=f'{bucket}/{key}/' candidates={i:v for i,v in files.items() if i[1].startswith(prefix) and i[1].endswith('/xl.meta.bkp')} changed={i:v for i,v in candidates.items() if i not in allowed or any(v[k]!=allowed[i][k] for k in ('size','sha256'))} @@ -218,7 +219,7 @@ def additive_backups(after,before_metadata,after_metadata,bucket,key,before_entr return result,proofs -def transition(before,after,before_metadata,after_metadata,bucket,key,before_entries,after_entries,*,inspector_sha,references=None, +def transition(before,after,before_metadata,after_metadata,bucket,key,before_entries,after_entries,*,inspector_sha,disk_count=4,references=None, append_order=None,backup_metadata=None): """Retained versions stay byte-identical; new ones match immutable references. @@ -227,18 +228,18 @@ def transition(before,after,before_metadata,after_metadata,bucket,key,before_ent Marker creation/deletion changes only the ACK-derived version population. """ require(before['formats']==after['formats'],'format changed during version race') - old=key_state(before,before_metadata,bucket,key,before_entries,inspector_sha=inspector_sha) + old=key_state(before,before_metadata,bucket,key,before_entries,inspector_sha=inspector_sha,disk_count=disk_count) old_ids={v['versionId'] for v in before_entries};new_ids={v['versionId'] for v in after_entries} deleted=old_ids-new_ids allowed=[old['versions'][vid]['dataDir'] for vid in deleted if not old['versions'][vid]['deleteMarker']] - originals=file_index(before) + originals=file_index(before,disk_count=disk_count) backups={(disk,f'{bucket}/{key}/{v["dataDir"]}/xl.meta.bkp'):originals[(disk,f'{bucket}/{key}/xl.meta')] - for v in old['versions'].values() if not v['deleteMarker'] for disk in range(4)} + for v in old['versions'].values() if not v['deleteMarker'] for disk in range(disk_count)} backup_proofs=[] if append_order is not None: backups,backup_proofs=additive_backups(after,before_metadata,after_metadata,bucket,key,before_entries, - after_entries,append_order,backups,backup_metadata,inspector_sha=inspector_sha) - new=key_state(after,after_metadata,bucket,key,after_entries,inspector_sha=inspector_sha,allowed_orphans=allowed,allowed_backups=backups) + after_entries,append_order,backups,backup_metadata,inspector_sha=inspector_sha,disk_count=disk_count) + new=key_state(after,after_metadata,bucket,key,after_entries,inspector_sha=inspector_sha,disk_count=disk_count,allowed_orphans=allowed,allowed_backups=backups) refs=references or {};needed={vid for vid in new_ids-old_ids if not new['versions'][vid]['deleteMarker']} require(set(refs)==needed,'exact independent references required for every new data version') retained=[];created=[] @@ -254,7 +255,7 @@ def transition(before,after,before_metadata,after_metadata,bucket,key,before_ent 'new version reference must be a separate exact baseline control') expected=next(v for v in after_entries if v['versionId']==vid) require(expected['payload']==ref['entry']['payload'],'new version ACK bytes differ from reference payload') - reference=key_state(before,before_metadata,bucket,ref['key'],[ref['entry']],inspector_sha=inspector_sha)['versions'][ref['versionId']] + reference=key_state(before,before_metadata,bucket,ref['key'],[ref['entry']],inspector_sha=inspector_sha,disk_count=disk_count)['versions'][ref['versionId']] require(item['coding']==reference['coding'] and all(item['layout'][k]==reference['layout'][k] for k in ('data_blocks','parity_blocks','block_size')), 'new version coding-index bytes differ from immutable reference') @@ -266,5 +267,5 @@ def transition(before,after,before_metadata,after_metadata,bucket,key,before_ent verifiedMetadataBackups=new['verifiedMetadataBackups'], backupSemanticProofs=backup_proofs, deletedGenerationFilesAbsent=new['deletedGenerationFilesAbsent'],garbageCollectorResponsibilityProved=False, - allLiveVersionsFourShards=True,exactAckVersionPopulation=True,casePassClaimed=False, + allLiveVersionsFourShards=disk_count==4,allLiveVersionsFullSet=True,shardsPerVersion=disk_count,exactAckVersionPopulation=True,casePassClaimed=False, overlapProved=False,publicApiDeletionProved=False) diff --git a/scripts/validation_heal/version_race_runner.py b/scripts/validation_heal/version_race_runner.py new file mode 100644 index 00000000..97806c9e --- /dev/null +++ b/scripts/validation_heal/version_race_runner.py @@ -0,0 +1,244 @@ +"""Frozen G04 metadata-loss races on the owned auto-testing H1/H2 fleet.""" +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +import secrets +import time +from urllib.parse import quote +import uuid + +try: + from . import race, version_race_fixture as model, version_race_adapter as physical + from .heal_probe import Probe, canonical, require, save + from .heal_workload import Recorder + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .cursor import validate_missing_batch, verify_part_manifest + from .heal_fault import bounded_fault_id + from .race_io import set_fleet as io_cap + from .checkpoint import records as journal_records + from .persistence import exact_records +except ImportError: + import race, version_race_fixture as model, version_race_adapter as physical + from heal_probe import Probe, canonical, require, save + from heal_workload import Recorder + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from cursor import validate_missing_batch, verify_part_manifest + from heal_fault import bounded_fault_id + from race_io import set_fleet as io_cap + from checkpoint import records as journal_records + from persistence import exact_records + + +CASE='heal.race-markers-and-batch-delete' + + +def receipts(recorder): + requests=sorted(recorder.output.glob('*-request.json'));acks=sorted(recorder.output.glob('*-ack.json')) + require(len(requests)==len(acks)==recorder.sequence,'G04 SDK journal has an orphan request or receipt') + values=[] + for index,(request_path,ack_path) in enumerate(zip(requests,acks,strict=True),1): + request=json.loads(request_path.read_bytes());ack=json.loads(ack_path.read_bytes()) + observation=ack.get('observation') or {} + require(request_path.name==f'{index:06d}-request.json' and ack_path.name==f'{index:06d}-ack.json' + and request.get('sequence')==ack.get('sequence')==index + and request.get('runId')==ack.get('runId')==recorder.probe.binding['runId'] + and ack.get('requestFile')==request_path.name + and ack.get('requestSha256')==hashlib.sha256(request_path.read_bytes()).hexdigest() + and ack.get('operationCheck')=='PASS' and observation.get('resultUnknown') is False + and observation.get('operation')==request.get('operation') + and observation.get('params')==request.get('params'), + 'G04 SDK request/receipt identity, hash or result differs') + values.append(ack) + return values + + +def partial(probe,fleet,path,token,reference,before,targets,bucket): + deadline=time.monotonic()+120 + while time.monotonic()0 and complete and len(pending)>=32: + target=targets[-1] + if model.TARGET not in pending or (target['node'],3,target['relativePath']) in race.files(snapshot): + raise race.WindowNotObserved('G04 selected metadata repaired before the frozen partial window') + return snapshot + if not pending:raise race.WindowNotObserved('G04 all metadata repaired before the partial window') + require(value['summary']=='running','G04 original root stopped before physical partial repair') + time.sleep(.25) + raise race.WindowNotObserved('G04 no nonzero physical partial window within 120 seconds') + + +def foreground(recorder,ledger,plan,probe,fleet,path,token,reference,branch,target_node): + require(len(plan)==1,'G04 requires one selected source mutation') + samples=[] + try: + with ThreadPoolExecutor(max_workers=1) as pool: + worker=pool.submit(model.submit,recorder,ledger,plan[0]);deadline=time.monotonic()+90 + try: + while not worker.done() and time.monotonic()0 and not authority: + try:fleet.stop();fleet.snapshot('retained-failure-scene',quiescent=True) + except Exception as error:fleet.record('failure-scene-error',{'type':type(error).__name__,'message':str(error)}) + fleet.record('last-ACK-rescue-required',{'state':'DIRTY','namespacesRetained':True, + 'lastACKPhysicalAuthorityVerified':False,'originalGenerationBlindRestorationProhibited':True}) + result['cleanup']='FAIL' + else:result['cleanup']='PASS' if fleet.cleanup() else 'FAIL' + fleet.record('branch-result',result) + return result diff --git a/test_validation_heal.py b/test_validation_heal.py index 001366a4..c66b2b90 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -963,6 +963,12 @@ def snapshot(self): def test_mrf_profile_is_explicit_and_cannot_enable_unrelated_background_sources(self): self.assertEqual(runtime_config("manual"), BACKGROUND_CONFIG) + legacy = runtime_config("frozen-race-legacy") + self.assertEqual(legacy["RUSTFS_SHARD_INTEGRITY_WRITE"], "false") + self.assertEqual(legacy["RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED"], "false") + self.assertEqual(legacy["RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE"], "false") + self.assertEqual(legacy["RUSTFS_HEAL_MAX_CONCURRENT_HEALS"], "1") + self.assertEqual(legacy["RUST_LOG"], "info") replay = runtime_config("offline-replay") self.assertEqual(replay.pop("RUSTFS_HEAL_MRF_ENABLE"), "true") self.assertEqual(replay, {key: value for key, value in BACKGROUND_CONFIG.items() if key != "RUSTFS_HEAL_MRF_ENABLE"}) diff --git a/test_validation_heal_race.py b/test_validation_heal_race.py index e4da0f03..f347f11d 100644 --- a/test_validation_heal_race.py +++ b/test_validation_heal_race.py @@ -72,7 +72,7 @@ def test_g05_controller_delegates_all_public_branches_and_refuses_old_source_bef with patch('scripts.validation_heal.mpu_race_runner.run_branch', return_value={'owned': True}) as execute, \ patch.object(race, 'Fleet', side_effect=AssertionError('generic fleet must not run')): candidate.write_text(json.dumps({'source_sha': race.REFERENCE['candidateCommit']})) - with self.assertRaisesRegex(ValueError, 'exact-source multipart'): + with self.assertRaisesRegex(ValueError, 'exact-source multipart or marker'): race.run_branch(args, 'H1', ['vm000'], 'upload-continue') execute.assert_not_called() candidate.write_text(json.dumps({'source_sha': race.COPY_CANDIDATE})) @@ -80,17 +80,18 @@ def test_g05_controller_delegates_all_public_branches_and_refuses_old_source_bef self.assertEqual(race.run_branch(args, 'H1', ['vm000'], branch), {'owned': True}) execute.assert_called_with(args, 'H1', ['vm000'], branch) - def test_g04_preserves_marker_and_batch_delete_semantics(self): - expected = [{"key": race.TARGET_KEY, "versionId": value, "isLatest": value == "v2", - "deleteMarker": False} for value in ("v0", "v1", "v2")] - with_marker = [*expected, {"key": race.TARGET_KEY, "versionId": "marker", "isLatest": True, "deleteMarker": True}] - self.assertEqual(race.operation_plan("rv-heal-0123456789abcdef", "create-marker", expected)[0]["operation"], - "delete_object") - self.assertEqual(race.operation_plan("rv-heal-0123456789abcdef", "delete-marker", with_marker)[0]["params"]["VersionId"], - "marker") - batch = race.operation_plan("rv-heal-0123456789abcdef", "batch-version-delete", expected)[0] - self.assertEqual(batch["operation"], "delete_objects") - self.assertEqual(len(batch["params"]["Delete"]["Objects"]), 2) + def test_g04_uses_complete_frozen_marker_controller_and_rejects_legacy_before_bootstrap(self): + with tempfile.TemporaryDirectory() as d: + candidate=Path(d)/'candidate.json';args=SimpleNamespace(case_id=race.CASE_IDS[3],candidate=candidate) + with patch('scripts.validation_heal.version_race_runner.run_branch',return_value={'owned':True}) as execute, \ + patch.object(race,'Fleet',side_effect=AssertionError('generic fleet must not run')): + candidate.write_text(json.dumps({'source_sha':race.REFERENCE['candidateCommit']})) + with self.assertRaisesRegex(ValueError,'exact-source'):race.run_branch(args,'H1',['vm000'],'create-marker') + execute.assert_not_called() + candidate.write_text(json.dumps({'source_sha':race.MARKER_CANDIDATE})) + for branch in race.G04_BRANCHES: + self.assertEqual(race.run_branch(args,'H1',['vm000'],branch),{'owned':True}) + execute.assert_called_with(args,'H1',['vm000'],branch) def test_native_branch_remains_in_denominator_for_each_case(self): for case_id, branches in race.CASE_BRANCHES.items(): diff --git a/test_validation_heal_race_physical.py b/test_validation_heal_race_physical.py index d09f82c2..eaf6f38b 100644 --- a/test_validation_heal_race_physical.py +++ b/test_validation_heal_race_physical.py @@ -77,32 +77,6 @@ def scene(self, branch): for r in final] return before, after, mapped, {"entries": final} - def test_real_g04_transition_accepts_markers_batch_delete_and_unversioned_delete(self): - for branch in race.G04_BRANCHES: - with self.subTest(branch=branch): - before, after, mapped, final = self.scene(branch) - survivors, proof = race.physical_transition(before, after, self.bucket, mapped, final, branch) - self.assertEqual(len(survivors), {"create-marker": 3, "delete-marker": 3, - "batch-version-delete": 1, "unversioned-delete": 0}[branch]) - self.assertTrue(proof["physicalProofBeforeGETOrRescue"]) - - def test_g04_rejects_retained_part_drift_marker_drift_and_deleted_generation_resurrection(self): - for branch in race.G04_BRANCHES[:3]: - for fault in ("part", "data-dir", "duplicate-version", "unknown-marker", "marker-latest", "old-directory"): - with self.subTest(branch=branch, fault=fault): - before, after, mapped, final = self.scene(branch) - value = after["node1"] - meta = next(f for f in value["files"] if f["path"].endswith("xl.meta")) - part = next(f for f in value["files"] if f["path"].endswith("part.1")) - if fault == "part": part["sha256"] = "changed" - elif fault == "data-dir": meta["metadata"]["versions"][0]["dataDir"] = str(uuid.UUID(int=999)) - elif fault == "duplicate-version": meta["metadata"]["versions"].append(copy.deepcopy(meta["metadata"]["versions"][0])) - elif fault == "unknown-marker": - meta["metadata"]["versions"].append({"versionId": str(uuid.UUID(int=888)), "deleted": True, "isLatest": False}) - elif fault == "marker-latest": meta["metadata"]["versions"][0]["isLatest"] = not meta["metadata"]["versions"][0]["isLatest"] - else: - value["files"].append({**part, "path": f"{self.bucket}/{race.TARGET_KEY}/{uuid.UUID(int=999)}/part.1"}) - with self.assertRaises(ValueError): race.physical_transition(before, after, self.bucket, mapped, final, branch) class RaceFaultAdmissionTests(unittest.TestCase): @@ -112,7 +86,7 @@ class FaultBoundaryReached(BaseException): for case_id, branches in race.CASE_BRANCHES.items(): # G05's dedicated controller checks all 257 fourth-drive G05 fault # IDs in MpuRunnerTests.test_complete_controller_proves_physical_before_final_get_and_only_then_cleans. - if case_id == 'heal.race-multipart-complete-abort': + if case_id in ('heal.race-multipart-complete-abort','heal.race-markers-and-batch-delete'): continue with self.subTest(case_id=case_id), tempfile.TemporaryDirectory() as folder, ExitStack() as stack: root = Path(folder) diff --git a/test_validation_heal_race_retention.py b/test_validation_heal_race_retention.py index 30e9aa09..d7a59b7f 100644 --- a/test_validation_heal_race_retention.py +++ b/test_validation_heal_race_retention.py @@ -294,6 +294,60 @@ def test_g06_original_copy_and_metadata_journals_require_explicit_source_case_id path.write_text(json.dumps({'runId': 'gh-12345-1-h2', 'operation': 'abort_multipart_upload'})) with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) + def test_g04_unknown_batch_and_all_negative_peer_proofs_stay_bound_for_isolation(self): + with tempfile.TemporaryDirectory() as directory: + root,candidate,folder=self.archive(directory) + original=next(root.glob('*/run/*'));case='heal.race-markers-and-batch-delete' + run=original.with_name(case);original.rename(run) + old=run/'cases'/(controller.CASE+'.json');value=json.loads(old.read_text());value['case_id']=case + old.rename(old.with_name(case+'.json'));(run/'cases'/(case+'.json')).write_text(json.dumps(value)) + old=run/'evidence'/controller.CASE;evidence=old.with_name(case);old.rename(evidence) + old=evidence/'H2/overwrite-vs-deep';folder=old.with_name('batch-version-delete');old.rename(folder) + summary={'caseId':case,'cleanup':'FAIL','results':[{'caseId':case,'cleanup':'FAIL','verdict':'FAIL', + 'topology':'H2','branch':'batch-version-delete'}]} + (evidence/'healthy-summary.json').write_text(json.dumps(summary)) + binding=json.loads((folder/'00001-binding.json').read_text());bucket=binding['bucketPrefix'] + for journal,operation,params in [('seed-ACKs','put_object',{'Bucket':bucket,'Key':'objects-0256.bin'}), + ('foreground-ACKs','delete_objects',{'Bucket':bucket,'Delete':{'Quiet':False,'Objects':[ + {'Key':key,'VersionId':str(i)} for i,key in enumerate(['objects-0256.bin']*3+['objects-0002.bin'])]}})]: + request={'runId':binding['runId'],'operation':operation,'params':params} + (folder/journal/'000001-request.json').write_text(json.dumps(request)) + # All peers may have read-only diagnostics even after another final assertion failed. + for i in range(1,5): + journal=folder/f'negative-read-ACKs-node{i}';journal.mkdir() + params={'Bucket':bucket,'Key':'objects-0256.bin','VersionId':'old'} + request={'runId':binding['runId'],'operation':'get_object','params':params} + path=journal/'000001-request.json';path.write_text(json.dumps(request)) + (journal/'000001-ack.json').write_text(json.dumps({'runId':binding['runId'],'requestFile':path.name, + 'requestSha256':hashlib.sha256(path.read_bytes()).hexdigest(),'observation':{'operation':'get_object'}})) + (journal/'000001-negative-proof.json').write_text(json.dumps({'expected':{'params':params,'httpStatus':404}, + 'state':'FAIL','canonicalCasePassClaimed':False})) + plan=recovery.recovery_plan(root,12345,candidate)[0] + self.assertEqual(plan['kind'],'unresolved-g04-race') + proof=folder/'negative-read-ACKs-node4/000001-negative-proof.json' + original=proof.read_bytes();value=json.loads(original);value['state']='PASS';proof.write_text(json.dumps(value)) + self.assertNotEqual(recovery.recovery_plan(root,12345,candidate)[0]['ackFilesSha256'],plan['ackFilesSha256']) + for fault in ('foreign-bucket','foreign-key','unexpected-operation','digest','foreign-peer'): + with self.subTest(fault=fault): + proof.write_bytes(original);request=folder/'negative-read-ACKs-node4/000001-request.json';raw=request.read_bytes() + if fault=='foreign-peer': + foreign=folder/'negative-read-ACKs-node5';foreign.mkdir() + elif fault=='digest':request.write_text(raw.decode()+' ') + elif fault=='unexpected-operation': + item=json.loads(raw);item['operation']='delete_object';request.write_text(json.dumps(item)) + else: + item=json.loads(original);item['expected']['params']['Bucket' if fault=='foreign-bucket' else 'Key']='foreign' + proof.write_text(json.dumps(item)) + with self.assertRaises(ValueError):recovery.recovery_plan(root,12345,candidate) + request.write_bytes(raw) + if fault=='foreign-peer':foreign.rmdir() + proof.write_bytes(original) + # A pre-foreground failure retains seed uncertainty without inventing a writer receipt. + shutil.rmtree(folder/'foreground-ACKs') + summary['results'][0]['verdict']='NOT_OBSERVED';(evidence/'healthy-summary.json').write_text(json.dumps(summary)) + self.assertEqual(recovery.recovery_plan(root,12345,candidate)[0]['kind'],'unresolved-g04-race') + + def mpu_archive(self, directory, *, foreground=False): root,candidate,folder = self.archive(directory) original = next(root.glob('*/run/*'));case = 'heal.race-multipart-complete-abort' diff --git a/test_validation_heal_version_race_adapter.py b/test_validation_heal_version_race_adapter.py new file mode 100644 index 00000000..0db47fd4 --- /dev/null +++ b/test_validation_heal_version_race_adapter.py @@ -0,0 +1,148 @@ +"""Real H1/H2 oracle adaptation, source metadata fault shape and negative probes.""" +import copy +import hashlib +from pathlib import Path +from types import SimpleNamespace +import tempfile +import unittest +from unittest.mock import Mock,patch + +import test_validation_heal_version_race_physical as fixtures +from scripts.validation_heal import version_race_adapter as adapter,version_race_physical as oracle +from scripts.validation_heal import version_race_fixture as model,version_race_runner as runner,race +from scripts.validation_heal.heal_workload import Recorder + + +def fleet_scene(entries,width=4): + snapshot,metadata=fixtures.physical(entries) + slots=[str(i) for i in range(width)];distribution=list(range(2,width+1))+[1] + result={f'node{n}':{'formats':[],'files':[]} for n in range(1,2 if width==4 else 5)} + for slot in range(width): + node=f'node{slot%4+1}' if width==16 else 'node1';disk=slot//4 if width==16 else slot + value=result[node] + value['formats'].append({'disk':disk,'value':{'format':'xl','id':'pool','xl':{'this':slots[slot],'sets':[slots]}}}) + for row in snapshot['files']: + if row['disk']!=0:continue + current={**row,'disk':disk};current['sha256']=hashlib.sha256((row['sha256']+str(slot)).encode()).hexdigest() + if row['path'].endswith('/xl.meta'): + key=row['path'][len(fixtures.BUCKET)+1:].removesuffix('/xl.meta') + decoded=copy.deepcopy(next(v['decoded'] for v in metadata['rows'] if v['disk']==0 and v['key']==key)) + for version in decoded['versions']: + if not version['deleted']: + version['erasure'].update(index=distribution[slot],distribution=distribution, + data_blocks=2 if width==4 else 12,parity_blocks=2 if width==4 else 4) + current['metadata']=decoded + value['files'].append(current) + return result + + +class VersionAdapterTests(unittest.TestCase): + def test_h1_and_interleaved_h2_map_real_set_slots_and_all_coding_indices(self): + entries=[fixtures.entry(1,latest=True)] + for width in (4,16): + with self.subTest(width=width): + scene=fleet_scene(entries,width) + normalized,metadata=adapter.view(scene,fixtures.BUCKET,[fixtures.KEY],fixtures.INSPECTOR_SHA) + proof=oracle.key_state(normalized,metadata,fixtures.BUCKET,fixtures.KEY,entries, + inspector_sha=fixtures.INSPECTOR_SHA,disk_count=width) + self.assertEqual(set(proof['versions'][fixtures.uid(1)]['coding']),set(range(1,width+1))) + self.assertTrue(proof['allLiveVersionsFullSet']) + self.assertEqual(proof['allLiveVersionsFourShards'],width==4) + if width==16: + self.assertEqual([row['disk'] for row in metadata['rows'][:4]],[0,4,8,12]) + + def test_h2_missing_peer_geometry_duplicate_slot_and_wrong_inspector_are_refused(self): + entries=[fixtures.entry(1,latest=True)] + for fault in ('peer','slot','geometry','inspector','unknown-key','decode'): + with self.subTest(fault=fault): + scene=fleet_scene(entries,16) + if fault=='peer':scene.pop('node4') + elif fault=='slot':scene['node1']['formats'][1]['value']['xl']['this']='0' + elif fault=='unknown-key':scene['node1']['files'][0]['path']=fixtures.BUCKET+'/foreign/xl.meta' + elif fault=='decode':next(v for v in scene['node1']['files'] if 'metadata' in v).pop('metadata') + elif fault=='geometry':next(v for v in scene['node1']['files'] if 'metadata' in v)['metadata']['versions'][0]['erasure']['data_blocks']=8 + with self.assertRaises(ValueError): + normalized,metadata=adapter.view(scene,fixtures.BUCKET,[fixtures.KEY],fixtures.INSPECTOR_SHA) + oracle.key_state(normalized,metadata,fixtures.BUCKET,fixtures.KEY,entries, + inspector_sha='b'*64 if fault=='inspector' else fixtures.INSPECTOR_SHA,disk_count=16) + + def test_complete_g04_fault_plan_contains_257_metadata_members_on_original_fourth_drive(self): + keys={f'objects-{i:04d}.bin' for i in range(257)} + entries=[fixtures.entry(i+1,key=key,latest=True) for i,key in enumerate(sorted(keys))] + for width in (4,16): + scene=fleet_scene(entries,width) + ledger=SimpleNamespace(bucket=fixtures.BUCKET,bound=True,poisoned=None,keys=keys,entries=lambda:entries) + targets=adapter.metadata_targets(scene,fixtures.BUCKET,ledger,'create-marker',fixtures.INSPECTOR_SHA) + self.assertEqual(len(targets),257) + self.assertEqual({(v['node'],v['disk']) for v in targets},{('node1',3)}) + self.assertTrue(all(v['relativePath'].endswith('/xl.meta') for v in targets)) + self.assertEqual(targets[-1]['key'],model.TARGET) + wrong=copy.deepcopy(scene);wrong['node1']['files'].pop() + with self.assertRaises((ValueError,KeyError)): + adapter.metadata_targets(wrong,fixtures.BUCKET,ledger,'create-marker',fixtures.INSPECTOR_SHA) + + def test_metadata_reconstruction_changes_only_one_metadata_file(self): + entries=[fixtures.entry(1,latest=True)];before=fleet_scene(entries) + row=next(v for v in before['node1']['files'] if v['disk']==3 and 'metadata' in v) + target={'node':'node1','disk':3,'relativePath':row['path'],'key':fixtures.KEY} + after=copy.deepcopy(before);selected=next(v for v in after['node1']['files'] if v['disk']==3 and 'metadata' in v) + selected['sha256']='f'*64 + self.assertTrue(adapter.reconstruction(before,after,fixtures.BUCKET,target)['allDataShardsUnchanged']) + for fault in ('data','other-meta','missing','format'): + wrong=copy.deepcopy(after) + if fault=='format':wrong['node1']['formats'][0]['sha256']='changed' + elif fault=='missing':wrong['node1']['files'].remove(next(v for v in wrong['node1']['files'] if v['disk']==3 and 'metadata' in v)) + else:next(v for v in wrong['node1']['files'] if ('metadata' not in v if fault=='data' else v['disk']==0 and 'metadata' in v))['sha256']='e'*64 + with self.subTest(fault=fault),self.assertRaises(ValueError):adapter.reconstruction(before,wrong,fixtures.BUCKET,target) + + def test_source_partial_window_requires_repaired_control_32_pending_and_missing_selected_metadata(self): + keys=sorted({f'objects-{i:04d}.bin' for i in range(257)}) + entries=[fixtures.entry(i+1,key=key,latest=True) for i,key in enumerate(keys)] + before=fleet_scene(entries);snapshot=copy.deepcopy(before) + targets=[{'node':'node1','disk':3,'key':key,'relativePath':f'{fixtures.BUCKET}/{key}/xl.meta'} for key in keys] + snapshot['node1']['files']=[row for row in snapshot['node1']['files'] if not + (row['disk']==3 and row['path'].endswith('/xl.meta') and row['path']!=targets[0]['relativePath'])] + fleet=SimpleNamespace(record=lambda label,value:value,snapshot=lambda label:snapshot) + status={'httpStatus':200,'response':{'summary':'running','settings':race.MARKER_REFERENCE['effective'], + 'outcome':{'counters':{'processed':1}}}} + with patch.object(runner.race,'observe',return_value=status): + self.assertEqual(runner.partial(None,fleet,'path','token',race.MARKER_REFERENCE,before,targets,fixtures.BUCKET),snapshot) + fleet.snapshot=lambda label:before + with self.assertRaises(race.WindowNotObserved):runner.partial(None,fleet,'path','token',race.MARKER_REFERENCE,before,targets,fixtures.BUCKET) + + def test_h2_deleted_generation_is_not_live_but_its_unreferenced_bytes_are_reported(self): + before_entries=[fixtures.entry(1),fixtures.entry(2,latest=True)] + after_entries=[fixtures.entry(1,latest=True)] + before=fleet_scene(before_entries,16);after=fleet_scene(after_entries,16) + # Retained bytes stay original even when metadata latest is promoted. + for node,value in after.items(): + for row in value['files']: + if row['path'].endswith('/part.1'): + row['sha256']=next(v['sha256'] for v in before[node]['files'] if v['disk']==row['disk'] and v['path']==row['path']) + value['files'] += [copy.deepcopy(v) for v in before[node]['files'] if fixtures.uid(1002) in v['path']] + proof=adapter.transition(before,after,fixtures.BUCKET,before_entries,after_entries,fixtures.INSPECTOR_SHA) + key=proof['keys'][fixtures.KEY] + self.assertEqual(key['deletedVersions'],[fixtures.uid(2)]) + self.assertEqual(len(key['unreferencedDeletedGenerationFiles']),16) + self.assertFalse(key['garbageCollectorResponsibilityProved']) + bad=copy.deepcopy(after) + next(v for v in bad['node4']['files'] if 'metadata' in v)['metadata']['versions'].append( + copy.deepcopy(next(v for v in before['node4']['files'] if 'metadata' in v)['metadata']['versions'][-1])) + with self.assertRaises(ValueError):adapter.transition(before,bad,fixtures.BUCKET,before_entries,after_entries,fixtures.INSPECTOR_SHA) + + def test_negative_diagnostics_drain_read_failures_but_stop_on_evidence_io(self): + expected=[{'params':{'Bucket':fixtures.BUCKET,'Key':fixtures.KEY,'VersionId':fixtures.uid(i)}, + 'httpStatus':404,'errorCode':'NoSuchVersion'} for i in (1,2)] + with tempfile.TemporaryDirectory() as tmp: + recorder=SimpleNamespace(output=Path(tmp),call=Mock(side_effect=[ValueError('wrong original 403'), + {'ResponseMetadata':{'HTTPStatusCode':404},'Error':{'Code':'NoSuchVersion'}}])) + proof=runner.negative_checks(recorder,expected) + self.assertEqual([v['state'] for v in proof['results']],['FAIL','PASS']) + recorder.call.reset_mock() + recorder.call.side_effect=ValueError('wrong original 403') + with patch.object(runner,'save',side_effect=OSError('fsync failed')): + with self.assertRaises(OSError):runner.negative_checks(recorder,expected) + self.assertEqual(recorder.call.call_count,1) + + +if __name__=='__main__':unittest.main() diff --git a/test_validation_heal_version_race_runner.py b/test_validation_heal_version_race_runner.py new file mode 100644 index 00000000..217b82ed --- /dev/null +++ b/test_validation_heal_version_race_runner.py @@ -0,0 +1,214 @@ +"""Actual Recorder/controller boundaries; synthetic fleet data is not live proof.""" +import copy +from contextlib import ExitStack +import hashlib +import json +import os +from pathlib import Path +import tempfile +import threading +import time +from types import SimpleNamespace +import unittest +from unittest.mock import Mock,patch +import uuid + +from scripts.validation_heal import version_race_runner as runner,version_race_fixture as model,race +from scripts.validation_heal.heal_probe import CLOCK_DOMAIN +from scripts.validation_heal.heal_workload import Recorder + + +class SDKProbe: + def __init__(self,binding,trace): + self.binding=binding;self.trace=trace;self.rows={};self.sequence=0;self.versioned=False;self.fail=None + + def s3(self,operation,params,body=None): + started=time.monotonic_ns();self.trace.append(('s3',operation,copy.deepcopy(params))) + if operation==self.fail:raise OSError('unknown original request; no retry') + response={'ResponseMetadata':{'HTTPStatusCode':200}};key=params.get('Key');self.sequence+=1 + if operation=='put_bucket_versioning':self.versioned=True + elif operation=='put_object': + vid=str(uuid.UUID(int=10000-self.sequence)) if self.versioned else 'null' + etag='"'+hashlib.md5(body,usedforsecurity=False).hexdigest()+'"' + row={'key':key,'versionId':vid,'deleteMarker':False,'payload':{'size':len(body),'sha256':hashlib.sha256(body).hexdigest()}, + 'properties':{'ETag':etag,'ContentLength':len(body),'ContentType':params['ContentType'], + 'Metadata':copy.deepcopy(params['Metadata']),'StorageClass':'STANDARD'}, + 'tags':[{'Key':'fixture','Value':'DV' if self.versioned else 'DB'},{'Key':'suite','Value':'heal'}]} + self.rows.setdefault(key,[]).append(row);response.update(VersionId=vid,ETag=etag) + elif operation=='delete_object': + response['ResponseMetadata']['HTTPStatusCode']=204 + if 'VersionId' in params: + original=next(v for v in self.rows[key] if v['versionId']==params['VersionId']) + self.rows[key].remove(original);response.update(VersionId=original['versionId'],DeleteMarker=original['deleteMarker']) + elif self.versioned: + vid=str(uuid.UUID(int=10000-self.sequence));self.rows[key].append({'key':key,'versionId':vid,'deleteMarker':True}) + response.update(VersionId=vid,DeleteMarker=True) + else:self.rows[key]=[] + elif operation=='delete_objects': + response['Deleted']=[] + for item in params['Delete']['Objects']: + original=next(v for v in self.rows[item['Key']] if v['versionId']==item['VersionId']) + self.rows[item['Key']].remove(original) + response['Deleted'].append({**item,'DeleteMarker':original['deleteMarker']}) + elif operation=='get_object': + wanted=next((v for v in self.rows.get(key,[]) if v['versionId']==params['VersionId']),None) if 'VersionId' in params else (self.rows.get(key) or [None])[-1] + marker=bool(wanted and wanted['deleteMarker']);code=405 if marker and 'VersionId' in params else 404 + response={'ResponseMetadata':{'HTTPStatusCode':code,'HTTPHeaders':{}}, + 'Error':{'Code':'MethodNotAllowed' if code==405 else 'NoSuchVersion' if 'VersionId' in params else 'NoSuchKey'}} + if marker:response['ResponseMetadata']['HTTPHeaders'].update({'x-amz-delete-marker':'true','x-amz-version-id':wanted['versionId']}) + elif operation!='create_bucket':raise AssertionError(operation) + return {'operation':operation,'params':copy.deepcopy(params),'response':response, + 'acknowledged':200<=response['ResponseMetadata']['HTTPStatusCode']<300,'resultUnknown':False, + 'clockDomain':CLOCK_DOMAIN,'clientCallInvoked':True,'callStartedMonotonicNs':started, + 'callFinishedMonotonicNs':time.monotonic_ns()} + + def inventory(self,bucket,**unused): + self.trace.append(('inventory',bucket));rows=[] + for values in self.rows.values(): + for i,row in enumerate(values):rows.append({**copy.deepcopy(row),'isLatest':i==len(values)-1}) + return {'bucket':bucket,'entries':sorted(rows,key=lambda v:(v['key'],v['versionId']))} + + def admin(self,path,query,body): + return {'httpStatus':200,'response':{'clientToken':str(uuid.UUID(int=1))}} + + +class FakeFleet: + def __init__(self,nodes,directory,run_id,*unused,**kwargs): + self.nodes=nodes;self.evidence=directory;self.run_id=run_id;self.faults=[];self.records=[];self.cleaned=False + def record(self,label,value):self.records.append((label,value));return value + def deploy(self,*unused):pass + def stop(self):self.records.append(('stop',{})) + def start(self):self.records.append(('start',{})) + def snapshot(self,label,**kwargs): + self.records.append((label,{}));return {node:{'formats':[],'files':[]} for node in self.nodes} + def remote(self,node,action,**params): + self.records.append(('remote',{'node':node,'action':action,**params}));return {} + def cleanup(self):self.cleaned=True;return True + + +class VersionRunnerTests(unittest.TestCase): + def test_actual_foreground_uses_one_sdk_receipt_and_releases_cap_without_sdk_retry(self): + for reject in (False,True): + with self.subTest(reject=reject),tempfile.TemporaryDirectory() as tmp: + trace=[];binding={'bucketPrefix':'rv-heal-owned','runId':'gh-12345-1-h1','maxInventoryBytes':512*1024**2} + probe=SDKProbe(binding,trace);seed=Recorder(probe,Path(tmp)/'seed') + ledger=model.seed_fixture(seed,binding['bucketPrefix'],'batch-version-delete',controls=3,size=17) + ledger.bind_baseline(probe.inventory(ledger.bucket));plan=model.mutation_plan(ledger,'batch-version-delete',size=17) + entered,release=threading.Event(),threading.Event();real=probe.s3 + def call(operation,params,body=None): + start=time.monotonic_ns();entered.set() + if not release.wait(5):raise TimeoutError('test worker did not drain') + if reject:raise OSError('original reply lost') + value=real(operation,params,body);value.update(callStartedMonotonicNs=start,callFinishedMonotonicNs=time.monotonic_ns()) + return value + probe.s3=call;writer=Recorder(probe,Path(tmp)/'foreground');fleet=FakeFleet(['node1'],Path(tmp),'gh-12345-1-h1') + path='/rustfs/admin/v3/heal/'+ledger.bucket;token=str(uuid.UUID(int=1)) + def status(*unused): + self.assertTrue(entered.wait(5));a=time.monotonic_ns();b=time.monotonic_ns() + value={'httpStatus':200,'path':path,'query':[('clientToken',token)],'clockDomain':CLOCK_DOMAIN, + 'callStartedMonotonicNs':a,'callFinishedMonotonicNs':b, + 'response':{'summary':'running','settings':race.MARKER_REFERENCE['effective'], + 'outcome':{'counters':{'processed':1}}}} + release.set();return value + with patch.object(runner.race,'observe',side_effect=status),patch.object(runner,'io_cap') as cap: + if reject: + with self.assertRaises(ValueError):runner.foreground(writer,ledger,plan,probe,fleet,path,token,race.MARKER_REFERENCE,'batch-version-delete','node1') + self.assertTrue(ledger.poisoned) + else: + values,overlap=runner.foreground(writer,ledger,plan,probe,fleet,path,token,race.MARKER_REFERENCE,'batch-version-delete','node1') + self.assertEqual(len(values),1);self.assertTrue(overlap) + cap.assert_called_once_with(fleet,runner.CASE,'batch-version-delete',False,nodes=['node1']) + self.assertEqual(writer.sequence,1) + + def controller(self,branch,*,overlap=True,fail=None,width=4): + tmp=tempfile.TemporaryDirectory();self.addCleanup(tmp.cleanup);root=Path(tmp.name) + candidate=root/'candidate.json';candidate.write_text(json.dumps({'source_sha':race.MARKER_CANDIDATE,'package_sha256':'c'*64})) + rc=root/'rc';rc.write_bytes(b'client');nodes=['node1'] if width==4 else ['node1','node2','node3','node4'] + args=SimpleNamespace(candidate=candidate,evidence=root/'evidence',rc_bin=rc,tool_directory=root, + testing_commit='a'*40,binary_sha='b'*64,case_id=runner.CASE) + instances=[];probes=[];trace=[] + def fleet(*a,**kw):value=FakeFleet(*a,**kw);instances.append(value);return value + def probe(binding,endpoint=None): + if not probes:probes.append(SDKProbe(binding,trace)) + return probes[0] + real_seed=model.seed_fixture + def seed(*a,**kw): + probes[0].fail='put_object' if fail=='seed' else None + return real_seed(*a,**kw,size=31) + def foreground(recorder,ledger,plan,*a): + trace.append(('foreground',));probes[0].fail=plan[0]['operation'] if fail=='foreground' else None + values=[model.submit(recorder,ledger,step) for step in plan] + runner.receipts(recorder);return values,overlap + def mapped(snapshot,bucket,entries): + return {(row['key'],row['versionId']):{'cohort':[]} for row in entries if not row['deleteMarker']} + def targets(snapshot,bucket,ledger,*a): + return [{'node':nodes[0],'disk':3,'key':key,'relativePath':f'{bucket}/{key}/xl.meta', + 'sha256':'d'*64,'size':300} for key in sorted(ledger.keys)] + def physical_transition(*a): + trace.append(('physical',)) + if fail=='physical':raise ValueError('original physical result refused') + return {'physicalProofBeforeGETOrRescue':True} + def root_records(*a,**kw): + return {'original-root':{}} if fail=='root' else {} + def streams(*a,**kw): + if fail=='streams-io' and a[2]=='G04-final-full-streams':raise OSError('fsync failed') + return {'count':kw['count']} + with ExitStack() as stack: + stack.enter_context(patch.dict(os.environ,{'GITHUB_RUN_ID':'12345','GITHUB_RUN_ATTEMPT':'1'})) + for owner,name,value in [(runner,'Fleet',fleet),(runner,'Probe',probe),(runner,'verify_inspector',lambda *a:{'binarySha256':'e'*64}), + (runner,'validate_topology',Mock()),(model,'seed_fixture',seed),(runner.physical,'metadata_targets',targets), + (runner.race,'map_fixture',mapped),(runner,'verify_part_manifest',streams),(runner,'validate_missing_batch',Mock()), + (runner,'io_cap',Mock()),(runner,'partial',Mock()),(runner,'foreground',foreground), + (runner,'terminal',lambda *a:{'summary':'finished'}),(runner.physical,'transition',physical_transition), + (runner.physical,'reconstruction',Mock(return_value={})),(runner.race,'terminal_proof',lambda *a:{'completed':True}), + (runner,'journal_records',Mock()),(runner,'exact_records',root_records)]: + stack.enter_context(patch.object(owner,name,value)) + result=runner.run_branch(args,'H1' if width==4 else 'H2',nodes,branch) + return result,instances[0],probes[0],trace + + def test_all_source_g04_plans_preserve_257_metadata_faults_and_full_negative_contracts(self): + for width in (4,16): + for branch in model.BRANCHES[4:]: + with self.subTest(width=width,branch=branch): + result,fleet,probe,trace=self.controller(branch,width=width) + self.assertEqual((result['verdict'],result['cleanup']),('PASS','PASS')) + faults=[v for label,v in fleet.records if label=='remote' and v['action']=='apply-fault'] + self.assertEqual(len(faults),257);self.assertEqual({v['disk'] for v in faults},{3}) + self.assertTrue(all(v['relativePath'].endswith('/xl.meta') and v['faultId'].startswith('G04-') for v in faults)) + self.assertEqual(fleet.faults,[]);self.assertTrue(fleet.cleaned) + foreground_index=trace.index(('foreground',));physical_index=trace.index(('physical',)) + final_reads=[i for i,v in enumerate(trace) if v[0]=='s3' and v[1]=='get_object'] + self.assertTrue(foreground_index Date: Wed, 21 Oct 2026 15:58:08 +0800 Subject: [PATCH 276/343] fix(validation): allow stopped race root census --- scripts/validation_heal/fleet_node.py | 6 ++++- test_validation_heal_journal_fault.py | 34 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index dbffc396..a41f048d 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -1376,7 +1376,11 @@ def advance_captured_checkpoint(request, roots, owner, module, fault, before, ca def journal_operation(request): state, roots, owner = owned(request["runId"]) require(digest(state / "rustfs") == owner["binarySha256"], "durable-record candidate changed") - require(owner.get("runtimeProfile") in {"root-restart", "mrf-crash", "journal-bounded", "incarnation-checkpoint", "incarnation-mrf"}, "durable records require the owned recovery runtime") + race_root_census = (owner.get("runtimeProfile") in {"manual", "frozen-race-legacy"} + and request.get("action") == "journal-census" and request.get("family") == "root" + and set(request) == {"runId", "action", "family", "taskId"}) + require(race_root_census or owner.get("runtimeProfile") in {"root-restart", "mrf-crash", "journal-bounded", "incarnation-checkpoint", "incarnation-mrf"}, + "durable records require the owned recovery runtime or stopped read-only race root census") require(set(request) <= {"runId", "action", "family", "taskId", "disk", "relativePath", "faultId", "kind", "expectedBefore", "expected", "captureId", "mrfIdentity"}, "durable-record adapter accepts no arbitrary system paths or commands") if owner.get("runtimeProfile") == "journal-bounded": diff --git a/test_validation_heal_journal_fault.py b/test_validation_heal_journal_fault.py index 321f3c20..e8a4a5c9 100644 --- a/test_validation_heal_journal_fault.py +++ b/test_validation_heal_journal_fault.py @@ -887,6 +887,40 @@ def test_bounded_journal_census_reads_verified_candidate_views_and_rejects_other with self.subTest(family=family, action=action), self.assertRaises(ValueError): fleet_node.journal_operation({**request, 'family': family, 'action': action}) + def test_race_root_census_reads_original_bytes_only_after_stopping_owned_service(self): + for profile in ('manual', 'frozen-race-legacy'): + with self.subTest(profile=profile), tempfile.TemporaryDirectory() as temp, \ + patch.object(fault, 'mount_id', return_value=1): + root = Path(temp).resolve(); owner, request, _, _ = self.fixture(root) + owner['runtimeProfile'] = profile + raw = json.dumps({'schema': 3, 'task_id': self.token, 'heal_type': {'type': 'erasure_set'}}).encode() + path = root / 'data/.rustfs.sys' / ('root-heal-' + self.token + '.json'); path.write_bytes(raw) + request = {'runId': owner['runId'], 'action': 'journal-census', 'family': 'root', 'taskId': self.token} + with patch.object(fleet_node, 'owned', return_value=(root, [root], owner)), \ + patch.object(fleet_node, 'digest', return_value=owner['binarySha256']), \ + patch.object(fleet_node, 'fault_module', return_value=fault), patch.object(fleet_node, 'stopped') as stopped, \ + patch.dict(fleet_node.__dict__, journal_fault_backend=journal): + result = fleet_node.journal_operation(request) + stopped.assert_called_once_with(owner) + present = [row for row in result['records'] if row['present']] + self.assertEqual(len(present), 1) + self.assertEqual(base64.b64decode(present[0]['rawBase64']), raw) + self.assertTrue(result['readOnly']); self.assertTrue(result['quiescent']) + self.assertFalse(result['directoryEnumerationUsed']) + self.assertEqual(path.read_bytes(), raw) + with patch.object(fleet_node, 'stopped', side_effect=ValueError('service remains active')): + with self.assertRaisesRegex(ValueError, 'service remains active'): + fleet_node.journal_operation(request) + for change in ({'family': 'checkpoint'}, {'family': 'mrf'}, {'action': 'journal-observe'}, + {'action': 'journal-apply', 'disk': 0}, {'action': 'journal-recover', 'disk': 0}, + {'expected': {}}, {'relativePath': '.rustfs.sys/foreign.json'}): + with self.subTest(change=change), self.assertRaises(ValueError): + fleet_node.journal_operation({**request, **change}) + self.assertEqual(path.read_bytes(), raw) + path.unlink() + result = fleet_node.journal_operation(request) + self.assertTrue(all(not row['present'] for row in result['records'])) + def test_bounded_root_admission_reads_the_actual_service_view_without_mutating_it(self): with tempfile.TemporaryDirectory() as temp, patch.object(fault, 'mount_id', return_value=1): root = Path(temp).resolve(); owner, request, _, _ = self.fixture(root) From 81f771e06ebfda66efb6c1a71a6f02cc6971ae3e Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 21 Oct 2026 16:54:11 +0800 Subject: [PATCH 277/343] feat(validation): record caller-only TCP disconnect --- .../workflows/validation-catalog-tests.yml | 2 +- docs/validation-heal-operations.md | 6 + scripts/validation_heal/caller_abort.py | 126 +++++++++++ test_validation_heal_caller_abort.py | 211 ++++++++++++++++++ 4 files changed, 344 insertions(+), 1 deletion(-) create mode 100644 scripts/validation_heal/caller_abort.py create mode 100644 test_validation_heal_caller_abort.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 3ff8b34a..3f6954d1 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -159,7 +159,7 @@ jobs: sudo apt-get install --no-install-recommends -y nftables fi - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - - run: python3 -B -m unittest -v test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py + - run: python3 -B -m unittest -v test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index a7cf115c..3921df32 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -709,6 +709,12 @@ Final committed/staging/control proofs precede all-peer S3 verification. Asserti The additive `mpu-race-scene` node action in `mpu_race_physical.py` adapts the frozen `heal_h1_mpu_physical.py` upload-ID and byte-continuity oracle to the current owned four-drive hosts. Each request binds the owned race runtime, bucket, candidate binary and all three ACK-selected uploads. It inventories the complete fixed key prefix, hashes only selected staging files under bounded entry/depth/byte limits, rejects unknown uploads and links, and preserves original process identity during a live capture. Stopped captures additionally verify the owned service remains stopped. Across H1/H2, pending part/sidecar populations and existing sibling bytes remain exact; a completed upload must lose its staging directory and retain all three previously witnessed encoded parts in committed storage. These are staging/continuity witnesses; independent inspector coding/bitrot checks, product Heal proof, real overlap and full SDK inventories remain required. +### I05 caller-only TCP disconnect component + +`caller_abort.py` sends one signed, exact historical-version S3 GET to a bound endpoint and closes only that caller TCP socket. Invoke it with `python3 -B scripts/validation_heal/caller_abort.py --binding --bucket --key --version-id --output `. Credentials use the existing `HEAL_ACCESS_KEY`, `HEAL_SECRET_KEY` and optional `HEAL_SESSION_TOKEN` environment variables. The request intent is persisted before connection; a completed write checkpoint precedes shutdown, and the final receipt binds the original socket addresses, descriptor, request digest and monotonic clock. The evidence directory cannot be reused. There is no request retry, redirect, proxy or response-derived success. A persistence or transport error closes the socket and retains its uncertain result. Signed headers and credential values are not stored. + +The component always reports `resultUnknown=true`, `acknowledged=false` and `serverOwnershipProven=false`. Successful local command execution means the client transport completed its disconnect sequence. It does not prove that the server admitted the call, acquired a lock, continued after disconnect or released its owner correctly. I05 remains outside canonical dispatch until the controller binds actual server admission and owner events to this exact call, proves the registered timeout and stale-generation fences, and verifies final object generations, drained locks and file descriptors across every required H1/H2 branch. Existing exact-source native owner tests remain separate evidence. + ### J02 original opaque metadata probe The `unknown-negative` diagnostic includes the source procedure before the supplemental legacy-object diagnostic. A separate acknowledged object first receives an independently healthy Deep receipt. With the owned services stopped, the existing descriptor-relative fault backend flips byte zero of every original `xl.meta` copy: four copies in H1 and sixteen in H2. Each mutation has its own original backup, receipt and complete physical delta; the candidate-bound decoder must refuse every exact damaged copy. No other metadata, part or format may change. diff --git a/scripts/validation_heal/caller_abort.py b/scripts/validation_heal/caller_abort.py new file mode 100644 index 00000000..b2e368d8 --- /dev/null +++ b/scripts/validation_heal/caller_abort.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Disconnect one signed, exact-version GET; never infer server ownership.""" + +import argparse +import hashlib +import http.client +import json +import os +from pathlib import Path +import socket +import time +from urllib.parse import quote, urlencode, urlsplit +import uuid + +try: + from .heal_probe import CLOCK_DOMAIN, Probe, canonical, require, save, validate_binding, validate_s3_request +except ImportError: + from heal_probe import CLOCK_DOMAIN, Probe, canonical, require, save, validate_binding, validate_s3_request + + +def disconnect_get(probe, bucket, key, version_id, output): + """Record intent, send once, close the caller socket, retain UNKNOWN. + + The caller must independently establish server admission, lock ownership, + drain and final object/FD state. Sending bytes alone proves none of these. + A populated output directory prevents accidental replay of this request. + """ + validate_binding(probe.binding) + require(isinstance(key, str) and 0 < len(key.encode()) <= 1024, 'bounded exact object key required') + require(isinstance(version_id, str) and str(uuid.UUID(version_id)) == version_id + and uuid.UUID(version_id).int != 0, 'exact nonnil historical version required') + params = {'Bucket': bucket, 'Key': key, 'VersionId': version_id} + validate_s3_request(probe.binding, 'get_object', params) + bound = canonical(probe.binding) + require(probe.endpoint_name in probe.binding['endpoints'] + and probe.endpoint == probe.binding['endpoints'][probe.endpoint_name].rstrip('/'), + 'caller endpoint differs from the current binding') + endpoint = urlsplit(probe.endpoint) + path = '/' + quote(bucket, safe='') + '/' + quote(key, safe='/~') + target = path + '?' + urlencode({'versionId': version_id}) + from botocore.auth import S3SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + access, secret = probe.credentials() + token = None if probe.credential_pair else os.environ.get('HEAL_SESSION_TOKEN') + outgoing = AWSRequest(method='GET', url=probe.endpoint + target, headers={ + 'Host': endpoint.netloc, 'x-amz-content-sha256': hashlib.sha256(b'').hexdigest(), + 'Connection': 'close', 'Content-Length': '0'}) + S3SigV4Auth(Credentials(access, secret, token), 's3', 'us-east-1').add_auth(outgoing) + output = Path(output) + output.mkdir(exist_ok=False) + parent = os.open(output.parent, os.O_RDONLY) + try: + os.fsync(parent) + finally: + os.close(parent) + intent_path = output / 'request.json' + save(intent_path, {'schemaVersion': 1, 'runId': probe.binding['runId'], + 'candidateCommit': probe.binding['candidateCommit'], 'harnessCommit': probe.binding['harnessCommit'], + 'candidatePackageSha256': probe.binding['candidatePackageSha256'], + 'candidateBinarySha256': probe.binding['candidateBinarySha256'], + 'bindingSha256': hashlib.sha256(bound.encode()).hexdigest(), + 'endpoint': probe.endpoint_name, 'endpointUrl': probe.endpoint, + 'operation': 'get_object', 'params': params, + 'clockDomain': CLOCK_DOMAIN, 'preparedMonotonicNs': time.monotonic_ns(), + 'retryAllowed': False, 'canonicalCasePassClaimed': False}) + evidence = {'schemaVersion': 1, 'runId': probe.binding['runId'], + 'requestSha256': hashlib.sha256(intent_path.read_bytes()).hexdigest(), + 'clockDomain': CLOCK_DOMAIN, 'requestInvocationStarted': False, 'requestWriteCompleted': False, + 'responseObserved': False, 'resultUnknown': True, 'acknowledged': False, + 'retryAllowed': False, 'serverOwnershipProven': False, 'canonicalCasePassClaimed': False} + connection_type = http.client.HTTPSConnection if endpoint.scheme == 'https' else http.client.HTTPConnection + class OnceConnection(connection_type): + attempts = 0 + def connect(self): + self.attempts += 1 + require(self.attempts == 1, 'caller-disconnect request cannot reconnect') + super().connect() + connection = OnceConnection(endpoint.hostname, endpoint.port, timeout=5) + sock = None + try: + validate_binding(probe.binding) + require(canonical(probe.binding) == bound, 'caller binding changed before connection') + connection.connect() + sock = connection.sock + require(sock is not None, 'caller socket unavailable') + evidence.update(socketDescriptor=sock.fileno(), localAddress=list(sock.getsockname()), + peerAddress=list(sock.getpeername())) + validate_binding(probe.binding) + require(canonical(probe.binding) == bound, 'caller binding changed before request') + evidence.update(requestInvocationStarted=True, callStartedMonotonicNs=time.monotonic_ns()) + connection.request('GET', target, headers=dict(outgoing.headers.items())) + evidence.update(requestWriteCompleted=True, requestWrittenMonotonicNs=time.monotonic_ns()) + save(output / 'request-written.json', evidence) + evidence['disconnectStartedMonotonicNs'] = time.monotonic_ns() + sock.shutdown(socket.SHUT_RDWR) + evidence['shutdownCompleted'] = True + except BaseException as error: + evidence['errorType'] = type(error).__name__ + raise + finally: + try: + connection.close() + finally: + evidence.update(connectionAttempts=connection.attempts, clientClosedMonotonicNs=time.monotonic_ns(), + clientSocketClosed=sock is None or sock.fileno() == -1) + save(output / 'disconnected.json', evidence) + return evidence + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--binding', type=Path, required=True) + parser.add_argument('--bucket', required=True) + parser.add_argument('--key', required=True) + parser.add_argument('--version-id', required=True) + parser.add_argument('--endpoint') + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + proof = disconnect_get(Probe(json.loads(args.binding.read_text()), args.endpoint), + args.bucket, args.key, args.version_id, args.output) + print(json.dumps(proof)) + + +if __name__ == '__main__': + main() diff --git a/test_validation_heal_caller_abort.py b/test_validation_heal_caller_abort.py new file mode 100644 index 00000000..470c09fc --- /dev/null +++ b/test_validation_heal_caller_abort.py @@ -0,0 +1,211 @@ +"""Real TCP evidence for the I05 client subset; no server-lock PASS claim.""" + +from datetime import datetime, timedelta, timezone +import hashlib +import http.server +import json +from pathlib import Path +import threading +import tempfile +import unittest +from unittest.mock import patch + +from scripts.validation_heal import caller_abort +from scripts.validation_heal.heal_probe import Probe + + +VERSION = '00000000-0000-4000-8000-000000000001' + + +class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + self.connection.settimeout(3) + self.server.requests.append((self.path, dict(self.headers))) + try: + self.server.eof.append(self.rfile.read(1)) + finally: + self.server.disconnected.set() + + +class CallerAbortTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + self.server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Handler) + self.server.requests = [] + self.server.eof = [] + self.server.disconnected = threading.Event() + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.binding = { + 'schemaVersion': 1, 'runId': 'gh-12345-1-h1', + 'leaseExpiresAt': (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(), + 'jumpHostOnly': True, 'exclusiveHealService': True, 'candidateCommit': 'a' * 40, + 'harnessCommit': 'b' * 40, 'candidatePackageSha256': 'c' * 64, + 'candidateBinarySha256': 'd' * 64, 'rcBinarySha256': 'e' * 64, + 'bucketPrefix': 'rv-heal-0123456789abcdef', 'defaultEndpoint': 'node1', + 'endpoints': {'node1': f'http://127.0.0.1:{self.server.server_port}'}, + 'maxInventoryObjects': 512, 'maxInventoryBytes': 512 * 1024**2} + self.probe = Probe(self.binding, credential_pair=('private-test-access', 'private-test-secret')) + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + self.thread.join() + self.tmp.cleanup() + + def invoke(self, output=None, **changes): + return caller_abort.disconnect_get(self.probe, changes.get('bucket', self.binding['bucketPrefix']), + changes.get('key', '中文/literal%2F space.bin'), changes.get('version', VERSION), + output or self.root / 'call') + + def test_one_signed_exact_version_get_closes_real_socket_and_retains_unknown(self): + proof = self.invoke() + self.assertTrue(self.server.disconnected.wait(3), 'server must observe this caller EOF') + self.assertEqual(self.server.eof, [b'']) + self.assertEqual(len(self.server.requests), 1) + path, headers = self.server.requests[0] + self.assertEqual(path, '/rv-heal-0123456789abcdef/%E4%B8%AD%E6%96%87/literal%252F%20space.bin?versionId=' + VERSION) + self.assertTrue(headers['Authorization'].startswith('AWS4-HMAC-SHA256 Credential=private-test-access/')) + self.assertEqual(headers['Host'], f'127.0.0.1:{self.server.server_port}') + from botocore.auth import S3SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + signed = headers['Authorization'].split('SignedHeaders=')[1].split(',')[0].split(';') + checked = AWSRequest(method='GET', url=self.probe.endpoint + path, + headers={k: v for k, v in headers.items() if k.lower() in signed}) + checked.context['timestamp'] = headers['X-Amz-Date'] + auth = S3SigV4Auth(Credentials('private-test-access', 'private-test-secret'), 's3', 'us-east-1') + expected = auth.signature(auth.string_to_sign(checked, auth.canonical_request(checked)), checked) + self.assertEqual(headers['Authorization'].split('Signature=')[1], expected) + self.assertEqual(proof['connectionAttempts'], 1) + self.assertTrue(proof['requestWriteCompleted'] and proof['clientSocketClosed'] and proof['shutdownCompleted']) + self.assertTrue(proof['resultUnknown']) + for field in ('responseObserved', 'acknowledged', 'retryAllowed', 'serverOwnershipProven', 'canonicalCasePassClaimed'): + self.assertFalse(proof[field], field) + intent = self.root / 'call/request.json' + self.assertEqual(proof['requestSha256'], hashlib.sha256(intent.read_bytes()).hexdigest()) + written = json.loads((self.root / 'call/request-written.json').read_text()) + self.assertTrue(written['requestWriteCompleted']) + self.assertLessEqual(written['requestWrittenMonotonicNs'], proof['disconnectStartedMonotonicNs']) + self.assertLessEqual(proof['disconnectStartedMonotonicNs'], proof['clientClosedMonotonicNs']) + self.assertEqual(json.loads((self.root / 'call/disconnected.json').read_text()), proof) + text = ''.join(p.read_text() for p in (self.root / 'call').iterdir()) + for secret in ('private-test-access', 'private-test-secret', 'Authorization'): + self.assertNotIn(secret, text) + + def test_existing_evidence_directory_prevents_replay_without_another_connection(self): + self.invoke() + self.assertTrue(self.server.disconnected.wait(3)) + original = (self.root / 'call/disconnected.json').read_bytes() + with patch.object(caller_abort.socket, 'create_connection', side_effect=AssertionError('no replay')): + with self.assertRaises(FileExistsError): + self.invoke() + self.assertEqual((self.root / 'call/disconnected.json').read_bytes(), original) + self.assertEqual(len(self.server.requests), 1) + + def test_intent_persistence_failure_prevents_any_network_request(self): + with patch.object(caller_abort, 'save', side_effect=OSError('fsync failed')), \ + patch.object(caller_abort.socket, 'create_connection', side_effect=AssertionError('network before intent')): + with self.assertRaisesRegex(OSError, 'fsync failed'): + self.invoke() + self.assertEqual(self.server.requests, []) + + def test_evidence_failure_after_send_still_closes_socket_without_retry(self): + original_save = caller_abort.save + def fail_written(path, value): + if path.name == 'request-written.json': + raise OSError('written receipt unavailable') + original_save(path, value) + with patch.object(caller_abort, 'save', side_effect=fail_written): + with self.assertRaisesRegex(OSError, 'written receipt unavailable'): + self.invoke() + self.assertTrue(self.server.disconnected.wait(3)) + self.assertEqual(self.server.eof, [b'']) + proof = json.loads((self.root / 'call/disconnected.json').read_text()) + self.assertEqual(proof['errorType'], 'OSError') + self.assertTrue(proof['resultUnknown'] and proof['clientSocketClosed']) + self.assertEqual(proof['connectionAttempts'], 1) + self.assertEqual(len(self.server.requests), 1) + + def test_connection_failure_is_recorded_and_cannot_retry(self): + with patch.object(caller_abort.socket, 'create_connection', side_effect=ConnectionRefusedError('private detail')) as connect: + with self.assertRaises(ConnectionRefusedError): + self.invoke() + self.assertEqual(connect.call_count, 1) + proof = json.loads((self.root / 'call/disconnected.json').read_text()) + self.assertEqual(proof['errorType'], 'ConnectionRefusedError') + self.assertEqual(proof['connectionAttempts'], 1) + self.assertFalse(proof['requestInvocationStarted']) + self.assertFalse(proof['requestWriteCompleted']) + self.assertNotIn('private detail', json.dumps(proof)) + + def test_directory_sync_failure_prevents_network(self): + with patch.object(caller_abort.os, 'fsync', side_effect=OSError('directory sync failed')), \ + patch.object(caller_abort.socket, 'create_connection', side_effect=AssertionError('network before durable directory')): + with self.assertRaisesRegex(OSError, 'directory sync failed'): + self.invoke() + self.assertEqual(self.server.requests, []) + + def test_shutdown_failure_still_closes_original_socket_and_retains_unknown(self): + original = caller_abort.socket.create_connection + sockets = [] + class FailingShutdown: + def __init__(self, sock): + self.sock = sock + def __getattr__(self, name): + return getattr(self.sock, name) + def shutdown(self, how): + raise OSError('shutdown failed') + def connect(*args, **kwargs): + sock = FailingShutdown(original(*args, **kwargs)) + sockets.append(sock) + return sock + with patch.object(caller_abort.socket, 'create_connection', side_effect=connect): + with self.assertRaisesRegex(OSError, 'shutdown failed'): + self.invoke() + self.assertTrue(self.server.disconnected.wait(3)) + self.assertEqual(self.server.eof, [b'']) + self.assertEqual(sockets[0].fileno(), -1) + proof = json.loads((self.root / 'call/disconnected.json').read_text()) + self.assertTrue(proof['requestWriteCompleted'] and proof['clientSocketClosed'] and proof['resultUnknown']) + self.assertEqual(proof['errorType'], 'OSError') + self.assertNotIn('shutdownCompleted', proof) + self.assertEqual(proof['connectionAttempts'], 1) + + def test_changed_endpoint_or_binding_is_rejected_before_request(self): + self.binding['endpoints']['node1'] = 'http://127.0.0.1:9400' + with patch.object(caller_abort.socket, 'create_connection', side_effect=AssertionError('endpoint drift')): + with self.assertRaisesRegex(ValueError, 'endpoint differs'): + self.invoke() + self.binding['endpoints']['node1'] = self.probe.endpoint + original_save = caller_abort.save + def drift_after_intent(path, value): + original_save(path, value) + if path.name == 'request.json': + self.binding['candidateBinarySha256'] = 'f' * 64 + with patch.object(caller_abort, 'save', side_effect=drift_after_intent), \ + patch.object(caller_abort.socket, 'create_connection', side_effect=AssertionError('binding drift')): + with self.assertRaisesRegex(ValueError, 'binding changed'): + self.invoke() + proof = json.loads((self.root / 'call/disconnected.json').read_text()) + self.assertEqual(proof['connectionAttempts'], 0) + self.assertFalse(proof['requestInvocationStarted']) + + def test_foreign_bucket_null_or_missing_version_and_expired_lease_never_connect(self): + with patch.object(caller_abort.socket, 'create_connection', side_effect=AssertionError('invalid request connected')): + for changes in ({'bucket': 'foreign-bucket'}, {'version': 'null'}, {'version': ''}, + {'version': '00000000-0000-0000-0000-000000000000'}, {'key': ''}): + with self.subTest(changes=changes), self.assertRaises(ValueError): + self.invoke(**changes) + self.binding['leaseExpiresAt'] = (datetime.now(timezone.utc) - timedelta(seconds=1)).isoformat() + with self.assertRaisesRegex(ValueError, 'lease expired'): + self.invoke() + self.assertEqual(self.server.requests, []) + + +if __name__ == '__main__': + unittest.main() From 525e473f74e0fb3d1f274b086a9a5aba99ca8dff Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 21 Oct 2026 17:04:26 +0800 Subject: [PATCH 278/343] docs(validation): clarify remaining I05 evidence requirements --- docs/validation-heal-cases.md | 2 +- docs/validation-heal-environment-gates.md | 4 ++-- scripts/validation_heal/execution_policy.py | 4 ++-- scripts/validation_heal/recipes.json | 6 ++++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index daf2a807..9b82076a 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1770,7 +1770,7 @@ Topologies: H1,H2. Branches: object-lock-timeout, caller-disconnect, stale-read- Execution policy: `evidence-blocked`; disabled by default: `false`. -Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 需要真实 lock owner、超时、调用方断开和 stale read-repair 的服务端所有权/提交顺序；黑盒请求耗时只能证明 TCP 断开，不能证明锁提交屏障，候选包没有 reviewed e2e-test-hooks。 +Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集，以及单次签名历史版本 GET 的客户端 TCP 断开组件；仍缺服务端请求 admission/所有权绑定、完整 timeout/stale-generation 编排和 H1/H2 最终对象/锁/FD 验收。精确源码 cfg(test) 原生证据可独立构建，不要求给优化服务候选加入隐藏 hook；客户端断开或原生子集不能充作完整 canonical PASS。 1. LOCK_WINDOW用本例慢IO/受控原生锁屏障持有目标对象锁，先证明真实争用超过预注册预算。 2. 同时heal/覆盖或删除；调用方断开实例仅断本次TCP连接，不杀服务器或关闭跳板机；记录请求开始/断开/服务在途结束。 diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index 84b79753..114f04de 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -40,10 +40,10 @@ These observations do not certify a clean environment. Independently audit retai | --- | --- | --- | --- | | C11 | `heal.bucket-incarnation-recreation` | H1/H2 旧元数据盘重返、原始 checkpoint/resume 与 committed MRF 重放编排已实现；精确候选的 9 项存储原生回归和 3 项 manager incarnation 前置证据已独立核验。完整现场分支尚无独立审计结果 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | | F07 | `heal.checkpoint-and-mrf-corruption` | 已实现 H1/H2 checkpoint 截断、未来 schema、committed MRF CRC 故障及独立安全重扫；10 项原生回归作为前置证据。真实较早代次回放与私有限额卷的 journal ENOSPC 已实现；仍缺完整 native cut 审核，完整 live 尚未审计 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | -| I05 | `heal.lock-timeout-and-caller-abort` | 请求耗时/TCP 断开不能证明服务端锁 owner 与提交顺序 | lock owner、timeout、caller abort 和 stale read-repair 的服务端屏障 | +| I05 | `heal.lock-timeout-and-caller-abort` | 已有同源码 native owner/取消/stale 子集和单连接签名历史版本 GET 断开组件；缺完整 controller 与 live 证据 | 实际服务端 admission/owner 绑定、timeout/stale-generation 编排、H1/H2 最终对象/锁/FD 闭环；原生与客户端证据独立保留 | | J02 | `heal.outcome-counters-and-error-truth` | 全部夹具与 manifest 对账均已实现。候选 `7951bb629821246868ee3761b321dd07c06db45c` 修复整桶重试覆盖 repaired 记录，210 项 manager 与 5 项独立进程集成测试通过。新实机 `38874900846/1` 完成 20 个分支：16 个报告通过，4 个原始全元数据损坏探针返回永久 Failed，保留 NOT_OBSERVED；四组混合任务与全部 50 条节点清理记录已独立审计。真实 legacy Unknown 已观察，不能代替原始探针 | 原始元数据损坏分支的 Unknown 仍未观察；完整 source-procedure 验收尚未闭合。原始 Failed、NOT_OBSERVED 和完整用例门控保持不变 | -I05 和已接入 A09、G01–G06 的精确屏障分支仍受候选包对应、经过审核的 `e2e-test-hooks` 限制。第一批 G07/G08 的 native 分支与 H04 容量锁窗口也保留 BLOCKED；不能用外部请求重试或最终对象正确来声称精确 lock/version/marker/MPU/copy 提交屏障已命中。 +I05 与 A09、G01–G06 的精确屏障证据允许使用同一不可变候选源码单独构建的 `cfg(test)` 原生测试执行文件。不得给优化服务候选加入隐藏 hook 冒充原包，也不能把缺少 release hook 本身归为外部环境阻塞。原生子集、完整服务调用和现场最终 ACK/物理检查分别验收；已验收 native 证据不能代替尚缺的 controller 或完整 live 结果。第一批 G07/G08 的 native 分支与 H04 容量锁窗口保留各自未完成条件；外部请求重试或最终对象正确均不能证明精确锁或提交窗口。 资源适配器协议使用 stdin/stdout JSON，不调用 shell。runtime 中的 `adapter` 包含绝对可执行路径、SHA-256 和操作 allowlist；`environment_cases.required_operations(case_id)` 给出所需操作。每个请求携带 run ID、原候选 commit、sequence、拓扑/分支 scope、bucket prefix、租约和参数。响应必须回显上述身份、按 `heal_probe.canonical(request)` 的 UTF-8 SHA-256 绑定原请求，并把真实观察写在 `observation`，禁止返回 verdict。资源适配器需独立实现平台 power control、故障层、pool lifecycle 或 KMS/tier 隔离；本 PR 未向当前共享环境部署适配器。 diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py index 0eb4e367..78cccb2d 100644 --- a/scripts/validation_heal/execution_policy.py +++ b/scripts/validation_heal/execution_policy.py @@ -139,8 +139,8 @@ def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, . ), "heal.lock-timeout-and-caller-abort": _native( "I05", "NATIVE_LOCK_OWNER_ABORT_BARRIER", - "需要真实 lock owner、超时、调用方断开和 stale read-repair 的服务端所有权/提交顺序；黑盒请求耗时只能证明 TCP 断开，不能证明锁提交屏障，候选包没有 reviewed e2e-test-hooks。", - ("server_lock_owner", "lock_timeout_fence", "caller_abort_sequence", "stale_read_repair_fence", "reviewed_e2e_test_hooks"), + "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集，以及单次签名历史版本 GET 的客户端 TCP 断开组件；仍缺服务端请求 admission/所有权绑定、完整 timeout/stale-generation 编排和 H1/H2 最终对象/锁/FD 验收。精确源码 cfg(test) 原生证据可独立构建，不要求给优化服务候选加入隐藏 hook；客户端断开或原生子集不能充作完整 canonical PASS。", + ("server_lock_owner", "lock_timeout_fence", "caller_abort_sequence", "stale_read_repair_fence", "exact_source_native_evidence", "complete_live_orchestrator", "final_object_lock_fd_closure"), ), "heal.outcome-counters-and-error-truth": _native( "J02", "EVIDENCE_MIXED_OUTCOME_ORCHESTRATION", diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index eb805aa1..111ec405 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -3253,13 +3253,15 @@ "dispatchable": false, "code": "I05", "reasonCode": "NATIVE_LOCK_OWNER_ABORT_BARRIER", - "reason": "需要真实 lock owner、超时、调用方断开和 stale read-repair 的服务端所有权/提交顺序；黑盒请求耗时只能证明 TCP 断开，不能证明锁提交屏障，候选包没有 reviewed e2e-test-hooks。", + "reason": "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集，以及单次签名历史版本 GET 的客户端 TCP 断开组件；仍缺服务端请求 admission/所有权绑定、完整 timeout/stale-generation 编排和 H1/H2 最终对象/锁/FD 验收。精确源码 cfg(test) 原生证据可独立构建，不要求给优化服务候选加入隐藏 hook；客户端断开或原生子集不能充作完整 canonical PASS。", "requiredCapabilities": [ "server_lock_owner", "lock_timeout_fence", "caller_abort_sequence", "stale_read_repair_fence", - "reviewed_e2e_test_hooks" + "exact_source_native_evidence", + "complete_live_orchestrator", + "final_object_lock_fd_closure" ], "enableVariable": null, "capabilitiesVariable": null From a80863ab6a3428722efc58ee4bad9e7b4044f62e Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 21 Oct 2026 17:32:57 +0800 Subject: [PATCH 279/343] feat(validation): observe I05 caller admission and drain --- .../workflows/validation-catalog-tests.yml | 2 +- docs/validation-heal-cases.md | 2 +- docs/validation-heal-environment-gates.md | 2 +- docs/validation-heal-operations.md | 10 + scripts/validation_heal/caller_abort.py | 13 +- scripts/validation_heal/caller_socket.py | 105 ++++++ scripts/validation_heal/execution_policy.py | 2 +- scripts/validation_heal/fleet_node.py | 9 +- scripts/validation_heal/healthy.py | 5 +- scripts/validation_heal/lock_observer.py | 255 ++++++++++++++ scripts/validation_heal/recipes.json | 2 +- scripts/validation_heal/scanner_repair.py | 8 +- test_validation_heal_caller_abort.py | 35 +- test_validation_heal_lock_observer.py | 310 ++++++++++++++++++ 14 files changed, 748 insertions(+), 12 deletions(-) create mode 100644 scripts/validation_heal/caller_socket.py create mode 100644 scripts/validation_heal/lock_observer.py create mode 100644 test_validation_heal_lock_observer.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 3f6954d1..757bc65e 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -159,7 +159,7 @@ jobs: sudo apt-get install --no-install-recommends -y nftables fi - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - - run: python3 -B -m unittest -v test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py + - run: python3 -B -m unittest -v test_validation_heal_lock_observer.py test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index 9b82076a..e37531b2 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1770,7 +1770,7 @@ Topologies: H1,H2. Branches: object-lock-timeout, caller-disconnect, stale-read- Execution policy: `evidence-blocked`; disabled by default: `false`. -Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集，以及单次签名历史版本 GET 的客户端 TCP 断开组件；仍缺服务端请求 admission/所有权绑定、完整 timeout/stale-generation 编排和 H1/H2 最终对象/锁/FD 验收。精确源码 cfg(test) 原生证据可独立构建，不要求给优化服务候选加入隐藏 hook；客户端断开或原生子集不能充作完整 canonical PASS。 +Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集、单连接历史 GET 断开和服务 admission/owner/socket 观测编排；观测编排尚无 live 验收。仍缺实际断连边界的精确所有权、完整 timeout/stale-generation controller 和 H1/H2 对象/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断观测、客户端断开或原生子集不能充作完整 canonical PASS。 1. LOCK_WINDOW用本例慢IO/受控原生锁屏障持有目标对象锁，先证明真实争用超过预注册预算。 2. 同时heal/覆盖或删除；调用方断开实例仅断本次TCP连接，不杀服务器或关闭跳板机；记录请求开始/断开/服务在途结束。 diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index 114f04de..742a4edb 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -40,7 +40,7 @@ These observations do not certify a clean environment. Independently audit retai | --- | --- | --- | --- | | C11 | `heal.bucket-incarnation-recreation` | H1/H2 旧元数据盘重返、原始 checkpoint/resume 与 committed MRF 重放编排已实现；精确候选的 9 项存储原生回归和 3 项 manager incarnation 前置证据已独立核验。完整现场分支尚无独立审计结果 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | | F07 | `heal.checkpoint-and-mrf-corruption` | 已实现 H1/H2 checkpoint 截断、未来 schema、committed MRF CRC 故障及独立安全重扫；10 项原生回归作为前置证据。真实较早代次回放与私有限额卷的 journal ENOSPC 已实现；仍缺完整 native cut 审核，完整 live 尚未审计 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | -| I05 | `heal.lock-timeout-and-caller-abort` | 已有同源码 native owner/取消/stale 子集和单连接签名历史版本 GET 断开组件；缺完整 controller 与 live 证据 | 实际服务端 admission/owner 绑定、timeout/stale-generation 编排、H1/H2 最终对象/锁/FD 闭环；原生与客户端证据独立保留 | +| I05 | `heal.lock-timeout-and-caller-abort` | 已有同源码 native 子集、单连接签名历史 GET 和服务 admission/owner/socket 观测编排；缺完整 controller 与 live 验收 | 实际断连边界所有权、timeout/stale-generation 编排、H1/H2 最终对象/锁/FD 闭环；服务观测尚未 live 验收 | | J02 | `heal.outcome-counters-and-error-truth` | 全部夹具与 manifest 对账均已实现。候选 `7951bb629821246868ee3761b321dd07c06db45c` 修复整桶重试覆盖 repaired 记录，210 项 manager 与 5 项独立进程集成测试通过。新实机 `38874900846/1` 完成 20 个分支：16 个报告通过，4 个原始全元数据损坏探针返回永久 Failed，保留 NOT_OBSERVED；四组混合任务与全部 50 条节点清理记录已独立审计。真实 legacy Unknown 已观察，不能代替原始探针 | 原始元数据损坏分支的 Unknown 仍未观察；完整 source-procedure 验收尚未闭合。原始 Failed、NOT_OBSERVED 和完整用例门控保持不变 | I05 与 A09、G01–G06 的精确屏障证据允许使用同一不可变候选源码单独构建的 `cfg(test)` 原生测试执行文件。不得给优化服务候选加入隐藏 hook 冒充原包，也不能把缺少 release hook 本身归为外部环境阻塞。原生子集、完整服务调用和现场最终 ACK/物理检查分别验收；已验收 native 证据不能代替尚缺的 controller 或完整 live 结果。第一批 G07/G08 的 native 分支与 H04 容量锁窗口保留各自未完成条件；外部请求重试或最终对象正确均不能证明精确锁或提交窗口。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 3921df32..09b00d2e 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -715,6 +715,16 @@ The additive `mpu-race-scene` node action in `mpu_race_physical.py` adapts the f The component always reports `resultUnknown=true`, `acknowledged=false` and `serverOwnershipProven=false`. Successful local command execution means the client transport completed its disconnect sequence. It does not prove that the server admitted the call, acquired a lock, continued after disconnect or released its owner correctly. I05 remains outside canonical dispatch until the controller binds actual server admission and owner events to this exact call, proves the registered timeout and stale-generation fences, and verifies final object generations, drained locks and file descriptors across every required H1/H2 branch. Existing exact-source native owner tests remain separate evidence. +### I05 service admission and caller drain observer + +`lock_observer.py` observes an existing run-owned H1/H2 service using the explicit `lock-timeout` runtime profile and reviewed source `757b751482bb970bc321e92059690a7698554358`. The profile enables only the existing object-lock diagnostic target at TRACE, sets the actual object-lock acquisition budget to one second, disables early PUT rename ACKs, and preserves the standard disabled scanner/automatic-Heal/MRF settings. It does not modify the service binary. Run `python3 -B scripts/validation_heal/lock_observer.py --binding --nodes vm000 [vm001 vm002 vm003] --bucket --entry --output `. The entry contains the original `key`, `versionId`, `deleteMarker=false`, and complete ACK-bound `payload.size`/`payload.sha256`. This component neither deploys nor cleans a namespace; the enclosing case owns those steps and recovery. + +After persisting and sending the original single GET, the observer waits up to ten seconds for its actual server-generated request ID and `get_object` lock attempt. A unique User-Agent marker connects the exact version URI and method in the `http-request` ancestor span to that lock's owner/namespace/attempt. The marker is correlation data, not an identity asserted by the client. GET's lock diagnostic omits `requested_version_id`; its HTTP span must carry the exact historical version URI instead. Free-text messages, nested native-test capture rows, client-supplied request IDs, repeated lifecycle events and foreign endpoint/owner/version records cannot prove admission. Cursor-bounded raw journald pages remain in evidence, and source message loss prevents completion. + +Before disconnect, two `/proc` samples must connect the exact client/server TCP tuple to an actual established socket descriptor of the original candidate process. A network-namespace table entry alone is insufficient: its inode must also appear in that process's FD links. The read-only adapter accepts no arbitrary PID, path, command or service port. Process birth, binary, environment and mount namespace must match, and its network namespace cannot change during sampling. A second journal capture must still show acquired without a diagnostic drop. An unavailable/already-ended window records incomplete evidence and closes the original caller socket without retry. + +After the caller closes, the observer follows the same request/attempt/owner until its `guard_dropped` event, requires two complete samples without the original owned socket, and waits up to thirty seconds for each peer's FD count to return to its baseline. It then reads and verifies the exact historical version's full content and samples resources again. Every transport result remains UNKNOWN. Diagnostic guard drop does not prove distributed release, snapshots do not prove ownership across the disconnect instant, and aggregate FD counts do not prove absence of every leak. Those claims stay explicitly false. The timeout/stale-generation native fences, queued-successor lock closure, complete DL/DV physical and ACK verification, every H1/H2 branch and independent live acceptance remain required; I05 stays outside canonical dispatch. + ### J02 original opaque metadata probe The `unknown-negative` diagnostic includes the source procedure before the supplemental legacy-object diagnostic. A separate acknowledged object first receives an independently healthy Deep receipt. With the owned services stopped, the existing descriptor-relative fault backend flips byte zero of every original `xl.meta` copy: four copies in H1 and sixteen in H2. Each mutation has its own original backup, receipt and complete physical delta; the candidate-bound decoder must refuse every exact damaged copy. No other metadata, part or format may change. diff --git a/scripts/validation_heal/caller_abort.py b/scripts/validation_heal/caller_abort.py index b2e368d8..d88b7b94 100644 --- a/scripts/validation_heal/caller_abort.py +++ b/scripts/validation_heal/caller_abort.py @@ -18,7 +18,7 @@ from heal_probe import CLOCK_DOMAIN, Probe, canonical, require, save, validate_binding, validate_s3_request -def disconnect_get(probe, bucket, key, version_id, output): +def disconnect_get(probe, bucket, key, version_id, output, *, before_disconnect=None): """Record intent, send once, close the caller socket, retain UNKNOWN. The caller must independently establish server admission, lock ownership, @@ -36,6 +36,7 @@ def disconnect_get(probe, bucket, key, version_id, output): and probe.endpoint == probe.binding['endpoints'][probe.endpoint_name].rstrip('/'), 'caller endpoint differs from the current binding') endpoint = urlsplit(probe.endpoint) + marker = 'auto-testing-i05/' + str(uuid.uuid4()) path = '/' + quote(bucket, safe='') + '/' + quote(key, safe='/~') target = path + '?' + urlencode({'versionId': version_id}) from botocore.auth import S3SigV4Auth @@ -45,7 +46,7 @@ def disconnect_get(probe, bucket, key, version_id, output): token = None if probe.credential_pair else os.environ.get('HEAL_SESSION_TOKEN') outgoing = AWSRequest(method='GET', url=probe.endpoint + target, headers={ 'Host': endpoint.netloc, 'x-amz-content-sha256': hashlib.sha256(b'').hexdigest(), - 'Connection': 'close', 'Content-Length': '0'}) + 'Connection': 'close', 'Content-Length': '0', 'User-Agent': marker}) S3SigV4Auth(Credentials(access, secret, token), 's3', 'us-east-1').add_auth(outgoing) output = Path(output) output.mkdir(exist_ok=False) @@ -62,10 +63,12 @@ def disconnect_get(probe, bucket, key, version_id, output): 'bindingSha256': hashlib.sha256(bound.encode()).hexdigest(), 'endpoint': probe.endpoint_name, 'endpointUrl': probe.endpoint, 'operation': 'get_object', 'params': params, + 'callerMarker': marker, 'requestTarget': target, 'clockDomain': CLOCK_DOMAIN, 'preparedMonotonicNs': time.monotonic_ns(), 'retryAllowed': False, 'canonicalCasePassClaimed': False}) evidence = {'schemaVersion': 1, 'runId': probe.binding['runId'], 'requestSha256': hashlib.sha256(intent_path.read_bytes()).hexdigest(), + 'callerMarker': marker, 'requestTarget': target, 'clockDomain': CLOCK_DOMAIN, 'requestInvocationStarted': False, 'requestWriteCompleted': False, 'responseObserved': False, 'resultUnknown': True, 'acknowledged': False, 'retryAllowed': False, 'serverOwnershipProven': False, 'canonicalCasePassClaimed': False} @@ -92,6 +95,12 @@ def connect(self): connection.request('GET', target, headers=dict(outgoing.headers.items())) evidence.update(requestWriteCompleted=True, requestWrittenMonotonicNs=time.monotonic_ns()) save(output / 'request-written.json', evidence) + if before_disconnect is not None: + # Observe the service without reading this request's response. + # An observer failure still reaches the socket-closing finally. + before_disconnect(json.loads(json.dumps(evidence))) + validate_binding(probe.binding) + require(canonical(probe.binding) == bound, 'caller binding changed during service observation') evidence['disconnectStartedMonotonicNs'] = time.monotonic_ns() sock.shutdown(socket.SHUT_RDWR) evidence['shutdownCompleted'] = True diff --git a/scripts/validation_heal/caller_socket.py b/scripts/validation_heal/caller_socket.py new file mode 100644 index 00000000..d4a05e32 --- /dev/null +++ b/scripts/validation_heal/caller_socket.py @@ -0,0 +1,105 @@ +"""Read the exact caller socket from one run-owned Linux candidate process.""" + +import ipaddress +import os +from pathlib import Path +import re +import socket +import struct +import time + + +def require(value, message): + if not value: + raise ValueError(message) + + +def address(value): + require(isinstance(value, list) and len(value) == 2 and isinstance(value[0], str) + and type(value[1]) is int and 0 < value[1] < 65536, 'exact numeric TCP endpoint required') + ip = ipaddress.ip_address(value[0]) + if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped: + ip = ip.ipv4_mapped + return str(ip), value[1] + + +def proc_address(value, ipv6): + parts = value.split(':') + require(len(parts) == 2 and re.fullmatch(r'[0-9A-Fa-f]{32}' if ipv6 else r'[0-9A-Fa-f]{8}', parts[0]) + and re.fullmatch(r'[0-9A-Fa-f]{4}', parts[1]), 'invalid Linux TCP address') + packed = b''.join(struct.pack('=I', int(parts[0][offset:offset + 8], 16)) + for offset in range(0, len(parts[0]), 8)) + ip = ipaddress.ip_address(socket.inet_ntop(socket.AF_INET6 if ipv6 else socket.AF_INET, packed)) + if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped: + ip = ip.ipv4_mapped + return str(ip), int(parts[1], 16) + + +def connections(raw, ipv6, client, server): + require(isinstance(raw, str) and len(raw.encode()) <= 4 * 1024**2, 'bounded Linux TCP table required') + lines = raw.splitlines() + require(lines and 'local_address' in lines[0] and 'inode' in lines[0], 'Linux TCP table header differs') + found = [] + for line in lines[1:]: + fields = line.split() + require(len(fields) >= 10 and fields[9].isdigit() and re.fullmatch(r'[0-9A-Fa-f]{2}', fields[3]), + 'incomplete Linux TCP connection row') + local, remote = proc_address(fields[1], ipv6), proc_address(fields[2], ipv6) + if local == server and remote == client: + found.append({'inode': int(fields[9]), 'state': fields[3].upper()}) + require(len(found) <= 1, 'ambiguous exact TCP connection in network namespace') + return found + + +def run(request, node): + require(set(request) == {'runId', 'action', 'expectedProcess', 'clientAddress', 'serverAddress'} + and request['action'] == 'caller-socket', 'caller socket accepts no arbitrary PID or path') + client, server = address(request['clientAddress']), address(request['serverAddress']) + require(server[1] == 9000, 'caller socket requires the bound service port') + _, _, owner = node['owned'](request['runId']) + require(owner.get('runtimeProfile') == 'lock-timeout', 'caller socket requires the isolated I05 runtime') + expected = request['expectedProcess'] + require(isinstance(expected, dict) and all(name in expected for name in node['PROCESS_FIELDS']), + 'caller socket requires the original process birth') + def process(): + observed = node['process'](request) + require(type(observed.get('pid')) is int and observed['pid'] > 1 + and all(observed.get(name) == expected[name] for name in node['PROCESS_FIELDS']), + 'caller socket crossed a process birth or runtime change') + return observed + before = process() + root = Path(f'/proc/{before["pid"]}') + network_namespace = os.readlink(root / 'ns/net') + def sample(): + started = time.monotonic_ns() + fds, vanished, owned_sockets = list((root / 'fd').iterdir()), 0, {} + require(len(fds) <= 65536, 'owned process descriptor census exceeds its budget') + for fd in fds: + try: + target = os.readlink(fd) + except FileNotFoundError: + vanished += 1 + continue + match = re.fullmatch(r'socket:\[([0-9]+)\]', target) + if match: + owned_sockets.setdefault(int(match[1]), []).append(int(fd.name)) + matches = [] + for table, ipv6 in (('tcp', False), ('tcp6', True)): + # /proc/PID/net describes the namespace, not process ownership. + # Only a matching descriptor inode establishes the latter. + with (root / 'net' / table).open() as stream: + raw = stream.read(4 * 1024**2 + 1) + for row in connections(raw, ipv6, client, server): + if row['inode'] in owned_sockets: + matches.append({**row, 'descriptors': sorted(owned_sockets[row['inode']]), 'table': table}) + require(len(matches) <= 1, 'caller TCP connection maps to ambiguous owned sockets') + process() + require(os.readlink(root / 'ns/net') == network_namespace, 'caller socket network namespace changed') + return {'startedMonotonicNs': started, 'finishedMonotonicNs': time.monotonic_ns(), + 'fileDescriptors': len(fds), 'vanishedDescriptors': vanished, 'ownedMatches': matches} + samples = [sample(), sample()] + process() + return {'process': {name: before[name] for name in node['PROCESS_FIELDS']}, 'clockDomain': before['bootId'], + 'clientAddress': list(client), 'serverAddress': list(server), 'samples': samples, + 'networkNamespace': network_namespace, + 'readOnly': True, 'continuousAbsenceProven': False, 'canonicalCasePassClaimed': False} diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py index 78cccb2d..65b426ae 100644 --- a/scripts/validation_heal/execution_policy.py +++ b/scripts/validation_heal/execution_policy.py @@ -139,7 +139,7 @@ def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, . ), "heal.lock-timeout-and-caller-abort": _native( "I05", "NATIVE_LOCK_OWNER_ABORT_BARRIER", - "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集，以及单次签名历史版本 GET 的客户端 TCP 断开组件；仍缺服务端请求 admission/所有权绑定、完整 timeout/stale-generation 编排和 H1/H2 最终对象/锁/FD 验收。精确源码 cfg(test) 原生证据可独立构建，不要求给优化服务候选加入隐藏 hook；客户端断开或原生子集不能充作完整 canonical PASS。", + "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集、单连接历史 GET 断开和服务 admission/owner/socket 观测编排；观测编排尚无 live 验收。仍缺实际断连边界的精确所有权、完整 timeout/stale-generation controller 和 H1/H2 对象/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断观测、客户端断开或原生子集不能充作完整 canonical PASS。", ("server_lock_owner", "lock_timeout_fence", "caller_abort_sequence", "stale_read_repair_fence", "exact_source_native_evidence", "complete_live_orchestrator", "final_object_lock_fd_closure"), ), "heal.outcome-counters-and-error-truth": _native( diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index a41f048d..586cdce1 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -33,6 +33,12 @@ def runtime_config(profile): + if profile == "lock-timeout": + logging_filter = "info,rustfs_ecstore::object_lock_diag=trace" + return {**BACKGROUND_CONFIG, "RUSTFS_OBJECT_LOCK_DIAG_ENABLE": "true", + "RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT": "1", + "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE": "false", + "RUSTFS_OBS_LOGGER_LEVEL": logging_filter, "RUST_LOG": logging_filter} if profile == "frozen-race-legacy": # The frozen G04/G05 fixture predates protected shard sidecars. Keep # its exact file population and serial Heal scheduling explicit. @@ -551,7 +557,7 @@ def start(request): "J02 phase commit is incomplete; service cannot start") if owner.get("outcomePhase") == "legacy": outcome_legacy_phase({"runId": owner["runId"], "action": "outcome-legacy-phase"}) - if owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf", "outcome-legacy", "frozen-race-legacy"}: + if owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf", "outcome-legacy", "frozen-race-legacy", "lock-timeout"}: runtime_configuration(state, owner) command("systemd-run", "--unit=" + owner["unit"], "--property=Type=exec", "--property=KillMode=control-group", "--property=TimeoutStopSec=30", @@ -1866,6 +1872,7 @@ def main(request): actions.update({"replace-drive": replace_drive, "replacement-state": replacement_state, "replacement-probe": replacement_probe}) actions.update({"format-export": format_export, "foreign-format": foreign_format, "restore-replacement": restore_replacement}) actions.update({"process": process, "process-metrics": process_metrics, "process-signal": process_signal, "process-graceful": process_graceful, "process-crash": process_crash}) + actions["caller-socket"] = lambda req: caller_socket_backend.run(req, globals()) require(request.get("action") in actions, "unsupported node operation") try: result = actions[request["action"]](request) diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 2a2e6e3f..593c5d17 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"frozen-race-legacy", "outcome-legacy", "outcome-grace", "queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation", "incarnation-checkpoint", "incarnation-mrf"}, "unsupported Heal runtime profile") + require(runtime_profile in {"lock-timeout", "frozen-race-legacy", "outcome-legacy", "outcome-grace", "queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation", "incarnation-checkpoint", "incarnation-mrf"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -185,6 +185,9 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() + if action == "caller-socket": + source += "\nimport types\ncaller_socket_backend = types.ModuleType('caller_socket')\nexec(" + repr( + NODE_SOURCE.with_name("caller_socket.py").read_text()) + ", caller_socket_backend.__dict__)\n" if action in {"mpu-race-backups", "version-race-backups"}: source += "\nimport types\nmpu_race_backups_backend = types.ModuleType('mpu_race_backups')\nexec(" + repr( NODE_SOURCE.with_name("mpu_race_backups.py").read_text()) + ", mpu_race_backups_backend.__dict__)\n" diff --git a/scripts/validation_heal/lock_observer.py b/scripts/validation_heal/lock_observer.py new file mode 100644 index 00000000..6821a4f8 --- /dev/null +++ b/scripts/validation_heal/lock_observer.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +"""Observe one I05 caller admission and drain; never certify the whole case.""" + +import argparse +import hashlib +import json +from pathlib import Path +import re +import time +import uuid + +try: + from .caller_abort import disconnect_get + from .fleet_node import PROCESS_FIELDS, runtime_config + from .heal_probe import Probe, canonical, require, save, validate_binding + from .healthy import Fleet + from .scanner_repair import JournalWindow +except ImportError: + from caller_abort import disconnect_get + from fleet_node import PROCESS_FIELDS, runtime_config + from heal_probe import Probe, canonical, require, save, validate_binding + from healthy import Fleet + from scanner_repair import JournalWindow + + +TARGET = 'rustfs_ecstore::object_lock_diag' +CANDIDATE = '757b751482bb970bc321e92059690a7698554358' +ADMISSION_SECONDS = 10 +DRAIN_SECONDS = 30 + + +def caller_attempt(rows, sent, bucket, key): + """Bind flattened production logs through the actual HTTP ancestor span. + + External S3 request IDs are server-owned. The client User-Agent is only a + unique correlation marker; a client request-id header cannot stand in for + server admission. GET's lock diagnostic has no requested_version_id; the + exact version must instead match the HTTP span URI. + """ + require(re.fullmatch(r'auto-testing-i05/[0-9a-f-]{36}', sent.get('callerMarker', '')), + 'missing caller correlation marker') + selected = [] + for row in rows: + if row.get('target') != TARGET: + continue + spans = [row.get('span'), *row.get('spans', [])] + spans = [span for span in spans if isinstance(span, dict) and span.get('name') == 'http-request'] + marked = [span for span in spans if span.get('user_agent') == sent['callerMarker']] + if not marked: + continue + require(len({canonical(span) for span in marked}) == 1, 'caller has ambiguous HTTP ancestor spans') + span = marked[0] + request_id = span.get('request_id') + require(isinstance(request_id, str) and str(uuid.UUID(request_id)) == request_id + and uuid.UUID(request_id).int != 0 and row.get('request_id') == request_id + and span.get('method') == 'GET' and span.get('uri') == sent['requestTarget'], + 'caller server request ID, method or exact version URI differs') + require(row.get('event') == 'object_namespace_lock' and row.get('component') == 'ecstore' + and row.get('subsystem') == 'object_lock' and row.get('op') == 'get_object' + and row.get('mode') == 'read' and row.get('bucket') == bucket and row.get('object') == key + and row.get('requested_version_id') is None and row.get('timeout_ms') == 1000, + 'caller lock diagnostic escaped its exact GET identity or runtime budget') + require(all(isinstance(row.get(name), str) and 0 < len(row[name]) <= 2048 + for name in ('owner', 'namespace', 'lock_attempt_id')) + and str(uuid.UUID(row['lock_attempt_id'])) == row['lock_attempt_id'] + and uuid.UUID(row['lock_attempt_id']).int != 0 + and type(row.get('acquire_ms')) is int and row['acquire_ms'] >= 0, + 'caller lock owner, attempt or timing is incomplete') + if row.get('state') == 'failed': + require(row.get('level') == 'ERROR' and isinstance(row.get('failure'), str), + 'caller lock failure lost its error classification') + else: + require(row.get('level') == 'TRACE' and type(row.get('hold_ms')) is int and row['hold_ms'] >= 0, + 'caller lock lifecycle lost its diagnostic level or hold timing') + selected.append(row) + if not selected: + return None + identity = ('request_id', 'lock_attempt_id', 'owner', 'namespace') + require(len({tuple(row[name] for name in identity) for row in selected}) == 1, + 'caller marker maps to multiple requests or lock owners') + states = [row.get('state') for row in selected] + allowed = (['acquiring'], ['acquiring', 'acquired'], ['acquiring', 'acquired', 'guard_dropped'], + ['acquiring', 'failed'], ['acquiring', 'cancelled']) + require(states in allowed, 'caller lock lifecycle is missing, duplicated or out of order') + return {**{name: selected[0][name] for name in identity}, 'states': states, 'events': selected, + 'admitted': True, 'observedAcquiredWithoutDrop': states == ['acquiring', 'acquired'], + 'diagnosticGuardDropped': states == ['acquiring', 'acquired', 'guard_dropped'], + 'distributedReleaseProven': False, 'heldAtDisconnectProven': False} + + +def same_process(observed, expected): + require(isinstance(observed, dict) and all(name in expected and observed.get(name) == expected[name] + for name in PROCESS_FIELDS), 'I05 observation crossed a service process birth or runtime change') + + +def observe_disconnect(probe, fleet, bucket, entry, output): + """Use an already deployed exclusive I05 service; do not alter the fleet.""" + validate_binding(probe.binding) + require(fleet.runtime_profile == 'lock-timeout' and fleet.run_id == probe.binding['runId'] + and probe.binding['candidateCommit'] == CANDIDATE, + 'caller observation requires its reviewed candidate and isolated runtime') + endpoints = {f'node{index}': f'http://rustfs-node{index}:9000' for index in range(1, len(fleet.nodes) + 1)} + require(probe.binding['endpoints'] == endpoints and len(set(fleet.nodes)) == len(fleet.nodes) + and ((len(fleet.nodes) == 1 and fleet.run_id.endswith('-h1')) + or (len(fleet.nodes) == 4 and fleet.run_id.endswith('-h2'))), + 'I05 caller endpoints differ from the H1/H2 fleet') + require(entry.get('deleteMarker') is False and isinstance(entry.get('payload'), dict) + and set(entry['payload']) == {'size', 'sha256'} and type(entry['payload']['size']) is int + and 0 < entry['payload']['size'] <= probe.binding['maxInventoryBytes'] + and re.fullmatch(r'[0-9a-f]{64}', entry['payload']['sha256']), + 'caller drain requires an acknowledged historical content manifest') + output = Path(output) + output.mkdir(exist_ok=False) + bound = canonical(probe.binding) + save(output / 'scope.json', {'candidateCommit': CANDIDATE, 'bindingSha256': hashlib.sha256(bound.encode()).hexdigest(), + 'runId': fleet.run_id, 'entry': entry, 'admissionSeconds': ADMISSION_SECONDS, 'drainSeconds': DRAIN_SECONDS, + 'canonicalCasePassClaimed': False, 'mutatesServiceConfiguration': False}) + expected, baseline = {}, {} + def metrics(label): + values = {} + for node in fleet.nodes: + value = fleet.record(label, {'node': node, **fleet.remote(node, 'process-metrics')}) + identity = value['identity'] + require(identity.get('binarySha256') == probe.binding['candidateBinarySha256'] + and all(identity.get('settings', {}).get(k) == v for k, v in runtime_config('lock-timeout').items()), + 'I05 resource sample has another candidate or process configuration') + if node in expected: + same_process(identity, expected[node]) + else: + expected[node] = identity + require(type(value.get('fileDescriptors')) is int and value['fileDescriptors'] >= 0, + 'I05 descriptor count is incomplete') + values[node] = value + return values + baseline = metrics('I05-caller-baseline-resources') + window = JournalWindow(fleet, 'I05-caller', event_targets={TARGET}, strict_json=True) + node = fleet.nodes[list(endpoints).index(probe.endpoint_name)] + admitted = None + network_namespace = None + def capture(sent): + validate_binding(probe.binding) + require(canonical(probe.binding) == bound, 'I05 caller binding drifted during observation') + events = window.capture() + attempts = {host: caller_attempt(rows, sent, bucket, entry['key']) for host, rows in events.items()} + require(all(value is None for host, value in attempts.items() if host != node), + 'caller marker appeared on a foreign endpoint') + return attempts[node] + def sockets(sent, label): + nonlocal network_namespace + value = fleet.record(label, {'node': node, **fleet.remote(node, 'caller-socket', + expectedProcess=expected[node], clientAddress=sent['localAddress'], serverAddress=sent['peerAddress'])}) + same_process(value.get('process'), expected[node]) + require(value.get('readOnly') is True and value.get('clockDomain') == expected[node]['bootId'] + and len(value.get('samples', [])) == 2, 'caller socket census lacks its original process or samples') + namespace = value.get('networkNamespace') + require(isinstance(namespace, str) and re.fullmatch(r'net:\[[0-9]+\]', namespace), + 'caller socket census lost its network namespace') + require(network_namespace is None or namespace == network_namespace, + 'caller socket moved to another network namespace between admission and drain') + network_namespace = namespace + return value + def before_disconnect(sent): + nonlocal admitted + deadline = time.monotonic() + ADMISSION_SECONDS + while True: + observed = capture(sent) + if observed is not None: + require(observed['states'] in (['acquiring'], ['acquiring', 'acquired']), + 'NOT_OBSERVED: caller lock already ended before controlled disconnect') + if observed['observedAcquiredWithoutDrop']: + connection = sockets(sent, 'I05-caller-before-disconnect-socket') + require(all(sample['vanishedDescriptors'] == 0 and len(sample['ownedMatches']) == 1 + and sample['ownedMatches'][0]['state'] == '01' + for sample in connection['samples']), + 'NOT_OBSERVED: admitted caller lacks its exact established service descriptor') + # Re-read the journal after the socket census. A later + # native fence must still prove the exact disconnect edge. + observed = capture(sent) + require(observed is not None and observed['observedAcquiredWithoutDrop'], + 'NOT_OBSERVED: caller lock ended while sampling its service descriptor') + metrics('I05-caller-admission-resources') + require(time.monotonic() <= deadline, 'NOT_OBSERVED: caller admission observation exceeded its budget') + admitted = {'node': node, 'attempt': observed, 'socket': connection} + save(output / 'admission.json', admitted) + return + require(time.monotonic() < deadline, 'NOT_OBSERVED: caller lock acquisition window was not observed') + time.sleep(.1) + try: + sent = disconnect_get(probe, bucket, entry['key'], entry['versionId'], output / 'transport', + before_disconnect=before_disconnect) + deadline = time.monotonic() + DRAIN_SECONDS + while True: + observed = capture(sent) + require(observed is not None and all(observed[name] == admitted['attempt'][name] + for name in ('request_id', 'lock_attempt_id', 'owner', 'namespace')), + 'caller drain lost its original service request or lock owner') + connection = sockets(sent, 'I05-caller-drain-socket') + resources = metrics('I05-caller-drain-resources') + require(time.monotonic() <= deadline, 'NOT_OBSERVED: caller drain observation exceeded its budget') + if (observed['diagnosticGuardDropped'] and all(sample['vanishedDescriptors'] == 0 + and not sample['ownedMatches'] for sample in connection['samples']) + and all(resources[host]['fileDescriptors'] <= baseline[host]['fileDescriptors'] for host in fleet.nodes)): + break + require(time.monotonic() < deadline, 'NOT_OBSERVED: caller diagnostic, socket or FD count did not drain') + time.sleep(.2) + read = fleet.record('I05-caller-final-exact-version', probe.s3('get_object', + {'Bucket': bucket, 'Key': entry['key'], 'VersionId': entry['versionId']})) + require(read.get('acknowledged') is True and read.get('resultUnknown') is False + and read.get('response', {}).get('VersionId') == entry['versionId'] and read.get('payload') == entry['payload'], + 'caller drain changed the acknowledged historical version or its complete content') + deadline = time.monotonic() + DRAIN_SECONDS + while True: + capture(sent) + verified_resources = metrics('I05-caller-after-verification-resources') + require(time.monotonic() <= deadline, 'NOT_OBSERVED: I05 verification resource observation exceeded its budget') + if all(verified_resources[host]['fileDescriptors'] <= baseline[host]['fileDescriptors'] for host in fleet.nodes): + break + require(time.monotonic() < deadline, 'NOT_OBSERVED: I05 verification did not return to its FD count baseline') + time.sleep(.2) + result = {'observationComplete': True, 'admission': admitted, 'finalAttempt': observed, + 'finalSocket': connection, 'finalResources': resources, 'afterVerificationResources': verified_resources, + 'exactHistoricalContentPreserved': True, + 'callerResultUnknown': True, 'heldAtDisconnectProven': False, 'distributedReleaseProven': False, + 'noGlobalFdLeakProven': False, 'canonicalCasePassClaimed': False, + 'remaining': ['exact native disconnect/timeout/stale-generation fences', + 'complete H1/H2 DL/DV ACK and physical closure', 'queued-successor lock and FD closure']} + save(output / 'observation.json', result) + return result + except BaseException as error: + save(output / 'failure.json', {'observationComplete': False, 'errorType': type(error).__name__, + 'state': 'NOT_OBSERVED' if str(error).startswith('NOT_OBSERVED:') else 'INCOMPLETE', + 'canonicalCasePassClaimed': False, 'originalTransportEvidenceRetained': True}) + raise + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--binding', type=Path, required=True) + parser.add_argument('--nodes', nargs='+', required=True) + parser.add_argument('--bucket', required=True) + parser.add_argument('--entry', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--endpoint') + args = parser.parse_args() + probe = Probe(json.loads(args.binding.read_text()), args.endpoint) + args.output.mkdir(parents=True, exist_ok=False) + # The component uses an existing owned deployment and never calls cleanup. + # Keep its SSH/log evidence separate from the exclusive request directory. + fleet = Fleet(args.nodes, args.output, probe.binding['runId'], runtime_profile='lock-timeout') + fleet.secret_values.update(probe.credentials()) + observe_disconnect(probe, fleet, args.bucket, json.loads(args.entry.read_text()), args.output / 'caller') + + +if __name__ == '__main__': + main() diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index 111ec405..42261087 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -3253,7 +3253,7 @@ "dispatchable": false, "code": "I05", "reasonCode": "NATIVE_LOCK_OWNER_ABORT_BARRIER", - "reason": "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集，以及单次签名历史版本 GET 的客户端 TCP 断开组件；仍缺服务端请求 admission/所有权绑定、完整 timeout/stale-generation 编排和 H1/H2 最终对象/锁/FD 验收。精确源码 cfg(test) 原生证据可独立构建，不要求给优化服务候选加入隐藏 hook；客户端断开或原生子集不能充作完整 canonical PASS。", + "reason": "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集、单连接历史 GET 断开和服务 admission/owner/socket 观测编排；观测编排尚无 live 验收。仍缺实际断连边界的精确所有权、完整 timeout/stale-generation controller 和 H1/H2 对象/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断观测、客户端断开或原生子集不能充作完整 canonical PASS。", "requiredCapabilities": [ "server_lock_owner", "lock_timeout_fence", diff --git a/scripts/validation_heal/scanner_repair.py b/scripts/validation_heal/scanner_repair.py index 9987ae75..e0f3bfae 100644 --- a/scripts/validation_heal/scanner_repair.py +++ b/scripts/validation_heal/scanner_repair.py @@ -30,9 +30,10 @@ def null_version(value): class JournalWindow: - def __init__(self, fleet, label, *, event_targets=None): + def __init__(self, fleet, label, *, event_targets=None, strict_json=False): self.fleet, self.label = fleet, label self.event_targets = event_targets + self.strict_json = strict_json self.events = {node: [] for node in fleet.nodes} self.cursors = {} for node in fleet.nodes: @@ -52,7 +53,10 @@ def capture(self): self.cursors[node] = observed["afterCursor"] for line in observed.get("journal", "").splitlines(): try: event = json.loads(line) - except json.JSONDecodeError: continue + except json.JSONDecodeError: + require(not self.strict_json, "journal window contains a malformed JSON event") + continue + require(not self.strict_json or isinstance(event, dict), "journal window contains a non-object JSON event") # Raw pages remain in immutable evidence; keep only source # attribution events in memory, not unrelated debug noise. if isinstance(event, dict) and (self.event_targets is None diff --git a/test_validation_heal_caller_abort.py b/test_validation_heal_caller_abort.py index 470c09fc..304e9cde 100644 --- a/test_validation_heal_caller_abort.py +++ b/test_validation_heal_caller_abort.py @@ -60,7 +60,7 @@ def tearDown(self): def invoke(self, output=None, **changes): return caller_abort.disconnect_get(self.probe, changes.get('bucket', self.binding['bucketPrefix']), changes.get('key', '中文/literal%2F space.bin'), changes.get('version', VERSION), - output or self.root / 'call') + output or self.root / 'call', before_disconnect=changes.get('before_disconnect')) def test_one_signed_exact_version_get_closes_real_socket_and_retains_unknown(self): proof = self.invoke() @@ -71,6 +71,8 @@ def test_one_signed_exact_version_get_closes_real_socket_and_retains_unknown(sel self.assertEqual(path, '/rv-heal-0123456789abcdef/%E4%B8%AD%E6%96%87/literal%252F%20space.bin?versionId=' + VERSION) self.assertTrue(headers['Authorization'].startswith('AWS4-HMAC-SHA256 Credential=private-test-access/')) self.assertEqual(headers['Host'], f'127.0.0.1:{self.server.server_port}') + self.assertEqual(headers['User-Agent'], proof['callerMarker']) + self.assertEqual(path, proof['requestTarget']) from botocore.auth import S3SigV4Auth from botocore.awsrequest import AWSRequest from botocore.credentials import Credentials @@ -97,6 +99,37 @@ def test_one_signed_exact_version_get_closes_real_socket_and_retains_unknown(sel for secret in ('private-test-access', 'private-test-secret', 'Authorization'): self.assertNotIn(secret, text) + def test_observer_runs_after_durable_write_while_caller_socket_is_open(self): + observations = [] + def observer(sent): + self.assertEqual(json.loads((self.root / 'call/request-written.json').read_text()), sent) + self.assertFalse(self.server.disconnected.is_set()) + observations.append(sent['callerMarker']) + sent['serverOwnershipProven'] = True + proof = self.invoke(before_disconnect=observer) + self.assertEqual(observations, [proof['callerMarker']]) + self.assertFalse(proof['serverOwnershipProven']) + self.assertTrue(self.server.disconnected.wait(3)) + + def test_observer_failure_closes_only_the_caller_and_retains_unknown(self): + def observer(sent): + raise ValueError('no service ownership window') + with self.assertRaisesRegex(ValueError, 'no service ownership'): + self.invoke(before_disconnect=observer) + self.assertTrue(self.server.disconnected.wait(3)) + proof = json.loads((self.root / 'call/disconnected.json').read_text()) + self.assertTrue(proof['clientSocketClosed'] and proof['resultUnknown']) + self.assertNotIn('disconnectStartedMonotonicNs', proof) + self.assertEqual(len(self.server.requests), 1) + + def test_observer_binding_drift_aborts_and_closes_without_another_request(self): + def observer(sent): + self.probe.binding['candidateCommit'] = 'f' * 40 + with self.assertRaisesRegex(ValueError, 'binding changed during'): + self.invoke(before_disconnect=observer) + self.assertTrue(self.server.disconnected.wait(3)) + self.assertEqual(len(self.server.requests), 1) + def test_existing_evidence_directory_prevents_replay_without_another_connection(self): self.invoke() self.assertTrue(self.server.disconnected.wait(3)) diff --git a/test_validation_heal_lock_observer.py b/test_validation_heal_lock_observer.py new file mode 100644 index 00000000..961187ac --- /dev/null +++ b/test_validation_heal_lock_observer.py @@ -0,0 +1,310 @@ +"""I05 server admission, process-bound socket census and honest evidence limits.""" + +import hashlib +import json +import os +from pathlib import Path +import socket +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from scripts.validation_heal import caller_socket as sockets +from scripts.validation_heal import fleet_node, lock_observer as observer +import test_validation_heal_caller_abort as wire_fixture + +VERSION = wire_fixture.VERSION + + +REQUEST = '10000000-0000-4000-8000-000000000001' +ATTEMPT = '20000000-0000-4000-8000-000000000001' +SENT = {'callerMarker': 'auto-testing-i05/' + REQUEST, + 'requestTarget': '/rv-heal-0123456789abcdef/target.bin?versionId=' + VERSION, + 'localAddress': ['127.0.0.1', 39001], 'peerAddress': ['127.0.0.1', 9000]} +BUCKET = 'rv-heal-0123456789abcdef' +HEADER = ' sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode' + + +def event(state, sent=SENT): + return {'target': observer.TARGET, 'level': 'TRACE', 'event': 'object_namespace_lock', + 'component': 'ecstore', 'subsystem': 'object_lock', 'state': state, 'op': 'get_object', + 'bucket': BUCKET, 'object': 'target.bin', 'namespace': 'namespace-one', 'owner': 'owner-one', + 'lock_attempt_id': ATTEMPT, 'request_id': REQUEST, 'mode': 'read', 'timeout_ms': 1000, + 'acquire_ms': 1, 'hold_ms': 1, 'span': {'name': 'http-request', 'user_agent': sent['callerMarker'], + 'request_id': REQUEST, 'method': 'GET', 'uri': sent['requestTarget']}} + + +def identity(): + return {**dict.fromkeys(fleet_node.PROCESS_FIELDS, 'unchanged'), 'pid': 123, 'startTicks': 456, + 'bootId': 'original-boot', 'binarySha256': 'd' * 64, + 'settings': fleet_node.runtime_config('lock-timeout')} + + +class LockEventTests(unittest.TestCase): + def test_incomplete_or_non_json_journal_page_cannot_complete_i05_observation(self): + from scripts.validation_heal.scanner_repair import JournalWindow + class Fleet: + nodes = ['vm000'] + journal = 'truncated candidate JSON' + complete = True + def record(self, label, value): return value + def remote(self, node, action, **params): + return {'complete': self.complete, 'beforeCursor': params.get('afterCursor'), + 'afterCursor': 'cursor-next', 'journal': self.journal, 'hasMore': False} + for raw, complete in [('truncated candidate JSON', True), ('[]', True), ('{}', False)]: + with self.subTest(raw=raw): + fleet = Fleet() + window = JournalWindow(fleet, 'I05', event_targets={observer.TARGET}, strict_json=True) + fleet.journal, fleet.complete = raw, complete + with self.assertRaises(ValueError): window.capture() + def test_flattened_production_event_binds_server_id_and_exact_version_uri(self): + rows = [event('acquiring'), event('acquired')] + rows[0]['spans'] = [rows[0]['span']] + proof = observer.caller_attempt(rows, SENT, BUCKET, 'target.bin') + self.assertEqual(proof['request_id'], REQUEST) + self.assertTrue(proof['admitted'] and proof['observedAcquiredWithoutDrop']) + rows.append(event('guard_dropped')) + proof = observer.caller_attempt(rows, SENT, BUCKET, 'target.bin') + self.assertTrue(proof['diagnosticGuardDropped']) + self.assertFalse(proof['distributedReleaseProven'] or proof['heldAtDisconnectProven']) + + def test_rejects_forged_request_id_foreign_key_version_owner_or_incomplete_events(self): + mutations = [lambda e: e.update(request_id=ATTEMPT), lambda e: e.update(object='foreign.bin'), + lambda e: e.update(owner='another-owner'), lambda e: e.update(timeout_ms=5000), + lambda e: e.update(acquire_ms=True), lambda e: e.update(hold_ms=-1), + lambda e: e.update(requested_version_id=VERSION), + lambda e: e['span'].update(uri='/other-version'), lambda e: e['span'].update(method='PUT'), + lambda e: e.update(lock_attempt_id=REQUEST)] + for mutate in mutations: + with self.subTest(mutate=mutate): + rows = [event('acquiring'), event('acquired')] + mutate(rows[1]) + with self.assertRaises(ValueError): + observer.caller_attempt(rows, SENT, BUCKET, 'target.bin') + for states in (['acquired'], ['acquiring', 'acquiring'], ['acquiring', 'disabled'], + ['acquiring', 'guard_dropped'], ['acquiring', 'acquired', 'acquired']): + with self.subTest(states=states), self.assertRaises(ValueError): + observer.caller_attempt([event(s) for s in states], SENT, BUCKET, 'target.bin') + + def test_free_text_client_markers_and_native_nested_fields_do_not_prove_admission(self): + for row in ({'message': str(SENT)}, {'target': 'foreign', **{'fields': event('acquiring')}}, + {'target': observer.TARGET, 'fields': event('acquiring')}, + {**event('acquiring'), 'span': {'name': 'http-request', 'user_agent': 'different'}}): + self.assertIsNone(observer.caller_attempt([row], SENT, BUCKET, 'target.bin')) + + def test_cancelled_and_failed_attempts_never_count_as_acquired_or_dropped(self): + for state in ('cancelled', 'failed'): + end = event(state) + if state == 'failed': end.update(level='ERROR', failure='timeout', acquire_ms=1001) + proof = observer.caller_attempt([event('acquiring'), end], SENT, BUCKET, 'target.bin') + self.assertFalse(proof['diagnosticGuardDropped'] or proof['observedAcquiredWithoutDrop']) + + +class CallerSocketTests(unittest.TestCase): + def request(self): + return {'runId': 'gh-12345-1-h1', 'action': 'caller-socket', 'expectedProcess': identity(), + 'clientAddress': SENT['localAddress'], 'serverAddress': SENT['peerAddress']} + + def node(self): + return {'owned': lambda run: (None, None, {'runtimeProfile': 'lock-timeout'}), + 'process': lambda request: identity(), 'PROCESS_FIELDS': fleet_node.PROCESS_FIELDS} + + def test_frozen_linux_tables_include_zero_port_listeners_and_mapped_ipv6(self): + ipv4 = HEADER + '\n 0: 0100007F:2328 0100007F:9859 01 0:0 00:0 0 1000 0 678\n' + ipv4 += ' 1: 00000000:2328 00000000:0000 0A 0:0 00:0 0 1000 0 123\n' + self.assertEqual(sockets.connections(ipv4, False, ('127.0.0.1', 39001), ('127.0.0.1', 9000)), + [{'inode': 678, 'state': '01'}]) + self.assertEqual(sockets.proc_address('0000000000000000FFFF00000100007F:2328', True), ('127.0.0.1', 9000)) + self.assertEqual(sockets.proc_address('00000000000000000000000001000000:0000', True), ('::1', 0)) + for raw in ('', HEADER + '\n truncated', ipv4 + ipv4.splitlines()[1]): + with self.assertRaises(ValueError): + sockets.connections(raw, False, ('127.0.0.1', 39001), ('127.0.0.1', 9000)) + + def test_namespace_tcp_entry_without_owned_fd_does_not_count_as_owned_socket(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'fd').mkdir(); (root / 'net').mkdir() + (root / 'ns').mkdir(); (root / 'ns/net').symlink_to('net:[123]') + (root / 'net/tcp').write_text(HEADER + '\n0: 0100007F:2328 0100007F:9859 01 0:0 00:0 0 1000 0 678\n') + (root / 'net/tcp6').write_text(HEADER + '\n') + (root / 'fd/3').symlink_to('socket:[999]') + with patch.object(sockets, 'Path', return_value=root): + proof = sockets.run(self.request(), self.node()) + self.assertTrue(all(not row['ownedMatches'] for row in proof['samples'])) + (root / 'fd/4').symlink_to('socket:[678]') + proof = sockets.run(self.request(), self.node()) + self.assertTrue(all(row['ownedMatches'][0]['descriptors'] == [4] for row in proof['samples'])) + changed = self.node(); calls = [identity(), identity(), {**identity(), 'startTicks': 999}] + changed['process'] = lambda req: calls.pop(0) + with self.assertRaisesRegex(ValueError, 'process birth'): + sockets.run(self.request(), changed) + + def test_path_pid_port_and_profile_injection_fail_before_proc_access(self): + for change in ({'path': '/etc/shadow'}, {'pid': 1}, {'serverAddress': ['127.0.0.1', 22]}, + {'clientAddress': ['hostname', 39001]}, {'clientAddress': ['127.0.0.1', True]}): + with self.subTest(change=change), patch.object(sockets, 'Path', side_effect=AssertionError('no proc read')): + with self.assertRaises(ValueError): sockets.run({**self.request(), **change}, self.node()) + node = self.node(); node['owned'] = lambda run: (None, None, {'runtimeProfile': 'manual'}) + with self.assertRaises(ValueError): sockets.run(self.request(), node) + + def test_fleet_transport_embeds_the_actual_read_only_socket_backend(self): + def execute(*args, **kwargs): + source = kwargs['input'].rsplit('\nmain(', 1)[0] + scope = {'__name__': '__i05_socket_test__'} + exec(compile(source, 'i05-socket-node', 'exec'), scope) + self.assertTrue(callable(scope['caller_socket_backend'].run)) + self.assertEqual(scope['runtime_config']('lock-timeout')['RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT'], '1') + return SimpleNamespace(returncode=0, stdout='{}', stderr='') + with tempfile.TemporaryDirectory() as tmp, \ + patch('scripts.validation_heal.healthy.subprocess.run', side_effect=execute): + fleet = observer.Fleet(['vm000'], Path(tmp), 'gh-12345-1-h1', runtime_profile='lock-timeout') + self.assertEqual(fleet.remote('vm000', 'caller-socket', **{k: v for k, v in self.request().items() + if k not in {'runId', 'action'}}), {}) + + @unittest.skipUnless(sys.platform == 'linux', 'requires actual Linux /proc TCP tables') + def test_actual_linux_socket_is_bound_to_accepted_descriptor_and_disappears_after_close(self): + with socket.socket() as listener, socket.socket() as client: + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(('127.0.0.1', 9000)); listener.listen(1) + client.connect(listener.getsockname()) + accepted, _ = listener.accept() + node = self.node() + actual = {**identity(), 'pid': os.getpid()} + node['process'] = lambda req: actual + req = {**self.request(), 'expectedProcess': actual, 'clientAddress': list(client.getsockname())} + try: + proof = sockets.run(req, node) + self.assertTrue(all(row['ownedMatches'][0]['descriptors'] == [accepted.fileno()] for row in proof['samples'])) + finally: + accepted.close() + proof = sockets.run(req, node) + self.assertTrue(all(not row['ownedMatches'] for row in proof['samples'])) + + +class ObserverFlowTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + # Reuse the existing real local wire fixture without inheriting its tests. + self.wire = wire_fixture.CallerAbortTests('test_one_signed_exact_version_get_closes_real_socket_and_retains_unknown') + self.wire.setUp() + self.addCleanup(self.wire.tearDown); self.addCleanup(self.tmp.cleanup) + self.probe = self.wire.probe + self.probe.binding.update(candidateCommit=observer.CANDIDATE) + self.probe.binding['endpoints']['node1'] = 'http://rustfs-node1:9000' + self.probe.endpoint = 'http://rustfs-node1:9000' + connect = socket.create_connection + def local_wire(address, *args, **kwargs): + self.assertEqual(address, ('rustfs-node1', 9000)) + return connect(('127.0.0.1', self.wire.server.server_port), *args, **kwargs) + self.connect_patch = patch('scripts.validation_heal.caller_abort.socket.create_connection', side_effect=local_wire) + self.connect_patch.start(); self.addCleanup(self.connect_patch.stop) + self.entry = {'key': 'target.bin', 'versionId': VERSION, 'deleteMarker': False, + 'payload': {'size': 1, 'sha256': hashlib.sha256(b'x').hexdigest()}} + self.stage = 'before' + self.sent = None + self.change = None + self.labels = [] + owner = self + class Fleet: + nodes = ['vm000']; run_id = owner.probe.binding['runId']; runtime_profile = 'lock-timeout' + def record(self, label, value): + owner.labels.append(label) + return value + def remote(self, node, action, **params): + if action == 'process-metrics': + return {'identity': identity(), 'fileDescriptors': 10 if owner.change != 'leak' else + 11 if owner.stage == 'after' else 10} + if action == 'caller-socket': + matches = [{'state': '01', 'inode': 678, 'descriptors': [4]}] if owner.stage == 'before' else [] + return {'process': identity(), 'clockDomain': identity()['bootId'], 'readOnly': True, + 'networkNamespace': 'net:[2]' if owner.change == 'namespace' and owner.stage == 'after' else 'net:[1]', + 'samples': [{'ownedMatches': matches, 'vanishedDescriptors': 0}, + {'ownedMatches': matches, 'vanishedDescriptors': 0}]} + raise AssertionError(action) + class Window: + def __init__(self, *args, **kwargs): pass + def capture(self): + rows = [event('acquiring', owner.sent), event('acquired', owner.sent)] + if owner.stage == 'after' or owner.change == 'early-drop': rows.append(event('guard_dropped', owner.sent)) + if owner.change == 'owner' and owner.stage == 'after': rows[-1]['owner'] = 'rebound' + return {'vm000': rows} + self.fleet = Fleet() + self.window_patch = patch.object(observer, 'JournalWindow', Window); self.window_patch.start() + self.addCleanup(self.window_patch.stop) + original = observer.disconnect_get + def transport(probe, bucket, key, version, output, *, before_disconnect): + def on_sent(sent): + owner.sent = sent + before_disconnect(sent) + value = original(probe, bucket, key, version, output, before_disconnect=on_sent) + self.stage = 'after' + return value + self.transport_patch = patch.object(observer, 'disconnect_get', side_effect=transport); self.transport_patch.start() + self.addCleanup(self.transport_patch.stop) + self.probe.s3 = lambda *args: {'acknowledged': True, 'resultUnknown': False, + 'response': {'VersionId': VERSION}, 'payload': self.entry['payload']} + + def invoke(self): + return observer.observe_disconnect(self.probe, self.fleet, BUCKET, self.entry, self.root / 'observe') + + def test_complete_component_keeps_original_unknown_and_all_missing_case_proofs(self): + proof = self.invoke() + self.assertTrue(proof['observationComplete'] and proof['exactHistoricalContentPreserved']) + for name in ('heldAtDisconnectProven', 'distributedReleaseProven', 'noGlobalFdLeakProven', 'canonicalCasePassClaimed'): + self.assertFalse(proof[name]) + transport = json.loads((self.root / 'observe/transport/disconnected.json').read_text()) + self.assertTrue(transport['resultUnknown']) + self.assertFalse(transport['acknowledged'] or transport['serverOwnershipProven']) + self.assertTrue((self.root / 'observe/admission.json').is_file()) + + def test_closed_before_disconnect_is_not_observed_and_closes_original_socket(self): + self.change = 'early-drop' + with self.assertRaisesRegex(ValueError, 'NOT_OBSERVED'): + self.invoke() + self.assertTrue(self.wire.server.disconnected.wait(3)) + self.assertFalse(json.loads((self.root / 'observe/failure.json').read_text())['observationComplete']) + self.assertEqual(len(self.wire.server.requests), 1) + + def test_rebound_owner_fd_leak_or_wrong_final_content_cannot_complete_observation(self): + self.change = 'owner' + with self.assertRaises(ValueError): self.invoke() + self.assertFalse((self.root / 'observe/observation.json').exists()) + + def test_network_namespace_move_cannot_hide_original_socket_during_drain(self): + self.change = 'namespace' + with self.assertRaisesRegex(ValueError, 'network namespace between'): + self.invoke() + self.assertFalse((self.root / 'observe/observation.json').exists()) + + def test_fd_count_must_return_to_baseline_and_original_failure_is_kept(self): + self.change = 'leak' + clock = [0] + def advance(seconds): clock[0] += observer.DRAIN_SECONDS + 1 + with patch.object(observer.time, 'monotonic', side_effect=lambda: clock[0]), \ + patch.object(observer.time, 'sleep', side_effect=advance), \ + self.assertRaisesRegex(ValueError, 'caller drain observation exceeded its budget'): + self.invoke() + self.assertEqual(self.labels.count('I05-caller-drain-resources'), 2) + self.assertFalse((self.root / 'observe/observation.json').exists()) + + def test_late_admission_cannot_turn_an_expired_observation_into_completion(self): + with patch.object(observer, 'ADMISSION_SECONDS', 0), self.assertRaisesRegex(ValueError, 'exceeded its budget'): + self.invoke() + self.assertTrue(self.wire.server.disconnected.wait(3)) + self.assertFalse((self.root / 'observe/observation.json').exists()) + + def test_final_version_must_match_the_original_ack_content(self): + self.probe.s3 = lambda *args: {'acknowledged': True, 'resultUnknown': False, + 'response': {'VersionId': VERSION}, 'payload': {'size': 1, 'sha256': '0' * 64}} + with self.assertRaisesRegex(ValueError, 'historical version'): + self.invoke() + + def test_unreviewed_source_cannot_read_or_send_any_request(self): + self.probe.binding['candidateCommit'] = 'a' * 40 + with patch.object(self.fleet, 'remote', side_effect=AssertionError('no remote call')): + with self.assertRaisesRegex(ValueError, 'reviewed candidate'): + self.invoke() + self.assertFalse(self.wire.server.requests) From fa79b5214a7fa4298a060b02bba6e6c5ff026a62 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 09:29:47 +0800 Subject: [PATCH 280/343] feat(validation): diagnose I05 object namespace lock timeouts --- .../workflows/validation-catalog-tests.yml | 2 +- docs/validation-heal-cases.md | 2 +- docs/validation-heal-operations.md | 8 + scripts/validation_heal/caller_abort.py | 18 +- scripts/validation_heal/caller_socket.py | 6 +- scripts/validation_heal/lock_observer.py | 49 ++- scripts/validation_heal/lock_timeout.py | 299 ++++++++++++++++++ test_validation_heal_caller_abort.py | 20 +- test_validation_heal_lock_observer.py | 157 ++++++++- test_validation_heal_lock_timeout.py | 194 ++++++++++++ 10 files changed, 732 insertions(+), 23 deletions(-) create mode 100644 scripts/validation_heal/lock_timeout.py create mode 100644 test_validation_heal_lock_timeout.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 757bc65e..8ff84a2d 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -159,7 +159,7 @@ jobs: sudo apt-get install --no-install-recommends -y nftables fi - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - - run: python3 -B -m unittest -v test_validation_heal_lock_observer.py test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py + - run: python3 -B -m unittest -v test_validation_heal_lock_timeout.py test_validation_heal_lock_observer.py test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index e37531b2..e7a7f1f7 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1770,7 +1770,7 @@ Topologies: H1,H2. Branches: object-lock-timeout, caller-disconnect, stale-read- Execution policy: `evidence-blocked`; disabled by default: `false`. -Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集、单连接历史 GET 断开和服务 admission/owner/socket 观测编排；观测编排尚无 live 验收。仍缺实际断连边界的精确所有权、完整 timeout/stale-generation controller 和 H1/H2 对象/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断观测、客户端断开或原生子集不能充作完整 canonical PASS。 +Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集、单连接历史 GET 断开和服务 admission/owner/socket 观测编排；新增 timeout 诊断控制器将一次真实 PUT 的持久化 ACK、服务端 request ID/精确写锁 timeout、仍被反压的原 GET socket 及物理/S3/FD 收尾关联起来，尚无 live 验收。仍缺实际断连瞬间的精确所有权证明、stale-generation controller、queued successor 锁闭环及 H1/H2 DL/DV 完整对象/ACK/物理/FD 验收。精确源码 cfg(test) 原生证据独立保留；诊断观测、客户端断开或原生子集不能充作完整 canonical PASS。 1. LOCK_WINDOW用本例慢IO/受控原生锁屏障持有目标对象锁，先证明真实争用超过预注册预算。 2. 同时heal/覆盖或删除；调用方断开实例仅断本次TCP连接，不杀服务器或关闭跳板机；记录请求开始/断开/服务在途结束。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 09b00d2e..314e6da2 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -725,6 +725,14 @@ Before disconnect, two `/proc` samples must connect the exact client/server TCP After the caller closes, the observer follows the same request/attempt/owner until its `guard_dropped` event, requires two complete samples without the original owned socket, and waits up to thirty seconds for each peer's FD count to return to its baseline. It then reads and verifies the exact historical version's full content and samples resources again. Every transport result remains UNKNOWN. Diagnostic guard drop does not prove distributed release, snapshots do not prove ownership across the disconnect instant, and aggregate FD counts do not prove absence of every leak. Those claims stay explicitly false. The timeout/stale-generation native fences, queued-successor lock closure, complete DL/DV physical and ACK verification, every H1/H2 branch and independent live acceptance remain required; I05 stays outside canonical dispatch. +### I05 object-lock timeout diagnostic controller + +`lock_timeout.py` extends the caller observer with one real contender PUT while the exact historical GET still owns its read lock. Run it only against the already assigned exclusive `lock-timeout` H1/H2 service and the current auto-testing nodes: `python3 -B scripts/validation_heal/lock_timeout.py --binding --nodes vm000 [vm001 vm002 vm003] --bucket --entry --output `. The original historical payload must be at least 32 MiB so the controller can apply and record a bounded 64 KiB client receive buffer, then observe the same established service-owned TCP socket with a non-empty send queue after the contender response. + +The contender body, exact PUT request, endpoint, structured 503/`ServiceUnavailable`/`Retry-After: 1` response and durable ACK are bound together, including the recorded operation, parameters, payload digest and ordered timestamps in the collector's clock domain. The pinned candidate's `rustfs/src/error.rs` maps typed `StorageError::Lock(LockError::Timeout)` to this response and directly tests its header. Cursor-complete server logs must associate that response's distinct server request ID and write-attempt UUID with the same bucket/key namespace and the exact one-second `put_object_commit` timeout. The same-node GET and PUT share the set's owner label; that label is not a per-request identity. The original GET attempt must still be acquired after the timeout and socket samples. Failure, UNKNOWN, a missing journal event, an empty send queue, a small fixture, or binding drift is retained as incomplete or `NOT_OBSERVED`, never retried or passed. The controller checks the original physical scene before any S3 read, compares complete versioned inventory before and after, and waits for process FD counts to return to the original baseline. + +This establishes a live timeout diagnostic subset only: it does not certify ownership at the precise disconnect edge, stale read-repair fencing, queued successor behavior, H1/H2 DL/DV source branches, or canonical acceptance. The native abort/timeout/stale fences and independent live audit remain required. + ### J02 original opaque metadata probe The `unknown-negative` diagnostic includes the source procedure before the supplemental legacy-object diagnostic. A separate acknowledged object first receives an independently healthy Deep receipt. With the owned services stopped, the existing descriptor-relative fault backend flips byte zero of every original `xl.meta` copy: four copies in H1 and sixteen in H2. Each mutation has its own original backup, receipt and complete physical delta; the candidate-bound decoder must refuse every exact damaged copy. No other metadata, part or format may change. diff --git a/scripts/validation_heal/caller_abort.py b/scripts/validation_heal/caller_abort.py index d88b7b94..3b68a91e 100644 --- a/scripts/validation_heal/caller_abort.py +++ b/scripts/validation_heal/caller_abort.py @@ -18,7 +18,8 @@ from heal_probe import CLOCK_DOMAIN, Probe, canonical, require, save, validate_binding, validate_s3_request -def disconnect_get(probe, bucket, key, version_id, output, *, before_disconnect=None): +def disconnect_get(probe, bucket, key, version_id, output, *, before_disconnect=None, + receive_buffer_bytes=None): """Record intent, send once, close the caller socket, retain UNKNOWN. The caller must independently establish server admission, lock ownership, @@ -26,6 +27,9 @@ def disconnect_get(probe, bucket, key, version_id, output, *, before_disconnect= A populated output directory prevents accidental replay of this request. """ validate_binding(probe.binding) + require(receive_buffer_bytes is None or type(receive_buffer_bytes) is int + and 4096 <= receive_buffer_bytes <= 1024**2, + 'invalid bounded caller receive buffer') require(isinstance(key, str) and 0 < len(key.encode()) <= 1024, 'bounded exact object key required') require(isinstance(version_id, str) and str(uuid.UUID(version_id)) == version_id and uuid.UUID(version_id).int != 0, 'exact nonnil historical version required') @@ -64,6 +68,7 @@ def disconnect_get(probe, bucket, key, version_id, output, *, before_disconnect= 'endpoint': probe.endpoint_name, 'endpointUrl': probe.endpoint, 'operation': 'get_object', 'params': params, 'callerMarker': marker, 'requestTarget': target, + 'receiveBufferBytesRequested': receive_buffer_bytes, 'clockDomain': CLOCK_DOMAIN, 'preparedMonotonicNs': time.monotonic_ns(), 'retryAllowed': False, 'canonicalCasePassClaimed': False}) evidence = {'schemaVersion': 1, 'runId': probe.binding['runId'], @@ -87,8 +92,17 @@ def connect(self): connection.connect() sock = connection.sock require(sock is not None, 'caller socket unavailable') + receive_buffer_effective = None + if receive_buffer_bytes is not None: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, receive_buffer_bytes) + receive_buffer_effective = sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF) + require(type(receive_buffer_effective) is int and receive_buffer_bytes <= receive_buffer_effective + <= receive_buffer_bytes * 4, + 'caller receive buffer did not retain its bounded setting') evidence.update(socketDescriptor=sock.fileno(), localAddress=list(sock.getsockname()), - peerAddress=list(sock.getpeername())) + peerAddress=list(sock.getpeername()), + receiveBufferBytesRequested=receive_buffer_bytes, + receiveBufferBytesEffective=receive_buffer_effective) validate_binding(probe.binding) require(canonical(probe.binding) == bound, 'caller binding changed before request') evidence.update(requestInvocationStarted=True, callStartedMonotonicNs=time.monotonic_ns()) diff --git a/scripts/validation_heal/caller_socket.py b/scripts/validation_heal/caller_socket.py index d4a05e32..e6d2aee5 100644 --- a/scripts/validation_heal/caller_socket.py +++ b/scripts/validation_heal/caller_socket.py @@ -46,7 +46,11 @@ def connections(raw, ipv6, client, server): 'incomplete Linux TCP connection row') local, remote = proc_address(fields[1], ipv6), proc_address(fields[2], ipv6) if local == server and remote == client: - found.append({'inode': int(fields[9]), 'state': fields[3].upper()}) + queues = fields[4].split(':') + require(len(queues) == 2 and all(re.fullmatch(r'[0-9A-Fa-f]+', value) for value in queues), + 'invalid Linux TCP queue counters') + found.append({'inode': int(fields[9]), 'state': fields[3].upper(), + 'txQueueBytes': int(queues[0], 16), 'rxQueueBytes': int(queues[1], 16)}) require(len(found) <= 1, 'ambiguous exact TCP connection in network namespace') return found diff --git a/scripts/validation_heal/lock_observer.py b/scripts/validation_heal/lock_observer.py index 6821a4f8..cfb6e063 100644 --- a/scripts/validation_heal/lock_observer.py +++ b/scripts/validation_heal/lock_observer.py @@ -12,13 +12,13 @@ try: from .caller_abort import disconnect_get from .fleet_node import PROCESS_FIELDS, runtime_config - from .heal_probe import Probe, canonical, require, save, validate_binding + from .heal_probe import MAX_BODY, Probe, canonical, require, save, validate_binding, validate_s3_request from .healthy import Fleet from .scanner_repair import JournalWindow except ImportError: from caller_abort import disconnect_get from fleet_node import PROCESS_FIELDS, runtime_config - from heal_probe import Probe, canonical, require, save, validate_binding + from heal_probe import MAX_BODY, Probe, canonical, require, save, validate_binding, validate_s3_request from healthy import Fleet from scanner_repair import JournalWindow @@ -93,22 +93,39 @@ def same_process(observed, expected): for name in PROCESS_FIELDS), 'I05 observation crossed a service process birth or runtime change') -def observe_disconnect(probe, fleet, bucket, entry, output): - """Use an already deployed exclusive I05 service; do not alter the fleet.""" +def validate_scope(probe, fleet, bucket, entry): + """Refuse another service, source or target before any remote observation.""" validate_binding(probe.binding) require(fleet.runtime_profile == 'lock-timeout' and fleet.run_id == probe.binding['runId'] and probe.binding['candidateCommit'] == CANDIDATE, 'caller observation requires its reviewed candidate and isolated runtime') endpoints = {f'node{index}': f'http://rustfs-node{index}:9000' for index in range(1, len(fleet.nodes) + 1)} - require(probe.binding['endpoints'] == endpoints and len(set(fleet.nodes)) == len(fleet.nodes) + expected_nodes = ['vm000'] if len(fleet.nodes) == 1 else ['vm000', 'vm001', 'vm002', 'vm003'] + require(probe.binding['endpoints'] == endpoints and fleet.nodes == expected_nodes and ((len(fleet.nodes) == 1 and fleet.run_id.endswith('-h1')) or (len(fleet.nodes) == 4 and fleet.run_id.endswith('-h2'))), 'I05 caller endpoints differ from the H1/H2 fleet') - require(entry.get('deleteMarker') is False and isinstance(entry.get('payload'), dict) + require(probe.endpoint_name in endpoints + and probe.endpoint == endpoints[probe.endpoint_name].rstrip('/'), + 'I05 caller endpoint differs from its binding') + require(isinstance(entry, dict) and entry.get('deleteMarker') is False + and isinstance(entry.get('key'), str) and 0 < len(entry['key'].encode()) <= 1024 + and isinstance(entry.get('versionId'), str) + and str(uuid.UUID(entry['versionId'])) == entry['versionId'] and uuid.UUID(entry['versionId']).int + and isinstance(entry.get('payload'), dict) and set(entry['payload']) == {'size', 'sha256'} and type(entry['payload']['size']) is int - and 0 < entry['payload']['size'] <= probe.binding['maxInventoryBytes'] + and 0 < entry['payload']['size'] <= min(MAX_BODY, probe.binding['maxInventoryBytes']) + and isinstance(entry['payload']['sha256'], str) and re.fullmatch(r'[0-9a-f]{64}', entry['payload']['sha256']), 'caller drain requires an acknowledged historical content manifest') + validate_s3_request(probe.binding, 'get_object', { + 'Bucket': bucket, 'Key': entry.get('key'), 'VersionId': entry['versionId']}) + + +def observe_disconnect(probe, fleet, bucket, entry, output, *, while_admitted=None, + receive_buffer_bytes=None, before_verification=None): + """Use an already deployed exclusive I05 service; do not alter the fleet.""" + validate_scope(probe, fleet, bucket, entry) output = Path(output) output.mkdir(exist_ok=False) bound = canonical(probe.binding) @@ -134,8 +151,9 @@ def metrics(label): return values baseline = metrics('I05-caller-baseline-resources') window = JournalWindow(fleet, 'I05-caller', event_targets={TARGET}, strict_json=True) - node = fleet.nodes[list(endpoints).index(probe.endpoint_name)] + node = fleet.nodes[int(probe.endpoint_name.removeprefix('node')) - 1] admitted = None + admitted_work = None network_namespace = None def capture(sent): validate_binding(probe.binding) @@ -160,7 +178,7 @@ def sockets(sent, label): network_namespace = namespace return value def before_disconnect(sent): - nonlocal admitted + nonlocal admitted, admitted_work deadline = time.monotonic() + ADMISSION_SECONDS while True: observed = capture(sent) @@ -182,12 +200,17 @@ def before_disconnect(sent): require(time.monotonic() <= deadline, 'NOT_OBSERVED: caller admission observation exceeded its budget') admitted = {'node': node, 'attempt': observed, 'socket': connection} save(output / 'admission.json', admitted) + if while_admitted is not None: + admitted_work = while_admitted(sent, admitted, window) return require(time.monotonic() < deadline, 'NOT_OBSERVED: caller lock acquisition window was not observed') time.sleep(.1) try: + transport_options = {'before_disconnect': before_disconnect} + if receive_buffer_bytes is not None: + transport_options['receive_buffer_bytes'] = receive_buffer_bytes sent = disconnect_get(probe, bucket, entry['key'], entry['versionId'], output / 'transport', - before_disconnect=before_disconnect) + **transport_options) deadline = time.monotonic() + DRAIN_SECONDS while True: observed = capture(sent) @@ -203,6 +226,7 @@ def before_disconnect(sent): break require(time.monotonic() < deadline, 'NOT_OBSERVED: caller diagnostic, socket or FD count did not drain') time.sleep(.2) + verification_precondition = before_verification() if before_verification is not None else None read = fleet.record('I05-caller-final-exact-version', probe.s3('get_object', {'Bucket': bucket, 'Key': entry['key'], 'VersionId': entry['versionId']})) require(read.get('acknowledged') is True and read.get('resultUnknown') is False @@ -217,7 +241,10 @@ def before_disconnect(sent): break require(time.monotonic() < deadline, 'NOT_OBSERVED: I05 verification did not return to its FD count baseline') time.sleep(.2) - result = {'observationComplete': True, 'admission': admitted, 'finalAttempt': observed, + result = {'observationComplete': True, 'baselineResources': baseline, + 'admission': admitted, 'finalAttempt': observed, + 'admittedWork': admitted_work, + 'verificationPrecondition': verification_precondition, 'finalSocket': connection, 'finalResources': resources, 'afterVerificationResources': verified_resources, 'exactHistoricalContentPreserved': True, 'callerResultUnknown': True, 'heldAtDisconnectProven': False, 'distributedReleaseProven': False, diff --git a/scripts/validation_heal/lock_timeout.py b/scripts/validation_heal/lock_timeout.py new file mode 100644 index 00000000..8dff960e --- /dev/null +++ b/scripts/validation_heal/lock_timeout.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""I05 real PUT lock timeout behind an observed GET; diagnostic subset only.""" + +import argparse +import hashlib +import json +from pathlib import Path +import re +import time +import uuid + +try: + from . import lock_observer + from .heal_probe import CLOCK_DOMAIN, MAX_BODY, Probe, canonical, compare_inventories, payload, require, save, validate_binding + from .heal_workload import Recorder + from .healthy import Fleet, physical_content +except ImportError: + import lock_observer + from heal_probe import CLOCK_DOMAIN, MAX_BODY, Probe, canonical, compare_inventories, payload, require, save, validate_binding + from heal_workload import Recorder + from healthy import Fleet, physical_content + + +OBSERVATION_SECONDS = 10 +CONTENDER_BYTES = 257 * 1024 + 123 +MIN_BACKPRESSURE_BYTES = 32 * 1024**2 +CALLER_RECEIVE_BUFFER_BYTES = 64 * 1024 + + +def require_exact_inventory_entry(inventory, bucket, entry): + require(isinstance(inventory, dict), 'I05 timeout requires an S3 inventory object') + entries = inventory.get('entries') + pagination = inventory.get('pagination') + require(inventory.get('bucket') == bucket and isinstance(entries, list) and entries + and all(isinstance(row, dict) and isinstance(row.get('key'), str) and row['key'] + and isinstance(row.get('versionId'), str) and row['versionId'] + and type(row.get('deleteMarker')) is bool and type(row.get('isLatest')) is bool + for row in entries), 'I05 timeout inventory has incomplete version identities') + identities = [(row['key'], row['versionId']) for row in entries] + require(len(set(identities)) == len(entries) + and all(sum(row['isLatest'] for row in entries if row['key'] == key) == 1 + for key in {row['key'] for row in entries}), + 'I05 timeout inventory has duplicate versions or incomplete latest generations') + for row in entries: + content = row.get('payload') + require(content is None if row['deleteMarker'] else + isinstance(content, dict) and set(content) == {'size', 'sha256'} + and type(content.get('size')) is int and 0 <= content['size'] <= MAX_BODY + and isinstance(content.get('sha256'), str) and re.fullmatch(r'[0-9a-f]{64}', content['sha256']), + 'I05 timeout inventory lacks complete bounded payload digests') + require(isinstance(pagination, dict) + and all(isinstance(pagination.get(name), list) and pagination[name] + and all(type(size) is int and 0 <= size <= 1000 for size in pagination[name]) + for name in ('versionPageSizes', 'currentPageSizes')), + 'I05 timeout inventory lacks complete typed pagination') + require(type(inventory.get('objectsAndMarkers')) is int + and inventory['objectsAndMarkers'] == len(entries) + and type(inventory.get('bytesRead')) is int + and inventory['bytesRead'] == sum(row.get('payload', {}).get('size', 0) + for row in entries if not row.get('deleteMarker')) + and sum(pagination['versionPageSizes']) == len(entries) + and sum(pagination['currentPageSizes']) == sum( + bool(row.get('isLatest') and not row.get('deleteMarker')) for row in entries), + 'I05 timeout requires a structurally complete cross-checked S3 inventory') + matches = [row for row in entries if row.get('key') == entry.get('key') + and row.get('versionId') == entry.get('versionId')] + require(len(matches) == 1 and matches[0].get('deleteMarker') is False + and matches[0].get('payload') == entry.get('payload'), + 'I05 timeout target is not the exact content verified in its complete S3 inventory') + return matches[0] + + +def validate_timeout_ack(ack, request, request_bytes, run_id, endpoint_name, bucket, key, body): + expected_params = {'Bucket': bucket, 'Key': key, + 'Metadata': {'heal-fixture': 'I05-timeout-contender'}} + expected_payload = {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()} + require(ack.get('schemaVersion') == 2 and ack.get('runId') == run_id + and ack.get('sequence') == 1 and ack.get('requestFile') == '000001-request.json' + and ack.get('requestSha256') == hashlib.sha256(request_bytes).hexdigest() + and ack.get('operationCheck') == 'PASS' + and ack.get('casePassClaimed') is False + and request.get('schemaVersion') == 2 and request.get('runId') == run_id + and request.get('sequence') == 1 and request.get('operation') == 'put_object' + and request.get('params') == expected_params and request.get('identity') is None + and request.get('expectHttp') == 503 + and request.get('casePassClaimed') is False + and request.get('payload') == expected_payload + and ack.get('clockDomain') == request.get('clockDomain') == CLOCK_DOMAIN, + 'I05 timeout ACK is not durably bound to the exact contender PUT') + observation = ack.get('observation') + require(isinstance(observation, dict) and observation.get('endpoint') == endpoint_name + and observation.get('operation') == 'put_object' and observation.get('params') == expected_params + and observation.get('payload') == expected_payload and observation.get('clockDomain') == CLOCK_DOMAIN + and observation.get('clientCallInvoked') is True + and observation.get('acknowledged') is False and observation.get('resultUnknown') is False + and observation.get('clientCleanupError') is None + and type(observation.get('callStartedMonotonicNs')) is int + and type(observation.get('callFinishedMonotonicNs')) is int + and all(type(value) is int and value > 0 for value in ( + request.get('preparedMonotonicNs'), ack.get('collectorCallStartedMonotonicNs'), + ack.get('collectorCallFinishedMonotonicNs'), ack.get('recordedMonotonicNs'))) + and request['preparedMonotonicNs'] <= ack['collectorCallStartedMonotonicNs'] + <= observation['callStartedMonotonicNs'] <= observation['callFinishedMonotonicNs'] + <= ack['collectorCallFinishedMonotonicNs'] <= ack['recordedMonotonicNs'], + 'I05 timeout PUT has an unknown transport result or incomplete durable refusal') + response = observation.get('response') + require(isinstance(response, dict), 'I05 timeout PUT ACK lacks its structured refusal') + return response + + +def timeout_proof(rows, response, holder, bucket, key): + """Reconcile one structured S3 refusal with its actual write-lock attempt.""" + metadata = response.get('ResponseMetadata', {}) + request_id = metadata.get('RequestId') + # Exact candidate error.rs attaches this header to typed namespace timeouts. + require(metadata.get('HTTPStatusCode') == 503 and type(metadata.get('RetryAttempts')) is int + and metadata['RetryAttempts'] == 0 + and response.get('Error', {}).get('Code') == 'ServiceUnavailable' + and metadata.get('HTTPHeaders', {}).get('retry-after') == '1' + and isinstance(request_id, str) and str(uuid.UUID(request_id)) == request_id and uuid.UUID(request_id).int + and request_id != holder['request_id'] and response.get('VersionId') is None, + 'I05 contender lacks an exact structured lock-timeout refusal') + selected = [row for row in rows if row.get('target') == lock_observer.TARGET and row.get('request_id') == request_id] + if not selected: + return None + require([row.get('state') for row in selected] in (['acquiring'], ['acquiring', 'failed']), + 'I05 contender acquired a lock or lost its exact timeout lifecycle') + for row in selected: + require(row.get('event') == 'object_namespace_lock' and row.get('component') == 'ecstore' + and row.get('subsystem') == 'object_lock' and row.get('op') == 'put_object_commit' + and row.get('mode') == 'write' and row.get('bucket') == bucket and row.get('object') == key + and row.get('namespace') == holder['namespace'] and row.get('requested_version_id') is None + and type(row.get('timeout_ms')) is int and row['timeout_ms'] == 1000 + and type(row.get('acquire_ms')) is int and row['acquire_ms'] >= 0 + # new_ns_lock reuses this SetDisks owner for same-node requests; + # the server request and attempt UUIDs distinguish contenders. + and row.get('owner') == holder['owner'] + and isinstance(row.get('lock_attempt_id'), str) + and str(uuid.UUID(row['lock_attempt_id'])) == row['lock_attempt_id'] + and uuid.UUID(row['lock_attempt_id']).int and row['lock_attempt_id'] != holder['lock_attempt_id'], + 'I05 contender has another lock budget, object, namespace or owner') + require(len({(row['owner'], row['lock_attempt_id']) for row in selected}) == 1, + 'I05 contender changed its original write owner') + if len(selected) == 1: + return None + start, end = selected + require(start.get('level') == 'TRACE' and end.get('level') == 'ERROR' and end.get('failure') == 'timeout' + and type(end.get('acquire_ms')) is int and end['acquire_ms'] >= end['timeout_ms'], + 'I05 contender did not observe actual contention beyond its registered budget') + return {'requestId': request_id, 'lockAttemptId': end['lock_attempt_id'], 'owner': end['owner'], + 'namespace': end['namespace'], 'events': selected, 'typedTimeoutBeyondBudgetObserved': True, + 'writeLockAcquired': False, 'canonicalCasePassClaimed': False} + + +def observe_timeout(probe, fleet, bucket, entry, output): + """Use existing owned service and fixtures; preserve every failed request.""" + lock_observer.validate_scope(probe, fleet, bucket, entry) + output = Path(output) + output.mkdir(exist_ok=False) + bound = canonical(probe.binding) + save(output / 'scope.json', {'runId': fleet.run_id, 'entry': entry, + 'candidateCommit': probe.binding['candidateCommit'], + 'bindingSha256': hashlib.sha256(bound.encode()).hexdigest(), + 'minimumCallerPayloadBytes': MIN_BACKPRESSURE_BYTES, + 'receiveBufferBytesRequested': CALLER_RECEIVE_BUFFER_BYTES, + 'canonicalCasePassClaimed': False}) + try: + return _observe_timeout(probe, fleet, bucket, entry, output, bound) + except BaseException as error: + save(output / 'failure.json', {'diagnosticComplete': False, 'errorType': type(error).__name__, + 'state': 'NOT_OBSERVED' if str(error).startswith('NOT_OBSERVED:') else 'INCOMPLETE', + 'canonicalCasePassClaimed': False, 'originalRequestsAndResponsesRetained': True}) + raise + + +def _observe_timeout(probe, fleet, bucket, entry, output, bound): + def binding(): + validate_binding(probe.binding) + require(canonical(probe.binding) == bound, 'I05 timeout binding changed during observation') + def unchanged_physical(label): + binding() + after = fleet.snapshot(label) + require(physical_content(after) == physical_content(physical), + 'I05 rejected timeout PUT changed an original physical file or object generation') + return {'originalFilesUnchanged': True, 'canonicalCasePassClaimed': False} + physical = fleet.snapshot('I05-timeout-original-physical') + before = fleet.record('I05-timeout-original-S3', probe.inventory(bucket)) + target = require_exact_inventory_entry(before, bucket, entry) + require(target['payload']['size'] >= MIN_BACKPRESSURE_BYTES, + 'NOT_OBSERVED: I05 timeout target is too small to maintain response backpressure') + unchanged_physical('I05-timeout-baseline-physical-after-public-read') + recorder = Recorder(probe, output / 'contender') + body = payload(f'{probe.binding["runId"]}/I05-timeout-contender', CONTENDER_BYTES) + def contend(sent, admitted, window): + binding() + node, holder = admitted['node'], admitted['attempt'] + recorder.call('put_object', {'Bucket': bucket, 'Key': entry['key'], + 'Metadata': {'heal-fixture': 'I05-timeout-contender'}}, body, expect_http=503) + ack_path = recorder.output / '000001-ack.json' + request_path = recorder.output / '000001-request.json' + request_bytes = request_path.read_bytes() + response = validate_timeout_ack(json.loads(ack_path.read_text()), + json.loads(request_bytes), request_bytes, probe.binding['runId'], probe.endpoint_name, + bucket, entry['key'], body) + deadline = time.monotonic() + OBSERVATION_SECONDS + while True: + binding() + all_rows = window.capture() + rows = all_rows[node] + current = lock_observer.caller_attempt(rows, sent, bucket, entry['key']) + require(current is not None and current['observedAcquiredWithoutDrop'] + and all(current[name] == holder[name] for name in ('request_id', 'lock_attempt_id', 'owner', 'namespace')), + 'NOT_OBSERVED: I05 original GET owner ended before contention was observed') + foreign_request = [row for host, host_rows in all_rows.items() if host != node + for row in host_rows if row.get('request_id') == response.get( + 'ResponseMetadata', {}).get('RequestId')] + require(not foreign_request, 'I05 contender request ID appeared on a foreign endpoint') + proof = timeout_proof(rows, response, holder, bucket, entry['key']) + require(time.monotonic() <= deadline, 'NOT_OBSERVED: I05 contender timeout observation exceeded its budget') + if proof is not None: + socket_state = fleet.record('I05-timeout-caller-backpressure-socket', { + 'node': node, **fleet.remote(node, 'caller-socket', + expectedProcess=admitted['socket']['process'], + clientAddress=sent['localAddress'], serverAddress=sent['peerAddress'])}) + samples = socket_state.get('samples', []) + require(socket_state.get('readOnly') is True + and socket_state.get('process') == admitted['socket']['process'] + and socket_state.get('clockDomain') == admitted['socket']['clockDomain'] + and socket_state.get('networkNamespace') == admitted['socket']['networkNamespace'] + and len(samples) == 2 + and all(len(sample.get('ownedMatches', [])) == 1 + and sample['ownedMatches'][0].get('state') == '01' + and type(sample['ownedMatches'][0].get('txQueueBytes')) is int + and sample['ownedMatches'][0]['txQueueBytes'] > 0 + and sample['vanishedDescriptors'] == 0 for sample in samples), + 'NOT_OBSERVED: timed-out PUT did not overlap a backpressured original GET socket') + current = lock_observer.caller_attempt(window.capture()[node], sent, bucket, entry['key']) + require(current is not None and current['observedAcquiredWithoutDrop'] + and all(current[name] == holder[name] for name in ( + 'request_id', 'lock_attempt_id', 'owner', 'namespace')), + 'NOT_OBSERVED: original GET ended while sampling post-timeout backpressure') + require(time.monotonic() <= deadline, + 'NOT_OBSERVED: I05 backpressure census exceeded its observation budget') + proof['backpressureSocket'] = socket_state + proof['readResponseBackpressureObservedAfterTimeout'] = True + save(output / 'timeout.json', proof) + return proof + time.sleep(.1) + caller = lock_observer.observe_disconnect(probe, fleet, bucket, entry, output / 'caller', + while_admitted=contend, receive_buffer_bytes=CALLER_RECEIVE_BUFFER_BYTES, + before_verification=lambda: unchanged_physical('I05-timeout-final-physical-before-public-read')) + binding() + public = fleet.record('I05-timeout-final-S3', probe.inventory(bucket)) + require_exact_inventory_entry(public, bucket, entry) + agreement = compare_inventories(before, public) + unchanged_physical('I05-timeout-final-physical-after-public-read') + deadline = time.monotonic() + lock_observer.DRAIN_SECONDS + while True: + binding() + final_resources = {} + for node in fleet.nodes: + value = fleet.record('I05-timeout-final-resources', { + 'node': node, **fleet.remote(node, 'process-metrics')}) + lock_observer.same_process(value.get('identity'), caller['finalResources'][node]['identity']) + require(type(value.get('fileDescriptors')) is int and value['fileDescriptors'] >= 0, + 'I05 final timeout descriptor census is incomplete') + final_resources[node] = value + require(time.monotonic() <= deadline, 'NOT_OBSERVED: I05 final timeout FD census exceeded its budget') + if all(value['fileDescriptors'] <= caller['baselineResources'][node]['fileDescriptors'] + for node, value in final_resources.items()): + break + time.sleep(.2) + result = {'diagnosticComplete': True, 'callerObservation': caller, 'inventoryAgreement': agreement, + 'finalResources': final_resources, 'rejectedWriteChangedOriginalFiles': False, + 'timeoutContenderAcknowledged': False, 'canonicalCasePassClaimed': False, 'wholeCasePassClaimed': False, + 'remaining': ['same-source native exact cancellation/timeout/stale fences', + 'complete source DL/DV workload and H1/H2 final ACK/physical/lock/FD closure']} + save(output / 'diagnostic.json', result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--binding', type=Path, required=True) + parser.add_argument('--nodes', nargs='+', required=True) + parser.add_argument('--bucket', required=True) + parser.add_argument('--entry', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--endpoint') + args = parser.parse_args() + probe = Probe(json.loads(args.binding.read_text()), args.endpoint) + args.output.mkdir(parents=True, exist_ok=False) + fleet = Fleet(args.nodes, args.output, probe.binding['runId'], runtime_profile='lock-timeout') + fleet.secret_values.update(probe.credentials()) + observe_timeout(probe, fleet, args.bucket, json.loads(args.entry.read_text()), args.output / 'timeout') + + +if __name__ == '__main__': + main() diff --git a/test_validation_heal_caller_abort.py b/test_validation_heal_caller_abort.py index 304e9cde..aa871f0d 100644 --- a/test_validation_heal_caller_abort.py +++ b/test_validation_heal_caller_abort.py @@ -60,7 +60,8 @@ def tearDown(self): def invoke(self, output=None, **changes): return caller_abort.disconnect_get(self.probe, changes.get('bucket', self.binding['bucketPrefix']), changes.get('key', '中文/literal%2F space.bin'), changes.get('version', VERSION), - output or self.root / 'call', before_disconnect=changes.get('before_disconnect')) + output or self.root / 'call', before_disconnect=changes.get('before_disconnect'), + receive_buffer_bytes=changes.get('receive_buffer_bytes')) def test_one_signed_exact_version_get_closes_real_socket_and_retains_unknown(self): proof = self.invoke() @@ -84,6 +85,8 @@ def test_one_signed_exact_version_get_closes_real_socket_and_retains_unknown(sel expected = auth.signature(auth.string_to_sign(checked, auth.canonical_request(checked)), checked) self.assertEqual(headers['Authorization'].split('Signature=')[1], expected) self.assertEqual(proof['connectionAttempts'], 1) + self.assertIsNone(proof['receiveBufferBytesRequested']) + self.assertIsNone(proof['receiveBufferBytesEffective']) self.assertTrue(proof['requestWriteCompleted'] and proof['clientSocketClosed'] and proof['shutdownCompleted']) self.assertTrue(proof['resultUnknown']) for field in ('responseObserved', 'acknowledged', 'retryAllowed', 'serverOwnershipProven', 'canonicalCasePassClaimed'): @@ -111,6 +114,21 @@ def observer(sent): self.assertFalse(proof['serverOwnershipProven']) self.assertTrue(self.server.disconnected.wait(3)) + def test_small_bounded_receive_buffer_is_persisted_and_applied_to_real_socket(self): + proof = self.invoke(receive_buffer_bytes=4096) + intent = json.loads((self.root / 'call/request.json').read_text()) + self.assertEqual(intent['receiveBufferBytesRequested'], 4096) + self.assertEqual(proof['receiveBufferBytesRequested'], 4096) + self.assertGreaterEqual(proof['receiveBufferBytesEffective'], 4096) + self.assertLessEqual(proof['receiveBufferBytesEffective'], 4096 * 4) + self.assertTrue(self.server.disconnected.wait(3)) + + def test_invalid_receive_buffer_is_rejected_before_output_or_network(self): + with self.assertRaises(ValueError): + self.invoke(receive_buffer_bytes=1) + self.assertFalse((self.root / 'call').exists()) + self.assertEqual(self.server.requests, []) + def test_observer_failure_closes_only_the_caller_and_retains_unknown(self): def observer(sent): raise ValueError('no service ownership window') diff --git a/test_validation_heal_lock_observer.py b/test_validation_heal_lock_observer.py index 961187ac..cafc1818 100644 --- a/test_validation_heal_lock_observer.py +++ b/test_validation_heal_lock_observer.py @@ -7,12 +7,14 @@ import socket import sys import tempfile +import time from types import SimpleNamespace import unittest from unittest.mock import patch from scripts.validation_heal import caller_socket as sockets from scripts.validation_heal import fleet_node, lock_observer as observer +from scripts.validation_heal import lock_timeout import test_validation_heal_caller_abort as wire_fixture VERSION = wire_fixture.VERSION @@ -101,6 +103,113 @@ def test_cancelled_and_failed_attempts_never_count_as_acquired_or_dropped(self): proof = observer.caller_attempt([event('acquiring'), end], SENT, BUCKET, 'target.bin') self.assertFalse(proof['diagnosticGuardDropped'] or proof['observedAcquiredWithoutDrop']) + def test_timeout_proof_requires_server_request_id_and_exact_write_lock_timeout(self): + holder = {'request_id': REQUEST, 'lock_attempt_id': ATTEMPT, 'owner': 'owner-one', 'namespace': 'namespace-one'} + contender = '30000000-0000-4000-8000-000000000001' + response = {'ResponseMetadata': {'HTTPStatusCode': 503, 'RetryAttempts': 0, 'RequestId': contender, + 'HTTPHeaders': {'retry-after': '1'}}, + 'Error': {'Code': 'ServiceUnavailable'}} + rows = [ + {'target': observer.TARGET, 'request_id': contender, 'event': 'object_namespace_lock', + 'component': 'ecstore', 'subsystem': 'object_lock', 'state': 'acquiring', 'level': 'TRACE', + 'op': 'put_object_commit', 'mode': 'write', 'bucket': BUCKET, 'object': 'target.bin', + 'namespace': 'namespace-one', 'owner': 'owner-one', 'lock_attempt_id': '40000000-0000-4000-8000-000000000001', + 'requested_version_id': None, 'timeout_ms': 1000, 'acquire_ms': 0}, + {'target': observer.TARGET, 'request_id': contender, 'event': 'object_namespace_lock', + 'component': 'ecstore', 'subsystem': 'object_lock', 'state': 'failed', 'level': 'ERROR', + 'op': 'put_object_commit', 'mode': 'write', 'bucket': BUCKET, 'object': 'target.bin', + 'namespace': 'namespace-one', 'owner': 'owner-one', 'lock_attempt_id': '40000000-0000-4000-8000-000000000001', + 'requested_version_id': None, 'timeout_ms': 1000, 'acquire_ms': 1001, 'failure': 'timeout'}, + ] + proof = lock_timeout.timeout_proof(rows, response, holder, BUCKET, 'target.bin') + self.assertTrue(proof['typedTimeoutBeyondBudgetObserved'] and not proof['writeLockAcquired']) + for mutation in ( + (lambda r, q: r[1].update(owner='foreign-owner')), + (lambda r, q: r[1].update(lock_attempt_id=ATTEMPT)), + (lambda r, q: q['ResponseMetadata'].update(RequestId=REQUEST)), + (lambda r, q: r[1].update(acquire_ms=999)), + (lambda r, q: r[1].update(namespace='other')), + (lambda r, q: q['ResponseMetadata']['HTTPHeaders'].clear()), + (lambda r, q: q['ResponseMetadata'].update(RetryAttempts=1)), + (lambda r, q: r[1].update(state='acquired')), + ): + copy_rows = json.loads(json.dumps(rows)); copy_response = json.loads(json.dumps(response)) + mutation(copy_rows, copy_response) + with self.assertRaises(ValueError): + lock_timeout.timeout_proof(copy_rows, copy_response, holder, BUCKET, 'target.bin') + + def test_timeout_inventory_requires_complete_cross_checked_exact_target(self): + entry = {'key': 'target.bin', 'versionId': VERSION, 'deleteMarker': False, 'isLatest': True, + 'payload': {'size': 5, 'sha256': 'a' * 64}} + inventory = {'bucket': BUCKET, 'entries': [entry], 'objectsAndMarkers': 1, + 'bytesRead': 5, 'pagination': {'versionPageSizes': [1], 'currentPageSizes': [1]}} + self.assertEqual(lock_timeout.require_exact_inventory_entry(inventory, BUCKET, entry), entry) + mutations = [ + lambda value: value.pop('pagination'), + lambda value: value.update(objectsAndMarkers=0), + lambda value: value.update(bytesRead=4), + lambda value: value['pagination'].update(currentPageSizes=[0]), + lambda value: value['entries'][0].update(versionId='foreign'), + lambda value: value['entries'][0].update(payload={'size': 5, 'sha256': 'b' * 64}), + lambda value: value['pagination'].update(versionPageSizes=[True]), + lambda value: value['pagination'].update(currentPageSizes=[-1, 2]), + lambda value: value['entries'][0].update(isLatest=1), + lambda value: value['entries'].append(dict(value['entries'][0])), + lambda value: value['entries'][0].update(payload={'size': True, 'sha256': 'a' * 64}), + ] + for mutate in mutations: + broken = json.loads(json.dumps(inventory)) + mutate(broken) + with self.subTest(mutate=mutate), self.assertRaises(ValueError): + lock_timeout.require_exact_inventory_entry(broken, BUCKET, entry) + + def test_timeout_ack_requires_exact_durable_put_receipt_and_endpoint(self): + body = b'contender' + request = {'schemaVersion': 2, 'runId': 'gh-12345-1-h1', 'sequence': 1, + 'operation': 'put_object', 'params': {'Bucket': BUCKET, 'Key': 'target.bin', + 'Metadata': {'heal-fixture': 'I05-timeout-contender'}}, 'identity': None, + 'expectHttp': 503, 'casePassClaimed': False, + 'clockDomain': lock_timeout.CLOCK_DOMAIN, 'preparedMonotonicNs': 50, + 'payload': {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}} + request_bytes = json.dumps(request, sort_keys=True).encode() + response = {'ResponseMetadata': {'HTTPStatusCode': 503, 'RetryAttempts': 0, + 'RequestId': '30000000-0000-4000-8000-000000000001', + 'HTTPHeaders': {'retry-after': '1'}}, 'Error': {'Code': 'ServiceUnavailable'}} + ack = {'schemaVersion': 2, 'runId': request['runId'], 'sequence': 1, + 'requestFile': '000001-request.json', + 'requestSha256': hashlib.sha256(request_bytes).hexdigest(), 'operationCheck': 'PASS', + 'casePassClaimed': False, + 'clockDomain': lock_timeout.CLOCK_DOMAIN, 'collectorCallStartedMonotonicNs': 75, + 'collectorCallFinishedMonotonicNs': 225, 'recordedMonotonicNs': 250, + 'observation': {'endpoint': 'node1', 'clientCallInvoked': True, + 'operation': 'put_object', 'params': request['params'], 'payload': request['payload'], + 'clockDomain': lock_timeout.CLOCK_DOMAIN, + 'acknowledged': False, 'resultUnknown': False, 'response': response, + 'callStartedMonotonicNs': 100, 'callFinishedMonotonicNs': 200}} + self.assertEqual(lock_timeout.validate_timeout_ack(ack, request, request_bytes, + request['runId'], 'node1', BUCKET, 'target.bin', body), response) + for mutation in ( + lambda a, r: a.update(requestSha256='0' * 64), + lambda a, r: a['observation'].update(endpoint='node2'), + lambda a, r: a['observation'].update(resultUnknown=True), + lambda a, r: r['params'].update(Key='foreign.bin'), + lambda a, r: r.update(expectHttp=200), + lambda a, r: a['observation'].update(clientCleanupError={'type': 'Error'}), + lambda a, r: a['observation'].update(operation='copy_object'), + lambda a, r: a['observation'].update(payload={'size': 1, 'sha256': '0' * 64}), + lambda a, r: a['observation'].update(clockDomain='foreign'), + lambda a, r: a.update(collectorCallStartedMonotonicNs=150), + lambda a, r: a.update(recordedMonotonicNs=150), + ): + broken_ack = json.loads(json.dumps(ack)); broken_request = json.loads(json.dumps(request)) + mutation(broken_ack, broken_request) + broken_bytes = json.dumps(broken_request, sort_keys=True).encode() + if broken_request != request: + broken_ack['requestSha256'] = hashlib.sha256(broken_bytes).hexdigest() + with self.subTest(mutate=mutation), self.assertRaises(ValueError): + lock_timeout.validate_timeout_ack(broken_ack, broken_request, broken_bytes, + request['runId'], 'node1', BUCKET, 'target.bin', body) + class CallerSocketTests(unittest.TestCase): def request(self): @@ -115,7 +224,7 @@ def test_frozen_linux_tables_include_zero_port_listeners_and_mapped_ipv6(self): ipv4 = HEADER + '\n 0: 0100007F:2328 0100007F:9859 01 0:0 00:0 0 1000 0 678\n' ipv4 += ' 1: 00000000:2328 00000000:0000 0A 0:0 00:0 0 1000 0 123\n' self.assertEqual(sockets.connections(ipv4, False, ('127.0.0.1', 39001), ('127.0.0.1', 9000)), - [{'inode': 678, 'state': '01'}]) + [{'inode': 678, 'state': '01', 'txQueueBytes': 0, 'rxQueueBytes': 0}]) self.assertEqual(sockets.proc_address('0000000000000000FFFF00000100007F:2328', True), ('127.0.0.1', 9000)) self.assertEqual(sockets.proc_address('00000000000000000000000001000000:0000', True), ('::1', 0)) for raw in ('', HEADER + '\n truncated', ipv4 + ipv4.splitlines()[1]): @@ -163,6 +272,37 @@ def execute(*args, **kwargs): self.assertEqual(fleet.remote('vm000', 'caller-socket', **{k: v for k, v in self.request().items() if k not in {'runId', 'action'}}), {}) + @unittest.skipUnless(sys.platform == 'linux', 'requires actual Linux /proc TCP tables') + def test_actual_linux_backpressured_response_has_an_owned_nonempty_send_queue(self): + with socket.socket() as listener, socket.socket() as client: + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(('127.0.0.1', 9000)); listener.listen(1) + client.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 4096) + client.connect(listener.getsockname()) + accepted, _ = listener.accept() + try: + accepted.setblocking(False) + remaining = memoryview(b'x' * (8 * 1024**2)) + deadline = time.monotonic() + 3 + while remaining: + try: + remaining = remaining[accepted.send(remaining):] + except BlockingIOError: + break + self.assertLess(time.monotonic(), deadline) + self.assertTrue(remaining, 'the unread response must hit actual TCP backpressure') + node = self.node() + actual = {**identity(), 'pid': os.getpid()} + node['process'] = lambda req: actual + req = {**self.request(), 'expectedProcess': actual, 'clientAddress': list(client.getsockname())} + proof = sockets.run(req, node) + for sample in proof['samples']: + match = sample['ownedMatches'][0] + self.assertEqual(match['descriptors'], [accepted.fileno()]) + self.assertGreater(match['txQueueBytes'], 0) + finally: + accepted.close() + @unittest.skipUnless(sys.platform == 'linux', 'requires actual Linux /proc TCP tables') def test_actual_linux_socket_is_bound_to_accepted_descriptor_and_disappears_after_close(self): with socket.socket() as listener, socket.socket() as client: @@ -207,6 +347,7 @@ def local_wire(address, *args, **kwargs): self.sent = None self.change = None self.labels = [] + self.contender_rows = [] owner = self class Fleet: nodes = ['vm000']; run_id = owner.probe.binding['runId']; runtime_profile = 'lock-timeout' @@ -218,7 +359,8 @@ def remote(self, node, action, **params): return {'identity': identity(), 'fileDescriptors': 10 if owner.change != 'leak' else 11 if owner.stage == 'after' else 10} if action == 'caller-socket': - matches = [{'state': '01', 'inode': 678, 'descriptors': [4]}] if owner.stage == 'before' else [] + matches = [{'state': '01', 'inode': 678, 'descriptors': [4], + 'txQueueBytes': 4096}] if owner.stage == 'before' else [] return {'process': identity(), 'clockDomain': identity()['bootId'], 'readOnly': True, 'networkNamespace': 'net:[2]' if owner.change == 'namespace' and owner.stage == 'after' else 'net:[1]', 'samples': [{'ownedMatches': matches, 'vanishedDescriptors': 0}, @@ -228,18 +370,21 @@ class Window: def __init__(self, *args, **kwargs): pass def capture(self): rows = [event('acquiring', owner.sent), event('acquired', owner.sent)] - if owner.stage == 'after' or owner.change == 'early-drop': rows.append(event('guard_dropped', owner.sent)) + if owner.stage == 'after' or owner.change == 'early-drop' or ( + owner.change == 'timeout-drop' and owner.contender_rows): + rows.append(event('guard_dropped', owner.sent)) if owner.change == 'owner' and owner.stage == 'after': rows[-1]['owner'] = 'rebound' - return {'vm000': rows} + return {node: rows + owner.contender_rows if node == 'vm000' else [] + for node in owner.fleet.nodes} self.fleet = Fleet() self.window_patch = patch.object(observer, 'JournalWindow', Window); self.window_patch.start() self.addCleanup(self.window_patch.stop) original = observer.disconnect_get - def transport(probe, bucket, key, version, output, *, before_disconnect): + def transport(probe, bucket, key, version, output, *, before_disconnect, **options): def on_sent(sent): owner.sent = sent before_disconnect(sent) - value = original(probe, bucket, key, version, output, before_disconnect=on_sent) + value = original(probe, bucket, key, version, output, before_disconnect=on_sent, **options) self.stage = 'after' return value self.transport_patch = patch.object(observer, 'disconnect_get', side_effect=transport); self.transport_patch.start() diff --git a/test_validation_heal_lock_timeout.py b/test_validation_heal_lock_timeout.py new file mode 100644 index 00000000..202f67c9 --- /dev/null +++ b/test_validation_heal_lock_timeout.py @@ -0,0 +1,194 @@ +"""I05 timeout controller: real caller socket, durable PUT receipts, honest scope.""" + +import copy +import json +import time +import unittest +from unittest.mock import patch + +from scripts.validation_heal import lock_timeout +import test_validation_heal_lock_observer as fixture + + +CONTENDER = '30000000-0000-4000-8000-000000000001' + + +class TimeoutFlowTests(unittest.TestCase): + def setUp(self): + self.scene = fixture.ObserverFlowTests( + 'test_complete_component_keeps_original_unknown_and_all_missing_case_proofs') + self.scene.setUp() + self.addCleanup(self.scene.doCleanups) + self.probe, self.fleet = self.scene.probe, self.scene.fleet + self.entry = self.scene.entry + self.entry['payload']['size'] = lock_timeout.MIN_BACKPRESSURE_BYTES + self.trace, self.change, self.puts = [], None, 0 + self.output = self.scene.root / 'timeout' + self.fleet.snapshot = self.snapshot + self.probe.inventory = self.inventory + self.probe.s3 = self.s3 + original_remote = self.fleet.remote + def remote(node, action, **params): + value = original_remote(node, action, **params) + if self.change == 'no-queue' and action == 'caller-socket' and self.puts: + for sample in value['samples']: + for match in sample['ownedMatches']: match['txQueueBytes'] = 0 + if self.change == 'late-fd-leak' and action == 'process-metrics' and ( + 'I05-timeout-final-physical-after-public-read' in self.trace): + value['fileDescriptors'] += 1 + return value + self.fleet.remote = remote + + def inventory(self, bucket): + self.trace.append('final-inventory' if self.puts else 'baseline-inventory') + entry = {**copy.deepcopy(self.entry), 'isLatest': True} + return {'bucket': bucket, 'entries': [entry], 'objectsAndMarkers': 1, + 'bytesRead': entry['payload']['size'], + 'pagination': {'versionPageSizes': [1], 'currentPageSizes': [1]}} + + def snapshot(self, label): + self.trace.append(label) + changed = self.change == 'physical-write' and self.puts + return {node: {'formats': [], 'files': [{'path': 'original', + 'sha256': 'changed' if changed else 'original'}]} for node in self.fleet.nodes} + + def s3(self, operation, params, body=None): + if operation == 'get_object': + self.trace.append('exact-version-get') + return {'acknowledged': True, 'resultUnknown': False, + 'response': {'VersionId': fixture.VERSION}, 'payload': self.entry['payload']} + self.assertEqual(operation, 'put_object') + self.assertEqual(params['Bucket'], fixture.BUCKET) + self.assertEqual(params['Key'], self.entry['key']) + self.puts += 1 + self.trace.append('put') + self.assertFalse(self.scene.wire.server.disconnected.is_set()) + self.scene.contender_rows = [{ + 'target': fixture.observer.TARGET, 'request_id': CONTENDER, + 'event': 'object_namespace_lock', 'component': 'ecstore', 'subsystem': 'object_lock', + 'op': 'put_object_commit', 'mode': 'write', 'bucket': fixture.BUCKET, + 'object': self.entry['key'], 'namespace': 'namespace-one', 'owner': 'owner-one', + 'lock_attempt_id': '40000000-0000-4000-8000-000000000001', + 'state': state, 'level': 'TRACE' if state == 'acquiring' else 'ERROR', + 'timeout_ms': 1000, 'acquire_ms': 0 if state == 'acquiring' else 1001, + **({} if state == 'acquiring' else {'failure': 'timeout'}), + } for state in ('acquiring', 'failed')] + if self.change == 'no-journal': self.scene.contender_rows = [] + response = {'ResponseMetadata': {'HTTPStatusCode': 503, 'RetryAttempts': 0, 'RequestId': CONTENDER, + 'HTTPHeaders': {'retry-after': '1'}}, 'Error': {'Code': 'ServiceUnavailable'}} + if self.change == 'unexpected-success': + response = {'ResponseMetadata': {'HTTPStatusCode': 200, 'RetryAttempts': 0}, + 'VersionId': '50000000-0000-4000-8000-000000000001'} + return {'endpoint': self.probe.endpoint_name, 'operation': operation, 'params': params, + 'payload': {'size': len(body), 'sha256': lock_timeout.hashlib.sha256(body).hexdigest()}, + 'clockDomain': lock_timeout.CLOCK_DOMAIN, + 'clientCallInvoked': True, 'acknowledged': self.change == 'unexpected-success', + 'resultUnknown': self.change == 'unknown', 'response': response, + 'callStartedMonotonicNs': time.monotonic_ns(), 'callFinishedMonotonicNs': time.monotonic_ns()} + + def invoke(self): + return lock_timeout.observe_timeout(self.probe, self.fleet, fixture.BUCKET, self.entry, self.output) + + def test_complete_timeout_keeps_real_caller_unknown_and_physical_check_precedes_public_get(self): + proof = self.invoke() + self.assertTrue(proof['diagnosticComplete']) + self.assertFalse(proof['wholeCasePassClaimed'] or proof['canonicalCasePassClaimed']) + timeout = proof['callerObservation']['admittedWork'] + self.assertTrue(timeout['typedTimeoutBeyondBudgetObserved']) + self.assertTrue(timeout['readResponseBackpressureObservedAfterTimeout']) + self.assertEqual(self.puts, 1) + self.assertLess(self.trace.index('I05-timeout-original-physical'), + self.trace.index('baseline-inventory')) + self.assertLess(self.trace.index('I05-timeout-final-physical-before-public-read'), + self.trace.index('exact-version-get')) + transport = json.loads((self.output / 'caller/transport/disconnected.json').read_text()) + self.assertTrue(transport['clientSocketClosed'] and transport['resultUnknown']) + self.assertEqual(transport['receiveBufferBytesRequested'], lock_timeout.CALLER_RECEIVE_BUFFER_BYTES) + self.assertTrue((self.output / 'contender/000001-request.json').is_file()) + self.assertTrue((self.output / 'contender/000001-ack.json').is_file()) + + def test_small_target_is_not_observed_without_sending_contender_or_get(self): + self.entry['payload']['size'] = 1 + with self.assertRaisesRegex(ValueError, 'too small'): self.invoke() + self.assertEqual(self.puts, 0) + self.assertEqual(self.scene.wire.server.requests, []) + self.assertEqual(json.loads((self.output / 'failure.json').read_text())['state'], 'NOT_OBSERVED') + + def test_h2_preserves_all_four_peer_resource_checks(self): + self.fleet.nodes = ['vm000', 'vm001', 'vm002', 'vm003'] + self.fleet.run_id = self.probe.binding['runId'] = 'gh-12345-1-h2' + # JSON object order does not define the endpoint-to-host binding. + self.probe.binding['endpoints'] = {f'node{index}': f'http://rustfs-node{index}:9000' + for index in range(4, 0, -1)} + proof = self.invoke() + self.assertEqual(set(proof['finalResources']), set(self.fleet.nodes)) + self.assertEqual(set(proof['callerObservation']['baselineResources']), set(self.fleet.nodes)) + self.assertFalse(proof['wholeCasePassClaimed']) + + def test_unexpected_put_success_retains_ack_and_closes_caller_without_public_read(self): + self.change = 'unexpected-success' + with self.assertRaisesRegex(ValueError, 'unexpected put_object result'): self.invoke() + ack = json.loads((self.output / 'contender/000001-ack.json').read_text()) + self.assertEqual(ack['operationCheck'], 'FAIL') + self.assertTrue(ack['observation']['acknowledged']) + self.assertTrue(self.scene.wire.server.disconnected.wait(3)) + self.assertEqual(self.puts, 1) + self.assertNotIn('exact-version-get', self.trace) + self.assertFalse((self.output / 'diagnostic.json').exists()) + + def test_unknown_put_retains_receipt_closes_caller_and_does_not_retry(self): + self.change = 'unknown' + with self.assertRaisesRegex(ValueError, 'unknown transport'): self.invoke() + ack = json.loads((self.output / 'contender/000001-ack.json').read_text()) + self.assertTrue(ack['observation']['resultUnknown']) + self.assertTrue(self.scene.wire.server.disconnected.wait(3)) + self.assertEqual(self.puts, 1) + self.assertFalse((self.output / 'diagnostic.json').exists()) + + def test_holder_dropped_before_timeout_observation_never_proves_contention(self): + self.scene.change = 'timeout-drop' + with self.assertRaisesRegex(ValueError, 'owner ended'): self.invoke() + self.assertTrue(self.scene.wire.server.disconnected.wait(3)) + self.assertFalse((self.output / 'timeout.json').exists()) + + def test_empty_post_timeout_send_queue_cannot_prove_backpressure(self): + self.change = 'no-queue' + with self.assertRaisesRegex(ValueError, 'backpressured original GET socket'): self.invoke() + self.assertTrue(self.scene.wire.server.disconnected.wait(3)) + self.assertFalse((self.output / 'timeout.json').exists()) + + def test_physical_change_stops_before_any_verification_get(self): + self.change = 'physical-write' + with self.assertRaisesRegex(ValueError, 'changed an original physical file'): self.invoke() + self.assertNotIn('exact-version-get', self.trace) + self.assertNotIn('final-inventory', self.trace) + self.assertFalse((self.output / 'diagnostic.json').exists()) + + def test_missing_timeout_events_exhaust_budget_without_retry(self): + self.change = 'no-journal' + with patch.object(lock_timeout, 'OBSERVATION_SECONDS', 0), \ + self.assertRaisesRegex(ValueError, 'observation exceeded its budget'): + self.invoke() + self.assertEqual(self.puts, 1) + self.assertTrue(self.scene.wire.server.disconnected.wait(3)) + + def test_inventory_cannot_leave_unclosed_fds_after_caller_drain(self): + self.change = 'late-fd-leak' + clock = [0] + def advance(seconds): clock[0] += fixture.observer.DRAIN_SECONDS + 1 + with patch.object(lock_timeout.time, 'monotonic', side_effect=lambda: clock[0]), \ + patch.object(lock_timeout.time, 'sleep', side_effect=advance), \ + self.assertRaisesRegex(ValueError, 'final timeout FD census exceeded its budget'): + self.invoke() + self.assertIn('final-inventory', self.trace) + self.assertFalse((self.output / 'diagnostic.json').exists()) + + def test_unreviewed_source_cannot_even_inventory(self): + self.probe.binding['candidateCommit'] = 'a' * 40 + with self.assertRaisesRegex(ValueError, 'reviewed candidate'): self.invoke() + self.assertEqual(self.trace, []) + self.assertFalse(self.output.exists()) + + +if __name__ == '__main__': + unittest.main() From 704bd606071676dded130dca91d3d5278455bcaa Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 10:27:00 +0800 Subject: [PATCH 281/343] feat(validation): wire manual I05 timeout diagnostics --- .../workflows/validation-catalog-tests.yml | 2 +- docs/validation-heal-operations.md | 32 ++ docs/validation-migration-map.md | 44 ++- scripts/run_validation_case.sh | 2 +- .../validation_heal/lock_timeout_runner.py | 223 ++++++++++++++ scripts/validation_heal/run_case.sh | 17 ++ test_validation_heal_lock_timeout_runner.py | 275 ++++++++++++++++++ 7 files changed, 592 insertions(+), 3 deletions(-) create mode 100644 scripts/validation_heal/lock_timeout_runner.py create mode 100644 test_validation_heal_lock_timeout_runner.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 8ff84a2d..164ae10b 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -159,7 +159,7 @@ jobs: sudo apt-get install --no-install-recommends -y nftables fi - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - - run: python3 -B -m unittest -v test_validation_heal_lock_timeout.py test_validation_heal_lock_observer.py test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py + - run: python3 -B -m unittest -v test_validation_heal_lock_timeout_runner.py test_validation_heal_lock_timeout.py test_validation_heal_lock_observer.py test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 314e6da2..7e63d2ba 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -733,6 +733,38 @@ The contender body, exact PUT request, endpoint, structured 503/`ServiceUnavaila This establishes a live timeout diagnostic subset only: it does not certify ownership at the precise disconnect edge, stale read-repair fencing, queued successor behavior, H1/H2 DL/DV source branches, or canonical acceptance. The native abort/timeout/stale fences and independent live audit remain required. +`lock_timeout_runner.py` provides the enclosing diagnostic deployment through +the existing Functional Chain workflow. After auditing prior cleanup and +ownership, explicitly select `validation_case=heal.lock-timeout-and-caller-abort`, +the exact `auto_testing_sha`, candidate build run/attempt and successful +inspector build run/attempt. The existing shared-fleet concurrency group and +auto-testing bootstrap install the immutable package. Local source, helper, +workflow identity, vm000–vm003 binding and inspector checks run before +bootstrap; Fleet verifies the installed binary digest before starting each +owned service. H1 runs on vm000, followed by H2 on vm000–vm003 only after +complete H1 diagnostic and namespace cleanup. A fresh artifact tree is required; +the runner accepts the enclosing script's setup/test logs but refuses any old +topology or summary before deployment. + +Each topology uses the existing Fleet, Recorder and DV seeder: two keys have +three original versions each and one current marker; the selected historical +version is 32 MiB to maintain bounded TCP backpressure. This small fixture is a +diagnostic subset, not the complete source DL/DV matrix. After timeout and +caller-drain checks, every endpoint must preserve the entire original ACK +manifest, payloads, metadata and tags. Physical comparison brackets those +reads, and every original process's FD count must return to its baseline before +cleanup. Failed/unknown seeding, contention, observation or peer verification +stops the owned services, saves the original physical/log scene and retains +the namespace and original receipts. A failed diagnostic or cleanup prevents +the next topology. SIGTERM follows the same failure capture and retention +path. The summary records diagnostic completion while keeping whole-case and +canonical claims false. The manual entry returns canonical `BLOCKED` after a +successful diagnostic with clean finalization, preserves diagnostic failures, +and reports `INFRA_ERROR` for an incomplete summary or failed finalization +after a zero exit. The workflow gate therefore remains non-green even when +the diagnostic succeeds. I05 remains excluded from the automatic matrix; +this manual route does not certify its native barriers or complete source case. + ### J02 original opaque metadata probe The `unknown-negative` diagnostic includes the source procedure before the supplemental legacy-object diagnostic. A separate acknowledged object first receives an independently healthy Deep receipt. With the owned services stopped, the existing descriptor-relative fault backend flips byte zero of every original `xl.meta` copy: four copies in H1 and sixteen in H2. Each mutation has its own original backup, receipt and complete physical delta; the candidate-bound decoder must refuse every exact damaged copy. No other metadata, part or format may change. diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index bec87573..ae1cfe0c 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -9,6 +9,13 @@ live run proves each source assertion on the required topology. ## Current acceptance and remaining work +Count reconciliation (2026-10-22): the checked-in per-case audit below supports +the 104-case baseline. Later PR summaries state 108, but the four additional +case identities and complete acceptance records have not yet been reconciled +to this baseline. The table therefore describes the documented baseline; this +update adds or revokes no canonical acceptance. Partial I05, native and recovery +diagnostics cannot supply the missing four records. + All 158 canonical IDs are retained. Independent complete live acceptance is **104/158**: the prior 101-case census plus B07 `heal.below-read-quorum` ([run 38727279347/1](https://github.com/rustfs/rustfs/actions/runs/38727279347)), A10 `heal.admin-option-contract` ([run 39197088183/1](https://github.com/rustfs/rustfs/actions/runs/39197088183)) and B06 `heal.read-quorum-boundary` ([run 39223475622/1](https://github.com/rustfs/rustfs/actions/runs/39223475622)). Their complete source matrices, immutable candidate/package/tools and cleanup were independently audited; earlier failures remain retained. | Acceptance state | Cases | Remaining requirement | @@ -219,7 +226,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -Four Heal cases remain outside canonical dispatch: C11, F07, I05 and J02. C11/F07/J02 have explicit diagnostic routes that preserve their blocking canonical verdicts; I05 still needs exact server barriers. The 13 environment-gated cases have disabled-by-default control flows and oracles in `environment_cases.py`; actual execution requires an assigned and reviewed auto-testing resource adapter, and canonical results remain review-gated. See [the environment gate reference](validation-heal-environment-gates.md) for each missing resource, capability and native barrier. +Four Heal cases remain outside canonical dispatch: C11, F07, I05 and J02. All four have explicit diagnostic routes that preserve their blocking canonical verdicts; I05 currently executes only the timeout diagnostic subset and still needs exact server barriers and its other source branches. The 13 environment-gated cases have disabled-by-default control flows and oracles in `environment_cases.py`; actual execution requires an assigned and reviewed auto-testing resource adapter, and canonical results remain review-gated. See [the environment gate reference](validation-heal-environment-gates.md) for each missing resource, capability and native barrier. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. @@ -2267,3 +2274,38 @@ focused checks and the 832-test Heal suite pass (nine platform skips). The original FAIL remains unchanged; later pressure behavior and complete fresh candidate-bound H1/H2 execution remain unproven. Canonical acceptance stays 104/158, and I03 still uses its separate frozen helper. + + +I05 manual timeout deployment (2026-10-22): `lock_timeout_runner.py` now joins +the existing four-argument dispatcher and Functional Chain manual case input. +It validates the exact 757b candidate, immutable helper, inspector and current +vm000-vm003 binding before package bootstrap, then runs independent H1 and H2 +timeout diagnostics with original versioned ACKs, full peer content/metadata/tag +verification, physical comparisons and original-process FD closure. Dirty +H1, unknown writes or failed finalization stop the sequence and retain the +original stopped namespace/log scene. SIGTERM follows that retention path. +The wrapper's pre-existing log directory is accepted; prior topology or summary +files are refused before any new deployment. A clean diagnostic returns +canonical BLOCKED, never PASS; the automatic matrix remains unchanged. + +Final focused verification ran 106 tests (104 passed, two Linux-only skips); +31 existing shell/outcome/nolock compatibility tests also passed. All 14 new +runner tests passed, including the actual CLI, real SIGTERM, retained receipts, +credential redaction and executed shell verdict block. The first focused run +had an earlier receive-buffer prerequisite failure in the empty-send-queue +negative test; the single test and complete focused rerun passed. A separate +300-connection macOS probe observed one kernel buffer increase after connect, +so this local transport limitation remains explicit; the bounded guard is not +relaxed and actual Linux component coverage remains in hosted CI. Catalog +158 and matrix 84/302/534 checks, shell syntax, actionlint and whitespace pass. +These checks do not establish live or canonical I05 acceptance. + +Read-only environment run [39710469108/1](https://github.com/rustfs/rustfs/actions/runs/39710469108) +completed against candidate package 39364893106/1 and helper fa79b52. It +observed sixteen independent eligible endpoints, H1/H2 mounts and no port-9000 +listeners. vm001-vm003 retain failed units from cancelled F07 38715204273/1; +its separately recorded successful recovery is [38726570148/1](https://github.com/rustfs/rustfs/actions/runs/38726570148). +The inspection performs no mutation and certifies neither historical-scene +recovery nor global workflow idleness. Fresh ownership/cleanup and immutable +artifact checks still precede any new fleet diagnostic. H4's twenty-four +endpoints and H6 platform restart authority remain unavailable. diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 980c54f7..87f36741 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -29,7 +29,7 @@ case "${CASE_ID}" in SUITE=rustfs-security-test.sh; EXTRA_ARGS=() ;; oidc.login-and-session) SUITE=scripts/validation_oidc/run_case.sh; EXTRA_ARGS=() ;; - heal.outcome-counters-and-error-truth|heal.checkpoint-and-mrf-corruption|heal.bucket-incarnation-recreation|heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-markers-and-batch-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update|heal.mixed-version-upgrade-rollback|heal.dedicated-host-forced-restart|heal.race-ilm-expiry-and-transition|heal.race-replication-resync|heal.with-expansion-rebalance|heal.with-rebalance-stop-resume|heal.with-decommission|heal.with-decommission-cancel-clear|heal.after-retired-pool-removal|heal.ec84-multi-set-restart|heal.slow-io-and-eio|heal.kms-and-tier-outage|heal.long-soak|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) + heal.lock-timeout-and-caller-abort|heal.outcome-counters-and-error-truth|heal.checkpoint-and-mrf-corruption|heal.bucket-incarnation-recreation|heal.admin-healthy-noop|heal.admin-invalid-request|heal.admin-authorization|heal.admin-nolock-safety|heal.race-unversioned-overwrite|heal.race-versioned-put|heal.race-version-delete|heal.race-markers-and-batch-delete|heal.race-multipart-complete-abort|heal.race-copy-and-metadata-update|heal.mixed-version-upgrade-rollback|heal.dedicated-host-forced-restart|heal.race-ilm-expiry-and-transition|heal.race-replication-resync|heal.with-expansion-rebalance|heal.with-rebalance-stop-resume|heal.with-decommission|heal.with-decommission-cancel-clear|heal.after-retired-pool-removal|heal.ec84-multi-set-restart|heal.slow-io-and-eio|heal.kms-and-tier-outage|heal.long-soak|heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution|heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.node-outage-rejoin|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.status-partial-disabled-unknown|heal.incremental-cursor-and-truncation|heal.admin-stop-and-stale-token|heal.cancelled-task-restart|heal.admin-overlap-and-force-start|heal.root-graceful-coordinator-restart|heal.below-write-quorum|heal.read-quorum-boundary|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.full-cluster-restart|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash|heal.mrf-offline-write-replay|heal.disk-replacement-unformatted|heal.performance-abba|heal.admin-scope-isolation|heal.bucket-metadata-integrity|heal.foreign-format-and-unknown-schema|heal.encrypted-objects|heal.coordinator-process-crash|heal.disk-and-node-flapping|heal.enospc-inodes-and-quota|heal.network-partition-and-lock-quorum|heal.multi-pool-target-isolation|heal.conflicting-subquorum-authority|heal.dangling-and-orphan-safety|heal.admin-option-contract|heal.queue-pressure-and-admission) SUITE=scripts/validation_heal/run_case.sh; EXTRA_ARGS=() ;; *) printf 'validation executor is not migrated: %s\n' "${CASE_ID}" >&2 diff --git a/scripts/validation_heal/lock_timeout_runner.py b/scripts/validation_heal/lock_timeout_runner.py new file mode 100644 index 00000000..ccf1bd20 --- /dev/null +++ b/scripts/validation_heal/lock_timeout_runner.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Deploy and retain the I05 timeout diagnostic; never certify the whole case.""" + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import subprocess +import sys +import time + +try: + from .heal_probe import Probe, compare_inventories, require, save, validate_binding + from .heal_workload import Recorder, seed, verify_seed + from .healthy import Fleet, physical_content, validate_topology + from .inspector import verify as verify_inspector + from .lock_observer import CANDIDATE, DRAIN_SECONDS, same_process + from .lock_timeout import MIN_BACKPRESSURE_BYTES, observe_timeout +except ImportError: + from heal_probe import Probe, compare_inventories, require, save, validate_binding + from heal_workload import Recorder, seed, verify_seed + from healthy import Fleet, physical_content, validate_topology + from inspector import verify as verify_inspector + from lock_observer import CANDIDATE, DRAIN_SECONDS, same_process + from lock_timeout import MIN_BACKPRESSURE_BYTES, observe_timeout + + +NODES = ('vm000', 'vm001', 'vm002', 'vm003') +CASE_ID = 'heal.lock-timeout-and-caller-abort' + + +def interrupted(signum, frame): + raise KeyboardInterrupt(f'I05 diagnostic interrupted by signal {signum}') + + +def preflight(args): + """Validate immutable local inputs before reserving or starting a node.""" + candidate = json.loads(args.candidate.read_text()) + require(candidate.get('source_sha') == CANDIDATE, + 'I05 timeout diagnostic requires the exact reviewed candidate') + require(re.fullmatch(r'[0-9a-f]{64}', candidate.get('package_sha256', '')), + 'I05 package digest is missing') + if args.binary_sha is not None: + require(re.fullmatch(r'[0-9a-f]{64}', args.binary_sha), 'I05 binary digest is missing') + require(re.fullmatch(r'[0-9a-f]{40}', args.testing_commit), 'I05 requires an immutable helper commit') + root = Path(__file__).resolve().parents[2] + head = subprocess.run(['git', '-C', str(root), 'rev-parse', 'HEAD'], + capture_output=True, text=True, check=True).stdout.strip() + require(head == args.testing_commit, 'I05 helper differs from the selected checkout') + changes = subprocess.run(['git', '-C', str(root), 'status', '--porcelain', '--untracked-files=normal', + '--', 'scripts/validation_heal'], capture_output=True, text=True, check=True) + require(not changes.stdout.strip(), 'I05 diagnostic source differs from its immutable helper commit') + for name in ('GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT'): + value = os.environ.get(name, '1' if name == 'GITHUB_RUN_ATTEMPT' else '') + require(re.fullmatch(r'[1-9][0-9]*', value), 'I05 requires the owning workflow identity') + nodes = (os.environ.get('RUSTFS_NODES') or ' '.join(NODES)).split() + require(nodes[:4] == list(NODES), 'I05 requires the current vm000-vm003 Heal node binding') + inspector = verify_inspector(args.tool_directory, candidate) + require(args.rc_bin.is_file(), 'I05 requires the pinned client binary') + return candidate, inspector + + +def run_topology(args, candidate, inspector, topology): + require(topology in ('H1', 'H2'), 'I05 timeout supports only H1/H2') + # Repeat the source/tool guard for callers which bypass main(). + require(candidate.get('source_sha') == inspector.get('candidateCommit') == CANDIDATE, + 'I05 timeout candidate and inspector differ') + directory = args.evidence / topology + directory.mkdir(parents=True, exist_ok=False) + run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + nodes = list(NODES[:1] if topology == 'H1' else NODES) + bucket = 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile='lock-timeout') + access, secret = secrets.token_hex(16), secrets.token_hex(24) + fleet.secret_values.update((access, secret)) + def error_info(error): + message = str(error) + for value in (access, secret): + message = message.replace(value, '[REDACTED]') + return {'type': type(error).__name__, 'message': message} + binding = {'schemaVersion': 1, 'runId': run_id, 'jumpHostOnly': True, 'exclusiveHealService': True, + 'leaseExpiresAt': (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat(), + 'bucketPrefix': bucket, 'endpoints': {f'node{i}': f'http://rustfs-node{i}:9000' + for i in range(1, len(nodes) + 1)}, 'defaultEndpoint': 'node1', + 'candidateCommit': candidate['source_sha'], 'harnessCommit': args.testing_commit, + 'candidatePackageSha256': candidate['package_sha256'], 'candidateBinarySha256': args.binary_sha, + 'rcBinarySha256': hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), + 'maxInventoryObjects': 32, 'maxInventoryBytes': 512 * 1024**2} + validate_binding(binding) + fleet.record('binding', binding) + fleet.record('inspector-build', inspector) + probe = Probe(binding, credential_pair=(access, secret)) + result = {'topology': topology, 'diagnosticComplete': False, 'cleanup': 'PENDING', + 'wholeCasePassClaimed': False, 'canonicalCasePassClaimed': False} + try: + fleet.deploy(args.binary_sha, bucket, access, secret) + fleet.record('actual-topology', validate_topology(fleet.snapshot('deployed-topology'), topology)) + recorder = Recorder(probe, directory / 'fixture') + # A bounded diagnostic fixture, not the full source DL/DV workload. + expected = seed(recorder, bucket, 'DV', 2, [MIN_BACKPRESSURE_BYTES, 1024]) + target = next(row for row in expected if not row['deleteMarker'] and not row['isLatest'] + and row['payload']['size'] >= MIN_BACKPRESSURE_BYTES) + baseline = fleet.record('original-ACK-bound-S3', probe.inventory(bucket)) + fleet.record('original-ACK-proof', verify_seed(expected, baseline)) + diagnostic = observe_timeout(probe, fleet, bucket, target, directory / 'timeout') + require(diagnostic.get('diagnosticComplete') is True + and diagnostic.get('canonicalCasePassClaimed') is False + and diagnostic.get('wholeCasePassClaimed') is False, + 'I05 timeout did not complete its diagnostic-only contract') + physical = fleet.snapshot('all-peer-verification-before-public-reads') + for endpoint in binding['endpoints']: + peer = Probe(binding, endpoint, credential_pair=(access, secret)) + observed = fleet.record('all-peer-S3', {'endpoint': endpoint, **peer.inventory(bucket)}) + fleet.record('all-peer-ACK-proof', {'endpoint': endpoint, + **verify_seed(expected, observed), **compare_inventories(baseline, observed)}) + require(physical_content(physical) == physical_content(fleet.snapshot('physical-after-all-peer-reads')), + 'I05 peer verification changed the diagnostic physical scene') + baseline_resources = diagnostic['callerObservation']['baselineResources'] + require(set(baseline_resources) == set(nodes), 'I05 diagnostic omitted a peer resource baseline') + deadline = time.monotonic() + DRAIN_SECONDS + while True: + validate_binding(binding) + resources = {} + for node in nodes: + resources[node] = fleet.record('after-peer-verification-resources', { + 'node': node, **fleet.remote(node, 'process-metrics')}) + same_process(resources[node].get('identity'), baseline_resources[node]['identity']) + require(type(resources[node].get('fileDescriptors')) is int + and resources[node]['fileDescriptors'] >= 0, 'I05 final peer FD census is incomplete') + require(time.monotonic() <= deadline, 'NOT_OBSERVED: I05 peer verification FD census exceeded its budget') + if all(resources[node]['fileDescriptors'] <= baseline_resources[node]['fileDescriptors'] for node in nodes): + break + time.sleep(.2) + result['diagnosticComplete'] = True + except (Exception, KeyboardInterrupt) as error: + result['failure'] = error_info(error) + finally: + if result['diagnosticComplete']: + try: + result['cleanup'] = 'PASS' if fleet.cleanup() else 'FAIL' + except (Exception, KeyboardInterrupt) as error: + result['cleanup'] = 'FAIL' + result['cleanupError'] = error_info(error) + else: + # A successful contender or uncertain seed must not be erased by + # namespace cleanup. Stop and retain the original files and ACKs. + result.update(cleanup='FAIL', namespacesRetained=True) + try: + if fleet.reserved: + fleet.stop() + fleet.snapshot('original-failure-scene', quiescent=True) + for node in fleet.reserved: + fleet.record('original-failure-logs', {'node': node, **fleet.remote(node, 'logs')}) + except (Exception, KeyboardInterrupt) as error: + result['failureCaptureError'] = error_info(error) + fleet.record('diagnostic-result', result) + return result + + +def run(args): + require(args.topologies in (['H1'], ['H2'], ['H1', 'H2']), 'I05 topology order must be H1 then H2') + require(isinstance(args.binary_sha, str) and re.fullmatch(r'[0-9a-f]{64}', args.binary_sha), + 'I05 deployment requires the installed binary digest') + candidate, inspector = preflight(args) + # run_case.sh creates this parent and opens test.log before invoking us. + # Reuse only those wrapper files; a prior topology or summary is a replay. + require(not args.evidence.is_symlink(), 'I05 evidence directory must not be a symlink') + args.evidence.mkdir(parents=True, exist_ok=True) + require(all(path.name in {'timeout-preflight.json', 'setup.log', 'test.log'} + and path.is_file() and not path.is_symlink() for path in args.evidence.iterdir()), + 'I05 requires fresh topology and summary evidence before deployment') + results = [] + for topology in args.topologies: + result = run_topology(args, candidate, inspector, topology) + results.append(result) + if not result['diagnosticComplete'] or result['cleanup'] != 'PASS': + break + summary = {'diagnostics': results, 'requestedTopologies': args.topologies, + 'diagnosticComplete': len(results) == len(args.topologies) + and all(row['diagnosticComplete'] and row['cleanup'] == 'PASS' for row in results), + 'wholeCasePassClaimed': False, 'canonicalCasePassClaimed': False, + 'remaining': ['exact disconnect-edge ownership and complete native timeout/stale fences', + 'stale-generation and queued-successor controllers', + 'complete H1/H2 source DL/DV ACK, physical, lock and FD acceptance']} + save(args.evidence / 'timeout-diagnostic-summary.json', summary) + save(args.evidence / 'healthy-summary.json', {**summary, 'caseId': CASE_ID, 'complete': False, + 'diagnosticBranchesPassed': summary['diagnosticComplete'], + 'cleanup': 'PASS' if results and all(row['cleanup'] == 'PASS' for row in results) else 'FAIL'}) + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + argv = sys.argv[1:] + preflight_only = argv[:1] == ['preflight'] + if preflight_only: + argv = argv[1:] + parser.add_argument('--candidate', type=Path, required=True) + parser.add_argument('--case-id', choices=(CASE_ID,), default=CASE_ID) + parser.add_argument('--testing-commit', required=True) + parser.add_argument('--binary-sha', required=not preflight_only) + parser.add_argument('--tool-directory', type=Path, required=True) + parser.add_argument('--rc-bin', type=Path, required=True) + parser.add_argument('--evidence', type=Path, required=not preflight_only) + parser.add_argument('--topologies', nargs='+', choices=('H1', 'H2'), default=['H1', 'H2']) + args = parser.parse_args(argv) + require(args.topologies in (['H1'], ['H2'], ['H1', 'H2']), 'I05 topology order must be H1 then H2') + if preflight_only: + preflight(args) + print(json.dumps({'preflightPassed': True, 'candidateCommit': CANDIDATE, + 'diagnosticOnly': True, 'canonicalCasePassClaimed': False})) + return 0 + signal.signal(signal.SIGTERM, interrupted) + return 0 if run(args)['diagnosticComplete'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 09a7a42d..0d5c4539 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -23,6 +23,7 @@ case "${case_id}" in heal.bucket-incarnation-recreation) driver=incarnation.py ;; heal.outcome-counters-and-error-truth) driver=outcome.py ;; heal.checkpoint-and-mrf-corruption) driver=persistence.py ;; + heal.lock-timeout-and-caller-abort) driver=lock_timeout_runner.py ;; heal.node-outage-rejoin) driver=node_outage.py ;; heal.status-partial-disabled-unknown) driver=status.py ;; heal.incremental-cursor-and-truncation) driver=cursor.py ;; @@ -101,6 +102,11 @@ if [ "${driver}" = race.py ] || [ "${driver}" = nolock.py ]; then --candidate "${candidate}" --case-id "${case_id}" --testing-commit "${testing_commit}" \ > "${evidence}/native-preflight.json" fi +if [ "${driver}" = lock_timeout_runner.py ]; then + python3 -B "${ROOT}/scripts/validation_heal/${driver}" preflight \ + --candidate "${candidate}" --case-id "${case_id}" --testing-commit "${testing_commit}" \ + "${tool_args[@]}" --rc-bin "$(command -v rc)" > "${evidence}/timeout-preflight.json" +fi python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ --result-file "${artifacts}/cases/${case_id}.json" --verdict RUNNING --cleanup-status PENDING @@ -219,6 +225,17 @@ if [ "${case_id}" = heal.outcome-counters-and-error-truth ] && [ "${cleanup_stat verdict=BLOCKED; reason="J02 manual diagnostics require independent acceptance of all original source procedures; canonical admission remains disabled" fi fi +if [ "${case_id}" = heal.lock-timeout-and-caller-abort ]; then + # This route is an explicit diagnostic invocation, never canonical admission. + # Even an all-green timeout subset leaves the source's other branches open. + if [ "${cleanup_status}" = PASS ] && [ "${case_rc}" -eq 0 ] && \ + jq -e '.diagnosticComplete == true and .complete == false and .canonicalCasePassClaimed == false' \ + "${evidence}/healthy-summary.json" >/dev/null 2>&1; then + verdict=BLOCKED; reason="I05 timeout diagnostic completed; exact native fences, stale-generation/queued-successor controllers and complete source H1/H2 DL/DV acceptance remain required" + elif [ "${case_rc}" -eq 0 ]; then + verdict=INFRA_ERROR; reason="I05 diagnostic summary or cleanup is incomplete; canonical PASS is prohibited" + fi +fi args=("${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool --result-file "${artifacts}/cases/${case_id}.json" --verdict "${verdict}" --cleanup-status "${cleanup_status}" --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --binary-sha256 "${binary_sha}" diff --git a/test_validation_heal_lock_timeout_runner.py b/test_validation_heal_lock_timeout_runner.py new file mode 100644 index 00000000..c7b464cb --- /dev/null +++ b/test_validation_heal_lock_timeout_runner.py @@ -0,0 +1,275 @@ +"""I05 deployment boundary and retained failure scenes through real Recorder.""" + +import copy +import hashlib +import json +import os +from pathlib import Path +import signal +import subprocess +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch +import uuid + +from scripts.validation_heal import lock_timeout_runner as runner +from test_validation_heal_lock_observer import identity + + +class TimeoutRunnerTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.rc = self.root / 'rc'; self.rc.write_bytes(b'pinned-client') + self.candidate = {'source_sha': runner.CANDIDATE, 'package_sha256': 'a' * 64} + self.inspector = {'candidateCommit': runner.CANDIDATE, 'binarySha256': 'b' * 64} + self.candidate_file = self.root / 'candidate.json' + self.candidate_file.write_text(json.dumps(self.candidate)) + self.args = SimpleNamespace(candidate=self.candidate_file, testing_commit='c' * 40, + binary_sha='d' * 64, tool_directory=self.root / 'tools', rc_bin=self.rc, + evidence=self.root / 'evidence', topologies=['H1', 'H2']) + self.events, self.fleets, self.entries = [], [], [] + self.change = None + owner = self + class Probe: + def __init__(self, binding, endpoint=None, *, credential_pair=None): + self.binding = binding; self.endpoint_name = endpoint or binding['defaultEndpoint'] + self.credential_pair = credential_pair + def s3(self, operation, params, body=None): + owner.events.append(operation) + response = {'ResponseMetadata': {'HTTPStatusCode': 200}} + if operation == 'create_bucket': owner.entries = [] + if operation == 'put_object': + version = str(uuid.uuid4()); response['VersionId'] = version + for row in owner.entries: + if row['key'] == params['Key']: row['isLatest'] = False + owner.entries.append({'key': params['Key'], 'versionId': version, + 'isLatest': True, 'deleteMarker': False, + 'payload': {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}}) + if operation == 'delete_object': + version = str(uuid.uuid4()); response.update(VersionId=version, DeleteMarker=True) + for row in owner.entries: + if row['key'] == params['Key']: row['isLatest'] = False + owner.entries.append({'key': params['Key'], 'versionId': version, + 'isLatest': True, 'deleteMarker': True}) + return {'acknowledged': True, 'resultUnknown': False, 'response': response} + def inventory(self, bucket): + owner.events.append('inventory-' + self.endpoint_name) + if owner.change == 'peer-error' and 'diagnostic' in owner.events: + raise ValueError('incomplete peer inventory') + return {'bucket': bucket, 'entries': copy.deepcopy(owner.entries)} + class Fleet: + def __init__(self, nodes, evidence, run_id, *args, **kwargs): + self.nodes = nodes; self.evidence = evidence; self.run_id = run_id + self.secret_values = set(); self.reserved = []; self.sequence = 0 + owner.fleets.append(self) + def record(self, label, value): + self.sequence += 1 + runner.save(self.evidence / f'{self.sequence:05d}-{label}.json', value) + return value + def deploy(self, *args): + owner.events.append('deploy'); self.reserved.extend(self.nodes) + if owner.change == 'reserve-error': raise ValueError('partial reservation') + def snapshot(self, label, **kwargs): + owner.events.append(label) + return {node: {'files': [], 'formats': []} for node in self.nodes} + def remote(self, node, action): + owner.events.append(action + '-' + node) + if action == 'process-metrics': + return {'identity': identity(), 'fileDescriptors': 11 if owner.change == 'fd-leak' else 10} + return {'journal': 'original log'} + def stop(self): owner.events.append('stop') + def cleanup(self): owner.events.append('cleanup'); return owner.change != 'cleanup-fail' + def diagnostic(probe, fleet, bucket, entry, output): + owner.events.append('diagnostic') + self.assertFalse(entry['isLatest'] or entry['deleteMarker']) + self.assertEqual(entry['payload']['size'], 2048) + if owner.change == 'diagnostic-error': raise ValueError('original timeout not observed') + if owner.change == 'secret-error': raise ValueError(str(probe.credential_pair)) + if owner.change == 'sigterm': signal.raise_signal(signal.SIGTERM) + return {'diagnosticComplete': True, 'wholeCasePassClaimed': False, + 'canonicalCasePassClaimed': False, 'callerObservation': {'baselineResources': { + node: {'identity': identity(), 'fileDescriptors': 10} for node in fleet.nodes}}} + for name, value in [('Fleet', Fleet), ('Probe', Probe), ('observe_timeout', diagnostic), + ('MIN_BACKPRESSURE_BYTES', 2048), ('validate_topology', lambda *a: {'topology': a[1]})]: + p = patch.object(runner, name, value); p.start(); self.addCleanup(p.stop) + p = patch.dict(os.environ, {'GITHUB_RUN_ID': '12345', 'GITHUB_RUN_ATTEMPT': '1', + 'RUSTFS_NODES': ' '.join(runner.NODES)}) + p.start(); self.addCleanup(p.stop) + + def invoke(self, topology='H1'): + return runner.run_topology(self.args, self.candidate, self.inspector, topology) + + def test_real_recorder_seeds_history_then_checks_every_peer_before_cleanup(self): + result = self.invoke('H2') + self.assertTrue(result['diagnosticComplete']) + self.assertEqual(result['cleanup'], 'PASS') + self.assertFalse(result['canonicalCasePassClaimed'] or result['wholeCasePassClaimed']) + self.assertEqual(self.fleets[0].nodes, list(runner.NODES)) + requests = sorted((self.args.evidence / 'H2/fixture').glob('*-request.json')) + acks = sorted((self.args.evidence / 'H2/fixture').glob('*-ack.json')) + self.assertEqual(len(requests), 9) + self.assertEqual(len(acks), 9) + for request, ack in zip(requests, acks): + self.assertEqual(json.loads(ack.read_text())['requestSha256'], hashlib.sha256(request.read_bytes()).hexdigest()) + self.assertEqual(sum(not e['deleteMarker'] for e in self.entries), 6) + self.assertEqual(sum(e['deleteMarker'] for e in self.entries), 1) + for endpoint in ('node1', 'node2', 'node3', 'node4'): + self.assertIn('inventory-' + endpoint, self.events) + self.assertLess(self.events.index('all-peer-verification-before-public-reads'), + self.events.index('inventory-node2')) + self.assertLess(self.events.index('physical-after-all-peer-reads'), self.events.index('cleanup')) + + def test_unreviewed_source_or_mismatched_tool_refuses_before_fleet_creation(self): + self.inspector['candidateCommit'] = 'e' * 40 + with self.assertRaises(ValueError): self.invoke() + self.assertFalse(self.fleets or self.events) + + def test_failure_keeps_original_journal_stopped_scene_and_skips_cleanup(self): + for change in ('diagnostic-error', 'reserve-error', 'peer-error'): + with self.subTest(change=change): + self.args.evidence = self.root / change; self.change = change + self.events.clear() + result = self.invoke() + self.assertFalse(result['diagnosticComplete']) + self.assertEqual(result['cleanup'], 'FAIL') + self.assertTrue(result['namespacesRetained']) + self.assertNotIn('cleanup', self.events) + self.assertLess(self.events.index('stop'), self.events.index('original-failure-scene')) + self.assertIn('logs-vm000', self.events) + if change != 'reserve-error': + self.assertEqual(len(list((self.args.evidence / 'H1/fixture').glob('*-ack.json'))), 9) + + def test_post_peer_fd_leak_retains_scene_and_never_cleans(self): + self.change = 'fd-leak' + clock = [0] + def advance(_): clock[0] += runner.DRAIN_SECONDS + 1 + with patch.object(runner.time, 'monotonic', side_effect=lambda: clock[0]), \ + patch.object(runner.time, 'sleep', side_effect=advance): + result = self.invoke() + self.assertFalse(result['diagnosticComplete']) + self.assertIn('exceeded its budget', result['failure']['message']) + self.assertNotIn('cleanup', self.events) + + def test_serial_driver_stops_before_h2_after_dirty_h1_and_retains_summary(self): + self.change = 'diagnostic-error' + with patch.object(runner, 'preflight', return_value=(self.candidate, self.inspector)): + summary = runner.run(self.args) + self.assertFalse(summary['diagnosticComplete'] or summary['canonicalCasePassClaimed']) + self.assertEqual([row['topology'] for row in summary['diagnostics']], ['H1']) + self.assertFalse((self.args.evidence / 'H2').exists()) + self.assertTrue((self.args.evidence / 'timeout-diagnostic-summary.json').is_file()) + + def test_cleanup_failure_blocks_next_topology_even_after_complete_diagnostic(self): + self.change = 'cleanup-fail' + with patch.object(runner, 'preflight', return_value=(self.candidate, self.inspector)): + result = runner.run(self.args) + self.assertFalse(result['diagnosticComplete']) + self.assertEqual(len(result['diagnostics']), 1) + self.assertEqual(result['diagnostics'][0]['cleanup'], 'FAIL') + + def test_preflight_checks_exact_source_helper_tools_and_workflow_before_remote_use(self): + with patch.object(runner.subprocess, 'run', side_effect=[SimpleNamespace(stdout='c' * 40 + '\n'), + SimpleNamespace(stdout='')]), \ + patch.object(runner, 'verify_inspector', return_value=self.inspector): + self.assertEqual(runner.preflight(self.args), (self.candidate, self.inspector)) + self.candidate_file.write_text(json.dumps({**self.candidate, 'source_sha': 'e' * 40})) + with self.assertRaisesRegex(ValueError, 'exact reviewed'): runner.preflight(self.args) + self.assertFalse(self.fleets or self.events) + + def test_uncommitted_or_untracked_controller_refuses_before_inspector_and_fleet(self): + with patch.object(runner.subprocess, 'run', side_effect=[SimpleNamespace(stdout='c' * 40 + '\n'), + SimpleNamespace(stdout=' M scripts/validation_heal/lock_timeout.py')]), \ + patch.object(runner, 'verify_inspector', side_effect=AssertionError('no inspector read')): + with self.assertRaisesRegex(ValueError, 'immutable helper'): runner.preflight(self.args) + self.assertFalse(self.fleets or self.events) + + def test_in_memory_credentials_cannot_escape_into_failure_or_serial_summary(self): + self.change = 'secret-error' + with patch.object(runner, 'preflight', return_value=(self.candidate, self.inspector)): + result = runner.run(self.args) + self.assertIn('[REDACTED]', result['diagnostics'][0]['failure']['message']) + for value in self.fleets[0].secret_values: + for evidence in self.args.evidence.rglob('*.json'): + self.assertNotIn(value, evidence.read_text()) + + def cli(self): + return ['lock_timeout_runner.py', '--candidate', str(self.candidate_file), '--case-id', runner.CASE_ID, + '--testing-commit', self.args.testing_commit, '--binary-sha', self.args.binary_sha, + '--tool-directory', str(self.args.tool_directory), '--rc-bin', str(self.rc), + '--evidence', str(self.args.evidence)] + + def test_actual_cli_accepts_wrapper_directory_and_keeps_canonical_summary_blocked(self): + self.args.evidence.mkdir() + for name in ('timeout-preflight.json', 'setup.log', 'test.log'): + (self.args.evidence / name).write_text('wrapper evidence') + with patch.object(runner.sys, 'argv', self.cli()), patch.object(runner.signal, 'signal'), \ + patch.object(runner, 'preflight', return_value=(self.candidate, self.inspector)): + self.assertEqual(runner.main(), 0) + summary = json.loads((self.args.evidence / 'healthy-summary.json').read_text()) + self.assertTrue(summary['diagnosticComplete']) + self.assertFalse(summary['complete'] or summary['canonicalCasePassClaimed'] or summary['wholeCasePassClaimed']) + self.assertEqual(summary['cleanup'], 'PASS') + self.assertEqual([row['topology'] for row in summary['diagnostics']], ['H1', 'H2']) + self.assertEqual((self.args.evidence / 'setup.log').read_text(), 'wrapper evidence') + + def test_old_topology_or_summary_refuses_before_any_fleet_deployment(self): + self.args.evidence.mkdir() + for name in ('H1', 'H2', 'healthy-summary.json', 'timeout-diagnostic-summary.json'): + with self.subTest(name=name): + path = self.args.evidence / name; path.write_text('original evidence') + with patch.object(runner, 'preflight', return_value=(self.candidate, self.inspector)): + with self.assertRaisesRegex(ValueError, 'fresh topology'): runner.run(self.args) + self.assertFalse(self.events or self.fleets) + self.assertEqual(path.read_text(), 'original evidence') + path.unlink() + + def test_actual_sigterm_retains_h1_and_stops_before_h2(self): + self.change = 'sigterm' + self.addCleanup(signal.signal, signal.SIGTERM, signal.getsignal(signal.SIGTERM)) + with patch.object(runner.sys, 'argv', self.cli()), \ + patch.object(runner, 'preflight', return_value=(self.candidate, self.inspector)): + self.assertEqual(runner.main(), 1) + summary = json.loads((self.args.evidence / 'healthy-summary.json').read_text()) + self.assertEqual(len(summary['diagnostics']), 1) + self.assertEqual(summary['diagnostics'][0]['failure']['type'], 'KeyboardInterrupt') + self.assertTrue(summary['diagnostics'][0]['namespacesRetained']) + self.assertNotIn('cleanup', self.events) + self.assertIn('original-failure-scene', self.events) + + def test_foreign_node_binding_refuses_before_tools_or_fleet(self): + with patch.dict(os.environ, {'RUSTFS_NODES': 'other000 other001 other002 other003'}), \ + patch.object(runner.subprocess, 'run', side_effect=[SimpleNamespace(stdout='c' * 40), + SimpleNamespace(stdout='')]), patch.object(runner, 'verify_inspector') as inspector: + with self.assertRaisesRegex(ValueError, 'node binding'): runner.preflight(self.args) + inspector.assert_not_called() + self.assertFalse(self.events or self.fleets) + + def test_real_shell_diagnostic_verdict_never_certifies_the_canonical_case(self): + root = Path(__file__).resolve().parent + dry = subprocess.run(['bash', str(root / 'scripts/validation_heal/run_case.sh'), + '--validation-case', runner.CASE_ID, '--dry-run'], capture_output=True, text=True) + self.assertEqual(dry.returncode, 0, dry.stderr) + shell = (root / 'scripts/validation_heal/run_case.sh').read_text() + self.assertLess(shell.index(' preflight', shell.index('if [ "${driver}" = lock_timeout_runner.py')), + shell.index('(bootstrap)')) + verdict = shell[shell.index('verdict=FAIL reason='):shell.index('args=("${ROOT}/scripts/validation_result.py"')] + self.args.evidence.mkdir() + path = self.args.evidence / 'healthy-summary.json' + for code, complete, cleanup, expected in ((0, True, 'PASS', 'BLOCKED'), (1, True, 'PASS', 'FAIL'), + (0, False, 'PASS', 'INFRA_ERROR'), (0, True, 'FAIL', 'INFRA_ERROR'), + (1, False, 'FAIL', 'FAIL'), (0, None, 'PASS', 'INFRA_ERROR')): + if complete is None: path.unlink() + else: path.write_text(json.dumps(dict(diagnosticComplete=complete, complete=False, + canonicalCasePassClaimed=False))) + checked = subprocess.run(['bash', '-euc', verdict + '\nprintf "%s" "$verdict"\n'], + capture_output=True, text=True, env={**os.environ, 'case_id': runner.CASE_ID, + 'cleanup_status': cleanup, 'case_rc': str(code), 'evidence': str(self.args.evidence)}) + self.assertEqual(checked.returncode, 0, checked.stderr) + self.assertEqual(checked.stdout, expected) + + +if __name__ == '__main__': + unittest.main() From 756e3701c72ae3264a27459ba08399eb53173ecc Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 11:02:58 +0800 Subject: [PATCH 282/343] fix(validation): preserve I05 pre-contender failure scenes --- .../workflows/validation-catalog-tests.yml | 6 +- docs/validation-heal-operations.md | 27 ++- docs/validation-migration-map.md | 32 +++ scripts/validation_heal/lock_observer.py | 13 +- scripts/validation_heal/race_retention.py | 5 +- .../validation_heal/recover_environment.py | 6 +- scripts/validation_heal/retained_scene.py | 10 +- scripts/validation_heal/timeout_retention.py | 222 ++++++++++++++++++ test_validation_heal_lock_observer.py | 33 ++- test_validation_heal_timeout_retention.py | 188 +++++++++++++++ .../i05-historical-get-39711742431.ndjson | 3 + .../i05-pre-contender-39711742431.md | 13 + .../i05-pre-contender-39711742431.zip | Bin 0 -> 95717 bytes 13 files changed, 544 insertions(+), 14 deletions(-) create mode 100644 scripts/validation_heal/timeout_retention.py create mode 100644 test_validation_heal_timeout_retention.py create mode 100644 testdata/i05-historical-get-39711742431.ndjson create mode 100644 testdata/validation/i05-pre-contender-39711742431.md create mode 100644 testdata/validation/i05-pre-contender-39711742431.zip diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 164ae10b..7b6f2776 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -55,6 +55,8 @@ on: - test_validation_heal_incarnation.py - test_validation_heal_incarnation_replay.py - scripts/validation_heal/** + - testdata/i05-*.ndjson + - testdata/validation/i05-*.zip - docs/validation-heal-*.md - rustfs_heal_test.sh - test_validation_oidc.py @@ -122,6 +124,8 @@ on: - test_validation_heal_incarnation.py - test_validation_heal_incarnation_replay.py - scripts/validation_heal/** + - testdata/i05-*.ndjson + - testdata/validation/i05-*.zip - docs/validation-heal-*.md - rustfs_heal_test.sh - test_validation_oidc.py @@ -159,7 +163,7 @@ jobs: sudo apt-get install --no-install-recommends -y nftables fi - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - - run: python3 -B -m unittest -v test_validation_heal_lock_timeout_runner.py test_validation_heal_lock_timeout.py test_validation_heal_lock_observer.py test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py + - run: python3 -B -m unittest -v test_validation_heal_timeout_retention.py test_validation_heal_race_retention.py test_validation_heal_retained_scene.py test_validation_heal_lock_timeout_runner.py test_validation_heal_lock_timeout.py test_validation_heal_lock_observer.py test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 7e63d2ba..9bd17c84 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -719,7 +719,7 @@ The component always reports `resultUnknown=true`, `acknowledged=false` and `ser `lock_observer.py` observes an existing run-owned H1/H2 service using the explicit `lock-timeout` runtime profile and reviewed source `757b751482bb970bc321e92059690a7698554358`. The profile enables only the existing object-lock diagnostic target at TRACE, sets the actual object-lock acquisition budget to one second, disables early PUT rename ACKs, and preserves the standard disabled scanner/automatic-Heal/MRF settings. It does not modify the service binary. Run `python3 -B scripts/validation_heal/lock_observer.py --binding --nodes vm000 [vm001 vm002 vm003] --bucket --entry --output `. The entry contains the original `key`, `versionId`, `deleteMarker=false`, and complete ACK-bound `payload.size`/`payload.sha256`. This component neither deploys nor cleans a namespace; the enclosing case owns those steps and recovery. -After persisting and sending the original single GET, the observer waits up to ten seconds for its actual server-generated request ID and `get_object` lock attempt. A unique User-Agent marker connects the exact version URI and method in the `http-request` ancestor span to that lock's owner/namespace/attempt. The marker is correlation data, not an identity asserted by the client. GET's lock diagnostic omits `requested_version_id`; its HTTP span must carry the exact historical version URI instead. Free-text messages, nested native-test capture rows, client-supplied request IDs, repeated lifecycle events and foreign endpoint/owner/version records cannot prove admission. Cursor-bounded raw journald pages remain in evidence, and source message loss prevents completion. +After persisting and sending the original single GET, the observer waits up to ten seconds for its actual server-generated request ID and `get_object` lock attempt. A unique User-Agent marker connects the exact version URI and method in the `http-request` ancestor span to that lock's owner/namespace/attempt. The marker is correlation data, not an identity asserted by the client. Its HTTP span must carry one exact historical version selector; an optional `requested_version_id` diagnostic field must agree with that selector. Free-text messages, nested native-test capture rows, client-supplied request IDs, repeated lifecycle events and foreign endpoint/owner/version records cannot prove admission. Cursor-bounded raw journald pages remain in evidence, and source message loss prevents completion. Before disconnect, two `/proc` samples must connect the exact client/server TCP tuple to an actual established socket descriptor of the original candidate process. A network-namespace table entry alone is insufficient: its inode must also appear in that process's FD links. The read-only adapter accepts no arbitrary PID, path, command or service port. Process birth, binary, environment and mount namespace must match, and its network namespace cannot change during sampling. A second journal capture must still show acquired without a diagnostic drop. An unavailable/already-ended window records incomplete evidence and closes the original caller socket without retry. @@ -770,3 +770,28 @@ this manual route does not certify its native barriers or complete source case. The `unknown-negative` diagnostic includes the source procedure before the supplemental legacy-object diagnostic. A separate acknowledged object first receives an independently healthy Deep receipt. With the owned services stopped, the existing descriptor-relative fault backend flips byte zero of every original `xl.meta` copy: four copies in H1 and sixteen in H2. Each mutation has its own original backup, receipt and complete physical delta; the candidate-bound decoder must refuse every exact damaged copy. No other metadata, part or format may change. A separate protected prefix is checked during the fault. The original key-level task must retain complete truthful counters and an unresolved disposition, with no invented healthy, repaired, absent, version or incarnation claim. Unknown must be counted within skipped and carry the legacy warning. If the candidate reports another truthful unresolved disposition, record `NOT_OBSERVED`; the legacy diagnostic cannot promote it to PASS. Archive the original terminal result and unchanged complete fault scene before restoring original metadata bytes. Verify the full original physical and SDK inventories and the unchanged original token after restoration. Fixture rescue is never product repair. J02 remains outside canonical dispatch until the separate version-specific grace Deferred/new-token procedure and complete live evidence are available. + + +### I05 pre-contender retained-scene isolation + +`validation_cleanup_run_id=39711742431` selects the narrow I05 preservation +route with the original package build 39364893106/1 and an immutable helper +containing `timeout_retention.py`. The driver must remain on the original +Functional Chain branch. First check the shared current auto-testing fleet +for other work. The planner accepts only original helper 704bd60's failed H1 +boundary before contender submission: nine exact successful seed request/ACK +pairs, complete version/marker/payload/public metadata baseline, the single +closed historical GET, original owner/candidate/inspector and unchanged stopped +four-drive scene. Unknown writes, additional actions or H2 evidence refuse it. +It reuses retained-scene audit/isolate/post-audit, retaining all four roots and +state under private per-run parents without S3 calls, backup restoration or +deletion. Verify all original archived files byte-for-byte and all five final +private parents independently before recording active namespace isolation. +`ISOLATED` never changes the original FAIL or supplies canonical acceptance. + +The first live run's GET already dropped its lock before observation. The +version-field correction correlates those original events but cannot reopen +that ended window. The exact candidate source and runtime log differ on the +optional version field; the cause remains unresolved. Any subsequent I05 +controller must establish a real streaming/IO ownership window with evidence; +repeating the same warmed full-read premise does not provide that proof. diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index ae1cfe0c..5031c219 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2309,3 +2309,35 @@ The inspection performs no mutation and certifies neither historical-scene recovery nor global workflow idleness. Fresh ownership/cleanup and immutable artifact checks still precede any new fleet diagnostic. H4's twenty-four endpoints and H6 platform restart authority remain unavailable. + + +I05 first manual live diagnostic [39711742431/1](https://github.com/rustfs/rustfs/actions/runs/39711742431) +used helper `704bd606071676dded130dca91d3d5278455bcaa`, candidate +`757b751482bb970bc321e92059690a7698554358`, package build 39364893106/1 and +inspector build 39566504743/1. H1 retained FAIL/cleanup FAIL after all nine seed +requests received successful bound ACKs and the original full S3 baseline +passed. The single historical GET was written and its client socket closed; +no contender PUT was invoked and H2 did not start. Official artifact +12397696964 has digest `e9ab49593f0423f0649066aa71325d59fadc0e7e6d0328071e2f06f45790325f`. + +The observer incorrectly required an absent diagnostic version field. It now +accepts an absent field or the exact unique HTTP version selector and rejects +any other version. The three original production events are regression +fixtures: they correlate to the exact GET but already show `guard_dropped` +within approximately 0.36 ms, so they cannot prove a held timeout window. The +reviewed source passes no version to this diagnostic while this log carries +one; that difference is not yet explained. Neither a wrong package nor a +source mix is inferred from it. Do not replay the same warmed baseline premise +or claim that the field correction establishes lock contention. + +`timeout_retention.py` admits only this old helper's H1 pre-contender failure +boundary, exact candidate/package/binary/inspector and owner, all nine successful +seed request/ACK pairs, deterministic bytes, public metadata/tags/versions, +closed read-only historical GET and unchanged stopped four-drive physical +scene. Another case, H2, contender, uncertain write or physical drift refuses +recovery. Existing retained-scene isolation preserves four data roots plus the +private state by same-filesystem non-overwriting renames with durable intent, +all-peer preflight, interrupted-move retries and five private archive parents. +It sends no S3 request and never restores backups, deletes the namespace or +rewrites the original FAIL. Live isolation is pending until separately recorded +and audited. Canonical acceptance and its count are unchanged. diff --git a/scripts/validation_heal/lock_observer.py b/scripts/validation_heal/lock_observer.py index cfb6e063..191f9382 100644 --- a/scripts/validation_heal/lock_observer.py +++ b/scripts/validation_heal/lock_observer.py @@ -8,6 +8,7 @@ import re import time import uuid +from urllib.parse import parse_qs, urlsplit try: from .caller_abort import disconnect_get @@ -34,11 +35,17 @@ def caller_attempt(rows, sent, bucket, key): External S3 request IDs are server-owned. The client User-Agent is only a unique correlation marker; a client request-id header cannot stand in for - server admission. GET's lock diagnostic has no requested_version_id; the - exact version must instead match the HTTP span URI. + server admission. The exact historical version is bound by the HTTP URI; + an optional diagnostic version must agree with that same selector. """ require(re.fullmatch(r'auto-testing-i05/[0-9a-f-]{36}', sent.get('callerMarker', '')), 'missing caller correlation marker') + query = parse_qs(urlsplit(sent.get('requestTarget', '')).query, keep_blank_values=True) + require(set(query) == {'versionId'} and len(query['versionId']) == 1, + 'caller requires one exact historical version selector') + version = query['versionId'][0] + require(str(uuid.UUID(version)) == version and uuid.UUID(version).int, + 'caller historical version selector is invalid') selected = [] for row in rows: if row.get('target') != TARGET: @@ -58,7 +65,7 @@ def caller_attempt(rows, sent, bucket, key): require(row.get('event') == 'object_namespace_lock' and row.get('component') == 'ecstore' and row.get('subsystem') == 'object_lock' and row.get('op') == 'get_object' and row.get('mode') == 'read' and row.get('bucket') == bucket and row.get('object') == key - and row.get('requested_version_id') is None and row.get('timeout_ms') == 1000, + and row.get('requested_version_id') in (None, version) and row.get('timeout_ms') == 1000, 'caller lock diagnostic escaped its exact GET identity or runtime budget') require(all(isinstance(row.get(name), str) and 0 < len(row[name]) <= 2048 for name in ('owner', 'namespace', 'lock_attempt_id')) diff --git a/scripts/validation_heal/race_retention.py b/scripts/validation_heal/race_retention.py index f88698b6..e0769acb 100644 --- a/scripts/validation_heal/race_retention.py +++ b/scripts/validation_heal/race_retention.py @@ -30,7 +30,7 @@ def validate(plan): fields = {'kind', 'runId', 'nodes', 'bucket', 'owners', 'businessSceneSha256', 'candidateCommit', 'harnessCommit', 'binarySha256', 'ackFilesSha256'} require(set(plan) == fields | ({'mpuKeys', 'mpuSceneSha256'} if multipart else set()) - and plan['kind'] in {kind for kind, _ in CASES.values()} + and plan['kind'] in ({kind for kind, _ in CASES.values()} | {'unresolved-i05-timeout'}) and re.fullmatch(r'gh-[1-9][0-9]*-[1-9][0-9]*-h[12]', plan['runId']) and len(plan['nodes']) == len(set(plan['nodes'])) == (1 if plan['runId'].endswith('-h1') else 4) and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in plan['nodes']) @@ -42,7 +42,8 @@ def validate(plan): for node in plan['nodes']: owner = plan['owners'][node] require(owner.get('runId') == plan['runId'] and owner.get('bucket') == plan['bucket'] - and (owner.get('runtimeProfile') == 'manual' or + and (owner.get('runtimeProfile') == 'lock-timeout' and plan['kind'] == 'unresolved-i05-timeout' or + owner.get('runtimeProfile') == 'manual' and plan['kind'] != 'unresolved-i05-timeout' or owner.get('runtimeProfile') == 'frozen-race-legacy' and plan['kind'] in {'unresolved-g04-race', 'unresolved-g05-race'}) and owner.get('binarySha256') == plan['binarySha256'] diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index 31aa21d1..ed497842 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -16,6 +16,7 @@ from .outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace, interrupted_opaque_plan, acknowledged_legacy_plan from .network_retention import archived_plan as network_retention_plan, isolate as isolate_network_scene from .race_retention import archived_plan as race_retention_plan, isolate as isolate_race_scene + from .timeout_retention import archived_plan as timeout_retention_plan except ImportError: from healthy import Fleet, physical_content from heal_probe import require, save @@ -23,6 +24,7 @@ from outcome_recovery import acknowledged_grace_plan, isolate_acknowledged_grace, interrupted_opaque_plan, acknowledged_legacy_plan from network_retention import archived_plan as network_retention_plan, isolate as isolate_network_scene from race_retention import archived_plan as race_retention_plan, isolate as isolate_race_scene + from timeout_retention import archived_plan as timeout_retention_plan def quota_recovery_plan(artifact, failed_run, candidate): @@ -149,6 +151,8 @@ def network_recovery_plan(artifact, failed_run, candidate): def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") + if list(artifact.glob("*/run/heal.lock-timeout-and-caller-abort/cases/*.json")): + return timeout_retention_plan(artifact, failed_run, candidate) if (list(artifact.glob("*/run/heal.race-unversioned-overwrite/cases/*.json")) or list(artifact.glob("*/run/heal.race-versioned-put/cases/*.json")) or list(artifact.glob("*/run/heal.race-markers-and-batch-delete/cases/*.json")) @@ -652,7 +656,7 @@ def main(): if plan.get("buckets"): fleet.snapshot_buckets = plan["buckets"] fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) - if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race", "unresolved-a09-race", "unresolved-g04-race", "unresolved-g05-race", "unresolved-g06-race"}: + if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race", "unresolved-a09-race", "unresolved-g04-race", "unresolved-g05-race", "unresolved-g06-race", "unresolved-i05-timeout"}: complete.append({"failedRunId": args.failed_run_id, **isolate_race_scene(fleet, plan)}) continue if plan.get("kind") == "unresolved-network-load": diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py index 0898f138..bbbb71e6 100644 --- a/scripts/validation_heal/retained_scene.py +++ b/scripts/validation_heal/retained_scene.py @@ -180,7 +180,7 @@ def mounted_paths(): def inspect(request, node): state, roots = node["paths"](request["runId"]) expected = request["expectedOwner"] - per_run = expected.get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy"} + per_run = expected.get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout"} retained_state, retained_roots = destinations(state, roots, per_run_parent=per_run) if per_run: old_state, old_roots = destinations(state, roots) @@ -190,7 +190,7 @@ def inspect(request, node): require(isinstance(expected, dict) and expected.get("runId") == request["runId"] and expected.get("unit") == node["unit"](request["runId"]) and expected.get("hostname") == node["socket"].gethostname() - and expected.get("runtimeProfile") in {"mrf-crash", "root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy"}, "retained scene archived owner differs") + and expected.get("runtimeProfile") in {"mrf-crash", "root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout"}, "retained scene archived owner differs") current = node["mounts"]() require(len(current) == len(expected.get("mounts", [])) == 4 and all(all(actual[key] == old[key] for key in ("target", "source", "fstype", "uuid", "maj:min")) @@ -234,7 +234,7 @@ def inspect(request, node): intent = retained_state.parent / (request["runId"] + ".intent.json") node["checked"](intent, state.parent) network = expected["runtimeProfile"] == "lock-quorum" - if expected["runtimeProfile"] in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy"}: + if expected["runtimeProfile"] in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout"}: original_sha = request.get("expectedOriginalSceneSha256", "") require(re.fullmatch(r"[0-9a-f]{64}", original_sha) is not None, "restored fixture requires its original physical scene digest") @@ -270,7 +270,7 @@ def inspect(request, node): def run(request, node): fields = {"runId", "action", "expectedOwner"} - if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy"}: + if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout"}: fields.add("expectedOriginalSceneSha256") if request.get("expectedOwner", {}).get("runtimeProfile") == "lock-quorum" or ( request.get('expectedOwner', {}).get('runtimeProfile') in {'manual', 'frozen-race-legacy'} @@ -309,7 +309,7 @@ def run(request, node): "originalSceneRestored": False, "canonicalPassClaimed": False, "backupRestorationUsed": False, "credentialContentsRead": False, "inspectionOnly": request["action"] == "retained-scene-audit", "mutated": mutated, "activeNamespaceIsolated": "active" not in locations} - if request["expectedOwner"].get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy"}: + if request["expectedOwner"].get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout"}: result["privateArchiveParents"] = [] for target in (*retained_roots, retained_state): info = target.parent.lstat() if target.parent.exists() else None diff --git a/scripts/validation_heal/timeout_retention.py b/scripts/validation_heal/timeout_retention.py new file mode 100644 index 00000000..b1ffc6ab --- /dev/null +++ b/scripts/validation_heal/timeout_retention.py @@ -0,0 +1,222 @@ +"""Isolate the archived I05 H1 failure before any contender was invoked.""" + +import hashlib +import json +import re +import uuid + +try: + from .heal_probe import canonical, payload, require + from .heal_workload import verify_seed + from .healthy import physical_content + from .lock_observer import CANDIDATE + from .race_retention import digest, validate +except ImportError: + from heal_probe import canonical, payload, require + from heal_workload import verify_seed + from healthy import physical_content + from lock_observer import CANDIDATE + from race_retention import digest, validate + + +CASE = 'heal.lock-timeout-and-caller-abort' +HELPER = '704bd606071676dded130dca91d3d5278455bcaa' +FAILURE = {'type': 'ValueError', + 'message': 'caller lock diagnostic escaped its exact GET identity or runtime budget'} +LABELS = ('binding', 'inspector-build', 'preflight', 'integrity-fleet-attestation', 'reserved', + 'inspector-upload', 'started', 'deployed-topology', 'actual-topology', 'original-ACK-bound-S3', + 'original-ACK-proof', 'I05-timeout-original-physical', 'I05-timeout-original-S3', + 'I05-timeout-baseline-physical-after-public-read', 'I05-caller-baseline-resources', + 'I05-caller-checkpoint', 'I05-caller-window', 'quiescent', 'original-failure-scene', + 'original-failure-logs', 'diagnostic-result') + + +def archived_plan(artifact, failed_run, candidate): + """Refuse uncertain seed effects, later actions, mixed cases and physical drift. + + This is preservation, with no S3 call, backup restoration or verdict change. + The old helper's exact pre-contender boundary is deliberately the only route. + """ + require(type(failed_run) is int and failed_run > 0, 'I05 requires an exact failed run') + paths = list(artifact.glob('*/run/*/cases/*.json')) + require(len(paths) == 1 and paths[0].stem == CASE, 'I05 cannot omit another failed case') + def read(path): + require(path.is_file() and not path.is_symlink() and path.stat().st_size <= 2 * 1024**2, + 'I05 archive member is missing, unsafe or oversized') + return json.loads(path.read_text()) + result = read(paths[0]) + provenance = result.get('provenance') or {} + require(result.get('case_id') == CASE and result.get('verdict') == result.get('cleanup_status') == 'FAIL' + and provenance.get('source_commit') == candidate.get('source_sha') == CANDIDATE + and provenance.get('candidate_sha256') == candidate.get('package_sha256') + and provenance.get('testing_commit') == HELPER, + 'I05 requires its original failed candidate and pre-contender helper') + evidence = paths[0].parents[1] / 'evidence' / CASE + summary = read(evidence / 'healthy-summary.json') + row = {'topology': 'H1', 'diagnosticComplete': False, 'cleanup': 'FAIL', + 'wholeCasePassClaimed': False, 'canonicalCasePassClaimed': False, + 'failure': FAILURE, 'namespacesRetained': True} + require(summary.get('caseId') == CASE and summary.get('cleanup') == 'FAIL' + and summary.get('diagnostics') == [row] and summary.get('complete') is False + and summary.get('diagnosticComplete') is False + and summary.get('requestedTopologies') == ['H1', 'H2'] + and summary.get('canonicalCasePassClaimed') is False + and not (evidence / 'H2').exists(), 'I05 requires the sole retained pre-contender H1') + folder = evidence / 'H1' + numbered = {label: f'{index:05d}-{label}.json' for index, label in enumerate(LABELS, 1)} + require({path.name for path in folder.iterdir()} == set(numbered.values()) | {'fixture', 'timeout'}, + 'I05 archive contains another action or lacks its original boundary') + def one(label): + return read(folder / numbered[label]) + binding, reservation, scene = one('binding'), one('reserved'), one('original-failure-scene') + run_id, bucket = binding.get('runId', ''), binding.get('bucketPrefix', '') + require(re.fullmatch(rf'gh-{failed_run}-[1-9][0-9]*-h1', run_id) + and bucket == 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] + and binding.get('candidateCommit') == CANDIDATE + and binding.get('candidatePackageSha256') == candidate['package_sha256'] + and binding.get('candidateBinarySha256') == provenance.get('binary_sha256') + and binding.get('harnessCommit') == HELPER + and binding.get('endpoints') == {'node1': 'http://rustfs-node1:9000'} + and binding.get('defaultEndpoint') == 'node1' and one('diagnostic-result') == row, + 'I05 original run, package, helper, binary or failure boundary differs') + owner = reservation.get('owner') or {} + require(reservation.get('node') == 'vm000' and reservation.get('reserved') is True + and owner.get('runtimeProfile') == 'lock-timeout' and set(scene) == {'vm000'}, + 'I05 requires the sole original H1 reserved owner') + inspector = one('inspector-build') + require(inspector.get('candidateCommit') == CANDIDATE + and inspector.get('binarySha256') == owner.get('inspectorSha256') + and inspector.get('built') is True, 'I05 original inspector identity differs') + original = scene['vm000'] + require(original.get('quiescent') is True and type(original.get('pid')) is int and original['pid'] == 0 + and original.get('hostname') == owner.get('hostname') and original.get('unit') == owner.get('unit') + and original.get('binarySha256') == owner.get('binarySha256') == binding['candidateBinarySha256'] + and len(original.get('formats', [])) == 4 + and {part.get('disk') for part in original['formats']} == set(range(4)) + and physical_content(scene) == physical_content(one('I05-timeout-original-physical')) + == physical_content(one('I05-timeout-baseline-physical-after-public-read')), + 'I05 stopped four-drive original physical scene changed') + + # Replay only the bounded seed ledger locally. No result is inferred from + # current LISTs, and a missing, unknown or failed mutation refuses recovery. + fixture = folder / 'fixture' + names = {f'{i:06d}-{kind}.json' for i in range(1, 10) for kind in ('request', 'ack')} + require({path.name for path in fixture.iterdir()} == names, 'I05 requires all nine seed request/ACK pairs') + expected, journal, public_properties, seen, domain = [], {}, {}, set(), None + for sequence in range(1, 10): + request_path = fixture / f'{sequence:06d}-request.json' + ack_path = fixture / f'{sequence:06d}-ack.json' + request, ack = read(request_path), read(ack_path) + operation = 'create_bucket' if sequence == 1 else 'put_bucket_versioning' if sequence == 2 else ( + 'delete_object' if sequence == 6 else 'put_object') + params = {'Bucket': bucket} + if sequence == 2: + params['VersioningConfiguration'] = {'Status': 'Enabled'} + elif sequence >= 3: + index, generation = (0, sequence - 3) if sequence <= 6 else (1, sequence - 7) + key = f'objects/{index:04d}.bin' + params['Key'] = key + if operation == 'put_object': + params.update(ContentType='application/octet-stream', Metadata={'heal-fixture': 'DV', + 'nonce': f'{index}-{generation}'}, Tagging='suite=heal&fixture=DV') + domain = domain or request.get('clockDomain') + require(isinstance(domain, str) and bool(domain) and request.get('schemaVersion') == ack.get('schemaVersion') == 2 + and request.get('runId') == ack.get('runId') == run_id + and request.get('sequence') == ack.get('sequence') == sequence + and request.get('operation') == operation and request.get('params') == params + and request.get('expectHttp') is None and request.get('identity') is None + and request.get('clockDomain') == ack.get('clockDomain') == domain + and ack.get('requestFile') == request_path.name + and ack.get('requestSha256') == hashlib.sha256(request_path.read_bytes()).hexdigest() + and ack.get('operationCheck') == 'PASS', 'I05 seed request/ACK binding differs') + observation = ack.get('observation') or {} + response = observation.get('response') or {} + require(observation.get('operation') == operation and observation.get('params') == params + and observation.get('endpoint') == 'node1' and observation.get('clockDomain') == domain + and observation.get('clientCallInvoked') is True and observation.get('acknowledged') is True + and observation.get('resultUnknown') is False and not observation.get('clientCleanupError') + and response.get('ResponseMetadata', {}).get('HTTPStatusCode') == (204 if sequence == 6 else 200) + and response.get('ResponseMetadata', {}).get('RetryAttempts') == 0, + 'I05 seed contains an uncertain or rejected mutation') + if sequence >= 3: + version = response.get('VersionId', '') + require(str(uuid.UUID(version)) == version and uuid.UUID(version).int and version not in seen, + 'I05 seed has an invalid or repeated version') + seen.add(version) + entry = {'key': key, 'versionId': version, 'deleteMarker': sequence == 6, + 'isLatest': sequence in (6, 9)} + if sequence == 6: + require(response.get('DeleteMarker') is True, 'I05 marker DELETE lacks its acknowledged marker') + else: + body = payload(f'{bucket}/{key}/{generation}', 32 * 1024**2 if index == 0 else 1024) + expected_payload = {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()} + require(request.get('payload') == observation.get('payload') == expected_payload, + 'I05 seed payload differs from the deterministic request') + require(isinstance(response.get('ETag'), str) and bool(response['ETag']), + 'I05 seed PUT lacks its acknowledged ETag') + public_properties[(key, version)] = {'ETag': response['ETag'], 'ContentLength': len(body), + 'ContentType': params['ContentType'], 'Metadata': params['Metadata']} + entry['payload'] = expected_payload + expected.append(entry) + for path in (request_path, ack_path): + journal[path.relative_to(folder).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + baseline = one('original-ACK-bound-S3') + require(baseline.get('bucket') == bucket and baseline == one('I05-timeout-original-S3'), + 'I05 original public baseline differs') + verify_seed(expected, baseline) + for entry in baseline['entries']: + if entry['deleteMarker']: + continue + require(all(entry.get('properties', {}).get(field) == value + for field, value in public_properties[(entry['key'], entry['versionId'])].items()) + and entry.get('tags') == [{'Key': 'fixture', 'Value': 'DV'}, {'Key': 'suite', 'Value': 'heal'}], + 'I05 original public metadata, ETag or tags differ from seed request/ACK') + + timeout = folder / 'timeout' + require({p.name for p in timeout.iterdir()} == {'scope.json', 'failure.json', 'caller'} + and {p.name for p in (timeout / 'caller').iterdir()} == {'scope.json', 'failure.json', 'transport'}, + 'I05 contender or later controller action may have been invoked') + scope = read(timeout / 'scope.json') + entry = scope.get('entry') + require(entry in expected and entry.get('key') == 'objects/0000.bin' and not entry['isLatest'] + and entry.get('deleteMarker') is False and scope.get('runId') == run_id + and scope.get('candidateCommit') == CANDIDATE + and scope.get('bindingSha256') == hashlib.sha256(canonical(binding).encode()).hexdigest(), + 'I05 caller target is not its acknowledged historical seed') + transport = timeout / 'caller/transport' + require({p.name for p in transport.iterdir()} == {'request.json', 'request-written.json', 'disconnected.json'}, + 'I05 caller transport contains another request or result') + request, sent, closed = (read(transport / name) for name in ('request.json', 'request-written.json', 'disconnected.json')) + target = f'/{bucket}/{entry["key"]}?versionId={entry["versionId"]}' + require(request.get('operation') == 'get_object' and request.get('params') == { + 'Bucket': bucket, 'Key': entry['key'], 'VersionId': entry['versionId']} + and request.get('endpoint') == 'node1' and request.get('endpointUrl') == binding['endpoints']['node1'] + and all(request.get(field) == binding[field] for field in ( + 'candidateCommit', 'harnessCommit', 'candidatePackageSha256', 'candidateBinarySha256')) + and request.get('bindingSha256') == scope['bindingSha256'] + and closed.get('clientSocketClosed') is True and closed.get('connectionAttempts') == 1 + and type(closed.get('clientClosedMonotonicNs')) is int + and closed['clientClosedMonotonicNs'] >= sent.get('requestWrittenMonotonicNs', 0), + 'I05 transport is not the sole closed read-only historical GET') + for value in (request, sent, closed): + require(value.get('runId') == run_id and value.get('requestTarget') == target + and value.get('callerMarker') == request.get('callerMarker') + and re.fullmatch(r'auto-testing-i05/[0-9a-f-]{36}', value.get('callerMarker', '')) + and value.get('clockDomain') == domain and value.get('retryAllowed') is False + and value.get('canonicalCasePassClaimed') is False, + 'I05 closed caller lost its exact request identity') + for value in (sent, closed): + require(value.get('requestSha256') == hashlib.sha256((transport / 'request.json').read_bytes()).hexdigest() + and value.get('requestWriteCompleted') is True + and value.get('resultUnknown') is True and value.get('acknowledged') is False + and value.get('serverOwnershipProven') is False, + 'I05 closed caller was retried or promoted to an acknowledged result') + for path in timeout.rglob('*.json'): + read(path) + journal[path.relative_to(folder).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + plan = {'kind': 'unresolved-i05-timeout', 'runId': run_id, 'nodes': ['vm000'], 'bucket': bucket, + 'owners': {'vm000': owner}, 'candidateCommit': CANDIDATE, 'harnessCommit': HELPER, + 'binarySha256': binding['candidateBinarySha256'], 'ackFilesSha256': digest(journal), + 'businessSceneSha256': {'vm000': digest(physical_content(scene)['vm000'])}} + validate(plan) + return [plan] diff --git a/test_validation_heal_lock_observer.py b/test_validation_heal_lock_observer.py index cafc1818..38954e55 100644 --- a/test_validation_heal_lock_observer.py +++ b/test_validation_heal_lock_observer.py @@ -33,6 +33,7 @@ def event(state, sent=SENT): return {'target': observer.TARGET, 'level': 'TRACE', 'event': 'object_namespace_lock', 'component': 'ecstore', 'subsystem': 'object_lock', 'state': state, 'op': 'get_object', 'bucket': BUCKET, 'object': 'target.bin', 'namespace': 'namespace-one', 'owner': 'owner-one', + 'requested_version_id': VERSION, 'lock_attempt_id': ATTEMPT, 'request_id': REQUEST, 'mode': 'read', 'timeout_ms': 1000, 'acquire_ms': 1, 'hold_ms': 1, 'span': {'name': 'http-request', 'user_agent': sent['callerMarker'], 'request_id': REQUEST, 'method': 'GET', 'uri': sent['requestTarget']}} @@ -72,11 +73,41 @@ def test_flattened_production_event_binds_server_id_and_exact_version_uri(self): self.assertTrue(proof['diagnosticGuardDropped']) self.assertFalse(proof['distributedReleaseProven'] or proof['heldAtDisconnectProven']) + def test_optional_diagnostic_version_must_agree_with_exact_http_selector(self): + rows = [event('acquiring'), event('acquired')] + for row in rows: row.pop('requested_version_id') + self.assertTrue(observer.caller_attempt(rows, SENT, BUCKET, 'target.bin')['observedAcquiredWithoutDrop']) + for version in ('', ATTEMPT): + with self.subTest(version=version): + rows = [event('acquiring'), event('acquired')] + rows[0]['requested_version_id'] = version + with self.assertRaisesRegex(ValueError, 'exact GET identity'): + observer.caller_attempt(rows, SENT, BUCKET, 'target.bin') + + def test_repeated_or_missing_http_version_selector_refuses_before_events(self): + for target in ('/bucket/key', SENT['requestTarget'] + '&versionId=' + VERSION): + with self.subTest(target=target), self.assertRaisesRegex(ValueError, 'one exact historical'): + observer.caller_attempt([], {**SENT, 'requestTarget': target}, BUCKET, 'target.bin') + + def test_original_live_historical_get_is_correlated_but_ended_window_stays_closed(self): + path = Path(__file__).with_name('testdata') / 'i05-historical-get-39711742431.ndjson' + rows = [json.loads(line) for line in path.read_text().splitlines()] + span = rows[0]['span'] + sent = {'callerMarker': span['user_agent'], 'requestTarget': span['uri']} + proof = observer.caller_attempt(rows, sent, rows[0]['bucket'], rows[0]['object']) + self.assertTrue(proof['diagnosticGuardDropped']) + self.assertFalse(proof['observedAcquiredWithoutDrop'] or proof['heldAtDisconnectProven']) + self.assertEqual(proof['request_id'], span['request_id']) + # The complete earlier lifecycle cannot be trimmed to claim a live holder. + rows[1]['requested_version_id'] = ATTEMPT + with self.assertRaisesRegex(ValueError, 'exact GET identity'): + observer.caller_attempt(rows, sent, rows[0]['bucket'], rows[0]['object']) + def test_rejects_forged_request_id_foreign_key_version_owner_or_incomplete_events(self): mutations = [lambda e: e.update(request_id=ATTEMPT), lambda e: e.update(object='foreign.bin'), lambda e: e.update(owner='another-owner'), lambda e: e.update(timeout_ms=5000), lambda e: e.update(acquire_ms=True), lambda e: e.update(hold_ms=-1), - lambda e: e.update(requested_version_id=VERSION), + lambda e: e.update(requested_version_id=ATTEMPT), lambda e: e['span'].update(uri='/other-version'), lambda e: e['span'].update(method='PUT'), lambda e: e.update(lock_attempt_id=REQUEST)] for mutate in mutations: diff --git a/test_validation_heal_timeout_retention.py b/test_validation_heal_timeout_retention.py new file mode 100644 index 00000000..e1e666c8 --- /dev/null +++ b/test_validation_heal_timeout_retention.py @@ -0,0 +1,188 @@ +"""Original live I05 evidence selects preservation only, before any contender.""" + +import hashlib +import json +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch +import zipfile + +from scripts.validation_heal import timeout_retention as controller, retained_scene as retained +from scripts.validation_heal import race_retention, recover_environment as recovery +import test_validation_heal_race_retention as race_tests + + +FIXTURE = Path(__file__).parent / 'testdata/validation/i05-pre-contender-39711742431.zip' + + +class TimeoutRetentionTests(unittest.TestCase): + def archive(self, directory): + root = Path(directory) / 'artifact' + with zipfile.ZipFile(FIXTURE) as archive: + archive.extractall(root) + artifact = root / 'rustfs-validation-39711742431-1' + candidate = json.loads((artifact / 'candidate.json').read_text()) + folder = artifact / 'run' / controller.CASE / 'evidence' / controller.CASE / 'H1' + return root, candidate, folder + + def test_original_live_failure_yields_exact_pre_contender_preservation_plan(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, _ = self.archive(directory) + plans = recovery.recovery_plan(root, 39711742431, candidate) + self.assertEqual(len(plans), 1) + plan = plans[0] + self.assertEqual(plan['kind'], 'unresolved-i05-timeout') + self.assertEqual(plan['runId'], 'gh-39711742431-1-h1') + self.assertEqual(plan['nodes'], ['vm000']) + self.assertEqual(plan['ackFilesSha256'], 'fc8576efabec0e2163f8b434d0971f307b3334b01c3afae90459e2280722b70c') + self.assertEqual(plan['businessSceneSha256'], {'vm000': + '6c9e808ff0c6d03cb5dfd5546bd588c3d3e1bcee3c167877dec21c4399fc1e6d'}) + + def test_tampered_or_unresolved_evidence_refuses_before_fleet_creation(self): + variants = ('candidate', 'helper', 'missing-ack', 'ack-hash', 'ack-unknown', 'ack-operation', + 'request-mutation', 'payload', 'marker', 'owner', 'physical', 'live', 'other-node', + 'contender', 'extra-fixture', 'other-case', 'H2', 'failure', 'open-socket', 'GET-mutation', + 'GET-version', 'GET-hash', 'GET-retry', 'extra-transport', 'inspector', 'extra-action', + 'metadata', 'tags') + for variant in variants: + with self.subTest(variant=variant), tempfile.TemporaryDirectory() as directory: + root, candidate, folder = self.archive(directory) + def change(name, mutate): + path = folder / name; value = json.loads(path.read_text()); mutate(value) + path.write_text(json.dumps(value)) + if variant == 'candidate': candidate['source_sha'] = '1' * 40 + elif variant == 'helper': change('00001-binding.json', lambda v: v.update(harnessCommit='2' * 40)) + elif variant == 'missing-ack': (folder / 'fixture/000009-ack.json').unlink() + elif variant.startswith('ack-'): + field, value = {'ack-hash': ('requestSha256', '0' * 64), + 'ack-unknown': ('observation', {'resultUnknown': True}), + 'ack-operation': ('observation', {'operation': 'complete_multipart_upload'})}[variant] + change('fixture/000003-ack.json', lambda v: v.update({field: value})) + elif variant in ('request-mutation', 'payload'): + change('fixture/000003-request.json', lambda v: v.update( + operation='delete_object') if variant == 'request-mutation' else v.update(payload={'size': 1})) + request = folder / 'fixture/000003-request.json' + change('fixture/000003-ack.json', lambda v: v.update(requestSha256=hashlib.sha256(request.read_bytes()).hexdigest())) + elif variant == 'marker': + change('fixture/000006-ack.json', lambda v: v['observation']['response'].update(DeleteMarker=False)) + elif variant == 'owner': change('00005-reserved.json', lambda v: v['owner'].update(runtimeProfile='manual')) + elif variant in ('physical', 'live', 'other-node'): + change('00019-original-failure-scene.json', lambda v: + v['vm000']['files'][0].update(sha256='0' * 64) if variant == 'physical' else + v['vm000'].update(pid=7) if variant == 'live' else v.update(vm001=v['vm000'])) + elif variant == 'contender': (folder / 'timeout/contender').mkdir() + elif variant == 'extra-fixture': (folder / 'fixture/000010-request.json').write_text('{}') + elif variant == 'other-case': + (folder.parents[2] / 'cases/other.json').write_text('{}') + elif variant == 'H2': (folder.parent / 'H2').mkdir() + elif variant == 'failure': change('00021-diagnostic-result.json', lambda v: v.update(failure={})) + elif variant in ('open-socket', 'GET-retry'): + change('timeout/caller/transport/disconnected.json', lambda v: v.update( + clientSocketClosed=False) if variant == 'open-socket' else v.update(connectionAttempts=2)) + elif variant in ('GET-mutation', 'GET-version'): + change('timeout/caller/transport/request.json', lambda v: v.update(operation='put_object') + if variant == 'GET-mutation' else v['params'].update(VersionId='foreign')) + elif variant == 'GET-hash': + change('timeout/caller/transport/request-written.json', lambda v: v.update(requestSha256='0' * 64)) + elif variant == 'extra-transport': (folder / 'timeout/caller/transport/ack.json').write_text('{}') + elif variant == 'inspector': change('00002-inspector-build.json', lambda v: v.update(binarySha256='0' * 64)) + elif variant in ('metadata', 'tags'): + def mutate(value): + if variant == 'metadata': value['entries'][0]['properties']['Metadata']['nonce'] = 'foreign' + else: value['entries'][0]['tags'] = [] + for name in ('00010-original-ACK-bound-S3.json', '00013-I05-timeout-original-S3.json'): + change(name, mutate) + else: (folder / '00022-another-action.json').write_text('{}') + with patch.object(recovery, 'Fleet') as fleet, self.assertRaises((ValueError, KeyError)): + recovery.recovery_plan(root, 39711742431, candidate) + fleet.assert_not_called() + + def test_main_routes_preservation_before_generic_cleanup_and_never_deploys(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, _ = self.archive(directory) + plan = recovery.recovery_plan(root, 39711742431, candidate)[0] + path = Path(directory) / 'candidate.json'; path.write_text(json.dumps(candidate)) + evidence = Path(directory) / 'recovery' + fleet = SimpleNamespace(nodes=plan['nodes'], run_id=plan['runId'], record=lambda label, value: value) + proof = {'runId': plan['runId'], 'cleanup': 'ISOLATED', 'canonicalPassClaimed': False} + with patch('sys.argv', ['recover', '--failed-run-id', '39711742431', '--artifact', str(root), + '--candidate', str(path), '--evidence', str(evidence)]), \ + patch.object(recovery, 'Fleet', return_value=fleet), \ + patch.object(recovery, 'isolate_race_scene', return_value=proof) as isolate: + recovery.main() + isolate.assert_called_once_with(fleet, plan) + summary = json.loads((evidence / 'recovery-summary.json').read_text()) + self.assertIn('ISOLATED', json.dumps(summary)) + + def scene(self, directory): + helper, state, roots, node, request, scene = race_tests.RaceRetentionTests().scene(directory) + owner = request['expectedOwner']; owner['runtimeProfile'] = 'lock-timeout' + for folder in (*roots, state): + (folder / 'owner.json').write_text(json.dumps(owner)) + for root in roots: + for path in (root / 'quarantine').iterdir(): path.unlink() + return helper, state, roots, node, request, scene + + def test_all_five_interrupted_moves_preserve_metadata_content_and_unread_credentials(self): + for interrupt_after in range(1, 6): + with self.subTest(interrupt_after=interrupt_after), tempfile.TemporaryDirectory() as directory: + helper, state, roots, node, request, _ = self.scene(directory) + original = {path: (path.stat(), path.read_bytes()) for folder in (*roots, state) + for path in folder.rglob('*') if path.is_file()} + opened, moves = [], 0 + original_open = os.open + def record(path, *args, **kwargs): + opened.append(str(path)); return original_open(path, *args, **kwargs) + def interrupted(source, target): + nonlocal moves + helper.local_rename(source, target) + if target.is_dir(): + moves += 1 + if moves == interrupt_after: raise RuntimeError('lost connection after rename') + with helper.root_stats(), patch.object(os, 'open', side_effect=record): + before = retained.run(request, node) + change = {**request, 'action': 'retained-scene-isolate', 'expectedBaseline': before['baseline']} + with patch.object(retained, 'rename_new', side_effect=interrupted), self.assertRaises(RuntimeError): + retained.run(change, node) + with patch.object(retained, 'rename_new', side_effect=helper.local_rename): + proof = retained.run(change, node) + self.assertTrue(proof['activeNamespaceIsolated']) + self.assertEqual(proof['baseline'], before['baseline']) + self.assertEqual(proof['locations'], ['retained'] * 5) + self.assertEqual(len(proof['privateArchiveParents']), 5) + self.assertFalse(retained.run(change, node)['mutated']) + retained_state, retained_roots = retained.destinations(state, roots, per_run_parent=True) + for old, (info, raw) in original.items(): + source = next(folder for folder in (*roots, state) if old.is_relative_to(folder)) + dest = retained_state if source == state else retained_roots[roots.index(source)] + moved = dest / old.relative_to(source) + actual = moved.stat() + self.assertEqual(moved.read_bytes(), raw) + for field in ('st_ino', 'st_dev', 'st_mode', 'st_uid', 'st_gid', 'st_size', 'st_mtime_ns'): + self.assertEqual(getattr(actual, field), getattr(info, field)) + self.assertFalse(any(path.endswith('service.env') or path.endswith('/credential') for path in opened)) + + def test_i05_does_not_open_other_profiles_quarantine_or_multipart_files(self): + with tempfile.TemporaryDirectory() as directory: + helper, state, roots, node, request, _ = self.scene(directory) + extra = roots[0] / 'quarantine/G01-target.original' + extra.write_bytes(b'foreign backup must stay opaque') + opened = []; original_open = os.open + def record(path, *args, **kwargs): + opened.append(str(path)); return original_open(path, *args, **kwargs) + with helper.root_stats(), patch.object(os, 'open', side_effect=record): retained.run(request, node) + self.assertNotIn(str(extra), opened) + + def test_plan_profile_cannot_be_reused_for_another_isolation_kind(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, _ = self.archive(directory) + plan = recovery.recovery_plan(root, 39711742431, candidate)[0] + plan['kind'] = 'unresolved-g01-race' + with self.assertRaises(ValueError): race_retention.validate(plan) + + +if __name__ == '__main__': + unittest.main() diff --git a/testdata/i05-historical-get-39711742431.ndjson b/testdata/i05-historical-get-39711742431.ndjson new file mode 100644 index 00000000..ce873b30 --- /dev/null +++ b/testdata/i05-historical-get-39711742431.ndjson @@ -0,0 +1,3 @@ +{"acquire_ms":0,"bucket":"rv-heal-bd855cc838e268dc","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"3a86c2cc-ada5-4e8a-9e0d-dbe3bb7c1489","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"objects/0000.bin","op":"get_object","owner":":::9000","request-id":"1ba2503b-78ea-40cd-b2ff-003f54662c9c","request_id":"1ba2503b-78ea-40cd-b2ff-003f54662c9c","requested_version_id":"d34e1ece-27de-4e0e-967e-e9238b61132d","span":{"content_length":"0","method":"GET","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"1ba2503b-78ea-40cd-b2ff-003f54662c9c","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-bd855cc838e268dc/objects/0000.bin?versionId=d34e1ece-27de-4e0e-967e-e9238b61132d","user_agent":"auto-testing-i05/7629f31a-8105-40e8-b287-cd86965b37ff","version":"HTTP/1.1"},"spans":[{"content_length":"0","method":"GET","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"1ba2503b-78ea-40cd-b2ff-003f54662c9c","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-bd855cc838e268dc/objects/0000.bin?versionId=d34e1ece-27de-4e0e-967e-e9238b61132d","user_agent":"auto-testing-i05/7629f31a-8105-40e8-b287-cd86965b37ff","version":"HTTP/1.1"}],"state":"acquiring","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(7)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T02:31:14.276873476Z"} +{"acquire_ms":0,"bucket":"rv-heal-bd855cc838e268dc","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"3a86c2cc-ada5-4e8a-9e0d-dbe3bb7c1489","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"objects/0000.bin","op":"get_object","owner":":::9000","request-id":"1ba2503b-78ea-40cd-b2ff-003f54662c9c","request_id":"1ba2503b-78ea-40cd-b2ff-003f54662c9c","requested_version_id":"d34e1ece-27de-4e0e-967e-e9238b61132d","span":{"content_length":"0","method":"GET","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"1ba2503b-78ea-40cd-b2ff-003f54662c9c","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-bd855cc838e268dc/objects/0000.bin?versionId=d34e1ece-27de-4e0e-967e-e9238b61132d","user_agent":"auto-testing-i05/7629f31a-8105-40e8-b287-cd86965b37ff","version":"HTTP/1.1"},"spans":[{"content_length":"0","method":"GET","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"1ba2503b-78ea-40cd-b2ff-003f54662c9c","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-bd855cc838e268dc/objects/0000.bin?versionId=d34e1ece-27de-4e0e-967e-e9238b61132d","user_agent":"auto-testing-i05/7629f31a-8105-40e8-b287-cd86965b37ff","version":"HTTP/1.1"}],"state":"acquired","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(7)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T02:31:14.276939705Z"} +{"acquire_ms":0,"bucket":"rv-heal-bd855cc838e268dc","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"3a86c2cc-ada5-4e8a-9e0d-dbe3bb7c1489","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"objects/0000.bin","op":"get_object","owner":":::9000","request-id":"1ba2503b-78ea-40cd-b2ff-003f54662c9c","request_id":"1ba2503b-78ea-40cd-b2ff-003f54662c9c","requested_version_id":"d34e1ece-27de-4e0e-967e-e9238b61132d","span":{"content_length":"0","method":"GET","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"1ba2503b-78ea-40cd-b2ff-003f54662c9c","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-bd855cc838e268dc/objects/0000.bin?versionId=d34e1ece-27de-4e0e-967e-e9238b61132d","user_agent":"auto-testing-i05/7629f31a-8105-40e8-b287-cd86965b37ff","version":"HTTP/1.1"},"spans":[{"content_length":"0","method":"GET","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"1ba2503b-78ea-40cd-b2ff-003f54662c9c","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-bd855cc838e268dc/objects/0000.bin?versionId=d34e1ece-27de-4e0e-967e-e9238b61132d","user_agent":"auto-testing-i05/7629f31a-8105-40e8-b287-cd86965b37ff","version":"HTTP/1.1"}],"state":"guard_dropped","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(8)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T02:31:14.277240674Z"} diff --git a/testdata/validation/i05-pre-contender-39711742431.md b/testdata/validation/i05-pre-contender-39711742431.md new file mode 100644 index 00000000..b8eddeae --- /dev/null +++ b/testdata/validation/i05-pre-contender-39711742431.md @@ -0,0 +1,13 @@ +The accompanying ZIP repacks all 58 original files from RustFS Functional Chain +[39711742431/1](https://github.com/rustfs/rustfs/actions/runs/39711742431), +artifact 12397696964. Every member's name and bytes were compared with the +downloaded original. It is a regression fixture, not the official ZIP bytes; +the official archive digest is +`e9ab49593f0423f0649066aa71325d59fadc0e7e6d0328071e2f06f45790325f`. + +The original result is FAIL/cleanup FAIL. Nine successful seed request/ACK pairs +precede one closed historical GET. The observer rejected its diagnostic version +field, no contender PUT was submitted and H2 never started. The original three +lock events also show an already dropped guard. This archive cannot establish +an acquired timeout window or canonical acceptance. Its stopped physical scene +and reserved owner are used only to authorize non-destructive isolation. diff --git a/testdata/validation/i05-pre-contender-39711742431.zip b/testdata/validation/i05-pre-contender-39711742431.zip new file mode 100644 index 0000000000000000000000000000000000000000..0fde8389f8015369f90629acea70d4dcb5aab248 GIT binary patch literal 95717 zcmeFZWo#u$lkaWjHZwD|nVFfHnfaKRnVH#bYBMvpsm;vHY`2-)*U!w}eRm%1)!i$- z(tMdCRY_H+Qpe7W{72-kB2qyb6buyz2nY)3QCUrw)g~X(1{es48WaeK_4BTyi<7gN z6RoSEjirg9vn9ZemYI{Ck&&H+iG`VwmXY4r(9Yyv-9tmRZ0!o7a|Ms&Ig>REMOFi+BFd?Xa*gAjQzV`mFYp=+-FZfbH%vD>=FV`3fe;GbF zse_50&tc>J974MPm*M+!+~OzX`WX;|uXsccm?QlbA3R@~7Cdd|!lza0`oXU+5x=_w zS2Jg3$ln%R($u^u(Z7$Y}UtMBgP`-Y3U&+{e!@*4kaZ zfXz^VSbuAN>G6+iE1UPOeQ?>@Kk^4f12uzNq)1~f7y+47b0k+F?Z>1F*(M^rf4&3% zYgcG(PMyeu00DJ@0RcJx54yt9#g5*>)X;{`24HMW>uhOj3UG0z{WNb{V?!GoQ%71u zBY>mxr@=d!I{gnG{If$<#{M-CI*-1B!cpk476@f2*4Kd_U zI86G7tJ7$)6mN`iiz)0PcvjkZ+LV5et>79F+^sXkbM9A13|xcSs9ouZ%1a0gToazj zI&I!Vo?>w!9k=k}B6&#e2QVXed>c6v*hJ`a=4^%pSZCOtWy&%lq)kV%%+w{7gv|Vh zKqozG1mt9K8?dHL%3=-UzSR;@3teNDW;u&tFzh2t+l+f1SlO*nJlxEMegeEwW`?rV zl8=^g*|lKc-FrC=>QgZHUr@MJsoQAwhCHUJ^+OGzSkwaURRPVho)p;(sW_nRsXE`Y zeyxcmD?S*$8*5n=n$kx4Te8yPsu_=cw^5ZAtY}V9vR_#TV=BQ7o$CTY8=@-zUY(*g zSx~|0g1)-yK#^e=oIC`wj{1#PE`AC&Wt=K;H$}uN&Btw`lQo)(Ec!tTT{_uQD~d@j zYSS8(6;i4gU5#DE9dH)vWl@Q~>0Y*6ZP8u*H7sr0Uc2>5yFfem?LA%M2ai0sQ@hJE z{l2LY!9lm~jQ9yGXhydb)6M1HBFjt74h#(VG(uSh5ZOs@>F5ID||fyU(mlAIP-*1$J*yShy9f5@c;h| z+|b`!Oce0afsNh#~vHULNI5cpa zY;36;iNV@(GR=CyfqLQv3^VTA%!lr_D>G-;`HI1m zMRyqzGhaeJ^RuV@y<1a*1#=qe!UPJRNQ4aB3wlT!A$TzjtBL6nSwVU0hQYf&2!13G z92MPGDmyq3btI}NG2yt8kqBA$o?p?s5W}}x5}W&o!K^A7IejF4Vq8vmyie<^Zc#ny z)P54|Pd-Z-B_|2Vz%<6M^HA!~Ewa+@&wibC@hqmJGLaI~{fS^hfCTFFf`$m`1t4==V>J@Bhu&cs`&`z=Hz;Nx}dD z#s94a$V6*t=VWhc>@E5ef2$NTe=)Mip|Z_hse3@A4_l5KmP~ruQTp^wQqX zzo%>37B1*;A{DmK1ft@08LZ+=1+s2hRu#E63_k+VVY_!|fEP?aA*fsTvqr6}q_!1jVdMKDpGU@E>eN10gw1L{;w;Oi5}F;hGyR~@Ut z%kxEyMj>`O5m&hbTUO7`lcj%PMiLb=FaqkfzQ}S{%4fC%K%)cvzAbzZk$}w*vy8RB z!J_0`t^xN{hTXl2cyX$55L494q#q7o9M*k9b@hsWv|ZWbuN1G4Wp%zS8S=L`ogO<^>(u+k< zJstIB)gvb}3#UWB&-k`^OKy`h+6Ix(5<9J&2;Rd|+4&}Zbsd7SoNJM$J-s_b0GlH) zvJ3JZ4v_y{L53XH8Ez$;CfF`A1gce^&g@Ut_E+G&FiFe_^&6BG<|E{E(%|sEa+;t7 z!6LXJEc_V5@xT~poo%>`2`&k)AC*T$=G?O~qz$p8H^Sh{f(;p$&$hB@JOs~ChWj-g z>y6ejccP;wQnTvRzFxO4lEA`iMjIF#)Qx4BE+Ag1!JKfaetPPrF;n8#$Si5unhsmL zs&6O-etbn_59Z2pBlZAgxncRUta{s1R>hd#y4{+ceA+tndOhttzn|}3_IKUuTKW7u z9Dd*2y*v+ABJ^ZJBxqWF0Px#`fx>UUF$&6Q65B%v;x zOR|M}J=@CjsEaX_ucJE)Ly37hIeWUA<@@dVI&;1@ct@`lRx){TW3K}x9Zc=KA$K7G ze%Pc$S?NZYEO?gKLWh&eiKfRCxoBWh0>|?7ohNBsB)=M~A?%8#j{M?D1yRN`txQL# zNbS@F)p!K^07VL(%K~;51;uS&B|k9ioaZEo%R%VT$9tlL+lIXD+@g4lDF~*>j|5p3 zg$#4Sx#WsQHqj!!9D!68!;l6=CSttpE*!}l{yeL3p0p$2H-%=}z2irlu`8Pt4&E45 zaeV!r9OM<+ts^e2l9GBIaB@nrd4Ey3oEIE!LrnBr1b=5v7N^TqMVsF(S$Es}dEB8;fAVag!DMaL&?i_(Xt&ccXcr zqkeFsr>q7($}0C!C*LPaN@1YB^}d4Y@yy?y+N*HZ6GCTlnmf@E=q{`g#zl@im4{NP z->M#~u^2y!J11*7UD0-V2g{B_B(uOLn~K4oW(^VOzPmU5}eu| zm83Fxr_-P9M3(r36ch?;v)7zPMe+O6y6R@IM8VPJ}%nr3>g7Flx3abvo}ZvvSw*Se~HJYrGAAeTvLxhS>y zzi&QjUhUC8^jmew9pW2j+bkj&B8FDE_$u%QXP9JXFToE}NIqU~$_2R*z3|2^LFz_( zu-f&}k5($`nR@YZN*G5r%xU!VfgjqWd-Z~HPfSm4T^SAKxRwT3GEFdjNj`yK$Cd1b zyVg?sPU9Fl3ZtUQI26L8U9Zm1-i3uuHE|G{J9IvY!`-@kp647a=P@U21+ga3H#ccl zA=#0|_;1JAK$M2@x8X`AmHOnc3r;N|+6vfYG z?`N|XbXjQ@m4OigE6qj{QFt|3GckNxSZ&xlemjmJo37+SF%u@4QjH2Uo;vzC!?{0? z@%-)hj(uvQRZNj6l0@<{RCc>Z7{UBv}iF$fn|;rCY?#lp`(%m1`d zh|d5w`){>Tme2UY+0@+8(%FO7%*NF8b6x1{Z0h9v2NC&mrKiLlH_w20`KTT}tJ6b0puy$7;aG`6{Ky%(3_wJiWlg=fe#G>APkB+j;i~!H#rIe2cpX_j*9z&e4H+OQ&jjZ;)E4qFPbofzBGd_ z&I^N~iU86EepvR%zA{=UDz%nu18*9?PPMBFTbH$^xy|-Ps-RvsY$2ztQc>)_{le?~ zRJbBm9o8@jwY|nMeve5_RrTlZwc>UW{y4uSX@?NgOiu0Uv4H6-jJqs#3}FxKhYMZx zr5r02?W?nQ1$`ilGQa7GhUtrj>C=xHziuPvI1@f)Bag|N&Ny?X8+vqJ{o4%W6UsJQ zZ*9Ja^jV==1;<4sVSK2&D-*#X(-2JmqV69e$@vsGD8Qj;Y~Sa=vQ7D(sHnMmT;e?B z?#SA_B}GMy$#X{hVMEEW7*`=|dQC(Oadq(zb|ZS{pzmKoX(svn2vhH^SPEf<`dAE+ z0lr;foZbl6xB_OJHt4_s=2a0ZllUb+XG>Wqc$2am>t5Sl|BVe0SuF!;ezE}pNFbny zztyr>X&p_SOdVZK{|s3NQ^xGp2~k5IJtAi+!{F6H6K;SuR8Q+qcY(s9QE-&DW?4o% zmduzYqEQdNsYJ4~Wq%xPBXF$eRjHjNgH}(epOqCSSF=m2cgrqM_&hop+qoLQV55&5 z#6neI%_U0rL6LB)!8Mr4+}BF9*AW=WlUtku3Fn}!5V7oI9i`!Sls&j(8m7Gx2}a2m zPT%0}+TzbmPt6b%cX?*(MtqspH5%#3Y53uK#V=DW@xtSZ|A?I3Y-yE1?ob*2Zttdy z+GFg8i#go>#^;Cx`Xy1LC{O}`$C!dVzyY=w9KV`dWV3VHSv6;LFq(V2r_rsseRp6E zV8>%)u0&_Qug6YVXL+8Dsq0}lsHpka4%uGHO7@HK@W0>VK*(zOwq5gW`%0*Q29%zI zLSQgjNL&_Ij1;~zT2W0y^$7&#A%b$Cf^xq@;TNHRCnuruhx^x7IKpUe z;E*Q~)EiaQQ(ej;S1k{5Nz_CXOAa#sapW^|;&tvZ>UR*B=MTl?$EjhM-BYlQjYm9t z)f8Esi7+f;5wHNguiD}lsi#yqW5Z1{2f;>phsvF!RcGaqF9(-5fX_=azFnSU{y-W6+rsq3O13c!@jC=Rd zM7>sx)!B_gSl9QEOY7r@fp)?_YTrpLBE*9@?+}(L`vv*_+?XR)?(bAo!?Y7s`R;An z2UG~dS(q{%`69(p<_;yn$tBtb3)NSsq;}Fe-kANJ?8Do5r+zNXZlEEoq35s48bWET zWdCXR!JkLM-)jGC|7GRoVs8U5{Bs`gjtj#7AVd_~;VmVs)P)P;2d~+I_RkPYTFs2K zleyCCaCp_b@rh}&(-YYkY@Z4WUq6v}e7>JwOuP5|8ttwl)lIe^&k=?ciS8shh?2}Z z$l$L9cMpfIXjDa5{Y^9z1grlk%qxRDg3H@(kH43!u+x4z z8As>;P=~5`S^0hjxX`P2G*vmmgz^#Prj6Jxon#odNa$8-TgTA5p=Dy0-it2deLp zdVV*Z=JO?{C4msMjGL$7+7Bxhp#=J27U`^7$x%4$>#ZfKmQ-M~v@ksAgs}35=SjEN zIDWjft>)$y?ljn71#`DoZOq`E%E~Mudctw%a?_0`?5gg`$rvBa{fQvj_W9fO6LR_} z3kXmAoC)G`(xF5oF^Awntlv$cr%WyO!}bXs0=-57D;Lotu)4akW|{Ugj!pr%b9!FW zsFnMxu(BD~;Tj@F@&VBFu*!Xf~5Xe`Gvf#o_IHbcy^!PUb4O)!rO8h|lMUsAa3o2vg0wdSX zjqo49M+V-%gtzyFnKq|9`z<{=E@k;m{X)0yR%q}G5ne30CfW;k1OYtmM6WZOq-SyP z2*R8%Cz7jx0FRxq9GovE24^Le25v1ZegU@i9+p$gX=e>vnNn+|PrmXUa{?>x=@*ZQ z{Y|EPC?CLi&L13?F>^MnI7NkfmS;gdP8?jhW_to$5)CX^kDr1PN553ByXB|5IQsFT zrbMffC+Jpv#zjkaTTX+{{^M0nb8d!%Q>MX=KSSn+-JMrI%bGO?M(>~_-^$p4JHSq#c{R`M)R8Y#txQ$Z`*R7D3L1eYztPW>|8!e%@5dE~_h^hEj`UWvG3Z>$mgR{nBQ44I#JKtoA|7`vgkR>jUD~q$0@Z+-X`bP zb^o+=KiOb3Bp<1%SZH#NYk(TIfczzy8wH-BESBoyv4rH^$iHl3bzId!mXj>TyuH5*18$vG31N_fW87} z!_ZrflbNN0`r+g}>)Y;iq<^?E0;&>R^J^CtM5SGBJ!LcP)d6aNUojXW!;SPh_7n%+ zZT?BP-SY@>p=VWRRenH!m9_fLZmIK6IYa&oQ2$=6%1LWz?CfG_^H)(ziIkiQ{`SS+dm|JR{83cILh#w&@YvTdFPH^NDI*%G4L5a zkF{66-aLR`O0Of0WLR6>E5C#Mb!1?iy;ev*9ZnlE5K!vh>2MesXaSCv=9YG!Vj`Fu~Q%>1X*NtXuTvcvgb{YD$gEfkyA?gF<(5Jkzi2+~0&kcg~rS-EU)Xb@cQ z{piZRww~jFuqt9uazkif7MA)}6o)Vo7S!Jle(_Wl9R)zZ=;wAKZe^O6iXI=ta42KZ z&{zGXHd(TW^CIiM&e!zv8k&06wov&?F7(2&;dh;^m47x= zQhFp^#Zv zTegdZmjh0k1aTBC@<&Ps2P-EmTj065)v3{im%Vde;+QfoIG}!aZ=JznxvV4c*DG#K zaoioeIL_Qu}*S5_9w(may}jJ9kalo#1O!RnSFc+A3f<}pJNB^h3lGk;F;(H zyrzNobtrNOQe{m?Sa`l7A{J+OmGtP&Kn|mXvaov#?@0oQ*b9t6>qju1^VLdm?>|2Uzt(bpZp-TJMgvAbzRn!ECTVeKU2PV zg6uV}OUP`qyuD``1i9>}eOFG2R`p!CkpOO{qSDeLD2`tD4inN)ZYem7P0zfuE89m! z^Aym3@ySOgpvk9YB?gglssZvKXd0^6{j=eRFT&ogn|}u&oW*>0+@+k5vF;*waqV z#-^?g@%ZxLh7efmY~>pVY+A2SfCh`9Mk7-B?7B1wlmZr(&lff8pdm_?&S4fbvPYQd z6s=n?n*<#4~X zG?k+Sqqqd5SUOVblguZgL@9$JM@8j0Au~HEEBh;1MsN<6=``v5o71&fhk#^61OgJT z1p>bp^+vi=|FlWfsYUAo`Cm zUJ!566`a^FVZ;?ouoJ`+_J=cBDiI}6Q7T1TCT!42^*L~}R73+_YX#((%=B7FyEh|QC zJ>tgexy`7#-8{_OmB}P|n9JX`#z)t`=Z?Iz`OkSaUQIuG4jvba8#w6SO}wnF0e-iG zw-oYjxw?$67+Y7DfS#uvNXFrZ#VPGIS$m$cxv*ir+9}iS^vS1=tCDBGHkt>7(t`!Q znCk;SIx~&Mm`8o>EH9@0!>vaG{*9-dgMxmsfs^R=>P1G~2wqD>kVu^>90skw?;d+#(*`R*hs`+i-DmxpuYMz?v!@)7-hZ zo^fa7_qif@vvmusIf9%UOu8m&amr8pu&*e*Ht}xc@#Mkj*`Z)QyxO8u6Vd|55fBF@ z5HTMcG4dXmG7+JhgRUC*retoWByV@aPodtG7oPp?`Yk0}5rH4yGbSnst)!C^B0*-R zP3c>Vjd#(_#Wl=$kFCI+#PqJVaPgDLM!qu}s7Sx?VkE?sGR-1plF#o4)=Bg@K2LG& z;_DAqgE>Ah#gjYKLxNsN68EGps(4XRk(v(IGp$f`1E=ZA3*VicAsP%6=0^oosEimA z_L~cn1h;tKZe9FXa%r9pY&zXvzf%K}dXEUx%ONBZ!j9K^sLzn`P^mp)$`T{aCQK@+ z0>IDkiUm@w!x>Q>Bg+rYWtkR4VUCqz#Ka?W@hVOOztTb4x7d-NWmmAa0ZYe#`dh<* zBIs%BtE2)ub{0g5#AQ%NB`IuY!yO@iMdCCtJ2DACtVn$$=L5k~0sEdfW`x*aQn#!- z`<@(y3Y+K=h0|!pE?~w@3S%b$9aFyvIq*wdhtwJeSK0=ZiBG#+&zqFEou*n!w9Q{l zXehYS(UDF<2kpw3tv7Lxb-g`4WJT zSnAe~sW;g22R922I&|62=06p0?1de^(u#p34s`qyjZ-kXT# zA)E4N=D8EacYs9>r{DK|YtOCQJCA#VHtmRI4tV-ZJszCfh035O%;sCT1pK;E+4@J1 z6|{36Jh?d)1$?=O!g`e@(^BKGr5>TT2hmd*&s}v0`fS<+xUBur_6D`tNU(9dH7 z&(w8h(#NTJzO{?xPx$(=j=v=m4P;?7IOx&EPBgkEUk5+6NLQyRpu2Ip#5oaP(JhY>?lMKO9c$EFTCnlzG^Hh-81;_W8@&n=TZmzdbGMII79M0Dt-T?S ztxcfZRy)pj5?R31leorQBUyJ%ri9(WSVJt^PH6 zf7{dTHK`>{XUD*uIGfFXQmG~A-1qTHXI;Hhk3rha35#>(XRsmIb(!C0e_mc&y95P$ z`~1V`UCB2*3fAe_Cnk`DCW;LIGPel#nhwZ%F~f*7ew~d;Z1b#gnp0QGt_p@oaI%e3 zN-na3OUvs%4D}=Z$dt{O_ToHlRt~?V_6)?^Wx>QF*~1mTiQZ&1$xJgYa zHtj^ozIB{|eJETl!t9Pm;^eGY9NArhYrl{4CfYxFW)qX7TG`Vj`WVx}3lph$1ST;N zZb61f0npv=ntHC6c3!8kX^ooI?tF1)C(%YYnM%GIJJs~Qtgo&rV-)s|kjws}0xQM*b+VZDchLGpgPRoKsnuw!I~*7EWm58mveRl9*i zB2O&jz#35UD+u)Hw4xlaGuz?zSeK~<^i zpmRVmOYr=$LK_Du+w3=}W6U%YC13?Ei+)9T%e@#MjA4890G)cYkveEBXG5rbq@p_-to)Kl zI4-Qq#$GwtbI#q|8IUOD$fkN5st}D@N-EN?wJc&P#Zz#h>g(Hx@i{F6p~ef)m5n^B zOP`Pb(MH3Kz#<@mEZ*2A;=u_0_ z{ICD^H+Ki-Vf8EsiP%Of#b5$EGR9`U{19p~jTR|!tL`iZ`BRz0x z$rCO6q_KA`u1LEGJkU*$47pGmE?i0KYaUfXQL&Pa5bc)`)oxPGHCKf-zpiF~r94O$M70SQ3hKyR3-H8Wj}0NVl7rsR~l!nR9Oq`Y9Jyt?7IVUY9_rn9aXcQ@d>rQMc^~G>Df{d@KP`H zA2pdtN&JfH%%mc^n6jM{=eR_}e1pm)(Hba&TniBo6oB|uKx$}%Y~zX@ZQA#m9jrW2 zcy#GPX?6s-t#5BmoM(sXu9PlkIUp`)KDggxy^t{4b z=Augzt6Xk$jPv1t|0xNVm&(8+YD|QpG7Lm#+r1qoN{iXn@a1aZCbl#N3fiR_mX&-7 zb1GlTM&DtBxY25rfw|{5V5##F)SgcVZB>=d8JCWGrL1@A7pVyYuUDXc8cv7SCyY*( z&Pz<;X2bmPJ4+iq?)Fj*iaC?9zsB43*B4uZy;N&!Z(?ac6?mTV6R2&JIKZ3$F9PwagcYi#Ox;AoPPN|D2aH&R>qYaq(qZX~wk->Fy zK+k9tgLCH%iGx6&?VcO0ZL}!8t~_ zNq{~j4lKaHoyY5fr&VZs=~js)AXz45>zFs;te%G=2ezmXS#^z1wJeIgq@`S!Lif-R@lM(-@}goJIPMO zm56j~{tL-g=7z8$wTChP?@0F5zex72n7!%u6wHh$`duZwJTX%}E}3>e_9nESe*fQ}*s(`&oAwb1VNvvJL+r+35d4vMuCqAarGt9J;i| z)VNA9y<*UbQ$)~6*~XFYC%Lsr7#x7g5@^mq1RqefEmAR#a&q-Dm~3KAF{jY&rZV$# z_hwjjD@3IP&-n&^)_HQAQjREw=e#l8 zy*ss-V%tIE*m*`Rn)e8wj(lsTOy(WL-+qeu@b68hT@7zAJl^j`b*WM70~UE;D;=Vw z`%aovF|UtdvVGjmPUy{nbfa%)py=(JIxt_X{KZ>b8#ic&d0Bf-Xce9 zesj9Dt$dNvMN7L;I(NZ|<}GvmI)I8b%fFUU81pDx?{ljAo`bkDl7Q9#r~%#Tlc?3 z*{br8^XYq_(2ba8y3Q=X{6mHS>7 zCW2Bn#h$eA{@N$#;hb9Y#!}UCT&t4;SW1gG)!bKB9k5dyqDRGOb{al&T0P`dV_M|~ z74CW6Ah{dtxqzep1Il)!{~w}kxBnhx109DcNS)o^zf~P(?@*k%iY=OW=aM;p!~9Lo ztYX0GMUgj3GsDSeX&Zmx+Lqg*x+xFagFC_bXr31kdu`E{>$c}uhY|GFq}*Ek9w~nF zij{Y^VdcUW0^8Gucv)kT`=Our^~Mg<`3RsC92Dh|i=U1E_HQ*&{}tnYlHC7m`Rf0$ z=5X=h6NzcDVYp_%h%O`*Muk&UJCXAacK-ppe?Z26!0ta__aCtP57_+&?EV9G|BqnTuCa5Z^RrsE z%JS!9x+`k=3K z$-N3qTT?FPuN~sp|8QwTK|fv3=f>Y@oq$ibI%}`Y8^U{Nn&MAVnzGRz5|z-8=74iQMoo8wARg=F!7QH~#h#b$gjhnTtRcwH761+blD9tcSx97FQz$a6hQSaq}nj33%j)->JYQXe#F zYy#r+Z5cRH1Th$En>fE@gM~TgAu2jk526G%ouQ^TpbeCtl)-A?Xp9KDndabCoK zo4l;;*`O7Vobze%kuO5_IJ&#b%%(tl5){vvuzpx{Uiz*}Yj1tfR&+F(Aa8vcDRy9q zu~nHm-+3?LJZqUqU$anO7QX-;es8fP9|G8+Xy;vv14Ivop+PW@C%(WsRzxyRb)(Eb&$6pa6koeW&RH^SD4u03f ziw;7WYnT`z3p#m5oQ3sS>fbT=PzRxNltn0uf@e1&OLH;p$#A89RWhP$*{DyPYWgXz zO?J^@J#fK9`ISa|7X^$s$|?68*2Ad4WdcD1+Xb1>JW&Lo2lW#>XhE$q?B&cdQ2`&374pXhS^9Gk7)JGg z%9*qMvg5R7q~j}+E$x`SD%G5j4F19Lxs^(mh6ykitcEV5EHN%IAT@y~xm*8$i24VI z0OJtwa}l`4S7M8vcKTc4Tu+i6U1y#w4obz+XSKIXYS1m zsz9Va-w(6n{>_E8jO_n)I=g*tG6!(`b1G}D#eRkkK3kum!#qfV?5U8V7N#gt(FKUC z5~{BiE)*(feM$;hSWz}a)cfNN@fF$fwQ*;JrS$vlEvenYhsRbcX+G7fEm}YT^2IXu;rV!)77D{+0?QxO1S-SY zN4r+cN4xLSl$4L2%=Et+COWYP$S^yW#}^n|D@mo5i8RIuzsW>StglA+`WY zKdia=d<6*GT|`tpHvWDeKBDgwU2|4&?e7LLWhI@+8TX`YW31&*uTSt5AICyVKn$18 z{RW09&S?-~0`8z{8c0g>2yY9M2@vkB@eygr>NK0YJ!dMryrF#POo%yYq>v zRS$-{N|vFw?YRDC)t=AqzJV^|!Q?WFkRw0o8%f;qpjW>A4z!xj$ggxwIp`3=1c6JY zA)zAb62lOC#!|q7+}7Gr^EU27I7UKgJN#>3a0nU^(f^xFU=-F%PCyD|X*ViatKBex z7?(NTiq&+pb0q~G=Qfa?>y+&42$T>D?khJ})w4Hqz8?%~*Fx3!y1jxO!7ZiQ!;o_% zUIj99wu(V>recMivC!ht2Hbm?=@W6L5)xugnQ9JdOPe$9&5t)@p6_dkG(XLJdCe}X z*Rsl_KbGjaT*`2wiu$^SUCy|ZO}^iWdr3aJ89qZqe^FmU+4oYS5~rk#q}WGonO%EC8~I^oy7iBNh_+lxrvPuRfs{1@6ONp4YvSsNR&t5#@UsZy zCD(1KRHgf{esvT22l@tebh31_kuP2w6hx0E4m<+zqx_+86ef=vBQZuE*mOq);LMB1 z__QIlym;VKW2t^%F1z{Ki*4@rb7zne6fwKFjEUyZu(SL+so z5ZM}_dwYn0{3!dNmwT`b2pUDde&jIS_k{CL-F$=krHb0ZrXRie;Xh9^nV8HF zHvi!eRNYS?i#DG+o{J#$IQqU;q^?wi1eq&Ms~sxv$HCDJ0%Jhwr7eO**PQDt69a&i}0ziuqz;`(G(B3!lf|x-S+Z z2d#sPrKyv#>3>8oqA`6iKNt{2uHGZ#8xjW4BR53^d7uLI5qGUHJ}YX-(`NnH%V_Z* z7yxGGp^=QadP-8=#=_{DVo}LxkBrkK;YD7Z)#a`S6yWz8!rO~_KtR(QP> zEWPSIq4FY75H@8*1cpP>R+^*7-)8ithM6diu4E7<6W!iNI-ckjw^nd|fU(c2SaE3a z0`ol^tJ+m||24I!z<{I)^K(CjswyC$w7>NQJm>#cOJ-(hY2)JfxkdG7Xa2cE^@Xl3 zK!-ii=dlLr#l;7&UV|p_HPyk@9|%jvwE>C&6D)|h!s1@b%Hy>k1+e|m?Xp&171vy8 zK_l*HLFmvvGA4~)A(yku+xC7|SktZ5$>T6d`}@5@|02kThik(|-TT|agi+U;&Fj#` z2MvCY@q)0HJ8u~Qxj^KHMBruaMx2ZX)CHIs@p#Em$69ianDW1BuXd}rxDojblJ(fdbK{ruB}QoysQPU8BpI?BY*Z`v1Zqu+wvN5F}6flr3mfd*!?3I%ej1W zL`20A7D%DY_dRfFjl$RAQY`D>W8!j;icEXP=aQ^Oy~hfpXbE*ZLKZYgV-dQdqEuo&??HB^2nDYN8RfKb>OkEc9G^6K%&DhL-C!oe zW?P0(bloE|g&nVaxPuv4 zfA`+s&As)i-k+~NRb4e(+k5()uAMV8-7{U7K5#y_I@KZ$Z;rLJv)3^QR~}p{9vm8;8sh!;3+mw)0*{D*>~==7~=!=36;W@R8I9_!Lq7ktb_p-l2LSdL>ayhD_V)= zd-xk;ZF+Ra^+!Ue9;ju5u_DhFz6mrteSmn zaL7;7cBxqFr15kNdE9O;`gU(!@J;AsjAJvW7&Qg-cxJoi+^JK=@FnkN!n0!S6>USQ z%4wqYV8XUCW&CAvd1%U1iy&(}ScA87^*4SF;P9RE2Hq_1<2imxYkQ|*!v3*)%PjIY=a$FpC=y!% z)j^Rw-3j}*6pu>A)U7&oxf-{buclup$$Ie(9?YA{IGpFbw z)7tmgwC(N^OesctU>f=h7pY^BPO7i-~1s zv>-u2uMT@;>-U3lF;-AwT%lt#o9Oy4PsB(pEC)BNxXNM4mD@tQiCtLt0)a<6RU&-i6gM<48SC+B~Y6N~Rt15MBrpXwh{%(dNv$+r|E3(*+NS z32Hl0v>YaqEa;eqw z>DIXkM*lq^y2I|iR=BDFLT6*BU4$Y_UIfI0zd;^qroLll+?pr#L- zSBzP8zFRD1c4t?r^QdK(?t0^;&*&de8I7+qeu$Mb6- zQQ$iFQWAZ-PKnZY`{WbP)H;kTN7@pswi9PBe(gw`shEpmX?9ZbLv-H7t z2KvD6)(j(^Nc^gyR*cGFOS=nrLP3#SG7;?u9Ps?w|k6J$_BpSHo;Ho;s~F|_4QDKTTWu&5)b9sJ5ygE8<|pCl*@8hgVr)7An}#M*Jkl2zh}pGlO^++QMO zsEM?o5Yx|SkRg+&sfxQn7H7y>7=JFxX{Xf?`a8wuPtj>nphO$)Y6rf9=6fW=S>byO zM_~(N0jjt#*t$P%wncBBL${@1!EXjTGh65;-lb&tu3H#^5v#AQ9tb)XLd$oN5>3xKg%;H zuhe5ZJ|tql9BO2r+WjCuO{|)!@l7W2zwGu02F5~3!VWNYw{sXw4{s!xFYF0R%S*DW z4pBAiY11x%fu~c5=9E*Dfplk?jRS-+9;!19yFc7uI_bNZ-9QP6fe9%Mm0N2vW9p!9 zOth~{TDaFB6!JyWVWOBm8R?eXkQ}S7+x^hARak?NqTIJ9EFOR@DIzSt^@wO+yA~;_ zh>>0hQ>8*}9ytaW#WGToP`10J=KRoUiz(nqgd!f;3s0S)gSJ+f%RPSLj6ks~Tp_H1P-e0Xsl+QYb?y@7vP&M?M zxFITpoSDc9Et5SZ{{dn2{8cOzlE&7*M9Cik2a$^q+RNXg;2a*P%9BMt~`h=?2{I>3mmV1@nXfB3Ngk$E4)T%xY8|E6ZH(#a5ULI z?983%-elu?FxEdhQT7!5NMHT&|&xB*&Wvpp9*2@XjDlO7S)7g zl@$y9iU0Ml`iym6jyCeYS{2k6p77!Zso( zT*dBx356-Y9Z&pVG<)$+A!zo5{RFr~|7!fpn~L2GAExs+p~<)Gff4P?dy|J(?LSs> z-rGgb2FlTDMvcH072R9}TW0GCph!Y68cPd*<}C0h5$2_((ofHEyVJ}KD%nD1G7}mx z{;pSZnnQvq?Jo)BPk!vsKyPD+u8^_#*JZ^&!OQ~vsB(YN`->W)791=62#1oq)AW+5 zavMh#vV}brc0T5Hqeks;{<7rqqJJ3+jzqQP-wHznRe*-;!rbICK;>}8Wmq#yta_ni zD4P2j+A)(?9LmU5d6d?35lsjCpPOwzdK}1iqS3mYyMxnP29*zC6HM!}*&b6$%A<|w z6XMRdJd*8yab);1?nCF%rfC$>5wSqi|Lm5k7H0o__dY(xEv4_O68(NEfIb_pJ?(b= za_nj#c^`*Z`?G39goUct#7-)+iCLn0V#}j8=gxpx*9~V%&(_N;1IyaLy?M(DPS4{k zL|fXIzF%bD)@{lYD?u0jzu z5a?RZfBpL;vHZX|Mcz0qR+CB1iqpb43eV{5YRjb~@N3P6e4Y-bcjm#03?13$StgIp z>%ap6fmZTq`hqMwT8>G;+f^IE+x3#&`&o$H+ezE|+uGx{?&dv+XR_lL_k4U^sAq~z zV>fo*JUxc(vSpRB;pYGeok-TM>hy(E*clvLn)_Q3&KtUQQ_h=<<8gcSbE=kHL6;eH zB)*b3wWVgwxXF3c1L%&fD zG%%*K&drQuYI}`9)~1li&U_NWhPXqkAiQ4A8)Se;vulXzOA}~g(aW7x{ZggZks9LZ z*?K%?wrPFn88JnD2trQ9TqqM&cs#mwcgqu?yUWR_zZbiSttNOC_@9$``LYJ2I9D(* zW^CmDzvo>s|8?Hge@*5;4q5z4LGpJ-?6C zF+le*d(Vt?J~)d|z3%96v*O$q9ZkElr>TCu9szk3=G+HEIDYfa(bh)YMlcd}q{1@T zi_enjeNd`~ow)PEnH+@JlxuqEOf}6kOLT)gk&k0*D!3C^+4E#ZRNL2Esm?krE+vrj z3XC{?L!J?^vBG5@tqBa2~pGetk?nS?gW#@Q${7*LOKeDz@uj)Twd( z=C^FJV~{e^!N9}A^K4?@#STlBz9o^d6%B3v_E6rCKC%q`{yK&^x1mzO?#K4DA2G?G zXFtkjkZAvA;2X$7Qj)Cr`R;~6ul;#H!uJ8~$z|KF%j@08hjPPUhe61vMu65^-~^e0 zKHg`5M^0FWT$e4CUdnk56=)J>=+1_XbWj-lSisMnwl7oN;$h>|*}|WFB0_jHK>Z{} zdBW1dz#sg*rM-%36UVy(S*B>;p1m?=;62)O&Xx^V@A2TsF9QK)Vrft;$ieM#rsF%C z_mZ*^2U^0BJuY{a!CRBO^sW~=L7T~xo;M%fbKYawMgb2t@|Z^4D&2+wcXp6Fr|Vn9 zG;eu_#>(yC_w(s)b-k`%y3e@` zgVCI+fObqsf+6SU6=QbEl+8c*`5s|Uj`p_NkU9Gtfb2jMN2BREY{shm;HmBNgXy~b6;)dYEi1MGKohFcF_u=C1`$a3QD8jpO1j>V zv&0i~WjKa{s4aY*KBJ#sQ`)XUO*iF&eZS4+(Uy8sho)=8KO$V+u18`Av0mXu@(LrW zBj`%Sj`_%E)xbVW)=L9&SN_fmGi5FVaT%===R)-8?9BXui9LTy`J^NLYOQcAQVyqJ z&Fpo!2D+B3spT9X-9)Ffwqaf8x7N7S$kyT;GRQB}{MYqqa7ockjsg!nrcz zD`pYgOGyQ?>=IZqwn}VQ2H2}aBxib`Mu}?4SSh$l;WVn*HN4ECOqRRco!2T__BQC7 z5121j>Xq~0G|0MDinR4ftPw%iZ>q+G^b|wK!_+VI`D$w~4*Au$mhV?}*1qYqaS!Ls z-^>2KWpe#&foJ695{`U?GLyS&_U4+Ub2nN?!2j$p%pO&4A}^)5+J;RQ%#`|+TtVKN z8QnO}^@e7@Q)qYz&dRcY263!58e=L$Mek+HA*u99jxjG)fy?bj?Am<1N z4u0p(Y7M$4B%UZg`-2;qrFv+u%}jK@#AS%Zk1g7z2!ncqUvo-uGY7nPo>Mx~Rd(t8OauOJJJo>cebrQ7k}VMInWpVTuisW8DjgNZZm-XCR67ybjP)p zN$r(z)SNPP96ev^?S~3zx)s{b0QHT?noVQayaaSb+c|YMn}@It?OCvOGG2wONi96@ z)@(w)Ymi_w=o7L}h}HdplaR+lpqSHT8o8Dqc$yGRSxV4tB~6NK2O#9TI{?*^i9Oav zBO)?52jv*L-cmxMz+T4MDlP-J3HcsTK&jG3W-qsQUH1Fjyl;5#p}*~$)MPu~PPXP$ zav`oxx=cCr5(Ro9u0uL0*PimcPa?u8+!d~-G0{H`53==9c!Xr2CV0woJc%gNbBmpZ z`pnksh{bBSS~~ZWM546rIgE-&N8TgiFih8kHSGJjO4--+PJ7O%+AGQ3dB{8k5%w5a zp&jVC69uPTyFp(?WWsiTg82J>aq-WJk*i}LeX5CS(oA>6SS+t39+_($!TZaZpYQYU zXCUaJ>-C0X#aLaY-uI%6BMQ$Li*NCewlz&2lS%9w*w%zA7*V9- zWG+Pe3Q}%R#FrXOke8m^YZ|*!vSleb4r0iw%R?S0B%>MLb;PH%$(ah2(-Ma+ETW*KVjQ~lVsak@$z zNX#=kDxV0r$fUS^o;OYMJh13Hz0RwvCQ#;6Z5z(iRv++=_AU!{|6}LQMn_cM%}__w z0&w_a$o0o=v8$}5UHk+L<1Fv8PQAS4Rm-DY_)AIxtt&v-XW(%aisz@BgQj9$qB0c` zh(I#5_L+EeV)jJGF7;^NnfrK>=iU>o!?uW~Pq`_E^DaDDFtm!`?zr!&E5eH*dzDMT z?3VV`Dj5Qit#h8dy^)vW4$9i&JJ0!y4Miz^xsAfbSrY~A<*HmD;F&}aHYA+ zeUg;F#3HL+9pdolHSCu*aboTq z)AtztY{k|%QOcUBSjh25*7F_q4lH@kb$`#K;+)l@6|>D%>xY=q;uz}H`G7TV^u>Co zh^3{hiC01lQDkc;fsP&va8D`y@LPD%bQJMf_f$J%xl}*4(G8}=M3+Wc_R~V5Btzm< z*@RjLROg&aOas`MtDN2=FRq_;hP{`V>8(DWhJ#hb0c^&G<#GY!((3QmP3vbGOVZVM z#HKD*$^3tsZQ}P2zI7VbpdBO2miavXlMr}4Z3eW4N6%I?w7X~EN^a7)YB6qcqpDiz zCaGeEDTQ0LC+638K1jTB`BNEg9-76uQ zD_GGI9s+rYiz9+h2A#M^(Vqf3yL2_+05v0azAsnL1WlC)dERSV3eSJ6UuSB;8A4m1GBZ&Sa`S-=qQ6MIRtqg%8wCH1s@k<1hR@&j7!3j-QSg?% z*LsI)e${HYRNZU_nBQgzxAB-a__uv7FWsNgqMoV3k~ocC2C@6+XAe&!>7p#xzN$qb zHUd|yMhByD*+!33HJ`-QAf79#qT8T2iJdwjv4nn>cGE%*0qP>%5@wV6mD zZ75iPP2|XZ3>Ai|@StD_plSP?_bR;6?2E5{yg_+*YZ>t*r&z%(SuNRJ{}5nWg*e=75fr-%UsXW?we zW1J!uKEj0uq8xL1!QeDy1@a;Yd~v99-pkf$&iJl8CBY%3`>~;(OC5Ki380nAfgLtyDFT>;luwoU=GCZHt-(6B5RKL&JvJAPQxItyQM0Xa z$)Z+qsj5j#QyNylMdG0#iAp!(aZpZ;6=nX!S<$PGB#T_`uTr5Nq!`Qu(~CIgZ(*Gp zT$q{ub;N@$o$kP`pF;aCdw`oi#_LQJ>^Vbh1l?^BGLI|jKtV&yQA?#DubsAGRL9~~ zB6TSL*q1Y1R`m%8%17grse7MPT)l{kx$^~%99t(+br@ukwbmeje~b!QB+>sxW(N(q`( z-g$hi!z{Pd)OLNf_T<6^y(tU@N91icDReYRxMe%|jKR~5vZ?^j5I+;-aAaXx@%u7X z;Mh2*NUkVea4VYmpUHZovv+YPavU2lXUw-X(ogjl-?c4p{(3Bz!6+st@XCM4Y9x|n zw^h%{pok3-6qmG}yQt@uj%=7JG zX3vomSkqFz(mMsN5=y>o=qX}14~ zqVIFZ_RX*bEGEfmNG1Cmc`|k&pv*TJNOo-gu=^VBxeLJYu7B`>8@YS*NHTlZ9uN|U zs9dkK=(%>a&uz@)j_r(}6cAZ|-8@%xA0lshwv>!ugzQ?-$)==JT zEol8Ik>fQtW_t`C!N+-=#c=<{emm@MkPpLR6nD25;k-}-!DHPCul&`2{u`m4fPM%$ zgn#qGPKHlX@-;YOE^Yh1$Ql1r7R)K?f5VOJjPGvvZ+3R)JXD+f*UN5~`>6P&X)MUa*BP@ZhQ$%?07rKQI5tmPzFiA zCv@?f(JnK)WvV2qO-@VF^iXeWQ{fio#fl`&v+ifsD2vkQtCXq}1yZ4dk_p_uTTzv= z^UD)MRff_Q5_Oj0)<#pERj^4m&y5n^pibO3&$EhM4~oGF-7lQWfFNl$0|W&L38%-1 zV~tns?*^6ZKMKeE0L#OKCf=v)s>M23yV&sag9!0Okbt99TS%yYd!@kn(oj~-O4ho2 zu;aBoJ!+!~Z|NIV>={VBI8o42hK*d^8o^;vVbSd94>XZR)OO)6m)tuhXe4b-31V~O z)@!-VwNjovn;+|M7HA&H? zG(RJ0tyI!-vMZOMZj$25ntTz>l8qPk5@}vxNY9Jvo=fJpgSWHuE|se|VngQ;X{nTX zFJ`P9N=7ZcdL90#2uh*BhRZ-Y1(4HT_&E75T3(<|3vi*3A{&A|z{xgwa|dA~TS-7^ zI#K@9Ut&}>BUouxy?cO@tocm#b#K?HH=wY15fL&0VZ=gBV$>N=?*rNkJVrqj zuBG4h=rbPsMu~Y+@rOZ#(eP@Iiu?Qm6MV6tvhy~Gh~m~WlEEn~zFY_FgNh)BD)y6w zX6_0ZZq3J^nlk3+G{i|U?B701B(iO2{_a3O*t2o0PbM%6_gX*W6st+9O$_JA$;X~V znY?<)%rDepHt9k_%=h`@G(daaZ-_CLsxN3#LtK-GX&^F*Rdk=e1e%lEjD3HY%Bgd` zKPA1{mA8{)&AGxK9oPFonkIeX;Zu4#WS`~RRTXuw$0E=?>1mR_^X=0*=(%s|D*cGD z+$?CY{;Xg!^lV833vKsg=40WgekC_jFo!-G>X66xiy z^jWyZ35*lIFJp298}IW`dTX!b+!X}x4>BB!${>Nx$1Wg2=le~Q-%FQXvBt8QmOsVa zb*rNt*i-?YgT_w{-r1g-7J?@p}^S+d!{=hyEZxE8qbcw_&EHh40#qo%5H&H_rFb53Ukhgkr2Z*{o?%DD^ zZMrs9X0UAR(#6bXdSHuq-NeCGd|UJeU$p);zpj7577G6bQ&)D0ZB`M9q`w({PzT>S z=fMBKqNEV@9&Q7E!;89o$vnSb_u5#}gi$=*O;vY@;S_zvG1EyWUK=*9bDg_ibJK!> z%#ktEwVZXOo==x%0vMO0yRiVRATTkod|N4KwR<*0f$`wuqTMRn4QP7&4)hPu*ee9lmM(ih}yK=_oc{O=a!QKwqhxFE!L`H| zjq8UPBOArvMQ>?}Y|Ou6=CddBYroPq?QanKysDz^bsL=3MKw?Bn%!RbkN zz3zSrjF(YIDmem_!?X?_>3D>6G0y-ZZufzB%KERzQKmU_agY8-dCI&?<3vaAj5smD zX9eLDMTmo$nm0Jb_Zfd4mHu$hVB9qN+KY`fVe*w-ptNx(;n#R9KN@gNk_nQ9r{E4bhC0*KUq0O8vlPhAg zp(APL=l7g)?2!qXc^3LQ2q#gn23f%oST4h7u%H|{3$mT?#)joy_vjTa~7 z7Fh?yucw&jsf+l#WzZ5v4i-58zX=vh0TE=a^%IF_b?=ReP4MF(iVHmxnvv>A!A`-L z-Jb*&F}sTT%nu$@dWB&qYQCOZ2DiP~DdmLXChFgf4bVn>Y{QIHKg(H2pbLZ<+zv$D zXfSlTS8Unpyn^s4Evrul4Ov~eadwMSl&*iNucg{4PC<5&+? z_h0B*Z*F3u6;|kvX7#JXvr?HW>ZkrU(*?N_0ZjgX(T*fDYJT)YL(67Zt@brX2bsQ-rDLb!-ivkI%xn&uaI4?A#vnGBJ)BUyXv4>^SrwrBTT}tjR4(QMJ47)=_Ju)emMI z)TIy;K6X}AaacOVRt(F8u*l|k_5EDxFKZB1&!v7dH56vjeab|iOkc``IG}v?z7@yD zMbUYiaP}}QVM0Yfczt_#di?}}OwnkJBh;ci#Roix}Bbv;&8 zGis%t(J*U?;c*NT!jA>*ce0O}M6w**ulI5-4!SIQyDVnVy1IHctzT0pE>&zkqt&&7 z*N?FJ_NcI@Mg0@tz(IOiGFy=|NPm4u$>DE*Fa20pBc_iNkq20aVBJZ4nS6*O6M zkEbJg))n1nP<_>y!=W&1x;SZQ{7AD)$lGAp{YZlw!k0S-;APplCJf!z8=aE$Dq7vI zy`D|+tofSw6}wkZ?uaZ@5IpxeHI*1+(;c(!f^T2lcv|VH1}PK~QoqKE{0?PVMWG&R zXM~z7YPekVT3M%RHI#sIL#<`7MrHS)WiKr;RwZf(GU#QD8DmU~HSKD?16ky~E!H$< z_vWx+G$P2S`?Fw3C+z0GW}0lWemc5kZb)h1DEHhQxs3)Z+PJULp=E3Ww(X|0zTCg< z%39f7Eyu4)c*G=UUa+*c_8p_Ds#!f*01|b5CiC4AI|~h3xGIAedRH>MOl&u^8g)J6 zWrVtjQM8L!hULgPKoQkaE=r}!e`?QKuOkPFitD+Zf_$WnjRy8v&X zt>={_g8r_dD8wmKwZE@=ppjzEa1qrG$>`s*6FEK8902$ zU~_~=SAi$cTZ)qW*|J%6+Lye;f{>>Cdh)~!FLgok*aqJwD3Q$Mhi6Jcd-#O1N@Qyp z^Z4F?MBS>7@#2PApYdNoN0Y{8_Ak4ziYuCak=Dq%gk1kPP9b61&JN3U%5- zyuL6nzn)PR>u@G21FO9BJ5q28leEIOiN@j|DO?oHQ*5=`gY0#x-_)2? zp&1 za6AeS2bx!$sHO>afZ`=(iJChCe%?6)PdGz!@HAuBClvvSt`3xHQOxe-M zsb!yWc0Ov1Q&dnK?N6dj0*3>sRn_hc$!?dro71%eif-FT-YSZslZy zV`Qa~5p28P30lfX*S*~hj7{7#K-z2N6$6)Dw88TT7(Ac6F(dmTUKxtxq6FMLNAkY9 zGUUxf{CHZqXXt0w(<_ucuQl&S2Zp@^=)n7M3Ab~`)n1UUR&%@W$O!j@tELr2#>R9h zhexEFIvSAB=0@ky^oT|6-a~79{!D0a#hYlCnyY0d1vPy#?rCs@P4nxT>+D85{{}6( zUpjitSor1!4joTnQd}*jRQ|qi$HT`!HkTW;&TxMXh4icXip}fhsbk;XpJ6 zKUodvadqDQkk51Hs%_~6%UZfRYGQ^zzI(Bt|GecBoVB=mv}}>M2<_InfX?^Ca=#oXv{LkQC4G zlI1=;tPDo$+TqO-*j$*yAOMsn6oRj{B^9lCEts!Md7%MZxJ!aTV_%!juJccSal7`u zs#jcqLS{%1hv15yI^7y^3Nb|5Y-ZL_ETh`1VCJ?hIXgGHOP~llthBwcSnpgG+R6yp zkBW`JG$KOjyaz&==MALi&!3x$kA7UQ_vZ*S;Tvbx9G8glwBH6X3WEi^*yUK7_yx~-G&-J*IAcy0X0cdAU}YNrqy5LXQNM{}zHy2as;>?{5a8O9I{bDtZ!)OG7KDYz*%?fJA2_I_Gnsr#DZN^(|AQ3boK3LnWr?%k7W~A4kC3UO8h2E_>3o-KMv> zQ24axEW2sNjsTwM)NEHB{me#qgHeNOi{!x~>Y%kq0XvjK3vFn1W=*oE23MP>4(M%3 z#`G0XmBG4z=uBH-Gp0sc#-hSwk>lNU(FXD=!c5m=#qq{P5NI0Yk==dPk{uuHcv|)d z)H#&0WX&i@X-@@Z5mfEMAc;pdJ$1ZjXEON)A{UDCo0(aMv9Dz}hYOW7HflaMB5ImS zI9H&5)s_3IUYuWQc%Fwj#>^Aa5BgcbW3QR@t*<2Bx)Wiilg$<4VO*!%%_w<*jz!#6 z&S=F9b)1Akt77VHncc-~%{kJBMIMVrQb|&jGe1-Ot0AT->lc$ih4gQlN!tca0k)G7 zD_lXQV}l;Ma-@yqBU0tc3hD+BO8{EH>CZNw>#2JQPMl@@<|bX0@Zq{pfuFbi%m;8$ z0BNVve6l&4#V(Lb(_|t{ z0`&AXZ8@9!7oB6_$HYIU zoxNq^QM^}>F*}8EI)H^EVK%S zz@=VxHRSPQN-)KSF1f;yIeXPpbxH_V?kFeRV?I`F;_E|Hnyj|6Fj zrbVV!fE8ytJAB>SoF9T;3B!|b;;Mf9s~hnB06v*Vqn_5UGn()|3a9uO-5*E{kkZtc z#;*`PwBL_mZ73=yP<5rnwV+kWR=>2D~=l#3<*vUN8l_iJF(a)L-YGiPCBXOjPl zd+J)_&B2+o0a01CYCbuUekx2ku@I_>3`H5H5P_yqUr;NTZu)+Hda=eG>K4YIt{X!i zzJ`x-S=7*~(SIDQA8H5OA)vXbAj^-qyK$Wvg3bU55CU|TqM>PIz?va?i>-?9=hFK^ z&gT+Tf>zHZk3k8a$j6lEqAi7l_=ylc7HSyW82MA(PsKvIKv%E=s9A6eqPbZJAv&d5 z$`a^uMlW?Az$o2r3u`~@NLT?sMT?wdWlUVq*HLVZPnMq={a2L&H79Y^Vv+nC1T_ii zV(;z6$IKzM5MROU;hUk>!EH&_S0VD~oMf5AVR7o)k(#x~M`m%(Ll=X{6sO~xM^_R( zftn|ub+O=cp{BvjjZ%~G;>0xS9_xBt`}l41iq$M6Os% z_Bb*Tk?NIFL(xEwbeE;s`Nr_mG~FRgyYa)HXJNl93s%)B;XVxc$DdyMH4Cb+#!8TT z;EtbK{MXG!1WD);khoB?sl054@o1v1Ng30=N_faRin1vPUCDD5>U8fgb<^tK-|8IM$0WQRPV`enqVv40gN=!C$A;KB3 zW{}u$@NbuazH|c@<|e1AdgoJokEgCTIyKJ^9qh2rX%3i&(ikWmep(Xyl!T&_Q)nK} zqw|KgbGAt6%dOGF^O`@Mpu$(Z*a72WY#f0jMY|fTyUU6|Wd*1DXtUxooGIj_-vmX! zd9XvuKx^PAi_w5A+poHY5%frRUMiGT5wM^CSJ0=ZgE2dY!K)U#c)(<7swkz;;evqQ zXD846!3Ir_Vn>AuCni71rKx>7Pga zwVFYQVRfy5Bb4BN?vncDT$uLuBYC;HXgEeDEXOcl*=n@*qO!C9<;jo}r=PueMW8Dj z&kV|Qr!C0WuS0_T%KV4_xp^|kOGIe`QPhQEz`o&+zstTZqCikNqq=KJ?fbe8&zwxO zK_$DH$)Cjz4uI5^19O{Z>5RjS%pbHhHdp;8`V5)ls*l!44(GEv`l+de#=# zWBWa0N6v}deoVND9Y-JUQXOYp(HrEBeP!>b++!VQ3J8I8ZKPrxQ8%^>_5sTW?bh8# zB_BUq6igzF%M-i0bN5!2(6c(u71!s&qyLQhnBu1pL)$cZV`YdL;yQgpHLviAecG{N@FPs zzueYV!AEO~Jx8GH@i+~~2hj0UPe40i=W7rpU0de4;KFesGeolwPrzg5EQ$#JzhXaF z|HOVEHDPA%@;8vMTfrs&GxFnd#PSVm zxm2jPs||N zu;dVbb&kqKr*CuCQaN+eVr{s(l_aqWeJEWasfq|Xhi3U`A_Y^m{ddO;g|!Gto!<>7 zaT8yGcj|O=loWD?PrTsyuXorW2J@&u`q3n23#xxBsL9za|5ew5^xly8Iq4g%|m` zwEP?q%IW_}+PF+XzWc2ZdKJtt)&i$|mGeMncBH3~Vs0mLkXG9AKgWMAEd?#}ivV)g z_lIXajPEOPOcO$RZ`=oQwHt5a&w&*6p zG%zMGlu@P_m%<$AXvluPFf?hmkZdJG!S1-$^TwEoG{D7^urr=|TE;5}6K?*lHyJbW z!X;p|V(*CGHvV5@L{E&#u=gQr+*y^6JW$`qj0GLK`c4Av%$O=8GNev4v&ftyYaf->^Fjs_ ztiw>&EKSgPruT~H_PPMK`s`uN>Rv{=ZQU+HaoE+Ry)skFN34P?N3>(7zn(iX2l!7T zB#IB5G}ii{IwinT(}N@cKcFZ@4li0YeuZ1Ey{e5Y0!9cE1Cr%$JiILNcDLH5l_-l-1#=qI4`}@nB zw`x>&5_A=YNNEkZ4q4{l^zw#(W3Fxg=WmAc9adm0v^iKKwK)527c2-Sj^h*upI(03 zNUpKh8fA2T{j8>m0uI+w_d}IDb{+aVLPA66FR_wtd~zUATA^0yw`aO``dXcXHD)Wj z;wSekwal^jhi>pVRYyao^3XVG#&7Mc%+^|}9Un7|sz)VlM9@xF-wTQ180ky|FKHgOd4>;2$)HXT zFDqn#5SC0H%F#j`&#Z@KW^FOH>zIF28(-fJO2_@f2gq!_3VPnhg0|18ir|-q@wAY+ zzU&kn*v0Jx`%kvAJGTmY!Ou8Fi?`tZgL@DpLq6&XL>mNsa}9J{PxhOjS~9w%RvrXs z2A)$UGa@@+MgK<}yxfg{4+a7Nte$E4XEJ{@oL`9BZ_&?I+d}{FslxuZ`O(*mOE-7q z<6o%Q_-Oyo{24hsnSbD$Ur08`RMMUh&p)?fu83Y_??p0|J<7^0JG0N~N@l83t06u#ln%p%VRD{N_mh0ZTNMk&X^A3iTIvLWw8y zEV>Rh6H|7%F+PXPw1_2i7!Lb91?cTp*#K|04Jpb^)b~ zgc9@LEM1jGGt|$f&p=qkB*Rm4v8$~pc_*$znmEc*ld;7Wj)c`Ge#NBW0L?R^%sZLV zTG0OC6%rw&G4;{~t<`tQ1XnY=TxaVAIgh|5WtYvh-_GCnL3A*NOYF^qO+)V@MLl>#2p z*NXPAKNGu_q;}`t8=xfv-jEA2eZObm^F`BE_ip zBBxFw?E1@39JW>sk6U z?FW?NT`TKosYar()5y>CEPPVsk%stx_gWm(b}jC#Pp?Z9cEC(!q^J`(HhQQi_(ZI~ z!N$bp{)?xRndV4iN!Z_eQ*Db~)sos*y1N#)yMY_doaay9vljGY`MEqZXKFZ7WzQuQ z<>v+R$z<>21&>`SN^f0fgvFmPxf@|Blq?Dsy%vsgJQHG87`k4r2R!QPZVm!Y+jVQO z&#KkfTgd3?!|a2ZNIdq8e#J?t>cv)#dnQ|*l9M#+SOO6X58aLll%&V2(-jkRX{h2PjomlakjB#w@1GBr3$_tzO z2)qKH+30v12fL+YEIQvqtpw{X=s<-LeP$8uhou7QNh*1d+N_nSaL=_(Dd4`-_5>kN zNAICI>UX*x#bIg4rrk+#OJ+Dq2p$PiBdT5ErI3~cBsvkG#ke0`Hh!JC50Sx&cSk9_K-MwnnoX>n#)oO+t^T$5J zn>Y?+{B?p%K}xc`HJW9Mk$lOoi3U-L67j*~mNNZ@IOE3$*b3n*-)^}r!^SU5o^}Gz zzS1SrJYj^XZ>Uz1wJ$H}ypKl(;$m2S;QPwW2)>O!yt^??j$eNXt|9+?U_48@d76 zL4XRFn1J=^kos6IGW0I3ka}eM_9kG6b_9n4f51t?01?^*CswlkQ6@roBk*}klZxcp zKm{4SC{El6|6J+ahTou+rM>}(<7;1EO< zd}2;vhmbwwaiX^dIr=plteoO)0D>nX;}+H%i*u$ilFn_SNF ziRr!em^Cnz4WAvd_Wt?K1jjv%n4HokCvb`w44mZWMeg|STL~giDrm_jNyq%fYS%a zxjGb)x4|tIJr)OY`zo93%4`zjdY{3@efQ*8kXx;TxjaF8D2p zg0%PWdEocVz?LtvPC;YRZZ%< zq8+3_%^;c;MO)rd~zqN?ubLA`4R;N*GPf4jZo)KhoX)cGp^(=_)) z5~=9hbV?pBS<{#bs(?sHi!|c~EQN$s(t1#@F=n^wX+`_*npvX~+8Y1cZlk@o>$wlf zEsJ>ZM^!$WGp2;=e??0y!_qAv>`X`AIptHga?4;# zf9^VXo;h^7mTqc+Zk9iA^pCPM@(2lB>1BMKet%z1b*}gc`B8=Arg{4N4_#e&6aRe= z>bry!?Jj4y=NGp|nT_$U?F^JdyGMDN-Wyf(L|)@$ip&2p_HG*fW$dX{siaggPNhsb ze{(8(Ou)g%T!v66KR2MuKdne+jW2gcQV=ua73#JQ;pV#^wzAm#MnmIO&#l(0Q0`%9 zVr1Pg<7LT55d2Q2@xNWYMAyGuJfUQ^hFI#V+XM*>Zw(^Cx zh;XvwU}Nbk_tUXk?>Kalkwu`y536Qn3*T>Z?Tv6d3~!zunVA-5Yl)ivm*<7a70Q~g z7ujqL@SCQ0mJRb7s=VWwMkWme(sV*s5sT3j6Y1@_wJ>VUwu=ZY>Q61gCl60e1C`#T z#6L5-RZeC;8M-#gl@EW{uGnv6_<7i#!_Sr-!eN5K>S|8;X9i{6mJZUUX&%n}VXsrP9Wm?$G zppgJ|0vZt*Nvyn-9ky%mch@}oLQ?8wlIoR(p<4`hXZk^z#03|Vz4{NUz>L{v?Fx>* z%Yy(y7$ZnEZyTo|-f7G(Yg=&e-8m!>L5XlQm>U25C}b71c$M+y0bk(NF>e&D)06`* zv`}>10NOgTu7bWxBvF&2=#mjneI}6OhlOOLSh_V)^_@_wO+BGy@c_mMG9EFzuuaGs zayyZq5JWIEA_}0v!|xn$3C?s}2A^RXD{4yJ2Vk88EPQsyA=2`h$M*HXVS|65F;M8s zcBO`Jpx6`d2oHt+r$Lu@hkLlIHuWPKI5|Z68LW zu(g%vX5s76_|FA^h~WY)WKc5rED6k3J5N*9VR)8WFPJIj1g#4@FmFO9+6h>23Q)yh z+@e!8hGUmx9tpmwVH`wYj4*Mx(6P`tO<|2vt*WMbNIp6^KezkLKd|EBCPS>B zSVkfTpS)c6)jP+ow-P|8x~u-psx1m z$U1I)&B@j!i)0qr?ptA_%nEJ9cuRa(Kwplyg@!;=MUc%ttopX7-FVdm!oIh*{_2AE zsVRNb)Ke|hgJ$41iP=%HHj7RdytDU=ANqn@}V?&Hn zZ!I}im+zum1QA)ovpInrIW85b6L&~kY#3Pr9U))NUe9cD`g9mcSH0^LHM`H;whXX! zxfcePx5qzCFMPbYoMgKqLZ$1vMOWb((bWdrxak2skC{&Y=oc&A>NB0JRubAT{b%!I zC}28SycHxE5r~(oeN3!pSpR2tgq!d2ld-k}xiyh}#FQJ7iD)0ICuwJXe(5eiy7?}; zn2L^GU}zLltn^_7QbKLrF(z2>;G>1&>1_*QKh|$MMCx6@X6ZeEt7z)omQ&A1uban+ z1p6Cp@~uY0!XaYxPJ`V~OW&@)jDRUH_(+-XjeB!le#oGWwk6jWhx9eNJMMQ zG3(GT$R7xy*}zgbZOi~vATHt^mraHi>|E4UB?n4;is95TD`?f&ficwXYlwCfpae!v zafYYcb246j0lRc4PO7mMH6Lx<6+B^KVol;HscU5^b_rJ?KxxWU(vfzV#Bs%lcg!x40ab2qy>wMCcS;nLsa#g2Z;_reJSiB-{c5F!-39v<_OxlECC@ zu&ZFteimh6q&do8QfZ3iwRO;>6frzOKUYzYW%Z1R7qnOG+Z?E(4~~^lzwNXCl#AiL zhe_)uSvY_(gp5PXEc`WO3^|KPP$(b>8UY2+;BlqaspoPaHHv2!jL1=fh6qXlCJT`Y z&5mM2ye8Zh>Vx*51_H6R`*5LDs3?pRx8^T?o|7pg(iTANyY4fNk${|+akBgNoXT5?)whnhXh2F2h8X2EG&PZ zL+|&Q`1Dx7NV|us8l!^t?MQuD2PWE)=umLxT^0lo!Z=H+*#`rCBbf(+(}9I>sEEKQ zV;0gXX#;mG@&KB$qZ~65_;(d<&zn|F*U6Z^rr0Dg*k)fzOVcybm-jIyTZvmM(8Rgf zvWkWF`vg>1Gb$~$6OI0AR0<_VG;;TsU9{Y`eU*aPHXQ>}QF6@C_I%EhgH2W8ktO>Q<&iR~HO&$vS%r5B@ zL4pu~AOY&wj0M5QfKvq+DxcFf=QXag=aS9G!C%bb1UTCfmC1Y-r#X&WrDWqfMg!_3 zYBlzY#hT!k8jygS&K;si6B5|Qn_Ry{Yz#O5nOWN%Ylb-dGqZ+KtfFPnW@+~FuVO~D zxptoy_bCFXnE8yt50%*n>HsKacv1fpGq-<=8F9+DVrKWPnA!fjm~pmuZm7Dm=Dxg~ ztiAgA&G2e!H;<3?<3ym{I6;mlr?G+37IXQ8Br2&v`p&>b4HU1UlEY%|l~LE^F+$<` zi(V9(w0sT^Og-RM8w&9gV$OQMb1W1|!hlss19x3#skuo+<`t5QtQ3&veCwB|;b7}@ zoSf5ZTWi!DB#ds0o6zqBp*F$a^c0ZZk7!7B*I+?{AoxJ?gJMIoNnONi&f6 zjJBDqF`eC4Z(jBf&q_CcRyZg|Iq4U85Ds=HLH^b>K?buD{54Gl%^HFt7K7+TPYHX< z7U6be*ZWlTI0xeBk7>01S)q8q36k>z0dni`G2zdmv@_DN9&>t=RMs<`e*<-ilmFiA zEF@a1!?f*R;I)}Hg^L>@9diWaX%R2YP1=jgGZcoC)C^(Aq%&>+O79;N3jMIh)x6g% zFj$&CY;9Yqm;o03G?iGYAco)-pz}XUFL97H&FWc@xR7x}5V|1s4ZucYXtphFG^vVy z^)L?xOIL2?sc@j@ex#v@r4lMONVTD|CsJ1Ko)oKe?-4zP6bH}E3TT4$shKf_{L??6 zV`Ds#2pKHse$B&)nr{Jv7`2Dr7L)8D15sE+TLFjHh zMU~-H5O?3lt$f~^Lhk868vOGr*-tVo<@Lt7*&1dhC1tQR2s1Y=Cq}GI#&a3Sk7zA3 z&oC*;Z8rgP>Bx&{`tBW`wG>ai`ADs|ucr?;0jUSh*xxU-d+?(8e!0=O>-6pk5@;Lm zxyPh;cH6->QZ#{QKmg%!=yB17rl(GnmZ0=y96mrBy79vXFvw0sM}l+jfOdh%;I`De zb+A*`%nCq>7x*jN0Gnpqcn!1;+ItOaalqsk9;O@95>bbTFarb!VHJ&DDLZ-s`AEMw zOZ=!rX<^v=^mheC3Q{7vAJ=ggrSEDE`plsaI54kpfw80A((0=R{=aK+eXVE7xmvh` zI7ePkRG131zu17u)e^6dwri&VCdPLCM}k&fm7{q5?+h0yDKjbM?OYopQVhWzQs<@+ zZw10)x?c&JQs#=1>J-XfrC8y2Ff)dLI2DjeV z4~0)PAak5gxDitug|g5_uN4%2l$RgGx?vRRfw1ua#!%YF7=9Sb0~Lsi5{g!yrr+k( zC~wk1aP}P%6cE90(xHL&juaTi^%e~E@4X#A%KrQjxN2}=e>>7+j=;U!P9di{%QJ2W z?}wd#75A;N)-In!-mRM?K_`(~RwOUfo{ntF%mk=6srY7;=kz%0s@wb$!Jty709NehtwEst$Y!^dw(+34J>rbkX;? z&FRZyO3w|NvT+rvRx%Xm6X(}i$*-4;+r{HlDYY@?lse|Cm$Vrt(N*OaW{Hux|*dGL{+Daw9VtP2JhIHH#*l&9*;=9xSy-!-&1+A@RW0 zeQeC3Ya1cgvBoW(98>BSQ)C)MIyqgBWoMSP%|?&CN+(T08Yk5gU8}4cC!?^9>tp!% zJcUTk=1m4T29t*I$$ApeW9q+m75K#tJjU+y*ja}uB6%1lK`Ec5x}^86MNPzRNpE#g zszY{H_o=ra%-*yhPL4BpLQe+M|GhzE_uM;uUF{w{xjmc={^4}W*pmZluRKc9VR0qa zWp}&59kNZ2i|>F6j3y}W*Sa`)DSY(z965hf2hiw&LJIy%TJDhou_ZCsKv5YIC_l&^ z2e~iurbpf(Kmie42=}H(fOa9>+X^jgpo`1s!_!#URgc&@(Rv|Nf>}R6ski4 z*T}!-P_`2*%Rg#z3w;S`L-r*C6M_hWhDX67rWXb@oW*Ry2kRA3@L7w8^=Tpee~L}w zHSQpv%Xmokyoz!aWNYFT;iga*v~%(enXZ%&Hk7{`b_RP==;F2#$A!ewG|a8&2kK3x zuydpmFneM?zKG`1VU;%&hfoc4G^Q?Uaaeo_GK5*X14cX%q#rAm;UAxSYf;wC-9>wr z-UQ;WGhCnxn77CQC^LY%7KSsFw608aCSd{R6KdewF!*Oq_)E8D#sNd|@i5)&Bjn+) zZMoLY`I4o5@c^B3@vPdarq4Q3g8l^|Tb!uQ+KIwggIfL6I*HRHfh!ARw<>PVp9f`< z7w$|#>c6Z4gKYmc4fZY#0*HVgkI7wHJ@pC`#IGIc4<#r|3v2iVl9>JPZj;qUv8a?O zMRJ1#VZuTiChHI-SdNoKz^CARK*@-7qY8*IzqSre#kI@|AFwzMK{ za(t!Jdw5%Ep!HoIIwmfE3J#xnMrsM4>o0%+R__1*+eGy+fe-gwgzw*b1yf+%)+LmtY* z5z2!&fuNC)zyLp0Ot|ZYojQsRXBM?4tigaHlIO3y7Rp@mdW`VCFk5|f^`9JfD_v%` z6UmlsHMfpGVj`)8e+UNry>V!x681*Rc#(T?HGqwIZwf2_)SeXH$`C}g9~H85BI1TT ztS89{oH6c5Z@@kFqVuOlKAzAUpuG7+^tQlIHhy->I`WePCcUo0UC>-TH%m(bRu8j- zHsHUe@1Pss#0c;nesbx>zaJTs!j`hSmtB6$v(=NV83gKBOYIAtk+M~MP^j?R3ak)b z;8ruQVrtLmi1b0W2M!SeE&rM}1p$B8&$6~RWo-%e2gd$0 z6@_}hiGA8Kqe|d!2z;*6q)y7_rOz%HAt@VsXDci0S7f&v$SjEEt+kJ_V^cwOMBtkJ zuNZOj79;qD3`6>mV~B`_go8rhQLu>_g#is`@Y9hZS5Bi|*Q-0yxXYFHHu&DHD96&L}Q zDr4GkMM-LPNbkbT+n3vhjR{sTpk~^V2I+?aLTm&gqz5_TpGkhz<7Tmwx-rZ)pX_ZW z{j*bh7cg0WS%&}YS&OF-wIx zKvv>OynTlP)eqypml(--wq(RxDn+0(+`(A<8?w^%vWjL^a96@}pNQ*WO?Q3mh$mlcLIa?TFP*-wiRXoovm=2cCZiVD zp_IxU_OBM@LL+D_I#dW;FeyKY>;|p?X?HWyu&-ZNu$_UFC2KWWgRHqj8UxWU- zPOgSv7TlCg2@e`*2QY;$aCxZU%Cz7p!<^Sa;etGi~tNzPR z?E8nGDEtMF-U?yUY4(E_NyBid6_c~a+r*>`MH&7;L+4v9KMAPizx*{ZY31z7h>k0& zc>c#vRQN<`8NWHz+yD>9W{N!ViCZuEolYDXsYY9abn#%G^vIU(6I>? z^4<`J+bIG^eEqG;a(8LH%58I&KcVy3f!&cn6RMOR+hh|7O{7elWWVxgnS8@j+W5ML zw0!L%l%3@S%A@LI;PeMfkG^cqq@ilFGhgL@mLnEtv67j=-ym#XjWQnQZp=+Ma$ zsga|f^X$<_?(N?1Pr&6n%%QE05}~?V{~=a2b!$X>bce>b(Qh7n7QjR}g! z&@LGxp)eIgSgI%F{SbjZv-df@j&c@wr$fHt>U%vHDqGcEVu+71yVT zq*O}fgS9+5d%DQZtkdU`V0@#pMR%)})kNJ>ZC|nA3%R{Z-MQKvb3DCSm;GxIsFfjQ zj^QLn(P#L-T-K=|A+cE(bHrx{Ruj1wEqQY6CyqDxabNJXHxceHzpIOVMLeF&tB~#! zDkZ71Hu!Ll&An!eNDAHY# zTS5y;abzR$Z+_V2Ex$qv1yM|468r`>zlAgc7C2tw`*fqkIo$l=9UbbJxF3Kc|eq^Vt^gpPTf?9WR9Mzd~@xXu2cGN^_#4U7$BGwOL9+AMI zNIzp8plU~FTv_9ALCir&7sA=?y_}S=QcczTiBSa6#$kh)+Gz~F8x4s%-#g?mTx_s( z!W2_7N)qD<0*1&GgRmF~!Ak1iw z>4P)Y2bA1$_pq1<`XShM8!f62NPV4t;1|)*AG|f{j|W_6BMj#XA$$DK>DRt+llleR z!jLyn@8xZW<8aqMZv@uvuWCu0|uGu7SHmIBh4PrgC{PU^#y9Iw9bjwMjA) zaiC8USB9_^nDSC)37ZK#(7$6iMc5T#+ueAlPqQr?h8h|d7W+f1LIIOXWmkZkF;V-_ zGI&IyZ^&*aAZ16oUb<*~x;mfhA);^}ylHA2^TUYi>8%eF zNxhwxUFfyV86J1=IOHf%MCImvf2yj6ZE3wy82Ij6Pp2TH3gMUpTzWCir>)37eEOwn zM!9%cxwylvFb~DQMz1(euOqG}#3-(P8`F>|YpeL$xTlRC6580)RUzv{_lu0vSn5C= zV?)_6cZ%R=xanXO=L^j$v$z2>MY6dJX>}0$CP)8Io1Ns!1J$^TbMlPH>qriItK}Il zR_68?thRJEaR}F={3n>&RjhdqfYOJATlL}ofCPuwCq(S?Zqj6~l6!WEnMM4b8aHY}j-Q=O$(D&D4vZzU*~hx%f!2!Jn}lgdntmJ0Sq}R1afwYX)-`WHNKI zeRi>12!3uqdpB>AE|?hl9yhsiw1RX*Q!e(V*oA-z)3bZ@h=ny6Ze22ZCDuZOc)~z=p>>D``8{+zDCP}N?CR8Muc?#UHXMJj$GRyZoVXwTIvyewx#uJlE1rtH{ddE z$dGB^N31Au-1CO?4Jb7GkTk0WCgWT*JmajVVQC7Y!$`rm6=mP(I&<;jG(>J;A<4*e zaWVYz@x6umg+zUbK~4lZ(vzneT1m9an9pIGTMpi`vEL2#c+nQTh1z8<&KO^)uT{CEeNwR#=Vw?PikuU^J4i%t43i#Y4do0$Ic zP#wFa>-MNheM*f}1zREE*GPn6#Hj;Yq}S?VuB9+hbXt;Q@9?&)9OW zrJkZuox&ll)J+}Y1py0A8VWiWM8%C2xM*;^Xh554fu$G2j>Bv&0%Ne9{hKhcmFKHwL?S<tpL65fbTTk3Wh*R8$j|#{T_ikyCw|6`%jXatng7?&)+(`} zk{0OzD+*(wMWq8HSt9>w@FlgJ89w7r=q7m!-G+Zcx94ut?UMqXX~e`$QgEv8?%x4i z+!_gDS4Gpz$9@fuJ$M*$geZck5&)c^j8yG5$o8EK`-tEW|=e;Q_o5xNejjp_vXC<*jq;dszu_O$YMydYn5r(~!yX zmze6r4)mf++1gA%Fz}*-g+cDo zqyQrtcZ3oULjE1N!*8j08%v6F7dW{apJ9U&PBTKld5ZB^;ghVzp|H*ec0H=TT zNb?@_PRn<^QQf9{^>nxAkp6aijJ<{$yC;C!8zM05!ZM|zedIb-yVj!vV|B#6)?+?O zX@nZXs6kv^(e@3clHH`gv}L2q6ST;;fAttU=0-Tdc#+8GCc}aULV?eRJ-S6d8J2LovT^ zO&S!tq2?0}te6zinexeLKTb5Px#s&lrrHm+w4XTf4PmaWhaj?*jy35#VRhh1wJaBB zo@uwo#pI}~PWVYV-r}3+YGeE`0a{A+y(kw=;G8Qrlndg&f9=X4UL@d)V z4^7vj@nwx-l1l=&yukN`yn+3WU`#Y;5AoTGwH$Lpb7dAs=Jy*FXu_IlaMcP$s7^_J zrT@w>LTg_hiNIY@pWWf3p)clmXgsLf@r`7uK6!v&_;7z>|9}(WQA2$+^xd$ zfgmLK8FGUNg|tbIe!V2%i!2lQP0|@}oT30Cf0rJ4lU`@^9U#yrPc@cInXjWHH>XsL2Lx=uuQ3zgNu9nM@VQjW?78FQZI1fTJ)1G!kh9 znw+bY{Jljv4q~$Z4*h&4Ageer^QA4-ybps3Tw?u-nA~aTzZ2cVG2WEHe6tc`g;lAw zS$~X~Uyj7zruVa4uQ9qM$S}IbDy_z;?0Ze(h&t7r$Z7VT>?lUbkR11C$4>&=dj-OgHI2p5rV6%=^}MQ zq%u}J8U5G8$_)|sKXGyn5Ad6eJ}th-GYP!dfTsc6c@khRF;%B5jD1Ma{&o+XGHY@| zu3G#&X?OFs^$TxfvKL2yj@TmKvh>b9-1&K_2*{yiuKUzC{;;FK3kmzH_71c(Aiy`g6Gj~Tx`6%9yt8S zb5t`-UF|g#$Ej`-XKFbYTqJj&@u~p#oCUZ!+bHIC5S7r(w=mgZlOlz0qtvUZA#H{s zKgav&C#*^5-t=lB9UUQJzZqh9nRVwjDQ%;xDSmIIi(@m!TDCo(_VW+E?fvFVt0%cl zDAgVW1;VQx%Yn^6x5K(FLgU&DR<|qiNZd)4-fGw%UQ8NwECnB(7rq)!hc5)-xH20O z&I-JE^+tO)z1pBGPhU)zNypv$pKD4~B~2Fvd95ov-eMlLFnLhM< z>{V>wMc1F_RGZRtVU2l)=+)_C$!NFi$yG+1;_ZAnOs$!$J^j)zcbdPd-5qFDsLeu^ z(W}SUI!{UHiEa`&J^>cKtID^vuU-Ax++28kX=((1%DWQK28(Rq&7!I=BvEkt9s->?PLL)HWDI9HP(GeVVCTL+vCB zl7jz^Ma!V2U)QC!2?qRn*Ryp));3?|ZiU?AVWsW+VK8G*lDH{&-Gs9HvfX%sAax^O z0+|((__3?I8nZ(TMPo88e|cXLhZHC18O<^+X&>TW_Vbi1E{xh9vL9r=k|2CQNv&9% zq~cBH$g7V=q__kHioT9_r8-SkkrIf2y!?e5o*fG*bWMDPH*(9!VKui_(!AizY#uYC z-yMV$@lsj`iy?IwZhtL&#Ux{V5ynpUT{4;yt|tSQ>j%FR904whocQF}ui6FAILt7T zvjnUvP$m8>D)^U==CMp#$+;m^N_^_pjjQZQ=6PwtGq~JPqk2{C_TroH zu3l}9r_GyMiXsspBfbC)#EQ!v%sc|Un}6pvKj0u`Zu)I@u!TM0PyLzzs9)a$>eng9 z!>MY|=AmljR87lik<&tc@g){g465%qcV_yiM#3NELugh|bW)^JFh4J{q(wQ8vM~ig z;k@r>r-POIYdBl#$E})n7kUCnyzpS*=ri8#fLdiGjqdZKo5BA{Z|cy2TA>^Ca5(=} z$l~x-=9<+u2y@c}_GKd24%x*fFlN_GtCzZmWz3r&A>_AH4G5EyF^c4Vi*@Odny$DQ zx%^yL-WdMo7GzaYQ)zw#vyGiBoFfhyrbR1C9a2EoxZ)q-F<+i}_VeXgc%1l8+Vgf; zr23hgt%d~kw~psa<&BMBkN_465w^PmjawJ{wltjyUw!E2AbIVVYFDB|hT{K;ml$ zB)yE{@>dhL|z)>Fb*`Z-&wgn}mO8j%y#tf0bo+wdEHpDn%RqT3w4d;(Jq<^aNU_5&{}hxOlv!R4D_Aic7t#ncWDuw}q^f6uo8WQQm_U$(&qTF zDtmy(D+Jyi(bjM6zAr+IeOiPsfcLLEVoF8prut5BV}$MyRUmI1sk$U9UJs z#RF-%#yjdLuU=>5ZaUe#-^t{}u9g{?>T&v24$Km``y;Ax4kgY*wzh{gHFWZ(!5Pm%4 z7F=?8SAfF5KQ-BHT|V4yr5;JOUDAOU#fQ#l3>wYSKrsD9)WqD&fi>II`$2T0jv-Go4l;X3M@IKmND_q4`bkjHGe6_uu{9pI+>IX`47Y=I4HPO3asj}9l7Ka zfx=Bv>1vo-7}b-m3}UUzi|**ftUp;9RnF;)Ldzn^=&VYnksr=_zC{5U$IYl z8CM=Dn&T6BnNr0D9^1t6j0bQ<;R^tLOm$|G-^L|ya>#Gb(x-sw9|KkQz*u^L=dw1B zTB1%|frw_y&6pkkUMm?n2eZ*aD%h=a%j^)csm1nbc{>-e?dj6-h33^UeP&-bq4nwZ zczBy$W~J7oO-ei)N&=4zip+jW{c_{Gi-+Z0?H+kK?^8k?PxEw4?*TPwS;)`~+ILf8 zDnk^%P-%_X$p?DRgjyBIQFL@)tQ1V^ZE4ao1?aqN$^w=s+7l0N`Cd0J#OwErm3U0+ zT{%4r@->PO_p_2PSL43Dw=axmt-G1&47L>9Y<>#+T6B zxy!>kK^ps z2j!qhRWJSyX6J+V0qPQ_FO6&KVg6A6R+_VuY+97}n*)ua9l(aOgIepLSG1!iDG7h( zuQb7Gc0y)P{ywc*WEB__rtUia2;vw#Fq@KJSDC&TE_*q)O+xqxGRc;BvFs6!@*drt#!4U>VZsRvr$#j3>$UMkxPUtnlq2DQ_c3 zQr@AWgJg-;hUcgkyI72seY*?K*BEFMET&kgc4!j9`4r_vF~h4HFJ9(Mc%(f_9LuU)yoUcGI6|hSk9ZwDjciXtXGaq1=$A+%vp$dJ(|+; z2P&rFf0aJl8FMo(S1OmESIpfBln3{?(=-RqG`{Sm+#e5=mKkeaJ`cj&O%0h_sbrnz z?^lduPetp}@Vs&^Ad9v4@uuO+eax&a zMYO-1qPeUTwx2q=CAHBf6%CK@dpF*OriKOrnjnMse5@_I2J)XU0ulpzW!59^@3+7v zg+!)6OgI}`8j|=39_Z9xSbY~%kAxIYU@F%j@N-+K>ZUrX2)kTZ(o{wp8o*d|3Mbt4 z_33)fT8>=gZ&-qCY;zjt4-bdEhj3Rb)ee_#)n?N1jE_EqC%BmldfmP}XRMwQ3^`In zg_Sjz+au2!_23$%vxFe2X1}=AzRx?xT|QE9vi|Y1$fdkFxxI7bVQ=rq^rFY8?X;TT=5W~t1+6_~e+~Vylp&qr|a77VL zUG3#zYssb(em;2wQ?l;xp^Yw^MB4llOeewVU=-$dZ_&jl^WmYq?($D0k`vkC5Nd-Y zE0(8K7I3a)onZ>23F&h>MbGovbM5`_sLjr9Qf_96+LMVv&FVDPI}2YK7VcwUF~Dc^ z5ghZv%b;Yx`N<8nh;Vxf`ezm$J67=Mn4BBP)_&K8JZc!DCUUYzZKgF?E%iP)IiI0Z zgSCIeWf4l)-ce_@AD@SttWT9GI#!bre7^Mk8O>8dWr$_j`1Sm8kvVDhG>MhX>m&TR zOxC$wW_l`rukWW3Im=(~a(+&)q-}pL`>C1cmDBIQm~l7QPsicO6y~+yF#|s|yR#&S zXt}eLy%u-amd}bj@cCwmqgJGZN2b1$?opy$)-p5gYtq&IXiCw7YedRjLfhIy>DqZO zM1D=S2|K2rcJ$=@s48Yl)cla$5rLHs`Qap?Z_S)E=+rB z%xIeP;^WK>>#-Yup(6$BOoeQt9@2GiMA?Fe!>*S zT>sUG7Zy6!oVy)g-{5iY-n9N|URLtfot~}Nq?gIEm-C`msDFNmuGM<3U>67o2rDQE zNG#yd&e7h%*q+8o&%)GD&%xB%iiU}Uje&uUnUR@^frf$3&e4kQtC5}st%bFLIgNv< zrIEFx1C5@QA&r5ag@utFjh?=>odcbblc}MRm4VU!=DFewboBJ}j0`k}rg|n;*7gpj z1~hg?_Kp?~v}X3!R-TbOa^MW`LAQ_SDm%2r7JE}*sNjXjjVX0v0d$f2Vm5^jt0@7X z<-g~>p1ickK}=!;jpmnl>hig24{MQaC9*M|Kbaddnq7q#GygW}P_kFWBS_cvV%8<> zV9uGKE*uHB%{x;Klu)?pd*w-Eu%qEQnV8YB9`v+7?Mt}H6UVlVD9#NV^cjvX0QvSB zuzJFOz@X^!74|A(WzF^#lHqg3S6p#lg~dA5R5P_cEvA-89GDb$Q8_o#&L5&u)CoTi z9l-zD=Pvo6rNdw#Ae4|GAQAt+^|`UBi-V(`(Ocis(=*WM8JPd+?mQ(K`$>A_=37-4 zyS$IFpZX=)6Bs@4VDVAw7%QqGWh|)FB=7gGT1lvEDX74Q_z?+AZkwks7qyqu(7T5Y zF(9u|L{~YuG~Zuwi}a>>8f>)AH;8OE5$Ed1CqSS~#j1bsX{saTR`z(m_eI;i5XGgdei!O^^`{W0y0tP zL>pO=nF{r|Ql7pHnUr+OKv!+W%J&lGv17PBqPFsn%UsQ_>yXp)#bIdu4EfNxUy(Oi zL?>+gY=mrOwmeD)jbjUzzqV;{9r{(Kc^w+~#ww0*prq1A2F^8RX|MVtS~V$nw!)ji zw`O&l^nWG0E35jRcI@#Oy!q?BmT~%G$1lO7HDx%rxR!$*yZWUfr|RHyRuh8_+Mq}n zm?E*ypuE#u&A!nRDCNpII8WdtqUooXG4w0h7)nuQBpkCEiKxG#MMoL0UOs~s&o_)1 z2RTCaDJ`5n*9~EZ74SolNUHvzm8@u+ez!g@@c^;;t_N*jw=waT_r7n)Bst~w(K=uO zFgvq$T=zrPXjYx7QU`hBFY>;&Vg-x^mklPC(P9|ZI6rV>asRTEL5NGxbNZdA?KeU{h==%c27LVl?qSQ# zmzk(u(i)f9U?kS<$v)P?TN~yWZ%u2BZujr9jZ;M>P?h0q(Jbo2@3K51vC)JR!&NU2 zd@K-+yj!)EHaIeB4v)^L55Fg%>RZ_xgE-qnhrb;ASVToB+V=Ksoh zXJ=&VXk`Cas8f!Xkqe+l4my3LhE^@vm#(!~-{}=V!a?dOqB26R7}M7bH?DkL1|ztd zYC7d+a-Eo@9lZF$iE~YTKgN}!imgLil5s2)r|`1EAWr~-=)MH^rH!*sm3XXIdgim= z7n&tuo!(<{EAD-9>uZb1_qBl%?mR)v+m%jUk&m1eWxR0C$Pxg>t*4@QJgiM z+#t-)^_~hZx?;^lwvaQP^Y-jpn?ssJI$KRWAGDKE_LDIxhz#CT>`1eqVp+$`W)4h* z+^CK!GUo1#t7E1e4^M7M8yNvYDLw9nmpLGHVnXdnLLM9_n$ZWS;WJPX%q-JlJu>nE; zAaJ|&Q6kP?Q!aNa8Pqo~Bm_bU%puwM=@e94l7!5XG47ya{KWlRuxI0)M}iS`na18W z7gm!k;Bxr!!}s{2j~cz+McUsk3Q2Pf*(|h5d#FD1+?S4gtltK+!$zom z!QX#M)uGGsg(dC{LlMex_gZR5xW+^$3w3{#6p=@lk+D(UHByJu2AfUYSD?~jQ3DSx z5S>v2ErHGc1}hm#oxM|04PWSqh!4T8n#Y-cGpbD&k)~uz%)ay9mlCJTB_d`l^~2I< zEi`%MkJgcl3AoIA?k3tcGr#yeK1JIPaO88KlhLUqjhfVzf}=0}PSmc2v4egUAt9}) z$-L8CkTQs{Znf9adH*iO~+NO4Xs8IV@&n2~j;hZCA1O~<(tMk!K z$sPKXa94odi7kfY;sezDBN~&aS4w6mJ9jpQVXzquyQ<_C*jJ(|jXIRa6D8|o3j?Q3 z8!AJv?w0@t{Bk!vqh((mn)>|x$0|zDBBYVIeM7*!g;=Q9$?0-@)(?!3FT(p6lZ&ZK z8y7$I%LlvZuvYawNS}}1lO7=P7=ghhf6vd6UWzc>H-Fr_iDspSC;7Dh^BxY>W$bbg`igBTKAqJ;^0;!!qxXPH7I$zERs<(G_(xXJl7k z{EvsnJIW82Om076D_(cKRpm#5rk)UwXsaf*w7~QUDdK95UDD)}p9=&N$!WPpG;SWB zogDSKwKLeb;WJJIeO{0iz|UF{>y_LOd3*5ZJbB2+H_G9xEeG;rEDd+pEr`9Zw9YE@ zMI4v%WC^(UWIV=;(*a*YrGSdluJ)d+XdsMs%(&n&?Rvw{uA+E$+V1Y(~r8oZVy(pMr;cn&iILZo<2~UrvN-)*|z-T$O{G0F1|Txj7p`v zMQZL7&i`ZXo4+&NmTqISW81ckj&0kvla6h5Y^!4%9d?|KZQGsu_TJ}w=i7VSamW4b z?2Pf`4_I&2nl)?AsufoTZmCcWekcV^9&jLBSroN5GGuvbGk0D(-aSN3Y%dgQ$^HYs zstW#yR(9%|uk<_{0;l~7519um$?J+O049zI&c9=f)3w#T>r9DwB4R6D8I0I|I&=vW zdJ?)Heuxd(v|QsB$00Z!`L3aXe$aVg+y`kl*;x+Sl=KG{&9D3xDuE%^&w#pOlc^fB z+`&=VK`Z$TW*266NKX>6cJ{S8K^O6sf^F7ABU@eIuRpMpe|XXXC=gI9)IX+JW%1-v+R#gUbxk|o83&?wJvSKnh7Y0EMv@pP+AEt zf}BqcVQL~kra}4a(=RImU?Qs=lM*OGQ#}=#;nI$8qthmQ&qu<;zH2_klo-g+ZG*#G z(V1B=QxZe4B$y~brqG12gdm^{Wy3|piE6BcS=ZDkflmEs(T&A7Uarva9zScPIMg=v zR+=hS^Qq-XN3w|H3xuQ7FeL||`_YET62$Twth7s-vw@vWwNGuePd9aZ_)D{HX;Bgn zS`ADl3^O4bTh?XRCf6#bdXU7S(z154{O}$s?5tZ}!d$ALLJ%F@K(lWQ0~^qeR-Y

@LR#Fk#JNj^N^#d&h2#q+pJ_b?=M0SXXTRAF}ty_X1$A5HngA>fG<7s`dQm`aF z)EbuoL7P3IVmH_nRuSS>YMe&p_ZzRjEX&X|x5 zSyIU715>^HvmI|^Qw!8&m69X&!RSgs2AmjEvAy?>mc(2mXD`~B%5U%qZr67D5r#3; zPg-ufWddf{t^8#AsH}wJJi9BTb-w%5(CO6+FN1N(4Rzsy%lm07umiO$Kh;hm?MpUH$Vi?hG*0<=3j0-o|STWPOa_ULcVZ(3m!=|HLD%^)? zs>EreOP{1;VzxZhc9bw9zm;$f)Wf|CV9G<<8EZqnvaQK9_oPitdN>Y~mWoUFUrn{$>snVIJ_z8jw){Fd8u=82F?K71U^%=M;1HDcyko=|)ZL$9TaANg%dG znjc&E2@+BXf0@^}lQ?=9-Ro{%pGSvnzA6s>SXh5o0Y6Qj z5!Bgb*4^dFvk>IcxzeY}p4_QEYD(YLl~;&~o9CJ^vSXYNZ=Cn*3zZ7TI(N11>$Xu? z@!KpLM597>gb7|BWNj9gqhKc~2R#LJcaFC?Mxx4yhkX=H)HWn1?u5^C%EFQeH7U*w z;tQwSbRvD9<39WU4RKs|O3K;*XwwZS2mc{eJj-7ZCsE#hjSy+*@|8-B|86!B&cMd4 z7Sa_O5AHmuT%VUZ?&H~d zfn39@iIQp-Pd_4CvFoUm95fjo9W(OPNVJ@Bpa=R}cv57gB-7M&rhRu|EDc|UBs`I8 zj&jNgVuRmcYy6!2Z2pO)aS@W1G<|~V?9rP(vB0irpIUmACDt7skxcoT&~u z)ja2e+96X-S1hJvI7-DfTF*#XbmYkoQpxd=mItaS<%d@9pGWh~e!Aa)K*Z4X*-MkS zbE|n6x)@i(Iie;rClAOy&)A#m$l)z^>q9eN`K_N=OI-PfIe%52Y@Ckp2vBB=-_G;w7~(OM!kp}K=l3$dPb0IYPe%$|U^Bk!ZF@uX&7Cvv zr566e=mz4pcL#M-0Y*LkLf4CI(6mIorHPvNv*tm+Sw3aorQc$_*B#s6301zuELcLH% z(0bRI-`~14E~5)IhXCMf`{w{3>;D2iS?50}hq{BuQE_OD{3nMINOidknUV4lCFe02 zNHD<&Ch#&ai;>Z%9t=V%O{bF*S=TwgPF;gd{;P4t)P23`39W99Z2PRqAIv^kRl4?@ zCr=FeH9*K@!em3qgaN%U`y#;{{tv^{C_zUC7LDKXp%G^cUYqd@mEsX=0S^!CGrm)% zS5Xl7R<#mv=u1vSqK=a7%TNs|S~Aw8AnRbNP{}5%H4UoFN5a62R}8ZmqaCUWNhvGq zT`LK)_p33NEfibIGQR4TFP5)Lx1`44wHt-Ad}n0rECV)01JhvvGl($^X+?^hUu91@ zZJbe`YTvRc+<1l(#59Xe_%t+@h0L_*U~8@7urAYl&ikFdGhSR;CpDeq!FA{AB60JLuH@tFHK+7RM}07|?#w zvtw9m{J5!c+UVsgdHOBb!vL5YenRZSj9s7s*6(KAfy^1X-ddJ+8$5+RZs#})p-hGWDcvxaWV59 z-;R8BQ~A(LB};LyE*{DH%Wj_}I3+81yarS+m3Rd;O<&zl^F^GNN!~ho^d(2u{Ktmx z@ubB_{LVd0N}r4QEXdmVf!lt;sc7kL)_yrb?Sp<_r>Sq(&&j=1mDGWA$~zC^t-z;; z6|JkG(M?60sTVCmay5b5Yk!I-FpG7)5U+-ubO0eMH<6JLEGr5yq7zbG0Aa*UNrkp8 z7~}o?bYyW_!gDUc&39nxBC_KCtYxHW(N0HHdQ!r2*ql}cE{E4b1 z-^yT3nvaWSH2b~qnD!fmeNFLWgBttkHlxVM{{{Ijk~Iih|5?b#`d8#r8n;_zMCyE{ zfy48%{CY*D4B=#vBoaj|hA3V$6sfgrW1@ssPHY$N&mc+Vd)3S`uzf+B%du*UHUa(P zDnPkzL-m!;KW)a+Ew^1obp!=@GWJ{co)TmK_Xt(7-@rFz@M#p}37H~1N|Nn2@bwf% zM0o{6PZ2TwD3`hqTjn*I7(1rmB7!SAZ&Vt#$WK^mXW7?N@*9>;r8C>kC~;oUdX)MD z_y$ZTvQ-#kgV>5=tnG=#KkWoN_@!(b2O|)_9g`aFKX334c~axMBUW-dPBD$QN_+NYTRH&vsw&cJ7L`Fi3t-bZ;X9t9KG99hSllWd23!r zCR&lMrs5dbei5NUN7k+!J)JX`dR)R(pud~VNEW^R2k=d%Wi=`2P>~hcU=~MLeiU6w zzhvEHFiLR>D0E}k$Y;{D&+*i3Dz%roeB(g6q6v233E(sSg~100yfmo?fw;c80P#vg zK_0B61k*i?CfBQ)oty00!zTRWt3tKW7?gpq8{2%ZDMRpf6Du7Col*?3;>4NCfW}g~ zp0#S=dZ*3bfe$_$-*5k)#rW9%1AMB|zk#p&^$*}Hfk#68bn&z`=DK^_daYq4_w{(| zTQa6{uvCCtR`iEAaAG1Eh2=^zg%iPMcekz|w^ZW%F3{x5u64c{m!3IXc4rK*eUHfi~)aW}G%{cS@xw1XNjVcmxocwr{ z2ZQ)xbyA(#oLB2a7?mvf6Ll6!j0$~OLNBZ;Q$FXeCy^y^J2Nq_=@1q1AffWH@GUWg zQ?Tmn=>i$+mZ1R6x`wa6X_7kLudsql@3h4WRtA&>Qq46 zx9a=m>ibK2Af!~5Van~~FrA~Hg0_DWKK9AA@hSi1m}IU}Pa&!6wpG>jMZ0^%?MRRW zr>h2>qas9`f?^?9A~= z1s-4UeHt(D>(RcGE0>taoF*p6-_9P5a{T-SkpO-Sbu70#FR!xBk27I&3aX7Bm-z+5 zsln2UugKY|2{M;!^NWrE&ssp5xK~@6VmyMS%01-fZ{W+!{yg36R%9Mo&zn6c&odaJ zLx1Xqbzf7HJ*N{N@hIiDVMFl(Tc^DuQE zl3B%`l!1XbdT&3@Xo7(TjVx36GzuPPC~QG`KE(0BwaXTJvuj=Mz&hzGv+Koh6bn9= z@|BXIt|?k|WV9L_vm(f3U8)Yvbh3Kgk1BPyQ%^!&TDt)YpG&hZMAym0cf#*5MbK#v zgeiG1@uT-6G9wGF@7#0Q9@6io_UQxE8PM`?xfunn#Ih1^xLlA}LDP zSbf4`3#)h2&e6Aw8!uJ()V)g1WS(Uj5I3_@Asjko(M$tEN|wo_42mVrd_->}7w0!m zJ(OO&#G0Vzv$X)+V}<$`?n&lQ)HccQM3S%W%lLI^7V*TD?)rK?U0nBjYuNkXV!WLB zz12U(QWZ5=4mWfxtBYsJ_3WJb3uFhW#=mqoUE(l~r*+_`7TLcM*v^l1D31>JZD>9( zUXpi03pGQ2OTd9N!~Ydtjm7QKFF{n6XG+!oOw^i1TM7MeN}&W5$(bX!H=`mYG1^z{ zVz>C+i*KHp)aTCSj^nQ#sP#ne`}Thp?6LhnVJ|<03Y4pHG+!*C0Fq=WkXW4^=c|fX zML5phrnHh20k@<1=YecZe*5NC>LZk#g+B=2dCQxwcL9rkpF3-VGPyFj<^mB<0E=X0f32 znovJ^lElz|kR6zJnye6`*I2&PaG&a}wrq>svUhlBQsf^qnvdH!UHPKsyS%`W0^) ztB8x;nf9Fd9rKftFl z>NGF$TbHMN;igxvME^0G4up$>kLxm!QbuMPky_!UvS1c))uUWoZ0T)efWDsmyfI$2 z)?HI?bM6Y-<%1YxpRKEJa!0Ef7SzRlZST88sD>7dn&kxOX2Wa7K_U9%_F9GK`LH%Z z*jNl4TGWNrWSacem~7J=th2gtCl_zy_{DjLY_TQ9a^jO=bsYP*QZ)MFy9&n8)!!5by1>w|~EZGL) z_HRyWs1A)CbiUA>n_6pcz7gW~6;q1mtPT*0091KUuQWYZ@3v(bO`d(jR)VgXYqktI zXlU!BfTaxD??098}chok&uRb z;xA3Bx3j<+I!P`I$JdM8Am+ai=T&B$ViZD0U=nE`g zi(`A^Etei`JyJjY;BNxx^wLtGPhAc8ayVCR&Fg~k;t-xYI9&!V$Ik|BZU;I3oIy$MhZnN}xcwpJ`wg2jw8SLnhC|hJo_}lUqth>rUGXH_HJ6*wUEZ8)YC9kH(&nbLtHShf+r-w-t>}iA0k)mi zhvE@`f_y{KdiZmW@h~SE89gJJaB?FmOLI<$zXO=l*pm~O+O4sX624vWqk6W@>ouG4 zSY_D4-;g3lsRa%nfE2}m?;p|{WdAEt#LLSM{@EI2@1i~xlumm%$PF5{;Ki&B?VsOK zDht7$H0$YTyw6+P$lUCl*lk_5Yslk1+lP(q^GI9JcH{5Z-Mes6TGsx2Vcmg3T3^8Bu*K3_(AWmWEFIZj1&f(S(g z&iWtiE5!oSK!f-HWU1pWWedS$l(@qZgs&T>!Mc(Rb+9cycKr3Mwi1vzcknl78qMy# z?hfEKZvN-=-#Gqr8>{}c|Aq(xwszf@k~uu*c&hcAh6HRW8Ob1119N|4`E>~{A(fg% zZmGQ9NMP@Ny+JR4PbPUX0kil5?^4rj>yzZ-CtV`9-NKPAW+1DK*! z0+9!tbjA{)l9UU-PHP+l!seJ8Fp>nS0;ME_nPW|+;2Of7Po8|jzN*_yv8-zAXtdUC z5mC>Qh~*L^6boV2Ga!|gL#m;q4_6thvsx|rrVG8_WLw>0YwPC<9N;*{^@BO*Fp_C( z1iRU?*JE|kWp1MD;_YyjnFtzoz$fER-AS11YlO|-xLw{HKO?91m$oS7b)AVr2<`7ttU*GS7;`;xmN+(YkE|C=0NX4o-VhJD}(269PV4jF79z^c{nP`iOwhPBinEnO@gnx#V+XmavQPlrDN)eSb!{d*Rx~?xb&8 z^2P%{ofcR1zw+hS(&=6{1+%yQ>{;?<%!OIt90|j#t!mZuuA}?G0l%+1vv&$~c1F(! z#E{%BSxS`D+sALtFXjFrthFE?{pFstV-DgkZzRvq;I(vozZYwFCSmwDam7!#6EER1a(Qa=r!7(!Uq zPmCfZrHfTxTe0Mp>1hCg|;cu*kPzsB73}6qM-67$?w5f)x<#=LROeO#!MsMLroq zvKs1scw+ET#Z;OgRvM^+fqXg%L2CU1nbO~#x7t5CZ|Tt z*|FGdpQqEkc3Vg4yZd2LD6o+w#ewi3H@3vjCLD^q@PNtgwVRg}R%@t@z~B-n`Gg;4 zuaBu!5xw`^_-M%n?7uA-Iw&Ixd1HZ1&bq%P&ck8t(uH4wrN!^X{etsX4zg$jw)tCh zYX9emP8@$lmcNTmKMgvS5Nw?+REtjxQ>@uS4bz6Lb%H8w3Vj}pYUcz-Ja2cIZnr!e z*3oBhkAo3e#R*=F08hf0xn{k{v#Sqne;+G3=&TV{bsAnyz<@{xi{~V2S?yW)Lg3v zfP}bQ+;7(#^#j+jb5S3fkihOJwCs~sGMcm|ioutMJabxKb*QgjL1-2#; zmDH>twIwNTWoOzZZQ=h*?+v^t()PF8;vWQhoc|m2>{rDBJ~wqF-aDn*E&?5=<9|inHFKAXx`Hj|i$KSfs*ljqi6A28=yb2k4;$bLOzA1S6LH&;KN#ZtblEMr##aHr1 zY7-(x`l3432J3K!%1^s2lLRjrQ`5MY3zCdV`ooptZP7-5w)uj|~b-ts`t?U%D5_-F&~?tZBu z!%4GSB|{Mj>-fmQD!j{V81$4{-sMiE9sR-WH!DE*?JmyCoUic5c z1weNC?YD^Q>IMb)E%>y)*u*mjI#+|E2s#_yq2ko_2>$7}V03XRIC+3(Y^X-G(&V*) z(EUDHQhu*6Rlb1F>(mX}<_FUX!4PxjR9j$xH&%A?*nJ6+Gp^ndzGZt(`GfU4d-g;e z&+wnkxA#AqZ}pet^|!q)jbZZnY&hC21FzcF0XBa&-|(^bBA~>R=^6Qq_oBB@0-;Iv zA2D+cJKAO}FMjFcK|H-wPwHqhO9$ z`9eSA)>ZRvNz3c0{%^lU@gIH*QT{;7X6x2fx)$aTg(?s_W_W64Y>Aq0aJG~Yvnh;` zfW1pf(rRWJqZ1a;QjFxck)fO07TgXvtK2_-`136P(qwPhZNfs-F++8bMN3p+N;b5K zCzi<%G>71v{7P)4ugdVJ)nYC9+i$A{_YT%?s|A?^O3m4Cs|Eh(J!H}D${$t>te7+V z4~<`%dRe)brV-qCPVr}_fDL_h!qmFd9AtG;ZnTjU*+xxuLwF?DtABXkNyLEP=MCk{{SDrZSf!A z`@7r1iN{aNXW&n_#rYWviCpQAtA#(@781ToEAH^=`?q|E(z6`D9@$@ZC(BKC4K9m( z&Xtpl6P?@nLSxoA8E>!HY$m4xnv=uVhAT1}%)>zQESaBH#J~E#kR~7aaL0e1GZt1v zuqy(%Ef!C=*+fQvYfgVt!8N-RM}BKg{~+Mw{C@%;5CnrBRTw*0V};Uv-B=U0K_}o9 z%9=jaX2ouAV4ip9E5m=;Etc&n4dgsRA+rK+MVdBm+%>Zgwv%L4U3nAQ*O{78bmpTO zos3k8Kz^k=GNu!%f(h(^IWm;)^ z#V9rjZ%|r~`A57(r$zr(os`CE8gT&KH@|SsL4fR(1dyG&SWKh;M|K*zWC{#{@+X(n z;_7ymcg%?#Ss#!p-PlS)t8DaA9ZXv|1{x{EiEYC$!?-42?bNijOB^vXr7@oipg&*> zZM%#D$WB^+$W9fq$ z{2UIXr`l6>q^Lwmo@3eIKM;NOXAhG(f))FpzufpvK1A!ysFgK)N+0LU4s8hb)*|)W zB3?v4>$*%s>exTbde`~avbXAUF^RWn6v9w!)sK!?b||-BOj8;vz}m*Mg$ipmny$Ev zS&?g4_X@7VR0wH+mZ|EB%w6B5C~z=jZvM5tN}yNzwFkhx#(xgk|&L_ufZ33+%_5!f}O7r$K6C8t z1Oy001*#7Xh3i1r#5kt8rthga^SC|7d;M8`wslOCydSz-##@;!MY~InK8bbwDSqiU z+mmr<264uaclBk;jni!j3A*sRf*3h8SdkC^mbvkh~bPM~Xh@ul@KqiaWlCso3VF6%|7oa&^ z{;4@(=2*n1p!kVcM`Zp@_s9Trk2+h0+_dJZ zMg@(YyZ%_$dGmj8uL&WIk>i{*oC$k~Sb!9mZ$@$t!6m|S>2xf2csMC+jALt!MIHZC zZPmBF^+b7)B>iSRq+{Lb6fRs1oJLjC07OZnTtmp&sEU?}DjHeiw*EU>h|=zCJTZl) zUUE@UG_^Np2Etm2E$-P;&sUodnaoT@Tk$UgpRA2NLL+I9a2TM)5;dYYCtVPPYFZ~A zHSe3(&G0oaqD~pnH#;;9J|`$h$OHVc^|YC+e;Nz@7up)4)lTbpnXScvwRKKi2~ z0wgC@V#F?d%2y(~@;taIR`#QOI+#M{nzl*QJ{Zh#{-Ut2#@Vlmhl*@jf+^^vX5dO$ zMuYh#5`iG3Av?Wd5#zMp5zLbJ!B+(K_DdrlVoZerdDs~M!iO0KYVBeDCThhv6-t#rOAh)?o)MR@?Q@Y;nYv}uF0?)D0s-3?&1m?A}B zU^q0CC)+iOq^0<|w67T4!hn**i!b{bnvSb=aca^^w>qo=4xWGy?;ZZH*O7J-e(^sH z^|=0qdVe@A#2$aah(y_Gk>g4wOMq%NjurqkCs>lzz_|QLadg!L>K%yZN2SKu5vL1W zy~(wiTeFVp08YF70_seEww})i9EBQ^;wys`4*DT*9j1;R0s3aAp$d zMA4k(F26y~7Zw0|F8QC?%9SUw4|Vom25j(|==ScCa24-Gwfowb3edl68OnO5jsEb^ ztBca?EJ^N9c^I&MWru?fYZQnZ|E)P~WHJxCGD4$7SPHv(0Z@;sGVujK_r?wZbnjM$ z_X1yy`!Uo(^BYKi1#x$koVP3I({|n6&Kb%EExWy1X6D&9&E>IG-aPqn?18mMv4K*i z+FPdLYyE}0KHiOxq#H8PF(HRBETI|hSqIsOI$S8%Xe%V8jFswZxpWk9Z-UVH{;kP0 zveOKfDbG15D$zNiM}n8`q92sj&Nxc%m*LL=UIsHmOB=vk z=pVzNs?i-VAWTR=uRdZD=NLajQ3K;d5MJ3@#)R!2e@^0sKV2qjEKkARx@|zV;haFE z1Q$+7_eC0B5-7mAPZm}jJ!xUXb5PQhhzqBw=dOK1cYmV%1)Xw>Ay0Hntv=4E2_YWh*Q1p!1yn^45JDJ< z(_qWG-zHgB8B&c;`pCDc^+(^s=%^{EbJOTPAsHzcgikVcz|+fHF*9RN7vtne(kSV2 zBP>$JElyTcLd2eGTa+4DU66G-f(D<2hK?w;55kWMK09cR?iF6kEW%h{){mRM2xYDv zuT^dZJ7`&vdClYwW%tdscd`#_!)iTEXiraGN6cyU2=VA^+jH1S_gpEBzfF@CYdY-c zaqsA{I1p6rcFUIgGzwEnUWOF^vbmU_{2DE?4=KZYHP++cd9dFzPvC=RRus30ts<}n zoC!^NScEKSEe*tkJ;244M5c8ZB8~3@Qa@Y{HIoP?@d@?m`_9Z_iRLdSWtEEeP_-Kf z{Z-$twjkEjc3Gc2R(Ta1{XqjB|)8X-UPshFe z;-Y3IW%|Jk>v zgCp^5S5Aq6wyb2vBU7c6t=O!L9W;&IYe0-DF#x785zMH)9KF|^@1?#- zc}S>N`2_V%O*Io&fj$PJM-cR&L2bImT$D(X@iRO`1ZxT%vs$E_!{kc0f8mwh5cr;! zHF}E$^cayX*&6#T*n)q$uJpczBuHSZ9U&AG1pP@%WItdGR1 z_~5eJVmmO$Wu6$YI^O~%tPYZQC`1`NPd`nO4%-{v2drocqxi@umVxK0i$aXX&Ha%D zD}VXP_}=4oFG^(G&=+R_{S7Brk^dGVoub3`ebSaV7ZGeSn!!L9wdx(rJE$C@JMk?D}04CNqk5~Uk~d-Ze=$`c>U>5 z?|scLDU}?i29eiHxxVTc-Ip*1YJ2@$d7rKcLTvl>(Lx;x&`;6|w(F6TXCE6Tw&IY- zbdBH%pplHjAAg*#xW44ALC{;9^tkoCR!}`PDA_el9XmwD;Pf80!5)X##7eF>%IObW z4v(HmysC0}ay|cTQkGHr!V3lv|D^ynRrt?~|9_3<(YZTWy11Cy{Urqew$cBjz}*%g z1&((W?s`YYl<1P^CC8y6e|b|aGa5kxQs8XLiB;rRwoGY*1@_OfOC+Py^|SZnPMqmF zXX(ryvr33>fZ)T&sfG{sC%)xdD%!e3Zr zfQDqCM95I;=W(g?8(D`X>|iSpjbl!@p%AI7h~ShL=slBf#H+RQ(QiGeEL2$v;z`pB zL4rgx#y(-uh>^9Lnd;_Jy;bzX%raWzd(8)X72cK(?H7x(szR8x+Ab}py<50C<> z4?(8jNQ?m%mcVEMz^VmcU}&aFu)>&4pO3RSPQ7~+&$o8roQb&r1GuPc)UCr3S^yo6 zYEe`z#qdx#t>#g@xbxguiKuVOABM+c{g$K=8r`qqd`nhj%?o2&@iWxBMx`o-{E%*1 zcU#k~_hUx!1dywNx;-XU=bBsgratQM@wVe-y@+gwFHAS2`=Vj;ZOx<*y-F|z+HQgJ z`GJ$n1I#R+D9VcMjmV-6tqF>%yR~g9F3S^(F&5dAnMc%2-ElGo$Xw?On^H5-Hg+K| zb4Au!1d>yZEuwozd#J->V%$<%P*Yx3`uKIsj@fRc^^-1=*I_d zxM7ylR54{MEm-j#^ElWAHqj=T+SO>!sQ+K_o%eSS#PgLIQyIAThQ^b@no0Zp`-`NGf;FI0(-S8!D z*3{H;C~w>Xe{6HR?$))nr81;$dSP!UdE2^q=NG2U;+Qs~DxK{gWF~2p_ zO4qdTd(y}%P>CT@S(9;5*Ug9tUK-UcTHvS!@-2QTJ@$8CjKJl*n)UoVNXn)#OevY9 z%8zNSEbgQB!FRy3c0IyJeTt6%Fl13(%wREtc3?o5wwI+$WIRR0EDkg3Ou8$HB_|3~ zFec``yN&f>%VFT_QP6_$-Jq|(jLSP}D4)GNU@T8qQI-#rb;vtdLg9P7rCL$2o|f&J zRBpStz*&w(ITnfJK`%D1Sg8^e5IvH_1T%YQ=}tOIfjn>@Mzxa!J0lmxdPpiHckH^u zgEMBt{*TRE>mJdETc6HlyHm@IandPI4&6m$chXh&YXF#-{Hr z4_23}%H6xpZC(ry(OnYr!)rQ=UsSCHavkYz8T>S6)#@!*$NC5+Lq@JlL6@pYY&D$B z@_D~8!qFDo(yp){$7dyL&6I3+>x(Zu>TDWz?Y8G#v&I=6SlU5EA)GDpphcxp27=MR zIUq@bGcbc=WXrCk)sI!a-OmT))?KKnuIi7j@MGMZ{8H>2-?aKrVsLi<(Ne0^y20OA zB2a~QwG-5!)}d-7aL^$z0_t{H9+1)(s2UhV)Bh8UI6eg=GLTdxT2mhkkESJosS78} z5kHLnqYF-6+q4>9t|e1hDnpvV+1F$Jan~R4Y{F{>ijX#RYI9X$ql(1UX$~+Zxwy~Ky=A~ZOvg{xH z@aXMa1dt^X9r7$Bnc^#A24PEjPE|b2@{g{y@>S)TJuT!46yl_4y%okIFXsL}wcdzv zuImk85{iJ^CH%jBR{x7aD2~UjGQxEp(tzZirg|wl#bp9Gghr#jV8pz*u7DPgCZX=$ zCJoQBm?9onJT5}mz!HE%OdMXra^yZv&V6IcRv**0jj>YoR+$V#Bigm>RvEO0w|b(r zo|d^TEtvIF#q~N^Ijk=fZQWpS-vl^vdxXn$l@?H`YE94I}j5$1(#(#rUBBE@ zA`@5iY?i0yBi2;W_b=$4MW$SLDxvU=c&6^_@8Ew;rdjq{)Zc5dq5)Nu-+w1SjBQK} z?OYuIi?+0^cS#aa8aQ;Y37jJri!}hFkZZk}GTjB2$F9Q>vZ{ z!RpC@H@a-TlgTF|FKX-@WujdQ2 z6{wsGNWd?E^9}p&obT@~W-b<m$dahVWDaNYtQz>Ka+6~@m!MjDC7T_Rb4m(TExyOzGj-u%)U1GhTi7~$&@g01{P zA&&2rTH&iqtH`NT#kqXG^_g!K#R;-*{&=!*#>%*@{KQWL5X&vfFR(9I2VVm;j7dS+ zS*GphvAXs71K9Q&CcUAcl@gY6w#w5}>$%YdWWMoArG;la$Qd%DGUycQpgu)UzXGbJu) zFGzs$WyD4win>tYCX$6a5YH3XDZ`S}TI*9!cn|>$RE%)Rc7fO^h*riv>Pag3?!1Y2 z^GXhASR!lRfm@SjaMb}_?ky)Bn`-YS;N(Q6LeX?hD4!9zKHMaAg1rc)aCVYynehCM8h1iyQ|Ka`$>2>V3R4UoU|kK zQ)!!USwAz?{MOcnkV~47O`6b~JA^6>?DRLW=N8&YQ?L_={P^waPVw(7@s==}_=fkr z!SWmgj)sjH>S+iF&jrsh|i4O=oeDK9&JB~MV1%sJN9G`$Ug z9YN$I>n~yV`ZmB-sz<9!WhA;uWXOvw%qWm-cmzoq^gY)DQ#2NX36er_P@=a#-8=R( z`kW@RTyK391u9i9Z0q+F8DOhhI1!lxOv_v8Xj8B&uUONrjw#rAyMQQ@$murvXpV;Z zK^b_1)zj+b76(n%B{!$ZA1ZWD;okzy+WU|K(iMyNs=sEBTd=Z^Kp5x0#-YDCy0_@+aghvb=ajwilSE9-o$ zmQL>;*Rz^w@s`6QcQ1ZVmGHxKBMS%&vZ*8=wPFE5cj?_lU#sWVMoe~2-hmRF(qj|E1lR|6ZW^ zzgA!V1&GlCvBVi2oWwy>)utvr<>YRf0}Xqg85v!tPLlCjLT~wcG*MBN#8yg=rKo@ix(eyyI*0U&d6vLmyD@#d}eZP7=tpYesBSogfL7S}HDL z9MlM}RIN&;noy9-s@8o3Ac_^jB8ONd6P+QZj(dBXkamg|jk zC9;efi|(4?)19LOeeo<1=gI3@UCFdG8M+f1zYBD07QCblmW{0j6knGcN z4jBfND2~##YKhfDR2wNNb<9fOiZx@5pTlr27(uhUk57{5RNEiMKSJ8UPRWPaI@_}c za5KYJgbzLy{!jIIIq^RKYDCN&zQ8T?tE9m zv;20!jf)?T|K<-uI1bWFfCQv9B@htZ|K|}onf`8t)7zRyG@Xd(>^8dRRgPc?FZ?hc?k2jd})lCVM4Azh%?w}-+U7k{a0ajX=!^(u1x z8eY!R-Jv0QWhir8I`xoR9mRlaoo+=aZ4m zjkmkgsNC@|_q3(Ujc#A(`;lM63mxO*OKCmHh&?^s-BScPevj`%Lt(LMr~EEU?9agM zNTxEt0w2(RDk6iGH1WXhkHEQT!Y+t!ahsgh_a}r^W~GJ@D29Xbu40|s{4nlv8H&>a zh+XruaMSdd(>j!4r_YqQPkF+@YVcwAqT!1P8LZA^P3^$>Hc9hPLD}YL7IQYhOw(Rk zaM(qMXZJaYUFEzRoxnZ2)~!S&+MtCBl+2!J7?fX=V#j90`9+{d?(hq4S`Yg&nli4Q z8oCnNCFhMa_8U=^6a9QxD>w-6rD7YH1L=3oD59GXdfYY;7|9V^yK&ufQQ<*rXjJOj zlfiOa8nt`Fui3$Le2Rc-p+{$!U$UYjOY6<6~buek8T$ybC>hV zr{mX&U)F=by%-l8Q8{WMaB+QqZHOk5M6Zwttt~>ufXk@s^OEH(5Nva{sp_VNr{OX{+#o*cQm^hv&Ms7rrvO_3b^6Zyopt$c@TL6+e0e%5`@`k zI5>+XH_~z4hHcR{`V>FBHX$XGguPUyxKGfhs#)9D9>Z|rvd21kU+^#6I3VDtx}P1C zRoE+k5_*VzMo_v0E%JRvws2&U?g3{2@#+<2YaCw!mo@>1G#gAxFJfEr3;RjzZ=`B8 zuNMtQIHB?hHON;b!4ypNm*3%cNK=+C9nWjc$TUW$+rbDc)(S^WbZ1hpu1-p!7=VRN zhzC62!!eq4vi-7{Qj&Ca=3WaFi})3x$h~xlKdNg5{?wMXFDNU6&qVuS;c;d? zl8|PWiL1(s_8FXhNUWe>-aW_05~7&+etsj(lB==7HwvY`CC3v1*L>W-k7xU|hmZ+5&qvg`Rnk7)E< zHZGiDUX`v4VRWD`u@w`pI5kRZt*VYBa$Nf)7?TPPu}WrTPl(c;3OD$=YjQ%> z(zWy^R5Q60DpWzfR0U}MSCTTAeoxJJcv&XIt0^n9sK)!LDBVqj)5_s^1BI*%%>}B4 zNy^1xeDE<)Ub<07Ia!^h1(uV0e1>yt5Shhw?4QI$HujM`ha;Yj*B)DS$2wzlFJ08^(JGC(?FT*+4&S{4lc`#zYlO1v-{@B@Fg<`7 zR!g;DP{0l3*WRWqWZZmEYNz)gS_td-n|S|(5C)>=mj*3pYih5=>i%9a=VWN4B-S%f z$Ggop@w0Mp7OM!JhIrS z83US~4cgt=R%udMwM@ZF zfLEQ;5q8l^!czB*svunm*QN~aoX=wjDcOO?&vDg=0tgYvK`9gy=9Uw3HLxeKdKyc6M|esMg6^W_j*8` zL;wc@l7s;QV*dZYb{Bhl8#-G^FFEs8_TeU;|jO2SrBmN48Z2kCdwAF{nQ@>&%2S*i&lehOtT{#49S1x;UrnE2ny2Uh(TLkJ1e3M5SbUSF%Ab z?ftAQUDLL3L5B;uu!S}d6}QV^1?NW~+lFOzkz2#aBM?2dM~4P@{v^~V)teP_dsK(7 zxa5;nGwhIPO?7wX!?N>0N5u|O6!joryFsC%m?%#$mGI3`W){Hq-&9Qy>J!K@Q#_|u zoNB-;@Xi&3><{?(5qW)NnvywMcU~g!!UFv7x2>~t;5Q!;s(<64RnEgc0*-b7uR@y_SYwf87F zA|E&PSbVNzf9}-6S}-^tCJ2`^;Nl>8ZS%BS`|;=vjd>%>4?4?s*YbIJrgIm>1%5qSF## z5);fhB8K<_RD?7R6~-THgS7br1MaiK^-!QUt{Iu-i(0)Zly;!TDW*pk3}+;`EY}X1o;JQmDnylalU&3mRypcVnkN z>hoOjnOj!FRp_$pq3_`-$Yk)?=MxtsPO9f`s&5_Mq%undD0CaT^bP3!q)zjfMjf5E zJqxRIMx2Gc)tb1Q?=&u~9lmB#W>_kH-uR?x`Aey_CG_&pfLGoNX@75*weRTVrJC#* zGn)n2%mMtTIjj-dED`w$#^bD~{BDR-`+HyWG=@f)l^s`Z$!MzO;Cavz<3zkN(U;<# z7jB`I$2)@w}d9tepebNY-{{HKCXJcaZARhM~*pw>4aG}oe9M8kC zaud--yi9T8w59?fpUfT8Nre_4NyT(4raoqThMa-qaZN4i9@MxTpJ^g1hWU^(KMN20c~hn|xuEgf{lSDZ5^X=o^TCIHI<1;!=o1j$ zOp_F72zl3CsyCe(J%RZj5w@1h@3P8U1Z)QBtcRadtFcNg$tHi|qpc$wOlpinQQi)% z4W}kAwUI2%C!c!TWso>1YZwjlN)*=gKePbXJw%{MYA=kZAZvsQ(lBq;s8(#oaX2(| zNwm2BWmkW~yA>_+WQd1ay&!$aQP$jOZFm7{e=u{bb_LT!!gpy!9EE`}DU#4iNxIve zfeV^vD|ljm#HyX9r}_DM1YcvQAwKP6jNqb?_W8C{0Xp(tYX^&n1y)WK5`VXH`t^1NEXDh@51X`-em)&mtv@GgLprt5aJ@!i^M0m_);}S z4_U>Qs%-V;ebN%dWwlsgz?=>RlTshKN%Sb>_|U6{gwEC| zzw;q?W8;fH(jE5V$rwAlD5P9*(o>wspwgZ4vLd)ILW2vXat-4^SwL z>~iL(S|u9DjO7tp?ZRr>Snrz5`y^6f^+$#a@r@ScKDlT-40?-an%!&GaH@%ng`q5M zw$bR5M&YD$yS;ZM<=Dz<5XgbX@nWyjd8p?A;KY9vfd z0>hrC>UB(Htt%Al|8~NcIhmxJPq>yk$H2?bB&50&@IQFp7L#ONGzt*9A=-jAL9DrIu?C>CMa{F zAaoJsn{AT>xJ5RITUzW%JdDk;S*XBja5ufo5xS9Jij7mzP{# zADACszJ5*d{Qck;p7LGD{PUJjKnz9Ky!vusVb7fZt5qZgbKJcGlVU2f+(g7&*t^BBTV5$8t`}r1v&L^6Rh+NulK7Q9GeYa!;|PNx~N6mK8SR4 zCqDR<&Joc!s*sU6uKFfWuhTP(UFr?47Po-T(*-LCMZcA;DBS$=Pt*Z&_*H0X-ieCm zoV`;&y7{fopBb^s_fhTFDcPSpZ=1=&Om2!`d}ZRLjtiI<+azNWp2c>OH}?W&{W_H-fo?5q^b&_qrkS&FC&`AzzvCoOk$MtQMi7zRUzx$dQ(}3gGOC0QiZmBM6rUOWZ;%+FV%R}^|hth zU}3iTLV}+B{5ZcZRPy~t;xY~kOjj0Rj7@>nPGdQZJ^O=4N~vw=MJ3% zjcNluNP?P}+X{j8JndHMkqHi6RBE##6Z z;ZLFE^}!D^VXgGCZk}VvHAio$6{b@w^(jf8pEU>#Q&gKBL=&m8xe5aT4W5?;$(N5U zo+MdjX{IMwjToE_yrQ-EP_BHCkqNz`a7tp-^YvSj=GiHG;dIyS!mHQ{G!O}cC&E!?KTeqXmQLzb%>$99!W)B zZB9}^+5qHr($%Vmp3ba!ClNh?&T-NM3tX9E+F}9t_q^bg_9l4)E24M$kLHTRcji|& z_D7NO5>p}~_uDbNMvT7434cT=Bk2=X(W-`j2XBh;L-OmpF;8kAAFm@b5GFOTGQ4A_ zTE$1FYS==Q_y%bpv~2D%MZ2^XaASqNvf?9g^+HZ+D<{QdT6oXO(7YZcbv|o{`+AF{ zvPS*|#kAwy$Cr>PPGnAw#q3T=+7|@XA`e6z(`^spdmuj~dp$EHQ&oP(P^Go(hh=3O z$5i@*0aUGnvk##$UcrHSMsKa_i5e`={XA7j=-Se!hptL1F4k(~;4PZwn>Xlr$mMC? z4*!%9a17a!K-$#iC;#)}aPsA$yo?Jhb_=0-M5_ng;`NzWDHS^#_1kf#=*W&z2^$IS zncQmFubFUjSRU45l$AKrXI-f>#>5}P92F_O|HBLP*~lf)zR2Eoq~ETRpp zau>pni@LrF#*0epcAE2o-PD#HjSBA>b|cC4Di%cJs+dm_kHYGTU&zoej0~(9cc+A| zxViZAPB+!8{`buk2bc17nKSCR`)KfJfQ1q<&cfMqCc=)XM2^7r0uQjgAPn5yEU18u z|NhrL2prJV+K}L~hPojFKYHy8luq3~u3O?e-TUVLK8EHp-WD0d^h$9K3OgXS1LJ zvVm3W{}bW=UC9g()3US&9^OUU9jUjjsv|d%fh#Hi0t7!|k6Hj~51dg44yGeqp_U%@ zj{h7_2e*pQZ(_$kWr4NB#-`L}K?Mv` zVbhY0oQQ7Lu9yCeYXCoh>&M|b&H(Gi5yg#Ir_BOj(_#}_WJ|O6aCSsk?A-3f7+mpj zf%Y>5e73)92sBG}Y^vHz5%a!HA>s6bI+tHUkzpv1xz+REjoQ(Mn6}(~8v#V3%-|uA zKo^@fh;FsP6Id@GJa^<5oNDXe=<993XZwVE?`h^7g$%&7 zBNvEAiT~ZlHwHghBW!YuV+M|YV^p9UM(n)4K9f~I9(4)W{X9eg>HuU8STsUl%Lu+{ z1Tcr%xf6pGzBiu*DnRE8Hf{Km-Zyv-2v*C+@V^} z#s`b;V(^zn_5?mgf)|VqmL^q#=SXuyFz50;0*psy|N5%k?W50)Lm;9IZk34DwQ5!mx!g%S+!WAML_?+JWN+GVik!6{iW=&saz1bsKpgHwHC@H=St1U@G5C)o4g zw160NyF+^feK*g8)6HS=Uxp?6M~_(hhT+zOVSW<69=c^B-d1!TjK$Yz#j_1greJ zA3y>JN@K92MgI=_ci1%8BU?|DZNzD97F57Rng3n&e@AVB>jy_FVCrvB{LA_`;}*`S X695OtaBw7mKTd8S^RWhSo(AVX`qtLC literal 0 HcmV?d00001 From c7e649915edffaeb7dbbb672f076ee8739ac2a5b Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 11:22:16 +0800 Subject: [PATCH 283/343] docs(validation): record I05 read-path hypothesis --- docs/validation-heal-operations.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 9bd17c84..6bab2e8b 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -795,3 +795,15 @@ that ended window. The exact candidate source and runtime log differ on the optional version field; the cause remains unresolved. Any subsequent I05 controller must establish a real streaming/IO ownership window with evidence; repeating the same warmed full-read premise does not provide that proof. + +Source review narrows the next experiment without changing the oracle. The +candidate defaults the object-data cache off, and an explicit `versionId` +excludes the direct-memory GET path. The 32 MiB fixture is non-inline, while +codec streaming rollout is off by default, so this run used the legacy duplex +reader. `ObjectLockDiagGuard` carries the real namespace guard; the +`SetDiskLockGuardedReader` empty-read path can release that wrapper before the +HTTP response completes. The live journal shows `guard_dropped` before the +seven-millisecond HTTP completion and a later `get_object_stream_body` +`dropped_incomplete` event. This is a source-supported hypothesis for a +controlled streaming/IO reproduction, not a confirmed cause or a canonical +acceptance result. From d100abeb8e85c4000a493aa994031e4a5d01518e Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 14:12:13 +0800 Subject: [PATCH 284/343] fix(validation): retain I05 GET lock during streaming --- docs/validation-migration-map.md | 31 +++++++++++++++++++++++++-- scripts/validation_heal/fleet_node.py | 2 ++ test_validation_heal_lock_observer.py | 4 +++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 5031c219..4028fcd1 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2339,5 +2339,32 @@ recovery. Existing retained-scene isolation preserves four data roots plus the private state by same-filesystem non-overwriting renames with durable intent, all-peer preflight, interrupted-move retries and five private archive parents. It sends no S3 request and never restores backups, deletes the namespace or -rewrites the original FAIL. Live isolation is pending until separately recorded -and audited. Canonical acceptance and its count are unchanged. +rewrites the original FAIL. Recovery [39715499193/1](https://github.com/rustfs/rustfs/actions/runs/39715499193) +completed `ISOLATED`: all 58 original artifact files match byte-for-byte, +pre/move/post physical baselines match, and four data roots plus private state +were moved into five root-owned mode-0700 same-device archive parents. +Canonical acceptance and its count are unchanged. + + +I05 lock-window diagnosis (2026-10-22): exact candidate source +`757b751482bb970bc321e92059690a7698554358` defaults +`RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE` to `true`. Its `get_object` path +acquires the ECStore read guard, marks downstream options `no_lock`, then +returns the reader without attaching that guard when optimization is enabled. +The original isolated-service attestation does not set this variable, so the +candidate default applies. The archived GET's exact `acquired` and +`guard_dropped` events share request `1ba2503b-78ea-40cd-b2ff-003f54662c9c` and +lock attempt `3a86c2cc-ada5-4e8a-9e0d-dbe3bb7c1489`; they are 0.367104 ms apart. +This explains the missing backpressured lock window without attributing it to +the caller socket or optional version field. + +The `lock-timeout` runtime profile now explicitly sets +`RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE=false`, causing the streaming GET to +retain its read guard until EOF or reader drop. The candidate binary and other +runtime profiles are unchanged. The existing remote process observer checks +every configured setting, so it will reject a deployment that omits or changes +this value. Focused verification ran 50 observer/timeout/runner tests: 48 +passed and two existing Linux-only tests were skipped on macOS; `git diff +--check` passed. Hosted CI and a fresh candidate-bound live diagnostic remain +required. The previous FAIL and its isolated scene remain retained; this local +runtime correction is not live or canonical acceptance. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 586cdce1..641c02ff 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -36,6 +36,8 @@ def runtime_config(profile): if profile == "lock-timeout": logging_filter = "info,rustfs_ecstore::object_lock_diag=trace" return {**BACKGROUND_CONFIG, "RUSTFS_OBJECT_LOCK_DIAG_ENABLE": "true", + # I05 needs streaming GETs to retain their lock through backpressure. + "RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE": "false", "RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT": "1", "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE": "false", "RUSTFS_OBS_LOGGER_LEVEL": logging_filter, "RUST_LOG": logging_filter} diff --git a/test_validation_heal_lock_observer.py b/test_validation_heal_lock_observer.py index 38954e55..5a39a5c6 100644 --- a/test_validation_heal_lock_observer.py +++ b/test_validation_heal_lock_observer.py @@ -295,7 +295,9 @@ def execute(*args, **kwargs): scope = {'__name__': '__i05_socket_test__'} exec(compile(source, 'i05-socket-node', 'exec'), scope) self.assertTrue(callable(scope['caller_socket_backend'].run)) - self.assertEqual(scope['runtime_config']('lock-timeout')['RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT'], '1') + settings = scope['runtime_config']('lock-timeout') + self.assertEqual(settings['RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT'], '1') + self.assertEqual(settings['RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE'], 'false') return SimpleNamespace(returncode=0, stdout='{}', stderr='') with tempfile.TemporaryDirectory() as tmp, \ patch('scripts.validation_heal.healthy.subprocess.run', side_effect=execute): From 66fe3a018ac32139bbc383dccb7545638d735838 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 14:59:07 +0800 Subject: [PATCH 285/343] fix(validation): validate I05 contender lock evidence --- docs/validation-heal-operations.md | 38 +++-- docs/validation-migration-map.md | 45 +++-- scripts/validation_heal/lock_timeout.py | 44 ++++- scripts/validation_heal/timeout_retention.py | 108 +++++++++--- test_validation_heal_lock_timeout.py | 44 +++++ test_validation_heal_timeout_retention.py | 163 +++++++++++++++++++ 6 files changed, 389 insertions(+), 53 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 6bab2e8b..72d53427 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -772,22 +772,32 @@ The `unknown-negative` diagnostic includes the source procedure before the suppl A separate protected prefix is checked during the fault. The original key-level task must retain complete truthful counters and an unresolved disposition, with no invented healthy, repaired, absent, version or incarnation claim. Unknown must be counted within skipped and carry the legacy warning. If the candidate reports another truthful unresolved disposition, record `NOT_OBSERVED`; the legacy diagnostic cannot promote it to PASS. Archive the original terminal result and unchanged complete fault scene before restoring original metadata bytes. Verify the full original physical and SDK inventories and the unchanged original token after restoration. Fixture rescue is never product repair. J02 remains outside canonical dispatch until the separate version-specific grace Deferred/new-token procedure and complete live evidence are available. -### I05 pre-contender retained-scene isolation +### I05 retained-scene isolation `validation_cleanup_run_id=39711742431` selects the narrow I05 preservation -route with the original package build 39364893106/1 and an immutable helper -containing `timeout_retention.py`. The driver must remain on the original -Functional Chain branch. First check the shared current auto-testing fleet -for other work. The planner accepts only original helper 704bd60's failed H1 -boundary before contender submission: nine exact successful seed request/ACK -pairs, complete version/marker/payload/public metadata baseline, the single -closed historical GET, original owner/candidate/inspector and unchanged stopped -four-drive scene. Unknown writes, additional actions or H2 evidence refuse it. -It reuses retained-scene audit/isolate/post-audit, retaining all four roots and -state under private per-run parents without S3 calls, backup restoration or -deletion. Verify all original archived files byte-for-byte and all five final -private parents independently before recording active namespace isolation. -`ISOLATED` never changes the original FAIL or supplies canonical acceptance. +route with the original package build and an immutable helper containing +`timeout_retention.py`. The driver must remain on the original Functional Chain +branch. First check the shared current auto-testing fleet for other work. The +planner accepts the original helper 704bd60 failure before contender +submission, and the reviewed helper d100abeb failure after one exact known 503 +contender refusal. Both require nine exact successful seed request/ACK pairs, +complete version/marker/payload/public metadata baseline, a closed read-only +historical GET, original owner/candidate/inspector and unchanged stopped +four-drive scene. The contender boundary also verifies its request/ACK digest, +server request ID, full bucket-incarnation read-lock lifecycle, and exact +failed target write-lock lifecycle. Unknown or successful writes, another +action, H2 evidence or physical drift refuse recovery. + +The recovery reuses retained-scene audit/isolate/post-audit, retaining all four +roots and state under private per-run parents without S3 calls, backup +restoration or deletion. Verify all original archived files byte-for-byte and +all five final private parents independently before recording active namespace +isolation. `ISOLATED` never changes the original FAIL or supplies canonical +acceptance. In run 39733064688, the server refused the contender after a 1001 +ms target write-lock timeout while the exact GET held its read lock for 4.148 +s; the validator failed because it combined the same PUT's bucket-incarnation +read-lock events with the target write-lock events. The failure and physical +scene remain preserved pending evidence-based isolation. The first live run's GET already dropped its lock before observation. The version-field correction correlates those original events but cannot reopen diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 4028fcd1..97d8ad14 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2330,20 +2330,37 @@ one; that difference is not yet explained. Neither a wrong package nor a source mix is inferred from it. Do not replay the same warmed baseline premise or claim that the field correction establishes lock contention. -`timeout_retention.py` admits only this old helper's H1 pre-contender failure -boundary, exact candidate/package/binary/inspector and owner, all nine successful -seed request/ACK pairs, deterministic bytes, public metadata/tags/versions, -closed read-only historical GET and unchanged stopped four-drive physical -scene. Another case, H2, contender, uncertain write or physical drift refuses -recovery. Existing retained-scene isolation preserves four data roots plus the -private state by same-filesystem non-overwriting renames with durable intent, -all-peer preflight, interrupted-move retries and five private archive parents. -It sends no S3 request and never restores backups, deletes the namespace or -rewrites the original FAIL. Recovery [39715499193/1](https://github.com/rustfs/rustfs/actions/runs/39715499193) -completed `ISOLATED`: all 58 original artifact files match byte-for-byte, -pre/move/post physical baselines match, and four data roots plus private state -were moved into five root-owned mode-0700 same-device archive parents. -Canonical acceptance and its count are unchanged. +`timeout_retention.py` recognizes two exact H1 evidence boundaries: the first +helper's failure before a contender, and the reviewed `d100abeb` helper's +failure after one contender PUT received a known 503. Both require the exact +candidate/package/binary/inspector and owner, all nine successful seed +request/ACK pairs, deterministic bytes, public metadata/tags/versions, closed +read-only historical GET and unchanged stopped four-drive physical scene. +The contender boundary additionally checks its only durable request/ACK, +request digest and clock ordering, structured `ServiceUnavailable` response, +request-correlated bucket-incarnation read-lock lifecycle, and exact failed +`put_object_commit` write-lock lifecycle. Unknown or successful writes, changed +physical evidence, another case, H2 or extra actions refuse recovery. + +Recovery reuses retained-scene isolation with same-filesystem non-overwriting +renames, durable intent, preflight, interrupted-move retries and five private +archive parents. It makes no S3 request and never restores backups, deletes the +namespace or rewrites the original FAIL. The first failure recovery +[39715499193/1](https://github.com/rustfs/rustfs/actions/runs/39715499193) +completed `ISOLATED`; all 58 original artifact files matched byte-for-byte and +the five scene roots moved into private archive parents. The second diagnostic +[39733064688/1](https://github.com/rustfs/rustfs/actions/runs/39733064688) +also remains FAIL/cleanup FAIL. Its server returned the expected 503 after a +1001 ms target write-lock timeout while the GET read lock was held for 4.148 s +and its service socket retained an 8.17 MB send queue. The validator rejected +the event sequence because it grouped the PUT's bucket-incarnation read lock +with its target write lock. The before/after stopped four-drive physical +census matched; the complete S3 inventory was captured before the contender, +and the server returned a known 503 with no successful VersionId. A +post-contender S3 inventory was not captured. The new recovery planner accepts +this exact archived boundary for preservation; it does not make a canonical +PASS claim or authorize replay. +Canonical acceptance and its 104/158 count remain unchanged. I05 lock-window diagnosis (2026-10-22): exact candidate source diff --git a/scripts/validation_heal/lock_timeout.py b/scripts/validation_heal/lock_timeout.py index 8dff960e..5dff0a74 100644 --- a/scripts/validation_heal/lock_timeout.py +++ b/scripts/validation_heal/lock_timeout.py @@ -70,7 +70,8 @@ def require_exact_inventory_entry(inventory, bucket, entry): return matches[0] -def validate_timeout_ack(ack, request, request_bytes, run_id, endpoint_name, bucket, key, body): +def validate_timeout_ack(ack, request, request_bytes, run_id, endpoint_name, bucket, key, body, + *, clock_domain=CLOCK_DOMAIN): expected_params = {'Bucket': bucket, 'Key': key, 'Metadata': {'heal-fixture': 'I05-timeout-contender'}} expected_payload = {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()} @@ -85,12 +86,12 @@ def validate_timeout_ack(ack, request, request_bytes, run_id, endpoint_name, buc and request.get('expectHttp') == 503 and request.get('casePassClaimed') is False and request.get('payload') == expected_payload - and ack.get('clockDomain') == request.get('clockDomain') == CLOCK_DOMAIN, + and ack.get('clockDomain') == request.get('clockDomain') == clock_domain, 'I05 timeout ACK is not durably bound to the exact contender PUT') observation = ack.get('observation') require(isinstance(observation, dict) and observation.get('endpoint') == endpoint_name and observation.get('operation') == 'put_object' and observation.get('params') == expected_params - and observation.get('payload') == expected_payload and observation.get('clockDomain') == CLOCK_DOMAIN + and observation.get('payload') == expected_payload and observation.get('clockDomain') == clock_domain and observation.get('clientCallInvoked') is True and observation.get('acknowledged') is False and observation.get('resultUnknown') is False and observation.get('clientCleanupError') is None @@ -120,7 +121,34 @@ def timeout_proof(rows, response, holder, bucket, key): and isinstance(request_id, str) and str(uuid.UUID(request_id)) == request_id and uuid.UUID(request_id).int and request_id != holder['request_id'] and response.get('VersionId') is None, 'I05 contender lacks an exact structured lock-timeout refusal') - selected = [row for row in rows if row.get('target') == lock_observer.TARGET and row.get('request_id') == request_id] + request_rows = [row for row in rows if row.get('target') == lock_observer.TARGET + and row.get('request_id') == request_id] + require(all(row.get('mode') in ('read', 'write') for row in request_rows), + 'I05 contender emitted an unknown namespace lock mode') + read_rows = [row for row in request_rows if row.get('mode') == 'read'] + write_rows = [row for row in request_rows if row.get('mode') == 'write'] + selected = [row for row in write_rows if row.get('op') == 'put_object_commit' + and row.get('bucket') == bucket and row.get('object') == key] + require(len(selected) == len(write_rows), + 'I05 contender made another namespace write-lock attempt') + if read_rows: + require([row.get('state') for row in read_rows] == ['acquiring', 'acquired', 'guard_dropped'] + and len({(row.get('owner'), row.get('namespace'), row.get('lock_attempt_id')) + for row in read_rows}) == 1, + 'I05 contender bucket-incarnation read lock has an incomplete lifecycle') + for row in read_rows: + require(row.get('event') == 'object_namespace_lock' and row.get('component') == 'ecstore' + and row.get('subsystem') == 'object_lock' and row.get('op') == 'get_object' + and row.get('bucket') == '.rustfs.sys' + and row.get('object') == f'buckets/{bucket}/.bucket-incarnation' + and row.get('namespace') == holder['namespace'] and row.get('owner') == holder['owner'] + and row.get('requested_version_id') is None + and type(row.get('timeout_ms')) is int and row['timeout_ms'] == 1000 + and type(row.get('acquire_ms')) is int and row['acquire_ms'] >= 0 + and isinstance(row.get('lock_attempt_id'), str) + and str(uuid.UUID(row['lock_attempt_id'])) == row['lock_attempt_id'] + and uuid.UUID(row['lock_attempt_id']).int, + 'I05 contender bucket-incarnation read lock changed identity or budget') if not selected: return None require([row.get('state') for row in selected] in (['acquiring'], ['acquiring', 'failed']), @@ -139,6 +167,14 @@ def timeout_proof(rows, response, holder, bucket, key): and str(uuid.UUID(row['lock_attempt_id'])) == row['lock_attempt_id'] and uuid.UUID(row['lock_attempt_id']).int and row['lock_attempt_id'] != holder['lock_attempt_id'], 'I05 contender has another lock budget, object, namespace or owner') + http_spans = [span for span in [row.get('span'), *row.get('spans', [])] + if isinstance(span, dict) and span.get('name') == 'http-request'] + if http_spans: + unique_http_spans = {canonical(span): span for span in http_spans} + require(len(unique_http_spans) == 1 and all( + span.get('request_id') == request_id and span.get('method') == 'PUT' + and span.get('uri') == f'/{bucket}/{key}' for span in unique_http_spans.values()), + 'I05 contender lock attempt is not bound to its exact PUT request') require(len({(row['owner'], row['lock_attempt_id']) for row in selected}) == 1, 'I05 contender changed its original write owner') if len(selected) == 1: diff --git a/scripts/validation_heal/timeout_retention.py b/scripts/validation_heal/timeout_retention.py index b1ffc6ab..b279cba7 100644 --- a/scripts/validation_heal/timeout_retention.py +++ b/scripts/validation_heal/timeout_retention.py @@ -1,4 +1,4 @@ -"""Isolate the archived I05 H1 failure before any contender was invoked.""" +"""Isolate archived I05 H1 failures with bounded, understood request effects.""" import hashlib import json @@ -10,32 +10,45 @@ from .heal_workload import verify_seed from .healthy import physical_content from .lock_observer import CANDIDATE + from .lock_timeout import CONTENDER_BYTES, timeout_proof, validate_timeout_ack from .race_retention import digest, validate except ImportError: from heal_probe import canonical, payload, require from heal_workload import verify_seed from healthy import physical_content from lock_observer import CANDIDATE + from lock_timeout import CONTENDER_BYTES, timeout_proof, validate_timeout_ack from race_retention import digest, validate CASE = 'heal.lock-timeout-and-caller-abort' -HELPER = '704bd606071676dded130dca91d3d5278455bcaa' -FAILURE = {'type': 'ValueError', - 'message': 'caller lock diagnostic escaped its exact GET identity or runtime budget'} -LABELS = ('binding', 'inspector-build', 'preflight', 'integrity-fleet-attestation', 'reserved', +PRE_CONTENDER_HELPER = '704bd606071676dded130dca91d3d5278455bcaa' +CONTENDER_HELPER = 'd100abeb8e85c4000a493aa994031e4a5d01518e' +FAILURES = { + PRE_CONTENDER_HELPER: {'type': 'ValueError', + 'message': 'caller lock diagnostic escaped its exact GET identity or runtime budget'}, + CONTENDER_HELPER: {'type': 'ValueError', + 'message': 'I05 contender acquired a lock or lost its exact timeout lifecycle'}, +} +COMMON_LABELS = ('binding', 'inspector-build', 'preflight', 'integrity-fleet-attestation', 'reserved', 'inspector-upload', 'started', 'deployed-topology', 'actual-topology', 'original-ACK-bound-S3', 'original-ACK-proof', 'I05-timeout-original-physical', 'I05-timeout-original-S3', 'I05-timeout-baseline-physical-after-public-read', 'I05-caller-baseline-resources', - 'I05-caller-checkpoint', 'I05-caller-window', 'quiescent', 'original-failure-scene', - 'original-failure-logs', 'diagnostic-result') + 'I05-caller-checkpoint') +LABELS = { + PRE_CONTENDER_HELPER: COMMON_LABELS + ('I05-caller-window', 'quiescent', + 'original-failure-scene', 'original-failure-logs', 'diagnostic-result'), + CONTENDER_HELPER: COMMON_LABELS + ('I05-caller-window', 'I05-caller-before-disconnect-socket', + 'I05-caller-window', 'I05-caller-admission-resources', 'I05-caller-window', 'quiescent', + 'original-failure-scene', 'original-failure-logs', 'diagnostic-result'), +} def archived_plan(artifact, failed_run, candidate): - """Refuse uncertain seed effects, later actions, mixed cases and physical drift. + """Refuse uncertain effects, later actions, mixed cases and physical drift. This is preservation, with no S3 call, backup restoration or verdict change. - The old helper's exact pre-contender boundary is deliberately the only route. + A contender is accepted only for the reviewed helper's exact refused PUT. """ require(type(failed_run) is int and failed_run > 0, 'I05 requires an exact failed run') paths = list(artifact.glob('*/run/*/cases/*.json')) @@ -49,25 +62,32 @@ def read(path): require(result.get('case_id') == CASE and result.get('verdict') == result.get('cleanup_status') == 'FAIL' and provenance.get('source_commit') == candidate.get('source_sha') == CANDIDATE and provenance.get('candidate_sha256') == candidate.get('package_sha256') - and provenance.get('testing_commit') == HELPER, - 'I05 requires its original failed candidate and pre-contender helper') + and provenance.get('testing_commit') in FAILURES, + 'I05 requires its original failed candidate and reviewed helper boundary') + helper = provenance['testing_commit'] + failure = FAILURES[helper] evidence = paths[0].parents[1] / 'evidence' / CASE summary = read(evidence / 'healthy-summary.json') row = {'topology': 'H1', 'diagnosticComplete': False, 'cleanup': 'FAIL', 'wholeCasePassClaimed': False, 'canonicalCasePassClaimed': False, - 'failure': FAILURE, 'namespacesRetained': True} + 'failure': failure, 'namespacesRetained': True} require(summary.get('caseId') == CASE and summary.get('cleanup') == 'FAIL' and summary.get('diagnostics') == [row] and summary.get('complete') is False and summary.get('diagnosticComplete') is False and summary.get('requestedTopologies') == ['H1', 'H2'] and summary.get('canonicalCasePassClaimed') is False - and not (evidence / 'H2').exists(), 'I05 requires the sole retained pre-contender H1') + and not (evidence / 'H2').exists(), 'I05 requires the sole retained H1 failure') folder = evidence / 'H1' - numbered = {label: f'{index:05d}-{label}.json' for index, label in enumerate(LABELS, 1)} - require({path.name for path in folder.iterdir()} == set(numbered.values()) | {'fixture', 'timeout'}, + labels = LABELS[helper] + numbered = [(label, f'{index:05d}-{label}.json') for index, label in enumerate(labels, 1)] + require({path.name for path in folder.iterdir()} == {name for _, name in numbered} | {'fixture', 'timeout'}, 'I05 archive contains another action or lacks its original boundary') def one(label): - return read(folder / numbered[label]) + matches = [name for current, name in numbered if current == label] + require(len(matches) == 1, 'I05 archive lacks one exact ' + label) + return read(folder / matches[0]) + def many(label): + return [read(folder / name) for current, name in numbered if current == label] binding, reservation, scene = one('binding'), one('reserved'), one('original-failure-scene') run_id, bucket = binding.get('runId', ''), binding.get('bucketPrefix', '') require(re.fullmatch(rf'gh-{failed_run}-[1-9][0-9]*-h1', run_id) @@ -75,7 +95,7 @@ def one(label): and binding.get('candidateCommit') == CANDIDATE and binding.get('candidatePackageSha256') == candidate['package_sha256'] and binding.get('candidateBinarySha256') == provenance.get('binary_sha256') - and binding.get('harnessCommit') == HELPER + and binding.get('harnessCommit') == helper and binding.get('endpoints') == {'node1': 'http://rustfs-node1:9000'} and binding.get('defaultEndpoint') == 'node1' and one('diagnostic-result') == row, 'I05 original run, package, helper, binary or failure boundary differs') @@ -173,9 +193,14 @@ def one(label): 'I05 original public metadata, ETag or tags differ from seed request/ACK') timeout = folder / 'timeout' - require({p.name for p in timeout.iterdir()} == {'scope.json', 'failure.json', 'caller'} - and {p.name for p in (timeout / 'caller').iterdir()} == {'scope.json', 'failure.json', 'transport'}, - 'I05 contender or later controller action may have been invoked') + expected_timeout_items = {'scope.json', 'failure.json', 'caller'} + expected_caller_items = {'scope.json', 'failure.json', 'transport'} + if helper == CONTENDER_HELPER: + expected_timeout_items.add('contender') + expected_caller_items.add('admission.json') + require({p.name for p in timeout.iterdir()} == expected_timeout_items + and {p.name for p in (timeout / 'caller').iterdir()} == expected_caller_items, + 'I05 archive contains an unreviewed action or lacks its exact failure boundary') scope = read(timeout / 'scope.json') entry = scope.get('entry') require(entry in expected and entry.get('key') == 'objects/0000.bin' and not entry['isLatest'] @@ -211,11 +236,52 @@ def one(label): and value.get('resultUnknown') is True and value.get('acknowledged') is False and value.get('serverOwnershipProven') is False, 'I05 closed caller was retried or promoted to an acknowledged result') + if helper == CONTENDER_HELPER: + contender = timeout / 'contender' + require({p.name for p in contender.iterdir()} == {'000001-request.json', '000001-ack.json'}, + 'I05 contender archive is not the sole one-shot request and ACK') + request_path, ack_path = contender / '000001-request.json', contender / '000001-ack.json' + request_bytes = request_path.read_bytes() + request, ack = read(request_path), read(ack_path) + body = payload(f'{run_id}/I05-timeout-contender', CONTENDER_BYTES) + response = validate_timeout_ack(ack, request, request_bytes, run_id, 'node1', bucket, + entry['key'], body, clock_domain=domain) + require(ack.get('operationCheck') == 'PASS' + and ack.get('observation', {}).get('acknowledged') is False + and ack.get('observation', {}).get('resultUnknown') is False, + 'I05 contender result is not a durable, known refusal') + holder = (read(timeout / 'caller/admission.json').get('attempt') or {}) + require(holder.get('admitted') is True and holder.get('observedAcquiredWithoutDrop') is True, + 'I05 contender lacks its original admitted GET lock holder') + windows = many('I05-caller-window') + require(len(windows) == 3, 'I05 contender archive lacks all three caller journal windows') + rows = [] + previous_cursor = None + for window in windows: + require(window.get('complete') is True and window.get('node') == 'vm000' + and window.get('unit') == owner['unit'] and not window.get('sourceMessageLoss') + and isinstance(window.get('beforeCursor'), str) + and isinstance(window.get('afterCursor'), str) + and (previous_cursor is None or window['beforeCursor'] == previous_cursor) + and isinstance(window.get('journal'), str), + 'I05 contender lock journal is incomplete or belongs to another service') + previous_cursor = window['afterCursor'] + for line in window['journal'].splitlines(): + try: + event = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError('I05 contender lock journal contains malformed JSON') from error + require(isinstance(event, dict), 'I05 contender lock journal contains a non-object event') + rows.append(event) + proof = timeout_proof(rows, response, holder, bucket, entry['key']) + require(proof is not None and proof.get('typedTimeoutBeyondBudgetObserved') is True + and proof.get('writeLockAcquired') is False, + 'I05 contender ACK lacks its exact failed target write-lock attempt') for path in timeout.rglob('*.json'): read(path) journal[path.relative_to(folder).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() plan = {'kind': 'unresolved-i05-timeout', 'runId': run_id, 'nodes': ['vm000'], 'bucket': bucket, - 'owners': {'vm000': owner}, 'candidateCommit': CANDIDATE, 'harnessCommit': HELPER, + 'owners': {'vm000': owner}, 'candidateCommit': CANDIDATE, 'harnessCommit': helper, 'binarySha256': binding['candidateBinarySha256'], 'ackFilesSha256': digest(journal), 'businessSceneSha256': {'vm000': digest(physical_content(scene)['vm000'])}} validate(plan) diff --git a/test_validation_heal_lock_timeout.py b/test_validation_heal_lock_timeout.py index 202f67c9..9ba6d34f 100644 --- a/test_validation_heal_lock_timeout.py +++ b/test_validation_heal_lock_timeout.py @@ -71,8 +71,24 @@ def s3(self, operation, params, body=None): 'lock_attempt_id': '40000000-0000-4000-8000-000000000001', 'state': state, 'level': 'TRACE' if state == 'acquiring' else 'ERROR', 'timeout_ms': 1000, 'acquire_ms': 0 if state == 'acquiring' else 1001, + 'spans': [{'name': 'http-request', 'request_id': CONTENDER, 'method': 'PUT', + 'uri': f'/{fixture.BUCKET}/{self.entry["key"]}'}], **({} if state == 'acquiring' else {'failure': 'timeout'}), } for state in ('acquiring', 'failed')] + if self.change in ('bucket-read', 'extra-write-lock'): + self.scene.contender_rows[:0] = [{ + 'target': fixture.observer.TARGET, 'request_id': CONTENDER, + 'event': 'object_namespace_lock', 'component': 'ecstore', 'subsystem': 'object_lock', + 'op': 'get_object', 'mode': 'read', 'bucket': '.rustfs.sys', + 'object': f'buckets/{fixture.BUCKET}/.bucket-incarnation', + 'namespace': 'namespace-one', 'owner': 'owner-one', + 'lock_attempt_id': '40000000-0000-4000-8000-000000000002', + 'state': state, 'level': 'TRACE', 'timeout_ms': 1000, 'acquire_ms': 0, + } for state in ('acquiring', 'acquired', 'guard_dropped')] + if self.change == 'extra-write-lock': + self.scene.contender_rows.append({**self.scene.contender_rows[-1], + 'op': 'delete_object', 'mode': 'write', 'object': self.entry['key'], + 'lock_attempt_id': '40000000-0000-4000-8000-000000000003'}) if self.change == 'no-journal': self.scene.contender_rows = [] response = {'ResponseMetadata': {'HTTPStatusCode': 503, 'RetryAttempts': 0, 'RequestId': CONTENDER, 'HTTPHeaders': {'retry-after': '1'}}, 'Error': {'Code': 'ServiceUnavailable'}} @@ -107,6 +123,34 @@ def test_complete_timeout_keeps_real_caller_unknown_and_physical_check_precedes_ self.assertTrue((self.output / 'contender/000001-request.json').is_file()) self.assertTrue((self.output / 'contender/000001-ack.json').is_file()) + def test_bucket_incarnation_read_lock_does_not_mask_target_write_timeout(self): + self.change = 'bucket-read' + proof = self.invoke() + self.assertTrue(proof['diagnosticComplete']) + self.assertTrue(proof['callerObservation']['admittedWork']['typedTimeoutBeyondBudgetObserved']) + + def test_an_additional_write_lock_attempt_is_rejected(self): + self.change = 'extra-write-lock' + with self.assertRaisesRegex(ValueError, 'another namespace write-lock attempt'): + self.invoke() + self.assertTrue(self.scene.wire.server.disconnected.wait(3)) + self.assertFalse((self.output / 'diagnostic.json').exists()) + + def test_incomplete_bucket_incarnation_read_lock_lifecycle_is_rejected(self): + self.change = 'bucket-read' + original = self.probe.s3 + def s3(operation, params, body=None): + value = original(operation, params, body) + if operation == 'put_object': + self.scene.contender_rows = [row for row in self.scene.contender_rows + if not (row.get('mode') == 'read' and row.get('state') == 'guard_dropped')] + return value + self.probe.s3 = s3 + with self.assertRaisesRegex(ValueError, 'bucket-incarnation read lock has an incomplete lifecycle'): + self.invoke() + self.assertTrue(self.scene.wire.server.disconnected.wait(3)) + self.assertFalse((self.output / 'diagnostic.json').exists()) + def test_small_target_is_not_observed_without_sending_contender_or_get(self): self.entry['payload']['size'] = 1 with self.assertRaisesRegex(ValueError, 'too small'): self.invoke() diff --git a/test_validation_heal_timeout_retention.py b/test_validation_heal_timeout_retention.py index e1e666c8..c02d50bf 100644 --- a/test_validation_heal_timeout_retention.py +++ b/test_validation_heal_timeout_retention.py @@ -12,6 +12,8 @@ from scripts.validation_heal import timeout_retention as controller, retained_scene as retained from scripts.validation_heal import race_retention, recover_environment as recovery +from scripts.validation_heal.heal_probe import canonical, payload +from scripts.validation_heal.lock_timeout import CONTENDER_BYTES import test_validation_heal_race_retention as race_tests @@ -28,6 +30,125 @@ def archive(self, directory): folder = artifact / 'run' / controller.CASE / 'evidence' / controller.CASE / 'H1' return root, candidate, folder + def add_contender_archive(self, root, candidate, folder): + def read(path): return json.loads(path.read_text()) + def write(path, value): path.write_text(json.dumps(value, sort_keys=True, separators=(',', ':'))) + + result_path = folder.parents[2] / 'cases' / f'{controller.CASE}.json' + result = read(result_path) + result['provenance']['testing_commit'] = controller.CONTENDER_HELPER + write(result_path, result) + binding_path = folder / '00001-binding.json' + binding = read(binding_path) + binding['harnessCommit'] = controller.CONTENDER_HELPER + write(binding_path, binding) + binding_hash = hashlib.sha256(canonical(binding).encode()).hexdigest() + + summary_path = folder.parent / 'healthy-summary.json' + summary = read(summary_path) + row = dict(summary['diagnostics'][0]) + row['failure'] = controller.FAILURES[controller.CONTENDER_HELPER] + summary['diagnostics'] = [row] + write(summary_path, summary) + diagnostic_path = folder / '00021-diagnostic-result.json' + diagnostic = read(diagnostic_path) + diagnostic['failure'] = controller.FAILURES[controller.CONTENDER_HELPER] + write(diagnostic_path, diagnostic) + + renamed = ( + ('00018-quiescent.json', '00022-quiescent.json'), + ('00019-original-failure-scene.json', '00023-original-failure-scene.json'), + ('00020-original-failure-logs.json', '00024-original-failure-logs.json'), + ('00021-diagnostic-result.json', '00025-diagnostic-result.json'), + ) + for old, new in renamed: (folder / old).rename(folder / new) + + first = read(folder / '00017-I05-caller-window.json') + first_after = first['afterCursor'] + middle_after = first_after + '-middle' + last_after = middle_after + '-last' + for name, before, after in ( + ('00019-I05-caller-window.json', first_after, middle_after), + ('00021-I05-caller-window.json', middle_after, last_after)): + write(folder / name, {'node': 'vm000', 'unit': 'rustfs-validation-heal-' + + binding['runId'] + '.service', 'complete': True, 'beforeCursor': before, + 'afterCursor': after, 'hasMore': False, 'sourceMessageLoss': [], 'journal': ''}) + write(folder / '00018-I05-caller-before-disconnect-socket.json', {'readOnly': True}) + write(folder / '00020-I05-caller-admission-resources.json', {'readOnly': True}) + + scope_path = folder / 'timeout/scope.json' + scope = read(scope_path) + scope['bindingSha256'] = binding_hash + write(scope_path, scope) + run_id, bucket, key = scope['runId'], binding['bucketPrefix'], scope['entry']['key'] + caller_path = folder / 'timeout/caller' + for name in ('request.json', 'request-written.json', 'disconnected.json'): + request_path = caller_path / 'transport' / name + caller_request = read(request_path) + caller_request['harnessCommit'] = controller.CONTENDER_HELPER + caller_request['bindingSha256'] = binding_hash + write(request_path, caller_request) + caller_request_bytes = (caller_path / 'transport/request.json').read_bytes() + for name in ('request-written.json', 'disconnected.json'): + request_path = caller_path / 'transport' / name + caller_request = read(request_path) + caller_request['requestSha256'] = hashlib.sha256(caller_request_bytes).hexdigest() + write(request_path, caller_request) + caller_request = read(caller_path / 'transport/request.json') + domain = caller_request['clockDomain'] + contender = folder / 'timeout/contender' + contender.mkdir() + body = payload(f'{run_id}/I05-timeout-contender', CONTENDER_BYTES) + params = {'Bucket': bucket, 'Key': key, 'Metadata': {'heal-fixture': 'I05-timeout-contender'}} + request = {'schemaVersion': 2, 'runId': run_id, 'sequence': 1, 'operation': 'put_object', + 'params': params, 'identity': None, 'expectHttp': 503, 'clockDomain': domain, + 'preparedMonotonicNs': 1000000, 'preparedAt': '2026-10-22T00:00:00+00:00', + 'casePassClaimed': False, + 'payload': {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}} + request_bytes = json.dumps(request, sort_keys=True, separators=(',', ':')).encode() + (contender / '000001-request.json').write_bytes(request_bytes) + request_id = '30000000-0000-4000-8000-000000000001' + response = {'Error': {'Code': 'ServiceUnavailable', 'Message': 'retry'}, + 'ResponseMetadata': {'RequestId': request_id, 'HostId': '', 'HTTPStatusCode': 503, + 'HTTPHeaders': {'retry-after': '1', 'x-amz-request-id': request_id}, 'RetryAttempts': 0}} + ack = {'schemaVersion': 2, 'runId': run_id, 'sequence': 1, 'requestFile': '000001-request.json', + 'requestSha256': hashlib.sha256(request_bytes).hexdigest(), 'operationCheck': 'PASS', + 'casePassClaimed': False, 'clockDomain': domain, + 'collectorCallStartedMonotonicNs': 1001000, 'collectorCallFinishedMonotonicNs': 2001000, + 'recordedMonotonicNs': 2002000, 'observation': {'endpoint': 'node1', 'operation': 'put_object', + 'params': params, 'payload': request['payload'], 'clockDomain': domain, + 'clientCallInvoked': True, 'acknowledged': False, 'resultUnknown': False, + 'clientCleanupError': None, 'callStartedMonotonicNs': 1002000, + 'callFinishedMonotonicNs': 2000000, 'response': response}} + write(contender / '000001-ack.json', ack) + + admission = {'node': 'vm000', 'attempt': {'request_id': '20000000-0000-4000-8000-000000000001', + 'lock_attempt_id': '20000000-0000-4000-8000-000000000002', 'owner': ':::9000', + 'namespace': 'pool-0', 'admitted': True, 'observedAcquiredWithoutDrop': True}} + write(caller_path / 'admission.json', admission) + rows = [] + def lock_row(mode, op, obj, attempt, state, **extra): + return {'target': 'rustfs_ecstore::object_lock_diag', 'request_id': request_id, + 'event': 'object_namespace_lock', 'component': 'ecstore', 'subsystem': 'object_lock', + 'op': op, 'mode': mode, 'bucket': '.rustfs.sys' if mode == 'read' else bucket, + 'object': obj, 'namespace': 'pool-0', 'owner': ':::9000', 'lock_attempt_id': attempt, + 'requested_version_id': None, 'timeout_ms': 1000, 'acquire_ms': 1001 if state == 'failed' else 0, + 'state': state, 'level': 'ERROR' if state == 'failed' else 'TRACE', + 'span': {'name': 'put_object_with_old_current_size'}, + 'spans': [{'name': 'http-request', 'request_id': request_id, 'method': 'PUT', + 'uri': f'/{bucket}/{key}'}], **extra} + read_attempt = '20000000-0000-4000-8000-000000000003' + for state in ('acquiring', 'acquired', 'guard_dropped'): + rows.append(lock_row('read', 'get_object', f'buckets/{bucket}/.bucket-incarnation', read_attempt, state)) + write_attempt = '20000000-0000-4000-8000-000000000004' + rows.append(lock_row('write', 'put_object_commit', key, write_attempt, 'acquiring')) + rows.append(lock_row('write', 'put_object_commit', key, write_attempt, 'failed', failure='timeout')) + write(folder / '00021-I05-caller-window.json', {'node': 'vm000', 'unit': 'rustfs-validation-heal-' + + binding['runId'] + '.service', 'complete': True, 'beforeCursor': middle_after, + 'afterCursor': last_after, 'hasMore': False, 'sourceMessageLoss': [], + 'journal': '\n'.join(json.dumps(row, sort_keys=True, separators=(',', ':')) for row in rows)}) + return contender + def test_original_live_failure_yields_exact_pre_contender_preservation_plan(self): with tempfile.TemporaryDirectory() as directory: root, candidate, _ = self.archive(directory) @@ -41,6 +162,48 @@ def test_original_live_failure_yields_exact_pre_contender_preservation_plan(self self.assertEqual(plan['businessSceneSha256'], {'vm000': '6c9e808ff0c6d03cb5dfd5546bd588c3d3e1bcee3c167877dec21c4399fc1e6d'}) + def test_new_helper_failed_refusal_yields_exact_contender_preservation_plan(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, folder = self.archive(directory) + self.add_contender_archive(root, candidate, folder) + plan, = recovery.recovery_plan(root, 39711742431, candidate) + self.assertEqual(plan['kind'], 'unresolved-i05-timeout') + self.assertEqual(plan['harnessCommit'], controller.CONTENDER_HELPER) + self.assertEqual(plan['nodes'], ['vm000']) + self.assertEqual(plan['businessSceneSha256'], {'vm000': + '6c9e808ff0c6d03cb5dfd5546bd588c3d3e1bcee3c167877dec21c4399fc1e6d'}) + + def test_new_helper_refuses_unknown_success_or_corrupt_lock_lifecycle(self): + for variant in ('unknown', 'success', 'acquired', 'read-lifecycle', 'extra-action'): + with self.subTest(variant=variant), tempfile.TemporaryDirectory() as directory: + root, candidate, folder = self.archive(directory) + contender = self.add_contender_archive(root, candidate, folder) + ack_path = contender / '000001-ack.json' + ack = json.loads(ack_path.read_text()) + if variant == 'unknown': ack['observation']['resultUnknown'] = True + elif variant == 'success': + ack['observation']['response']['ResponseMetadata']['HTTPStatusCode'] = 200 + elif variant == 'extra-action': + (contender / '000002-request.json').write_text('{}') + elif variant == 'acquired': + window_path = folder / '00021-I05-caller-window.json' + window = json.loads(window_path.read_text()) + events = [json.loads(line) for line in window['journal'].splitlines()] + events[-1]['state'] = 'acquired' + window['journal'] = '\n'.join(json.dumps(event) for event in events) + window_path.write_text(json.dumps(window)) + elif variant == 'read-lifecycle': + window_path = folder / '00021-I05-caller-window.json' + window = json.loads(window_path.read_text()) + events = [json.loads(line) for line in window['journal'].splitlines()] + events[2]['object'] = 'buckets/foreign/.bucket-incarnation' + window['journal'] = '\n'.join(json.dumps(event) for event in events) + window_path.write_text(json.dumps(window)) + ack_path.write_text(json.dumps(ack)) + with patch.object(recovery, 'Fleet') as fleet, self.assertRaises((ValueError, KeyError)): + recovery.recovery_plan(root, 39711742431, candidate) + fleet.assert_not_called() + def test_tampered_or_unresolved_evidence_refuses_before_fleet_creation(self): variants = ('candidate', 'helper', 'missing-ack', 'ack-hash', 'ack-unknown', 'ack-operation', 'request-mutation', 'payload', 'marker', 'owner', 'physical', 'live', 'other-node', From 1b88cad3581e755d8e904697cb2ec471e04b5f8f Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 15:12:20 +0800 Subject: [PATCH 286/343] docs(validation): record I05 isolation evidence --- docs/validation-heal-operations.md | 14 +++++++++++--- docs/validation-migration-map.md | 15 +++++++++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 72d53427..76c5b04f 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -774,7 +774,7 @@ A separate protected prefix is checked during the fault. The original key-level ### I05 retained-scene isolation -`validation_cleanup_run_id=39711742431` selects the narrow I05 preservation +`validation_cleanup_run_id=` selects the narrow I05 preservation route with the original package build and an immutable helper containing `timeout_retention.py`. The driver must remain on the original Functional Chain branch. First check the shared current auto-testing fleet for other work. The @@ -796,8 +796,16 @@ isolation. `ISOLATED` never changes the original FAIL or supplies canonical acceptance. In run 39733064688, the server refused the contender after a 1001 ms target write-lock timeout while the exact GET held its read lock for 4.148 s; the validator failed because it combined the same PUT's bucket-incarnation -read-lock events with the target write-lock events. The failure and physical -scene remain preserved pending evidence-based isolation. +read-lock events with the target write-lock events. Read-only inspection +[39738071753/1](https://github.com/rustfs/rustfs/actions/runs/39738071753) +observed vm000 with no RustFS service, validation unit or port 9000 listener. +Recovery [39738426191/1](https://github.com/rustfs/rustfs/actions/runs/39738426191) +completed `ISOLATED` for vm000: all 65 failed-artifact files match +byte-for-byte, preflight/move/post-audit physical digests match, and four data +roots plus private state are retained in five root-owned mode-0700 same-device +archive parents. No backup restoration, namespace deletion, S3 request or +original FAIL rewrite occurred. Inspection also observed old failed F07 units +on vm001-vm003; the I05 plan did not touch those nodes. The first live run's GET already dropped its lock before observation. The version-field correction correlates those original events but cannot reopen diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 97d8ad14..9d94b8ba 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2359,8 +2359,19 @@ census matched; the complete S3 inventory was captured before the contender, and the server returned a known 503 with no successful VersionId. A post-contender S3 inventory was not captured. The new recovery planner accepts this exact archived boundary for preservation; it does not make a canonical -PASS claim or authorize replay. -Canonical acceptance and its 104/158 count remain unchanged. +PASS claim or authorize replay. Read-only environment inspection +[39738071753/1](https://github.com/rustfs/rustfs/actions/runs/39738071753) +completed with stable observations of all four current Heal nodes and no +RustFS service, validation unit or port-9000 listener on the target vm000. +Evidence-based isolation [39738426191/1](https://github.com/rustfs/rustfs/actions/runs/39738426191) +completed `ISOLATED` for the exact vm000 scene: all 65 original artifact files +match byte-for-byte; preflight, move and post-audit physical digests match; +four data roots and private state remain in five root-owned mode-0700 +same-device archive parents. No S3 request, backup restoration, namespace +deletion or original verdict rewrite occurred. The inspection also observed +previous failed F07 units on vm001-vm003, which were outside this plan. The +original run remains FAIL/cleanup FAIL. Canonical acceptance and its 104/158 +count remain unchanged. I05 lock-window diagnosis (2026-10-22): exact candidate source From 22589bc00836b5d47f382429313e967c1dfc38a9 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 15:50:53 +0800 Subject: [PATCH 287/343] fix(validation): retain I05 H2 failure scenes --- docs/validation-heal-operations.md | 26 +- docs/validation-migration-map.md | 46 +++- scripts/validation_heal/timeout_retention.py | 215 ++++++++++++--- test_validation_heal_timeout_retention.py | 266 +++++++++++++++++++ 4 files changed, 492 insertions(+), 61 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 76c5b04f..0e495ba6 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -779,21 +779,29 @@ route with the original package build and an immutable helper containing `timeout_retention.py`. The driver must remain on the original Functional Chain branch. First check the shared current auto-testing fleet for other work. The planner accepts the original helper 704bd60 failure before contender -submission, and the reviewed helper d100abeb failure after one exact known 503 -contender refusal. Both require nine exact successful seed request/ACK pairs, -complete version/marker/payload/public metadata baseline, a closed read-only -historical GET, original owner/candidate/inspector and unchanged stopped -four-drive scene. The contender boundary also verifies its request/ACK digest, -server request ID, full bucket-incarnation read-lock lifecycle, and exact -failed target write-lock lifecycle. Unknown or successful writes, another -action, H2 evidence or physical drift refuse recovery. +submission and the reviewed helper d100abeb failure after one exact known 503 +contender refusal. It also accepts helper 1b88cad when H1 completed and cleaned +successfully before H2 reached that same exact contender failure. The H1 path +requires nine exact successful seed request/ACK pairs, complete +version/marker/payload/public metadata baseline, a closed read-only historical +GET, original owner/candidate/inspector and an unchanged stopped four-drive +scene. The contender boundary also verifies its request/ACK digest, server +request ID, full bucket-incarnation read-lock lifecycle and exact failed target +write-lock lifecycle. The H2 path additionally requires four reservations, +four inspector uploads, per-node lock checkpoints and balanced four-peer +journal rounds before disconnect, after admission, and during contender +observation, four stopped physical scenes and four service logs. +Unknown or successful writes, missing peers, another action, an unclean H1 or +physical drift refuse recovery. The recovery reuses retained-scene audit/isolate/post-audit, retaining all four roots and state under private per-run parents without S3 calls, backup restoration or deletion. Verify all original archived files byte-for-byte and all five final private parents independently before recording active namespace isolation. `ISOLATED` never changes the original FAIL or supplies canonical -acceptance. In run 39733064688, the server refused the contender after a 1001 +acceptance. H2 recovery requires the completed H1 diagnostic and its successful +cleanup; it isolates all four H2 peers together when the H2 contender boundary +fails. In run 39733064688, the server refused the contender after a 1001 ms target write-lock timeout while the exact GET held its read lock for 4.148 s; the validator failed because it combined the same PUT's bucket-incarnation read-lock events with the target write-lock events. Read-only inspection diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 9d94b8ba..1fb336ef 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2330,17 +2330,21 @@ one; that difference is not yet explained. Neither a wrong package nor a source mix is inferred from it. Do not replay the same warmed baseline premise or claim that the field correction establishes lock contention. -`timeout_retention.py` recognizes two exact H1 evidence boundaries: the first -helper's failure before a contender, and the reviewed `d100abeb` helper's -failure after one contender PUT received a known 503. Both require the exact -candidate/package/binary/inspector and owner, all nine successful seed +`timeout_retention.py` recognizes three exact I05 failure boundaries: the first +helper's H1 failure before a contender, the reviewed `d100abeb` H1 failure +after one contender PUT received a known 503, and current helper `1b88cad` H2 +failure after H1 completed and cleaned successfully. The H1 paths require the +exact candidate/package/binary/inspector and owner, all nine successful seed request/ACK pairs, deterministic bytes, public metadata/tags/versions, closed -read-only historical GET and unchanged stopped four-drive physical scene. -The contender boundary additionally checks its only durable request/ACK, -request digest and clock ordering, structured `ServiceUnavailable` response, -request-correlated bucket-incarnation read-lock lifecycle, and exact failed -`put_object_commit` write-lock lifecycle. Unknown or successful writes, changed -physical evidence, another case, H2 or extra actions refuse recovery. +read-only historical GET and unchanged stopped four-drive physical scene. The +H2 path also requires all four owners and stopped physical scenes, inspector +uploads, preflight and deployment proof, per-node resource baselines, lock +checkpoints, three complete lock journal windows, and service logs. Every +contender boundary checks its only durable request/ACK, request digest and +clock ordering, structured `ServiceUnavailable` response, request-correlated +bucket-incarnation read-lock lifecycle, and exact failed `put_object_commit` +write-lock lifecycle. Unknown or successful writes, changed physical evidence, +missing peers, an unclean H1, another case or extra actions refuse recovery. Recovery reuses retained-scene isolation with same-filesystem non-overwriting renames, durable intent, preflight, interrupted-move retries and five private @@ -2358,7 +2362,7 @@ with its target write lock. The before/after stopped four-drive physical census matched; the complete S3 inventory was captured before the contender, and the server returned a known 503 with no successful VersionId. A post-contender S3 inventory was not captured. The new recovery planner accepts -this exact archived boundary for preservation; it does not make a canonical +the exact archived H1 boundary for preservation; it does not make a canonical PASS claim or authorize replay. Read-only environment inspection [39738071753/1](https://github.com/rustfs/rustfs/actions/runs/39738071753) completed with stable observations of all four current Heal nodes and no @@ -2393,6 +2397,20 @@ runtime profiles are unchanged. The existing remote process observer checks every configured setting, so it will reject a deployment that omits or changes this value. Focused verification ran 50 observer/timeout/runner tests: 48 passed and two existing Linux-only tests were skipped on macOS; `git diff ---check` passed. Hosted CI and a fresh candidate-bound live diagnostic remain -required. The previous FAIL and its isolated scene remain retained; this local -runtime correction is not live or canonical acceptance. +--check` passed. The previous FAIL and its isolated scene remain retained; this +local runtime correction is not live or canonical acceptance. + +I05 H2 retained-scene recovery (2026-10-22): `timeout_retention.py` now accepts +the exact `1b88cad` H2 contender failure only after H1 completed with cleanup +PASS. The H2 plan binds all four reserved owners, inspector uploads, preflight +mounts, service identities/settings, journal checkpoints and balanced peer +sampling rounds, plus the four stopped physical scenes and failure logs. It +still requires the exact nine successful seed ACKs, complete public baseline, +closed historical GET and known contender 503/lock lifecycle. Unknown, +successful, foreign-peer, partial or physically changed evidence refuses +before creating a Fleet or moving any root. H2 isolation therefore keeps the +four peers together and never changes the original FAIL. An H2 archive fixture +and negative evidence cases were added. The matching local CI group ran 117 +tests successfully with two existing Linux-only tests skipped on macOS; the +previous single timeout FD test also passed when rerun in isolation. No new +live diagnostic or canonical acceptance is established by these checks. diff --git a/scripts/validation_heal/timeout_retention.py b/scripts/validation_heal/timeout_retention.py index b279cba7..c8a36497 100644 --- a/scripts/validation_heal/timeout_retention.py +++ b/scripts/validation_heal/timeout_retention.py @@ -1,4 +1,4 @@ -"""Isolate archived I05 H1 failures with bounded, understood request effects.""" +"""Isolate archived I05 H1/H2 failures with bounded, understood request effects.""" import hashlib import json @@ -9,14 +9,16 @@ from .heal_probe import canonical, payload, require from .heal_workload import verify_seed from .healthy import physical_content - from .lock_observer import CANDIDATE + from .fleet_node import runtime_config + from .lock_observer import CANDIDATE, same_process from .lock_timeout import CONTENDER_BYTES, timeout_proof, validate_timeout_ack from .race_retention import digest, validate except ImportError: from heal_probe import canonical, payload, require from heal_workload import verify_seed from healthy import physical_content - from lock_observer import CANDIDATE + from fleet_node import runtime_config + from lock_observer import CANDIDATE, same_process from lock_timeout import CONTENDER_BYTES, timeout_proof, validate_timeout_ack from race_retention import digest, validate @@ -24,11 +26,14 @@ CASE = 'heal.lock-timeout-and-caller-abort' PRE_CONTENDER_HELPER = '704bd606071676dded130dca91d3d5278455bcaa' CONTENDER_HELPER = 'd100abeb8e85c4000a493aa994031e4a5d01518e' +H2_CONTENDER_HELPER = '1b88cad3581e755d8e904697cb2ec471e04b5f8f' FAILURES = { PRE_CONTENDER_HELPER: {'type': 'ValueError', 'message': 'caller lock diagnostic escaped its exact GET identity or runtime budget'}, CONTENDER_HELPER: {'type': 'ValueError', 'message': 'I05 contender acquired a lock or lost its exact timeout lifecycle'}, + H2_CONTENDER_HELPER: {'type': 'ValueError', + 'message': 'I05 contender acquired a lock or lost its exact timeout lifecycle'}, } COMMON_LABELS = ('binding', 'inspector-build', 'preflight', 'integrity-fleet-attestation', 'reserved', 'inspector-upload', 'started', 'deployed-topology', 'actual-topology', 'original-ACK-bound-S3', @@ -41,9 +46,28 @@ CONTENDER_HELPER: COMMON_LABELS + ('I05-caller-window', 'I05-caller-before-disconnect-socket', 'I05-caller-window', 'I05-caller-admission-resources', 'I05-caller-window', 'quiescent', 'original-failure-scene', 'original-failure-logs', 'diagnostic-result'), + H2_CONTENDER_HELPER: COMMON_LABELS + ('I05-caller-window', 'I05-caller-before-disconnect-socket', + 'I05-caller-window', 'I05-caller-admission-resources', 'I05-caller-window', 'quiescent', + 'original-failure-scene', 'original-failure-logs', 'diagnostic-result'), } +def h2_labels(window_rounds=(1, 1, 1)): + """Exact journal shape when H1 cleans successfully and H2 fails in its contender window.""" + return ('binding', 'inspector-build', 'preflight', 'integrity-fleet-attestation', + *('reserved' for _ in range(4)), *('inspector-upload' for _ in range(4)), + 'started', 'deployed-topology', 'actual-topology', 'original-ACK-bound-S3', + 'original-ACK-proof', 'I05-timeout-original-physical', 'I05-timeout-original-S3', + 'I05-timeout-baseline-physical-after-public-read', + *('I05-caller-baseline-resources' for _ in range(4)), + *('I05-caller-checkpoint' for _ in range(4)), + *('I05-caller-window' for _ in range(4 * window_rounds[0])), 'I05-caller-before-disconnect-socket', + *('I05-caller-window' for _ in range(4 * window_rounds[1])), + *('I05-caller-admission-resources' for _ in range(4)), + *('I05-caller-window' for _ in range(4 * window_rounds[2])), 'quiescent', 'original-failure-scene', + *('original-failure-logs' for _ in range(4)), 'diagnostic-result') + + def archived_plan(artifact, failed_run, candidate): """Refuse uncertain effects, later actions, mixed cases and physical drift. @@ -71,14 +95,55 @@ def read(path): row = {'topology': 'H1', 'diagnosticComplete': False, 'cleanup': 'FAIL', 'wholeCasePassClaimed': False, 'canonicalCasePassClaimed': False, 'failure': failure, 'namespacesRetained': True} + diagnostics = summary.get('diagnostics') or [] + if len(diagnostics) == 2 and diagnostics[0].get('topology') == 'H1': + topology = 'H2' + row = {**row, 'topology': 'H2'} + h1 = diagnostics[0] + h1_dir = evidence / 'H1' + h1_results = list(h1_dir.glob('*-diagnostic-result.json')) + require(helper in (CONTENDER_HELPER, H2_CONTENDER_HELPER) + and h1.get('diagnosticComplete') is True and h1.get('cleanup') == 'PASS' + and h1.get('wholeCasePassClaimed') is False and h1.get('canonicalCasePassClaimed') is False + and len(h1_results) == 1, + 'I05 H2 isolation requires a completed and cleaned H1 diagnostic') + h1_result = read(h1_results[0]) + require(h1_result.get('topology') == 'H1' and h1_result.get('diagnosticComplete') is True + and h1_result.get('cleanup') == 'PASS' + and h1_result.get('wholeCasePassClaimed') is False + and h1_result.get('canonicalCasePassClaimed') is False, + 'I05 H1 diagnostic result does not prove completed cleanup') + require(diagnostics[1] == row, 'I05 H2 failure differs from its exact known boundary') + else: + topology = 'H1' + require(diagnostics == [row], 'I05 H1 failure differs from its exact known boundary') require(summary.get('caseId') == CASE and summary.get('cleanup') == 'FAIL' - and summary.get('diagnostics') == [row] and summary.get('complete') is False + and summary.get('diagnostics') == diagnostics and summary.get('complete') is False and summary.get('diagnosticComplete') is False and summary.get('requestedTopologies') == ['H1', 'H2'] and summary.get('canonicalCasePassClaimed') is False - and not (evidence / 'H2').exists(), 'I05 requires the sole retained H1 failure') - folder = evidence / 'H1' - labels = LABELS[helper] + and (topology == 'H2' or not (evidence / 'H2').exists()), + 'I05 requires one retained failure and no canonical acceptance claim') + folder = evidence / topology + if topology == 'H2': + archived_names = sorted(path.name for path in folder.iterdir()) + actual_labels = [re.sub(r'^\d{5}-', '', name)[:-5] for name in archived_names if name.endswith('.json')] + checkpoint_end = actual_labels.index('I05-caller-checkpoint') + 4 + first_socket = actual_labels.index('I05-caller-before-disconnect-socket') + first_admission = actual_labels.index('I05-caller-admission-resources') + last_admission = first_admission + 4 + first_quiescent = actual_labels.index('quiescent') + window_rounds = ((first_socket - checkpoint_end) // 4, + (first_admission - first_socket - 1) // 4, + (first_quiescent - last_admission) // 4) + require(all(rounds >= 1 for rounds in window_rounds) + and first_socket - checkpoint_end == 4 * window_rounds[0] + and first_admission - first_socket - 1 == 4 * window_rounds[1] + and first_quiescent - last_admission == 4 * window_rounds[2], + 'I05 H2 lock windows do not contain complete four-peer sampling rounds') + labels = h2_labels(window_rounds) + else: + labels = LABELS[helper] numbered = [(label, f'{index:05d}-{label}.json') for index, label in enumerate(labels, 1)] require({path.name for path in folder.iterdir()} == {name for _, name in numbered} | {'fixture', 'timeout'}, 'I05 archive contains another action or lacks its original boundary') @@ -88,33 +153,85 @@ def one(label): return read(folder / matches[0]) def many(label): return [read(folder / name) for current, name in numbered if current == label] - binding, reservation, scene = one('binding'), one('reserved'), one('original-failure-scene') + binding, scene = one('binding'), one('original-failure-scene') run_id, bucket = binding.get('runId', ''), binding.get('bucketPrefix', '') - require(re.fullmatch(rf'gh-{failed_run}-[1-9][0-9]*-h1', run_id) + suffix = topology.lower() + nodes = ['vm000'] if topology == 'H1' else ['vm000', 'vm001', 'vm002', 'vm003'] + endpoints = {f'node{index}': f'http://rustfs-node{index}:9000' + for index in range(1, len(nodes) + 1)} + require(re.fullmatch(rf'gh-{failed_run}-[1-9][0-9]*-{suffix}', run_id) and bucket == 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] and binding.get('candidateCommit') == CANDIDATE and binding.get('candidatePackageSha256') == candidate['package_sha256'] and binding.get('candidateBinarySha256') == provenance.get('binary_sha256') and binding.get('harnessCommit') == helper - and binding.get('endpoints') == {'node1': 'http://rustfs-node1:9000'} + and binding.get('endpoints') == endpoints and binding.get('defaultEndpoint') == 'node1' and one('diagnostic-result') == row, 'I05 original run, package, helper, binary or failure boundary differs') - owner = reservation.get('owner') or {} - require(reservation.get('node') == 'vm000' and reservation.get('reserved') is True - and owner.get('runtimeProfile') == 'lock-timeout' and set(scene) == {'vm000'}, - 'I05 requires the sole original H1 reserved owner') + reservations = many('reserved') + owners = {item.get('node'): item.get('owner') for item in reservations} + require(len(reservations) == len(nodes) and set(owners) == set(nodes) + and all(item.get('reserved') is True for item in reservations) + and all(owner.get('runtimeProfile') == 'lock-timeout' for owner in owners.values()) + and set(scene) == set(nodes), + 'I05 requires every original reserved node and retained scene') + owner = owners['vm000'] + windows = many('I05-caller-window') + sample_counts = {node: sum(window.get('node') == node for window in windows) for node in nodes} + baselines = {item.get('node'): item for item in many('I05-caller-baseline-resources')} + if topology == 'H2': + attestation = one('integrity-fleet-attestation') + actual = one('actual-topology') + uploads = {item.get('node'): item for item in many('inspector-upload')} + require(attestation.get('nodes') == nodes and attestation.get('allParticipatingNodesUseCandidate') is True + and actual.get('topology') == 'H2' and actual.get('nodes') == 4 + and actual.get('independentDrives') == 16 + and set(uploads) == set(nodes) + and all(uploads[node].get('installed') is True + and uploads[node].get('inspectorSha256') == owners[node].get('inspectorSha256') + for node in nodes) + and set(many('preflight')[0]) == set(nodes) + and all(many('preflight')[0][node].get('hostname') == owners[node].get('hostname') + and many('preflight')[0][node].get('mounts') == owners[node].get('mounts') + for node in nodes) + and set(many('deployed-topology')[0]) == set(nodes) + and set(many('started')[0]) == set(nodes) + and {item.get('node') for item in many('original-failure-logs')} == set(nodes) + and {item.get('node') for item in baselines.values()} == set(nodes) + and {item.get('node') for item in many('I05-caller-checkpoint')} == set(nodes) + and {item.get('node') for item in many('I05-caller-admission-resources')} == set(nodes) + and len(set(sample_counts.values())) == 1 and min(sample_counts.values()) >= 3, + 'I05 H2 evidence omits a peer, inspector upload, lock checkpoint or service log') + admission = {item.get('node'): item for item in many('I05-caller-admission-resources')} + logs = {item.get('node'): item for item in many('original-failure-logs')} + require(set(baselines) == set(admission) == set(logs) == set(nodes) + and all(type(baselines[node].get('fileDescriptors')) is int + and type(admission[node].get('fileDescriptors')) is int + and baselines[node].get('identity', {}).get('binarySha256') == binding['candidateBinarySha256'] + and admission[node].get('identity', {}).get('binarySha256') == binding['candidateBinarySha256'] + and baselines[node].get('identity', {}).get('settings') == runtime_config('lock-timeout') + and admission[node].get('identity', {}).get('settings') == runtime_config('lock-timeout') + and logs[node].get('unit') == owners[node].get('unit') + and isinstance(logs[node].get('journal'), str) and bool(logs[node]['journal']) + for node in nodes), + 'I05 H2 resource identity, runtime settings or original service logs differ') + for node in nodes: + same_process(admission[node].get('identity'), baselines[node].get('identity')) inspector = one('inspector-build') require(inspector.get('candidateCommit') == CANDIDATE - and inspector.get('binarySha256') == owner.get('inspectorSha256') + and all(inspector.get('binarySha256') == value.get('inspectorSha256') for value in owners.values()) and inspector.get('built') is True, 'I05 original inspector identity differs') - original = scene['vm000'] - require(original.get('quiescent') is True and type(original.get('pid')) is int and original['pid'] == 0 - and original.get('hostname') == owner.get('hostname') and original.get('unit') == owner.get('unit') - and original.get('binarySha256') == owner.get('binarySha256') == binding['candidateBinarySha256'] - and len(original.get('formats', [])) == 4 - and {part.get('disk') for part in original['formats']} == set(range(4)) - and physical_content(scene) == physical_content(one('I05-timeout-original-physical')) - == physical_content(one('I05-timeout-baseline-physical-after-public-read')), + for node in nodes: + original = scene[node] + node_owner = owners[node] + require(original.get('quiescent') is True and type(original.get('pid')) is int and original['pid'] == 0 + and original.get('hostname') == node_owner.get('hostname') and original.get('unit') == node_owner.get('unit') + and original.get('binarySha256') == node_owner.get('binarySha256') == binding['candidateBinarySha256'] + and len(original.get('formats', [])) == 4 + and {part.get('disk') for part in original['formats']} == set(range(4)), + 'I05 stopped four-drive original physical scene changed on ' + node) + require(physical_content(scene) == physical_content(one('I05-timeout-original-physical')) + == physical_content(one('I05-timeout-baseline-physical-after-public-read')), 'I05 stopped four-drive original physical scene changed') # Replay only the bounded seed ledger locally. No result is inferred from @@ -195,12 +312,26 @@ def many(label): timeout = folder / 'timeout' expected_timeout_items = {'scope.json', 'failure.json', 'caller'} expected_caller_items = {'scope.json', 'failure.json', 'transport'} - if helper == CONTENDER_HELPER: + has_contender = helper in (CONTENDER_HELPER, H2_CONTENDER_HELPER) + if has_contender: expected_timeout_items.add('contender') expected_caller_items.add('admission.json') require({p.name for p in timeout.iterdir()} == expected_timeout_items and {p.name for p in (timeout / 'caller').iterdir()} == expected_caller_items, 'I05 archive contains an unreviewed action or lacks its exact failure boundary') + timeout_failure = read(timeout / 'failure.json') + caller_failure = read(timeout / 'caller/failure.json') + require(timeout_failure.get('diagnosticComplete') is False + and timeout_failure.get('errorType') == failure['type'] + and timeout_failure.get('state') == 'INCOMPLETE' + and timeout_failure.get('canonicalCasePassClaimed') is False + and timeout_failure.get('originalRequestsAndResponsesRetained') is True + and caller_failure.get('observationComplete') is False + and caller_failure.get('errorType') == failure['type'] + and caller_failure.get('state') == 'INCOMPLETE' + and caller_failure.get('canonicalCasePassClaimed') is False + and caller_failure.get('originalTransportEvidenceRetained') is True, + 'I05 nested failure records differ from the exact retained diagnostic failure') scope = read(timeout / 'scope.json') entry = scope.get('entry') require(entry in expected and entry.get('key') == 'objects/0000.bin' and not entry['isLatest'] @@ -236,7 +367,7 @@ def many(label): and value.get('resultUnknown') is True and value.get('acknowledged') is False and value.get('serverOwnershipProven') is False, 'I05 closed caller was retried or promoted to an acknowledged result') - if helper == CONTENDER_HELPER: + if has_contender: contender = timeout / 'contender' require({p.name for p in contender.iterdir()} == {'000001-request.json', '000001-ack.json'}, 'I05 contender archive is not the sole one-shot request and ACK') @@ -253,36 +384,44 @@ def many(label): holder = (read(timeout / 'caller/admission.json').get('attempt') or {}) require(holder.get('admitted') is True and holder.get('observedAcquiredWithoutDrop') is True, 'I05 contender lacks its original admitted GET lock holder') - windows = many('I05-caller-window') - require(len(windows) == 3, 'I05 contender archive lacks all three caller journal windows') - rows = [] - previous_cursor = None + require(set(baselines) == set(nodes) and len(windows) == sample_counts['vm000'] * len(nodes), + 'I05 contender archive lacks each peer journal window') + rows = {node: [] for node in nodes} + previous_cursors = {} + window_counts = {node: 0 for node in nodes} for window in windows: - require(window.get('complete') is True and window.get('node') == 'vm000' - and window.get('unit') == owner['unit'] and not window.get('sourceMessageLoss') + node = window.get('node') + require(node in rows and window.get('complete') is True + and window.get('unit') == owners[node]['unit'] and not window.get('sourceMessageLoss') and isinstance(window.get('beforeCursor'), str) and isinstance(window.get('afterCursor'), str) - and (previous_cursor is None or window['beforeCursor'] == previous_cursor) + and (node not in previous_cursors or window['beforeCursor'] == previous_cursors[node]) and isinstance(window.get('journal'), str), 'I05 contender lock journal is incomplete or belongs to another service') - previous_cursor = window['afterCursor'] + previous_cursors[node] = window['afterCursor'] + window_counts[node] += 1 for line in window['journal'].splitlines(): try: event = json.loads(line) except json.JSONDecodeError as error: raise ValueError('I05 contender lock journal contains malformed JSON') from error require(isinstance(event, dict), 'I05 contender lock journal contains a non-object event') - rows.append(event) - proof = timeout_proof(rows, response, holder, bucket, entry['key']) + rows[node].append(event) + require(len(set(window_counts.values())) == 1 and min(window_counts.values()) >= 3, + 'I05 contender lock windows are incomplete or imbalanced across H2 peers') + require(all(not any(event.get('request_id') == response.get('ResponseMetadata', {}).get('RequestId') + for event in events) for node, events in rows.items() if node != 'vm000'), + 'I05 contender request ID appeared on a foreign endpoint') + proof = timeout_proof(rows['vm000'], response, holder, bucket, entry['key']) require(proof is not None and proof.get('typedTimeoutBeyondBudgetObserved') is True and proof.get('writeLockAcquired') is False, 'I05 contender ACK lacks its exact failed target write-lock attempt') for path in timeout.rglob('*.json'): read(path) journal[path.relative_to(folder).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() - plan = {'kind': 'unresolved-i05-timeout', 'runId': run_id, 'nodes': ['vm000'], 'bucket': bucket, - 'owners': {'vm000': owner}, 'candidateCommit': CANDIDATE, 'harnessCommit': helper, + plan = {'kind': 'unresolved-i05-timeout', 'runId': run_id, 'nodes': nodes, 'bucket': bucket, + 'owners': owners, 'candidateCommit': CANDIDATE, 'harnessCommit': helper, 'binarySha256': binding['candidateBinarySha256'], 'ackFilesSha256': digest(journal), - 'businessSceneSha256': {'vm000': digest(physical_content(scene)['vm000'])}} + 'businessSceneSha256': {node: digest(physical_content(scene)[node]) for node in nodes}} validate(plan) return [plan] diff --git a/test_validation_heal_timeout_retention.py b/test_validation_heal_timeout_retention.py index c02d50bf..b3fb54f3 100644 --- a/test_validation_heal_timeout_retention.py +++ b/test_validation_heal_timeout_retention.py @@ -4,6 +4,7 @@ import json import os from pathlib import Path +import shutil import tempfile from types import SimpleNamespace import unittest @@ -149,6 +150,230 @@ def lock_row(mode, op, obj, attempt, state, **extra): 'journal': '\n'.join(json.dumps(row, sort_keys=True, separators=(',', ':')) for row in rows)}) return contender + def add_h2_failure_archive(self, root, candidate, h1_folder, *, window_rounds=1): + self.add_contender_archive(root, candidate, h1_folder) + evidence = h1_folder.parent + h2_folder = evidence / 'H2' + shutil.copytree(h1_folder, h2_folder) + + def read(path): return json.loads(path.read_text()) + def write(path, value): path.write_text(json.dumps(value, sort_keys=True, separators=(',', ':'))) + old_binding = read(h2_folder / '00001-binding.json') + old_run, old_bucket = old_binding['runId'], old_binding['bucketPrefix'] + run_id = old_run[:-2] + 'h2' + bucket = 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] + + def replace(value): + if isinstance(value, dict): return {replace(key): replace(item) for key, item in value.items()} + if isinstance(value, list): return [replace(item) for item in value] + if isinstance(value, str): return value.replace(old_run, run_id).replace(old_bucket, bucket) + return value + + for path in h2_folder.rglob('*.json'): + write(path, replace(read(path))) + nodes = ['vm000', 'vm001', 'vm002', 'vm003'] + binding = read(h2_folder / '00001-binding.json') + binding.update(runId=run_id, bucketPrefix=bucket, harnessCommit=controller.H2_CONTENDER_HELPER, + endpoints={f'node{i}': f'http://rustfs-node{i}:9000' for i in range(1, 5)}) + binding_hash = hashlib.sha256(canonical(binding).encode()).hexdigest() + write(h2_folder / '00001-binding.json', binding) + + requests = {} + for sequence in range(1, 10): + request_path = h2_folder / f'fixture/{sequence:06d}-request.json' + ack_path = h2_folder / f'fixture/{sequence:06d}-ack.json' + request, ack = read(request_path), read(ack_path) + request['runId'] = ack['runId'] = run_id + request['params']['Bucket'] = ack['observation']['params']['Bucket'] = bucket + if sequence >= 3 and sequence != 6: + index, generation = (0, sequence - 3) if sequence <= 6 else (1, sequence - 7) + size = 32 * 1024**2 if index == 0 else 1024 + body = payload(f'{bucket}/objects/{index:04d}.bin/{generation}', size) + item = {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()} + request['payload'] = ack['observation']['payload'] = item + etag = hashlib.md5(body, usedforsecurity=False).hexdigest() + ack['observation']['response']['ETag'] = etag + requests[ack['observation']['response']['VersionId']] = (item, etag) + raw = json.dumps(request, sort_keys=True, separators=(',', ':')).encode() + request_path.write_bytes(raw) + ack['requestSha256'] = hashlib.sha256(raw).hexdigest() + write(ack_path, ack) + + for name in ('00010-original-ACK-bound-S3.json', '00013-I05-timeout-original-S3.json'): + path = h2_folder / name + value = read(path) + value['bucket'] = bucket + for entry in value['entries']: + if not entry['deleteMarker']: + entry['payload'], etag = requests[entry['versionId']] + entry['properties']['ETag'] = etag + write(path, value) + + scope_path = h2_folder / 'timeout/scope.json' + scope = read(scope_path) + scope.update(runId=run_id, candidateCommit=controller.CANDIDATE, bindingSha256=binding_hash) + scope_entry = next(entry for entry in read(h2_folder / '00010-original-ACK-bound-S3.json')['entries'] + if entry['key'] == scope['entry']['key'] + and entry['versionId'] == scope['entry']['versionId']) + scope['entry']['payload'] = requests[scope_entry['versionId']][0] + write(scope_path, scope) + caller_dir = h2_folder / 'timeout/caller' + transport = caller_dir / 'transport' + request_path = transport / 'request.json' + caller_request = read(request_path) + caller_request.update(runId=run_id, harnessCommit=controller.H2_CONTENDER_HELPER, + candidateCommit=controller.CANDIDATE, bucket=bucket, + bindingSha256=binding_hash, + requestTarget=f"/{bucket}/{scope_entry['key']}?versionId={scope_entry['versionId']}") + caller_request['params']['Bucket'] = bucket + raw = json.dumps(caller_request, sort_keys=True, separators=(',', ':')).encode() + request_path.write_bytes(raw) + for name in ('request-written.json', 'disconnected.json'): + path = transport / name + value = read(path) + value.update(runId=run_id, harnessCommit=controller.H2_CONTENDER_HELPER, + candidateCommit=controller.CANDIDATE, bucket=bucket, + bindingSha256=binding_hash, + requestTarget=caller_request['requestTarget'], + requestSha256=hashlib.sha256(raw).hexdigest()) + write(path, value) + + contender = h2_folder / 'timeout/contender' + contender_request_path = contender / '000001-request.json' + contender_ack_path = contender / '000001-ack.json' + contender_request = read(contender_request_path) + contender_ack = read(contender_ack_path) + contender_body = payload(f'{run_id}/I05-timeout-contender', CONTENDER_BYTES) + contender_request.update(runId=run_id, payload={'size': len(contender_body), + 'sha256': hashlib.sha256(contender_body).hexdigest()}) + contender_request['params']['Bucket'] = bucket + raw = json.dumps(contender_request, sort_keys=True, separators=(',', ':')).encode() + contender_request_path.write_bytes(raw) + contender_ack.update(runId=run_id, requestSha256=hashlib.sha256(raw).hexdigest()) + contender_ack['observation']['params']['Bucket'] = bucket + contender_ack['observation']['payload'] = contender_request['payload'] + write(contender_ack_path, contender_ack) + + records = {} + for path in sorted(h2_folder.glob('*.json')): + match = path.name.split('-', 1) + label = match[1][:-5] + records.setdefault(label, []).append(read(path)) + for path in h2_folder.glob('*.json'): + path.unlink() + + def clone(value): return json.loads(json.dumps(value)) + def by_node(value, node): + result = clone(value) + if isinstance(result, dict): result['node'] = node + return result + old_reservation = records['reserved'][0] + old_owner = old_reservation['owner'] + owners = {} + reservations = [] + uploads = [] + for node in nodes: + owner = clone(old_owner) + owner.update(runId=run_id, bucket=bucket, hostname=node, + unit=f'rustfs-validation-heal-{run_id}.service') + owners[node] = owner + reservation = clone(old_reservation) + reservation.update(node=node, owner=owner) + reservations.append(reservation) + uploads.append(by_node(records['inspector-upload'][0], node)) + + baseline_resources = records['I05-caller-baseline-resources'][0] + resources = [by_node(baseline_resources, node) for node in nodes] + for value, node in zip(resources, nodes, strict=True): + value['identity']['binarySha256'] = binding['candidateBinarySha256'] + value['identity']['settings'] = controller.runtime_config('lock-timeout') + original_scene = records['original-failure-scene'][0]['vm000'] + scenes = {} + for node in nodes: + scenes[node] = clone(original_scene) + scenes[node].update(hostname=node, unit=owners[node]['unit'], + binarySha256=binding['candidateBinarySha256']) + physical = records['I05-timeout-original-physical'][0] + old_physical = physical['vm000'] + snapshots = {node: clone(old_physical) for node in nodes} + started = {node: {'started': True} for node in nodes} + deployed = records['deployed-topology'][0] + deployed = {node: clone(deployed['vm000']) for node in nodes} + preflight = records['preflight'][0] + preflight = {node: {**clone(preflight['vm000']), 'hostname': node} for node in nodes} + checkpoint = records['I05-caller-checkpoint'][0] + checkpoints = [by_node(checkpoint, node) for node in nodes] + cursor_by_node = {node: f'{node}-checkpoint' for node in nodes} + windows = [] + source_windows = records['I05-caller-window'] + for stage in range(3): + for round_index in range(window_rounds): + for node in nodes: + source = source_windows[stage] + window = by_node(source, node) + window['unit'] = owners[node]['unit'] + window['beforeCursor'] = cursor_by_node[node] + cursor_by_node[node] = f'{node}-window-{stage}-{round_index + 1}' + window['afterCursor'] = cursor_by_node[node] + if node != 'vm000' or stage != 2 or round_index != 0: window['journal'] = '' + windows.append(window) + admission_resources = [clone(value) for value in resources] + for value in admission_resources: + value['identity']['binarySha256'] = binding['candidateBinarySha256'] + value['identity']['settings'] = controller.runtime_config('lock-timeout') + quiescent_source = records['quiescent'][0].get('vm000', {'stopped': True}) + quiescent = {node: clone(quiescent_source) for node in nodes} + logs = [by_node(records['original-failure-logs'][0], node) for node in nodes] + + attestation = records['integrity-fleet-attestation'][0] + attestation['nodes'] = nodes + attestation['allParticipatingNodesUseCandidate'] = True + actual = {'topology': 'H2', 'nodes': 4, 'independentDrives': 16} + diagnostic_result = {'topology': 'H2', 'diagnosticComplete': False, 'cleanup': 'FAIL', + 'wholeCasePassClaimed': False, 'canonicalCasePassClaimed': False, + 'failure': controller.FAILURES[controller.H2_CONTENDER_HELPER], 'namespacesRetained': True} + labels = controller.h2_labels((window_rounds, window_rounds, window_rounds)) + by_label = { + 'binding': [binding], 'inspector-build': records['inspector-build'], + 'preflight': [preflight], 'integrity-fleet-attestation': [attestation], + 'reserved': reservations, 'inspector-upload': uploads, 'started': [started], + 'deployed-topology': [deployed], 'actual-topology': [actual], + 'original-ACK-bound-S3': records['original-ACK-bound-S3'], + 'original-ACK-proof': records['original-ACK-proof'], + 'I05-timeout-original-physical': [snapshots], 'I05-timeout-original-S3': records['I05-timeout-original-S3'], + 'I05-timeout-baseline-physical-after-public-read': [clone(snapshots)], + 'I05-caller-baseline-resources': resources, 'I05-caller-checkpoint': checkpoints, + 'I05-caller-window': windows, 'I05-caller-before-disconnect-socket': records['I05-caller-before-disconnect-socket'], + 'I05-caller-admission-resources': admission_resources, 'quiescent': [quiescent], + 'original-failure-scene': [scenes], 'original-failure-logs': logs, + 'diagnostic-result': [diagnostic_result], + } + indexes = {} + for label in labels: + index = indexes.get(label, 0) + value = by_label[label][index] + indexes[label] = index + 1 + write(h2_folder / f'{sum(indexes.values()):05d}-{label}.json', value) + + shutil.rmtree(h1_folder) + h1_folder.mkdir() + write(h1_folder / '00001-diagnostic-result.json', {'topology': 'H1', 'diagnosticComplete': True, + 'cleanup': 'PASS', 'wholeCasePassClaimed': False, 'canonicalCasePassClaimed': False}) + case_path = root / 'rustfs-validation-39711742431-1' / 'run' / controller.CASE / 'cases' / f'{controller.CASE}.json' + case_result = read(case_path) + case_result['provenance']['testing_commit'] = controller.H2_CONTENDER_HELPER + write(case_path, case_result) + summary_path = evidence / 'healthy-summary.json' + summary = read(summary_path) + summary['diagnostics'] = [ + {'topology': 'H1', 'diagnosticComplete': True, 'cleanup': 'PASS', + 'wholeCasePassClaimed': False, 'canonicalCasePassClaimed': False}, + diagnostic_result] + summary.update(diagnosticComplete=False, diagnosticBranchesPassed=False, cleanup='FAIL', complete=False, + wholeCasePassClaimed=False, canonicalCasePassClaimed=False) + write(summary_path, summary) + return h2_folder + def test_original_live_failure_yields_exact_pre_contender_preservation_plan(self): with tempfile.TemporaryDirectory() as directory: root, candidate, _ = self.archive(directory) @@ -173,6 +398,47 @@ def test_new_helper_failed_refusal_yields_exact_contender_preservation_plan(self self.assertEqual(plan['businessSceneSha256'], {'vm000': '6c9e808ff0c6d03cb5dfd5546bd588c3d3e1bcee3c167877dec21c4399fc1e6d'}) + def test_h2_failure_after_clean_h1_yields_exact_four_node_plan(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, h1_folder = self.archive(directory) + self.add_h2_failure_archive(root, candidate, h1_folder, window_rounds=2) + plan, = recovery.recovery_plan(root, 39711742431, candidate) + self.assertEqual(plan['kind'], 'unresolved-i05-timeout') + self.assertEqual(plan['runId'], 'gh-39711742431-1-h2') + self.assertEqual(plan['harnessCommit'], controller.H2_CONTENDER_HELPER) + self.assertEqual(plan['nodes'], ['vm000', 'vm001', 'vm002', 'vm003']) + self.assertEqual(set(plan['owners']), set(plan['businessSceneSha256'])) + + def test_h2_retention_refuses_partial_peer_or_unclean_h1_evidence(self): + for variant in ('missing-reservation', 'missing-peer-window', 'missing-peer-log', 'h1-dirty', 'peer-contender'): + with self.subTest(variant=variant), tempfile.TemporaryDirectory() as directory: + root, candidate, h1_folder = self.archive(directory) + h2_folder = self.add_h2_failure_archive(root, candidate, h1_folder) + if variant == 'missing-reservation': + (h2_folder / '00005-reserved.json').unlink() + elif variant == 'missing-peer-window': + window = next(path for path in h2_folder.glob('*-I05-caller-window.json') + if json.loads(path.read_text()).get('node') == 'vm002') + window.unlink() + elif variant == 'missing-peer-log': + log = next(path for path in h2_folder.glob('*-original-failure-logs.json') + if json.loads(path.read_text()).get('node') == 'vm002') + log.unlink() + elif variant == 'h1-dirty': + summary_path = h1_folder.parent / 'healthy-summary.json' + summary = json.loads(summary_path.read_text()) + summary['diagnostics'][0]['cleanup'] = 'FAIL' + summary_path.write_text(json.dumps(summary)) + else: + window = next(path for path in h2_folder.glob('*-I05-caller-window.json') + if json.loads(path.read_text()).get('node') == 'vm001') + value = json.loads(window.read_text()) + value['journal'] = json.dumps({'request_id': '30000000-0000-4000-8000-000000000001'}) + window.write_text(json.dumps(value)) + with patch.object(recovery, 'Fleet') as fleet, self.assertRaises((ValueError, KeyError)): + recovery.recovery_plan(root, 39711742431, candidate) + fleet.assert_not_called() + def test_new_helper_refuses_unknown_success_or_corrupt_lock_lifecycle(self): for variant in ('unknown', 'success', 'acquired', 'read-lifecycle', 'extra-action'): with self.subTest(variant=variant), tempfile.TemporaryDirectory() as directory: From 0442d4dd814ebfb490dcbb3cc266fb6fbebb0365 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 16:07:26 +0800 Subject: [PATCH 288/343] docs(validation): record latest Heal environment inspection --- docs/validation-migration-map.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 1fb336ef..a36a5896 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2414,3 +2414,13 @@ and negative evidence cases were added. The matching local CI group ran 117 tests successfully with two existing Linux-only tests skipped on macOS; the previous single timeout FD test also passed when rerun in isolation. No new live diagnostic or canonical acceptance is established by these checks. + +Post-isolation read-only inspection [39745040693/1](https://github.com/rustfs/rustfs/actions/runs/39745040693) +used helper `22589bc00836b5d47f382429313e967c1dfc38a9` and candidate package +39364893106/1. It observed 16 independent eligible endpoints and available H1/H2 +mount shapes, with stable runtime observations and no port-9000 listeners. +vm000 had no loaded validation unit; vm001-vm003 still reported the failed H2 +unit from cancelled F07 run 38715204273/1. The inspection recorded +`mutated=false`, `cleanStateCertified=false` and +`globalWorkflowIdleCertified=false`; it neither changed that scene nor adds +canonical acceptance. From b56b848f6c9203c4b34acabb99c0c2429338cfaf Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 16:43:44 +0800 Subject: [PATCH 289/343] fix(validation): retain I05 drain-budget failure --- docs/validation-migration-map.md | 12 ++ scripts/validation_heal/timeout_retention.py | 204 +++++++++++++++++- test_validation_heal_timeout_retention.py | 46 ++++ .../i05-drain-budget-39746002648.zip | Bin 0 -> 161876 bytes 4 files changed, 251 insertions(+), 11 deletions(-) create mode 100644 testdata/validation/i05-drain-budget-39746002648.zip diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index a36a5896..b6103670 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2424,3 +2424,15 @@ unit from cancelled F07 run 38715204273/1. The inspection recorded `mutated=false`, `cleanStateCertified=false` and `globalWorkflowIdleCertified=false`; it neither changed that scene nor adds canonical acceptance. + +I05 drain-budget failure [39746002648/1](https://github.com/rustfs/rustfs/actions/runs/39746002648) +is now recognized by the retention planner under helper `0442d4d`. The plan +accepts only its H1 `NOT_OBSERVED` boundary: one closed, non-retried historical +GET with an admitted lock owner, one known 503 contender PUT whose target write +lock reached timeout without acquisition, the exact caller marker and lock +lifecycle, the two backpressure socket samples, all repeated read-only drain +socket/resource/journal samples, unchanged four-drive scene and the original +nine seed ACK pairs. Unknown or successful writes, extra requests, missing +samples, changed process identity or physical drift refuse before Fleet +creation. The archived failure remains `FAIL/cleanup FAIL`; no canonical PASS, +backup restoration, namespace deletion or verdict rewrite is authorized. diff --git a/scripts/validation_heal/timeout_retention.py b/scripts/validation_heal/timeout_retention.py index c8a36497..8ebc0901 100644 --- a/scripts/validation_heal/timeout_retention.py +++ b/scripts/validation_heal/timeout_retention.py @@ -10,16 +10,18 @@ from .heal_workload import verify_seed from .healthy import physical_content from .fleet_node import runtime_config - from .lock_observer import CANDIDATE, same_process - from .lock_timeout import CONTENDER_BYTES, timeout_proof, validate_timeout_ack + from .lock_observer import ADMISSION_SECONDS, CANDIDATE, DRAIN_SECONDS, TARGET, same_process + from .lock_timeout import (CALLER_RECEIVE_BUFFER_BYTES, CONTENDER_BYTES, + MIN_BACKPRESSURE_BYTES, timeout_proof, validate_timeout_ack) from .race_retention import digest, validate except ImportError: from heal_probe import canonical, payload, require from heal_workload import verify_seed from healthy import physical_content from fleet_node import runtime_config - from lock_observer import CANDIDATE, same_process - from lock_timeout import CONTENDER_BYTES, timeout_proof, validate_timeout_ack + from lock_observer import ADMISSION_SECONDS, CANDIDATE, DRAIN_SECONDS, TARGET, same_process + from lock_timeout import (CALLER_RECEIVE_BUFFER_BYTES, CONTENDER_BYTES, + MIN_BACKPRESSURE_BYTES, timeout_proof, validate_timeout_ack) from race_retention import digest, validate @@ -27,6 +29,7 @@ PRE_CONTENDER_HELPER = '704bd606071676dded130dca91d3d5278455bcaa' CONTENDER_HELPER = 'd100abeb8e85c4000a493aa994031e4a5d01518e' H2_CONTENDER_HELPER = '1b88cad3581e755d8e904697cb2ec471e04b5f8f' +DRAIN_BUDGET_HELPER = '0442d4dd814ebfb490dcbb3cc266fb6fbebb0365' FAILURES = { PRE_CONTENDER_HELPER: {'type': 'ValueError', 'message': 'caller lock diagnostic escaped its exact GET identity or runtime budget'}, @@ -34,6 +37,8 @@ 'message': 'I05 contender acquired a lock or lost its exact timeout lifecycle'}, H2_CONTENDER_HELPER: {'type': 'ValueError', 'message': 'I05 contender acquired a lock or lost its exact timeout lifecycle'}, + DRAIN_BUDGET_HELPER: {'type': 'ValueError', + 'message': 'NOT_OBSERVED: caller drain observation exceeded its budget'}, } COMMON_LABELS = ('binding', 'inspector-build', 'preflight', 'integrity-fleet-attestation', 'reserved', 'inspector-upload', 'started', 'deployed-topology', 'actual-topology', 'original-ACK-bound-S3', @@ -50,6 +55,10 @@ 'I05-caller-window', 'I05-caller-admission-resources', 'I05-caller-window', 'quiescent', 'original-failure-scene', 'original-failure-logs', 'diagnostic-result'), } +DRAIN_PREFIX_LABELS = COMMON_LABELS + ('I05-caller-window', 'I05-caller-before-disconnect-socket', + 'I05-caller-window', 'I05-caller-admission-resources', 'I05-caller-window', + 'I05-timeout-caller-backpressure-socket') +DRAIN_SUFFIX_LABELS = ('quiescent', 'original-failure-scene', 'original-failure-logs', 'diagnostic-result') def h2_labels(window_rounds=(1, 1, 1)): @@ -68,6 +77,26 @@ def h2_labels(window_rounds=(1, 1, 1)): *('original-failure-logs' for _ in range(4)), 'diagnostic-result') +def drain_failure_labels(folder): + """Accept only complete repeated H1 drain samples from the pinned failed helper.""" + names = sorted(path.name for path in folder.glob('*.json')) + labels = [name[6:-5] for name in names] + prefix, suffix = list(DRAIN_PREFIX_LABELS), list(DRAIN_SUFFIX_LABELS) + require(labels[:len(prefix)] == prefix and labels[-len(suffix):] == suffix, + 'I05 drain failure has a different caller or retained-scene boundary') + samples = labels[len(prefix):-len(suffix)] + first_drain = samples.index('I05-caller-drain-socket') if 'I05-caller-drain-socket' in samples else -1 + caller_windows = samples[:first_drain] + rounds = samples[first_drain:] + complete_rounds, terminal = divmod(len(rounds), 3) + expected_rounds = ['I05-caller-drain-socket', 'I05-caller-drain-resources', 'I05-caller-window'] * complete_rounds + require(first_drain > 0 and all(label == 'I05-caller-window' for label in caller_windows) + and complete_rounds >= 1 and rounds[:len(expected_rounds)] == expected_rounds + and (terminal == 0 or rounds[-terminal:] == ['I05-caller-drain-socket', 'I05-caller-drain-resources']), + 'I05 drain failure omits or reorders a complete socket/resource/journal sample') + return prefix + caller_windows + rounds + suffix + + def archived_plan(artifact, failed_run, candidate): """Refuse uncertain effects, later actions, mixed cases and physical drift. @@ -114,6 +143,9 @@ def read(path): and h1_result.get('canonicalCasePassClaimed') is False, 'I05 H1 diagnostic result does not prove completed cleanup') require(diagnostics[1] == row, 'I05 H2 failure differs from its exact known boundary') + elif helper == DRAIN_BUDGET_HELPER: + topology = 'H1' + require(diagnostics == [row], 'I05 drain failure differs from its exact H1 boundary') else: topology = 'H1' require(diagnostics == [row], 'I05 H1 failure differs from its exact known boundary') @@ -142,6 +174,8 @@ def read(path): and first_quiescent - last_admission == 4 * window_rounds[2], 'I05 H2 lock windows do not contain complete four-peer sampling rounds') labels = h2_labels(window_rounds) + elif helper == DRAIN_BUDGET_HELPER: + labels = drain_failure_labels(folder) else: labels = LABELS[helper] numbered = [(label, f'{index:05d}-{label}.json') for index, label in enumerate(labels, 1)] @@ -225,7 +259,8 @@ def many(label): original = scene[node] node_owner = owners[node] require(original.get('quiescent') is True and type(original.get('pid')) is int and original['pid'] == 0 - and original.get('hostname') == node_owner.get('hostname') and original.get('unit') == node_owner.get('unit') + and original.get('hostname') == node_owner.get('hostname') + and original.get('unit') == node_owner.get('unit') and original.get('binarySha256') == node_owner.get('binarySha256') == binding['candidateBinarySha256'] and len(original.get('formats', [])) == 4 and {part.get('disk') for part in original['formats']} == set(range(4)), @@ -257,7 +292,8 @@ def many(label): params.update(ContentType='application/octet-stream', Metadata={'heal-fixture': 'DV', 'nonce': f'{index}-{generation}'}, Tagging='suite=heal&fixture=DV') domain = domain or request.get('clockDomain') - require(isinstance(domain, str) and bool(domain) and request.get('schemaVersion') == ack.get('schemaVersion') == 2 + require(isinstance(domain, str) and bool(domain) + and request.get('schemaVersion') == ack.get('schemaVersion') == 2 and request.get('runId') == ack.get('runId') == run_id and request.get('sequence') == ack.get('sequence') == sequence and request.get('operation') == operation and request.get('params') == params @@ -312,23 +348,26 @@ def many(label): timeout = folder / 'timeout' expected_timeout_items = {'scope.json', 'failure.json', 'caller'} expected_caller_items = {'scope.json', 'failure.json', 'transport'} - has_contender = helper in (CONTENDER_HELPER, H2_CONTENDER_HELPER) + has_contender = helper in (CONTENDER_HELPER, H2_CONTENDER_HELPER, DRAIN_BUDGET_HELPER) if has_contender: expected_timeout_items.add('contender') expected_caller_items.add('admission.json') + if helper == DRAIN_BUDGET_HELPER: + expected_timeout_items.add('timeout.json') require({p.name for p in timeout.iterdir()} == expected_timeout_items and {p.name for p in (timeout / 'caller').iterdir()} == expected_caller_items, 'I05 archive contains an unreviewed action or lacks its exact failure boundary') timeout_failure = read(timeout / 'failure.json') caller_failure = read(timeout / 'caller/failure.json') + expected_failure_state = 'NOT_OBSERVED' if helper == DRAIN_BUDGET_HELPER else 'INCOMPLETE' require(timeout_failure.get('diagnosticComplete') is False and timeout_failure.get('errorType') == failure['type'] - and timeout_failure.get('state') == 'INCOMPLETE' + and timeout_failure.get('state') == expected_failure_state and timeout_failure.get('canonicalCasePassClaimed') is False and timeout_failure.get('originalRequestsAndResponsesRetained') is True and caller_failure.get('observationComplete') is False and caller_failure.get('errorType') == failure['type'] - and caller_failure.get('state') == 'INCOMPLETE' + and caller_failure.get('state') == expected_failure_state and caller_failure.get('canonicalCasePassClaimed') is False and caller_failure.get('originalTransportEvidenceRetained') is True, 'I05 nested failure records differ from the exact retained diagnostic failure') @@ -342,7 +381,9 @@ def many(label): transport = timeout / 'caller/transport' require({p.name for p in transport.iterdir()} == {'request.json', 'request-written.json', 'disconnected.json'}, 'I05 caller transport contains another request or result') - request, sent, closed = (read(transport / name) for name in ('request.json', 'request-written.json', 'disconnected.json')) + request, sent, closed = (read(transport / name) + for name in ('request.json', 'request-written.json', 'disconnected.json')) + caller_transport_request = request target = f'/{bucket}/{entry["key"]}?versionId={entry["versionId"]}' require(request.get('operation') == 'get_object' and request.get('params') == { 'Bucket': bucket, 'Key': entry['key'], 'VersionId': entry['versionId']} @@ -367,6 +408,21 @@ def many(label): and value.get('resultUnknown') is True and value.get('acknowledged') is False and value.get('serverOwnershipProven') is False, 'I05 closed caller was retried or promoted to an acknowledged result') + if helper == DRAIN_BUDGET_HELPER: + caller_scope = read(timeout / 'caller/scope.json') + require(caller_scope.get('runId') == run_id and caller_scope.get('candidateCommit') == CANDIDATE + and caller_scope.get('bindingSha256') == scope['bindingSha256'] + and caller_scope.get('entry') == entry + and caller_scope.get('admissionSeconds') == ADMISSION_SECONDS + and caller_scope.get('drainSeconds') == DRAIN_SECONDS + and caller_scope.get('canonicalCasePassClaimed') is False + and caller_scope.get('mutatesServiceConfiguration') is False + and scope.get('minimumCallerPayloadBytes') == MIN_BACKPRESSURE_BYTES + and scope.get('receiveBufferBytesRequested') == CALLER_RECEIVE_BUFFER_BYTES + and request.get('receiveBufferBytesRequested') == CALLER_RECEIVE_BUFFER_BYTES + and sent.get('requestInvocationStarted') is True + and closed.get('responseObserved') is False and closed.get('connectionAttempts') == 1, + 'I05 drain archive changed its bounded read-only caller or time budget') if has_contender: contender = timeout / 'contender' require({p.name for p in contender.iterdir()} == {'000001-request.json', '000001-ack.json'}, @@ -381,7 +437,8 @@ def many(label): and ack.get('observation', {}).get('acknowledged') is False and ack.get('observation', {}).get('resultUnknown') is False, 'I05 contender result is not a durable, known refusal') - holder = (read(timeout / 'caller/admission.json').get('attempt') or {}) + caller_admission = read(timeout / 'caller/admission.json') + holder = (caller_admission.get('attempt') or {}) require(holder.get('admitted') is True and holder.get('observedAcquiredWithoutDrop') is True, 'I05 contender lacks its original admitted GET lock holder') require(set(baselines) == set(nodes) and len(windows) == sample_counts['vm000'] * len(nodes), @@ -416,6 +473,131 @@ def many(label): require(proof is not None and proof.get('typedTimeoutBeyondBudgetObserved') is True and proof.get('writeLockAcquired') is False, 'I05 contender ACK lacks its exact failed target write-lock attempt') + if helper == DRAIN_BUDGET_HELPER: + caller_request_id = holder.get('request_id') + caller_lock_id = holder.get('lock_attempt_id') + caller_events = holder.get('events') or [] + caller_socket = caller_admission.get('socket') or {} + require(caller_admission.get('node') == 'vm000' + and holder.get('states') == ['acquiring', 'acquired'] + and holder.get('diagnosticGuardDropped') is False + and holder.get('distributedReleaseProven') is False + and holder.get('heldAtDisconnectProven') is False + and isinstance(caller_request_id, str) and str(uuid.UUID(caller_request_id)) == caller_request_id + and uuid.UUID(caller_request_id).int + and isinstance(caller_lock_id, str) and str(uuid.UUID(caller_lock_id)) == caller_lock_id + and uuid.UUID(caller_lock_id).int and len(caller_events) == 2 + and [event.get('state') for event in caller_events] == ['acquiring', 'acquired'], + 'I05 caller lacks the exact admitted GET owner at its intentional disconnect') + caller_uri = f'/{bucket}/{entry["key"]}?versionId={entry["versionId"]}' + for event in caller_events: + require(event.get('target') == TARGET and event.get('request_id') == caller_request_id + and event.get('lock_attempt_id') == caller_lock_id + and event.get('event') == 'object_namespace_lock' and event.get('component') == 'ecstore' + and event.get('subsystem') == 'object_lock' and event.get('op') == 'get_object' + and event.get('mode') == 'read' and event.get('bucket') == bucket + and event.get('object') == entry['key'] + and event.get('requested_version_id') == entry['versionId'] + and event.get('owner') == holder.get('owner') + and event.get('namespace') == holder.get('namespace') + and len([span for span in event.get('spans', []) + if span.get('name') == 'http-request' + and span.get('request_id') == caller_request_id + and span.get('method') == 'GET' and span.get('uri') == caller_uri + and span.get('user_agent') == caller_transport_request.get('callerMarker')]) == 1, + 'I05 admitted caller lock owner is not bound to its exact historical GET') + caller_lock_rows = [event for event in rows['vm000'] + if event.get('target') == TARGET and event.get('request_id') == caller_request_id] + require([event.get('state') for event in caller_lock_rows] == ['acquiring', 'acquired', 'guard_dropped'] + and all(event.get('lock_attempt_id') == caller_lock_id + and event.get('op') == 'get_object' and event.get('mode') == 'read' + and event.get('bucket') == bucket and event.get('object') == entry['key'] + and event.get('requested_version_id') == entry['versionId'] + and event.get('owner') == holder.get('owner') + and event.get('namespace') == holder.get('namespace') + for event in caller_lock_rows), + 'I05 caller journal lost or changed its original GET lock lifecycle') + dropped = [event for event in rows['vm000'] + if event.get('request_id') == caller_request_id + and event.get('event') == 'get_object_stream_body'] + require(len(dropped) == 1 and dropped[0].get('state') == 'dropped_incomplete' + and dropped[0].get('bucket') == bucket and dropped[0].get('object') == entry['key'], + 'I05 caller disconnect lacks its exact incomplete response-body event') + mutations = [] + for event in rows['vm000']: + spans = [event.get('span'), *(event.get('spans') or [])] + for span in spans: + if isinstance(span, dict) and span.get('method') in {'PUT', 'POST', 'PATCH', 'DELETE'}: + mutations.append((event.get('request_id'), span.get('method'), span.get('uri'))) + require(mutations and all(request_id == response['ResponseMetadata']['RequestId'] + and method == 'PUT' and uri == f'/{bucket}/{entry["key"]}' + for request_id, method, uri in mutations), + 'I05 caller window contains another or unknown S3 mutation') + backpressure = one('I05-timeout-caller-backpressure-socket') + expected_process = caller_socket.get('process') + samples = backpressure.get('samples') or [] + require(backpressure.get('node') == 'vm000' and backpressure.get('readOnly') is True + and backpressure.get('process') == expected_process + and backpressure.get('clockDomain') == caller_socket.get('clockDomain') + and backpressure.get('networkNamespace') == caller_socket.get('networkNamespace') + and backpressure.get('clientAddress') == sent.get('localAddress') + and backpressure.get('serverAddress') == sent.get('peerAddress') + and len(samples) == 2 + and all(sample.get('vanishedDescriptors') == 0 + and len(sample.get('ownedMatches') or []) == 1 + and sample['ownedMatches'][0].get('state') == '01' + and type(sample['ownedMatches'][0].get('txQueueBytes')) is int + and sample['ownedMatches'][0]['txQueueBytes'] > 0 + for sample in samples) + and len({sample['ownedMatches'][0].get('inode') for sample in samples}) == 1, + 'I05 contender is not bound to the original backpressured caller socket') + timeout_proof_record = read(timeout / 'timeout.json') + require(timeout_proof_record.get('requestId') == response['ResponseMetadata']['RequestId'] + and timeout_proof_record.get('lockAttemptId') == proof['lockAttemptId'] + and timeout_proof_record.get('owner') == proof['owner'] + and timeout_proof_record.get('namespace') == proof['namespace'] + and timeout_proof_record.get('events') == proof['events'] + and timeout_proof_record.get('typedTimeoutBeyondBudgetObserved') is True + and timeout_proof_record.get('writeLockAcquired') is False + and timeout_proof_record.get('backpressureSocket') == backpressure + and timeout_proof_record.get('readResponseBackpressureObservedAfterTimeout') is True + and timeout_proof_record.get('canonicalCasePassClaimed') is False, + 'I05 contender timeout proof differs from the exact archived lock and socket evidence') + baseline = baselines.get('vm000') + drain_sockets = many('I05-caller-drain-socket') + drain_resources = many('I05-caller-drain-resources') + drain_windows = windows[5:] + require(baseline is not None and len(drain_sockets) == len(drain_resources) >= 1 + and len(drain_windows) in {len(drain_sockets), len(drain_sockets) - 1}, + 'I05 drain archive lacks a complete service and socket sample pair') + def same_socket_process(value): + process = value.get('process') or {} + expected = baseline['identity'] + require(all(process.get(name) == expected.get(name) for name in ( + 'pid', 'startTicks', 'bootId', 'unit', 'executable', 'binarySha256', 'mountNamespace', + 'commandSha256', 'environmentFileSha256', 'settings')), + 'I05 drain socket sample crossed its original service process') + for socket_state, resource in zip(drain_sockets, drain_resources, strict=True): + same_socket_process(socket_state) + require(socket_state.get('node') == resource.get('node') == 'vm000' + and socket_state.get('clockDomain') == caller_socket.get('clockDomain') + and socket_state.get('networkNamespace') == caller_socket.get('networkNamespace') + and socket_state.get('clientAddress') == sent.get('localAddress') + and socket_state.get('serverAddress') == sent.get('peerAddress') + and socket_state.get('readOnly') is True + and socket_state.get('continuousAbsenceProven') is False + and len(socket_state.get('samples') or []) == 2 + and all(sample.get('vanishedDescriptors') == 0 and not sample.get('ownedMatches') + for sample in socket_state['samples']) + and type(resource.get('fileDescriptors')) is int and resource['fileDescriptors'] >= 0 + and resource.get('readOnly') is True + and resource.get('identity', {}).get('settings') == baseline['identity'].get('settings'), + 'I05 drain observation contains a changed process, socket or resource sample') + same_process(resource.get('identity'), baseline['identity']) + write_rows = [event for event in rows['vm000'] if event.get('mode') == 'write'] + require(write_rows == proof['events'] + and [event.get('state') for event in write_rows] == ['acquiring', 'failed'], + 'I05 drain journal contains a successful or unbound namespace write') for path in timeout.rglob('*.json'): read(path) journal[path.relative_to(folder).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() diff --git a/test_validation_heal_timeout_retention.py b/test_validation_heal_timeout_retention.py index b3fb54f3..5922d82f 100644 --- a/test_validation_heal_timeout_retention.py +++ b/test_validation_heal_timeout_retention.py @@ -19,6 +19,7 @@ FIXTURE = Path(__file__).parent / 'testdata/validation/i05-pre-contender-39711742431.zip' +DRAIN_FIXTURE = Path(__file__).parent / 'testdata/validation/i05-drain-budget-39746002648.zip' class TimeoutRetentionTests(unittest.TestCase): @@ -31,6 +32,15 @@ def archive(self, directory): folder = artifact / 'run' / controller.CASE / 'evidence' / controller.CASE / 'H1' return root, candidate, folder + def drain_archive(self, directory): + root = Path(directory) / 'drain-artifact' + with zipfile.ZipFile(DRAIN_FIXTURE) as archive: + archive.extractall(root) + artifact = root / 'rustfs-validation-39746002648-1' + candidate = json.loads((artifact / 'candidate.json').read_text()) + folder = artifact / 'run' / controller.CASE / 'evidence' / controller.CASE / 'H1' + return root, candidate, folder + def add_contender_archive(self, root, candidate, folder): def read(path): return json.loads(path.read_text()) def write(path, value): path.write_text(json.dumps(value, sort_keys=True, separators=(',', ':'))) @@ -398,6 +408,42 @@ def test_new_helper_failed_refusal_yields_exact_contender_preservation_plan(self self.assertEqual(plan['businessSceneSha256'], {'vm000': '6c9e808ff0c6d03cb5dfd5546bd588c3d3e1bcee3c167877dec21c4399fc1e6d'}) + def test_drain_budget_failure_yields_exact_bounded_preservation_plan(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, _ = self.drain_archive(directory) + plan, = recovery.recovery_plan(root, 39746002648, candidate) + self.assertEqual(plan['kind'], 'unresolved-i05-timeout') + self.assertEqual(plan['runId'], 'gh-39746002648-1-h1') + self.assertEqual(plan['harnessCommit'], controller.DRAIN_BUDGET_HELPER) + self.assertEqual(plan['nodes'], ['vm000']) + self.assertEqual(plan['ackFilesSha256'], '21e9471c9e9133186b88fb664f8a4deff4c93044761a9c399915a0b1db9d6f32') + self.assertEqual(plan['businessSceneSha256'], {'vm000': + 'c438f5bf295f56ddd168e9e840b41281863c3da06c0973eb9abead95edd63e89'}) + + def test_drain_budget_failure_refuses_unknown_mutation_or_incomplete_sampling(self): + for variant in ('success', 'unknown', 'extra-action', 'missing-sample', 'physical-drift'): + with self.subTest(variant=variant), tempfile.TemporaryDirectory() as directory: + root, candidate, folder = self.drain_archive(directory) + if variant == 'success': + ack = json.loads((folder / 'timeout/contender/000001-ack.json').read_text()) + ack['observation']['response']['ResponseMetadata']['HTTPStatusCode'] = 200 + (folder / 'timeout/contender/000001-ack.json').write_text(json.dumps(ack)) + elif variant == 'unknown': + ack = json.loads((folder / 'timeout/contender/000001-ack.json').read_text()) + ack['observation']['resultUnknown'] = True + (folder / 'timeout/contender/000001-ack.json').write_text(json.dumps(ack)) + elif variant == 'extra-action': + (folder / 'timeout/contender/000002-request.json').write_text('{}') + elif variant == 'missing-sample': + next(folder.glob('*-I05-caller-drain-resources.json')).unlink() + else: + scene = json.loads((folder / '00058-original-failure-scene.json').read_text()) + scene['vm000']['formats'][0]['sha256'] = '0' * 64 + (folder / '00058-original-failure-scene.json').write_text(json.dumps(scene)) + with patch.object(recovery, 'Fleet') as fleet, self.assertRaises((ValueError, KeyError)): + recovery.recovery_plan(root, 39746002648, candidate) + fleet.assert_not_called() + def test_h2_failure_after_clean_h1_yields_exact_four_node_plan(self): with tempfile.TemporaryDirectory() as directory: root, candidate, h1_folder = self.archive(directory) diff --git a/testdata/validation/i05-drain-budget-39746002648.zip b/testdata/validation/i05-drain-budget-39746002648.zip new file mode 100644 index 0000000000000000000000000000000000000000..a2f977b1a6712841f76a8e3bd92e219c3dc1e25e GIT binary patch literal 161876 zcmd43WprFip5-f9lEutmF*7rh#j=>0F~!Wx%xIx4W@ct)W@ct-bZe&G>#En?x90WC zx~*Kw%ukWABI4}W`}dD?en^6VLIM5pw-BJL^|vqo>kS_W2guIR-oe9(o%FIr~K&PY#2?UZjvSj|pzoQE@5HRQ+Fc1*vA5HxDY#HVA6XI_z zyWOLV!GQq*iGThWQ|}t4(>u8s$5J=K+=I3n-$+ z!<+tQ7Y0}LHve`Yw_0^`Nr^Mr=s2}K143`dFJmY46#@ZnVX>t787)s4rMsfy_uHT3 z#t6rXg_AqtHQLE%)yrfdN%Y zhH%xfq!@{=N@dfU0U!~+ z-nnxh1Vo@ygv=w&uh7Vu=PSS+rNOr^!tNZ3?1W_1QYrhr==)WO}Jn$rA)MMP(M$?e=9R5#k*h=r4tPj znQQf)YOSCfnpvuSas^+JPPS22UR)Eg3@)Eo(_82!sJEKIt=+kgU@{O;B_)FFG_G8g zD?bd9x>whqlz%q$ucoCuFMHr%V&<^z@f_VWX~?Q`KwTpcTwtS-5&m|+P;|PETV4fk zAmfy;W<%%l)tA){7|9Xo78}UtHYZhv^8}}qRSk3-=_`tPm*(^!OY;lxPOvz}n94Qs z64L=v3UNS4S24AJtUx|YKPGOZ-l$(Bq~<0}>KLaOXLsoVfeF{N6md-y@wE`xqCidR z`ID7&GB@5+xZZ9>%WAEK)UC+yk;JqLrI-87vpBHOivAk<8f9$}hT|9aqyP>WC2wsN zqsWQxS4hms8EUp0+e)vrL zt|C2qR78++OD@tB%vU}(L@^k`+C!Fr<1~eyMn-ntRs87}e9C%l+~3rrfu zj0Hr9Up+rSN_SSOS9l7@9BbE!t!borWiw^?MHViT+EILj2|bJiw=7Q1pf(>W1SdhR z3;QJHnpYS|Xgh)%GQ67$EpgfJL0X}UGT|;kTml{Cwetl;n|t>5*hZ0~8Xq!^!^EBz zPit-kKRRsau{elAXt z#vfpfbr!+T+|mzcbqs0RN)ld69Ov$u08$iQ_sgfQjek^L7>T{Hj!zxt`4nc_e?fT- z^sEj4PMu0%8g7+OeZ;)UCt)Z+`9VNAREz;4XP zPqz!rfDt*@Z#(M~>d8jS_q^)ahMqI6kxf+*<=!H1VvcX)BbqLwCub|ojz_M(qMM3lJu2@VH+BzvKu|%9Vdlw_nR14{j4Rub{vhtbpa|R~AijIL1^dTg8Z?!O zN(To5>f!_fqWKqw>2Dj9oul=ij?$UB1z?38?TyNh?@WK^P*1HMhT1p`E-drhFwD%^ zSO)_mL3l{&_dr|%vdX3RhI>))p45z}1b7PQG5D_}7dSdP*K7wCgDK0J%jB$v+SSCC z#uOvsf|{{YPl~y8`2KW8Xw(s=?#KnM?rKb!L84}n#nWuSIsjnp?eg%@(!udQ_l)}L z{zBC8{&^yrO-859C{(oG|>2X5|Wa&Q6& zwSM*HZl#jc)dU^#?uxMymQMusg0{}+VU{rz{+Xi2E+JHsGedmXZ~1Mg>)K7lDG1yk zSF~&SFEp(&Mi}xGTeiUq#m*{{167{TK^LYjHHjn&b#7J`d+bqrTvQ7j-tNo33wXL9 z?I(cFj<@NLEm~h!i7vffnIb`Jsd$vqsX3BCQz-S_vhrm<;S!*Z?(Y{;o47>@fCP`4^)KUV%4ZxQ9}=O$_;^q4c!D7t&y^Do%q~o zMD5*cjn6srxpusse8<)H_IAFznme1jqk;~-7e=(b2$+XAs_B)N2SzLE47oFxtl9f8 z{!l?@N1CcU3~e{$7b6>!S)afbW1}ADQ02wH2!Ez;TGHd>9*OI6=~Of>cS5NJ1w=RM zepP1o5sR+4#haX}a&i~!#O2G%6YwE#&~+b>z~huw`pSq>I@xXjp@#>~-6ST~&l{J+ z?Gqol2IG5lRYS@S13d3DObxmce`zyoADnO|km)~kPT(u~4XSoIyTHkY-KE9k)@g;C zZrG;=VJ_+_K$t>ZN2)P@fb$x$z*ODmXqj9^3T#Yp^P;=dXhc{Us^c5NIrss1-AjP1nLf^;;+@8acPmD79B%NiJ#F@#K1$y96CgBzUl8f}3tDxG z0i5{kjZPRLus1N5{13zf!a%`&FzT-mPd7Ru(Ruh=uo&+(+A>lSaQ9jEl-m6iRM29D zdFYifU%Fp1)=`PWvp)2#<)2>Bfg#xp%cZaHz%@oI9M2Q;NL1r~_$IC@g35|q zDuRAYg>9a2KX&g5ssA=skd>Tajm0Ddv-rp)<=6c=$Rg+RI~(g4*8?AjoqjFgH+HmxV$J%uebmqY0w{Jb(wDR=9XSW!ZZB~i%Q z8-=0wI>zFdi67QkUvRAN!jb~WnE3WuFmPXdtf1cr)S{tKOpN)8A%CHm`f7OWc>gt9 zQdvt+!SY;e)scWRVnAWQ)p=I~q?ZVbATL9SQtl!DW|gfwjNjE3a^{V~t!GD@cp6o* zO2<*OkV7|DR88WTpo+VPRpPBiBX2-((<0R*W`(4QQW>k!RM zE{^x8OQxQYf!huA8H zoc^Z2!={Rxbe~6<>u|&DnIjx4&e3cx}o!1mbNrb7>1lI_Th-m%t0}AXD9NV`dJn)|F}5-z%(X zsN_2?qHx-rP9^X%HI_s^L!TQgGu7+aAc8mkVJ*XHg=Af zZ*qBzJ%q2+il3pQOE4}NB(9${yUzC0z~RRZzahEv33rwPBPD+A;Be_pQwSP2U&Jji z54{tp!!JY1^IzLufA*GbVe9?M_m2*M#quY(<)=xa1OWnK`4=3(|6`PB>;aB;2LFOz zk{hyKp+{(Yp!$+&5sq{I9F<{*7_>`|mfMF=Kq@lnO=s3pS0NDP(bF)RKJj(IN(`f( z7p@|qq0rs@8dBhBT+(<*JqAlkDvWENUs463HcOQrK@$R@i!p+dV-p3G7%Y_itKa^? zXRHkoNd*yu6XiM|9=+(ncxg@KLve3a1p)EgJOw&U4V4LrYDz9<+TzZ)Xq~{eT{{q5 zHsw@F_&_gx?2~5CDRA^-(hAxNxREkoc?#S?u*FVllE`Lg-J-f%C_BlJiW7veMxvYz z9h5+eTTu6h#qe_spj8J9ikK0KT{8%pkrC_V>qQs8xK(jmt#CM7VGryyAJFU>s21Mq za5c60okb?Dxc4%q*}qguj;vgKZC(0C^pwUDj4Z8+T@r1i19C@IkhVg;6S+O@_b^Vc z12;MZtRTY`v=^sLL6Kl}K03Xll6o=noYcSfHnPeWy{oZ%dF37!4T~$`&s4U-^n8W> z$JGA&T!Z*unA%TY_rJk5;Ew#Y3qP%+9r?dz9sg+E&d3H}=Rj*^_@7~oGWC(T6;`yh z2bId<@Y0{5Wa84pT(Y<(=jxi67a9M|}f=f)gO9(w^Sw3=FPoo$yD&B+^{-j03E4;27_j-$@b@xcPXyUR+m zt?m73|L()PxyjWzzT^3U?!xQwLY8kI9o{8eJmHA93Xt8uc6w_9WW4#X9VDY_?hyR1VC{ zj7SG)I@gWN&tdUK_f}no{oV4su^5TPrUWI)GHf*xz0HlNQT@s`DQ(JV@ic|Cf7{|@ z6uGy2b)&hx}?`h&Fa| zE_9ML51$CJ0KaOe$4^m+q6_k-?IAwG)pohGfl3A$FU?#ps|3vY(VIH4sI`3FbPw1l zEF8RJXC8?TqcE5(g`&ObI>;Fi(GsmMJ~MbsO2{Ef9%5Yr^VzP00@VnF$4)UBnw!jC z{w~s-MM(u0i+EFw>^1AcJa#I8PHl4g2^oAWl8abykrGoV05VASVbeyWB{K#lmtHAi zt@3e^_QWopn?6N!MFG+s^i5~DqZ?*M(Su!Spn8=QD}FL8PFBMBs~o%kNWiRz;SR#j zaS`lA|1^=!IS7u-M8air`eJP;&4~I<1D)4e!{Q(*Aubu#N{?SKtEdNpF|g_8puz<) z@w5b4C|GyO>F&jv*bEC)hP!U^4*1>lkayrwE54{@j6{+lXNer^H6L`wH?7rZLUbRk0E z1xAP7dc$K5o%)>A*01gdFAt11JNx+w)9X=xGJ)>MY`Lhz?wQHNS~*gJ5E7Xs%0`Hu zg`9@H6=4NASj$j$j>~MF!H8s=^mQ?72$4_Zoe4Xt5l2VE(7t{Qlf3MnC+2WY#+f9} zPMcY!h@b#hI(E2_UpU$)=pdXM?e$NjHOLW~)~-)a(8n>Vrw8kp5%LTp%%zo9*UsM( zvNB^_hZ{Wpxx7TE3xI9?yt)9Ztg&Yf+X1o1N(MbX#_Ui5%wN)QrF$Bx>J}uS!8xEE zaGTWaK}%jDOeRN>yw~}gZ(Mr^+t6uM9_6Akza}>{F6iMA!v6Yg|Eo&+VSkTr{k7&q z&wBg!i?roI=@5REowe`rS;|*JwM@n9`~x1N^ltr=P$3uy=!}tf!ME9iJwy-SqAC;^r z)|5=D3GyD!)?%z9;HDngTe%`RLU_j!@;rF;u(QqyO%*E`c-Nz5mgN`^jFyW>QcvvR z)Y2d>%gZ@mN9EW1DXyXNulWOnEBl0xu>j-56|)C;$4UyjzT-|hfUUf8;J;D;B? zXF=N;u`T^HF3N9-|7=BG{_>@nrZSeP-icQ%l)m=pCnf zA*(!-Wg`w-kb~l=i_xfQ3<;Xydf;iTc+{Xw(#b{R9FD5~TrGPm6mAs$H>^u>9k>m; zk>x`sXj`azIN`fwDDo{RKHT#14;=YX&7ar&wo;yDGB=DL7VLadN|r`_q5+RTaC7G1 zVe&CHF}Wh*$l0mNIlI!qj z=4;e}^_vWY3@4>+iP&r%NuC{L< ze9ek?kCJ_ozuQzahgM%}J)LFUuBkBlyc{3P{>N71`@d*4OpWv`X)OT;7BmiKRz?6v z2bxa?M`NI8X=!9fqo@CA6#mqW{QF+#$N#ww7&)048d)0{{r~KV{r`oY{9hY3_rIpI z4u6!oECD9}nI-(k$Z{NxieUE{S#G`n0mc1Ao&H}d*3eAP#2R4lU}ivL?`UPEXXpB- zWHd5XK=um*Lg2+~nD_)j?sotLMucbt7Hxk@)wiVnX*I4g37C(yB_$_#(&K{QBX@vy zhZhV`#Eq&*(rH-=a^nJG=XdvGqy01x6|Odha3Ut|QTLgI2FIUL#i|Af$Hc4%{qk(% zHA7EE7oICNKPjNJ@&=iIZp6JD^$$*Iz<5J0914k#JqsNb-j`dG8VBY~a+AocxwWmu zK`57Pn!7$Ua<^q}lR`i2v@#JaJR5~|VFm7ngv+Hiw|AHXBuj+`pW9L2ou*1~6qOAVD>Ud?6 zYZ+!dd=F!2AMt3$cY}~*=@E;bvvMO6pRz>7ue3|^T9xsuABzZ>or?k&wo?8(U8S_o zH2JQmd?{sMJZXPM<5Sr9B8?hOB>DipW*kY1R+I@ARv#wX7$(`IXK$<*_uB~mb6&mM;N#W^6=B# zmluOey3ZLHCJCIDCofBn14j{rw>Tq5Od_dL=Q=CMZlij_`&yNRSWy2(nWpej_=_P6;E3dQ!`r`Nrjc$SPo& zj{i<3kl2M(%BrM!phke2LKPt2FXS5fV;8>$=2}qCQMUU)Bigm%giNJBL21HKY5&mH zC~i13JeI0X92qlxIFq%xW?AuE*tjT;l*aCbyh0QAsuLOU7qLxIxZbwyxLJyr&R59X zOR)?XKYU%nBwK32gjghh1fspbFjw{;VX+~7knn9WqthHv;G68@GDmzrpe~`tp@^7# zmbtqyW*{joyd-~6`7|KU-Z|T-=URPbm;4SvwUBycUCOdw2$Jq~kxmCQK62z3yLxS} zz~#~3+sUkD0`J2tXK1U=q(Zx9UBO|^fEB!E@kFOHQeHz>R}&p=fDj-vTLHlQ(e-%w zHR?MsChhNhlLI5H@*n#&+}Hj&eN!Y+UxWpqzvt0gZZ&T7FQJ_r%r%;Z))Y?kN^25( zR0I-P99v2fx2k{Bahgwy;LP2SKX7?Ni<*GtWLumCL?bZ{l%-IyXIJbTPOA>qdUvq-bq52)cjuh_AUUwCxH z?c9-Z`Ndt|UGm0xuEfw?$6>)ybi4Ve@T;VvrC9^dsq!S3envre#0>{q7bfIT?rXYk zi)D0@RS9gLKXI0RkS(&Nc}9L;XRwMJ6lYZ(2z*sNSA31|I;*{_^oMGS{XK$2r zmp?=ta}ShZIe)=&KRRdK!<7S6^(u44X?|Re9UPYqOOpHovBF}2V-}WU^ty4M{kjWl zrWI}{upXBK(Mj7h4U=*nI`!*B9g_9Wss?z*OPUHDK%aHzTjE^1imUJ+gG@jGLI?@8F2g zBbEhM>W59FSYGm1qz%RZr(-BD%_hv|Y?RAx%V9PA4!xid52m#YOXcj68RruzA>=XI z7T(u`Uzi5+@+n38Rovha`J$2gA);VsMO=4x7Zao{M@FCIc$wu;ET$ZMxlLhbpurE| z$3yZ(SDpYB#|q9M&6q?i9fFS8kED_*afCj2!WIu<{j3=V~jZsCv zpAPC#ap))`wiqDmslOwQgQERT{z-%iFy4sOU%I1 zNYDDu^?xmvC%l^ckG;@1-rF>!}z;#R*bK|Qdse1t7#LII-m z`WTtreQINRVXZTbV<@0(L#XTkeC0{2Z+jxF&sn}Y$Ju?H1I6ceXqmiMVo|!d`ozef~@AiLcI17|C z0C}P)UK16SuQ+K%(uuw;4T{##*b})R2wr$p939JRG!$Am zPH}Q6Cvwab6lc~MCf?LK*=!=i1MMOn^P9kWn;O+n1!lk@irPx)D?XH#;%!@CH%zdk zU>TCElpiD4XChQvMgO>MK$*8?P#yyA9*%g zHqZ%rZj$G$w@O6=&T$ps4i=Z?%>UV&BLyl0>?x4Ci*vA}P}o|kb`w{MX~%PN;)YLF zsvBsi3bkPrI6%Jz#{UIh9Y2g6KLVwFf9Z#M%{X9eZ! z6d$(~TW#HKbZh>mlk{tZI|?`;fx&o2T|dGu5^&$|KtbIRV~!-Etv_jlgKZUZNvY^%~mbapm;Wg^Tv~f5_*-dp1geG_*0NStZX;_pR-$E@7(uW4ye*k|fi zFp%$Lg^Ul%RAtBV)IlfrTHDQou)6&Bu?=f3FTT#Bb0@fl@nSjVBT)!GxK_*e$9*_$ za(Yr@-vou-J1>*YRULssFVcu^+uG8#+2Z!l_<5S7pnI$rs!^qzSgcNfTR$v{c0MF3 z)d!2?T<=#C)QvqL;DQvad(&7%NqM^$f6?OJkuFf}-1iyQxc)rJ`kiG}H7MkN>~vVN zSh_&C!1kq`l)pj2VHUZXpf5osiuog*jh1#oK0 z?Zspu2HPHSaoiH`NI+yn%BF_pZjafwZVeX9FBIStaUV2otBOzTaqOjp;NU|dj35eq zL6EVV*y)_$&x(zJ2o2oCbzpGaQkrWwJvO)CbXG_>#_*9F4IE^+vRg3ezLRnbafNyD z!52=vZZZ^uBJq)h?EH~d-&Ec$wmXV?S)gUWu#GI87cb~lomenP7Vu=iT6mecpL@zf zd*9kb4sFW-1AXCeW;$oCFaym+i@t%8+~opOU)CZ)_oQ6MwGKb#pm8+97h-F}V2JU? zp+4-^vZv%!3KT2m_J>0Gec2zuVeFEo!7mVY=_bK6#8iOM9VWOT_=+R|Nl++}hK38S zyb`f86RO46J0zXaU!&UuvtZT;Q@*dq*U3S*5^swI!xzx?IxE4phArA9RL#la9c*9K z?z%w#j!;@LW11S!a6tVEw`Ld*0NboqOW;x z$Go9vQx^hAWl0xF2eS+*Z`TNAJ+qC$CBeO%T`F)7@Z!y^rQ!6j{&%l=SIsY8C>kjr z8qw`*;s_g&Lj|_%;WSjN;IB5h<3UU;iAiV8>uE%f*|dpE4aM`0N`rfqYY}e70-hTt zr=6`Q{c8_L49XQPD$=EfT1zJoz=u-pM3YgK-*oJAuK?ezm0y{U?6%jC}WuC|6|62Jd0UHq-J zW@<29{0%UA#!4{Wp7GCyyFNUK^X)U!xqbx#3i~Tf6g?A-jh&IPrJ0H8pC7I7OOys= z&?5w0yih4;RMFQzT}m`5D#?^7)(sQ~1NVn(bZS_dPLqGM^|z>+j3%fLXGnbU(tJ#bm ztDouCZSJXXM=yrhL*N{7can2;RpcsFas~~dSWF^-_HadVpyYD| z6;6mi>QqEhb0;h$~r$td*etrW1^WG-NH`t<7UnU~ceN2guR zC&azIE}HF70g_1u|J>I8!rD1r-@kSYmz?sGk+1yR^XE&^2m3K5;CnCp11)hu?Pj2u zBxzJpUI#=gI`2!EYI+W&8A(A2^27JH+S7Y|NwCbmIsKr4xqmt!GoW}iHSynJk7Z{5 zukEoLfP7X2uL+&1Z4TyluoOj4eC)WqrPD%aGsiMYL+tp1f$rfH>Aab@i#kZdhU5_u zC7l@VjZu%CtI>gj_@z;^WkJpZCX)^@AogGLim267iC&ER6Z@9bEyeWUc!G!7yc*Ai z)2vc=nY3Ka6UW86&F0w7L~3WemYO(Ud%=O80-7h+I`KG*?h|9iQ}j4xi2EHziuY-o zYF)w0r{?f2mhSHO7w-fI%enZkHslvOpIi_xu_Rb#JReXoAhpg40)yyTao}y1Z_uUg zq!n1;$9+68PY$d~kXO)t!9dbTY_~`~Pm_?4&>9gsNMb+?6CX6Dly6@a>m5_9(eFO7 z&ff?Ks18XCRigL~NM!4EOZtHF*9xQ9@?%i=`}kG@83{mjgY9R4P4?`PR9N08nx~h! zHkA}7fqywR=ywej|KUckr>#3I#Qc*T6qG<`I0iBdG#Mv=7Z?NT#xjeHVVmTR-NUuK z2=gdbg!#@QGUMOrTJN}nxgT_2cFdTxfu#c}v`LHLykeuLf>Jiwj zXE(@s?3Zaj38=~md{~07Uj%>&?hEK;T`*LwsLrFvoIJCQRm`%+^Y$3E53U%SReO7Y zl1tB{`pq0(5x$2T(3L&2W#qnCI-b7EjIZ3ywL~2Sn|h{_jZn91lPwLrvfV^(lx|yG zl!)M%JbBc$O3XKG6nQzu?uXCa)tL^`}UeCB`yM=IVu zkv?3n>9A11-U~;T3F+P|X@^wqNuEV}m-nU4n$eKewR-|d$zIDD2VBH{$12$gJKNUP zF>j5)rBdgbXHtdq{0e(?-raUyX!8?qxYg*#*}&zhTEek_0n`T8u3t5k`>rZr?RNT)PkWGAH zD*-9z!wq_9_@M#}fgi04F*pLPHmszul3n)g;EJ-y)Y*TyE;^Fhm}_o1dtFKuHKF*$ zmNaD%jWPS^Lu?2w!-BZEI3ukM`4U&syAoDZz*H zz$t>N)Q3_ky*&tCBlC=7pP4h?C?puweGNV?D79805WaKxX_a~7yPZ@mdl#9!WSiBa zF1uOKF6C$;6Bb}9cBDQZ?;#V?u(WOafmz>pZP)7JgUBn~YRwSnGRe%o!UDrOuCBa` zwWdZ9c@@d3ZvKN0eiezuiVtp`{H&GmtAJ}6v&PQeka<6Qebn;_(EKRzYtaOVBfe0( z(G2A6hQFUh0%(2$_3R=qVbw&)2W2emLIv0p2GIblbJ9I2&A3=7oeMO@+5XGGT6fw` z1V~6r7JKGHob#EhIJsB&KW#N$5$#8ROd<5=b`$cKdL;%L{m;2Gvo`tD$uo+T1biOv z47hkg^{psN+p~MRE~EHD{0mfU>LpH7RKjv;vZoSpwgKWyO!w&R>h;Exm`Q5 zD3RtPPVdy|&D^uBopwg=9MA4ey{$G9E{YmR{W{WIy^fPh`{3-|i%ll_ZC^2RhJM7A zcAz$O;2i&HGud?>w($5t<>VwEL6vbUoZ( zjI0M&ocw1+?a}s}l3T7!Yj3{PIg-z%r04d@u5oxRXZ2hqPY5^82sw@6+V!Ev3bQFBuy;_ z5|ziyt2cEASs`A57JVrd$z~_P3lO@W5+B@UoUhM7{8E_vjpMPm*vUP=R3ArBOwMfL@Aoi_C5pTu?MtEO(Y`psc_oTweQ8)=lpl%} znfWMW`OF{~--ld(%Ew(dSIS07Q1@>Kw(RW)dB=Dy!~Pn0q&#|k#QH!6$i`F0P0e#b zKvnRTd;OEn=!!As|Fm3u&UM`CSM%@)BX3yMW~%RLo+O!FcY^qqApdbG`ID0=Bpaq(~5 zUp)6lzjbEoqAWEEn`Bj!kdJLd5{2wUarz4`b|>Q_s0eIWd5tK;!q_IRf{2pUb&~xe zE;A0o3`lsA4-yOr?R<#&slUTO^0N=|M1$~wxQa3o5~KOKPxkGSlo#KND5z*KHKpe& z4uD4J(+Wq0Cb#e+mzxX=jT$V1gAx=cOppl4lLLE500m8d0#Q^^LX&bIx*e=^#B5Wn zOqA>RGiniDf>FG`0a0|LPat|QCZZPd2}E6fS$r>1PS4U_bZmO;Z-(hNsJ2>DavN$W zUc2%#D_oJ#c^8QA4=3n3oS2fIU!kd}HUJM1p6=;nkwrzJ*V&JOv~~lgAn8V?9l{?; z(Fan1OND2MfslY;P@0aoY^+>XUd^&TeI}5x-e%4M8!>8>=`pflfqwxTcU{UVozZH>sy#-yP=nwUNUz@ zE<^vr2%-MEnL8^p8{mFr$%UcP9yI=_+NAxhYBSOQx|9)V zq_V%iDFD%S<6lrBX|<-faGdLSUrK3TI8x}u8mBbA)8FMQ%MBFs{WnM^>}yVO<^=F7-?@8U9<3Zs*3qNvG&zCDYe;nG; z(ag3~y}Ja&M|J2Hmz-34W!x_&6L_yPbJiRE$5k7p&#FzTn+z*VQ`Vg$PYkkhr6419 z;6-(}LCWP9d|(6`1Q8qrQK$_Q{6T@(_zW9g4inx_CK^h0;#rjD_6HMHz9#4A8D*!V zzk+7_xq2t<7V?Y&?t{N8j!7>NyBX)-NdzRMx4hqsM5O&AxhL5vvQJEgTH;s1qaoEv zjt>G>#8OJPbJ|t$L*pWoqg>*vJ_g$SQU6r+Twx))lO}CD11*Rv$_lGiS+XV7%D0nX z(lY5TIZIi}BsHm+Y0x#;aK}pCTm=h@(yo95JdL&oy#kiGBvARh`R72=>C^!BDuGp zjwy%sOI9=#qFiq+m>YVlY13|mgqSGQ*T`L6mo7W7T3qKu0m zIa)@_cP{UD(*TEfPbDwR+PxjYX07V&8d(YhHuob!tx=3SxMjHVC!jhUt9r(b$@Y^2E~Q`A**Rcf2TsE(|qMMafU<>_T=ejjEEsSnC* z%|((`INA*dLOy<-B>3#Y>yA4s@U3~ufR<#+*xaHsrvRYa-{vbNCO3zXkOt=K1j4o8 zqwCY(O0@a6TlNv3*Sn+v35HEzlL8I&D`)EUf{9!T^|+nE9m@8vI*`iHYD$a~S&BtQ z{lS~MF2fR*DU2rDL@+!M{T*p@ZX(v%k!f?jgM17;ONLWhNd33WLCn-*h<4v!I*|B* znv8G0JWmr)H5F66)YR>8Ymy+2m2cRMDbU;u$*|o2VncL-mY92IN0hRJ`N0xhpgTvb zEj{{KwGr2k{v)liNMK7)IZLIkSrPv`NYuBAk1*)-X(7h2e|1{>2cq;ZrnNH_E5OR9 zG_C6pJkESgU?@_N@MS;|nL^cA$$_BPbtL$)J*1r15UaRce(1dso3NHI4rFqY3|#X@ zz`34zp5^Z1{Pao=q1;2RrvzG3k|p2lW+`$riA5z}!x~Adl50^wf5sle6B2cLPdr@1 zUcfA6AcL_~!vuR=T-LSHfoH-JGT!{ys+C#~ZY9iTrPAkwnv1BH5U>Td2Nn(`I&mFf zhr9g{eRh21G}nddAoKJ<2GHg}beq%%mwCh)Jvru?C7l%YJzUX4SIm1w5o3`-9{lNN ztccJ#0GW8%mSV2IBAPM48}RJzP6+#(8?A z16Sc<#s2(qpI>If`eIED$US~iOwO6nb#Kta7PeLYNc@e1#ujn)p=vKP)=Y3K0{1DBZ`FrJllOy!yUYel?atXH; z_DzT2mYbE=6)*g5y>GTq>s^YV- zX4o=b%X^Qng52p+S{oke*%7W(P&ke`i(n6wtD=2zWfleAloKU?*L9^klRKCx&>2xV zHI!&k?Z#-+p_L!M(*fLM$MTq8a9Pmrvh7PcO@#4$ZpX$eK|iDo)a>|>I&*B0fY6Hc zW(&*ip0WJEsoT-I& zLQ|;);xEQs@iEn`9ri(votUf!Y{UZbo{19a_yJ8;TT1-E#w`YP$Mwq6{V&CoaYwEl zz+J9=L`j+H(rG~Bu|Y~oXx&c((jtLC{09_~K)Joa#8@E5TZjdE4tBzV`L#`Ga>NK& z*m5w85t#kGG4!^mPa$yW)IkQbFfxT#cxhPL&vK6^zgckGJrAN{a*DyILRN3I50fDn zdD(JjE)PfYI)MMt^;strM45g(*9W;Iz*?Mf4?P9~^>N0{H?Lji zb}dcq``RJJi|6WwG9aL~5~rggR(Y04jkkU6l|Vv!Iut>%isr=r>t)hF4@)_ez<^4d z4{;2G0h;#Y-~c0z99$JeS7MW11I-|9IFO-6q%S754JK9&dKxlqW|r$N8ro=>4S`eZ z>Q~e5s_yJC5MV%Z-Q0&|TP>n6hcJRF9VS+c$Cw618vKrFaf6t#F+5eP3{eArD|DHd=CkJ*L7ltisGp{6AWHrxM zm6x@>3CBaCKl9+bPeNgFf*;)gb;J2kYJz zsoQAG^Qa2Wev{7?0p+DY#~L8*w#VOrIGcxn!32L6&b72?G2;Xw5#+znCzJ=aSI{GI z+<ENBi zearNUS-!0U=OGu)cOHrUo2B$K4l+@H9pxR73{E$09TFPI_eh%tiRnOv%ziT*8dAJ( zC-(zN%l)qlUZ+u9p-Bzk_D77k)EY{s)7Ic#P7?=O=R?X?rj*);-l#fJs*M6+)ure9 zHwtVZ2(1?0h8IvTzy{|VZ&5Ra$;CQ5pzbn)H^)2c=ad66v5F*Tl) z;MT$*cnaB_1_sxH=wr_!HX$B|OoD@$jfz5ZgUv2>RsdfJJU(mJU$;C!I34K|%ZXdc zRwy)?Kbr*%kq&5 zxG5Q3XPpSAtwx4)Ql!Xf`L#LYl4Z#TY=5{O#}*-NIG4P0ml+rdZAD5hAKEOschG-S z4iAg{*w_CKfXl*Hav-5H!Tc+ien>379#dMM8&bSsqrM$TlL0m1{zXsY&||2Wg3%EE;i=CZYb5_fpYG?bTr|)vS6;T}%?Mklr4vtv8w{L1zW`3tK zp^Wua1ON}uH>!f#nmAMRQ20)1waQ!d{8UW4U|-SH#f4cvGWP7AmDU9HqcNtc3e=yx zOHm_3Aa|0h7Y$3>S<5m&6H--~ao8SfefTo;GC)QJJ8f1As|dbfgikmd zD8L8e-Bk8E$Ri})c8$ZDB=muIFRMCmzd+TpzzDKVOg7f@BvtFI%bmb>N~lv6imS7) z@2AN^Axt_lX8%p#O3i)?x&ybFM_r6&8|o=$CW50%pp^{a6xj+^vr2ehR@sK-eBOC4 zlNw$$z&dMOBp&@2*^CFegI%>cE0!a8mgO>e-cpqRB;X~FH#s&%^m$O!lXqt_`C?#= zKKkx#q)nSr2QY7irfPt$;#VX4KbU*R;Li80%{#VjTb*==9ox2T+qToOJ5I;8ZL?!1 z9ov|H?{n@sXP=p>*)z{mb?+B>pQ``*eZC9Vx|WK5!LFs<{dQU*ZziZy$ssBHC-a6j zCGU!@qe=h9wtcr%#j=7ay?U13*~-E7%Y#-8JJC|d-hYC zeUJDz$VT+zR8l{EGYec{ONpaNGIP1oDOQdr2N}vjF@rs*V~N5Pl(MpAkKR*B`~JDj zAbK;u;bVL7V2u%Kali2^9vRUz!T8~{GEt(~ACo#|bq$9Ii3DMDPebQUNoB2gWV@gV z{0NIw#3S_zS+F?e+D{I8O>iyCaHoZsa^*>lVwptz`Ej_#N~n2#5FG^w$3c8sx0(v! z_~)RF`Dk8rYm90GD-T;ZexKLNmY|FXp_b2?{LapSaA6jfEtzZ7ZL1|&esVBQn$ok~ z5*R)%{oMQRcve1yn}#86pDrd)+v)Lx=_ynF1Y_DFM&oS6)3hOTdseJeAO)7p9>*e0 zvHqS%%GI8@V2k}g+!Z-{UUZeruK1Y^eO9;Vn%vDLmQ-$Y#8L2jf-yaZ7+hl}e2tF4 zSf@CuQ-jYWpEvC^HKhqXt;TsHF@~mw5NyK`PCSIAcYh3sz7x(5Cd?XcC?T2+7wB^x zevJfvA~Ox9m25mrZmFXufs*q^OaB`nKOeOHcWu7>!5*N|&*<|&_y%lXBN=&V+!*vq z^c#3Nylm6yd)$r324Ne8Q0kV@RQ|Mq)ezQf_%@RBfwClG;-J_HAHmzHkY(qTisS3YHyk32F<9+wzACXCCO=fyAy zjMoVrJ|?=yow>ARHz{w(!gk?JfIl2o`O6%+wPd^QTGU_$zIG}$2m2NnK0Rel-PtsJ z{oY%+sRi?(`gx+O>Jv`mI^)sg=KM6i*5u9U*6G%NY_T!4w2S{eCcb}Ce*Wcmi^(R4 z5l!fY8$1u;8~orhzmjTWQUoe?e?_2@ciTZ$Nm?X7736K_f$)zX$+C)$1| z4;F{?`Sd5}?gvVd|s&Xc3S8fE0ro9Xh7#0~Zp z5X3(RzO#UV?;_ajqz20DLTN*$<(Xw6x@z1TvmB{rmB$x1`otRn6tW;yfo}`ht}4h; z=9#A{zy*o&@E6Zq!x##ZQ_;UH025z*p>AXbbx`KBe3YlC$10qk)m{_BhhH4l>`Z^D z&F(}b9)&c;&FXjhuF;os$gl&qoqU_U!+M&|J_s7S*#G0%`rofD|2M6p00#Fa=59{T z4*%yR=YK6xdWa>}HUTEpt$;0O0{6BO72P!ru9>_~t6d%K}E$a03@l zX#R+8RuGRD(z#(%!uEuN+bHdB}FHr`~5JQVCTFxAO?9$HD^pD>VWbofZiP_z&d=K zXxbdiTt#GCX+6NgQ-tuPVQWu}Fb|0%O zJZuRrUZK}f4DzcR3vg8OKS(~4WF^__37qC`Tsc76EhK6jAJFsBFx*3cdJiDkcsfyzeh7zp=iJpE zZsfmtu59`2_p?K_h@i(%DJ6ZMYiMjNo0-SQqQ*6oBG0vID^_A*YflNqQ2J zqo|~^40R-SMLs_uf)2n_~8qiO~wR!s8idYv_=Hc}(mekQ7EPNu|D4603u93oXAq<4bArMtRH%c5fr2 zAD{fhBVxF`75_TTZ4T5NknWHim?u!+E5_6P5Ky7N9lz7|qhid?Lt>^87}HvditWMs zddpJinPKp%%F@FYp~+{Ob)md~0XGhvx&cjSzK9|{DZp+H!&=`+So3Pt2Sf`fUG&21 zb~3%#I7w^AIB^e#){*}w?=21e(3M?~eD4!1y(`_O{!^_%Fx?cDaLpht{+4>E-JB(7 zx5Jal12Ajx#D8S=gKqjW@PHpVx(yrbu4PTL0F)xdPho%DI48+E_qVN16>u2 zF?)(gl2o;hktNB_P{1TfaKFVPHfX__mn@Pl)8V{A~$iG_Hy^!AJ)9Um3GD*h33_RoK?8vgYq6D>JKoS>N^4IB4LFBU=r~ z%?6YVaUt@QqyQa!m@?Xp65Vz`p|G=jR~~n^<>D`vmA{+?;mwF6XH$J=PLgyKzDU_o z?8APtB(~JnM)uBK06%0oTy=*<{!To&iW~wxwcx@ivXGcgQ|Br=Qq0)#0;Q8n{q-Abpm7V?8X>kWa`BwtLo=#L7E>*VXGrk< zL_I>lxo)DM%M5YpC&~rHD-%G(oY#ZEqird=k&5h@9Rsr#%SnCY_0=kiIA407bLjZCIjS z!-~Nfa+a5!4rQ2D3MO8#7y6U1IMFDFe%|D<6d8EMQ%upcckn`k%8CW~0Ou}Bf4F6= z;c$GEC|7t9POG)N+WMy)>l6%+7fQ9n>8JtoXWG*SrCDZQ>Y<3JE9By)Z~Lh6!u73Nsgz?g3KJ$k36<&-vR zl-AVFqKj}wm>gdW9O>IAhzmg*(x!hGJ>{aFbeODbb`$x1HB`j&LS?h@ z7=>7qvZNltES~X3%X&Hg;N2qMR2LhKo3Aln^nx|*^rGx!IH>bL#1!VK?da0jkrw?vFd-f z%Vdv&Sn>9weKXbyzM>~E{B@~C%v{B;akUVm5u&WH&8!sXXXZ>8MW#s&@9nsDL366% z3CET0J*~Lf;KAG1G{Rjm>~>&74EkXkuL-fIz3n~9Kfk!=7$FRMfF^r0)ZgZV{qom7 zSmiN?RaUh17g}~LC&j+O1hTcO-lPnQ1pAYk`SQ5Fn{ z6q1Tq)(DK_v=$BxjS6&vIRzPND_Xi74ETiPBvrB09(sl)UD8s9q*-^Lk8*3x^ZjgN zX$*X^!jE3RxF2!>q&mka)CzV&&Pl2*7$~Z*HB#1|pa-Kai>sB3T`kL@)(h_VD)=bG;lYR|LbFdA=w>C0Zu!eATLSsMt(I2u+0MJFF}RwzSh zFGs1W?rC6VYobcuw?ciQ9G^@Mk?XUvw$coO=G0%SBbyaYoYyPzH{csWZ?H~7IhNhS zoa#-#3z#-Go)5lrTKaJzj7pbvV6iaMp&&<5X>adDO@ASlC8Kpn`v|+j9*mXQgzY|h zre~i4Q5=EZEG+HIAx+nDcD={gk$>^=Q$IUlh0`xNs}CE&86e{&C-%sT55qq~wO(z! zFRr=V^FHIWJBOxyh*mZsq^o>qM*&3*2#t3UbZrBzf@0D*;G>?1r1``+hemdwaSS3v zojdvk6+#Y6L4^db86#m6qvFHWowi&wD=%ot>jAQ450FwTOP1YdRu zo_@B&#HrtrMXTZD=B9%VIZ`LO+B-u)UIDJ1h8?(&Mj@a_7$I&3&vIg9^>*5E)RE1e zyL1RaY$TOA*DzDupB>VFluta2mb^5=Si|g{YgB5kZ3wbC0!mN|TAq?enbOQ&KsYpCyvhs(>?cbmIjlW-}RcQ;m7WwR#O ziRr@4zV2}}V(v-t^?1|va0@N5^X&8mV*L53B9s&x=R*|p?wc#k0{e1)lC8H5JFM>$ zDKV?XNjF^|X3dw~AQ`jjtwa+z%v7++4Ei#TQ%vg5Hrqja`gcx6!}%>qa`Azd`)hgf z&Ybt3F}!Z=ZvUe=XpyGySqGr1@$Z8w_P>ZKNk@RVqVq-7&h3ai1m2Gbs&zMzb}5>2 zcd=nU^Eays2U1CtL7YyFUN_r_sMQ_P3#8q5c{;tl+a8^egloP?s{Lx3ZYj0iNeCl^ zFkfOz!T8u&B^SSz5|d~_i(srZy3*U;sv^JvH7kJNH_p9E@)hErTe^cnjmqr$ajb_q zcS-$OJa&UKs@alF*CKmVk6IBt(cfgvgq6O8ZZrT`0=7?)iVT%v$`MqexFsoVcAP>g zF*EtG&o8Dr_};`FfyHfkVsyTuh0|8G6$X6By)-G9nG%X3j7o~1Suwy)(ZtwfHZ$2E zp~##Q_P*3wW0N)MX@Qt8-oVo+z2H&2u$DyjSatn7qy?9L4F%Vi`_#-1^saBk!8c#MpeqvlOU2gdw zH=N&&T2IJdE83oSL!O*x)A29$gV;fX@6!Q`2d8??+{1%m^ph5F4(8V7=UHUKn^^CD zqmLZA>xS*nSxUm5<*0tIEc&?6xfH*cgE-6vQ#U+j_C-=J0L1ZvtC$(=s2N8ECxOeg+4|5hTw8)Xm;9Kk)Hcr)X>cu?XYV$1pz-qP8@|Y ziK&!Ty%#i_&y^rnq?))hHTcUD8cVB7oHEI1IMNHZv{wX@Rh#wU3_qywUK=rCMU~d7 zcddSxNW>B);1!G5qaeQ<|A=I+QYDEOAzYM@X4B7cO3S@m)7o`pr!*EgR-T~|-gR}q z$j?EMF->CfbJ`EO6!GkpUJ5<;9_&Lr;o&MYCt}II5+xNhX4=Gfjnmj7&?XyP$hIq6 zil4l52PW;TyZ(~VVBy?@E$T8sTLEYy<_c&^YDv2eG;L$0OJ{?QvaM=byJ}t+Cc`my zBGlfwKJY6$N$Zk;?G2$S9u(BY#XDMquL+m_iI`OcTMu({SycX^I*5thn3(D$FbFNu z%xeBapQhl2en*?9X9*sXD+a+5niFa40p_xsg znk&ghdN+J%bbr-F`$B-EAn16K0$OqpZ+8ruk2&_vxWn~4@kmu}TCYBSlu{<#e&_ae ztuXY0V5&WIVI|alX3A!0D)J&#fGfjW@J8GsPs3f2)cI*#N;P_SwyRSbM$9I*CGj6B*Uk}d;KX`MvR3Xn8P~#ys`<&zsfd& zMS;WdQEHQ?9kE-*m*8|i865TE64{WK7MASJAH5Z5_lEY^Px)finz4raXv@*b_$* zPg+{mKI*Kp6~UWaVr|;3^>fi00TQAHHY_e;Q zq97*#`l@BMthHkOnQ+#)WpDTL4CCv5AH<>MbW=^)$<2U4`ARZEYa2f<+V=pnP2v?B z&yP_0&EWn(Q8`=XVpbbd09?<@a0PZt#ChB=n)X87HcGqJ#gE6H)fP^k(aC zajUz5T3L*_l!DItMdq_CtW~;DYNIND@x0Qf4tGC+c1kDob)`-=mqk;vN${hBg^vJo ze{dli5eFv*>~}t8;4ZODJy@3)SD&+KfX73S)rsLZb0=vx$`tEH<5F_WcobGNc0aix zz-6&{$5`Us0WGLz#>rq%FTyn9Djt^ogUkT3RcLjh=$`VAbV~Ez(Dsn31N#JAtevE^A{^1C!bN;IMMAzYzLpaGpbO@kWh_e51FLl)RIWP9KlFEMX-BZU>d1g} z62cjHOnYXbLRGuZNz8fnn?E3(y|U8o;%}Xc8a(PMqXp^8&N?#=XTP=ZJ6){@i8po^ ziHfsN1y5y{^=C0EQXqHFAn=HBKNrasy_*K6Rl;6YB+6xi?+^?pBZfn!0nO}X!~A(e zg{mxz{JwkZ;{3Wvk-1YJ1tX@&BNE8!QT`1S6>=XG~3 zyo)7zuV%?uHMObC_~JgE9k43*wl zIm9?K{Di5Df2M%vizhvs(8OT0)|64jyzgE=zqTT!Xh{|tfTC`qo@+LeJ03=qC~gTu zDhZ1sDYxvWm#8;dG9Apk7^4q+HgSw%aapaKRHEtV`m@p*9ajc@X%<9P%wiIWR?46G z8=SmigaTHj-AJQxdSjhl)oRh5wPKpCHdNddV|M186pBnWR&qWc1{15NbPm4@k7G}>+PI2!g~HIrm%bb zytASQ-^PK}IJZlv))xm>c0;)n_%OTrTCQp3 zJa9mn(4d`eH?t<{JmlO3QYS!xh_23EQRF>R{*%`2S&)ej#| z$l{y{?YYjf9)|Cm9_;a*_rA>Xgzm-j4T0PmPoYcENbbW!O+jiW(V(1&w*7+$s9$rX z#7chZpNJcYM-FjR`s#ufs`Wi*y8}5p3!m>oinPa?^5m@`wHxF({5|{Mr8B z3s=m44Ohy4D*mD=`TtS-75}UD>rW+70&y}hef>wD%6YB`2fx7E>*d+y>WN5Pp;)zA zrx^m!M>{oZ48s` z2&93f{B0IVF<`t-K{fG*KvZ?%wTCkTJfQrOU{l%5UTr8f>@g=+vf)FcQvFA#D&A4Z znVhyn*q`=2v8XW%*wI+?#9H%2R|`mwq@I$Qaw?pP%)}6Gl{u&5%%s%(K*#wVy-8|o zOHuBra8-DH(dG_uJ2fuh`#N8-SL&7(_%o}Dvp4JaSoPIeepfcLz#T`Vv#&y{(ib1q z`=LmG(CT=8#d;!eFI4AuqiP*r!`~tfdVMOVKap{m@i4)*`VE@|b{2USf2f2o$@#^K zv91EqN`1{1Lg&XsQORB0WX-}Kv}*8iX$Gl^y0)OqC&HioaV*<;0g-v*su`POeI_=A zMj9`(Geo~5K%e5=7Ui+DL?A=D?25W<&i8e>29Q|c0urk?Kw_1Rvp+bK4UdD9{6}K7 z^%_*2AjdYrM7aY$xIL9H*M!2@>Ju%45~>WXSr4xqDk93N-T1ptrSMW2Dt>n2Y}Tu_ zYL5h96@RBJ2%t|jjDLh;y>zldIMkmI+hE(%Vp_Szrjn*z+x$$t!$bjg7-f@j-UwUi zn2gD88X}-voj2^DXfi5Bk)}ejBr>YtAk%E3OiW7w%{U8&-ll)B}&f z&-7HN3}-qt{CChLcc8QO&e~lffYeIlmJO2)aj7dS`&=ko>((OY>?M(k+Drv&0$e3= zP6BB{O{b=`02|_!XZ-w&t`5t+_FKMv#&dslQq7Uag)|}1>0gAy|C%Rdpf1M`+tb(A|74^g-CVUP#!r^w z)%p!mk)D_ma%r?hJ^%PlZanU)XSYZ9&#BegmgdXfOh>`~*N2+`9mU!|rr0Vxx3lp; z5LQ)*f*E)k$|W9Z`x@|!8--T^mUc>l4wDp*WVj16%cx5%4bW>ZG_VAhn^iMz&X67Ke{lR4JObJ zP#n)X)Y^aUvm#)k2goS!kRc3^TLh#z%H-vVM_Z(P25$ni$9Gr?6Sd6`byu5uC~wVu z2~xyqu!eflJ`XxGLZgP83$SunnxuDoms!fd2{l3Z*U_f)K{Y-=>h%KON@ zkbLHs+l%{mTJvZbCF7}((JuR_{A13C{H>pId8Yqq)P0;~2_#a*u= zXA)^q%a0JrFIb@4Lm=w!=D_CxhiUw#s^1zZ_r%iyAD8%^-M zpjP*vuMmHZK%b<3^=ZHumiq5AhGqI|uccCt!tc9cH|W_~kw3`BYNchF;etWw=L$Mp zjLh-dY6|BSauP?nR28f!ru9>ETij|dIu?%4DhXoZjZBO|J01}Nt)sKEC~nz>LMu8I z-Z!lla_Yyihf*C>hyrjvN5a8&HRh1Jp@jRZ_`6iJJpgx{wR)%X)t={_f!K!h#pW=1 zT+cnDBDzP4Oh-YQWSNqHl1L<9(~)xw2q&q~9I7EzlI>@vBo>z&BwgxCp(iTM=H<1h zGAkBqDrhoxg4~O~2)K8;M%nuXtqx|M;ih8&uYZCQD z4xcuzb>>XPtCe>E^Ub}T$mnPCPPInfJ|}!J(~y><)?m~?PyUdbOu1gWtE$tyg$ipO zo|t6!LHGGIc(~ptgh|}E6T7mEx&Z^-h7vlsCgi*BKty~P5LE;syw@L%4)6>ST!y3& zGpnM(v0iJtI*e<15JI77K>Kp~5(-gD;0=3Op^)P;V0rxRk# za|(}C6p*dECL3z+e#IXw#*)oza-kQ~{RK>IHwmi}Ka{V*JC+Ns`n*_9Y!!gp$kk_0-WFHUUkp~908M)q8zxM!uL0Qhv!3^w%L2j6gMD+omf zPY*eovsPD9VUb2TD8IGTu^%QxULD33V%J=Td`M2ZDRxKl3-x!Qlrh|k$kmk=TUj-q zsP8=~M2Wt**@-*3n{z1n_w;xDaCsj$o9YEK45^#`IxQ5POtR1$L(2umb=-jTwo@!6 zC|t5gU6M!plWA|>O*5|}!gDk%#1`@jH?{-u(#o21j>^7{Pl-Hq0MHE5$W`T7J+7#to|vE z>K71ZxHcISUIBYEyvp8jznUlw3HqbKwSYU+$W33up(~XdQzl+ zK@}&|6wlv8Lc1jxZCHI~lQ(R#ht1FDN1erT!5)g*oKTJoEloydwr?Y!d%zpY7z@o#QUNgb`s}~+?c+!nN1yf1bnwP`9ps5 z@YY?6RzK0sr(nWA9JEl?dQ3yUtB@I^tQ{m)x94RQc^rFA`d0+=if)-`Q`s20;PYaCLI@5YERRWk|0=CYSf{u9QCCNch#c} zIDKv@gO*x>WmfsG4oj zun`udwhbbQ!c$kq0`{e2$B`;%1@h^|-VrWHPkx+m-JI!%1%#jsv2(U!(lKJBdw*)$ zjjV$ZR+#V}yh~yo4la#QOG@hZt4wpu8awH^mgBX@mCifEex~y&MHhu0X81`o+@%en zD6)R+?OTbiAt(%zSNr(tYp;v!ozIxjZJ|k79{F8SYVrbQAf4qBp6qyQZo?hb&0%cn z2NN`o3CygtO%ANsPE4^(41IEpUg@*!goc^#!V_B)yIH(|)WiaMG;dzWA5F-qh_pMA zXzkW|6Q{)d)1sTH6m)*aLw>A}(}CnO&_`Zzr{teTYTMY(#oWoE#20a8zG`Qo3x=MKvFF+y1M=!IG%A&d@23xA}? zO{v5!|4NUMRxZru{w+P;wIXCfcX$?$fdc)7)Y6*_I0P?GopSN{3498?Ep-4Ok9w%M zsXEKpLVd31jWv#k8ML=^!QQ1=4hrvQCeGBk;*^@^|D7INb9B1OnrCCguD&EqauYJB zmQ(33kaU*I0AetcH&G{3yZ6N4#-yQTk{~0gLMsK*sYI)9s;}DYk@?`nOTzqBwL!2t zwrLy826|?G+DQ#}mXAK%%39KoCEjlJL}ztnTBq_#CKYw?oZ&Gp!1dm8L}T9FR)VIo zWH0LEV;b>a>9NUBKSS}DWnrSbq&Wk_JqC$SAeJqJ9VOj|Frr|;bKM~^--Bzz%o|NO zwEU6+Oy?kaY@GMlw?^dX+~_0rdl9Vs^h)Jv^W5Oz)|#xvisX24T916sTx=L5x-Jpi zg-G({|5a#g_U$nLZ^l8EzaE4Ne*|Is3oRVJhxt!2FLCC$-xV4fwFjdE;3Xb4Ax$V1 zL$?on^%U3mB}jRGYR1kv_FL%=?%j{Nwu@Kpx?m%@Y3!u4W}5JugHjDzi!4d)r0^W9 z%KBihH92#sVtN1K;MU8iiL0t3`d=LEgM#>jgAQ+c-+yxu5QAABle!}17XOoj z<(zvuYF7OxbT=kREiyP6Nt)^ja4V@pc4O=qaNmoVCyTL)edW15-rP=P^(4b2_)O)f zX8pH<9$G1jle#OCdW4ISnPSCbC?4JWTJFGFec~ZR2UFOB5oUKje86Z_PDm z&1dbLbY0s*&J|iGnhaPoZzxu=L=5S7;uLfC}xjDih04z;5yg zN0l7}{j)Adj>0mI4Oza_4}R*OY}~S6t{9%Iy7vsn8j$*?fmtI+%@!KyVHUB~1J>KE z{?m5Nl5lMO3xI>ue;*0-=f56;fO}^_jDUM*FSM*$4Mm9{!5Nhl9@D92_pR8Y)y$eO zv$Zo;_g$J1Kg&}nzPxQ@xSyFf=!mK`F85%FmU*BiS-Nn#4vnktM&UL-rEnVk+Ym_sqCc1raI^~#JX~8VhLKvGW zelp#60bJM^ZshPT>P`>ID0+^x^T}E?=8=s>3g)e6tv-Yr9>v$xg8lYb%J8H#NuV;LD?!0n6d*1nCwL9Lz1>6#he0AH719_$T)%C|3^!Dn(MJ}C zXuHC&?U3mMLccinz+=KSWQ{A`A|^&6CeDGSh^e5LpGc&oN8TU@mm*|m)j$K?6OUxU z7eJ)2c%_Kl>%YoTKGVZml^Y0jsQh#C=bPoy2`e|ds@51xvw(tQNNNp)ISF=NBg4t8 zL$GCIviZ6izw;pk=GycMp*Nh|B~|Pe_+T&%8toxonDj)1mCfS_bKLk&=q(NQ2>p1&%tWPR)?&^DGw$EHqr%ysdYf7WM9gn}H zdXfnD^T)P6P>R%hxz>9^g){QxK{$od0MT^oNx&eb)&%3$@bcjVc_IALEgaNM|M9Tr ztD(>>hp)vcMGUD5tnT=tUVSnmxO+q#fLzE)S0!eTE+}5e{%unak5^^Z*V;1Ar$U@O z+L4ljv(RFUZ}kyv{Q#W^4EEqY*O%%^ z5Hp3A=^lD|&v9Ys+)7c;zDVT`ALWC74`1uoR7ukTPpw@G3?mD79|S7i8hS zM!`%)FPG!8QGbo3evGI{>DJ`|iJ%#HbO2)=Gc=GVIB$4h%d&a-#iBTpQ9NP#=MQ`< zoA&H)rUYX9iywH;?+?5KAgt!PRRGM_jBv za_b?g+Pc3d>_zp<>+m`kk9*QDz`$+4)^&@#E>IDXH#+6EJan=f$ zg^BE8Cnwf{eWB?`JE`f?m*eMBSW)D4a#2`3`=V!}akxj)RN(ISo3iTI+POm5A(D7{qMUi|ym09`sawU9*Qsi?$Tto8H* z3}cXwqD;K8*0ohi>3uYUg~!}RnkJm&St|i$Q`u4{w4?DLf0P+Y{41LKJyc!}{way^ z;>k)@NN@hjN`u*1oeDsU>Y+>5=mX~!SCx7`SN5-vC2F455XcWyAe!j&?TJL&6@@? z)mS{#APl@aFtg;n^4*FRDPno#Km%dq$=P@rX{Nk{ZnG|_nXfUT#8qW1*kY_ua6`je z`VSd$u4>;S!m5iuX6s=1U$Mwu7)->sofCclE(wgLF^tkAr515in1K&(41|6&%x(in z7Cqef^T1L#7RW^TND>pl>OWFwM$l={%IS1#rHVZa9%I*67p{EO=iT_k0?%eELTtx~ zyJfxQZVeU)MKeWxsW5}@q8cNy-#i+^T#;pmUasxd`+^;tH<}q4o`0S_8eicnhW%q) z^(l%yo2SqA{mltM1$f@t@*H=5EXDIYrNDOWyY*#gh9bYA{Y8P-iDCl!=ZPjZaNK3J zdB+bHtGnr3)KLcpbHDvyH23U6)W-C%E(fA7*#!emM6pFfk9l-Dfem#K$9I?MfLr0k z|I29I;~s4S0UQWO9I(M0_usa&^?Td*GrhTuqn)v#ldS{2fwQ@l(f@I6t3zeg22B*n z`;b=T1q|8p;mu&H2aB_Ihi@mCOTD(Wtjv{ie3H(Q38}xU*u({Mm6)GPNGy4APTN~X z`L3k&{Wgcn1nF3*XnI$?PABE<^hc_?b^d}D2XcNBT>u(hhu#YAQ~=9{c}0Os-OvLN z1CCpp8hGvm6pHfoim5G{T^b(6MEMLWBzi;jt?8ijJkU|0odjhq2-t35h!7U)BTN~5 zW2A{0ux*X95kl=Z3alje$rXo6@X{P1gHed>cBB=~fTop`(?rQ$tO(+KMkd>uO?YX} za@q7&ThJ(f-@JtnVp6af66UeiS6I}n^Ht!k^3dBCAx{n^b`r{3snmmh%!8UdG#Agf z2kW1^yyc>0(kxClMT72XhmST;3tp(hx4Z9aaQvN>7J(%@hdDS z2rt+~K8b_~FSPs4v{!u^nfqD)R!WOTH)C&u$F=5yyAi53p0m(pZEBdX;XHC6Z$%_z!Dq1j1d0) zQpxECL1hi1p^Qs`y4@!?2!B=wU=(MRTU;Q&+uSsnuP1ortm=@N^_0>De&a8I8^k7v)*lauhSu7GOPlyA_O-YCkl2)SR*I}H zhU{7pe3`#4?fl7FI)#hyDN=v0s%@>_QtDQC^q0h}Dvgil&9gYL;Htqo<~mJ%36?X6 zXL1k+oU*Tus&Vv`=oJccN~XH~=8p0!YOXI&0r|bDqRfb$t)k43>`6wg^)ZWl^t*Pa zdOMG%)+e2=R<8G_+vmL)Yq2fwBZ$lLwg#Sg`7Hf*5)HnA%jWV*+mP z5Drvd_dr)I`WW3#Dv`P3sbNx@{J8{csF#z?@9s6x`m!|)C!wg(kH1bHFJ^h(Jzi!` zR|jrCX@nL{?qAz!K}iPDIIYW^iGd$9C{UHV5+(AV#5XbECUcM3^)A1#J1tP;5RW7LIlwYxHq7w9(*c=QSisx`ogNr|R?1z|FtaCmjm zQLo{=?OF8`DtWu9xzO_7bd9@(-U{>5y0^4+3?;?uD>oJfPWBAx?< zH+XS=%_&-W%AGB0;YL!2Ip0}w_ogN0?$iT`lTVvs*%x%FDlNiSA*6MsBX`}~IyVgRtY zvI(G9@ZWBP$injPKjc3Q`v0Gd5DUcQmHfDWG-i5Sk=O0RYa>IE2@hzIeU3A875?_Y zGkRU2IqQT40a`1R?bz8DyZ7W_@BVX+BBkSGY@-nb=(chtw|E8WC$r6M*^3pV(m4-Q zP@I-CX|!>JbdsBue19xs(Tq<7?r$SRRiV@aa0-G1A&=&q<$Nh29bm!Vw?y{~07i)Q zwe-as%#CL4^t8&8<)2qHB~?5SEfIx@DS#fq6cIfvA1lAzsX)8S#raCq)I_DU?cX+) zpQl7TJOo%7Mw=QNCixsS#lK;aJ!}P%sHQLICR-jr7A(L)4KvZI5o5=bz^UA9}2N~E+93*=vb8$R~5LN-Q5Z|L5UsxDg<=K$QuZq&> z%7R8xPq^@{f% za^S?|f!h3v^eN=0TQUV(r2R9~%LI3Vs`e{BcRHDPI{ZQ1j2qKi%*)e@F>Bi9TCbm%NfmwOU{$0biub6(_Y}gQ;>ghJ6=9jDZjj0hL=lH-?EA|-M$>b-A zH&h>=l$he(&0Qz1?;St09Syi1jahWa(();9lvm=nc%fX&m^IlK%;U+g46bKtY`Sy{ zt;c=OhHTpA+}u4GQ|J?VYL$f<&Rn)kAG|j9v&|MliVP$^${rnr_8Am151lbHzPEA$ z^|alGK2*vfDI9N{y8^n1MsGNjYmOwCqm(dZ*%Iy`gr61}qK$c2R@Cy7QqL6A6p#!a zM2UD_LZx-W?!_q z=)U~`_#hTCc=_~m@*`g}LF@r+5Z7Sn&wkq=qVCw7()G*#wn41*svP@-FtgEp-i~$F zYj4OZJ&s!v++H@J(im#v3jY;u?hHnqLod4BQ2Gm1TB(r_kT#d@EBK#V5M9=_hkpxh zFf;yt;D$E9Ko#wu2C9kw;)bZn7;Y;Gl2}AU%dipA%y|UKhj+nVm5unEJ{BAlhVx`g zXiEFx$-us`>@R`a%1@qc@IX1P`^niB6T)-L3sv_E^uxWKdYcbd%1yIa8W~d8qM^O~ zQAptr-%}S&PA@w?ixf~H-~Z_D6(r{;$mP11;N1y=8PxOl^juUC-(|mRWHR18Gb4w! zC{WqZm>vZfs4~VNK@fF^xpg+(Z?fcVh%+F-?8x>sh18a=%lL|(_>imneiCaE@f2Ye zDGlvRIPD^8fh0srQ>uTj)tV#fm-*74!&hHWm;-1w7LXSq)u4dkffc}sbpz%7h$4a6 z0Z9a>LE4RD!g(SV8tSmQRY}8M4?76IUOW>?JCPm6fzFlHv-9P1==dq8oJmJSa0wM4 z7nZ8eXQ1;OG?U6mXxLAJk=IKJjts^p<&E-!blH8_iCDAM2p~sER(Zn;l(9%HN-+qS z;HVn3yi$k?Yt(58I2j2e1Qk?ug(L?wJ1Sg17VwedYA+)iqaHbQnlIHf)UF1=L)8V~ zp=y$E+=)k4r`|d|r+)1d4@l(%$`0(}D7QkH(=zj=I=sOX%WI!FUg0Nx8QfCXJrS$J z-CWU0+G6aU8opK#O30D6k$LHVV zC}=4xzEM69d;d)iI$gB|^Yr2o06A&_#65;jI@TRXA(!A36PKUYw(>bP>4v;mLp;;&VI=8)Z?olK=nw*cW5v+`7XXfR zF6A9C9t4$)!Ms$9!f2KFsrwSVpTG@s*uYr$xQY_w8F!I-8Qk(<27Hs%gFHug&>?kmI$ImsT?br}La^)Ag+LS^lB|KM496`9dEK*xvuP%Y&C780Da@lCTTf zzRd+=ryBN3hu(}ybpbQd4t04Qv%-qq=B`m(jT8;jwP6F=_16m15u;KI{;UNbuDxi6 ziJW{3@&K^YmDkmO$drPKZWE3F4*AAs_Wz%_@o)LYzi~tCzqpaO|G(u%hSoiR8>=ut z(`^-SNOu*fgsUFjs zeXh~vO%lU_0k4V?(Z=m-*2P`M&Rt6FU6@7y<7HahvRlEfq4JcJhdNYD^3S4_YuybD z4jX6hpHW>zK)|T4FA&A_OCx^08Ue%V20?)-h-3ocgo7wdl8h1)r-pKMa};nSp%L{G zMH7MahJpwP71&wxvP#-2TRV|A;Fl<5!^j0-9fJ4jBeVY`awSpNYEL^Qr8LWCV{pt!g$I?1R5q zv+!qciA(}rIA%g@jPuJIjkV#sC9lg+*5KG`V2f26Tyh1CgLxAuZ`J&{ zGXNW;G;Ct_*-L+511P}t4Dmucm#AR=t!>kOERk;VvKC~BJxi-$2 zmqmSz2rbiU@n`#l$X-9bsF-N#M3sqWa>fAuEyKN+!z~2_V^WK|y_F4+DIOd3;#>NE z`Y!9Ml>ftb*=O3*G<3H61NmaPhdK=8@egVw-`AD=rbZKh8b5whL!D=7A2Za9vbycl z|6uMNqciKawcXgN*rN{HI|L9RXG*HcpNVD$1}R;0a?(=D!~&S9lOmh+AN#9Y_-f`M}9; zlLese7V?8cRZ0mkkCS4K~gOw<;f=e&Ps&sc*m zw+(mOr&DX`TslE3+|u@r{a`?j6yx+$X~F)a80A*m*{gTakSA#1p4EHtb?4nuKpzYm z_D$ThVh?dvdU?UVV=q%Gw(wIR-3PpH!soVDrO)w;N*%v;eiTt9eDzA(Fa0$Zo3>t1 zZ>*xIE5-x;e8YrqP#T8|yPU%+>W40R(@62dTqv1fF1wu75d{m1<}1Php)WGk!-yBX zVDq>whLIFb4jaq6hJdmtZ5;(2S-5Ax(~m1~mBm2nk_U_eXsM}7V9V5ZY#Y7`G>Cwq@3<2&$m5KG!g!-HZ)1IM5uq;2c4g(@ z(LSYhOUTQ%CkeIUb-j1>Jg5KItgcnm43yW z?ZUL0m!Ec4_^5_a2SIej=k35##7Q zam3teto`KM1oLI4F7^YVBxt8kd;a0l*PN~t6ZjRe61l{g-G}W58QPoOgJ=4c*!&=y z^Y9cmxTRJwk(M{N87Gq6sM+WvMgj#bPtI<+VT=9%Lwu%uE8Ei&Xh@+zzLqKu{{X$R zp9R_ye^b*L)YeP@aJDXDm*dVoC#KSX4$2YN??vhdj{>PZJi)YehjnjSGG~j0FXZ{X zHh;ZriU_mkHQCd)*dR@#(ZKma6lO|p!Z@+kRj9+TiC7)ny8(?otcmL9xI;y}0$ZXn z;ZM2-2J%S-m85c+(X-z-IFElZ`fapLI>{Xhh^!l4K&sA}Js8hdm}{zRpK`9}(N{A& z@qtvSGR8;g&$!Nk4b$}fAKMbYnoeM3d}s>!Sz5yia&gbA~GUdmH?8m;NT@3~*pC|3l%Cf`{-7=P;K zpY5r$lj-eijxp^MkDP5!tGx9Rm!re644G{`=VsiQZ@PX>U9ISjZC+?TCEe7XdcLsZ z%rbT<87Z?a&nTaNWn4mZEx*2neT+)GaJ4P>x`@dkLrd9QVXrse?xDTlF_Y=9!7<4- zRF;<&qvFIuv*&7gUH_F(N1ex2R@mpkN)|xnmP~U#r5TXqEipa4J>B*9?#ENNT!B0A zP*eaOihusG?qFnNYw}OanK}w%)_6<^z|PKaJ{PSHN+6=A2n-e_Ur$g{x%?~9=^3{V z+QUYf&XAA-0Yo%DQc&Ohn%BmcqF)l`o#$!$MrG4DH2$@NO|^WZdt;%S1}T#mm%4tq zofo*QMh^paDP18?oq^1jKQ+B}0rcC@MF^Tk!sY>DW)TKveu7K5ABQzoV5V1r!cz`@ zO8RtcZsH|I76?~BX6?AR(L3bGXp5OCs!vUg%I~hstBa?~H7ba|UUJLHGejphyg>lv z;z?5;=O2j{q9RXJ1u$YVNqWXL*e7yShHHXj=glW}&b`;RpaDLUsq6;ToR5`u+HLxR z1KQz0Ghh^GMv5V_#e^nvtG?jA%&#BmQ1@L=$?ejwh#q9XkA70dlxBjbauQb+qP`ea z<9969aF3sKqiFD(lv+v3xhZ-R)ES!Ho-b*0V0*|?Vc+bJ8Kc?Au-^lS@Lpyq z@j+BgYJynVghh&k%n_LFNu=(JNEB=ct|&Mst?B*!+IdzRJK+H;TQn{ZkkCKiy?5C~ul6Go=DvTMZ$FW{E>5r~e{>#tDD8nK%yX#H6jp7Rx^OL9i-O(6Z#OG~)@mKC-P4*QT- zNENJ!)D@naj65y0zZvEwhRl3O0d@jLI#y8{t&AX_5k+%0Kvt95n=vo{O%FoJ#x!Rr z-R7~El01A+-7b=nYOtZwE@s;fB}XL|leKrCd7*zt!fm=#y~wWm{4>H-RLbP;H+xE_x2wcZZlbiVj!6kb+9rgWBe_D#*6GAt{ji?Ft0j@ta^z~gfgpd z(&$+fYxw|{Y7~V?Km1C7H?F90VGp087J-G^9F0c2ih``z?oRyc_|8@aFo+9LB#AOD z`*g%SDY?M;1Oc0w=|4S6gBp)<9Dl~PSD251M-;=u*H)C3N_b|?=j^@^1c2f z*vX>)^wBc2k}j9!c6e>NmbYVt-yV*I)rS_{pT8K6x!U(>54=3vERHSakUvV8&i1b8 zr(vxyQ&&ot!Ab$X#8T=`{Oa_prIoxF-Gi1TS}VD&9Y3q9S!C?TchgcU z3TzZ+8uj?pBaiiZW&^jT>5lIMYyCHLxfzABZxs1IzT>~hZr!__>vcRibbAmH=WQdA z$t(uNg1?rv-W5IBmG6dqg}Nkx0fh8c>`Sm>S@yx;f?5xmFT1jKkV|f_>@x?b=VlW& zP1Lwj&0{47)tV)h3VG6Oxaf<(+lobN<<0p+;vX?-OtZBOut_g!t2A$LIr{-&!-->M zCP{PMaO|Ettre~KIG||?D08;d{#ha_I5pnlN!LW(c^Op=wkg^ajOeY7W{}vta*XkA z{bt)k7jqA=*!}%W2+-kLB+q&Ag-_}PI8Vck1u~3Zp|PW%g&b6KYuFVF>aoGk=+BYL zW6Yz_HB?w-#Pvqg8L8s8Z+kL;_J8rvmfEb$|F^?I-+V+#;G zgQ3S!UeW2?NO~n9eZH?gahSG@GWFnH*zffPWg8uYK2%j|Gu|}E5;pEZQPZ(huQdxA{t4lv_sKrm z4%@n<)I0=d;*i-l)`WC|4Wa0jx*@A@;(l4D4_3Gjn8QEs|6nqPHc&%S!Q%;j=pK-FSWu_CckpQ*XgfpvLs6AT_ED-EQlu+Gm;5HNrt8VVtPsV&?a{i3sdp{%3r~{#jvzlReaPL6h+7fdPsN3H)3MCyU zkO^kh0%R%TA$4eJIWzR=605M*w_NS*m@+(O!$zi;Zt_#n<$Je|TEBYoWTc0DE7u@2 zO4Ft@&Fyx%&79H`@N>F!W)`@VWk{UmxSri5T0g&dzk$`dd(_L4DUv>Sb-sLZ^m@z3 zs>=&-VfVO(5_VvJwQ?U+&m;X3w2>B2_3(V{hwGg|$RKoh$#cHrcqMLouYji+vb1}3 z^K#Q??8e>ZDxNJmJ#zV6Hv2dS4mrCt9~)AF9IUNN*TH4RpXEL~@^XS4?Q!2hQG@b& zJ$wPCE{l1Ni`Fr4jd>SUz3KhbQ%`z=xYBS9# z%z8PpPhj!>&_@sbal7_3)aR9}oCeNC^@}J;4$k8jiXOMr{P;CmtXoSC0+m_B#d>8m z7`%3R(Sau+=QSq`X%76nKEgOO#Z7$Ogqidj^N){ED$A|3q1`>U&bE`f0g^6~3p(_zn)zey;l6HyBb2yPjAf<^g9VXvwdfdEa#cdD*<>N!4r{G`5ZTj*Z;y}p z1S-r<>4d5nyGYr{zBBL|_WplyL<9bG`DD78MT&a?Up7T?E5{b_?2{jcADTn^| zyt>VZ>Cp!qeZPV{gXVaL3X>Tsw;ZaS?C(X`zM%@5X(F$b5S8U_J?}W%n2R?~eltH| zgR~cXq zp;3oU>NgfU4r`0kdnT&zEf&`~43k`HlO39i!5sK2EKgqKlw7PmfgRV%z^u0r^CchU z3?#RpP=k*U=U>jKarAuu_)Ca$KSO@iIEJ<$n1$8}u)(o5GF2%@`u6r(QLkjo68W6A zTZgPLA`-#7pypYHWG7A0dezPDPbV!p zqqwO^S?g3-)SCJj6REU}z)nj;;H{(Mij>7+yUzU~-WOuH8M3dQyblbzhc$OZW63O6 zcdoVQx_CKplpv8Vp#&G%Xi6M+(45htStHjLh3`XHyc=XlijcCe45uuynr@MXZjN4% zrfZZ5Uzz5c^ukp^bQAIv-1>{!vuWq&3GfOeE+q{Di&?!rB(tj?+`JRon$zt&&#t~b zRt&}iG#q`xh_6l^U?RWR%4kk%!=POG(N5s!V-#AC+q_QN$h}eni@tQ29cR=Mn}_YK zj;TkUFE_q>ew_M=bn^{PiL9}SbIzfRW5uwpjZ}wE=k*^t=vw+J0H1*yNgue8vj6iU zf-wMS4_R9S8{bX-dCR+#(jU{m`!onyW65$c)bi*dvm!_U-@+)7>Aq0yWpm>hG61yD zv^RqZo_OrSOu%x+lE#F+ca{wKtYuQOZ&k8AQQys}e+Nz;0IgEzHI+V5v|f`v{z@FT zcQ{t&9_FQ?NBbU{FGewLC+ds#XC3jfK80}DpG~>H zRzO&httB-uA|zWA^XcQ3^%jaMmdX=h<`e~a8wm)+tJ}2lX6hv>l6I|T_6?=JV{_Dl06d6Ir07@$m@|)$`M8)%{cq* zG3t;;m`-jg3~;ieVOQI4!=hXqhmUBZ!yqn1emU9wsV1-d!z@)GKjSj@(kGIf0k5zm zymKZU{`uoufX#X3h&a`BdbejWg_gPHDtb@x-jJ3D_Oc8@P5Aq}ouznjD|bW*d_Tsn zc4wwZk8LQiSzr9p2kk5?gdFND%xNjxW%GdvT~Y>)fqB6DeJy_Je3`R?mU@LWzjY^p zJAw&+|2M_JY3hM-mWNmCRF4^<{gbJse{j6`ol>*lfIFlJSmeq3&)XsYb=1(g*aIAa z<{o2JZY&{S-$p#sd@j<{`-gQ%gJ( zkm_c$cZj%uK5faKwF(?wBe-VG=kyjZI@e^i*$$}EyWR~A)*XvfT$dEV)~I;|{qEh! znUIEIJI|I6!|a-uvCocOi1B&Ss>s8HJBT*hq;a1Yf~o^V27@trU41Vd;*^vx-ia|X znSo@(&|1Yn3ze#28YRg%+Z^=E{UkAJ(05c=`sj67aqgn@-+SaOrMD_NNnNu-Y&`~} z_cQOu^B;_vE2g{%bJC}>?UT=NX;Prb!vqhSZWYmU$DRWbKlHdhZ z{?9~lZG3VT*r`$1y5BF0-3UDA6F!B4WEd(!u!Hl(6>-*)y|>y+qtHEI7oQ?2NJi3X!7!_syw`Qh zGl6%0a|*XiK|E^3gVB;5O$tv*uInxy(acs#$jh2raH3h?~k$b$Gu)BS+ZScQY&={nfqr6p3MJ6K_MMP?@TlJLf0*g2!} z2#q)UPt^0)A~bGsRi_>u1tI`iC!WigZQg?AbYkm_dilrv(zfOG9^Qn{ec!3y3pJ6! z6Jlof>zuYt)@nJgt`rSDn3RA#5@CzyiN2<`v$ubc5}6+fB7A{8Bo)Bt&%2`Ve|jzc z(BxkY0G7bJKL2T0R0(H7>SaU#xo`hCIu4Hwt3gGrvN&1~r6F}XDoaDszo9VN@9xst zbj?4!CeJ}hif^LJjENxe?qlg%OsFE(wx|d=AgM~GhHa|{^B|;`0k&mnnLj(em3&Z7 z79akAP^FK_@*{k9LpwgRKbi-!QLH{cf_&$L7Oi0WqL`W^(fvPkC>Pv+b8Z6?5OVQB zK;r&WP8|bdD}aLoz~-NyGN&3^Hrwo&URinu4<(S|OjL@fCfSs{k+GMuhH_Q~Vm#O| z=wbuQ!?8390BFCrdp>3KH2ca~$9t1JkdO(7U%y(s3_Ck<5CX+ghJU#-wyzWW2in}C zs!3Tl!l`!ks;L+6oJW+a#FM3>wFE!0OVc73{mku@EB`4|OlCNuy*QxUTCM^S>qb?X zE2~Fww#2&Y|9MQ3w8_kzg{CpZ=DU8c&ITF#N=TA7dhLwx_T|s6206S5j@otmY0XZ$ zvDM`m^MQnNgIA?z9U_UBbz>D(ZD-mxvJ`}sfT@h)_QU{Yu`Xxtd(>(1+36KN66^ki zxlUZw#Rbi)RjK@X9h^^iXt0&JB7R!&X;vED?3#k1nh6Q4hZV4?s)O%|!eVWH;>&U5 zrm4G%N^Et-Hz^VsoeDZ33bB`Cx#cS4M+!yK7M!K#N+Ye~acvT#J)zpih)Hn({i}Tq zlDyodek)CKFFyhBKSHV zrNj7j1bG7+rbDZOz+uS=dcAnYH&*S(d^rNwr$|0MMk+cIB?xm-^G9EOX^SgH3IFN* zs%jQ-3lE||B8dkWWS5&Rq}XoD&?hFAEgY^(GgV#+hE6gT-Ap_=(;=3tZSM?{_W*o1 zYH4CQ=PZFVoID9zi{6ZpvJx3>@9%r{OoOYxY?azuPXf)A%IYIqr`ZYdz+y6?_(~cw zLF(8^4elR#-k;lOujx0eV;^jJx?)L zF|h+*p*5j*qZ3di6AS}yY#5IV;hr?7+yr}BHg%u9{cfP+j8fF5*XjXXa9l?X8E#{w zjnHJ})QoN9;*tPfS*DyELNjr=%wJ#${V~u^qcrWIfZs$?N+(QO4a{DD&QSY7HI%QX zIov3=`T04ga5&turqV5wOj~GueQs@c`D3Sq4#afsc<%x(5db)n7}XJYB}auju-(qL ze2}V^#~UV*Ayta1NcZ>J#aHmWodf}Yu@l!S&`G>kd? z6(s5eTR~Q7YI|5tKJ|zHpl?6ZeYHrj-ImM}y1DgMnoQ1M7A5vJ%6Rar+Z24VLtmJ& z4HQP>1&n?EjS&`;UViexswtDR4z5Ry2--L*InGD=q46OISUmXf5E@8`Zd*;e3>A}_ z?}-jzu8v)h4zN+%@$w8#f;HPh5LSCkTc6BD(Bk>A#siTMWF`uN^+78g@g+|euouP1qxg6(R1+9bk;0=H-j2r2gP@swRBnBbE!tL;~jL5E6QM;@2g#x zRqYzT5i{7_z4|_O*H|>R(@ra;f|-)+(y<1L#vAn@f8T+%__1v9zLsSu5FqsJ+eM}a zt%ZV#7EJiph7h67J&xrk+P&@t2EYiHnju}5PmKo0&)`#q9sG~ITJnAu1e6c^auJq0 zZ44amBl^i7yVHV_>!{xIAdT0Rsj^%Vy z{-E#$KN;g^QPm~hO{O=aS4S;v{>(uGG=%;H2-prLOnuX*u#fLEL2*y>bTp~8B@{k_ z@OA?7=R<^8*5Kp7V<8st@zD zGqWB_euZB1rJN)AAdRRvZpJ_H50ew=`zrU7zzq5Z`p-0Mf0Gp0|If(@dlLs6Cwn6k zhkq&)3skgh^4Zb7rmCd{)(~JMr2Eh^ zYDvqjC6)7H$0r&pdtZ`dPAy5x(uX3pq4>!`hPB<=lK?VvZc>r?OBW|}o$a25= zmQ+@bJ=rZA=>R2+jEdAq(mW90*;V7%KYWronQXZi#&r ze0%%H2xkgSzF2F|lHsXVd<<>y-{e$B{(a>S(xnN+R=+VDgW>V<%!ynl_Fjv;# zjD(0E)}v=xn!>^{7WSw|0KOIBRJlim@hM@cj4%Y67-U%E(`i|`yBjR-Zs8p>V@HiN z%BUTat&BPkJr$%_TX;S~vm<4~mxR6H(}~rs)p83sgTRx1!*bm#0nTs)17|qCFc6a@ zKd<)vWrm~KUkFza3kCwxpFXBH{Gz#fS#>qj`s9g3(k7NRC1A{?elpj@l`ZNCl3sm2 zvwT+nfdp>9NLACx20H-x@M!KOBRTALqAhSF$ksDsV2rg@XXtR~gyTA5y?o2!IL!^c zV9BzfJ!CQLT)C@Ns`l~dZlT_Y@D+EW!axIS;?W>G1--&LzP;V@K~M3Z+`PYtmSJgI zPLt`f!c{<`Zq7e_&1BKe)#GZMqp`oPE?6~PG|=tKH$RO=O}KA5lsT-mX*3L3@RVRl(d}vL2io8SISHs zZp3$6=T=EX>!A^B4kJ|4Safrh`k1OSWj~=rcReD@c2~slArZ2`+&dkH zoE@q-bx5(=9JPu(@S5B??q^K3*md4A3w7=INr?qfHC7}`*%8N0P3+9n7jaOtG+y`~~(I@-@8d@JBn_rHUvEP=#~rpYVqzcD9d^PXT}R6BN5+Ud*UN*Xlgl2xJY~yVe!eF#^o{U z@D1FGh~!@N(K>lPXdp{Oj@ay#Irx#%CN>@QhMRs+CX zv@YZ}GA*jFt6^(H8B+gNckH)9lvQC9b_mI1gk%=pYk>q(x}zxDP;y$-J>C0C_D-pW zby#n9P|3Xl%D^#-cn>>}jmx0*Hm#uPwS0a)NX?&ACV^%9gZ|g4tY2A8 zE-liI3VB!-Q(7j+od)|^96TX)xCgX}y9l>F0)!3irL~bCht?(r5qGj*#OP0=`G`q5 z0(M!|CPm=CmQRC=!x?yx!(}GXy~CL2X7Tj8ydvo?5}aV z`SpNhpo@H}ek=r&wuIC6j$C{}k&?yq8g{-xU#{e3Q6lq-bE>otqxfV{o5EW6S8&QZQq?aVVnS2AW0-<3XK6E=ZK8gkIh z+WTc6fh9X>A`~rk@1a(6-mTX-Lmmk|HkB+qtbH&ASfL61?B1dWYHHW(kgAi4A=9&t zn=O?;el)hB%2agM70wbw_*KYT~ZHySBn^O!|i?I~;c|1k!COnDpuOL5mXjl?Uaq<-LZc zEt42KE$q_?42F)NmY_Z#c@Pw`aldtZuVR%3>ilKq;wmY6IxLg6j9H8-*)idCYnaCw z+^N!K%%@-VIlP_$_TcNxp~BZkZtVNRMTkXmLgKEI4g)o@-=`c#nLcm{q7o;53WOj| z-x2B-O8(CK3FiYUd=L16QSHicFKq+CbwFuWvt^x*nTNB4_DB9eguG`u#nb`cW6LbC zn9oqy4^jcXV3?!$%snFH#08=dy%adcI7$fmqa^SoP^dz%B!yC~wdbJ)YGNig!T=AA zcyl5jh4Fx4CRqE8>8J0`SM`?@gN8VG&$!q;i8k@K$}3#P0|$4 zkz=&d)+b}fG`No?w(WRjw(T7G!x@(2ddJ*XhWRdx@DbZ6wdgiL#F|F(;k|<#SUbL9 zaXw?qFwFFDhEx^$?KnIYc?{Wny_CDzWsHog7r@a&D$FA?7U0S%=o^_&pc!I(`$Wjz zExf;H?1`#GW?@0?He* z%c%ZD#&*!VuaL{rEp;EG`1#;38-Gv#?NYgv&()h zDh4@&qr#LP$98B{3KFQkDTiJ?d|ySni~ZT&T5jkB zNG~OEx4>cQYNI7hl)iVmrr+xmpd|6=WvOu-0Ttz=Ge`*%$!2#lf$_IRCITXng~tv7 zgf;?tRsQMq$*bDORz0GkxU5qdEo*VG8toUtbzQjtSRz1fQqTPJ6FmXJBZI=iwZ_c zsbh{__)}d>#8)k2@oZoa>RfbG$*ItqZQ!=aAYcd!KhCX8Z`dlhu6M3J{{RP)>(BK= zqmM{){sgY1P{dyWKMs4IkD)Z^V0rM=0kIHDqok4M+24wyzW>!PKjAslgi9rGsYt(S?ev6qS=jc zj&F6LRTVaS2Fdw#yIbU%Yu-gqyDI6ENli_K1lJdfw$_MY)KydW;4%r8PHV+uW124o z`Lbi0UQ%_w0sC`GMvTw9 zOqu~DcF0LlAx^$k_=JRUjwZS5K>+OXJ)(%kwfnlCgD)FiH`}tl57xBw5q%n#d4?jJ z9--WLX6DpZ^~df;pDzTM5?Y!{7o1dv2W;2EK@L0!2kF-()}6OcpKp@O7Wsj5ieSiqDqfQiKg=EE^)pH}PkE+}cHMdXh>4~`CN^eI{&cl(sRz_m&hnlwX; zI!3%=4K!^w;q193)sQ|nioil=p`fJzkXw?MNsM9Rdmfvp3 zXKJ&}mx(B1#~MIKpGu32ryiDAL=RjqkRNyc7#dp_qbxnYE-J(e|#DHv9-&P@>oGyLdnU!7&_#8(R5~U^FrR|HXD|56J~P5o*|-3u8#3 zvpEPAB&MQX_!#$Hw2db81n=$9bRltm(o94*NLWKH$evO@n<$A9IU~ww>ZYZNW5wN~2ioZD%ct>N=JfM>;h^_?v?%I~wn?Kf+s zq0~7*85;7oC{}AsY{|N)Yg?s@R@9qexM{&vb-wWj@~e35yYCL+H+l4cz4@fw)K6LL zsr-;QzqqC_hNPhQr>KTRcR(b4)FZJ`eQaZ)L7@$xuELE)Khr`WXGe|LRpHgmI}dV{ zdTrG@wM_$AD|X`F5OBorq4QbO5Eqw5{$&B$NR(FK0u-R)03K>7C+cp>J>i{Xh?aBX z*Y1q}7NBZ>BS09bZHdV+Yh6%|m`!b(s%}$;|;IG<``ZuPB+jZTR3#37DN39%7|==#y*Qx+3ey z+7{X(g>E{u#a~Oex=whaczL16ksIog7;)wIsN>u3H@g@!Zx2?Ih}tj9C+YD>XMpGeqWBdYmPJf&32>T?x|H6pg+~TsPo`&YN5bu$XU7&{>^sl z??qIMH&OY`cEj=6)VcoTq4ryTzMpBy#sTGL1lkE#4*V`!fK8^96BrS6nT%0_Oa%MF zf$-#A$rn(^x?`3kzM;XL0Qw^!q6us@QYiQbPP4Lz%2=tZy_Jm*43WT@>tA6jzf>## z3$`-nVG$Gv@fPediY)@_`|JJa)fHWHX{ylug}i@lStEFxzTYEZYe_pv4?j1h`>gXf z+s&fUiPXHHptWpVmMy80zz?av0*HYQ}e`C9eJg{H=M^qYE|IWty zg9-WXs75Pbr2*wFQ?F)uD(4b!F#ia;mV_(SQxa?DNK<8)om@I903%r@Z}#p~%e@vz_swc*=ogjNA5()bOX(3NYQ}qlB>p!q z`iG^$E7gS-PV3wrx|leNVme%S1Je6;+xi=y&_P6D9kT1b3)Gh`VJXT4A#B@`G8IBH zls+FU@13?lsri+wtR(OmtVotD-@ycU(*VFt`^35}C>OL{PSx~@ooduDkNRA_d3)_B zbpOze>HmUmbWQlJ8+nKDfV$D>U)POE?WN0DGCEchJJz9!1pkI^)Ei`D|HNOz%1yOH zv7!A57^zyp&eu_WY=0Tygp)2&YX#JeBTc1i zmq6XP@>@54@%ckHDlM$g{Z%)r{TsURiR2I6IA+o;+h_7Ox^eS?1gINzfx6Kajxct< zp0DYU>mvTi8~@u(zPuzHtNu353f^WU~!$}n(@oblDQjN1=dE1mlH9uLGfLnvaLi`+nJb-)eD{RDBhq&z z&Z4u&`$lh#V*29~-{;)E_EeUTC1$_G>5KPVE7p+uxND{ z(Pliw6$Wj~SD}e^k!JsHsak{Z^JU242&To28zWx?70FdNIg?BMdl?8sv@|2cj)`!h zy-LDgDhZvF0-Lt7s{emu&evvBgt^PF?*ek=-ki#dR2dOsY1j7ju1)m(tF$Fc>A@RWSv zB+XSAw>PcP10uv@@m+c-zB=CgG-W1~I$i`jC$6K~52s^q)QujL%)Y7_tc>rp0*tlL z!Crmv6D;K77bKH-u!1v52KU8Z7kr_8`r$uxhOir;2ot)ozPy*lM2szl?|u`P#M7lE zlGbcDn6w>gO1`%H_TDVpoQQq_u``3Vv%++h!#9powsG_iJ+nmh`y*yRgW^xe53{oT z?SK3?@xyx9~jX;aDacoh|b4nQhzemX**K8Uq)9x z)BOv6IAOVVv`~tdopMV8d)Dkd1c)C-{J-Ld8)^}YmKPd}I|%+?@xzJpvj2r2ZU|30 zwSVGKy_Ft#_7G}p7_6&ploGhX8+n6|^|%m=e+LI>B!DD+t_+l@<~)i8ylmMJOq1~beIIRWg$|r2ib%vhba%>W>yZ~T zH1809#3rU(h4B;on1#iD@FXbZ1#Qv}miJ{h4 zucWVx?@olL&PA73clz{NN_J8ohE4&F;5d6C3?0E8iEApdbwy%5z+(8L%WHK-?=_y|hIw)H*60VEzZZMiC_q%0MlyNj|tWqNjSg z0u94Rt+lI~qBUYiaHB&ZVW)ZNH=a?rEcKDhN!c#X8})_aQQVnUJ=5+VTM!|XeT0fL zPSw_J_2cv1MpH#%#%W?8Er&e)>@2v=g&F>m(GMgc->7-|scdVN!M&&@jK5qni%tzX zkC%d0CYDe5?9>{UGK8%ZmVD8|okK7wx#Dd4S)kd2-;GF8r!(aDlqTUoigm!mVi}NPSeS|K( zly7QunuB0Nqq(T!0+*vQf~`eLt9R78juB4LE*YoU5};OTV8GjRk%?n=7AS}tW?!7` zS8pO8ucp-3F=H?ucZrHUWSYBQm|IHgV{Ha`_#;+%zGBevXSq^AB?SBOHEPD{w2IV| z^?dbJMR<4#wRQt1{?DhgKu{T{xsL@rAIQu2Y!+;Z;4Zddw9&R0*1Puw>EQW=E9V!QbgXim+%=uA=)gA?$v1vHItuH-F;zeGD8JTI*;Sr* zjl^JVv=<*;6AC{C%r0VMH(nyW)}yVX4?(H0m8V@n3<^;8TXa{PMd}W*L z9iU=Fk$}r@?jOU${MJCH?W9A)4r6uHesY&+17ier$~mp|<>984GKs26a!gIerJ8Bc zv#50BgyWWlFF|>`QUgqt`1#Y4K*bE0tp@paHCyA#^`z6?P%TdiNttY8BAZ$oa=hW- zxE%(kBZqSUO?Cf6B5b&e@mgf&lDBS94RP6qYK1RZ$&S#oZUbNhuvF>eWIIn#xsX~N zmhpHcvsM8THJFAfH}rX8tnZsjEeQglTvbzu*0Ka}QqLKU{Ln5z36^B#3?+QV>T~H& zPw!FAk;?r;JYEz=jg$N;0*Iy7bMdm&x1W!dyB)~{nq4Hf&d-P;yVDn#rreq{)}QX4 z&A9qp?v2KYvE3uS#O&|yO9+VE&d$qGP;51uq>m#kUVjx2_xLH9tJe2vpmE=b zFg64wsgRZ(pz3UC`YnF*Yd4yrNqW5neKz6#eXGrnEK(xHGxc^4M#LahssD$)w+gN! z%eHpK%*@Qp%*@Qp%*dn?CM0=Z)xip?|0`@Sye9=*T~$nCvm|Dxp5`y4$fM zL=sbA4x1pMr{C(6&WT%6-OBxFIyGSUu4p>GR94Gqvo_O!w~N_00RJf`rQx)X&6Bid zSS+fFTj|DAe$gwH^}fp+4f|vb1sqNKaYJhXbu61#qL03RGINy6=q~PIU8aE%tcBGGilVBP9fV zD#j;))pQWMi-=Pp&OxqZXD^qRK8&;v@Au2gtZY0cBekBX@HIhqAZbtZwWBy0NbJNm zB6Fc-(8{O<6#b0pNQTfmBA%712GU7bZIS&?(T-k_C;FeAQ80PGrILzBO|#$^4Gfq# z63Z0$oM@1oyh>iKF8@5}HVy1D^=H``%xmhl7Q3l{!FqwWur8P6kwO>!yoLm0ZqF%( z!>WHj9*Y^gT&ZE${8X{5)7WfbJF$>a&8}f-K$CP0Ye!fT`iuPN4 zPqL|6Yxhhw6$FY3+bUPktNvn>v%pvFt#jRUU_CLLQTlGcSBYJyBo13E}`I*X-*V~!On@Smi zPw7WQOSaG&pe+z~@Jl##>_)ad3qh~@z%8jp&0v+}T7SD4zmM9EJ<4gU*InTIGsKSf zj)d~gCu1h)xdMM zrh1=yPkeo3ju6&KzaOTJ?)hlV*WmRhhLQ`DFyZ_e$3Ca?Fk~vkJVK_Z zfY)9>J}jMHU_4``JEphRhkKP9lVEML(WxIUGF*N|ibDak$V0Igk0W=j4sqpb9L@*n zGNz0&iSb`^!K=}EzI}@cTeFzBq-nCNGGErd5*ic*MlBke2*LPPZoH$y53GWp_H|D(ysh7?@Poh2TBI3G0)q9jed+jtPgiu^}*ePiZMdLnVFyS zT^DwoW4E{0i&pxLqAmXMVTpYbM*PC)S9paRR5uB_^^4gD9_QSzO7$zZj?u5sB zb`I{AIZ}w05OxZS-WBzTMP_kiRYs!qW7IoY-7C#!NPLUO0h3|M++#CW zeO|b(BeT`7Ji{m49s74o>##)>tP%-7#MV|EzM_>)R!7!8m!S!-(3H%f)uqCR~k zAX1bf8buM0g^(9$qujA*Do^OEz%jOC`h$*7K;monTH7! zVKQYu><2pE_-wQY!o!b*C{VqOCV~`f)LG$!U=8DAeUK-pMD>L2&IkNI6utp{BMaq& ztGj^Iq^u8XZKXnV-Mr6(5`vdD(0}U*k9?RUjciiAq-GF~w7~vZclEgNDDN&U^+y_DM+HL5d z&k{*18IcZh7X}^9&jwfiG0hGJV(SC#Qy;?Dj->7%Z|4@$ZhdH@z4T5>Q&az%gglI!UR7DCvB!nI4tOzLm%1>UYp$h z4Ns>%{tA!l^KMCfQ(aX2VEWJ$w1 zxUKwLV~1u+5JZVMW#q67w@}Q~O(I&<&1;P-9UFy2->t~vJu&RUp}PAPZuTPjd>2}Y z?Ybz~`?sd;RsybJDxI8N2)^MjqAi@``;-aGx%o(bW}*5@@j3 zsqnT#oBH~}vP4f>gzZC2N@v(^zwV~i_+>>TOSKq&jmw^;krvyeF7DTN*$L|VLFY^+ zB37YvSJGTee3EU~)70RwcY7w^qHfbQ;Zkm1s2cix6{LSY4KxuNj672*oQZPNA8xYA zcXMUu%lEerLj=7@d^|f%)`&`8nk2f%hPTgHPGHvT*~Jj*;NZ&ycfEl^M6eI0d>E&G zul=vQtJ*xGIiFb&s-3ZS;;y1mSrx|{jDdHNq1&&(L9SW~-6qLSCMeprC?|?y9jBGY zRkIBPE>^vYGkcYT`0flpk|L?S=$fS z5Z8U(AN)8z%h{%k9qpU71A5^lhYM;44GYBq%m-5kc66&OoREkjKe8|A_4RE|7Tl^C zbS#G|SS;Jsnj2HATyho=5d3z4gZM=7e^hNFR@-?L$IlXf^8zTLX8CDP!XAE8LaBYI zGFw&m|JH&k(sG6z`Ih>PXYx~2(4tbo=kt}?9;F{Hl=O2Q^>1FGNwI{0j@C&-!0$?^ zn&en}-4byjD+8n{O1LnR9Cp+~P2YDU|IPBPn1dQ(sB1jEq30?<;w zd1`Mn%XtCE1BGLRlORBme!O5s+RGc3R`}AB#Y)(oj<+-FqsdXRkv_7XoWe(uM_`eK znf~w*SLytzgip*r5>yE3st7JUPs(XTcndiwpUKo!KLVMtY;4hVPsIc_Q9SmJCeY z5D`2P1c)#&@m&!UDIY^s3wmoQZikQDOeUGE#kL7@b%PbM!9-y|Fsw#1-pxE|H za&hK4yvjp5gW}Oef2ZMkr>vYnXypl=LDBRXje?-X*UyeO7U+hJjt9!g7Qd8G2EQwz z5+{CFLj72@a)>b3FS~vfVx`up#_{$p|Band*-=eRJ3XRYXZk00#$^J)&ODBVv31A& zVrSaC{$OW@ywHEKGXf>Q*cmS`xyF?BZIq_S?4)h|uvXy_BB0*$c1D_a( zf2HYcKFa%MFqE170*Pb=RHD^wr~*{VZ;?-AbG75homATacrI$JiRsvbUUxcKceaI~ z=lN?Y&)?aZDfx2q!LThOuhAH-#=)$JI3|%{{w4Vu&Aqz;vqoK8X>HIDrf$0BjkS@A^$&Z zgnR}g7Gi^Af{Ob9$VIw+5_dvVAry6idc@Eh&94$K&RJW+&5MgA^sG{S!T^fhoUlAw zVWM5gK74jq(-b`JYTk80eGBs3fkf^{ID4<%h||q4ezmI@auivcpA^A!C*G(e8#`=& z*+H6sKb?%4X25H;&>iYb)fw&>JvDRvv10q(INd>GcoevDAHVWMcw|oE0P_35^?>1@ zmo}z%$7;`jZN>Uu-3e#?zf0m}cK8TTQu-V1s)k|6#UR2~T(wnnHFMrkGPVeOhzeax;2p2u>5& z@ArLdI^;$5An`$=DIx_LE$7W*86^unb+MKxhu@LltwUplg4OkIJ8`=udne|iSo4Y@ z-~@2;<|UUG!Kxg zTxvnCS$cNl>dLpAi>C()e#w$aG_OChj70T+L_9isp5Y&QNZ zpEFXq1znEEo9HfrwRZmv)Gr>|`Tv&BX|!b-D3mK|L0zS&IO`^TKq!wS)zv$ z`qO_$^)p)!arEFDQtdTw0oX;T9RRy{rT++@1&;~9F2q=WVHf$|_YHK)5KUh%p(ybK zdo%ul>bI%zwAg>f0M7Ro<_nWn_;F8sh$yR!KALI4 zY?^-@V%i250wK~NisT}SM(dsuivK`D$-NEY3PANut`apDo*R^J7v#=Ms`e6seA1l_ zjCL>`gxXFruaIu35rBk;lUR&GNP5X3gC&7JeT$xOCzt5VV+dmea2E+#S~%IMJ4t6% zfB-}*rr-2)gO$JZa~V+;djS1hodT`o*ni zVg0uI%|_SRx4r%(jI2ubO&;xR z)v;DjGvV%ZScH#W1Db+-BfhS|-3$lOrMQ#={uBIsD2RC@yya^qXsYO|fV>bcx1KuniY{Exkc;%|G6y*xrG-&Nnb^Ybs zYW>4)nh2#1gZ;T*oPLe+FHT_yAFlRp z8=qLjBH7>*2DqYfW@D&lk8yuV(P%5P(aqnt!e3v-^wtnu>BQw8i>_)Uxi2BI+?by$ z%%1`i5v%Lz+dzBQ6ZTHB)?)P=i6%wDGOgE9)@B%69Z^GJz;R%RH=dD!O$@nfYZ6f@ zp(DXb=t%@bc#}1DGb&rfvyU?sm#fSSWwF%9-Yk-QkP+P=v)iu5%1xI z+2mcq7jaZaF`pPyEoYf+-aR&Ohss^!BoC!`Lvy*ZvMG<2>JfI>Vh3LpCr@_oe?gtK zw^~n|)lEx6`{+%U4HY{t6tq!(0kun%iE>-suHo?_J5V(;qu9;5q9PEPXynUeJjiD0 z2zG#Wa?xcJ#(JEyH#gA~7BH;AjC5g0Wsup%=U@hYQw2NJpqIfeLg|U%6+=Z94z}F0cp*Ja?g@f z!`Y!%OQGXXsjih}WvO;^GF@?oYYo2~)nz(ULByUf2JN`WSW$A_iEgh(*P^Lszyr-) zajtGF@xh>JbjNFl#W^*u5n@$OVms=#z~yGyIm$2Xz>{5g5=S@U^Y8B{)0#n+{kEvx4xBM-nWqH9+4_) zb&ZGoS-wkug@;U}Yg)5whJ*bFWfKbDfoLZqKU;3-Wc9lwZcq_gy$VA^_FoO>^jh;ox0=91^#Pe%Rgnxj4W5>zzciuV!sp+I=;y zm@K=dDHG;^9dlR|+*=I+BfKZ1C*(idm9XYlH4_4AF3uQ#@ifTxJF)){AgA>c}uDQ0oiauwZ68Sw9TwZ!n;8L*Causc<{IK zSidanN#75phyt6{8p}oKW)n2NJj9_5oWL2Sz2L*+3%4ABD7$%~xHrj!!qJc0>81v# zkxJ6gV^J?!RrKkl!Xl%A-y1*oW6AJcuySVg@Hq2aG+N}Q2v9RB8KmBavx7Uxw3m=m zBM;7BJfBVo;BG??V)?$WB02^}r`H+{0Fyk~-UH24Q*8G(#@FFA`{|Vn<5AeJ6Pbq8 zGPhyXn0oobJBoTmCp@_=%pP*ZFIa;*4^3vjj`47HRXI&uo9Wqz4+|dj!p#!z@G$cQ z*{aHjb!ZES@R($S=|8VfZ==kI8k7WZ*`00?aXJw|VuNq?H!dJ*C^GMkl2Bwf6SyXW zU8zQ=1Lvz;#-XM|vCH?p+9iF-5u?p2_3${$n~^dDm(k#>#THVaQ8^Tur=wIus}fB) zHOEgv)wfdYeuePfDwDi5M~A4kObhy!i^XJ&fO!|;=R4hIZ7{s@>1crT$v9MqCBlTy zyQ9s_t|}pu(X=HK6MTV!ng;>3Y1Z$Ur@A$Ve4Qzh_#!tF%_l7t%LbYys`dd#2aQF6p(kQ|8J%FS%qlBb}T2}+jVaDfJ zqg-jT*7P%77F=59c6C6RHPfDX^?lr~XeX3mGcjrxW9U_MsNU#{o8+?aMX?VrZl(N} zvL@d5G$j}$J!xtvb@ECxWv7j_=*AG>bp+mE>gp5w))Q{yiu-pYq2QZnCSq^HxkTt6 zJXJsLnyyoj3dnmF&aA+bXLCpm3T>VbV~OVH`!RczRLE5%3p|YsHkI`AY8}E|2YoKv z@^Ml+RmG2R6F;J$Jy-ZsqpPyGqjd^~z%KSpjw3NwT9R(-`^>#mRo`Y_2=>#z$1{O~ z1r6RRFL@f6emx(s+CHCM|EC^a<7cH#ctGPd1Yn-_m+}H^e`)~zr$6-+UdL4?w2${M zFv6;^VyRLxXw;&rQ6Rt>ZXh^uOA#dvZfUXNQ5w~()SYo|1CmP`vEM_g_D=z;of(_QlGQ*KEXb^|Amd&K=)a&QG z8r0;#qNI+oCh-VbX~VNraZK5^xXV$-v))Jb30wweyQjob-zG-Ru&7+K#BzIdX&?1< zI`fjN$*406iB0ts4(j&jju*oFq1oN_OzuZs^uf5p9sS4dh}_J}nLO)3q%Hx=7s10Z z7Yn!8-KV^fOn4mg_f)ninAqW4=|e}pqy0&T{Oi&{E*##^k2@{RiQpJSUlLXJ)QmNx zLb)!OcHx-wZ1rJ9WRt5`+jqHD^%cOChIvz2iM_*QM!=frmEHK~HX>XHdFnmb>4UCV z`a0F2Y6Is{sD&du&+63VM!==r5XRBPK6a) ztGS)EgE_sEmA9)k;LT5OY~g0^Lhs~mVrOj%DAW80dX`gtZB;z!M1Vzf@MU}UF&AGA z$10?b2>)rGX;4n{Ml;7a={Nm37#Ml{m4(>bK*KnYND!Fd_&iviKDl(7-C-xixT$s0 zuj4O^h6@V|f&|pX3xQhCX)p1)kVCs8NO4lJRdFS|&~fusRe_!=g7^p&hwkf=e0XYE zaIs~MIv-j&wz+%d&z@O5KtdDE}!a`)~ailcHFs#G+W^X24V z@Xn-i#L7p$FedkYgO1~?@wg^0qoduI=f#+8%xB%&dyc@B<&|gicftLOgXYFBYft-k z2L~H(*POM64riTDIbM%U5dUI=pJUW~*~y|=uZ*zT5sGuLvzdp~nJ-nSUozb+1LEMzWn zR#?{W4qktK?`&>tV_fIleCb@|IikhicxvbMRlwEQOS^Lln|X&w2f;y?cM z9$*tyP2F917{*25h#-g{G1OOx8r5Zz=UK$#Z&c-x)v*;k5GbjZA_~ASV{a9x)ao>2jwn?q};daR3$}o}vK)Liv2KOh7%nV$op~ zHpLV$OlQ{L*OU}wBg#BaQeTnOap2P^C(TNH#;K>NKDQy+!B8*2E3S*-jz`qyo+B>* zk%R$XCt9V}7-g`Q+sYjd3daqps$89T3QLi=6jDtvaNC$J<z78D=1ysp%b z3HD*+qcSv5TKu7V+p!coA|UmUqbZfQ1`W=I&Zfs{2jA%xyXvA|w05bJ>U6S=6! zBS}_K){C-NDHLVITusdwmLaQBg9}NHIF#fD#kK?LP@K=UY<`q>com>7*IN#caut5M$@tLDbA4K%8C7v zm@$>?|4?)Ev{goJ@#?Z!va4<71+eIl22H5%Bvg#EaCvDj63%>mvrE5Z7Y%1ul=#4x zomFOiMD@YAc(dk+QC550d>bz<<0U|x^27GFk<=o!8`FkF1Ty8Kb(8Rri}Rx7P*`K9 zn-|}a(^B?vcM>{Vh2~sGK(@|esVR=*EH7PdPHu(g=irHqoQ!FWY~e=t(Rvqs4*?G# zWlwI(#p10V=Szc#Y(1>)LjG?o!RwL{e5WbI5S=F6i3g?CwS z{Nmm2s*92+sZSfg1y0zr2+VQxCCE1W_E_{`5f-TrCb1I8RH6)K z3T>i{LLZJ8xEJ*EOjJ{?0C7015^EGZ7tAIGKG9r^n^2@|5SD%w;8`f4Os8)38e74Q z$oHP}PYT7y%qGC3E)yu?OW(0I>pvW;VUo98$c*ZYg_QHG!b{e_vivL-iPy{RAwO=w3z!i!;rvYp542i&g*g zWbvz>fDhs`-qR<4gbQItg6s>|hk`7Y1!gi+DU$*!i-dgpNbuFBvY0p$vx?NN$sB5) zVpok+a`=Wi8^Bo!XRcL{BO1u0?m1*(A4S@t+GM_IhgcD|nHl;JGBd_4OVYKWg|Yc@ z>h6bV$trMsd-qozbz>_q&OaQQ#Rg(b@K}iV=F{ahD6T6SdCWi@PG+mr%YxM?ZE>IZd)%eLlNjIvAR`L^APMj|ga^EXihN zVK7xrhFlpKiI5l1;g z63M@$Z|4Ji8kFK$@WtaY#NIv7->^@CZuT86ce9X znpmka!?!Lojo{P)j8}SEfN*JGaDq6EJ=sJ^_A1qH(B!+)Gzz**Gv;*?-zd7OO4T@l zdg8`N{e;haufl3F-ik@K3Z9$%=j`ECEWHY4IG?(Xr5h77O|mVR3Xfp0%e?+2;}pRf zG1W38DUJ6S4`z&39F0hfvM$d2vf+WUvTR~7+8t3DqUAWQ$@r)TFnoJ>`%<`J(1CB* zDSP~WxFw?>M&7^lGN7BqKz6x zQn%b2f@ks|D=bueskDR_-O^YT#qG~-xy?#9^*r8I2TW6w!mJ|*}aRQ&>Je{Lq) zLmkw#Du{1r`XWDns8?cgPndW(BpKL-d)V$|sF2h2K~{o&88wYvo|=e}3VlEXjj;l| z?&zQuQ$>g@(yD8N0&~mj6#DA+59r{=rw!4nyf7_v5u%_3D`ib-I`${YaDM?ttUr8{4=lG3Fi_#MxvEeOov)3 z+;u(i{x|XA;L{-piSgJ98~cgX*H$B13+hbY#vSh4)h&nlxOukvg!L66PV*fMTA!3W zNwJwQ$9{+keIMaQrE%UF|GuHo7wbQ#rBYNiopIb>St*4ViF~1JQ)V-te?F1`OvqH> z%SE&G?MaWGCJzNto4!dBNQs)C;%F^V{&eoSnjyMu}>A4IDZ(;o$+wKltf17}-ADUb@6sM8|9q+BsI zlC5k}-cSYhr7=bKFG810KIKcb*^&WtRu7c;PeYGzGNyqArE>XZ6Z-fn2$iq*E1DsnP5B z#zLruPwuIZW9zix#{#~#@`Aq~#wT1o9|U6@VTIG)&J;*yE{Vz+fgC%m-QGG)NmSuV zP(8U}!>HN?sMGSJ#wy^Xi`zK|oO`t%f}_b}4Z9TD>8(kb zGaF+*Z!>{aha{pgxKs>`1(FNRMUWTHrl@#ms8rP*D=n%56M~{U5jPqD#-3~ujv-8k zsi7hf5e=H4=Zg%3H&OP0XT_b78OaHFfozB1Ue{C&T1%_<{_3w8pgagL13dfD=4mcB zuuhs|wwWcRfJN1-sZeW}+mc6b*ic9;Hke4~MWN~{>m-r|;?ivW{WRYq5F1TA(B~>p z-;+W{oK;(~1RxTiNXJ`lZqj2afe~+$irA5PBdw6&FMoez5Y&41Dl5+VxTT%P(MXhj zN(=P__p8dIOS}z(RF7EAoJ|y9sRxa+dlfa<;O_II*Oh|Ysr7uEQ3YPCNk26mK0KpO z9oYRE^G;SC^g0yc8~sHI@FsTLNCJLGiv9uk@kxxIkN%L%>zG?l`zvn)egJR3C z=ozG7NKcpbHONtejiD){^9t7qXb+_>&Z}M|SEq1qkgr*)S`m;`gvy$~7cc?$@2@}t z85r&-M-!hOjx;V1Lt%ZQlP*25!XHv$GhwEvgH51f+CyuY2Wu-~^flE`g!0qNO+KLx z=3mI3uNGKXmx7k#XyJ=$0G{7CNGwdSQV5Qzr%BA?(nuv7q-K!j%u-c24F4`blVz*# zo->|Q2zEyZd*16{;8XWZqC(VKP3vN?!`T~pS72G)>m&V6t6h*7nPyn7ML(hohKLoS zp3I`9lJ29jB~nWX>9LuG@sI|-yzuDd7K&N(GCvWsf0_`at}e|SEsKNZZ3^qm!u+Tg zT<}7>)@>d$X<=`x92uQ3Dw6Zz%k=<$c$^nkS0~X_6BBD*6fY=ck^}lze1j_NtTi)* zg!CA(uX-Yo+^Y=e^xEy^L8(nORt$GZ&J2CMJCBT#ttUwzdc9g)a9hXQNSs^H756F~ zEn)O`RfTldx|Bc(E$s|=~Q zG3@hU+wId+qD^dyI!u~mkYS{T%wHq1L~aH>&){`;;%3LqeYnfXauVxXF6s;h%M2M` zx|`-0xN_hxW91h+bR7-5ye}OcIXznL0<`hY?<+Y1UKQ4_&e#}*bo}aiHO)Ye!B+Q8GD+xps3xnKygwd`xpv1V_^+R7^E^7J5#}6cHz; zXwd1UlBJZB8Z)$AZ>u9K#gM{<>GVyc&Vr;#5=xE7J-@tkF;9&WnvZ*wGrKwvUNPu^ zNLGC6iEP^=xfG*EQ4yI((Me42kXSdsp<{-4w0nF{GqFjqGq$!$=A%PlRi5U`NI%L+ zSI(H}VK7!U2Og$6eCcc&cN;=cilaqd@syBbqa_D^j-f86Hh$zTZYwwbT)sb`f)t&VNnR@as!g^Cv=0EdDIZ^P4+r3o~{*Ty&2dPW9c_ zMGGR<-`!F7uKf?caS!DR<^qyiD<U1Jrwi7Kkhfv<==EV6&8k4?0`s(A?65XdN1X5 z$wdx_T-!Vxv+9+aQ$VCfas?n#GaZ>XN16RawH!9MFdl4teb>0uV z;HIvw7B@@q%reJ#tRvY%Kf6XrHwNwgrC^kpn-pPp#?$48^k51S7jB?AO=KkJ)a57A zG#$gVkm#RThmoe_{Aq{DrX(&wWI}Y3aL__rBm8lof}q|KEF@$YA->-S_Q-20?!;9! zby?do3)F_dqD>gYVk6Qz1W;=%M@7d?mmtB2E0U+kMOCPP{bYbX%>Z;d=n^EBH9^00 zI>@;IozB1VM?L(1@kiZbUT!(3>DtqsKll2|HfvwZ_)av~I}mStb2yxQ>oHb&x&%y; zaKq#@Q40f;&geP}#@Q8=jIs@#W)^QQ#~ef*HXep01x6m4PGc?MrKe?2>!8T#@rp#r z>5MZKvd^Mdy3O(>BkUFAu+vU)!I~%l>cb!`Q9*dQ78mSh=b@8|(dG%hlTdZ)~mxezUpS`@`mH z(~tB$;cL5Mf%uJYyjbOa#~HUfoW)l@u@fft^X+5edWD}UeXpX4HS%-f>7ROi-lo0Q z{RN}!$KTH+1<%RCF9vsJxUvQ~XgELu;>#L^OG;)WI%--az}}NoK0j;Bx7Wb5;W=>L zVuRrgoxR^Dq+3={%&>CrP_=irX7@*UUr-A_3qZaZ?>I+@u|$Egw}eKT=h-N{J-q!m4p0f80w#eC`-By0yM^|So5vLr-k=*KxznV`l z)(Hv-hWRqDHAJvU#E~3xm-w5(T9Vd0f67GP&B_44j88R+PJDG9V8-Xn05Ib#4un0& zqwEs}nDMdBxdF`hzQwG;K;AkuVa7Qv)Pt6Elq5{dei`b9dt@yhpGz^<8%R$^N8qOs z+1`#cic=!+MUUpn4`KegCTKoxCVDH|Qa}Zj?T8eDG8?H8@)RNoTrF~JssY>&OoC1l z4lL)4E{@huUM8bwGUz{O1qMn;xD#nX+ibJY;>y{8Aqj%=lw3bLq>SDl3Z5X3m=ONJ z+ynJdq(2$8|FaPh0`pJeR(n{GZBmVdpkC3AyMbW^ww1laK$jGF7S z_}bRGXVX<8?_C9S5H2YIb*icjB{fbfh1J{~WE`&D527ccf+8G5s3K+Z12mQ>>^mZO zD?J}Vl{0p0DQbt`_*G`rFvW;p4nve%j^4yfKoG4?E9EX5Rsf(*SDrGxjw>-@+Reot zNO|yjb)esOD%#M=P;$-JrLS-uH{f`e+5LH7ae=duZ6a#KB!E?aBarfWKwV=TqqVgK zukmKl?EubPwftIN6t-8lu<1*u+9_kNyfW=uITgA8(}>mRFBfOG#MStujvb$jwH&@{ zcz}!Z4sq^=VxZ#Hk;$8bw*r3B2xHu=xsKMo+hsa0<-_RqHZx-?$FaXo(TL04w9mSK z?g0A1`$D0kXO5ZY^Ijxfwi!c(BjUrkST?++&A5J(ZF7s{PeFZ}4#yaliUdCaJ5M0d z9~cu16AV`+s}}*$06F2g@DDks`7b8`Iid1;izcJTQT6COa`U>f4C(HuOBGI!-8`bJq{nU>Ur{^9r$X(o;sYFPNa77QtfIoLS}0pBG#s>DN3 zmvgdDm8FK`HAQtQhDs8*dQH)>e;pmI{nfdj+E6=BLospIhfP2FbON_cxYK^A-)j!08vSaT8TrY@!{cDWn`zKO`=hzQa!ftq z;$6kMUmkbl!Ip%!hJ=KBTz{kkFT_AYOaJS{?yTd%i6|B+wSkno84T>pCYpFeFY^1n zTTu@=`&!#5$3+u|`F%gQs!SqJ4G*lE_4&c|-I>4dGtb%G?E0V@7%6rxl^7|c_)`%c z8IB)sPhV#IJYHmOg8f;$n@f?-m&6a}@wN?!W93F)-~|8N5`ODeZo>!2n$VztfKva` zcgCNB;!Lf~0SP3I*8hRO_L;iuFKJWvMJlRWtEA0+!W;UN-jd>5bnp^`X79Y;YM{>+5LD)?-#a$BS;cx$|PVu2DZsgmCcY8Uulngu1^1CB`7V9H6jPdvRD! zLU=%=(BS()ra42!)?Ng@n1Awt=CvML(44Mv+O9)xvUuj+t(-Wpck_0=+0T?@3X=w_ z%?(%r`vH~bAqR#2z2s=ER@nPEm&Hp@h5Qvy3xmDp#36;r%39@$L(xXZ9a_>{7cDCh ztgx>vTJw-}s-@a_AV8>AuF)_=-r51YHx57texdAdFl4!6jQ7{e6~>eK z{=CnXFWs5rYGzdQQB4&E%4P&xNg)I1jdXp~krdh7!pmXc(#k0Z(4))>OBWi!aVC+3 z7>LQIQrPfv=Z(9|%5<3Gkxn{Zal37vFuH49@Sm*4yA8iJ=y&UG&|i>loUwvSWG7Ag z7exWzMdudUQZHVKyed4d_{%85ct{K#>4#%c<#Au}v#{U$ zoO$)Qk%LJ9-m@gVoIl;t(|&dw(|-2l$W^JQ|HKqdTx3fp*2VSP9#*uUzJ;fOlSe84) z94g*P^A_!R(0qui21@WzBFSXeGxAB$mm0>kd+PdD6yKtLJbORF8*#cu2 z=e5GK_sk0RW*`&=rN#UQNF^@9^nhI+_rq9t90vl&w|`n7*3u5ULj#0OL4UQdiSgwd$p&@p`Ma$85AsMe9T`=TE)lW2czc+(Uo&&ET^x1WP zS29g8beFfrV2pBzfXJZ5puj{sSQ(=k>ATW$zG4UiOUl%MxBys^f}T#u;*^jF zZq^oi4frr8PBFXpWS{Vv&xbH4AW{k}QtOSwt3^bzctxGQm?i z2m$G}lqY|-`Ze7dToV3pfsS0_9UL)4mO#Kjbw`DopxL7Yu*Q(&KC%Z8!^?xEI=`%b z(SNu4&6L&fL3BkFC#C>;08>Tuaec^7otJ@bGmkec*3uaq^WZI|)yPhb%yocT2E?I_ zuF$+XttjPmi?jhdVz67==M%z(H z!9t_qs6wOw=L>ByZZfdzcu1}=w$elpF{~nFF`)+~q403Dn$r%Gqm+th11Wn(b?SRy zD&gpBknK7sJbR?}nn$)GDv+i7h^R5JTkteIF)&cDkF>@}QhV1S5O@xtb2A*p8-ygS z99`MPX^nOZaYMUi8RWYUe(8Emzr2215`4FY$Ot@?TXedCBOPa@G2ajlt$ZFCobH%& z49q?_XXwd89vz-4-Yaa3@8td%Gfphh4rJ-Zlamx24sa0|`;=5ks~Ky2CCOqJB!l@Y z4~>=D#+Jt0!{T96WvPm7hj|R$_hCXqu5<9-cSaH4{GF!*t6wwQz8g6(E_bcy!}L}y z7rxx!MQZ5@*1h05LW%UjTGEL93RiC+_u5*?@?3pvB3(pj-_+#E(~xq;w-+1!6@c0A zsZXXJhwM}1Cv@&++g`@=*O}uSt0ji<5bD?5y)kYYPATUcP21&8-6PYiek+v{C7#--N@>HaU-|Czd<=;{c#h$l1%)HEReQh&(J56=4V{*CGM&ZT}Q;>II2ZDR5D zi7WfX&T+*hDLg3m2mB{0CmjURmq5pWe9jK#nVH`L(a291c+!}348H+7seT1xmSI+C zZu}P#9b@*^sqW*p`rKxPHSV_OvO&n_j~%5A`e*`ZI{(V-BDsA(RJ~w_<srUy*x1b+Y}45H*B^#Y!h$*D7-Q7g^#QJYhV-coOeZccXKjA$*>r-#pIY5`tedjN zOZuQ^kA5DT{Y)pq*(KTQMWDPJ2sw15Okwo(VzpigZhKIpP|f)IydSC;-nEG&4~l|9 zF~ts^C`Ay7r5Q7}(?AOWMMloD6wPL(RKfT$5uRXYU;WLKeOr9rn()-6duMD++5go> zKv#ZsY*~$>R*gqLJI6o@b8I=w;Brh=hwjbv$%7?VermE~sedB4Wp>Z5LC+2_ zm9Rd&rcrlkXN{)@=482~1@}IjNeFyanAETo9H(@AoN_%7?9#2F)lTvQ$;7v-nADuz z--~`P;CIQEUv1anu%HhNWJYQGGQ7#CqmLjaZr|YbDm-@RQ@TImv%itUsBY?!5aqn! z$_BJKG5YSL&(M7`_A0Wt&x=s|Q24D#~Nb=(X_!=>G7SbB>0 z#fhQ8WH*^rQ7yYvZ1TLR2-PxjSybp0Qv`CNbh8*W!Vt?*fwuVQ@Jsdrgc6_26g%$j zA2+Y9hJojd6tm=CGZa%V48SDJvZsKit!bz1i7o@2sC{>x4c_2lP2rANnzGU2&@TYxuHlker( z)hV&4T3!@G6NPgb(dj=-c9s6#S!SZY9hK|`g^Z~5*=!ZG1?dfO}0U}Y=It!m$F;eq7uw7_0J zV4-VDJ8ouc^t@$*0&;?030XINJV+Igm*GyS4q}6hBC#|pCL*HbyERkvSk^9SSkMla zXH*b+AY&*0T~~|>o(UKKgKMz`3pqJgQJdrEaQpl^HBK(+0(oV4SYo&=S=UMRTG%IQ ziUJih@m9-L&gir?KHxAnrpFM&@H=^z-$&Ku9Pg%D{pmsCqVq#huqOO1ZiIhk8PH z`0wGv;xTrw8t*IEeNzeay56#-`@?0GL*5m}xAV)Bx67Ir)nQY|ex2(T>MHlF^!#kc zyVUo(x2-zQEkqS6Ew~GnJi4Yn^YymE9|wEu`U=N)vT^$!4RabO$!km+ZmEH%8@~^`Jfz1HNBBJF9X;(9(Cj_h zEf=qk%Q`*=e`_awM{|h0zt?wc$QZQdrn;N{G*}wWF%E+1@61Ne8N_+Rl)RzlxpW$sj0&DG}@nG~`49-s?jt5RIKH z&)q!J%ewB-_YirBlD3|<3wgT6`|+B3*hbE0264*_KmIIZP!C}zc6}?ou-Wf}GzqqUlLEpZjSHPNn?-YR_mVCJE}d=xGH!*1J-Z9y<@1Evf zqxATs?COon%(^eEw6{Zk|GnPYDk${Us09tzN^xuh(-*sVzvx*0R+&-g{mw`5EV{{! z278=j)ZL)ZGv^kGn9Zm!7;zu=S3o|hR-XzXrZ5zFx5_0NULJ>|H+%V|z6!C3#jl?L z;asRp2n=laDL<^5frlUc`sG$h#18(0v-ddXrI+Xc^* z_FqSPV4V7b-Sq-!Hw=DYOU=Q7`mmld6yc;$1V5?$_-&3^eSh3&UE%!-bcp*9X&;H% zZROp{c^Etow(Idt;84vVr_5MB3qO=}v))vDj7!`?Cx=#^1JXc}QHKTo#>Vt3DkR+= zmwa3$gb6W*HR$(<(NI31M?KCgGcQ3+6u+8Mc3JdHEmSe1L*vp;xa>Pd>MZ}K#;FbV0fKs0BdmG>KRRfZ=44TR7@;HzY+Edk>{Z;6wyd2azdt_)VVv z@Z#`plqCFG6^+Ux(A~+3?1m@!9$9wp@3>qV0rLaSeXtkLuKUPPY-tRL4r4r7@;eht zge;ZI=3B61DwFn9k2xVl}n6S#Ck)`uRa{cn-vbRNY+Wl)kiLmAtF#9mvkMvA<^$B!68AM_& z)kuy!xcx^RLc5`yR7-Ut&PpJ(hU*BciSXvqAkAKMizes3HMS0mkAVTc z5%rD}hE0Yh=|Bd1YN3Rg;BX*{V5}(ctI%aA`Qs-{R8fp@T`ltaG8@N^vlR(Q4aF4F^*4aYNfvk)xs-TVk-d0AEd` zJaD{qhULQO>C2Jy%+2#R6LDCr*RIv9UnJm>P_`^YefVFy8k?XP6+lD-mWDF$g?J<- zwswj7h><}adAFBRzd;ecD!_!u!q;XaNU)Orn%zybh=ys{Y34BLPKE#`eB2y{AfB7# zM?4EKAOWk`&kf=bK(wU@5CAHi&aq5U)3`d)?-Umh_6(@)e?p%-!-l9Up9SSD)Kk>> zBv=uj4++3)QB(k3xXv(+%ut3Q zP$3L4hKHNJfl{RD{(Qsc2rk(~6w-sI2xr)bc+S!=2+`3w7FhCLh>;W|jIe1=O)gZx zTO$H2WLcqqtUuP7wn&vdnN+!Oqp@LbEc?fLEiovczS5(MQdjxAtD2HPE<%m#5kD}2 zzm`8dC(k_h6@tYxyHUER*z`6vmj|0c)= z1f1uW^k~Z0D>@}8kbF;WQG#oZ|JWtQh|8C&IY3Wl0DZctcufH@$bDion>>)nuXo0z032nN zbf!3T=p1qVygq)yL9?1U6YElKlsoXQg%q2zDW!zgi(w5_iN$cjEz4nB-!1ms8QUNsi7QUpZlTdC1 z!94wY7^niUWs#tOezkZ`6sL;hJ7Z2C5SGteVail6X^qgn`zeGGQL3%4PAT^J zn*AVupZ$}s?iOK%-Z-D?3c!Pj;8B>i`G0+LzKB7*QDHL90E>qIHTOk$B)PVjz0|Kd z4sqR4K(x)sLT)Mk$mKn3j@BM$%Y!PSA`YJn`UC_IznD_9{j&Fxvg-T3!Z|1gr8O#w zZ|lRiBM0ScW_W^JvVG<-2*q485MFIM10a1RXjYL_N|!`dFqQ!USG-QhW_hb4H;xGC zD!i9yjzQ-5a)Up+s=knsAb453rJ+NR@=lQ)mdWJ=wznjzv{*a|WzKq2C237yP19KG+||(9Gv#)!OIx zXn%l74C4%dWWaBW!K|6xpLJb?&4P3;r>s_w9hf*6?OWqnof$`wm=yp*w?Q-*aMLSk zj+h2!d}=>RXEu$byD?5RSp(A*f9+FFI@9oEr+y{ItQ>& z;Z&TURUSmR?4_pyTU=cIy`&a`S!pV;bw!yq09PCL1*KsQe|N=5q`inNBc=M`u}WvK zXtTnjqhY>+Bja+O+e!QLW_Qy2`4sW<;r8+Ke%JfcvbFV*-SX`ru4R1?An#i@u%~D- zuzy05zks{EmD3o*GGGzniE7odUA0+BiHuS*-Wu-*JK2-;$dEdnw774MRl5op8#qY# z>348k{HktH0~vwLsfn-=ss^YI%L9N3#?Q^L2G#)wup+zS4YUp3L=Cjj>5eA7%NQig z=+k9L|U!Lq+P{pZk(z_1`V>I6e;fxOo-wq6%FD@USgnOr7@nc}zSi2=&c0POm z>l!OsqyhPV3ExoGfAO&RKkgeUW=Jr75lZyw8NXj?B|iHL6AtP>0I03P08O=;Ny^qE z)c~~aOWiGDshsJ-gv2UZBn;Fo6-n_lQLl-^0d=2!5!Jiso76;7w;(7{;zYb!a%b{n zM3p|-SyFb9b~CHBY#Pf6iq_bgEE@@bs7zF_&dW)u+k@(Za=xJedH5AWwzGOH4!M;E z@K6l#=P#)ZjIP_gXsSH^9iUeI1yHO00;p;J6QFkTcYvDN{|-RS+=y*uLX9+)oXp9J z+Om@WmkEhLG|}gW4{AY4(u6E})s=9t0D(~-bXE)n!9Gn6F0!BN5NO(u6h(1hBDfhS zA_5Nzf}cNO*dKPQ?o{SQR25cZsP>D2NrP>52yoGT{6U@BoZ8_VV08R_VZYci@`uOu)4ayK>?ik>F}y{i$i!of{huvZ zbfh|Wy*|8>r4QKIt#IS^77b@W{?LsnJAbXIVw;<}dRji=|8q@sctNTy@6nz~EPZc_ z4JUFpwbrO(<})C3Gk^E^>5=36q=|LpVHzf)mwE(-_`%Vz$Dan0C&Zq0s&RA~#}q^H zwj>XSveipx0OYOiVfE?-i_Qk#n9-e+flB3Yv8S?b&rJ# z_*(sS*N)2p&->gBO>Pz#p-<5^Q39wRnDYPRr{Vv1{IrC>^V917jh~hzjdCmU#ZNP< z{d1nZs{IA|b!Zh*K~N*URGWp9D4_a}`DxaLjTZkU;NUF(;v4$EnK(49SJ^N#)B)wrmc@oF zyGLXJDF4!m88wwvrI0~%w5;H7*ysGV24_wEkDeN0M5p{Te#9DJ}pPS$s+iS*EhhTcOE^qR$r$_ySxNn9{4K9 z3{bsKS(Cofi(Pa>3Rkd{cG4`uJqwl|L;k~xFm%d9xa(i62>(uA{JRyQ(2;?i*8i9| zkWBgpqxBuU1z;4Fr4Gqqe-a?+Z%JZZ15JscM%$swBl!DVMZc2#tr~;NAVwVQEAUl~ z0i|49it3%wRL0R3w-hajFN9#}`l`mnRb8c*;kRi2V9DbE0n6^XH~O2Yp}fGCsbR#_ z<6owReE%;bj_UtR;)rRdJ`4KKCXSd(ML18WoLxPbeC1j%Bq>kK7vP)j|2}a9MBz97 z zpmy-1+3-y>T_?v+chJ^b{W{peh-DT=>pb+E3O3DUyq0y=BL=BPYQudW)|*Jy@hroV z*k@peYmaM;fBnJnU&pBSUv`)Mr)%^7->CL)Mzz=u-b(6U`-TL*_6>#n->ic%u>R#q zYh>qS^uGr9^#2*PS|dOWd~^++E(=09kjdi_aQ@R$*22+6xZ4*v*T@oab5Wlx#Y`;R zOz~$Y0{-xm^9p$I(WG>qYv1SecHYg5GXHdZ=8rXsll|uVT2ktIQb(Y$T{H2TSvG}{ z1g1iawlZ9;vDAH)SW7j&zATyP3DA#plx0GeJ*>k-yw>6e7tCLYuY`hOvbj??xI313 zvs07P`1$Q_Y1$$1Q`-8&o$0kbPM7>r6=E+uPI!;VX$=6g7&5Q2;CCBmMbu6MZ(Pj5 z78-o4C|?Lc8U>kv#z;(3PDX@oj)~t5@jDXH>R04ZtWe<&TO& zdV~9XF&JTfOKihZl*s<<))gk)`0WRd&=ZCwV8n`~C86(t1_f%(i2~e&M4z>l87$Nm zD%1h{drC&~k|IQlfe0=atg5E(B^l~=LPes}N9fHPb^rag+OLUA(D~65QmVu*c-|wK zQ0V8@QGGy>4=e`dZG7?&oV}7yum)(o?*M_HO$Ik)ORQGW4n+GD};2w8?X3$6Cyf7_Rnx{JUGkm;TjvavbJQy@c`ig?%;l zugtWp8<$TX)eEc>^h)eTatbL1++q_wSKKMczjpzMD=uz}Nx(67ji+d@(aXv~Ru@KF zjCJczkEQ|3gG$P7-rWyFETP9Ki=pAGK^gNGNh+# zQ^5zGmhC@^hGSFssZvWKsm}uo#DAWWOxAJc7PFbX!NR~!|o5vTbOoUbWaCVG+nWDr(mr9CA{&3^4f1(@3X7!^+e zLs3p;bF7#z+o6?lc^?+=6b>i5!Ylch0(ZPT_eXR6U2nE`TY{CTI=HYBbUK_t&rSz@S>-MDF1&mh7VW6*<1rZQB(BNPX3j5aCiVM#RvgVlrX<@7P1#AC;Vb~}ZK4PJ1x@EwUdV)hjud{sD0YN? zgvU&Xj@>ky4hJGi@YzJFLy{E&G(*Nbp$5Ekbl;)n#4at5CKr}-q$=QA1tlv($%x~a zN($^iAM=NmC~^b zM{LE11R8gzHDjd=^q3_neyW=aQ91eFNJ&dmltv0nbc z?i%0J0lf}m*cu{UCIT9(H9xG{$u5u+NpR5Ptkef7$RJUA*CeGr*MZ#3P!GPU{&Iq& zq1oVI&P5=k@Tat{;C>s4cO6-eob8<=opfE1NTSV7Sy6;>x7#+~3&K zOU(2l)YHgnUE65|#Hj)rg?VX2gK%>(wJpv=8WyQDp{++O&>sgu8!)mlaTH>|Zl^7YLbrP&W;yTw^t6njGI$1UDqSZ(Uyw2H2vs znVApSQn^V_OVWfcFB$%JWZkV%@ilkFOa1N8Bb1x6Tf>-MCwGponXyumr z*~Rn-xFz|Jdlxey3~e?@^5YW6(F$3-+9%VHK?Da9S&6>=W{^xG`mPG8-|1?6MjmLn z^17TuH|qpl>E`&`1ty5uXuNSK$SdEJpB_~yVv54=In{x#Q@Rkp01r-SqHjN`K=C>; z4YKD_n{%#0qx6w|&<d9-hwLmTKS6 zoOz!=UUmH)R0%0|ot!^(LK`n~DcD_C1Uyg)VZqE^a301Mfc9x7=QBBZ_-zih@1!&W~ zgTEh%(&S6ZAB^Ns6rV6bV`q!aCKb%&TK$?cIA#{U9-jNI`iUc**^VUT1sPy$~~nv5>RHuDgs8? z6s4%m$hNBLGWw>xpWCUEqo3ZlxW8FOnLiHbO>;{GQLMe50@XI+yEd;N$Nb+(O1z$E zTh$rTu<$!XSHG`&_y2Mhy_^^6%ze=l9o7?l)ZJZLpLVaGzAR!~lDDtQeXT4xyEtlJ zIdY!EJuABZB?`8ZOaM?qFbH*6t(_P{cg-ljWS3n!_Ris6LuiW6@x!|Cy9K*NI$)Iw zcke-(_LmB=;48;OggY@J#PL#}k)Xl%04FzWIYjg2Xvs%0@2t-+iIl?y?&{^-#S@l7 zDm=PLT>SKSpYXnYm_OCk$)Soek0%Sug8(1yHdrUJ5r*0TaV&*){<)bOI9NadX_ALl zHuppUNu)%Y?o05eai=oj{h|sa?M9J0z>{|C&YA98%7p8<5H8|r0Vh>goHEcN(|6U) zl)y-sD<#yf-f!a;v^n87-b^>_jFZCWXs_+aINoN>zr^Ks0Te;bEi0yV0* zRKNmtHz{E1t9lsKDxD9ft|w$ImTPtCX79exYnS2~&GgvmT+ypmgv%K)EbobIZnaeb z{KP<+t$oniX;n~sw2BRLc}&2eNMCf^Zggws3(*k;ScjZk%omU-UHfm(5+Oxxn=Cp# zE*u-??81Ij)@m4jI}j#tiO%52@#`qbTHx`h|KZr*5P4}vqyDFGOt<{JG5MSVk-C#| z_JS9!O-$ktgo9ywKdyCLBoj1qO8vu6`^(tB7KT;JLcQYr$o=?)f1~{sZJrcP_RN6R z6gsy^UA?&6L2rb;aRR#kyyU5r>T!C#qJtt8MF8Z=XnT`PJ$}<}GRM8ZS6Us)b67>V zV!MUS$p}eTBE%K(jq2GyYM6-3*8R7=9{VO^x{nz(@@fs7yD)HhkhHo`rZRm~5j2}J z@nSd{pLF-_a5)*fM-(|M=`?zX#vOptK@b>sb~V!G+FBvRQa3i#nZk5uykg^5hig0i zPoL7ujTXDq#6^jK-q15#nV9Z9mQ_(W}pa$e~{6Jfv!d$L6G{Ugq8oC<6O z8&Ao;X570l6ZPu&Aa~ohqwGv)RY9z=g{CvrKuFmxT=|`h+AfE=nrf@};S`7H#b} zVHydc5&K-=&7-mD3B~x+*LmSVPK>i|p8EoyoA(AfsN=oXhmxcP45@4f4+8YnB{u2x znnty^i3ka+ zCSLaC`W4&DNCX)?nUo|D?IW4Z#}~{cVL73SRFn%>;{XaI;m2!<3pjnD4}M1RFi*YS zzr8M=R;b7iR;j5Un01#Z*7#;SNbQd;&YfPe54k%hI2V=A87^WaffoC`*=1(?6-f0l z+frm@$f}G^N%wPenzJ4)lEN{}rZDY0t6^bqAt}A11Z&NpybJ02njJ-WRz+xO?yf-Y zE2qFPP|>vhj-chAqdmBmgn4omKo|dWMstG)>)VSy{9d8zvWV)4XmWwQ)U8I#xS1FS z8A9){exK%x10BiN;nEaRp;etZ2TjfT{QAouuEZV89b(?X3~4+dN$6GzvP+C2k_Vk* z?0$lJh+~)q=bL+rU_;yyIYn#h;A;YMnvQ9yi9mb3S2z~0fPuG-$or8 z(T^hW@E{|KTUyl+q;t=CzdD zR@O}<)_KK$F}DyiPpCG2Y0U-qA_YZ7{?K-YUS5fIyW`+-Y2)y-1>4Q_k2#8Tt|`6Y z&Q;9RJ@)oy;e+}1L-dg+?kBd-#H=qZhl%MCO8mBLH0kU?S? zb_^rYE^zdA_h&<-GCnWM1Jt&NNBGl|od;&p)_E9JBp&|b_%m%R&2T3hRT`ALj^S@? zX2w~mpnw#~GBM(c?V6=)B63{jZBaiM%}u@%JEEc`_X&PKeXxFBfnUgqn;oV*M}pU! z%T-8N#7r0daJ@a`tva_pu!T_|u-phlgEs;QjGMCGR(%f!jLYP4n6*vh1?CpU6odMW zK_2yzw^3wZQpwmF_#bSDhTNO_laXAAukbKj5<LBr8HX;mX(nOhwNrd&WQT^EJ? znB;_ob$h9*3URIpLncR84KLzAQ(9_}9-HM}bZjAfgo;q~4Ghx01j}Z`6h)J6>;>JZ z62k-k-e{E50a3-zBWB8-(QiRJXIYS3@yk3T=*PGtG+QyCnT97y8c(hptRaQ%tO`EyV>)W)6Fn>g4nLS^bIfD~0~s1biojdYVTXk*oOr zi()>Hc6LiPP~HNsL|oGEQ3j)q67ku)38Bx&HcI9eqMg)`)f)<6mHF+pLTTQ~(LF>) z&NH#)LBH$E)Ti_MQ_$}xrX;2BB(Riil~%)hTeVtH<)!`BmPopINQqr9Xc5Z?Me23^ zK_$g~7@Df2R}FJfk^?Ih%G&ad>8K1cz?G?sduM{oGi)0Uw_Q^U%5r{elteHP(Mjy7#yTvbRg8e7m1p+|KAKSMRXpL#=-n#H51VfiZyN`DTJs;LF zj>w2{`V%85o@MO$^;vH_v47}jYuzTgiV&Akuct!{XHiM>75o#|FR9~bMl=(0J6{^2 z*$9ku1HelUo(2h3uqlTXvbBK%&B2| z!OA3d^6^O=mva#*DabonRWX4&Co0t!R9s%JR7FOru0)xZL#x-S&wl?-KSo$~wlayB*j|U58U< zFjnD$j{fsfh@uurnemT+Tyng>USkk-QX_QGjy5W73r&~Xxrw)!2NCO*$H0M8B=MgY zS?(6xTb_S}{G(ZXlZ?Vow4rXPDElUndMcZV z(;Se|+Tiny4M%!JXL;Y73zQGQ0fCW5eEhuvs3blsAvLR2%8^~p=~x6v)Da^Yqp6F9 zW*{C;1_l=1fQV_}70DINB(s(Uc1!tSOnwlVJyDokBJPub`PMRBm}3DJT`h~^qfi9$ z7>k8V!1UByGDdp8Q^iF|VkKdN3|FQj=XlKB6GcR{<FzG&N%aWwrY=FKi>ct+Iah)F$y?O`d9qB;D z(Mr2HlBb@Z3E)%@sgK`V`X@zty0zNiy(#o`(Dk`u`8^}KHurkbt%qVU)2vQ`a7VJn zas0fc!S4Ok@*f}}spy^cDcsBpXK4OORl&(pNu;7niE_!9;N}y@Ek7 z#Y+H0YZ2998?Xu0SqZ`)=duAkPypGS;W(}thoeNjR_Ak!@4*4Z-Y#I|lA67$=T zQ{|-k>S}ah*rco=!TvedBU2mpPoyi$$cLDfJ(@|gI0N2MNv0vf7)OP%#;h$mO0$`J zjCc>k<;Rm#f*TaxAGstVu4E9v^kq{_))Lk7gyR6EYi8+5*I$8DD!HSd&2En;BSF(I z^uP>>^g1tHl%I!*pEur~ucN0JpD(_dDlM1-5QvW8_KiOB#h%dAQe7zcEr&n6--v>? zP^JhEfYYc@6oo2SZrTP3u;*5{5>KoMD0-l$&V%b2s{`e`1SwmVBaNbX*_@U5WV*U= z0p=%Mrc~a*mfM@9G*8$0p+R!E$g8>)TH?u39l2JrpLwoc>!_eFy;y8nJd0swy($GY z#I0wBixYGaTVTB@`-s>@ATc!$iU3I3K+LgQf$+RVilfDD-= zV+i9d;l0o0))l;FOOkh0usumXW)ra(i(UW}L{A4`YM>PvoWD?u3_@65Cy+QfbO~pB$00n_}8^AYgkt+aUHdpT{z6)naj_~8>h#tUykE(X_UNNFk-=-4W$fj ztgRIbb1|=Wr{b(u;*9!p*B9(MZeF^Tu2vb(+8BO5R5`VWBg-nlVxin@9bIOjchKRj zJiFr1pdWqJO4?nQ#E!Uqy zbnL%u{CRhqa*zD+YMZB+@S!sPyG-YTvVQ^=?gchgQ~2GoL^LhXyC)Muax+cKq%e5_{96wZr4Sz2c6V~B;C1J8 zEn5!R#>NUh8SJ+ z78nEoOo+h>94Eg8@yd3qoi+v4$FM$gCLQwiPz8;aB>Fb#FGtFZb29S=_TUA-rM_V& zXSalg^TWC2u(9)tj*rMW5>G2x4u{8!-iGo$9Dd!$4yw{r7XI}r{#tz)#uDw!`>8-U z3MV$H?Erbv=9JYq8++x<=ThUMo$wJ3!Rkt(zP#WVJt2ZI3uCs7Jirlqy7p&DbBAr) z$aq*PbY#l0cg<$UDHhiCk~JpV{Hs^%yQmcM2@xh;f_gESf-fBUYHal`Q1OV zqw!I1{Dr*eb%w0#wD6T~{3X2PhOJZnyy^J#Uuc!Z%d@##+K1|Ix%6U|uB_=7ax*~a z!F6H!Faj9>4FH92*Uo>uhs#isfo_*5nk~M<@=;!i1NVXjLc^h8k}*i>CFv6Mf@Pxq zYgD&zW_(ofq=%PHkIM~T@pM%m{RE?MHzPO;iYeKYR93P8v#&A8IN0F-5Vx%!Sv={& zwmumCoNY77?%2{tY*~8ZRyuq?&o$kpKFT&5jg*5a$W;KsIKqrdPAJT^=amM=vU|p) zU$DS)|1_*KdAvKHn&~)el0?d%s|<0_BM8DQT6P0wveZq{O+juF?+f?afxhmkZke=; zshf#^oT>krhM9y9QCpjCQbd0he>$P_*rS1xrMp;yBz6sxLej-A?;GRDi~vIgNkWIy z5?f)>jTWvklhe=r$$A9e0o|$rdt`_mN!bfV@KF^tm1y>5Iu_jla@FOSsgqr@wb*BoK_ zLv<@%)2km3in{uK@|h%EvA7G-_I*~ei9yI?=wl0{weX(DD34@btX4EVT1((lH z51eY=$YMSnXASZu;yj8pEt&6EBR_byV(&agqw~G@Z4w8rKIO>7`$Py~kiE>#m8- z6bj)@Wk8ZFFJf<0)U^1{pC|K4s~?{q^0b!Rf%@Hx^1xTXBs^t%Ke3uLSs2jC4IZ;kyl($fT-dUExY{UgtR-t4qL1E>eRSQvKzZqyik95)im= zETq4g$U=Z2ZXT6l4E93RhZ;1x8aCN}7BLIsfk8fZ1ruuCXkd%D&L3(~S;|~dYS4$%ukIVX`neUfRVJ3)fOSxJF=`l)w2A^pv?X{Ef&z7NONgY3Q z_=$cqajSqBp}%*`Q1cRjqRS{iAVZEF4wAl^9zc>To zb~hC~O5ssv(r)A(#x?r6s*TT{S8*Sl_75kZ z?6LpseLJMbF{-Grao9*}eSPMTOqss?64~#rZRIlVD}iy$bCN3Rd22#_&p_uG z0=D8|c`TBQ@s8l(-s;qk$amPn=l9F1-RoFKap7^|FzQM#U=^!PY1+$BsP( zCJBi*9bg0q*l;!TV%&YXj#zZf*_aasYz&%j<^$IL{IbCQghX^1YxTz#uWqBiU2WxR zSu9ca8bf74mcGj|8@xN8S@=3~t6a(th*6p@u^{hWcq+51sN|-P#p-cFRU~s!*09~a~FK>O0;+GbJ8na`K=A8rujpj#x242Gte7_fuHeM8I){66z6 zK{IP1wK*nb)b5hNl;?e&qHr*uE-kvI*4dc+RFY50OI23AzO=d_JARn7sVdye)l@e( z@N`~L)Un!m=Yw9Rdxr_H#*VjK8AJ@QNRRIz=&`t?BAvVbZD7YnY;f3&b3>6+%Lm3HocChM8OjkX_{XpdJ1kVUYS_1EA`W(EhH*P5oAH zgK}3BasmT#aG?WzlOAJ8QJ%hI2(me9CWJRX@Cw2y(kMNTSb4~>j~JMYS%QtJ@e1j) zR69gfQYhwE5FNukg&c#ia43Fr<9;yX2G$0h|1_qQ77woB*bRmbjER>Z>J}P#C+XwD ztNX}(h41U~=z)p~3u5eCmCcQrE?a%607N~ic4vC?aw78q==k8F2Mf%g&^oD4{IJIi z?n3RYGjvw*kquOJ?>o!q>b-^Aa$49#YF?g{U4);|*tT|G#onu_>d*5>_AW}Jh{A;q z#^-h(+?KP>Rnd(ri!7_#CgJCwPF~Ri+n?{fJ7y|(R6k!&>JtqcAp%Zr@YIQ)%nGcP z?Jn=>XFAvX&DUI63Ut8&g(mSMJKfEAmTwmW{ev4s7hnOdo_gQPLQ+7kAparnZ*Q-c z9~i#x3O57wFiVL45WnAhkf z{nc3W*LCPV4d5dfAygRJ3^~lBKt6ZizAR;(iBv0+kI~{|hN(T6=;CT5zhb~hehVZ2 zboUd!$zhAdWk$deS0nxv0Y>nf75V4u8SBlhi&pB>W^G0MPa7$I+Kru}S4pJcu6A7I zLdma_ongzaHf76cac)BMM1w!>@t3UHb>b63Jp zcwt&U#VxCd6r=lr3UXg{W)+KMQcJYQGKO&vPy-5Xn55n)CFj`4G1O8<^bN?uU*h6G z52{)~Bt5wBwBe450tUkP(qeaY8nz9j;zTBtjqo?IN3)TP@i?1}SWPZ@^_@aoa3Mfn zQaqg@VDf~8qtcyCnIj*<3mEYA>Os*7+dj57xIj}NU*cj%PxMiVj>etaP2ri!Sa$_7h5gcLpPH7Cruc7% zNoDtoTi8QT4GBj1Up?}edB}`}$Rx$qQ*1Iy*n_?g%m@P9ytm+4>04^JAS*z-Q{C43 zCVZ_x9I#*Fey(l`sP)^q%F4Rkvwa|)paoh2`I3wm4FYKnw4qp&yWVNVqmMmkJrP25 zfuN5X4QK36u{{$iDDfUeKf(!fzXMnS?(S+1usT9A9*0YoirNj(E_L>D3>zmu16yd5 z=E(r}>pClUL?DZGmR~@t(2@Ows+JTWCQ&;rDO`+Ff4(spP)MqWCP69Wp~_PzR}(c1 zNdiIk7npYY)c-Y_YUY@K+y(1pOk}`{9A|xi&Hn14ER`pktMy)uM`m)6troxlP}A2_ zf5>PGa@sN|pD*J=3u_(Uw98DMHTp zX!`0FEkDhHu}<9ItFwxl0)JEEz|OZ7iBRagA3#IkUvhM%LCCNC6CMaA!`dcyvhtH+ zysi={F^vd42)SR$HsH5RNa(v-P_(PDUjr<~S9df8)c68xeC6T#yDgt;D-AlmGqAxO zgaKqJQiSzV$^#oh2muZDbByS(PU2*pLG}U z3a`r&dQB$b3e00hVbSwh`RYK*BKQy}!aqa=v|lQ>|LSG{G;GKvUa?~vI%rpUstBCT zU-w{d6V4FH=`^XyeI%q-;g%OMhH~&+2y~HDpshRZSgxR4TFSw{kocNCX8x{3AY%~o z>o}Jri%|bpN)AUb)(y%Ss|-^z1O{YZ;h#}#L=De%y>2bf@Pv_}y^n#?(Lam^#Kl80 zO4bVpX6x;?pNb_WEeq2%bLte7r z7Hc*zd*hCl=1Al}YsWmgq+L3-6`Mr;_un9)N*mOq+V= z-UE$d16IYbKFrUH-{-IAGOG`0v;5Ovhpeb{lYHAL4_;0i3#OwA#=RS*y#ShRjPT(L zdHExax;dAbsHMMSI3GUV1}G-b?~sX+cDBUl^!0SRy?5im$yI}6Z&)hm$8bQZ0j&H^`+i12xg8(_cKqPy!pH64~ zUcHgHprvkCBVX_;>{$rEgXk-vjZKzvX&|rTWOmj2^2wgGrgd4k_G*sN{zuE@_kpe}I?B9Y@u#HmkFruJ))CJP*cOwS){OK!i_vNql zff>Q|Vfs*mDZeQa6bMLlCmznIbtOW6we;`N(h*G3%rU@m!1ti4iUVy&(<}dtv9kb* zW9!;=5)ve6fCPdjxCM6&?nCh45L^cvT!I95cY?dS334+D$uxkC1{-cBenK9`aE#acp z_Iuh3!B;-7k=|lc(JN@@E{)k(&}0C3`g-qE!8ZHmYqcYXS|3;F5bPv&>g}SR>4t+Q zzQXwL>m}cXO`+ujnF3fMVlel^p?7-J<{SMnv(iY;4;f`%VB@gT&4V0=Y!-@n&z!9H zn{+T|RThKSK5_Ycd9MH)&9DC?_|d?vM!pc~d`o02DFP3qwi}7n#2#eGzE&cPtw@d*6^-T@K1}!hBrug zdv6!gAVZu;05#{_MBhU0+}rf|339}YM}Smn*A+6;CIHHyVZ{N%cu0_dO9TR!adC{}yj->9ca z(ARh>=NOBx`OD>06E`xl=gOFTv(LncPR4^aI^kA~C542uB1AnwPUD$oqv%!fSl6xieurXWGtTqaLt#5SQgBIWI;hpw%pVzt;KzzJmS3JYz z;exO})?z2hygffb+Jo$lPNPr!=D|OMV%ICM+<=KtRA!7OBTRLm|{#(DX!7$ShBFtlqfaOMGIW^%)Un(m3LbB!FTgw zhAG+A$mQ%rQ7*3MCtmeP{l+f`hhgw}E;6U1^6pk9AO)rLp|%M;k#pR>70FH5SOS>E z>ScatjsvrX@Ju{QTAcSo zgJc4ZcGV6;8P?6RrBmKbQDv;T?v3@z&YprrjXYDC=7IW)*#x)7xQT&%stQAKiz(^#Kwd5ZP9s> zqZ;@>|2O$N0NRu=vpFkE){zW#aA7Aa>>a^m|LTZ?`UeIvBy_L#Ct0<6^PkN>Hd%}f zkU`}RBFA2_nl#23+KE=`Jibl?0cJTXbd#IqvB%qqIAs+90q-<~XcWk-_cQE%gfxxN zRU8mZXC|{G)5sLv+EzRdR?whdD#396n~qwZ`@PeGtTCWu{H2|B|J0qkh(z z>U^B$o@anOx5>0L#5uXo3v&+%94WoIV!vFhQGr?VH@w^4UftTB&Un}*xOcmbX|utH zba~B+q**0z-XArvlj*fI95A>DP<)I+3n)bw*xp_c+&Z1Okrn#Q3;;(7eIcTFgQ-F4 zIWl@D^4phsidPtW;07gQ=5o%Vb<03!Y@9(FW;C-lK8rrTmqH*f3!Za;)wGDNaJqz^ zvq8mLP90C~nQLc^JZiCJ1yvjT#qjG3rVE-`+h_YTQG0yvUdCpIe*H}I;T;;H->adZ zS6xx>u#F3Q5MFt|5O|*U{Y$A8#Z<8LX%aq?CMK$xWee9zi^0+77~05uK-!0rAbOC# z@K`aXY<&OQa>}Hhb7}(1npXo0k^^L%?qob*Q~RO|w4I=s5p)5BH199o(P$Dfgwqkt z_r$r8GzsLM=={TqL=-%(nUeQTS{$j)skn`E$ZbZ&jWG;#-puuAWSvs&j6iSsYPpeE zN25LJwyks%?Pj)%iAmMDn$OqmVNN2(UY{!8>w8lW3%&o?3ok3;$0(4h^8TY-$`IMQ zBaF3&`UkA;SC29YePRD3qPEiozL&KGrHkkzAF$zJJw3Fap$Uzw#eFinL_h0xO@A;Gl$dJnOncsSR z7Ifby0pf7wsAGcZ86{-bR1vhs7?qnM^lvluqsTF~m??!n9SqEjT1=(fn%aIj6W@pi zvNclml15URoBrN&GIoY=LMjy%vIRDW9c>Ac`9?HkBl^{frl{XUi>}K$D%|_AahH+V ztNf;$)pS#7=32D2W*m1|p#ltc(jqu*DhwG2-yj#+EAc3t_6&g}c&|1_`y7-}4UfTS zd%&i<>joAl^JvZRIlC~V>}Kw-_v&xF335|y+j}rj7 zR%Q$tUKq3(aZBV+>!j@nxpBZ(@zn8NCnZbko$i?(r6`Lgg^I>vstGiV9-P-ZZ~(R| z9F2&`Rn|!2ZHm8eV(6Q=>oBHf*^)#GsLDfc9QTijYGHr_RP)P2h&E;rS0nI9#Us-P zzr9HxXT2)}^s(O0%~TD9YAlzp@Y;_LtPKqtbG4?{6dP2VeE-1`N>jM~E@@2K+(naG z);ac?8#U{&iCKUvRm2mup6Zyd(cx-9&m5xwh+=patbc}F0Q~A&(J{`JT<*ehXIMWn z9S*Hk+BFS@+=ktsIj544AcJ=S#O}98U5j^l?;qChX6c&E9u77hrk7IXn%`4UFCt}% zN3wei$c!$UZQdDW3pv-es+yVx51aHRMy{GuOIeW3(U|4$V%Au6Y_ayt-pGWt^B}rZ zC$=XSze*yOURMpYUNArNt;ehR&Z3ESu(M(who0?CsUz_ue0-7s#i##pV_|J2)>GUlS`5 z2Ni++s47}ngyo?>!jEm9*mqrEwGI8bPL<7TpTU8rv4wl>)075$5-ZzB{yLE-;YP}y z12OpA&(Epn?(6?o1` zg~pYeuv)A6s} z<#WgZRlIYKmHi_ht;F!qecPX8kq*6Uz5~jSf45k5dtn28U5ES!EE_0jaP7~nKI1*g z5f6-2zs4i~G^-lZ0|HYkSo2`(Svrvi67GHfd20Q8iMAT6vO#cAtJpZat|?u5Z=t31 za8&PF=lV1?@}fmPaPN{=AhsZr)Bkr|{WJDwDBoe0o&gcu)C3)ovMm-0 za;Q&e#LTVR^5+Mn)HOs@9OVPPbP8bWGrjQw5z*;o^oi&TPn4S2o5DgwEt8W%lORDg42c$kP%tC7eCMr@9maR@vlzJ=#&4L#%|z3qB=t9o&mgq$C` zlX0DUAD(+}(U|hP<`mb|8u=J_t)Rbe9|fpJFEG*4fuynNiJQL{Hd|l>p6G~32K*cu5gw2 z(ucILCo>7#C2!sE+@OcW=aQP^zKlvnF#$@yrQqugJ%$8Njy4=RX9b@;^#`&gLuPHsDiwZPTH~xH}HqE;_`xGY;TQWE`zmR(0ve z7n=!73=1(Id64lgczVnG49W`{zI)b{abhiyn~zA-=TWjhboK_aXGSzMkm1 zDA!Oe)mqJ!Kw=zO&o#KAj~^69`3aqDMg&uZ@`r*-bkE#cJl8#brP+pQB8$K~(5E07 zcj0v^S)Y0ng{TEpu+k*!-5k-%BJP*RYv47l(Pbl8*L}gxg@XV^&74raRUxPc8FMx&94r)Ln5nD=-dJe8X9s&^RUcoHoP|= z@wOvZ;5QTCqFHjVg~{rPs{c#_X5aCb*APkK$`i`gtDgl#4{eDp;~+LItRl`u1l}0P z&WWtq&Um%4y+Sj72!v(Ah`U}qPa4eLVJi_!xu3T?eHHrdX1-=NPZ<=EwLQ3X!8?JE zJ0UEklh;`5)ZnsofAOhi|BiV3MBmkFG^D;DH7EE;&}npXo9BVav2T$KatS&0VXn^C zX{>ceuNaD*TrHTLCOf?Fv5y}tYHCzJsF8JI8CZ(Ke8=R(0iivzD`&fP5vw-8xxbFR zc6F`72DM%8UX6sGCP&Vus(=LB^h z>B%tm=HSH5ywJts?IZg@tOj)|FyF8W$f{w?ebkD(a5L__J?(T?d7xwVWLUBBJ#>IH z;^QG*HIX%-W44EstCG&X=}K`V1(}HEj(8g1f|G2(9U~umGH&2i-H~(5(E5Fcls?F9 z&%ohs`5;_5Qco!lOXO^SrbL40kQMpxgu}64)!^zvmHA-der#6tMZ+G^zE{maLt|sD zqcSRR7qzAUe8yXiU5PDauUrG!!Ni@Ulxa;V81Bf|)TjJYvr;BRf;x#>Pd6-x3 zX9s{?7G%3ucHP8Xo3JAy%We)Ma!xNz1K>fXh5dZ)i!b8Y7H2Xd`%+!6?ub!4_fAKf zI}18a6RG-?N8PijlkvCZ;wxJAx>2{anlHo!Mt(OOFdugjy5G(>yWZ6-Ha+adYE`@2 zpFEfy%IM!Fdawau<+7)D)?`Lg&2sLH$b^#7vy1DD4h91QXP3bz)Vc{sltKW^&D-P=g&0#kXzQ@<}cN+sf=7;g<+Q&wNSwBzrA0JPGMeWPP4W z8~QR;+I!!i2rPeL$3*!d_$VP|flv_t-qDBk6ry!r%uhS^&gzlk?-_G6m6#tWoiD^~ zInhgM276n5ys_#f_D%HU1Jiq%hB512SNAF~t)Is_gXftSr}kXEX_P5J=*dSm9>68s zOW39-Szx;C9O{7b>d$dK9p3RTQ~J|`bt-DF`TO%phq_0;vn?Szz$`q;>iP?!{m2DGb8_>*Z?C2b?b+X8G%7W znvS$jr z;&u6f*AMkk?$d~Piy0pI=eUoxxX#QB*kI61&_42y@m_ZBSpM4Z$X_P%By#UAj1Ae? zwn*|AX`<@Fee-H7RVat&$SI4t z#MXJYkj6~xds@L*xd>|&MLsu%ba;VwffS3pwR_#{tn_K{{(|U~b+kPB(R)#>TiE6n z;34j#KzQ1K(3dSj`P&d*QilE-Mh0`%D{&f2!TY>wNK~n9!D=cNpk<6ERfyI9`0Z3R zS3osGJvY!~sM?-m>s13`8c2>;u4yvcrJ=ZLwBZ-^r>tdvq?ZI9qDyZsW$xNF9tpA!k-@Nv-w_|;C zP*Kt0tckzQi!)q!LRM+q2fb`E8%1jD6imHLRaq_=c`YBW>diSuRBuEbJIuu%3*@x@NPs z$3*+5&FfKUCEdgGJxCya>Zq0OhSu^NW5vy<}Q*M0mS zEEfP-Puudkn(ZirHKHrb5jn{JV)LQGz8D;iJN^EiT7Z`ZUC%`0BJde8Jgc2HNg0N= z$FmSDk+QEyct{g|WCDygyYllz)gnbPV{CR6d7FVc&LZ+ROxILeZ~0%*u%F#7!evpOiLHrUtSfbUGrL>hT|e=3h| zV8WGLMya?hR6dQEXig^=z>mc1LvO1%TbQlSHQkH!3-0i_%QM19x!rx~g0B0>A8Ywz zG$oGL`&(tj)?`NY{(c~T&|jwVB&p~>Fm}0zuE%^>ZwfVc=Bi;K|}F|S^mua zBp9BSavG2Pa~CpkJDLV4f_PjrEc=ChZ8&y|_qRrsHga_xLN{0f<6x%W8tF^1UTc#j z(~?EgsQg>8KP*)iW85z#e%`%0;lb=FxxQ(bb1h7NSU+&*4a||AR zdjj}#e#-x*+`>FHrP*?yp@egG#0oBZQT#IrQ5naQha`SYZB0$SHg}1h9SOy&?MWhm zvX3onUvok?zXbST9jWtTp~=kUa64$XXmQ1;7o5OZ*|Fl+%LD zE9$&fP#?-LMhn?#IyCu3eS}xCdVJoVCl*;l{!4Zt<(X&;^zRZ#U9)Q)r2fwVa>TbE zo3zTfpfXBrJkqewkuPm%Qwy$sTbMJLjsqQq71WZR0T_^a;sF{z!vCU(z|rw zG)V%(SnGe88iw2d<*98@B?z_Vy-;4eiY^Y@pWe;y-bgXw>_!Cn?)w}m)W$-8&>E}4 zyhgX$z78C1y7Z5sV>&!b*K+A3BHv97yxKvGC2G3fE8*Q3Iy zb|K=@T~2CW0RLHeDfcIrz}-6V%KSpH_s#kF5d=?TeWB2ZmTGe(G^hD@M!)(Yud#8^ zgH=YZwA)6j4Z*j=Rs=XY>>KoP21gGp7Dw4g5i-rO%y55^Yy>t&jgG(sT-LC^N7(lD zHOsqGVlui+dk%-?DY%E;x~4c@zp;`S&5k>z6;qfy~5T%a|@1?u>PV1`Roex}F zNdw=seHFeW7eQJkYG)XDoIhg3}s|UdSFhdW-g8b zaz3o0SH6`8x6y6-8&BzgbpGL=MqijIaELL3`@9wU_Qgd$;SVQ@iB;`plM6}#U@4QR z#HT2NyBQ3ZefaCuHI4w%1=dM!d$2?Aa$)6*VPgS)ow0i)8KXfG0u{q%%+A#bJk-4a z&W^A!GZNe$k@8LQX`3YFNqRM>g7Izr8=g}YpWnFhCZm?hMWs|+!^mU^sZ}9OMj7j6 z7!ooye0!ZMA{oP5SH`Ee3-SSsKeYeqn9Z~yn^wcDN?V9ra+N?zxZ#?k$T4?cO_c43 zERJ)W7=$z8p`Ps6-AWC@6Xk9z%%w-c*UCb^%<1rpw@+K7PkXl1<))A%S^Mif@7d;n zs&2Zig{&#SHQq75-Ye@+5T1jTIL%kpdtkv&-Sgde??@Mwx&M`zB*RrszPO>*v9|Yl z-87Z%%%^#f2GdunGmcCe@N~8b*TdX|@mUGy1W)Hac&mF1E5yuqxoUSfNvgzE$7$a> z-xNPTIIS9&gjN#+AE$2*JY)LCI| zQm=HNp6KQFoqSo`0X@b0QI5eUYyCIZ>QPi#4#%ButTi6_XRgx&dgSj0=qX7G#ayWu z&o!bn%W9t2(boMdQjT1;{IiY!?BH=fGO@dw24P>Yx#q?fID2RY1tS(R1@aWhhOxo$ zYI>Nm)$WoKiDAjq;QFajwknRj{XWj7!lBH%-)yJIu7ZnCA`{#sZu7`L52bNq8!>~t zPK3jdN1XPNzpBGeLQ?n0Aqh8vOl#zmXfnuXIxMta@)v^`C^tUs#mv7`)Ri54y?7w2 z1L`K)rbUns-UTDEZl4*XVZ_&z#zYQi+pk2vd#8N+o7iQJpgOr1j=$~w3X+u%yI|7C zK{P`@rgrFRsOtzlO7>tADEDJw6;7A`YP;16yHyf90FOKboBj5hz}cFS+<3`u{jIvTe*#;epwgqU+_1kJ2)+S??%mdN$wVS%-fge|KvG!6~ak=qvGspbrE{mLmSC{y*IWxUBq~q#X+d>yPO?Sy0dD ztA^P4uCPuNd%-WgNlm5~igpQNBc>rzPLZ#(l;VYSPl?RJWre2iKT(qeQcHYb=-Ol9Vs7687hIjOZ^Sio zpHk(Tazo(wRxa%5gNHlSOdpBuS;Of?L6Y@Z-xN7jQ-9N>q}vffpjn|i?P@`ZTJOb9 zWb<_#-(*9T0bl;9{i6F#+q-bP8MCdum6=6umy>}};DWU%#APu>`r=?G=Oz?&^SUX@ zP7l}G4;u8#U>>Y9iRtJ~H&&WW=tJ&DhxCP$om8Y#Vj4Qm<}^1gfXbfGuZUmThm@ED zC(5wpUk76Dj49)6R(PHGdyT4e0^;yyKIPqg_%^BbCYUjMw7Ot;aQ>=xrGSpMs$<$z zr>Hue+G_beb6@uu(BYO?x_@hPnlZ{J^ni9Jc#1j6pdW94Yl(y%0pJ<=Bq@2gceeupR&!m+vZ|zt>1jW{#(u^6 zyBxB7Xl%t^M*GO$jXWbK8?crx^NLv{sQyHt^ms^cZ7r5hLjGepv;L8P?nm=#YpV-t52F4qRb7O1zZ5 zC3D00?<&@LVr;w(SN(QS%gasruEzei=m|ZxPoq8W z4E|I0c&vVCQLjW|KjQUgY@deM5iAcB4FyG3jxf=9{Qr$D7mWR_g7PjV#-EhqPu=5B z`6IIRuax7TeJT^PGiZa-_QJ0Xz9A7D0Kn*R90=zC($vv)P`(K)Ztb0^1=H~>kOo8Q zbMpy9`W<60t_JxbhRJZ~7o6`=p!9Hj{@kS5dG_kTm z66?u5_}vsPlvDsX&0%Ij9v#Gu^Tc^P8PkH1vo_<2LS-yJG+5q-C^M|@)Rf66xV(2Q ziM?IHcsKp^LA_I1t0oFEPVRoj;9hKRI^d?T{U=Z+f(FXJr#q_t8z_e?Ts83|9{d$3 zpFIW2YiQ6CVG6}#pxoMB?r1z+*thT$D07~bCvktWe+-ldp@H(tzXIjig+g6^9LU~d zpggpb-g?Q+#M?2pBn>a%OYuW-!4ESDYoz|;)!xq1>RfQ>Icxh?=2o!D22dn9KRbEa*BU4^NXdFnkBu^;ipR0s z8TsY`ANOQpW;+{{4BtbdBacFaB8bMjHAPk9ra&v0`&5RXy|q7uU=%C_mUf+wT74uf01`2n+a@}b_$lfDNv+rCL<*gqanO$&3x z?}A3y4)-Vz5^qv}#X-SN@Fq(WH$s1sjrpdeo$R%AvT;%NveCxqwo!R2?Ej3yk`SU3aZn0Jw5)7689jS{4o;&1)WvUkcm7mRguDE}NU z$9Y)`9FCq~%DA<+JoODfYV@(CHs?R+iK~1a7am?;62>o&T{u%E6%9S;@tkMV#;@_6 z(!jny>ns@u6@%INe zK=-Ni5kMkxnuhvqqA#$?@p?ebVmlc)SAnpoacBE*w5+}uI}&0hh#JmLa^T+%c7e?c zqDhR68K^&)cfYPHES0D0>lt;bsZE(hoZsJN|x!95pygZ#vQF~64I#v25iBS3Vb31arXY)VSar* z*u_~4qoN5p+rQIzI2$Clw5D=zO>keH861gQ8i{k<{ed8290}mNiEsk&C= zM#XeCHkkLfo=IY$Qf+dH3?8*L_$3W}xy^phL9B)mRiq4Hiw+@WN;E7r-_KoJ7f)>= zsHRy+#ji%-3nFZ?rF5WDL5vS${=sA-TyGE`qIiAcZeYL|)ct+WampAoQi%{rV8J%CP#~qCV65JYBiT zUuh@FZ^BT=Iz}m4A((2m-C{Ad!2oaS>g&)l5+ParEy~+exqU-_DF=V`>mz%-s%~oI zeu<<*0sTQwXDJ1oSa4ksJKE#PAv5SThCCs(vJ`OZgAh8om1zwCZ1GkRmg(7ID7)VtuX z>8uMDxAQYj{CaR>GgM?VY42rA2ETtzV+PT?DZ~*HPnM&-)Xeh#x|tk$);K@RmHJEN zowqSPjHIroOyrJHmCP`j!?hO-$#-zKn1bD-x4RsP zUfP2kG(m_MFx<4qfAiU?lH&)NfIcU?;m(+T_R#e@!jlxQjuber4*wn?LXsGqTfXu8 zlJ=ZRCiLaG8D{9rxvHJ)SZ-2jGO!i{mRhPLDx?ZCXN|RN_F}X6J)9lyL|-L882C0- z-e)VY@bDk(ArTh37 zKVS3G6>xnr=)Y2kb;S!Ribx|R8- zM;1d%76T+_L6u5BTBfpEb|`Q2VQ8W5j5Q+3@Ump9SL}OUGAH3XdY5Gqw&&kIpjb0l zejc;()kMzv5@p}z@!*_KxznFjPsyTU)eF!`{iuICbGYOKjF`Wih>Wg91$@9qaJ3!)q)|}&$w|VMwxOnpYHC4_-CkR9O5%Z z;-uqQ;P_v^@L9Iw~m&qy)iJ&B6hC6MI#;c~nd&8$vZ_k60k*0(=HoOuBFV~!0D zxO=`rI5FL26^pADd#zRZ534xj-(MVm=f-mhKtbsiBr?xw7d325Q$4(sThZpmM(u3- zHOp#APk_$h@%AQs@v1n}zV&s5)+eroD_(l^SsFJle-^WET=n?3UbfeVkf?>L4OMxn zA_c94a5$o^v^Tc<(%I$b?KA3rG;(2R;Ez<|xXI`vhnAV}39dr2Nt;QahMhMl!6YMJ zB|O)B6tYK?j7sOsc;NMo8YWvREP1!knTNxVXDT?*`rdQ!*^E#U+RDklP5H}7f0r>C7H8rHf|q&v@-*~{~ioR>3|SxV*x*8W_bP=J_%E_hI&x>L@%i4EL_ne z%)v-(Z>QV;EAw{mb3eU>@So;)?#f>zBZS_{t)g%YQ0BtJ~h6egi!P;WEXbrxg+}}CO%%6 zuh7h(IpOysfNAg=>j%{Cww9c&rSIG%)-T>e!tK?nBq;kx85bZ!4us$R%ncfsb&#E> zLM-6!KP2;B;_|@odRqZc`rm&P>HZEWKxrQmTs7Q@ukg<{c~!~uia@-yAA!+;g8m6*})ESvi%`wm>6& z4x zY~&O{9P^DD(XP6;KGsPLut_kRhkO1Gtd=Oo%t<{2bq3PEXI4Am_XO_m70#cdO&L(( zZjAqQ+~2Y|M2U#+Ena=AZ3aH;4rlT3b1)LxbKM9TcazSaZPl<8X&eihicklP#N*J` zV7)=*DGZFKNe(6n#Pd5H!N5`{QDT3?oDqc|;{1rPVW7?F>YoOfRz4Q34GZMA!YcV{uwMJ8&vPpA`1Ae zEa*rGz8>yl+Dw!>DHl5sfelF77~I|r=Mw67mVl?X7igW}9;TkcmH80e;9Qumu7pa8 zhZBdH2&fEb#<|2Hh!O!It-T-_U{%cX{h4+X2=1i=@V0He5#`s;4d|ZdK|^yarmRPb;53LhEYCzB$ZQQ*sS>Z=Etx5jMo&nV2mP^j!R~dTttrt(6*>^P!*z71YJF3>Z4vuz3l9F< zwm9aS(AQnLL5x%*@J$c7@Ik#48Q*qd??i>aDe$@}RIaNVS!UJAGKhFl7X=6WRA92q zs*+tsHN^L%mJSLS?=&XU9KA2YF|<6AO2tWEEdD%`yn9z<0+Eg$INKMDmcBz?+_3NC zlj%)&tKH2c+$pIwz4!{Y!Ei-3icM|kShS26PoPs?>^VE8n(u5R>2q=2Zgw}InZ?92 z{BgX3^hp&#$=gPpT|zjj%#ECVJ*iaTb+(FlfN;FD+??8dn-;M69crJV8g&3cK&YF_ zpt5~(Iatm}OoSwc((~%KiydqayL&T}&+qP6J4_rC`839KQb6F_)eiSD z^5s$CJKL(d`5R!?+^3_V;@G-=_q(I?%f7_e#rf{L<+d)IVqc)fpk&D6s(y;O>UCdV zGU4F86R1J`vMW&Oq_Od#v3|ZvtD}U;z2VxaD(S&>za2GpL#t#rHFA_@fi`x&cRz-2 zYdN8TA!z_>cyh6(zIpFp154SIg}d@PM!D(aT;=r0G-duWLTdp8D7jwVIN#u;8;O+J z0X8H>>knSI&kVGQvhfX-?79{gt!{Lns=HVBw`g4p7gnF#dgbsz9BX*)GOI%!H3=sY zuhJK$QcIfF-LFfWhMUJt?{5ww9vT}~hR!_m6Dm(na@^ZpZyZ1kxw-(l`=cR6b{VOX zd-JQ~U6p!gM^V+1Y28$+}u4(DUdN;JCdYfjAyHDYCanDV7U$fYM>6`@cMFR`OyAPq*_LTG02J#20mO;3xfiAVyB)JC_-m=vNLQ0x zz+w!`vulScv6JzNqgk;^=X^G{`qd1J=N0|#3G+0!X9JT3*- zJ9Lm#zf^1RFTMxK6k?!~HGZrwDqFd`%c+yr$t}>)i7D}AXPe>pird{i?qX3zu7YUV zphoq5uJjALb9(1mNc?7VNnc+f-6BXO7PeMBa(!GMRghqFQtQMz+Fb*TR|ITI#Nr9q zzWH{>qO5Ml6g@sdXR$#CHfoydUeY4GzZs64OkK3U>flRBNNX<2;8q*bBXCzwb{Hfe zbxv_4fS!;=^H1YU9XW_8Fbq(dFeln>v^Le!k-sFXGAg zanISXvXv6{JjX9H(3y?q9NW#}Y=M(IuQ~VO{jAK#(&z2{T0?~O5J!{)f66;XE!Sia zdaY>dz%Aqoq(ku?lVjRZG0)r2*Gxs4?fkP)*7)l0#*uJNfqiNsmK|}W30?<2@SYRXR`8-;Vht} z?Ypd;H*;uUz0lrug&{Vp1@W!3m&{k&o$ABjmhz)U&btn{OdXhpP@8am>)w@PCr6rO~^pd zA6XThIx}`6*1~4aJe#$0ewUadiEhN?qZs-y`ZfzG;H`*MNb4($Je@k^pBGJ&R%}Vx zbRlE=^Bx0rx3(cG-ujT;5aaQ( zBBXFB8DgBS$UL~&a>Y@RkTJS{HyJ!o-#MFr5c`ZM$t_2f&k7jxp4I(wR_=`MU!dsl z1Elj3&={;#`U`Ol45>Bq$Ld5+DGVXQF=P#G z>l4I;gT|j%kKwJsyAB}Shi{1;z{64>9y8Gz z822y$;cc)c65CTC`bFn5)?m!z($VNP$+R-5@F=rX+i&bFa9kw3JEM9;73T%;DGpU0 zXHR@dnoiHJhw|gA3F4ci0+Qgf-U@RA4(PD`=E(e#i|#W9o6?<0QO1jluAwdUJ~_^m z#1ghoWLVj29#cwC!28(iN&sx3eCk|;yYED!OxeT`+>DqC#CG6qi1 zFWF;5{EN71UB>b+;v0{ja$uDGt;_b2OG|W+ z6t+KO^@Z=eEWC0#Vf+D+CJ4fS;m0huAxtthLC44?!CfFDO8?Q8JTwbi-=iUG_)LJ}w(>M4hRNt9L6eGz@M|p&^aQZH!Z2{FAKw&C;gT1z7&Ob-H6FQY6IH9+ zz@nN?wb|Kn9l~v&wk%SPx4zGbvW;>z7Qy#R_3a$0F5Lp<6BjtUf(jhtdPD z8FakgF?1)m8BsMXD^H{+B^1-{R|5!_lBSZwA$6QHh!LCxqa>@KwUTZkj8`Q`sf2&V zFL2#cq!bXXQi*53PD+QSIMwX+_HY1nc=P`EAA(`C7><%^#yurB&z*&lxZ$AFC(h31 zDIqY7Unpi*76_`}a;2qn2_0%;#R4y@BG*qqL(i=b@#0BG8b|B3Ed4>wo%Vc+Nm}$Y z$mu#lZCzw7En4$DKrt!3`)A`nUG@ZFjLEz=Jk^3hM?3~EM7RbU>D#vyQbRC59Yt}B zfGF9Tg!&F(&U_4B!0fhHRa3n-_hT;Py`11!i7OQvR#1nQbUJF{=YQMfaN}>emUQKL zassh{UUy+)ZRi|!$BDz?l#tpt0MFx}+!UyYsp&twj>E)eDUXgmYQZ>Ce}blGFvpgM z2%JxG>#qdxd)!kA3CL`!i|5~`432PV3Uv^n*#Y$F-BDk+46HU}{<%c<-Olq5 zbS@cW7D-zwBnClN^{|6YI9}uP;;`fRh3F%(W_ZZ!V#7=~ei7{NnFjbBTiJ`)wj0S= z9i(lof_l-vMyU;bem`>nTB1opt@oZGkpt-Uczo>?y7N#PQ_=2`D`3sW1VcdTw53>Y zrNCU45!^<)z0Zu9DS~x38@pTugT$9>(UB;Df96K}pSiIIog0&|n-LxxKcVRmRSa^1 zAP#a)nW)>Dl!QUmw2dU|S{bU;FV>T(!(v#lsi-}SeOSMc9aYKgO2}mCPT%mKEE%sb zT=lhNkaQjOMe(p@u&4h!fnz-{!1Y2P&TD_T7-uXhe8q&x@G%W(XIhf43F5MzoZ;5! zl)%691mB-6yPnY}*#fv2uwt1y{!-k5MR|52@A1ZwpZc5fPk)0<)a2VZ#*v^IT8jhr zMOUKiVusf+8-V>g&!GQ#db?$34@e(a0g0|M-VnfVFO2$M{ z_Udm5GpuI&t)T^q@Cq6Q>Sd-QfKmsQh98y>#%A<*F{Y$GakvfC<(ly;JQgR=X5yG0 zixaC;l#uYg)~a+7aq!}F$`XeIN=jR0BGY5l$cU+dhjZ{fOF!ol9yE7bo*q3kTY?so zR)y?h8PlB^@#CK=6jJ}JP{j9Fp#yf`pic@4KV`>Bf@sW_pY?v8A#N6)TUL@>QRMn- zT1CzV`oT>&7pMpHHHCdO`Z$vJSv?}skCwIh>z$6m3(DMNAVsKY}>?64`OB_Qr|+MHw=0?RKVmd7JR{M zs56V0dLuemOiv&XM1oD{m&uG)VnmiUg$)yW)pUCI<0&~F`X9-$=}8E;823%Tlg>8-RboVPT?#WBVS=;C*4ogDN~kSnhl+e! zFape<>O>d8)n`CKZB@T9jmy6-&Ez%E0-d&xj{Un21b)8mGQ-Ut4q`76AP3THdBH6W zvk(~v!e{>GWQ{uCkP{!PLl4mE&@ETqONUnp`XhdH1KqmJ#pNeU`IsSG%9uwSQ!{;O zoud<&=EIS|&XsK=zY;EhdCh8RH2tI$dQ-YvKOxLbaL`5enC5gm z)YzL3n9qK_1UGw*=Gb*ciM=3sE|UOAB%2?!NE`!HGtJtvqamoyZB-LRPi?O?L!SRi z6p~2S?}^cG)Wdu8vgahZ&mP4B#WA-^W`cKuSLGQt2)>kvt6Ex$k`YeRd%*IS|7MG_ zb!g>P_3XkV4f5;~jImI>vEvvE!$vX3cgz>ZuQxgwjs@7-yWY%jx$o+n5SZ$l7#cn= zc5tfh&L+cFAafD?R_AJP!%UKt_%T#i6RVR9Iov#ez$H<|Z)xkYpaLgrfFk)=h~+vKvIBaBV1){>2gX<1#y9^o*D7 z@rkIa{MZ)gm3K-P<7Z7&Qg+fElJjw&dX39dBh0Yjm)bg9Xd6KZE*tof12jd?FPz9S zV{qJYb?xx`r?`7827}{*<=hOOnR(Ek*sQXBfiS_)8rBiLU&s)CH8j=0>EaEVcvUv zc{d)f$>r!hRn1(pjL)x4aL5uh0A07Z1U6DG2XSa=e*P`ZiQe8*LIemL#F)Y`53@|V zjM3++;?agQ-`*n%yhCveD~~Jb^=Tt6ze*=7v<)A;Bv~1!Zlwd6AaKjbKW@_y1|`EugAwy8mw)1f@&5yAIu< zh%`udAG)Nwq@+8fL0Y;S6bU7yrID1D5Tx|IeD37F`TdROc^59%aUItZ=K5TF=DYXo z*)uaPBe?wB1u0xdB*nX!3a98LB|F4y$SOl&o9IJtZf$8{5W^BuPhs91X)``?xZO+~ zg~BJao7_o^a5)r8@;%J{6A1ZL3KF@d-6N46ebn*8{ge56>yji|LoqBsM)GMOG~K%( z2T56={aaIj`nDi#`~$qPTVAm|HylU7X#}6lO70#h>Yt*UUduo;TUe z?7NZLElJJlSHflzh@&ew9;308DJiU5AH>?1p4@Ic?g|> zIe*;t5_7pXdu9b&XaZW1s2Yk>O0u38Yk>fTXH=*E`CRj~H?|W#p8W#DGsY6PPrU@Y zM{I3tyLgu8PlAv{60xy#;~ClSj^2F0Cfbk6@t-f9)P1PLZXvHlCcc~qY}8{aqST}h zmD1NVDiqV#JPEQODs`=2efRs9($NjclG0K0z=GS^#=%cWfB(|mflBO>+|_|plMYr~ ztYH!x4bado7z@yldBJgOf?aFG_V+J1=8jn^IOe~#{MREa=>>P&KI^io)p%Q|wuy@4 zTC!#`Tc~!1=@b$AA~;|)mnvuRoFSv03hCm_VGjiPs+*wG2SWO~;Xb}ZYc4B7)yPFw z-p!3?t8-7>Ep|#8>VEs;sWqdiYU1dqqH6N57N`~wEa%@mZv;hA4>(%}Ta7L-L1Z$A!A|3weeZu+th_wbjEDLA)zRG0YEUOa;DHkvU zUCqIXgAM`I@#jsc@>z=IU8*hep0rzH#UE!hlwzJ^@uJXJ^K7B(keW1mzR#o?qZ$zw z_YHE!tK!!KrJEo@IX9_U3-*H zk5QUvS|{(2R=3`4rA!-FoIdbwT9rc{uGrhR`@*SYcYwJM#zfqg`xQ~h`VpgvtRD`o zCr>cH5$*g)!8B*SjAa;$)$9Y0;t{5q(yh52-)YL8{86bCqADSntdAYyz9>iTQ3l#F zXQ~q8iM~XBx40GKo>=1=;(N5mlx9=oPt)q24Hl2N|eU7|L4>rYwOn}FHD3O0c75AeupD%Q+hN0d*7q{}zj6MiG zZ+o+=6tm}Bf4FtFxmWOBvpoBFzcgL>yzo7S{ObNCk@@dESw*m*7;DsjaR21My-O0VHE6(3zhi+JMp+wWuM8hb*%FKDG zXh(=oQIOL!q7aZ>Un?G}BxAcZb}qf(I=v-*l5}veKX-;8o-P-SAbzIeGX`OzISn9`=a)HVxXcfiDt515=l{zcaFH>}fQ~9sbbZc`-|>L-%K0z5)}f#0v45 z_K!m!45q?XDF6(W-Bq2+bY`?$!z|H68D?X?P@@F-@E$IxO3P(rbd+F4sTu)*xO ziFYBK?md6H_m=AbRXBatN(g1X19Uey#kw>3u_651kh=vpKGPWx!48Xxzd%N=$2iD4 zdB?Y1IWtMWyU{=9xu8@mq6?OS_BZ`2;3}s%e3^vb-GelXYBQ+!Ou6gy;PnODcy1XKHPJgL!7yBx@% zW;kyuAmF0$qrORk%*Rs0EMBQaPdFbJoS7W7v7}}~*iM<)xq*2HtpS=Q0uKBo(*>DI zh1_Zf8F4t100MI&FA4ZMo;fP~a0fAD>0`Sz0a8}ranXK?WS!29%Yl)-r zQSXl~85(3yjTMeYD3*+^PK|X1i;w5og?x;yWXEb)-Xvi1X1&gNlbgq!gymFWQ`tjJ zb(ACnmw?}uMc2=X{(dE#O7q6+bb6#|><4wr9m>xsdD}{=rLtCIwNupF)7>sCoSP0& zOcQa>dR(6d_VOhxS8JO$c^Em}IAPCwKQswuTN>{w3_lPFQ)Eiwj&nTa+$T*@z#Js2t>dGTb8g=CNkWwiwsw|J z-*M5UuX~F@H)>ZnyMXQ1ZgXGhgZ6_Cr@{QV;F{i9Dh91~3>|GeJLc?2epV*$lauY& zp}OJh4NuA+G$j?gUYuxDk(gK&4_h&{YcytQ7qvEZ(we8qJekp3**L{zPUy83e!)Y4 zi^-F2;taK9SKmXn5Z`b(MkG&cf23Kl!)}9JVBDcl1zyzlI%3@nOiVWk^>FfzsCH?E z@+REKt~xj%O%Wv*X7PU9Ur@Xj7e0;V%TH=($uI&C7>s?7tn-20zL4YC@trwuGekjx+i z8DPCFQt07WEP6!aOT{0zM$%)&Jb_!=S#FNsQ+wtk@1&&0{Al3iod1fwk>mCoH9v~oD1HKT zDcgyD+SD=R8@E`bNGArX)gUV@F-E7hU>5g7B7{OI35nKS4O-Z@A4fV8zE%z;QaOsD zcSsq3?DOCE3v+djdoOEE*H%va;1UE6-2? z?8SMlCK)3yhKuTP?bt1FDUDu&_j1WZ$<%QKN5veLVYmIZbwCMKmrS}XZtw)(rwelF z4}}d*My~y|0woA-5UHJcEPf*DTbA34tYJYZ1Wg%YHr~h zeU8le39+1%I69)d#7&0QYMnoxCpVPEf7QUgIK{n^)J}ub-^bCJz~z$&0Swe&a&W{1 zCD}$bD(t8Rk{f}@Yo3bC0FMoh9?8vBL&Wu&srJ(_qZp$b_>>v06v!;e!SgUo)6%l$ zcX484Z%ttP$=!rENIYYnn_I?vuqlk~{3SprOg1coWbxFxuAgYGFQ0^81f$Lft}6T8 zk{qIJ$J;B0zJZ+9y?DO7@c(R_OL!*44YES8s2-ul;TIst9c$!*U`fJRt z)Pep|1o_hoZlSXr)F?!KTSO#JV-%lo)07`EPR~tt&!LPb*4!V9#CV?#rLa$^#~f** z)twya7-8!fvACU2m5wj;kcaSIYIkkS#k`$0#Y|bg*p9vV@>%bMuII_d7kMdY7)%HV z2n2`;e-&*KYD^JhXb6aQcnF9j2m%N@M|%fjdqyXHOEV*V2QwRMMm8QUc1{)+R!(+q zMi8@|qcyWBSl^P#(#Ft&(ZS3LY~$#_sBdk=XsB;#3ASU@H?XmDUUq6m6NDz=PA0Z(i zU;q$^8YOL;X--Vf!7_w%g0ZLK50rca3Tji`_U;?M#&0A;6%Pri5jP?k+6f{1;`wx? z%vxeZANMB0_!!R`kxtCo=F1hZ+`86LDYN@D9>kj_yWD&+q{-Ua)3MgZJIT{HP^1*9 z+PNIhKxVW=gdv~My4a_@7d28UUL1!Yj5N1*8pepD;9$DPJ?^2`qoIjS%}G5xdKOL> zzQmp~j#*L%R!(M9G+!y9EN%{}3Cc#rfbj4ugm2;Sn0RHZiRdaHB^HUBnhl%YyFaFF>u+;Tq_e^*Ek zNDqxqqiYasTK=Ta50FC$){>Aqd64vdK0GX`nWJGdtg*~UdfQME-|*SLBpN~_Etg$5DV|hiji313*d=1v@%v12_6$x5j}MMtTe7w_ zR4gS?v{7NeGbm@3HO3}M4B^f8Y+CexiF$4nj~%J5QyG*Y{i0LKO^Eu@s3fv=)jRH1 z9?ezy3%O4x+Pvuz-jO4F9!IM#1CRF>631MkjS|xPR13*Fyis>l%xcSvMku5fUoPi> zXT91&@%y~fx~-c=TwL8*5*cGVh6~A3ocKOec?#~@$#K_Ym8uthR`yUgFk@CoU6a8{ z$!O$*=-@kk;jT)9KwI-=$C)5er_wbXV}~dD)(S?zm|Q%+ZzPUIVi#+Md2YB$Zql}5 zvOK}?n{L#OCA|=Z`0dksw&EbH*T>wQ49WATG;|jJa`^ByZ>|mG!Y6HSmD*cu z2D7fMyIr0ibLP?rjUJ7Cf!t+EgOV&q`K-FzT_Qj1G2Xw#MQUwfLK@MCy2}x>Vz2C3 zf#3|?bdVi?t2=zI8f24;zF#_|xsE)$gXe&oH-o6op;yvuXvJ5crvCaIqS2-G6Vhef zph131+Zv3G{Hg;wvX4d+ajJBq(_GW-4eN#QQwAB>}Bx1ZW zbba?4(>A#IOF5_EHSP#(Wg7aV*UZ&$_K`o=9z{fyA)FeI_~@-02^9i`=o^)Z#QQ=z ziEgRKRxWaImb|U+d_s48d|Zf#Hh0`x?JV&;Dg(ZIsTVa;JmOho-qMZ8v1iY`ofU)# zk*U@kXYcb9D5c(v2%3d_r+>VYSw-uk`pP9ZZ3p?V3KTAgjF=YwL0=vWQC4%5e}7*u zI7hAt-Za(5kgtr6gd9VIu7lLQI!Zk`TiOUiM#ZO{G*eRfVE|&o+fN7dd3K3 zIMS_Y|Jx|tz;0oN-TCl{{1>O^EId!q)w)P6CrSjvY5mGOLWJ>#-D zuGO)+nCO}tPnqB5Jg0SPW`5@FC z7uyFWW+5jEo_(iZyv##OXj^mEg1CunkcE)Nx?kjAgti#OxZ5~!dQHuEx3*L?(I-B1 z-~71xIZ}q(j6ao#NT*>Ahj(kos@aPa&UWh4a|$tb!So13E0hSh*uBMu$Pab1q+Ynh z1adma$c1CRl3rd#IUTmP>;i5eJddY|!dO;=@&SolQ7$+_cW>w`&>SXB(OQelvFW&dN^A~%SLijhS z53SjgJls}kDA8ScKZv=h7ZFz0q}>>zD^^wkF+Wgdec15i*~D#ncm!c3Et;|5!)PVp z$ini78|dYZNvvzQ4A?o<;n_?>DHX@1dD&?wl4k?7`()*ugme#LN5qoxD(Fgf%*ht9 zm9q(Q%for1P}F%wNekb{V&5vi`%I}SRA(lmjwcxUCJ(&ggOZpnB&wLVekDXNKUO7) z+ciqj7Ro+4uE+xxW0^tGTau1ra1_eQNYu|p2pooU192-x707OX7{6^rxA`oGsnwWl z^ksCbGY8187TwJuvn3%Q+A4_hl_VX4nt=cEy9lbqJ5Uwyui}e8W}g{VB_bfap&cxJ zvSqPi3mb!B&eC6Jne%E-qn_ABNXJzzQGjTYYs03 zHXy#Sf^K|!He0mmhR2=VqD--@P4j?&-$=gs6p((EF(yh+ohuIe$e!Mv(gOG~j4=UD3ir9SsLRO=gpr9ZzcD zP4&a9&p_eyUgpBP$Va{JUC2t{`0I_mmh6P@BSEht)ey@?FbAS1mK{uvC9F5&hB3XA zUmJ4}kDbKSuyo6PBF3E_@jm9sV4UZt()H7Y0$Sq>yZO8&byF8}9CHci=!#}yxkzKX z^TwwlDHoqDllD0726M=XJ>pMGEihZBd8D5UoQcOQrQQdAN09jB-0L3oqj&H0#3yJS zSmwkYqft+oRf}}nJ)F4T9aPoq;KILd`!;Q(H5QA#NOiQyD@|h}-w@AcgpVO3J)_7u z*|+~m`ja7*G?6ODR~2^JPJ&Jl51e_ZCvq3sAMWWqPSe7D7h^4G=)PHZW1#evi*?%T zHE;I=f*LKIK0ODDES_1@XBr|79o{XyPc)OZ;;}6(Y-^^c?jdl~OUahk8kZ5A%~Yz{ zRnotH?$HFUI9C{$cp2ZI)l$8^n9z%f;w+~f@stR{sgQ0QHf9>Rhqr5kf~2Iv7#jPQ za4?<8;}7b7$~UDf4XDNnL%l_oSn)if9F9t!#!}O6Rl*8xa-|pj{onjiyit)fvt)57qPkw9dpcuAK>-CTa>C8pI|xL&dubGSG1g z<7S>mAZLd&@hE~#MfYdfa;Q*Q@U22Sb{;#G&D{VQK$6?_qu+b`=C*3ZqYWh-%GzDY zR~oPV3VCM<$*p74KjMOz1;kPlZT!(5K=hCv48>zZwRERtK}<&Tk&CYqD@QC4dsmBG|&@ySA>};ZP)%&b}H*Qh{90Z zAe+WGI?E~qcDlMCJd(_)D4}H_ot8J(UL}F@R%N+4@Ke#-_RrvZRnW*C(1ADf9)>`+-Z3yOQr|~FP zeoa1|-14WrCr?m%;@FF|Qjt`0xO!b9NO{Fys!upoKCpZG{w|m<#X_jAsN%2_ln<^}ua4Jt6KIzQS_h~w zqXt_sy7hP$;O=Wo%chvV6iaSz;_H5aZ6G^mi=&`Sn3P0mGLRc8Pbo;NXH_FG-Y*sD z$Yqdb@0M(bW36jo7sA48)+?xv{#GhA6`Q4az;9ni?+%%M4!OHQY3KV~NxmBnVJBqI z&W=PrhwCL$O1=)H;J;g>$MIAiAs+cDonMH+TWw1>$St3E=j?i?GsSkS;O#23<(myec9Ddn#d7n zAS|_?;L&C9Mt%q#OR@&%AeR3XmZHM5J|g9qx!#I?x3Lb{={SNZgER#K2btuwo@rdI zq6atK%A4LE8W^ckEFcy+`NHgFcD~h*7b+E1EI<`k`4a8~!mh(p3BfLC;mRuLd09}9 zMhZi}cvq5SGLm~BD)bW!q17#`{F^<{FQ4CzUl#Dd&wKT#_jA3OuWo*x`%_qzm^n~( zHbFS&%{cy(RII6Dnl6D`U~VlKuHo1+eq)xzqxV&L4# z%-!>O3l>HERRt$DZgarwRp9z94}nq$XsNejC;YR+(MRb1Xol|%l!et}axi?Wz!th9 zI6zEKAtt~wHZqE0bpN(!EEFlK_`>jnxe|74u=QB&Yj+JjIixK!U!>f8*ivoKWOvjM z^Xs&Xx$Sdm38d7M_RY*JYS<%$igEJJj18pcQ=_d4Xz467mG0;%FnT?v)0gGYeAQ*jQsgIvI&EJm3#z>s#F~Eh{)TVW z>D=9CnL%n;cSwAqO&_RAFmSvz@@SIqQTOm4QlH%D+%mV5m=Hs$#!N-lKN7BfM@Hy= zXQ6rG(G2J3XLXzFJ_5rR>d`x5yn1(wX;{|O6nM(pqdX6zf?^RcX*-e^c-mLhjYr2Jr!gZoTpGrhu?RbP~ zJG6ia2J>!HcDP7ZwugniFH@&q-0y-9XS(0ZNS247_u_&%Qs@@e*S-T!r}uc9_9Ui~ z;Rqq5unV(TRQnYSy_wPBO<48`^lb59v2r!aJ?be#J*nc_2Xl&$?sR%`uv#<%Shm|C zy@G)rUXuhFb5`f+h}>nng+A2r0(ThVh)CV&7Mff0q0*IrJ|XYs|Gv=Y;Pk#QZ1W~^ zcUqy(lnWt?MzvS1JH}hT&eRUc7N~u@yd;{Z&y*N7+#}!=g9nxsC~s>t5JmvMws=>d z6VWYLISUHA4|$5q%F+-*ax|9aAu{FdphhdRMESe2`$Ecbm2-L}b`Z@b0V>nL5AA*& zwZ=PEYA0GB*ia!}w|)vGDQ~F6N)Yn^HH^6-?@3zWOGKC_!D=x_OtiZEfv>3URZF1p z9g3WN?)O1zNyi=i2>k5Es$(&VBbNeXcp<|lSjcHcGjdQ$Re16{7MK#h7)&O+qL;6| zDTPNLQ|64sD>$hrJS*RjpHPG(hQ_ibtN!m;2 zq9Z*cvQ5}*sV1T2mcdfY+B(;j>8}b<=mRLgnFoEQ>SQI`02rsiC^jA@v_eUNRt0Q9 z88&B5B{orMAdwl0q4_`~hQb@4ZsE@#bAJkDu1WxX_6*v6Glv;(+CnrMf(#}Q z#5QxWkY7;4-UEYA47ennNQ_^CcLC$G+6V1po*iif_N=1zE-I#^Bq?{DbS0#4x&j85 zU=6QEf7;y4k_k#>D#uEh(znm11ndd68o+vX7NJ7D?CGS$yf@SnqMt2_&UVTm zk-ut6*sZ{B0HP{Kil$*%#;hEJX~<)CH&l^B@X*SL+obgs7*8835geZtzah<8%O6?^ zIX1T%lM`(!>l&2)k51by)QNg=7<<+r^8yut^ABk zfuyd4-cDzctHb{&+oYt!P2vl^dUi}mqHduYW3MI*3a+1W9IKi_lAFT9oib{8r}RT<0mc-{y3%QWBuVPThTts$`YV3DLhP9L!^hSpc5_M>~s2=R&e-$v6aPxkg(`i zchVo#AG{{A)OmJ0b;d|GIw@SVP3sOk&omPjqk3bZ zPeOI6DbrExL#7uUizm$Y>o#K_b-2{p;n(#wkUgx$l3OmeF+tQ`QWSBw=UW6L=;Fw$ z)ee`IHaCR{Zc=yin5ptXQ18>XGue+4R!o?#gor<>d8SLlgKd?C)M%9yA7)_ww8@}a z5)Dzj8+;Lhd*`srdHa^;V&q64*o~)3j9 zwybySE_w$Odu5t z2V{{`uQL-B?Ep5hGjnidG`0kTfxNVX1K8f-KY8qL=7Dk*1R_USP`8g&!-n6K zhj8^$xyzHHHX@Ru%9DgV7MN4xc3OT|i9`CCFLoB1lXy-?gSmOP8111Yp3Q@RcA;b(!ot?Cq(CR;C@TYJ=b6S!e)eweMP z#*KnI+U?a|1m(Q7DFYSCc_N_+Qg)_^nOF){i($J)h|Qdtp*M#MuLh4t^;MwkvOdT%-90+`^o6* zG!0fBnu3p$hVPWi+l`~$B|X5?B0(;aq4d6uNns%~jj zb3{CXOG>@D?K0=As3Y-$%g){7T)#PF!H+p;E_5Gh0w`pEy;jVc7TMATw_C3%gLANJ zW@;Q_pUg@}Sg8uJr93oABYarWAnzG4T{<#`wB^c?G+|r2h!43J=I-@)7S`F>U#dBk z(N?~WLxHF?udRyz;eb1YiJTL~nyDvlm_?U-e{@L=s3xhpIH4-=iG_hKYaMRSctTIy zY6;i~lI$()8jU6V2M{?5iQDCO|$~;H>s!$}_WyRvjg4f=6)ZzIVPa!$Y zm+Y>%MO{GGf-R%`>_fD$rddqmJ=y5&92?#~&3Tt^&pVJz4hc(L8mBpsl>}9DPn^@_ z5&L<#d<#r41yQ1J#VRJbdm;_NqC-;8edM?THCt22&PgdyWA!LO*O%at$-&umM68EA zIPo9-5qm5s{Y91SL0wYioG0t;rM{I|4syan`W4Z#5OjGzv|dvfWi`4| z!4#dTmJ(|htzm2&z8y-(mu<`abuD}=edZAcT3PhRkyF^4lEIvmF^w-JID%@bO-qIp z2INb7?qy0%-s40!e{AsP6g3YYe>aBESSo%=|)q z_OL10RJ1%%Sfg>6F;0;PPqY#IWQKkqHqFt zSkMbd!Q=f^NJ%BuTm2#;8Zo~2RdWqS8S!4(B}E+s$j>^&LapgSRDXs#d?Ea(Q)Jg4 z+(!aA6_oepUU8+-QBPtYM!x_y%`u=d2ux2OklB04j_#Rrq(x4@+6@ZGINEq})^CwXnJ%-o zWIe-h^mJ){9Exis(xpqt=2L2!NmV~)tydM+PN@-rw_F+ixZ_0cz^;X9MD0N{ZzJz1 z^kuJ0Ut#(X{7bCca{bTNZ6*G(Zp%bVH%=DoE80XKa(?_Ermc}uCP<%!L~Zozy6t{? z8BSuNKrY*4Q|s0?CqmYoa@C=d8R_O!saH)||DEVb9Ad*QT}S#F-&Yg?E|&7G_zJI{ zUr?9U+~)eEL!^+fGU_aRI-JFmgA9$v%YzI9-OJN^K9^pj9AL&hR?F$Sy_Lzp$`YV5 zpmArN{}~}V-01zDkF@sXRb9L0>f4Bpo-+B^x*TgwT%gWLc^c5gvs2Re)YeyK3iYwcA#QQuV&2%0|m z^r0|25$+%ecYmVe#vx6xae4GdjNtKYvE)s;QM^3_5cgYHzXK(kYk*le_n_gc)TPE(ggC-+>!Y@RW zOD9|XN2wS&9n8ghWU`C`>18p2{$E7uH7VDh7s)QMM_|WBCODp5(8(Bnwh)?HlFeS( z?$BxsgMTd+|F+tqWANx?W;NY(G|&w?;8b-F{5XK{Db4F5CQczoHlLxU)%(Y#!8k0& zUuNr4salLa;!(e~zk~mZ8qqRc(iMsrzF5XES*B)KtoTmfQ6YgI4*4=EUvqcYasb`N z$56dj{2|f(A#Ap^L6z9bdq&f95VIrQgbtGPP_prdCg9gTXqy8Eo)bI8#2@FTvJirV z76lM!44ASGy+3;KBU1L8XbX5@EIMs7bV}eXwn*OLR9(RdHKi(VV%C_no!PJ=+|!^V z-KA@Z?a#I7&V1ox?u4)cVeSg=C_t0-wrA_AWm}ibS@HouHV%PukCf{S)>(P}EgN=V zdmBePL$LifBd1(tO`E5jm|lZrxc&B~9xsxVJV`KOa+9Taqg1Ag?$m;AH{J|um&PkO zYMU@7VVkw&EIRdDKJZ*yw2u%H%#fiPDH<3(v}2vMfrJ>bx1eKHUcV!Re_&$InV=n= z3>|N$HsD=9a6vkrs~QiX|p#T8ja%73bh8Gp#B1=Ad8(C;|m1q-Ku;||W6 z3qS5ACvaYbwP-A&E9y3j^t`=+<5)sIonW?uk+9;=tMy16Ie8%uCrW{g|6R<@$}n0umX)XzROrW<{-}zS{Rojr#a1u3*SkuI7niAIm(1%~*$8jXBXxAVH<8{c_#I(cNMSgyMZ`EkRAm)zb^tw8*GTK@B16y~yey)Aulj1W94be7uiE543>2$?z zy`$H6tC1C#h)RjjH(_*nW`ro#r$n{=&2XQs5rIaEKvZ#z`szmUJm41mFD;N-Bo0tk||YPKAJSD zawR#Gla{MhgPH9(WDoT3c@!5NeYy*UDqQ+Nr(K82NU^kN__O6UFN*48aLcB0saKH_ z=OjnX#k$N>Ws$LJAz6woTrrv?&L2z~<~TX#+{#tvo5S7aq#bBY zz4%9Dvz9IT#sjG8iUU56uScT(C?>JB0~=eKnV5dN1Wh@d2l+V*DulZZT0@>~M7WiP z5JYQvEtfte@dHal|4)3r5pL{k%oW?!qB^_Vfi()KiKW{pyw1-JkqOG98EGH#`N(Ic z`gsjMvKNzB>&~KP6KAQ4E-=o4%tNc@K5VaFRlq;9h!||M?A-!sJUjH93QqM}f;G%N zUjNVQL;P9?_n(5k|9tuVFPZ<{>oe512ir6MzpC&q$EW#W4`QR=XsX1UpNOdpn-sFq;VyJMA@1$_kn0#zO}-o?>UE(RLr-WSj6k<3XmHD~tT zV4K&Kr>qeiz9G8(sPbYDyUiEO)X!*G}B+Dc(C@1P@Nnq?x83V8gZDd+#2i zX=~V*f)&%nL$<9L$e$0({SvKrFuFFnUAI&@e+QMcYE+vv)H#dXyF^dzy_*L5+dBENV zr(QVftWOCe^3j1SUiD;GIMif)25mEy(zA3wSUys`OQ&2MX04%oLs5ltGBaKw{+9fs z8=~RZxTzp9PK00?sW+!x>gFxAI_7qBQbUqAUzyX-Ejy*pY`lw%7(p|#O33@7y4tnf z2)VctQP26@0s$2PGpmFYSAGhX^g$0*j~H*32)uKi&H@=wC0#(3jW@-*asT9 z+0$-ItF#jOewfl4KTu`1N=PF|353N|UTIBH3cg5QT+=(-Zj$N*hP9NCS(tpycSAe@ zG~xQ29nto8l25x(9}0;GGQF{rtuQJBcQ$P`S=@djkvB8|r&vWv>Q}%Xi9rh^)u=!s zYQ-@n_8zs|T*(Kn1es}6%sT17QJaU?IlUC=_=3(MG|pQnkM%)1bt%tG<2lX;){01j z`O^0}A95*(nWxz^WSixhG9J%kR+#NCJWT`*=i!*cSQv^;<=@ z%1W@m6}$eU(Dk=g{wm&eJHqTohJ}ET02*ffy%iu63-Q!f<2zb$u(7dZw9>aWGX`>` zzYm!2Yxkz)N`D*Bf&kDM)1`Jtfv5Qvogq*y{I$3k{6lT_n94?^08wiPF9ZbRr8dvt zzTM`3Sb^H*A8L}rG85hsK%@f#W{yiuULb$F$v-iHGUl&Q?RPYNb^oFGy=e%Dul?sO z>bINzTBQBGeF%B|kxm})UhQc9`Cd7J^3HGB@@q?WU|XQ%o5{-P|J~%(aQ834-M#&N zO@7+}e1re$hmpVZFrELqhxxM{{%2m1{iUap#rvPVqCcPN|EJgK{^x^#m87%ldGN0b zK)_}NlmGXd|4UOhLa|WofI)u%1TQha1N^3`<@nd~=Klyrjm-2-tZnQa%nTXr9j&bN z?OeY@QU2oct-pc10xW-EG)fUK8<;R8oAi7=$V%`E`Oi= zdswVSpI@nt=Q7tw$8jkj`3b9?Moe^*0KM^Rp;p2rb?lV8MlKKwAu>WIn@>^iJMcrtA5FdL^&qJd0a&@f=Q37}`Y(aS`BPY3 zF8V{a0HnZwVWn&S5?G+0!fFy-@7@49pc^np3BS|{owR=mELKK+LyI4Z4|lKwnU(=B zT#xWiBx?O%0*UR1ks4&Eyk-HU+J7MpgMSGm?jJ@{{FKtQbS09L`7eRQ^TSAKOOd7T z0i=e1S&78vmq6n9VI(}&nC+D-kp}I52_*I(Mw(gBJiFTMy1V=mNZdaqTduc zr~Q|ZU%34eSfC%alvYs3*wvPb^ZF%_cz()K+R%HA^*|KS`meYT+xM5iV*4qqiFa6G zjaOnNKl&xG*nbMkGv#4mxw^}|SiuL_E~H0hv=y)LA$6JAzed#1fD z*wDeoj?uu;%+lzG<{f#VPVyI5=EtCtuaOFLsgC`-b1i?Z%lko^DgSi<T)UbMI^!DY*jfH3bO9uXaer;X`n&D;a;}k%mHWS-`<^VIUH)~^ z{ay_7jV$2R41cW6{g%n3E4Xe3LmObXy0sD5?i)0JX~ltLOnEpEQs@J*ZOSEO-};4U zzG^+N`1mu~&B=lK$+{SNq}nX}V&xm-8EdJh2W{m~bJe*&d( zkia3D-&1M%9d&v?Qqu>N%lt8(>JSM zzPMT`0)Fp$zbOTO%XRbowkZ4!nYbDy@)F9~uY%%Wrw_z}Ur&nu-h!=)8rzQ!z{rsT z))FskUuWwtVxPaDen$HmUX+-L1Auw}VCE(5yY>8+12i%N6vNsY26S=aEeJ)faLq!0HpuY2*aAYMlRONhsXb3d}jbe_CO7< zy*;2Km+zqXzOtZrI5_@30QTRt<%Nsa2+MIr(&OiP=@xOvV<71UG4?N6kzBh@KrY6I zj%HwcL$LLa?Pq+2*gbu9mMZ`KHS)0jZ*p*VKXL9~Et>ON*GR{HNmKrtodExL>7?J@ zH3D+{w*{}n)oZ<17raUyTq76zrSifnZtD3ym3$QSaHj~EwCef%)7AMOUn3pse_QYx zKx3$23kc)axa)F^duVvAgClU^i-V1=4Y1Gi$96^PNE|9b0@Jc(;Bz@Aw*z~fR9uX}v88rDI=ODwlg_<* z??wguHPWzMu^#brfyXmx79|rPvikQH0L7cv33y2}ep(R?k2aLPmXpr{o$>MpsW8;* zr2B7|IR9@hMfB_By5z@xV(Hd#0Mqu5;g)e}QiF$ija;mk;`~1o2f%{N<>S@9&rSEi zcUJHLGg5Bg+=2cX!m893nb;$?(1eb0LMW-Slj_SbYa*i&P;u^U?pv(J+Z0&4p zjDILc_xGFh$}g!%G39lF{Wg2~|{msz1j_rbsy135-YD ze+4YaBG<^s$-?-z!%_e=N6R0}2YqgR&_D?I227v~;qsov^gGwc1-g`GHZ%nTX-XU5 z%7Y)0n=<+y#~Xlu@cEa22o<|VHnvMa%TM@+=whYwtM_8+Nn9fr`=#;r6I`NaNO`qD zU;LUme|kwrsHLuv3v|h!8-R^%fL!@stHM8%HV-Etng4nyKMpm!QwBOUvtnd?u}{onJ~ zbJJ_2W4q)IenMLC9K4m(0aqXlxPr%*q~(SAHF5#Bzym%TZ0GcYr6nS2N4Q_B6Ov?6 z{^7Oft*#M=gXiD+0puzFIOP2MX5l2%m?BpXF#f%3WMjLu#PQPu?C(>c%dxV<$7`hf z&a1`0&w`h)$@+V(xEDOsH7*N$&!t)4M|k;4roR&&0s-gs)VBib9^bs6>HFv|U$}CK z4*44C*sfSb{fT`Vm+yu6I~_g7HPU_0EfL>mzRRb|{~d4u=UM>&NUZ$GVR2WN_{#k5 zx__%yzIFS5hV#dn@89=^FQ54S_ct8(zkZ}|*TH{dN(>7W`vIw6;7 Date: Thu, 22 Oct 2026 17:19:39 +0800 Subject: [PATCH 290/343] fix(validation): capture I05 descriptor ownership evidence --- docs/validation-heal-operations.md | 2 + docs/validation-migration-map.md | 32 +++++++++ scripts/validation_heal/caller_socket.py | 91 ++++++++++++++++-------- scripts/validation_heal/fleet_node.py | 6 ++ scripts/validation_heal/healthy.py | 2 +- test_validation_heal_lock_observer.py | 67 +++++++++++++++++ test_validation_heal_lock_timeout.py | 41 +++++++++++ test_validation_heal_performance.py | 1 + 8 files changed, 211 insertions(+), 31 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 0e495ba6..b358e232 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -725,6 +725,8 @@ Before disconnect, two `/proc` samples must connect the exact client/server TCP After the caller closes, the observer follows the same request/attempt/owner until its `guard_dropped` event, requires two complete samples without the original owned socket, and waits up to thirty seconds for each peer's FD count to return to its baseline. It then reads and verifies the exact historical version's full content and samples resources again. Every transport result remains UNKNOWN. Diagnostic guard drop does not prove distributed release, snapshots do not prove ownership across the disconnect instant, and aggregate FD counts do not prove absence of every leak. Those claims stay explicitly false. The timeout/stale-generation native fences, queued-successor lock closure, complete DL/DV physical and ACK verification, every H1/H2 branch and independent live acceptance remain required; I05 stays outside canonical dispatch. +For the I05 runtime, process resource baselines and later samples also retain a bounded `descriptorCensus`: FD numbers, descriptor kinds, hashed link targets and TCP rows whose socket inodes belong to that exact process. The caller socket's two samples include the same census. Raw descriptor paths and descriptor contents are excluded. Collection rechecks the original process birth and network namespace, counts descriptors that vanished during the read, and marks every census as a snapshot without a no-leak claim. These records help diagnose a count that does not return to baseline; they do not exclude any descriptor from the original FD threshold or extend the drain budget. + ### I05 object-lock timeout diagnostic controller `lock_timeout.py` extends the caller observer with one real contender PUT while the exact historical GET still owns its read lock. Run it only against the already assigned exclusive `lock-timeout` H1/H2 service and the current auto-testing nodes: `python3 -B scripts/validation_heal/lock_timeout.py --binding --nodes vm000 [vm001 vm002 vm003] --bucket --entry --output `. The original historical payload must be at least 32 MiB so the controller can apply and record a bounded 64 KiB client receive buffer, then observe the same established service-owned TCP socket with a non-empty send queue after the contender response. diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index b6103670..1b515e28 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2436,3 +2436,35 @@ nine seed ACK pairs. Unknown or successful writes, extra requests, missing samples, changed process identity or physical drift refuse before Fleet creation. The archived failure remains `FAIL/cleanup FAIL`; no canonical PASS, backup restoration, namespace deletion or verdict rewrite is authorized. + +I05 drain-budget scene isolation [39750591531/1](https://github.com/rustfs/rustfs/actions/runs/39750591531) +completed `ISOLATED` using helper `b56b848`. Independent audit compared all 101 +original artifact files byte-for-byte, rechecked the unchanged original +`FAIL/cleanup FAIL`, and verified identical preflight/move/post-audit tree, +owner, device and inode baselines. The original business scene digest remains +`c438f5bf295f56ddd168e9e840b41281863c3da06c0973eb9abead95edd63e89`. +Four data roots and private state remain in five root-owned mode-0700 +same-device archive parents. No S3 request, backup restoration, namespace +deletion or original verdict rewrite occurred. This preservation result adds +no canonical acceptance; the documented count remains 104/158. + +The retained run proves the caller socket disappeared and its diagnostic read +guard dropped, while the process FD count remained 23 against baseline 22. +It does not identify the extra FD. I05 resource and socket samples now include +bounded descriptor kinds and target fingerprints plus TCP rows joined to the +original process's owned socket inodes. File contents and raw descriptor paths +are excluded. The same original process birth and network namespace must +survive collection; snapshots still make no global no-leak claim. The FD +threshold and thirty-second drain budget are unchanged. A real local HTTP/1.1 +503 integration confirms the actual `Probe.s3` SDK path closes its keepalive +socket without retry, but does not identify the original production FD. Fresh +live descriptor evidence is required before attributing or correcting it. + +Verification: all 70 focused observer/timeout/transport/performance checks +passed in Linux, including actual `/proc` descriptors and network-namespace +refusal. The nine direct socket/census components also passed without network +access in Linux. All 63 retention/runner/migration checks passed on macOS; +catalog 158, matrix 84/302/534 and whitespace checks passed. The macOS focused +suite encountered the previously recorded receive-buffer prerequisite in one +physical-change negative test; that test passed independently and in the full +Linux suite. The prerequisite and failure oracle were not relaxed. diff --git a/scripts/validation_heal/caller_socket.py b/scripts/validation_heal/caller_socket.py index e6d2aee5..808e6028 100644 --- a/scripts/validation_heal/caller_socket.py +++ b/scripts/validation_heal/caller_socket.py @@ -1,5 +1,6 @@ """Read the exact caller socket from one run-owned Linux candidate process.""" +import hashlib import ipaddress import os from pathlib import Path @@ -35,7 +36,7 @@ def proc_address(value, ipv6): return str(ip), int(parts[1], 16) -def connections(raw, ipv6, client, server): +def tcp_rows(raw, ipv6): require(isinstance(raw, str) and len(raw.encode()) <= 4 * 1024**2, 'bounded Linux TCP table required') lines = raw.splitlines() require(lines and 'local_address' in lines[0] and 'inode' in lines[0], 'Linux TCP table header differs') @@ -45,16 +46,64 @@ def connections(raw, ipv6, client, server): require(len(fields) >= 10 and fields[9].isdigit() and re.fullmatch(r'[0-9A-Fa-f]{2}', fields[3]), 'incomplete Linux TCP connection row') local, remote = proc_address(fields[1], ipv6), proc_address(fields[2], ipv6) - if local == server and remote == client: - queues = fields[4].split(':') - require(len(queues) == 2 and all(re.fullmatch(r'[0-9A-Fa-f]+', value) for value in queues), - 'invalid Linux TCP queue counters') - found.append({'inode': int(fields[9]), 'state': fields[3].upper(), - 'txQueueBytes': int(queues[0], 16), 'rxQueueBytes': int(queues[1], 16)}) + queues = fields[4].split(':') + require(len(queues) == 2 and all(re.fullmatch(r'[0-9A-Fa-f]+', value) for value in queues), + 'invalid Linux TCP queue counters') + found.append({'inode': int(fields[9]), 'state': fields[3].upper(), + 'localAddress': list(local), 'remoteAddress': list(remote), + 'txQueueBytes': int(queues[0], 16), 'rxQueueBytes': int(queues[1], 16)}) + return found + + +def connections(raw, ipv6, client, server): + found = [{k: v for k, v in row.items() if k not in {'localAddress', 'remoteAddress'}} + for row in tcp_rows(raw, ipv6) + if tuple(row['localAddress']) == server and tuple(row['remoteAddress']) == client] require(len(found) <= 1, 'ambiguous exact TCP connection in network namespace') return found +def descriptor_census(root): + """Read owned FD links and TCP metadata, never descriptor contents or paths. + + This is a bounded snapshot. A descriptor may disappear or be reused while + it is read; neither a stable count nor these fingerprints proves no leak. + The caller binds process birth and network namespace around this census. + """ + started = time.monotonic_ns() + fds = list((root / 'fd').iterdir()) + require(len(fds) <= 65536, 'owned process descriptor census exceeds its budget') + descriptors, vanished, owned_sockets = [], 0, {} + for fd in sorted(fds, key=lambda entry: int(entry.name)): + try: + target = os.readlink(fd) + except FileNotFoundError: + vanished += 1 + continue + match = re.fullmatch(r'(socket|pipe):\[([0-9]+)\]', target) + kind = match[1] if match else ('anonymous' if target.startswith('anon_inode:') else 'path') + row = {'descriptor': int(fd.name), 'kind': kind, + 'targetSha256': hashlib.sha256(os.fsencode(target)).hexdigest()} + if match: + row['inode'] = int(match[2]) + if kind == 'socket': + owned_sockets.setdefault(row['inode'], []).append(row['descriptor']) + descriptors.append(row) + tcp = [] + for table, ipv6 in (('tcp', False), ('tcp6', True)): + # The namespace table alone does not establish process ownership. + with (root / 'net' / table).open() as stream: + raw = stream.read(4 * 1024**2 + 1) + for row in tcp_rows(raw, ipv6): + if row['inode'] in owned_sockets: + tcp.append({**row, 'descriptors': owned_sockets[row['inode']], 'table': table}) + return {'startedMonotonicNs': started, 'finishedMonotonicNs': time.monotonic_ns(), + 'fileDescriptors': len(fds), 'vanishedDescriptors': vanished, + 'descriptors': descriptors, 'ownedTcpConnections': tcp, + 'descriptorContentsRead': False, 'rawDescriptorPathsIncluded': False, + 'snapshotOnly': True, 'noLeakProven': False} + + def run(request, node): require(set(request) == {'runId', 'action', 'expectedProcess', 'clientAddress', 'serverAddress'} and request['action'] == 'caller-socket', 'caller socket accepts no arbitrary PID or path') @@ -75,32 +124,14 @@ def process(): root = Path(f'/proc/{before["pid"]}') network_namespace = os.readlink(root / 'ns/net') def sample(): - started = time.monotonic_ns() - fds, vanished, owned_sockets = list((root / 'fd').iterdir()), 0, {} - require(len(fds) <= 65536, 'owned process descriptor census exceeds its budget') - for fd in fds: - try: - target = os.readlink(fd) - except FileNotFoundError: - vanished += 1 - continue - match = re.fullmatch(r'socket:\[([0-9]+)\]', target) - if match: - owned_sockets.setdefault(int(match[1]), []).append(int(fd.name)) - matches = [] - for table, ipv6 in (('tcp', False), ('tcp6', True)): - # /proc/PID/net describes the namespace, not process ownership. - # Only a matching descriptor inode establishes the latter. - with (root / 'net' / table).open() as stream: - raw = stream.read(4 * 1024**2 + 1) - for row in connections(raw, ipv6, client, server): - if row['inode'] in owned_sockets: - matches.append({**row, 'descriptors': sorted(owned_sockets[row['inode']]), 'table': table}) + census = descriptor_census(root) + matches = [{k: v for k, v in row.items() if k not in {'localAddress', 'remoteAddress'}} + for row in census['ownedTcpConnections'] + if tuple(row['localAddress']) == server and tuple(row['remoteAddress']) == client] require(len(matches) <= 1, 'caller TCP connection maps to ambiguous owned sockets') process() require(os.readlink(root / 'ns/net') == network_namespace, 'caller socket network namespace changed') - return {'startedMonotonicNs': started, 'finishedMonotonicNs': time.monotonic_ns(), - 'fileDescriptors': len(fds), 'vanishedDescriptors': vanished, 'ownedMatches': matches} + return {**census, 'ownedMatches': matches} samples = [sample(), sample()] process() return {'process': {name: before[name] for name in node['PROCESS_FIELDS']}, 'clockDomain': before['bootId'], diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 641c02ff..78ae2569 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -694,6 +694,12 @@ def process_metrics(request): "threads": int(fields[17]), "fileDescriptors": sum(1 for _ in (root / "fd").iterdir()), "io": {name: io[name] for name in sorted(required_io)}, "readOnly": True, "ioScope": "owned process counters; cached logical IO and storage IO reported separately"} + if owned(request["runId"])[2].get("runtimeProfile") == "lock-timeout": + namespace = os.readlink(root / "ns/net") + result["descriptorCensus"] = caller_socket_backend.descriptor_census(root) + result["fileDescriptors"] = result["descriptorCensus"]["fileDescriptors"] + require(os.readlink(root / "ns/net") == namespace, "resource sample crossed a network namespace change") + result["networkNamespace"] = namespace after = process(request) require(all(after.get(name) == before[name] for name in PROCESS_FIELDS), "resource sample crossed a service identity change") return result diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 593c5d17..7622f39a 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -185,7 +185,7 @@ def remote(self, node, action, **params): params["buckets"] = self.snapshot_buckets value = {"runId": self.run_id, "action": action, **params} source = NODE_SOURCE.read_text() - if action == "caller-socket": + if action in {"caller-socket", "process-metrics"}: source += "\nimport types\ncaller_socket_backend = types.ModuleType('caller_socket')\nexec(" + repr( NODE_SOURCE.with_name("caller_socket.py").read_text()) + ", caller_socket_backend.__dict__)\n" if action in {"mpu-race-backups", "version-race-backups"}: diff --git a/test_validation_heal_lock_observer.py b/test_validation_heal_lock_observer.py index 5a39a5c6..905af1f9 100644 --- a/test_validation_heal_lock_observer.py +++ b/test_validation_heal_lock_observer.py @@ -251,6 +251,36 @@ def node(self): return {'owned': lambda run: (None, None, {'runtimeProfile': 'lock-timeout'}), 'process': lambda request: identity(), 'PROCESS_FIELDS': fleet_node.PROCESS_FIELDS} + def test_descriptor_census_distinguishes_owned_tcp_and_hashes_private_paths_without_opening_them(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'fd').mkdir(); (root / 'net').mkdir() + private = '/private/test-secret/credentials.env' + for fd, target in ((3, 'socket:[678]'), (4, 'pipe:[999]'), + (5, private), (6, 'anon_inode:[eventpoll]')): + (root / f'fd/{fd}').symlink_to(target) + (root / 'net/tcp').write_text(HEADER + + '\n0: 0100007F:2328 0100007F:9859 01 0:0 00:0 0 1000 0 678' + + '\n1: 0100007F:2328 0100007F:985A 08 0:0 00:0 0 1000 0 777\n') + (root / 'net/tcp6').write_text(HEADER + '\n') + proof = sockets.descriptor_census(root) + self.assertEqual(proof['fileDescriptors'], 4) + self.assertEqual([row['kind'] for row in proof['descriptors']], ['socket', 'pipe', 'path', 'anonymous']) + self.assertEqual(proof['descriptors'][2]['targetSha256'], hashlib.sha256(os.fsencode(private)).hexdigest()) + self.assertNotIn(private, json.dumps(proof)) + self.assertEqual(len(proof['ownedTcpConnections']), 1) + self.assertEqual(proof['ownedTcpConnections'][0]['descriptors'], [3]) + self.assertFalse(proof['descriptorContentsRead'] or proof['rawDescriptorPathsIncluded'] or proof['noLeakProven']) + readlink = os.readlink + def vanish(path): + if Path(path).name == '4': raise FileNotFoundError('closed during census') + return readlink(path) + with patch.object(sockets.os, 'readlink', side_effect=vanish): + changed = sockets.descriptor_census(root) + self.assertEqual(changed['vanishedDescriptors'], 1) + self.assertEqual(changed['fileDescriptors'], 4) + self.assertEqual(len(changed['descriptors']), 3) + def test_frozen_linux_tables_include_zero_port_listeners_and_mapped_ipv6(self): ipv4 = HEADER + '\n 0: 0100007F:2328 0100007F:9859 01 0:0 00:0 0 1000 0 678\n' ipv4 += ' 1: 00000000:2328 00000000:0000 0A 0:0 00:0 0 1000 0 123\n' @@ -305,6 +335,43 @@ def execute(*args, **kwargs): self.assertEqual(fleet.remote('vm000', 'caller-socket', **{k: v for k, v in self.request().items() if k not in {'runId', 'action'}}), {}) + def test_fleet_metrics_embed_the_same_descriptor_backend(self): + def execute(*args, **kwargs): + source = kwargs['input'].rsplit('\nmain(', 1)[0] + scope = {'__name__': '__i05_metrics_test__'} + exec(compile(source, 'i05-metrics-node', 'exec'), scope) + self.assertTrue(callable(scope['caller_socket_backend'].descriptor_census)) + return SimpleNamespace(returncode=0, stdout='{}', stderr='') + with tempfile.TemporaryDirectory() as tmp, \ + patch('scripts.validation_heal.healthy.subprocess.run', side_effect=execute): + from scripts.validation_heal.healthy import Fleet + fleet = Fleet(['vm000'], Path(tmp), 'gh-12345-1-h1', runtime_profile='lock-timeout') + self.assertEqual(fleet.remote('vm000', 'process-metrics'), {}) + + @unittest.skipUnless(sys.platform == 'linux', 'requires actual Linux /proc descriptor tables') + def test_metrics_keep_same_process_fd_fingerprints_for_live_file_and_socket(self): + actual = {**identity(), 'pid': os.getpid(), + 'startTicks': int(Path('/proc/self/stat').read_text().rpartition(') ')[2].split()[19])} + with tempfile.TemporaryFile() as file, socket.socket() as sock, \ + patch.object(fleet_node, 'process', return_value=actual), \ + patch.object(fleet_node, 'owned', return_value=(None, None, {'runtimeProfile': 'lock-timeout'})), \ + patch.object(fleet_node, 'caller_socket_backend', sockets, create=True): + proof = fleet_node.process_metrics({'runId': 'gh-12345-1-h1', 'action': 'process-metrics'}) + census = proof['descriptorCensus'] + by_fd = {row['descriptor']: row for row in census['descriptors']} + self.assertEqual(by_fd[file.fileno()]['kind'], 'path') + self.assertEqual(by_fd[sock.fileno()]['kind'], 'socket') + self.assertEqual(proof['networkNamespace'], os.readlink('/proc/self/ns/net')) + self.assertEqual(proof['fileDescriptors'], census['fileDescriptors']) + self.assertTrue(census['snapshotOnly']) + with patch.object(fleet_node, 'process', side_effect=[actual, {**actual, 'startTicks': 1}]): + with self.assertRaisesRegex(ValueError, 'identity change'): + fleet_node.process_metrics({'runId': 'gh-12345-1-h1', 'action': 'process-metrics'}) + with patch.object(fleet_node.os, 'readlink', side_effect=['net:[1]', 'net:[2]']), \ + patch.object(sockets, 'descriptor_census', return_value=census): + with self.assertRaisesRegex(ValueError, 'network namespace change'): + fleet_node.process_metrics({'runId': 'gh-12345-1-h1', 'action': 'process-metrics'}) + @unittest.skipUnless(sys.platform == 'linux', 'requires actual Linux /proc TCP tables') def test_actual_linux_backpressured_response_has_an_owned_nonempty_send_queue(self): with socket.socket() as listener, socket.socket() as client: diff --git a/test_validation_heal_lock_timeout.py b/test_validation_heal_lock_timeout.py index 9ba6d34f..49534a00 100644 --- a/test_validation_heal_lock_timeout.py +++ b/test_validation_heal_lock_timeout.py @@ -1,7 +1,9 @@ """I05 timeout controller: real caller socket, durable PUT receipts, honest scope.""" import copy +import http.server import json +import threading import time import unittest from unittest.mock import patch @@ -129,6 +131,45 @@ def test_bucket_incarnation_read_lock_does_not_mask_target_write_timeout(self): self.assertTrue(proof['diagnosticComplete']) self.assertTrue(proof['callerObservation']['admittedWork']['typedTimeoutBeyondBudgetObserved']) + def test_actual_probe_closes_known_503_keepalive_socket_without_retry(self): + from scripts.validation_heal.heal_probe import Probe + closed, received = threading.Event(), [] + class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + def log_message(self, *args): pass + def do_PUT(self): + received.append(self.rfile.read(int(self.headers['Content-Length']))) + body = b'ServiceUnavailableone-refusal' + self.send_response(503) + self.send_header('Content-Length', str(len(body))) + self.send_header('Content-Type', 'application/xml') + self.send_header('x-amz-request-id', 'one-refusal') + self.send_header('Retry-After', '1') + self.end_headers() + self.wfile.write(body); self.wfile.flush() + self.connection.settimeout(3) + try: + if self.rfile.read(1) == b'': closed.set() + except TimeoutError: pass + self.close_connection = True + server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + binding = copy.deepcopy(self.probe.binding) + binding['endpoints'] = {'node1': f'http://127.0.0.1:{server.server_port}'} + probe = Probe(binding, credential_pair=('test-access', 'test-secret')) + result = probe.s3('put_object', {'Bucket': fixture.BUCKET, 'Key': self.entry['key'], + 'Metadata': {'heal-fixture': 'I05-timeout-contender'}}, b'one-refused-body') + self.assertFalse(result['acknowledged'] or result['resultUnknown']) + self.assertEqual(result['response']['ResponseMetadata']['HTTPStatusCode'], 503) + self.assertEqual(result['response']['ResponseMetadata']['RetryAttempts'], 0) + self.assertNotIn('clientCleanupError', result) + self.assertTrue(closed.wait(4), 'actual SDK close must reach the server socket') + self.assertEqual(received, [b'one-refused-body']) + finally: + server.shutdown(); server.server_close(); thread.join() + def test_an_additional_write_lock_attempt_is_rejected(self): self.change = 'extra-write-lock' with self.assertRaisesRegex(ValueError, 'another namespace write-lock attempt'): diff --git a/test_validation_heal_performance.py b/test_validation_heal_performance.py index a01b4f50..42190e60 100644 --- a/test_validation_heal_performance.py +++ b/test_validation_heal_performance.py @@ -181,6 +181,7 @@ def __truediv__(self, name): return Proc(name) def read_text(self): return proc[self.name] def iterdir(self): return iter(range(7)) with patch.object(node, "Path", return_value=Proc()), patch.object(node, "process", return_value=identity), \ + patch.object(node, "owned", return_value=(None, None, {"runtimeProfile": "manual"})), \ patch.object(node.os, "sysconf", side_effect=lambda name: 100 if name == "SC_CLK_TCK" else 4096): value = node.process_metrics({"runId": "gh-1-1-h2", "action": "process-metrics"}) self.assertEqual(value["cpuTicks"], 150) From f43151595648d83b184ab95fd3e908e7611d0dae Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 17:42:10 +0800 Subject: [PATCH 291/343] fix(validation): preserve I05 descriptor drain failure --- docs/validation-heal-operations.md | 22 ++++ docs/validation-migration-map.md | 10 ++ scripts/validation_heal/timeout_retention.py | 106 ++++++++++++++++-- test_validation_heal_timeout_retention.py | 75 ++++++++++++- .../i05-descriptor-drain-39754893467.zip | Bin 0 -> 176594 bytes 5 files changed, 202 insertions(+), 11 deletions(-) create mode 100644 testdata/validation/i05-descriptor-drain-39754893467.zip diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index b358e232..5d8d7db4 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -835,3 +835,25 @@ seven-millisecond HTTP completion and a later `get_object_stream_body` `dropped_incomplete` event. This is a source-supported hypothesis for a controlled streaming/IO reproduction, not a confirmed cause or a canonical acceptance result. + +I05 descriptor run `39754893467/1` pins helper +`dd7e54e6672cab76b5a6245ab3b4fbd051ddd39d` and the unchanged package from +`39364893106/1`. H1 retains its original FAIL/cleanup FAIL at the unchanged +30-second drain budget; H2 did not start. The baseline owns 22 descriptors. +Every drain resource snapshot owns 23, with all baseline fingerprints intact +and exactly one additional TCP descriptor from remote `168.63.129.16` to local +port 9000. The caller's exact GET socket is absent, and its read-lock journal +contains `guard_dropped`. These snapshots establish external connection +interference; they do not prove the platform probe configuration, continuous +socket absence, no leak, or complete canonical acceptance. + +Preservation recognizes only this additional exact helper and its original +known 503 refusal, closed read-only GET, complete journal lifecycle, stopped +physical scene and bounded request/ACK set. The original 101-file artifact is +retained in `testdata/validation/i05-descriptor-drain-39754893467.zip`. +Descriptor counts, unique FD identities, socket-link fingerprints, owned TCP +joins, snapshot flags, process birth and network namespace must agree. +Missing census, unknown/successful mutation, extra request and physical drift +refuse preservation. Drain journal windows follow the numbered first-drain +label rather than a fixed count of preceding caller windows. This change +permits preservation only; it performs no network change and grants no PASS. diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 1b515e28..b591052c 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -2468,3 +2468,13 @@ catalog 158, matrix 84/302/534 and whitespace checks passed. The macOS focused suite encountered the previously recorded receive-buffer prerequisite in one physical-change negative test; that test passed independently and in the full Linux suite. The prerequisite and failure oracle were not relaxed. + +I05 descriptor diagnosis `39754893467/1` retains FAIL/cleanup FAIL under exact +helper `dd7e54e6672cab76b5a6245ab3b4fbd051ddd39d`; H2 was not started. Complete +FD snapshots attribute the 22-to-23 increase to an external TCP connection +from `168.63.129.16` while the intentional GET socket disappears and its read +lock drops. The concrete platform probe configuration remains unverified. +The preservation planner now validates this helper's complete census and +original bounded refusal, caller, journal and physical evidence. The real +101-file fixture and corrupt-evidence regressions exercise preservation +without S3 access, restoration, verdict change or canonical acceptance. diff --git a/scripts/validation_heal/timeout_retention.py b/scripts/validation_heal/timeout_retention.py index 8ebc0901..6910b660 100644 --- a/scripts/validation_heal/timeout_retention.py +++ b/scripts/validation_heal/timeout_retention.py @@ -1,6 +1,7 @@ """Isolate archived I05 H1/H2 failures with bounded, understood request effects.""" import hashlib +import ipaddress import json import re import uuid @@ -30,6 +31,8 @@ CONTENDER_HELPER = 'd100abeb8e85c4000a493aa994031e4a5d01518e' H2_CONTENDER_HELPER = '1b88cad3581e755d8e904697cb2ec471e04b5f8f' DRAIN_BUDGET_HELPER = '0442d4dd814ebfb490dcbb3cc266fb6fbebb0365' +DESCRIPTOR_HELPER = 'dd7e54e6672cab76b5a6245ab3b4fbd051ddd39d' +DRAIN_HELPERS = {DRAIN_BUDGET_HELPER, DESCRIPTOR_HELPER} FAILURES = { PRE_CONTENDER_HELPER: {'type': 'ValueError', 'message': 'caller lock diagnostic escaped its exact GET identity or runtime budget'}, @@ -39,6 +42,8 @@ 'message': 'I05 contender acquired a lock or lost its exact timeout lifecycle'}, DRAIN_BUDGET_HELPER: {'type': 'ValueError', 'message': 'NOT_OBSERVED: caller drain observation exceeded its budget'}, + DESCRIPTOR_HELPER: {'type': 'ValueError', + 'message': 'NOT_OBSERVED: caller drain observation exceeded its budget'}, } COMMON_LABELS = ('binding', 'inspector-build', 'preflight', 'integrity-fleet-attestation', 'reserved', 'inspector-upload', 'started', 'deployed-topology', 'actual-topology', 'original-ACK-bound-S3', @@ -97,6 +102,59 @@ def drain_failure_labels(folder): return prefix + caller_windows + rounds + suffix +def validate_census(value): + """Keep the reviewed helper's complete snapshots without inferring no leak.""" + require(isinstance(value, dict) + and value.get('descriptorContentsRead') is False + and value.get('rawDescriptorPathsIncluded') is False + and value.get('snapshotOnly') is True and value.get('noLeakProven') is False + and type(value.get('startedMonotonicNs')) is int + and type(value.get('finishedMonotonicNs')) is int + and 0 < value['startedMonotonicNs'] <= value['finishedMonotonicNs'] + and type(value.get('fileDescriptors')) is int + and 0 <= value['fileDescriptors'] <= 65536 + and value.get('vanishedDescriptors') == 0 + and isinstance(value.get('descriptors'), list) + and len(value['descriptors']) == value['fileDescriptors'] + and isinstance(value.get('ownedTcpConnections'), list), + 'I05 descriptor census is incomplete or claims more than a snapshot') + descriptors, sockets = {}, {} + for row in value['descriptors']: + fd, kind = row.get('descriptor'), row.get('kind') + require(type(fd) is int and fd >= 0 and fd not in descriptors + and kind in {'socket', 'pipe', 'path', 'anonymous'} + and re.fullmatch(r'[0-9a-f]{64}', row.get('targetSha256', '')) + and set(row) == {'descriptor', 'kind', 'targetSha256'} | ( + {'inode'} if kind in {'socket', 'pipe'} else set()), + 'I05 descriptor identity is missing, duplicated or contains an unreviewed field') + descriptors[fd] = row + if kind in {'socket', 'pipe'}: + inode = row.get('inode') + require(type(inode) is int and inode > 0 + and row['targetSha256'] == hashlib.sha256(f'{kind}:[{inode}]'.encode()).hexdigest(), + 'I05 descriptor inode does not match its link fingerprint') + if kind == 'socket': + sockets.setdefault(inode, []).append(fd) + seen = set() + for row in value['ownedTcpConnections']: + inode = row.get('inode') + require(type(inode) is int and inode in sockets and inode not in seen + and row.get('descriptors') == sockets[inode] + and row.get('table') in {'tcp', 'tcp6'} + and re.fullmatch(r'[0-9A-F]{2}', row.get('state', '')) + and all(type(row.get(name)) is int and row[name] >= 0 + for name in ('txQueueBytes', 'rxQueueBytes')), + 'I05 TCP census is not joined to the original owned descriptors') + seen.add(inode) + for name in ('localAddress', 'remoteAddress'): + endpoint = row.get(name) + require(isinstance(endpoint, list) and len(endpoint) == 2 + and isinstance(endpoint[0], str) and type(endpoint[1]) is int + and 0 <= endpoint[1] <= 65535, + 'I05 TCP census has an invalid numeric endpoint') + ipaddress.ip_address(endpoint[0]) + + def archived_plan(artifact, failed_run, candidate): """Refuse uncertain effects, later actions, mixed cases and physical drift. @@ -143,7 +201,7 @@ def read(path): and h1_result.get('canonicalCasePassClaimed') is False, 'I05 H1 diagnostic result does not prove completed cleanup') require(diagnostics[1] == row, 'I05 H2 failure differs from its exact known boundary') - elif helper == DRAIN_BUDGET_HELPER: + elif helper in DRAIN_HELPERS: topology = 'H1' require(diagnostics == [row], 'I05 drain failure differs from its exact H1 boundary') else: @@ -174,7 +232,7 @@ def read(path): and first_quiescent - last_admission == 4 * window_rounds[2], 'I05 H2 lock windows do not contain complete four-peer sampling rounds') labels = h2_labels(window_rounds) - elif helper == DRAIN_BUDGET_HELPER: + elif helper in DRAIN_HELPERS: labels = drain_failure_labels(folder) else: labels = LABELS[helper] @@ -348,18 +406,18 @@ def many(label): timeout = folder / 'timeout' expected_timeout_items = {'scope.json', 'failure.json', 'caller'} expected_caller_items = {'scope.json', 'failure.json', 'transport'} - has_contender = helper in (CONTENDER_HELPER, H2_CONTENDER_HELPER, DRAIN_BUDGET_HELPER) + has_contender = helper in {CONTENDER_HELPER, H2_CONTENDER_HELPER} | DRAIN_HELPERS if has_contender: expected_timeout_items.add('contender') expected_caller_items.add('admission.json') - if helper == DRAIN_BUDGET_HELPER: + if helper in DRAIN_HELPERS: expected_timeout_items.add('timeout.json') require({p.name for p in timeout.iterdir()} == expected_timeout_items and {p.name for p in (timeout / 'caller').iterdir()} == expected_caller_items, 'I05 archive contains an unreviewed action or lacks its exact failure boundary') timeout_failure = read(timeout / 'failure.json') caller_failure = read(timeout / 'caller/failure.json') - expected_failure_state = 'NOT_OBSERVED' if helper == DRAIN_BUDGET_HELPER else 'INCOMPLETE' + expected_failure_state = 'NOT_OBSERVED' if helper in DRAIN_HELPERS else 'INCOMPLETE' require(timeout_failure.get('diagnosticComplete') is False and timeout_failure.get('errorType') == failure['type'] and timeout_failure.get('state') == expected_failure_state @@ -408,7 +466,7 @@ def many(label): and value.get('resultUnknown') is True and value.get('acknowledged') is False and value.get('serverOwnershipProven') is False, 'I05 closed caller was retried or promoted to an acknowledged result') - if helper == DRAIN_BUDGET_HELPER: + if helper in DRAIN_HELPERS: caller_scope = read(timeout / 'caller/scope.json') require(caller_scope.get('runId') == run_id and caller_scope.get('candidateCommit') == CANDIDATE and caller_scope.get('bindingSha256') == scope['bindingSha256'] @@ -473,7 +531,7 @@ def many(label): require(proof is not None and proof.get('typedTimeoutBeyondBudgetObserved') is True and proof.get('writeLockAcquired') is False, 'I05 contender ACK lacks its exact failed target write-lock attempt') - if helper == DRAIN_BUDGET_HELPER: + if helper in DRAIN_HELPERS: caller_request_id = holder.get('request_id') caller_lock_id = holder.get('lock_attempt_id') caller_events = holder.get('events') or [] @@ -566,7 +624,10 @@ def many(label): baseline = baselines.get('vm000') drain_sockets = many('I05-caller-drain-socket') drain_resources = many('I05-caller-drain-resources') - drain_windows = windows[5:] + first_drain = next(index for index, (label, _) in enumerate(numbered) + if label == 'I05-caller-drain-socket') + drain_windows = [read(folder / name) for label, name in numbered[first_drain:] + if label == 'I05-caller-window'] require(baseline is not None and len(drain_sockets) == len(drain_resources) >= 1 and len(drain_windows) in {len(drain_sockets), len(drain_sockets) - 1}, 'I05 drain archive lacks a complete service and socket sample pair') @@ -594,6 +655,35 @@ def same_socket_process(value): and resource.get('identity', {}).get('settings') == baseline['identity'].get('settings'), 'I05 drain observation contains a changed process, socket or resource sample') same_process(resource.get('identity'), baseline['identity']) + if helper == DESCRIPTOR_HELPER: + resource_states = [baseline, one('I05-caller-admission-resources'), *drain_resources] + for resource in resource_states: + same_process(resource.get('identity'), baseline['identity']) + census = resource.get('descriptorCensus') + validate_census(census) + require(resource.get('node') == 'vm000' and resource.get('readOnly') is True + and resource.get('clockDomain') == baseline['identity']['bootId'] + and resource.get('networkNamespace') == caller_socket.get('networkNamespace') + and resource.get('fileDescriptors') == census['fileDescriptors'] + and resource['identity'].get('settings') == { + **runtime_config('lock-timeout'), 'RUSTFS_HEAL_TASK_TIMEOUT_SECS': '300'}, + 'I05 resource census crossed its original process or network namespace') + for socket_state in [one('I05-caller-before-disconnect-socket'), backpressure, *drain_sockets]: + same_socket_process(socket_state) + require(socket_state.get('networkNamespace') == baseline.get('networkNamespace') + and socket_state.get('clockDomain') == baseline['identity']['bootId'] + and socket_state.get('clientAddress') == sent.get('localAddress') + and socket_state.get('serverAddress') == sent.get('peerAddress'), + 'I05 socket census crossed its original caller or network namespace') + for sample in socket_state.get('samples') or []: + validate_census(sample) + matches = [{k: v for k, v in row.items() + if k not in {'localAddress', 'remoteAddress'}} + for row in sample['ownedTcpConnections'] + if row['localAddress'] == socket_state['serverAddress'] + and row['remoteAddress'] == socket_state['clientAddress']] + require(sample.get('ownedMatches') == matches, + 'I05 caller socket census lost its exact owned TCP join') write_rows = [event for event in rows['vm000'] if event.get('mode') == 'write'] require(write_rows == proof['events'] and [event.get('state') for event in write_rows] == ['acquiring', 'failed'], diff --git a/test_validation_heal_timeout_retention.py b/test_validation_heal_timeout_retention.py index 5922d82f..c0b324b1 100644 --- a/test_validation_heal_timeout_retention.py +++ b/test_validation_heal_timeout_retention.py @@ -20,6 +20,7 @@ FIXTURE = Path(__file__).parent / 'testdata/validation/i05-pre-contender-39711742431.zip' DRAIN_FIXTURE = Path(__file__).parent / 'testdata/validation/i05-drain-budget-39746002648.zip' +DESCRIPTOR_FIXTURE = Path(__file__).parent / 'testdata/validation/i05-descriptor-drain-39754893467.zip' class TimeoutRetentionTests(unittest.TestCase): @@ -32,11 +33,11 @@ def archive(self, directory): folder = artifact / 'run' / controller.CASE / 'evidence' / controller.CASE / 'H1' return root, candidate, folder - def drain_archive(self, directory): + def drain_archive(self, directory, run=39746002648): root = Path(directory) / 'drain-artifact' - with zipfile.ZipFile(DRAIN_FIXTURE) as archive: + with zipfile.ZipFile(DRAIN_FIXTURE if run == 39746002648 else DESCRIPTOR_FIXTURE) as archive: archive.extractall(root) - artifact = root / 'rustfs-validation-39746002648-1' + artifact = root / f'rustfs-validation-{run}-1' candidate = json.loads((artifact / 'candidate.json').read_text()) folder = artifact / 'run' / controller.CASE / 'evidence' / controller.CASE / 'H1' return root, candidate, folder @@ -420,6 +421,74 @@ def test_drain_budget_failure_yields_exact_bounded_preservation_plan(self): self.assertEqual(plan['businessSceneSha256'], {'vm000': 'c438f5bf295f56ddd168e9e840b41281863c3da06c0973eb9abead95edd63e89'}) + def test_descriptor_failure_preserves_exact_scene_without_leak_or_pass_claim(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, folder = self.drain_archive(directory, 39754893467) + plan, = recovery.recovery_plan(root, 39754893467, candidate) + self.assertEqual(plan['harnessCommit'], controller.DESCRIPTOR_HELPER) + self.assertEqual(plan['kind'], 'unresolved-i05-timeout') + self.assertEqual(plan['ackFilesSha256'], + 'f114c1f2f4d8ddbbc5c98549e1e32f8da61241b4ccf3123bf89f9f3aed88879d') + self.assertEqual(plan['businessSceneSha256'], {'vm000': + 'f921b69ea51622f89e9ad4a78de8c890caf0b7ffefa0b401457d6f9796fc7d31'}) + result = json.loads((folder.parents[2] / 'cases' / f'{controller.CASE}.json').read_text()) + self.assertEqual((result['verdict'], result['cleanup_status']), ('FAIL', 'FAIL')) + + def test_drain_window_boundary_follows_labels_after_variable_caller_wait(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, folder = self.drain_archive(directory, 39754893467) + empty = folder / '00023-I05-caller-window.json' + value = json.loads(empty.read_text()) + self.assertEqual(value['journal'], '') + self.assertEqual(value['beforeCursor'], value['afterCursor']) + empty.unlink() + for path in sorted(folder.glob('*.json')): + index = int(path.name[:5]) + if index > 23: + path.rename(path.with_name(f'{index - 1:05d}' + path.name[5:])) + plan, = recovery.recovery_plan(root, 39754893467, candidate) + self.assertEqual(plan['runId'], 'gh-39754893467-1-h1') + + def test_descriptor_failure_refuses_corrupt_census_and_unbounded_effects(self): + variants = ('missing-census', 'count', 'inode', 'duplicate-fd', 'foreign-tcp', 'tcp-join', + 'namespace', 'process', 'leak-claim', 'raw-path', 'vanished', 'socket-match', + 'success', 'unknown', 'extra-action', 'physical-drift') + for variant in variants: + with self.subTest(variant=variant), tempfile.TemporaryDirectory() as directory: + root, candidate, folder = self.drain_archive(directory, 39754893467) + path = next(folder.glob('*-I05-caller-drain-resources.json')) + value = json.loads(path.read_text()) + census = value['descriptorCensus'] + if variant == 'missing-census': value.pop('descriptorCensus') + elif variant == 'count': census['fileDescriptors'] += 1 + elif variant == 'inode': next(row for row in census['descriptors'] if row['kind'] == 'socket')['inode'] += 1 + elif variant == 'duplicate-fd': census['descriptors'][-1]['descriptor'] = 0 + elif variant == 'foreign-tcp': census['ownedTcpConnections'][0]['inode'] = 999999999 + elif variant == 'tcp-join': census['ownedTcpConnections'][0]['descriptors'] = [0] + elif variant == 'namespace': value['networkNamespace'] = 'net:[1]' + elif variant == 'process': value['identity']['startTicks'] += 1 + elif variant == 'leak-claim': census['noLeakProven'] = True + elif variant == 'raw-path': census['descriptors'][0]['target'] = '/private/path' + elif variant == 'vanished': census['vanishedDescriptors'] = 1 + elif variant == 'socket-match': + path = next(folder.glob('*-I05-caller-drain-socket.json')) + value = json.loads(path.read_text()) + value['samples'][0]['ownedTcpConnections'][0]['descriptors'] = [0] + elif variant in ('success', 'unknown'): + path = folder / 'timeout/contender/000001-ack.json' + value = json.loads(path.read_text()) + if variant == 'success': value['observation']['response']['ResponseMetadata']['HTTPStatusCode'] = 200 + else: value['observation']['resultUnknown'] = True + elif variant == 'extra-action': (folder / 'timeout/contender/000002-request.json').write_text('{}') + else: + path = next(folder.glob('*-original-failure-scene.json')) + value = json.loads(path.read_text()) + value['vm000']['formats'][0]['sha256'] = '0' * 64 + path.write_text(json.dumps(value)) + with patch.object(recovery, 'Fleet') as fleet, self.assertRaises((ValueError, KeyError)): + recovery.recovery_plan(root, 39754893467, candidate) + fleet.assert_not_called() + def test_drain_budget_failure_refuses_unknown_mutation_or_incomplete_sampling(self): for variant in ('success', 'unknown', 'extra-action', 'missing-sample', 'physical-drift'): with self.subTest(variant=variant), tempfile.TemporaryDirectory() as directory: diff --git a/testdata/validation/i05-descriptor-drain-39754893467.zip b/testdata/validation/i05-descriptor-drain-39754893467.zip new file mode 100644 index 0000000000000000000000000000000000000000..79a6302c8b3ae8295cf40f07c51cb689c66436ca GIT binary patch literal 176594 zcmeFZWl&_>vbKxUxHj(ExVuB+?yiNqJB_Sab5YQJOxNsG17Rzi%OJE=%N>Csmmd|JHog5sE z9cY~OEzJz|9nEa4X_z?JSeV&4n3!4FXc*`K`qqa3dc}yw!N$oRU_@)~U}Ie%KWx21 zkJ#}*1(IPAj&uGTm2Qs|v`dec+mBd4COqX$XV%tGB@pG=+ccIo39(=$hSA6iUlrd} z=wW^hC2%w$X*{g)6H7`ejBB4?QWdd2Q;i-`>kDEJV+19~CMp;ScqluB-~Pd@wpWO7 z3aA*o2-o?@*hMeKOM3zzszNNDU~G{MOqEJEZRsH^p2_^b>-VmU(5$9 zu{YlJX*Qlu6Vm?QnD0-!{T!F+rAG|7;1=Fv3iJMb@BYF#?`}04GNn}03x0KuSnLX1 z$&{KRdy{icUG=I!_clWN-HEp(Y#h|5f4~<|#^OdyGl-#$j3%y>A~B*?;z~WSpcTfn zn-A>HeEf8~&}uU(?}odxKB6CLzh)(=GU4 zhhkz*d`K1q2&e-L2*~k&F%8uU8z)DaPxqz)=v!JE+0*D7 z*w{OMI=q9C!~fzB{~S=I(SJ>Z_5&)gEQ`2p#gU36BJgKOVSVc1f^m_m*mz*#p{}G@ zgFc?G($ULnDNE{S*Lyjo&cm5C!8_mp=$&eh@?%-aMZsSH+dU5#kwf@3)2s)MTlNVz z1pgJyL(T{fdcauQ*Vq9L4eml%HyIe)usmTl(N$3}m$lMo+pwSEbK3ey2)x-Vl+a_r z@JRt@`|;rtJduDVBgqOFR=Op+tgas`evLvH`$tjNJgSGZ?A?0k14+p0YjCveL$2w7 zCigKniRgglbHwm`SxByXFavlzOBq9$Sg2E`bov+=N0`pVZzY6C>-MCni3>_GsoD2_ z4muVH$nm09tOE?IlshdL>5W}@`0?Yd=%`uDWF^Vj z4=RTpR(&7W4&uZxPGCH45YaMHS8y#1*_49vt8)DyXha>0?25c~DDsHC0iQ}z^kb3N zMkJ$nFSU>4#AQ?Ua6vr{X_%n-MMg@^d4;J`t9(=~RyMx~ir_8B!Oe6L3OR~#YK#gg zyNLj>-pI=Vd3CM>*Ar{lX*}|w18|9bRPif`+>X)CHsf`yfp}yQ57KZ_37!gJ%p&0n zmatUN0tJ{dY|8c~r(Par6?A>NuPa|C7M%7jk#VeH9=uRmg@-nxLrKPNl+0zCXg|Q) zrB3q}o2Z_@c~T4?obZl&>E0h*G9^6ZPra&{C}jH7cha>LDX3bN$qPteBr!=St6!L7 zIItwBvA};#l^3PpJ<1|IfjV zoXrf4tN}*<3m>NwW1yq|yfM%in0U@+`52=YFvz0UAn% zK>m)maqv{IHuN5eRPgn|Jx-$HaC&2w>HcN@?#^7Q!tyBz7SME}dSY?sBoN%c#r!)l0pS=~Sisl<6DFujS*0ej6;{uU^1vpjyw{>Fb8>Sow~F{VX`CjjYVHOc*w` zAr7Xo9mNHcFWL*TfP_xfY{56$aED2?;8xlU}6)P9xVFi-1sS^xP0VxqGC@f1`;AE}K-?=sK$Wzx-qwMbu2T}$# z@cf*1ByN^jn!kV3a0r4&GK4`d1Q2Z_DTGdNoS_u$ z`r8ii#s);hvq+4o3r&?^2>QuBHEB=w9@(@z{fvP@*C!68MjwO#0B7*OrTyb(EOgW)~!VesHw=6C}C}4!qjZAj!Q$VsOt)Ep) zTclP**P_YGdkc9~v}WrbQ58E8ZFB3rduA3JSUce6mwh`wIB)YGE_jbR4i63lBn}M( z^z(0ZK}H%gYX@5+fTN8)je(PyrQsj0*rBpwjVg-hbwDNZ0)}Mq@Mf^tgTYn1&A%PU ztzO$&R^~!JHbL#cfY{enWbBN-LMXufT`ckUXI>$rd{}Z=WpPc|GPrVbO@HARVWZV7Uj5E}1QS3&jf@zo+qimFzVa|g>R!WOO5xem zzm}HryyD?AW#q8y^%~nWY07MH{JKUcxWGmu^Bwnoq2zQOud)UKAmf~`ZcFD1;mc|d zjO>Jbiw)#+o0B5Ld4f~Ust&e|41sFiqcti5veZTz|K! zZMEJ)>Q;EU>Ai`f%&@JEqRgP|Nm{MdF^hcUyLP8~JFlh|olaLP_xscB^X|55O*5~z zo89~M?ekNAS)8u%gUkgCSamsGlss-!ZL-!e9*}9$Yn{^a%x&I%IFWB?@f#> zjY5hs?^t+2Egft6vlM|u8eN|nMJjZxCXP61>npofsH## zQSh^NM+WlZ>y14Qje>$|4RCxyyh(3Qm;PF-ZgYbalPM*X--?oM(Jl)M*p<4%E> z!ko13EfqCwiFOuE82m~a^1yP@nqTdVjfnxyNNLwBqwx-!(S9R}I3D*Cf5vb6ufuRV z>Ens{ND1LXktp!(#$4^S@MnW7YAHMX>ajAeDJ;_7SPuXtEO2R9GZs)GevSM9DZM$V zzVFjO=2*MVY%Qa$E1St9FS76%)K21~Oz2^xconhofckvs5S)1V9_*9kYu@ibLfaAC zP~pG0&=QvY9;6j}DC6(q#U;>DU%OvGb-3qlk8PDWY6+lHIZPaA@wMky2%^J=A4`KM zRBx0IRhR(}qK@$z4i_{Xo&nNh5b@0LF`2w%m@j*aix1J~o4WI5g(uNr2oqY^EcnM( z2Zf1@p2>74TVaJ>fjN0Xnrv0a;o*EgXq>gvvv+~V_lqG&Fah!W-*^r^~dR#vBZ2VQbc{;M^-agX@HwYl0u)kHinP_b7jf^eLOiUgB zQ0@K%0h=^>)WM4LEK-Gp`r~|i z;Fpe%y7fu#)8Kh`+;N@)=H+ty*KPz&z*@HZEz>l+`hpr(=C}1h4<@*Y=fekj9ehW^ z{kdl0rf+c?90{j)M*>(18imts4fTaNCk#nj2zPMM9JWzu?rj6eE_okx!F06fW94k281 zG(`jJs9U4QU9P=LRj%}1^<+}Yu}sBZ={7;2-=n^5Qih!>f8&oQ zDwB}Ff{)W#Yfwqxt3R)q6Iz7kG)!d!||8&3|Ag+mBO zqbS&FCighjO0pRh6E#+dR!m}4*BVA4+!<&IRr2%bXS1cV{=~>*>Pj>w?#lQ7$Z^ru4T-Wo zofP6T$4&oRos{`A!EiJ(u{U#cqcOHLGWuK^IyxFTIQ{`e{#@)Sa7EA2Bc4B~M$BmS zQeKSHsuCi$LK7m&5rl>EtgEp*@7mR(6TWlBt$`9!=37v&>|8Ls?+?Ng0KSrq=m;i4 zj3SzN57G;_=@Gepy9tIKI-yX@@}V%IV}<2>zys8d$RwBZb-|=QqRd*{E}IKL>xMCD z<3IccXz{_TH3Ul+@u0Q}_y<*!%{Q3BtFd)ZP%0~lD&m%PnpQJt2ltDki?FpGDX%>& zCRnZESfjm?Q8?=E5eynphY`iSHCZ@8S>t_F9&AJ3usbkn?Stm5&>gZ?u|B?XpPnc# zunxvuy1dx)#jAEJALBdTQ9Hg-JKlU8^64@xi_TaJ`=$X1C4>VzcBv)UsTI8BSJ2fb3Y&_9iUE3q#-5`DkzvGtMg6M1zC6e* z`iyMVT1b>j7$cG2<1QT+fnf#8syCNQAH^DJfAxLyFv8qR2*Mrp370~5H-B>?AegXC2{0l=UA{3;&t9O8M%5|^f*r! zJBqs{BAx?>-azzqNxAM(o`=_Gp>?BZl(Z||2j<_IKC zPprVi1H_9N)ImZ~@A&&;J10_BLGMtbcdN{=jUFHRcEXJLB`(cLo9&Sf!OYhHVT^&c zwA@GBpq*}Jmc56VtIZPhK7zgP?10cpQu1XY)6o{WeZvv>%852IS{1T!4z8$zH1G+z zANS6)fHZxlX0yb#;{HPEBxmT}Kw4d)8%R3#w)MVS3kl+K{6V+H_OIX`pLcXV@gU~ha5}E~xJ_tEqz<@TN@QFuCp@b^ z)|1+n589_z0TZA-Yrfm=GG}vJc1T<_YAf46C^J_~j*S^!iJemS@NaJrp8Hqc$g$(- zl8qd??kJe8IP|RS7YvUiXI*_3cHTqqhJ%oDgV*s%q;agLB1^oN4NIwt3bs-)@kk_8 zv8UwZL(6knRAXCHH3?H5#4t6)Q|`X3=fsPYR+rZs&M|s-*4T0!0r<9-dY5$nIs*c~ z0TgUMJwM>{i1}M*0PDZ4-JEPKZS?<~2|Qy0@V^KUMYec~2+Fl#{rSMFwxE1cMBpw)blU7*bgsQ38?1G{ul2W1`iHC@i9I~s&HYZgbEk@M)spBW-TlcPj1-3EAl{D> z&(lxuqXBydizaVSK~R}1oC<>3`xxwzLY9EPw9u3~e)`wpv0k4S0QrnOyPwD3OIO%v zKBJ7it1l1ge9)L(H+4T1UwH{;Y-8N*%6#<$40@%c<*C` zo}DkTM~;bDb1d=s9*Z6iP?^k|Raa`&4&zJ@7mh_oux|Pz?Dc&{AuxNG=#%ZPiEuC& z(PC4|RhuzQxCIVmt>ZU`VOb<7jO~z;Ibw!(Z*?{&Ki3_>Aqi$0dj?gOjGwPGFayL{ z%Llm1JAwXcr+%4Cb=FVB%Z>;H6#KWjF*}W+(Wm#i85z}#g8i^kx}=lL-es8<51^dbu$I-H{?TT zKaq|Jk07%9kg>M*yFV6jW_jZ+3KVA|ymaCY*|t!*vUFM5@U8y7@~n)z?80h2P{**O zvX1CsKO24B^889p94SESaU^d&{as7IEk?cBh<#FloWA>`2+7<-Bn)AX?FE4>lD3%%2&>o5TDVc8yNz1(b4;HJLL`E+T zGe^R~*rT3g?>s*ona|+dy}!?{n?9PmvEt_4+YENtrDuC^b7r#2+{W}yUom46pAg7w zo34KN>163f&7kgg{RW(QJLUC5l$uQG9F~UNgqKra?==|Ui`#Qzi3Z!voQuXmSp&M* zBI5jw?Y>N?hWD?B!^fF1pJoGDcWa8r%Y7}I5#-Y}c51-%DCv^6_tF^;Z%w1gu7N^+ zcF9Rp&!kZul}XX&Zvta)E?*#!KvngFkXjWORn{?)#2_GEB$8dI_W4;MwJQCys;CJ? zxMl7ny>lILZ9iy$xol!e@cz80w2f8}10_mc_el^U2iO44WDI?g{tH{;2tjo`B|4(D zkhLIwFnqbwZv;^5i}`70@eZ{f5$#Iyobu``D@ilrqk&7S8Y}gOPBbX~ZvxFl^>TQf zSae{}I@u9uIiON<)-QyU&bMceo-mP(?wKqs78JRjSl>=Kl7jH=#m`d+5Hrn)7{J3u zMLWM*&cEEdv9Eifyxet)j}q4R_4Z@Q%8g7#zvy%;V&^(aNz5z$07>IZignYMP?V*o z*YR1$o1nDv8X&2YeV!(eQ!<0yODF%L$4!$zm@lUMQv(;G_OAQP(ZcgNTco(Xb!@Ek z=~_5keR|G)h6^i#lMh!|+GQ=jZQ1S7x#6Ka`6$ldH+6%cuT76A(vCvCgqO7^l@I{- z(RW%;u&}siGWX{IyIH87uf4=}bO#JWmjEli!F!H@v20ihFycU9FH?HV9O9r4hKO)T z#-c*h20B{Ul-M4Stj~lzK}jx7$j?|%)eFHfo5gnoh7UQ0o1*^JdhPAtIiztZXjJqv z4p!}Qfn;q|mk6u`tVc(0f{ve; zvFBY?suq$SDzLwgGOGf_WW)N+p~x{`wbyj9DxE8x|8pH=Id~s&@u_aepDF9#3tBm7 z^Z|}e`j&syxP+fPvVEUX>ES)BlqoT_uS8MBdehhu#bAiMw9=e1RfwR^dyuA8C_$HCQ5Bm&u) zNTS-iKRxxq?2)Tla3&^wZX&1tkdQ}+FPT`DQlg#$19ABMBPETqO%PDuOTQ&fK3DcGX%9D z0|6!eogs*Up2o)B%*4$4(>ww~k~9W3PM=?2Dl+{UhLR^8G+7WosR|V%uc)JHSAPX4 z4#qMj1ZPJ*$EifwT;Jl>Q_Jks@W;e6hne;@fS@}7+0zxzLUl%UMzrna^?mcwb|B52 za_4lkcj@wXJCa87;ukGvX%dkO^JHABH$x712lcWYsy8>+#~<#cTS^m69jdQ!b9QfF zw>RLENs3j&TNSV;Pv%cnE3Nt?jZP&&RFFIUoQe{Zh0503f~So$gOxXTOjt%832gA# zPgCoLgn8UWTS`HaD~7G?&s59lO<#q=$=D-$>}+g?eru_>z}#j+p-5A-KJHFP=2Y z7a{h_m0w9JGs=Jf2K9J1Pa!zJaoI6W#_9aQ@uJt++z3OQ;ThnLpE>-z2A^Fv&6{#S zO2fK9&fe;feP_(O&N`;@;8ZZ^4Z7&-5hl3AKC$+=JRPsI1mwO~g0_(EO-xiXFJHhS{B85AiC{=1G^-%10Bb5 z()$y%YK5&vG8$&GQR|6HXq=&1HOlytJQC)YLhEuKa2Sm$Z1d&2w_BqNjL?smfUxkn zrb=A>5KFIohg~(N4r_Ad(+Qg(pt!hUrz%82N>hpHYb>xBGn`iY`^#yC=~9Jn2@L{+ zNKw-i#WbRA1ArJdAWdli&8yn+4^sH=OK~7Gh;+vQ>t+=$^I2FZ+6pc!)0{sGT zD0)$0Av>7~*M32B!v1b^Q6hUGdO;ybfn=D(C#sJ{30DL~4iC?8KxT4KRP<3W3+3o9 z(Q458kCjEEYU;7<=c>Y|8VD%!Zyf_h8gY7-|0zlNABnd$b#pNLEKd1T-nBIywAiA4 zJXDDtIe8%?gV~fUL-r>!EyvISD3(U73zUGMG&&MB);nFdrJuU!qDWR+>kUN#T(N>h zxyQW0Y(CwPK5tH08y8+T+q3m8Plf`}X)24lJK zBg3n$^~=&a#dHx zf~B-G9uWM28ZY{dLr)+uB~|~>2$~7lV+bcZI{XD)+MVRGC%4&ZcIH~UV5J6$gw9~*fPsMsvE9s zwdSY4kPCU6J~l<(Rm@hP)$Nf+{`OvWb8meT9q@d8ofgHm{;+y`>RlTr=fdXW-B^7x zdf>ae7$_)X-*}$>JuonF?q4K}XT|I7*?*k%Uft1lcRPr$jH`*wXJZ)&M?UN9`mqS% zTrh?n%=GmBBJ_?=5x(};du`t&bJ*WSXRM?W`GAj&;OsaA>Am6A-15Fd!9bVtA@ou- z>CB36^S0+remJ9Oa>09b(W|@UwZ9sO@yH`H+dsnYEIVy(`~*k0B?A8XkALw9aT=$`FDLnrLKsC~dB^&qdlq;%xseKi)}*0({Y3 zu{z`an8flh$%og>$(rD;O2(M(F|NZ$Z-Hw4A{R*B!r&naC1#S=S14fZs}0lO=6g2GmV!7?$m-5uPFmKP^_;l}W$Y%rF zlcmvYI)#H-B@1$buo}=`2*r1S9eiOG`a!DXl!NLuqzPjI6!5t08HqVD`M)Bed>!h6 zxl@wEVG0-JI&NB-H8TJvvFoqf=GVTZadHqjQ%akw5FK(syWQ9aIb`USW<;#uj3pW% zJCSrL-{R{0h9N?KfeZZ_m_Q;cy3b)iCae=1VNlt}CVinm*_v_`o&O$TIHgkwpTd9u zjcR@%&~E!eO!liJ%Dq?%J9ryg=(ZrvASlAGUvyuBz~` zlmQ8HBla1rkA4?xE<>SGFHeJci$$6w`u7BB;ZN|2U!ZJXYX^=iAkn9agOzqb4b4>i zYASZo<`NmXluX*iYqJ<*Oce+xjZ9O_y0?4zedyy2!)|I^8Q!Cl#p+m$X2uJ{GQSrb zfSjhji-(FJ%EVygVT3}Hm2%pPm-CZFa`@?LNq3!jdl0~A5XRNvsg4+8)gHdD{8R); z8?waQ2WW+h%ux0~;`fuGRo!Q5SvEXbhDeLiWD0HBS&xq%DN*ya%8d4}Z#5HtsNqc{ zLQWb7R~2MRN6Ttl*S5#^D+^X-n0+X$M?_9iZCmEKj6SOyJ5*qbSf#gom|u*0Dwuu* z4;wf)PpQPQwitTd)RIFMLDZU@ck*`r;7nlen!Iq<=p^iKch@G`J4S~o zSmDNyc4=={UYL+4RjB-KHJ;}^5AZmWQ zavc64`grDO%OLy6Hupt<$5e%`*yW6|p_1l+aU&L8X`P#0{(=&e%St5e?wdQ6`2v~I zX0A9QbTi>WavFT z8WX&;_Sek;+IyR#9fhNyiA!#Corn~&iC#IwD<1eu?GJKxsGxNIL8Ei_RsnjTe)HL! z3>JY2T3BMZM)Z198d(4X537L3`0EPd@1h}e!HCK<4h&xkdhBNCbyIPbBDpcb^9$sh=eO=0LS(;E0DyN^lV_OhC=D7N#bhP$`szk-#T759^U^K9c$q=~%pv?OCg7iQ< z=;vY&kUu9pU{r~;%R`AI=ei)U(Bo2Rw!e-vW2R{+`h*7&&Z!W@hU;w69$3aU31Ghy z=D)l`y^H41a@cKZO7fZ^>-#BPtIV>)x^ZU|!aF6A-<2S`n?Rz&z=ur3X%WHwnh!wp zHwM^S6^BOBiKQRy4`EzI>sz z0y%W1*sD(1L(xJ=LoU|^mu=_N7O;0RShr~(8DE}qDSf3%hO+*>FD`{#v-?0`I7wbAmX`IR;(@Q{ThL&5Fx zml#gpFaB{ld79LkY?A?RuTbjVu{t*FjzYpE=ES@-*85%r z>Kt+C`1u4|0HSFZDe2I3;1?~Dk{y*$|;Jg8VCsogA+*wt3PJu!JH5}#P0c$&pR*daGtB_%>gIza=v z5(T8|TyzjW)FlIXvF5xg%n82$EO)`NfOJcbyDyQT^vz5J$-ZITw(&RfRXn4ZqHXn8 zsPIrFP`?sFXNgT*h&CDMQ$3d?G2hcXmEvMyjoT@EAQZSH6%CjWUWw|R3uewn;Iu;L zXT~ss>qiSoC_qzqx6G^i8GaeitnbXMl- zyyjZU!B-IA^w5TCZ349$)*1w>i@IOt%vGnS?U)7FGH~WX#uKxj2@qMr5s3zJ17V-X zhRv}sXVuhMrRf;fNS18aXb`O_X%HE~t9s^5bH-8{Dhcio_?_5OvCkSL@KG*=f>-^&IwUy$#t0Dt-) zXDr2TeIa(-v?c%og(<%gCDo<=Ty%owFZ0G5=7EQ)j{o{6_ZW6Y2e4!5Y`b(NfFzMT!jc&3PlbIM6Gs>_@}-881JxmCe_=AEmT zFDH1}7rm?mvs1q-T7=|d!q{7-B5{lqMs|@A<_;wg=|M>i$3pRwe$9}%H8vfNISJ^N z`ErW1?TiG%qTZ)qC2UK)A$u}>NCX5q#EU$*%_DlhZ5e=;AJxU6NF_iKITSw!<*WMu z*m>XFjB><1suY3*8wuqGuLzZnBvYF8dqDOYg=0Y>^P&GyxfA|HGidzl^f6Qr6lf!|`lh8@=x*cG;0+{E#BC ziS%)9NfFe`VJR5(t#e_xT~Kp&(jon4Z?gD8bu~|=_eIkH2GtaaISr*0F2a#Q6kFXp zQi!CS-Z@o*zbN_I4qX;7-&{95?P9|*mDKR$aHU^bvYizr8_Db&%8rX!q~OvY=nX6A z=H~)}^E)6ruu8yy5R8oY(qdIi%`+MsfFln*u`(eq)c~a$ktE)DIX^mF0`oGUxK6OJ zMYU~gcs*Wkx4L*Vy*pcOMvu1Ebsp65in64%VML&?t_FjfM2P|^*kbn>nJlonNy-VV zF*TJ$clW~u#GprXKK!}##}??*tjD}d8G6mP>qCHp;Y7E6jb`~_{rk(sw91({OatSW z2FU~~S*SF0)X5<_7Gg=(p#Yr&_bz~U#mJ=!)_o*r=^^J=G$7R)N@@;p?fGmA2rLUf zt$JAM9XhwfD;HmXj9z62V__R_wumRi#v1G!RknSml6tg&I<~5T96TJEmEHblgYu@s zeEauH;2SK9d9{Lf&*m=fpIPMBlTX8M9y>p6F5dY5ii4PNJU9sdi*P6Zttf=){~3k+ z595}!!Djnp-0!GC6UIzJ@VXt~RtUl<*{1$l$oN6KQ}PuO=jLDg;9l3bG`e=5eSceR zp;f|Dg)otPka(x3%pY{Ty{RwF0WeK;&nJVT_x&$_1q^J)S`ZQbEc@YHeHXXKK4w%A zb#Z&}{(f*&5(MB_yVps`%8xoh&(5M7@#MHm#o-;P&a6t@Xso~%8uV!;koA2`J+Ftyy<#CpIG%w8jW$p0x!y_xmvog&c7>nJf`AMlfJ}`IX_Jg z*=_+cX9utP*1f24BK)pn+Zs#u(XnRx%rWV|88^p8Xgbegji=s8uZmW0BB-zoqJdzH zTDzVEUb`(#6i5G;ockI7Dm;M8CJZ<})~?nU6LkMfZAnZOa>j^*;MTnTG&axrjH|a# z#yzeoLSEtu0jpEWE;eIHlej)<{5eV*G@=E&3DnU82rhI7zxKwDhMhZ@csbO^sG{9L z=v18Ir^^Qg9uL;!bw`IHMLTo;hjDZL$+%_zWZYbBj9F{TficmQIqQKYwLe_or}j+Q z_T5Iq$IqbP@o-E#QlcN=>gK&PkG3{|8tHf-whXnz*W}Jj?W_8MVy~6zETS@DPp6n? z;ulE`?sAq_VGM@~9HMKeW=eqiFvMcx1#tmZUXdr8_qBytb zdXy?*9un0T-4C01msc(R+TGAB1jJgYIr&IwzOleC0f!D!;90Pi*dCTNdtsGqsJ5i$ ztiXoY-B9Xcgmr-`j7N8tLo5x+9^pBQ$Ti#$r)qVwShd88R}&J0=b_rLyrBQaxFP?= zxH}UK&G$a9sSp%I{tw1Yy@p#3sx<$zv=Ak-Gs^RLeLte*VXCHeoBR6=gkOF}`x&Z( zA8o~o-#vqrqO*D!5vd9lR%xXTW;BL2WzFIv{cM-!h$5Id^L@+BX>6g!#Zjtu`m@}lnzzwvLJd@ zSp&4=4<+@CboLVuSzfytXi+SWu}Z}unwIVl;OwgwP+TSkS6#TLUqO~4d4pjhc0qGk z-%l@@`J3=Cn$4rS)wE`R9+gvRYl$7l-sP%)zXqma9^+j>^_p|f2fIX^0r>o8Bul~) z+h75}gf0n*)UV&EM|7DBd|9uA)1M^>G z&ET`g(9+D>h~~cuFO9yj<7Wq=t&@SJ8Q`;C@V_dg&i-M(|1jTwnD0N#_aEl_5A*$p z`ToOv|6#uWFyDWe??24h?F$E-3lw>H2z@!~wBc6|XrDOt0Q(5F z2%xSFfgy!!xWFeEE}*x?YD}ZFgZgu`4T}PlPG(sAtRjD9&yl z2XR!z`90cz2$s=G-Gzb2;;E@lrO+3VtudkLJI@GKTuNhp2^nC#vvo|cOJT0n=C1DL ze;Ul$76>tK?Qmuvjz{+fp-OwhJzCXDbABS>t&4hB%^=C9u5W-`dcZFrOG4~E0KOur z1i91gfj9p=$2+R+#+S)Krd+2=i5w_N>$`(k?q$h7XpGD>hO+g2W*G@vJd6k$ zzxhE%F|<+pCh(P&D*P3~=A1pPR|N=Iu%Th1c_;+13NtC-1|hE&Q_hU!DKYE{L*NBc zqP?3)FJLJlI5X~45y`Ob8-GUvvk+;NP$(!?^BV%Lt2`sCiHmo&^}3h)247`jJJD&n zET?q35Fg9s=`y0NJY?aQb(oP(Rtt79TBTF!xwH=cEA1D>SjFC@Bo?D@*4A2hO=yUW z1U7n7n1$^SrDjBPlp>@zDpQ8)$Rb~4L7@CRj=)gF87xRh1qUm7e@70?R}Mze4#w?V zIeISgabZUk`q#Y=#tEEs5097Ox@aK1GaqK1D$MU)gXg~Q?RqA?LUPNB4)T%Ue9YnF zN(Y<7^%PCpUa#D$zVf)rhuf4y%R=}M-(J3+y0UY=Mcp6Z56;Hj$YxF9ko)UGki+RG z7Hz-v2FegBIm!R(5!m{G$;q4ux8;dUX?s@HZoS0m@@+zVCV3%U}AQW!c=EE+sm2=SYuxD@xwARXMql zG=s7K(ID!v7We*mqtSBYYRhijY%jHmKykcADeWZGdsf`sL-NRHL>bZ4UL4L1V^VgQ z7@!%rk$3|+5x&QHPob}=Pjd+}!Vo|dO_Nx^SX;`w4wxejVH#oB3@}4VZaywZv-v4V zWkuSAAa$Y+Mcepg4FtKciMUuWg=Wec&W}gVxW7>kJP9&+l2GPiq&%oZmw~)Yjkb3z zp&9MFf^@N3QiI5El3<;KD;QSS1Vc<1zXXk+2*w1scN&mD{liG+|d^3sDPyy8BzzC zBuWYs@4Xz<03if7>0r>JqGAyn$%UN&AJ<}L1-ByVWk0S|^Y;1x-8*C&QO2N_^Oo)S zmeAAHLO{|d76*@|2@ItqT*dBC9L`KhpC^UN9ZK>QqE<>G7!_)|t-Ve;D|p(7c}dmg zo^3vjFRy7d{1tB{$e~+clU{3iM}kTv#D?f+?uJism(n5XamxtNQReWZ;&u2~b)XXBi5Pvvz!U zGA&i>&&uwaR^Qf%ocC&5j0&sEA}78a0qwv^Dt6-~ft}^YMPQO3K@^2`rw(NHCFMz+ zl{G!3Ew6OuGW$eph*cneA5+4xVt3`BMj`YY*q*>JE<;U-fPD$xw3QL7u`_@mM23F} z#`Kcj+LmCq#8x7Rg^D)b{(cf5c^R+oPly6!BjZkIdr4I}gtigXYXW|bUys5^BU^`I-2<+rOw_B-DIix%Sb`l>yH(46)a3pn~j(mO33W;~0o20X6`3oR$) zF0^gXc2-)%!p8c}WkQeEjxVVBq%kZ|>q|VEcFHbNSrX(8lG@{t7isn-x~n zACpz|ul2x2)Doz|l=MM4hm9Z>T(CdPwZEXc8J5M6Cgsnd3B13!#$@CVRhNS{HrOI# zOtoCCK6=GeO&Zs^@qJ59s(YHJ6_0&Fp~HIl?H+>{X$9MgtX&ph6^v&k;AmPWj7ewz z%T>hGmwtKM0ra^v0r71EF7I6UKuD|VK(G^E6K;N@{J`kikf%!24eB>f`N1<~P`(5S z<|ryq0ZH7G`}yf^^$G&d?KEnBjek(KZ<)Vi@rG}`GM%&Dy`bPc+%@AOvlXNc&R9Mg z+L%FD4ZQ}f5*6#4o+Z_|LD;s}WCPUsN{(U^h0A##?RSm0_nldjo?Hu$;%7Tme`l<8 z2Ra8NN`0Es`$lu+%VG$pqpVu`qU8jP3YQRCMlOkwBkHFk)7QEmgw2rJBl= zXzsRg4$kK3epbH_!l-+JcKjfCQUN5Y_F2X{+Z~ySlg8+0uPLi;6-Y(8x2fRmRb6#^ z6a0Lv^=Lin)Z^X~B9>9$7(x2)<^o8H-7$#jH1T>IGB<5M=Fy*hODp-Yn}UMBV2YDY zBs`#i6IuHs66f0CgPG*!G+P_wX-R6O>@{qZthcU}tXf;1vVIpY++R6E?0tW=bh0!? zw5(XmqX^Gl0zH)Y=5#2%^Fty%{YCdW+WamBFMZ&>)&=E@ec78mUp9slwqZD5n?(5J z!CUWK|AG0_Aw<*wdOOD|C&Myu3yjgq5C3Hn-C~G?z)jR)zg8l<45A)r{&Mk@Y%p?Z zG;~K!uJ2Dq4bN7-lUMFPn3T_}R?=qU->VeePb8b*#Q~2#C$!F%!)j@==dtJKzpjxY zDyGsvcOex@$iC5zaup^FHko$a{k-R9wP)e+9PH}K^K?so=~KXm@H*-l!LH}R5W(KU zZVQrOOnRZ+)=JGs-2y8(qTDh{O%Qk2X|^V#5WlieJ#OV=SO{&A8FTv8hvMLsHwJ!; zO+-6Fza_feI?4CGX35DP$=N>Cj}qn)u-jD4d72PC<(uK|7#(A<>>FyZD@@Mo%R4nm zv&HTGG+PGwNG4l1`K=ehoHc{H&HEa3VShSlM>oKY$yu>VltSBx{PaDn`XvWMHu-&C z@7WYbBhZ_C7AW}vcx`e^pM-Aoi>{HI#SMXbU6=2=ci>!ve0o9u$%iLD+f`XB7M7^7 ztf&>lSyvU4BA${2cu>ZTR85kMc*c;Ip znmGV$tgSzH&HUWG8(?AdXNW!1(3HgYX@-ee+qTWdR%6?GXR+5_UHjSpvG)7^$Jo!o zdosuGAO|yZjxq1+`r>o%n#zKZiD5q>d(Hr>bmNI)nyxN~#>6Jc`%tOR$cxGmWG6`7 zW#+puxl{gFa*w!q)QH|L0G3?IpG&TS*(lRURpIs*@ga)XBQuu4%%Vy{D=jaqnl|<7 zhG|*!A-@TWh1PVgSh|K$AS$AEJcQiXi>!GEhtg$zqnNvt25y#;UD`)o#4kU zlk(pPp&K@I6P5ZbVYXtS87=ypYaT=pa*6W%tHz)Qr9=B@A}CRPVe%R2a$)2&fv4Xg znxbY!Ia3-}QbE4;CH(}_(elwdC_8~E3Mpe_Lix^~nf#o7bDD?Qdm0D?VtR&cgYa#* z4J`-w76!kLK;;DgDLL^5wbvzNwnL;^CrAZ3qzD>xjP!J_=6l5OtFbXKaHFoOmeDht zI<7-p#}zz>8>7^3f?MD4m3+Az*#qhd3I{#{%j~#W+a^6iei3@|A%z_CIKI3IXxZSo z;cH{Lb-%^(a9}aDiN9YfU&MjAkqn1-dCC&1=5qFA=HR$HeLZ3`_f+GmjBMykv9j#+ z4S`n$6?tx%`q7g>zDov?CV<{f>sdiU-AQ<*t;rP_rQyb=e)YhuzGejWplnx461*jA z#golvW#BA&yJ2K*{$Z7_${>@TK!?V&usLPmJ{1~N$BaD zdG}JoHAwsreybnGoc0evqa`?8M)l~)%aoUHO+hHX8EB7G(sySO(iReqb)IohikZI8 zNTBpp#w0OtR@5}d&qaUq9aQUnI~TwpUP<}RRigja_IzPkSd|oi=&qjm+2ljB>;v4m z#mfAYLEPA2e{2zrsu^SbGOct;lwvT0ZCra*e~#`1-D6!Gw{VAw70m3QC~QMAFn#C>`qF&n&m1SMt;!)*#QFK_6(z4Qug;dBa@o-ceQ*)pFn z3^Lb5hzhdUfa3kPb(r;eDhpe-0zF5}K|X~REuaMm~Xx9J2?302cExAf{~Nj90kIjaHdIsBZs=z? zI?*74*RBFc)zOj+hM)P-4>f^{nplwVLIi4-afi9qun(b!;BWK&rYLy2n*(Hm!`ixJ zQ3#ET$n_E)WaSwcic33m@KSY;&S7|ouxnwZ?k5-qVy)Mtg$Tc#8gociOWq=L7@(9v zKt4F{X?)`j4H``%!+?;WRtR>kElIufbAi%n*oxBLJ|)l@On+Ubx=ZMkWW&SnCT)eA zj|c^k2eene{bneNAOWv#I=V$#N*(t5LTRIt^Hl-3i_N&=Lfg0;!KXbbdMlXK1XKx| z)*H9X2gXY7gJ@ex>SGUmuSB(?F4%UYo{!MmYhMtBT-CJ{P4FBEZ4MEsgUdtE9Q-e} zG$awdb%&AYRce;J%~AN#Wkmf`l#mQhqTYw#P+#Iik+lv4Ew}^tB}6JyiF>I**Lw9V zY% znlluc#%mMTfjVhZkY4fOA&HQ%>{Q+lI3Izg(LSe8FUV^#nJ>0uhqkh9e&P8BNS7FoFVnl3;$co5e+PzEBT8av78S}dn5pJ)OXppf zbz1fCvVyaWJn7vomgG$SsVxmHT!%)gWUyKP3H_+BsEly!_=8IQ`VDCte*dJxa;?V%0B`Q z^;N*u{9i>+(ZB!m;r^$kWh7F<(vJ?t_a+&r!jj^gPdHfFe$mmp5IUMQLY}*jC|y+H zM6H`|eO^gO1IMoX8wa#0{PTph2^~Mw5 z3HAO|X@Tm)!|EG8Ua5R0(a1k`dV8S-X*F^cjT z*pCqEm^uL_A63Z?d_iK!l98Z79jaNNtP|^FyV%&YMd`M42_h=_AZ6vouVUa=v!(@8 z^D-2G8tCp3A=P_t%aFd5dr&6~G1@Mj>F$|Ntij58v$y)iNM1Z`R9X*>>%Bh8F%qt; z+LU8gMpA5>{tKfeie*0VGhh%C#d`HB?Y}vQG0^?_yVB9OFtM}yeyz%1>$ z_Rd~WQ{jF?BX4pf1Y-F~^5)tfVZwb?+yL>Mt?0z&9v*vRm?MN$dP4f)S;sKa$iAvr zUEaRXrOq-=K*>m8r3r}TKtuDY{^v-=xzLo!B`>{V__|RHvS?itQZl8UiVF{y*;Y+M zOfehaXbh|RO%nnf4JMOQc3;Ewj&q*=x@?=HeSs^~3cUTpH<+?b!!=z@&dsM0)wYp4 zDjxF|CR%Z~?lMzdy;RHkenBGnslcF|k)gX7Xy&vsIHABN&hkZh*DA+@4`;D@vP@<< zgfjJ;NUU%?V(~E>our*WIwbU69Eg%cB{&t4*<_igpIlGg#TqZhs`pz}RrXt1?}S%k z6r&>G%c~uPhJ>>Awwz+e7$Nwf4u~RmnTl_VB9j)e~%Aseag!0Dl%c>e%h}x7unSY^R(lOIfZeN^;4Fd1IjYCD+lvp9cdWKBr#_mP_H(s|kF)%%d;IpXQ? z0V5UWWibtAOtQ@^b;R$ zUOLZN+#?66`wb~d>&ded*qZ1a>2LxRXY+#WoH&K6b=)l6Jk~61V9A6yub?8$`uVpq#yKUMaWzx- zg&=5Z-j<{QM}rUGXpD?o!hSjns8lG1R_Cf*RW5lL}eefseI~IRqg`w zbF*8F4cWFuBIU7dV2ZQKJB2dj#CH$|S7nmc>rjwY-x#1RXE0Ljf0`QcUtz=&&VZi>@JDQRQ2FIaFhXMMuRkbYe9G5ygu)Mf1)# z_*>-8HFO`oJXv6X6-_<(>a-LRs~eT<*zc6K?pqFTY-sSdl>X*v=uv|M2jUWyd} zE}|0C(UATnq&GKRLLXgdIcJylqzyRS^0ksqIrkG%Gr-b#`KzT7D1eEYAVUI`${z&< ztU{8*7wo$$F+X29*0sx!co>}Y#AkCsU9X|Jc{3*{Ps@YG%!-=zs;gKGEb|izti3NG z$GSin^QXAqxIZ{;fB_-@S4FvAI<=g9eSP`p zwbSI!x*Of!=T8TC#{0z1o>=QS=6r4-W+_muA}P4KXo#1Ae2CxGo*tnNCdIauz z9e8N0;K>q670W6m0AGzINwU(4x`k!&X`wAG9(?C|P#b9*{JRB0vz7Lqn~NGAOB9#Z zRm4;7rWnF{vrc%=OOd$+LV*HpYoi6oFuhMv+M#!|lmp zF|kd1kyNtTMoeLoY15vm6DZtl!EAckKBY-*6(>axr^w$qftt{5M0v|nQv>x%3C-f@ zMyqeFbZqF`oAc?+zNQB3{ie)jkMkDbQ46<|b}q+Ncjp_@-&GwD6(84Bz?CN&UjOC6 zkAeQr!^EEkzatGZfLe|AyrM;rt;N<}dCbDHS8k89f;P86D6#t54C+%RnOX={0a0u> z?EUI0pKO%DPUXjVFyr|*pGNp(cY5t`KUf|V3HjSdv%G(=j1InIwpa^qsGDCTM&obX z=hQK5Cm;L%n}J*sr8k9L149cD)FbQJBOL>d-PSNh#tt!w=@XuWkePfCb*Usg!t>YI zm~1l#m-2F>JaCnOF^B+`JaD_K!1s$@2>kUbbrKS$#KZ|_{z&8mJ6*=nPds!|X4NFc z0twj)K4Y87nZ<$Vag%fc$#g}lUAic0&OveZT~0w?rcVfGMhjeS3k%Lm;!rs%qS#Zg zUrrzWl4W9*^bF z&&g_*4s<#}?r}154?pl@`rFDPN>0Q@Wj14@9fw}CMReNnVsIdHu)rx8@SK7v5gjRI z7B0YJ8ODJzt_yk-zf5r!!s}pl^ZaC-v3;3VkvE$oaG%O$sjK2I$1mr zos3UPD4F;Xm?#pp#X9gdfauwgG0Sq(dD$h_8pIDW2Ynq?1cRc#)e3^c zd(!e7)l5a))cg5D2nq!g9!QK!qpZM`SZY?_!HGuwE)O{jVvIuRupyTQ%7o*(gT(@n z89mur!h`*)6%10kJ)?vRE~YKz5hq||{5spbl@{JD5$798KZ@t75foz`KT`4i51fsej;TIq$UuD~-8CxyogSK&Cu^xBcvDKSr z(!r0z|2`I9zX|76*2knM3dgb}7{T-K9JH?p>B{1RvmoB)%Z^#{U%1Ra#HtU?dt8AQ z*7cRqQjI;-Y472S+zBS%@Yn*Kq;^a5;9p^?5K2i&!FyDt)^TZI;{?r~KhlOe1~D?K z;f!?9x1y?{JIhe#K3xDmti+M^q_z=Yo zBZVcmS}i*jr&lit|KTd1yL!t}2}ok5v*`23tEe*lG6%jfiDXu>&6cOl^G9vvRPTa7LCPl=L$| zi|4a;@vv5$;uvpQoKY|imXLtaKc8>t6!XQgJBIqJymDRVq5+OFN3D>ciEz#-N} z&2+lDWq_Of%90LGQ{n)uS%T+k*O%g|E7(LnYjJgZXGTspGb2@1SG8)SA2*_jS+A`6 z`TnCGl=g$NJa{ubm;%^U_VW{B{NT&Z<|Y#a4&r%{$Xq{hM~au&@|&_j3`_!3-+aVO z!N8vn-&jHsR-wB=j}ZeMd9`++w+3x&XUmWeR^1`af>nkc_HNI2Urvn5h1o$6!-MbxF}II%LwfWdyS+H zmDVkOIHNx8SJ6aJORF7Ild92WiEh)IMgB9Gs%FL|88yEi29zXPP0tnH@`oWX@lHX= z{Csc|K9Akhg1^D^t=PCvbe&!*+SX*MoatA6YZg}g>tUqD^jN{J?63bDpKaL?+&IUG z$z-h-@@p(Vo89FIuNGujcVhG64Y1T`l(|qncDkoPeG*&m{!7FTO~e4A^TW1*-oY$a zE4J%5<@N9!3VqX|MZJ@lQf3?_XVt$RFqB5>O?&_PPN!a`+kq=fDvr8Gsm&=5H{9NL zqQ)u+#k#iu=6A%V98E23yf(JhAQ$@<&oq}<4~3pCgt%+pmRxP7_L)~|z~->!zaAty zOraFz$`ie(|M4mO_&6jwB+2I>ov(>+XLq$WwX&QMZY9^6R)pK0nF>*BO0Y-RV;{5L zw61BATNnp}XP$-KhnLkcv4Klv7B3hXle!Bwis{pL=iJ=EVHGXmt0+rn^p(Fcr~6!H z5M9`p7F%Om9N37G5;<~2W-hkyeBE_6gvF{hd)CRha%JD=y01`Z(Vt~H zOldf%%gw)E+c5*gg*X@c0YU)k3Lw{lY1B)?*0L^#Z0rhy<$cY0g77|J$N;xjPrI!NVP>>8@j);)do|D3tbBP=h%BZ77~4+_Z0` z1$y^kX!-&p*-fkTJOyo@CwUKD9b(M6Ah1je?(vP`YTgC<%<0V*rPlxnDO;bus!_44 zhoHtXQ?V7R)!T)X#^*Eyt3Qp#b|yNF1{sV;<2whQ5Z{=AAL5Kwo8z(PG{^Z911&`jt(fNBI%&F9LNeCN6zdzp`D#BuWP5tvz=bT)ui|g-0pzV%6T^*8o;v3#qLI6dpf^vC%j=a zo3x%m3>wf@IC_4lFA(~HNN^z3lHJa-xfFsZp8)m?o@!SVpBQJRkfJ`Up9=m_q3=*a zCuc9|CvykWDw?`g?uKccYc=S*hh;%<1&@l|g8{Sy^G`TJrHl@>%S}GbDUDQ~gPALm+_bH9Y`+gu4)C*W?e6>03bZUlAn zplzwGC%(rr>~wAM7=0nQ(BT9Cg{NH zO*gV2q``6gFx}@*P!(Doec+rA_q&r>FTlS~({WSdrRcvr(!BqwH1|7a^N+wWAb$i< z=j`kOj_GeU>AW4L5VGfu*7~=3emlIvd)#tgY+Vf=vV!id%_%$Fbg{Jr{#l z26Dx`n`&*qAg5$008^1NO`f^oyU_*$2J92S=ftS?1J6ylcg%^5!4{X5ZBQZp(8HBds0o|OO<01tx+26=)yR1)Syn{_;woz z)y8aWv=4}HAQ*!%wu7t*O^yP?hy)P*;FnzU3c3Z}*!K0jD+^ynAEZPxTnN7~z45^p z7m9Fe3 zI;(1u32B-F4_H3du6ML``+9)KnfWxZShc=B(BxNAMrM2RWZgFF>}ZyRHKR42mJH`2)UwL{-6dri zu}i9`y18&JTg4%Cz1eh=W*m2g$=mVK!OOw54_KCbIc3>&dAX&i(P^EAPW^HL}dg=k(}N%$ctn8TbV)nq9kAST9*`0;gg3j zu%f-Q8=pg|4uXlOuR(df8bO}QhA&jfyx_j7&y=C?S^bAZ1?>+~n5mTfeoFM85><=D zPl?KYZ@$^VE+dR&@#xbN^mRMlk`Ewt3mrFier5sj*VL^7K%;tPLL5j zoXQqtsSLH6mDfPIU)y6zQR-Tp(}pRKp0&N(AeL7N7Um$mXVfR7 z0}#5kl8mGLmW`@z(+uI#49CzcaYy!4>vC??ht4vt8ibJ`R0^4ccoOk9Jkrnj(X;@M z6dK@>@_HY#Lw)I`L^-RmoC5nOJQed#k&5|;NVRn)C^DY!v~j*m>mm19#pE(>gwvG= z2YZ7ypULnWv5MkzS5oC&U4&SF`fV~Fwr^1j;TXZx>B-x#9WEHEYURc1Ou&F!p){gp z5SMmob}N&Ymo>69J8469HWG^=LqniaY0Rs#_s7!m==|!mD9BDa@T2=kSahha`0lnr z(de8}OQ-67H%YtxFiG`)n50(!-Xz`mHzq0czcoowVx$~nJ$DJ38v!P%mug9mqzEQu zDo_m}Ju{thHRgZrkwQx!oJRmW(zNd$sn({=;f&Vi-H6uaxQs@#-il)T#?U!=TwpjY z0lf9XFPeo^&*)gxv2Nsjt; zsQ*bS8;R_&^utHtzwz+1(ZH19&exf?SVeoMp}u~eEo1UZB0Bd)^X!Q~-j3wPkl{jZ z+P(@jWJGKk8z6{{$niXE+ws^)%@fiyXdUu)|nz8B()CAq1->kcF-$stx+e^uGpQXb>W_#@O(L z((k;l(Wa5xtl#LFn9u6K$^~(BgC{0X=##bf8 zep(tdm0En6h-$l*aNwAm-vLKY4cJ$!P-QbcVX%O`CMg+2R z{i1O-5ef^X%eDyG%E@<-YYtBg+BOKvH@O>j@Fz5BxcuX$=)}c?#`e>FHq9=({1;y_ zNoA{;!vvL@&4^i!SkpOA;f}7!sBNPv2uF;==@vE^VfPDB5gu2=t1z8*c-b%H>*W$7+!0ffne%hI* ztvO@GlOT)*+H4h|As=zBM>NeUfuDt$D8mDSM`EU*#*Xe_c(O6q?{g;Opj|i~P{5lo z*mEkVdzfd#ihgc#F3UN4pT+#u)x`QL*yT+;g88(BE5~cLmosM2 zPHu{IKC_fgENaPxiY{2m{l$hL(s6eFm5FlQL@0q5)!R{w-q8q3XF5E&Uz%t(7oK6K z(VUOO{T^!J5U9$P1rm{>3{1XdyPJ7`&Iuh)c(2k&3R}5f1JA1x<-hBnblM4eR)~Xb zh{IaJM5lfP7w7~Qnu}yPg1`HePm?k|zNXTCsq1`8{^Hzp-;5(kOe$#-XGB`TxQWxCw-*E* z*eIhe=(`h^pZFg|W2A_ZnDq4nD)Jh7q`ty7>bJRZjiJTCu(TAW|am2CWI zi7p;8r}1gWqhc&2!opmVBiYBTaLs|Z6H6sEvLS*NO%Au|uDUfBsS&xvm%v23GfPWU zU*w)uugj6qa-tUnJ(W=pA}^tJqF30mUHKA}m-o;h@XG3F0R0aPnO2k2<#CgyU3gu; z6lO{D#DmQtzR~bKDK`RQRC+NY3M#8ne!8EqwI+h`CbeZ-bM)e@$!b?^RtuWc#$3iN z(YFSL0V3C?jA$FY;6#*Q)OHcIJ*@LoROcVZwU6L)9W266m>2F(?o49VyI-x?vg zRfG@ehor(4eJU;s;Cu!JgV0V#JYaOvJ~Xg;?}57NkyPuiGa-8w*2jRGV?M}zyB&6~ z^t4xu-|H~Jr2$v_w1J;sPYj1D^^Mi#POaX1Q04_LO{GS&w)G^lq%F6T7P@_Gb6els z=%{=S$LQ8vP`@mxq6x@UuQCpnE%aaaX!Op$pq+9lx3jQZCoB)g7ROm^uUMQkMtdIYHhJQwyc<0XW*8;8^kDJ8 zOKq(E>`QMr8#ksy4Igs0p|U5_u&i#CjSmT~DNNubXwVthUQb392-*d+>)f3auU+3UTR$prpc27pVt3Z}c9PP9?j1#^C>YHB(=?t~4rOeO0G@_3FRM zMr8P{C#!F(V`BOHzN#YiRs_l8L~H!q4(t5G=j#sG+`dJ1y(Ep1+Rw2<`f?!}e(DA; zs#(t`SO%eCulbTxO5$X`@HrUr>+{R2-wk*+yS}u>q9Llfo~W@P^9mpEwS45H28tmr zeg1@Z;79ch9MF=*vs4-!lqk}91Gx6-dzON#)=fsfC9yTOn2sKD+95riIttNZ!`p(i z(8p$tGor9&Epbal;X>TVAa?LN49jR-K_#+|wj*p&i*|9zlst5ccaPaZYa#_Hmqo>k z9TJc+QQ3NQvK&1QOF9l`Y->W;HB=ZcO)su|JOG4fZXhF}zp)2>7+Su3Seu|PF#RGM zNv3WKi5-xhBQ|KWj;!;EE;>Oc6@oNsCVWm<7DeW^y~Qz6lw(u!I6Gl$Iw*sVxG zL*_;sm)uf)g|i=ZopyP2+`Z70ZIOHGJ-{d>_2k*Mc{vs6?4V6~?#Op!IP}=R@ox)z z4tiG?aN3<$;=7pRCe3slO>3Eq8P| z&vZRzbh!59Jh^ImtACQ33U5YoAGm(##=4(1wQ-!kxQF-Tub!61mE$m+cGdT21JCX= z<~SGzw*16H9jjN9g#6O}!nJKrJ2e3p95i%ZGKj`tm>`^jo>ymZF=PQJT`Bhx=%?rn zk#V}*lblzoNw?gcg~stFioA1dV(@dZ~!QK(bOsrKfz1qW2-{T)V_n=4^?)Ri`ODGN<$Ww4<#x@6j$qovjE- zIKw$Ci~WdGn~A`qCV|nl5T6+pDI?+C^`4`LZ4<*+nAJQf^|HnyigLX@Bb=XMecUq%lprT zm9<%0jRE9fqt>NTX+J2BC$!`B8=4{%e^c!_aB4|h7@Rk{2u^mD>WHKzYrz`=UpMSO zuUiZX?EzB2ZESttLlwpCKSLFsMEMNUKqr@MWEMahA}*@&Mzmz01J(BQ&L3M1#x@g{ zCm_xQqr_VyiR7$^1}?Eg7#{YC7)95t7>0pSmrTP8MR-nCz2z8Uky6b0PlFPCO@LKF zfIx!7D2nH7>y508z4abp!+OP=iN^+CE;7u|j&&h1P*!aNyR2KZHsAYV zKD&dlDO>TGfyc!bZMReG6`L{8cw=#aXv8(Sx|&Ui-EJM*SCE)NMQ3`aq1CF$UOkjC zCUo?jpvFXKf8Mi8N}`Lq7r5I7KKY*OEsD=FZFysH3dr?N!fThl__q}>um_)Lb1rA> ze^M7GF?*gpu2XAzk@--(sV6A^S;+v;2zoQQD@Uxd;Gq_1eY89$u7xAi2oX0ruAzyB zHe~-Do@d_(CW_|qRu5u*QI&}e@ZH;9XH(oJ3iTDj&uAyd0>eYX*~D|mOntJC$in&( z>H$DHut@Xxk^j+49mT4L1Vg~S8hwlEgEv#@GT^1{$(!ke5E>r#DwMHi+*rA{gBzr? zSD$J%uZ96*Xra4O=NHPMr-~sGO03lv(?ISMn zN}QH8D805BA0w_ycQrw`EZ(FyO_jT-WAID>e#I}=b+YPeYET45AqsnnnUtw{y$V&U zz}k3>-{4Yd`MQl^&X#J*uhKnaj9O;s)C{RV)Eh>iJx40hoO2A(Hc_7(Ahpp3yUf++ zB>k8h?9$UoqYybaypM=~&JCtlGe4LYt+wKYh9&YuQ1R+OeFVmPQMT&0{L1*lM1~2Z zok7_(Z<(V|1W-O$pQPj>xi?Kf-&rWWxr04kl416#c@5-iYL;RXiP%?)rafj#xdFpHi=7uz+S-BUb))DeQOV#mNiBFXlz8{txEG zEG_~=y5-*y!R^~;V859%>I$K442(Omeb`M)g%h0YFnQTHONU+CG@KS>r3UrHyo_MU z*|$9lm44B+ttEd3P9e?8!h;I;+>e0E>zW)`n>1#3FuCzI>9FG7_jy?H$&5ZlTeb z%RRyK@qsMlxaHu)zUH!M-e$a>JvZ8kULQDJeHu7DybL&=SD)j$z0+mI6t}rsJJg+^ z%3P&pZA%ar%D>Y8&b;VrTPqx+RWrf9SiR3$J{IwObOyC#!^IB8D#3=nM#SOWPnI82S_D|+TeD%@~=Ecn0 zAVy125AmV)P6gVf!$|V#iDb0_RZDjT?-2MRRo>2X2M7hSs`vLLI_2Q}DixN>9b2yr z83QMq7>acE;|B*A;)kraMBUyb+EBzqQnD@FOV&3EnT6pDZi4NVc68ThMjb0skWcc` zjVNA{zCzbQ8lt3+P@78N7&4Nmv-swGN3kJ2yvXnadT~){+z5bP7yzIbRsiS)DuZE< z!JDOGMvepR@q>36n=~2DR63e5Ou_PWj>K=Q-i=d4M2xa~HLDgM%<~BZI8_>)e92Bk z7>K8ht>h>wO+{3gv-VAo$ov@)P2VDS-MYT@y+2@pRm`~wYIji>svlkGKN!_;pI%Pi zH~<7hkADk@meqjpLXEoU)hjBRevT2qNQO>qFgAx;;P-c)(kOhse+FOMePfME0zF0B z5h9E6$c5(Xi*7Q(vq6UcbC{_R&H`c_36`geqMcwr!2RxXW=6QQuaW+Q^4nhxB|%U?LusX1&RX>t8}VBW zggZZ}%W(|Tir8R7xu&2WvKr&ON2uB;gw(5U!WICXUa(qbc?6s^9?yRFrl}zE_kWQW=85 z<9)&3|4T)Qo$Vg#Bd@Y$15v`w2O@DN%v0qM0xwcZH2!Y*%&kwD8D49Tezxj%Arf5( z#U@QzdS%=J%=o=)uR_8Upn&P<4kg|^eBj&hMF+u+pjPS@p{)PrG2|8opvSv8DZXsLUg_wGNuH;~?Qv!>o<03o9=Cvb#8Ge0l z;yOon?ldoUpHyokU|r=AYGC)eZD#KL`i6yw9Y+kp+3m41Bg;1_(|uW7#dOF=C0g0a*~i_NPqYOLuE8fdDT`VA zK_A3ln>}-t)_8W-Na;FlL>2a%^&gn}K4HyP*A*5mvN_hZva$6sE5csTsqx<`mJcpB zH&X0d9nUY6wS{+Rq{a78Zx#teDA~QAaujfW+B5^tws1RYZ~M6F?tb%^c2Ic;yx;?1 zi1@F!gBboD(qJT#9?%X#;CGe$l+WiIW^ssLSbawtC`)y4)rYv&&rBv8(*%ASFZ|xB zx@de@N+xa^lS+ZspoUgrQeKL5K`~7Vr7lLJ&@rLK@32c%3dGJmZGYyYtnnn&^mK9R zoLy$UYj)q0(cCwvi@R^H1#Jl?4&KNGl1lgp418)==0_N`rH-W6!=4pnlqkFbu7(nt zkMLpSRU<|DOt@bC)vLDxj_E6Ft>V?tnisMyWbK$wR5te)xE!8wPx!cWgMN*nrf@8{ z2;nt`9?w{au5%{24}JYMSs^)DBH36uAdW83ki?2!iVKQ7!aOl-UdesTI$%!XU3J^oqD-n0B(%vOH_=9+<4 z>4yJa%;qfqSYs|)8Lq_Q$5pSOzbiN7=D}8&p6oQk->d&P_46LrCu~oD!Qyk?=@ehI z(Zq2>`l;Z!@L&X=&28D_cz=Fv57rx_Gia8cNWxp9E6-pBDiKGl<9kZ<8NWVWk}eXZ zN>G;YqdFU9+Zc(!2mfZaQNeOHuU8ohsz&Pi-So$PJLw!gWW~aYi4Y%h^qWCHfA};- zV=I<`&rD3P_st8baACR#zq96fdTZ_0*2#J9J^R6Rz;e~-L-g@DmF!Z!I>%21XMjzC z>5C=7zB=sa?0_f!MYZQ6U+0h(+f$a#iTT&3uP3k`SkK5Wj&-JI3HP_zfMPcM*jTvh z)1*-~m#61@j`uI8ua|8mo`i0)Nrunmn_UCD34ALlNUW`;Z$0_){j&&{;PkdXyp$x? zpK>~Tp5VesZ?5;P%amQ{j0{unlQT|`ZMb+f)^QOC1THF$_67o)Y7RcDUr@MpZMvYsUEN&?7$$sAYHj8hUzv)Up@RlH=HWLvI-Ywd~SA*RmHamw(l=;Q+O4 zhd zOufwc8~RQAU56}y^3}92qLrTqpnPEql-u5ZF4zC+r9=&b*gR5?PuTifD|<1DKpD^R zM=P7-dn>!#S|x{09=Pj+vCX;m?!Rkg7Xez??|-ziLAN8#cul$=-txp2ym88#OSkiqb!iCHmq*ERw4w5;c!T`b7rFGg=3iOf&PH z&~!Gw6Lkmq7ttU|X2M&3@y_bk)ZX1hSuo%PqhX6P{Y0pWhpbCgO7A-8pMwSUNHn93 zwqEX}nLQm_d|`rV-*@xWvr_@49@v);z&Pf$8DYRfg9?n=u^*Y4Fi<?qe5rXj8a?)mf~UMoKt$bCnD3hhdT(UC z7iIpfwV1me+PuJk7Cy_wH!s`Z_;@fWlWj~$%ccsJbJhs9U+FA>%)iZhrhRJO!8Acl}0;zG3+mpx>0ZcO-JT zbzr7tg+zo~fxe?1%2VcHZ}sJ9m!nqAD6Pa0oX((J?~L7A?>E?p4YO)(xr6^bY@iX> zzZLnvLcjT~ko|vAze)Z_!T0H*Nir`94YBbd<@g8|a7}0HmqEf!d&Sj_%M;e#FV1gl z(z7$JpSp$#$My|X8uIt$uPm3TLP|%2n;k(dXPTP#|3H>H3fxeTSyjDS&heeL&H>4j zRF6x8hsSCtDJ7*;oo zz96yY?|C%UN1`;Fbt)>GEcJFm^Dr||e=`aA!E$yGeIx8kWvc#qV@2_VJOsI-@z>Lk zg*%m|Z(SvnT*C}6I(t->GH5NBg_sui88-3oYD)$K&9V-1ROk0o%iK(kC6gbm&Rt%V zcoL$LF-KdAS@~@2HqPs7y*|G-0M;9%+hUTIK+aI4r}$FmO)?Zk$LoY7jIbm|njjpk z@|GNttrVx$5@1aA8O$eX1t2)^|5S zFO2}m(v7O`$kIL^Pq=WbUMi`chXB|V-u(Sc->0X}7jRzA6x4MWOh(7Mo5qhiNzI-x za3o~sYZsT+0A%S|!q~3XMOuoKKWUfe4QW@^BwnJCozBbJPRP13UC)+JRftGcphw-> zqb>tZ##D62$FF!eFWyBT+KO}Nhds?A9GXa<&UR4i0mxE1%j{psQU(CB^zBjkcVuZE z18T3D=9ECH-=x7uO2hh?UI;;4qA|3hxjg=oKQ(;ep$7}^og!o+XJikGkks9jrzMHRtpcR|7?QGMa0Xx zIV@$e&Z*Lvrfbi3wz5L`=&dHzZQ}V3hler?TvEPf^N8JYQubFc zlzFPMdUz^zCrVWp$2@~_=b}Px3=ankFp32?kXL0)M)?ph<_{J*i2~z{~_Oi{X@P<`v+kL^)t7^WiVra@;?bPbe|1=5N1|lQm7v| zLTR!FO6%vfP*_&I7q!O@q2Y%3nQ}TykR#;A4MKW$%sipDHl;BvDT?}6dmsi ze{emCY5F!()@f0bQ6DlFJ7gIDrL&Zo?QJ%2D2uM}AH-69L@+7TPfTH)L1RL(3#fH> zA%7*7I-S}olkkFAFd4v9DFllO3Nk^$jX{z#OAh1nck0~GWtEYut3w7@_&|O4qQn7S zl=a_HZ_oki4a^VqhUPc*M*GFj4WQl#bRi7?Qg7J4t2faA^#+ITXTO(kUdI6+2`m$OiTT>b;%H%-z+}L<1H4LqZ(fSMkD{bBLlCfgb zz67vK9}2zbMb{jEIZ<=L z=aWui(LFC22|9eQt|_#_~)<)7A9&`?w0zgn)U%&bm;Ahnbv1EFh_UI z?=aLioSGwB(yORjSJAU@taWD#pFH8JA8^jyN-^$qkgLtE6h4~Y!=LF}lN*gx{TwxOzW(9qVeI)p1U(BAvETJNyW8OIS z#lw^iOb?6HNlB=$r~{41QNdn8V)-7~v(@>b_DBB4>Sz8&#65K*a6;PW~qC zf6m{4|J(e{#qar>ga5Djn_ue9|APF@)1UJ~rsr|;^`)RW^){txx0DXbe2px)$` z*D2ku63A!SAeIW!+oioINN6~Tu8jT6->`nq-z@z(f1_qJ$If|I`ZIs?{{N7_30BAZ zk-yRUmA{GjEq~+L^|JbJ@;8(JZ}K-t@>oUhvP}QT->4G8O#dJ9H3*i{0*NP ze)d{1i@%vc4%lx@liMFuy_F}g#7hw-F&NzZRHb*K2B)0AEGxaPUo(%Y$UiNs>Vcn@ z6%D7_r6o=ExIR1999KKob2cmu?RnXp?$FK%!e5{!nT3KRVUVLgG3|7g-!bhAt$@M^ zgJHMU__WZ+k{wBvjh5_W=mP`!IfsL5^Bmhz<1A}#WX$qOGYGIigJe2OaXPhq3av=3 z>5z>CC+zGf6mj;c=s%z)#lC+OM%FF?g^^Cb3<4bOI3zxx(O@ED(iHt+{2)5k7BEFAE20naA*SqD0I1&kIkZ~5UzU|Ud=(0~ zXFv(r52#7`^)IN2HJnrhobz`|`(iGsyGhcfD64h;+emSzkGQ0&q>tZc9wlGji{^h} znkY)$^K$x?FWHLx9r*?jrvX6q{sZ~uJ5=vq<(pocv42q7HB$eewBN;XM(a6XYXj@6}%{~J~)+McY3I{dDK6!i0sUENXANJldtjqjg+ZCl-x>GCyhQ@Xp^_YY9V8E37TIo955p8fpa+%Gr|U;n;dpX)kLt8@H| zJ?L_>laUsbAtl0CHd4sUw)u{XN4F}*P6W14(rL;5T{MeOWl>lj-cPO&i3;j2wHxI6Npf>}A z|H1TTxCs>B$b$H6e`w!)=d;=UZ9bb5dPs@ls|~4o!%``CX$rdbZf|e@ak-%&bV|}x zi~>2$B51I5Lvfrl$aqEj+;QmRSmtcis?=)Y1A&+cegVVqGTTyeQ2R#irhT)1)4qxL z)Amh~(Z{dZm4xab^i}&N?52H#^Sky< z%(wQ9I4|?Yj;3z~O{9F=kM_-#app(+23`NUeFGS0;*9@F`=+P?)V}G{{#EM z81G`A8(kFawr=ShZC>cu+nxyQ8q=*2o;fq(AtaYNo7*xPVDxTcd{BRyOd8_I6oGh> zFsE)luozy%>~(Cm-b-+p$a_(Fn-w1kvB_U10OpKXuu3^O&GbbgJ_c(9_@zv8y?wiH zc}szLxt}Rn(w8U;-F1)oV=eU4kkq1l+nt;CO=w~NwQ*(+O@AEr?6-V2_L87$J{zH2 z5h_gI7_m%z$HI#eui7Jl5voB5>Qb&+V@L}}dA)~E&!yNDn=G-jRo-g5acX_el~pvd z{ubYSb-s#k{1u-~T*WssNH3y~ulhHAK>y}Re=4YdQ)F`HP!{@2K3mB@;{I!u&h?MIvc(0E9_!_-W zv{~Ky4oOWj%O^*e@+(5HhN$^(`!^Ba`ZtbO{hPHv1~_{6K!jk<;^cyY>-8%y9Kvwj zi|fZshiu*2Um&zenM&1?D@NA7?wzC*B;?%KtdidFQLce48GtM>>G)B$LyB@ zj>3AOC^Dn8X5ma@yJjTx-NJzo#tu13Ds1e5du){yiNkD+9>_4>6AgaV$>tVh04kUY zxz+Q@1B*{K^~9huu?mXXeB$rduNzBC)IBg)gy8YhcMQ`8wMMk%w!30X#4fzTB+l z$zaj9kPojeQAD?ZOVq1)_%Wo)>aukRAt}wqE)yoF-6|BoQZu7-ZK=UXJwMlVurU1y zPp;|m9B2|RNM1z3?e*>C#?No(h3~Q2bo)%}ony5xfVt4-C9Q9oncH%NA@8uANh~)s)3(L~VRlG+O3Giee*_bOvVo8xVyTk*1tjIG_X z2c}0Y-q=E`7ETDJb+EPTwmrsp|Kf=pMrWFA>5{A8lLwDj6}e>NjhqS*PXy30WQ9?w zNP-@R@Q~-n8Ss7f@wu0vDD-)Vsxqq{QDV5W!4BzTbqCp|hNiF8wzknBpRBCrT0D^& zkqB2p)u$%c_zw`lOfO6r5LG#+*EVX6dpj3=#~?#(66!wXF0qW#)A(#__b3zRgM9ziExDHG9eyn6;kGCL^{c(GREf_{m_(>g(ByqkH}s_z~Fc< zOuc263`a@&U>T&7^LU9H#_`kBJSCzmvfU2&CH};wi=O!CSTX$owxLYdRkTTZ-jT&# zrKz)nm!>n*Pkp7`Iv4SWoe1-JS$h;K9jcEJetr#)ZOMN4SIo*D{4~e?MXdFoS=oPP zW&an<%A_J9`;~OpzA#d@7wlNxVNB^jHrTiuICHCwp)vZ6Wy>VLPS8PyeCB8PkH7i zk~gQkugr%sc zK);2zMm_e;6;bZ3=;Kk-Ab0JGP`GkMq>Grdd8a6NbAeuhCVj!x#p0G^cx?Ueg~_=% z0#UP2UBD$Qp4QHPc2IK+XG7Nhv`a1u$uI5O0QOgAWwnulMEFDFT7R6C`OXEuuFXYK z`oM+70b=zo=-jY+edC%}i5kX20>7ALP`KIzSOE4x1viKJsOLD_7sAhV(;+68ZElWv zblu;Mc}vdiv8{|1Nq1-c??pP3iX=}82Q9M(=^PA*=!Miy-Mj^5gWac#z*BUn$T&vc zOrtWw^g$LvqXhdg;d!#Lno5%f%P8c1HSeD60dGFS#qp1{7mE$OTof3YooGa10%`=v z-ZPFhEmqJE&rad3b0@t95oxI9JKbczC=ThUfU~+lm($oC9t@_2Z?#XXYKae`kqbib z(t}7ShwAza9)VNT_X+EW`W9J0BoyDBPN>PcdD&7q`X>X9w4>7o<^CyzwJMhguF|@; zXj65LffS1Vlf)Upe0HOKL8!-!}`n^TcnrpUYOZv{I0WesaP$BK*CuI09xq zT_dmC!nGDy>veOcL@ z+DzG1ihjgM3x+CRWnS=vWToeW@^M^BHC0`sx9s}k`n~H}*=QCq!^QJrADcQ@IwXc@ zz7Lb3iU`s*XwSm(RJ^F=XBA#FVKW-b4&tazG0GpNX7viNPeqg)F<6L&OWRSu4>2GP zmC5-3BG;lW3=zuvnk*HjKQQnhv(%w3(@|%_MXj^70*X?imAasrqM(MZK_k@hBSWTq zk>_gWe&xpafc%8N!}ugw|7zR2PktZBwrDKVBADSXmBjw!NOvV3$k}wg^|N^3TgAHg zHcF}wK|>EQ23OGE&b8ujEB=dO(g7M!v3Bybr{yQ?eRK7gL3t1a+Lhre_l;<-v3olx z{lTUDQ?AwVe3Y2u_bz2@(K3;r%>#!&%mXPP^T4w|H4n)B);v&iV;*=TtC9hJ_W`;P zu39Bt*xT!oym9EwNZyIx#~<~I>c-u%WCV*TU#~e9cN@6UF{1)kIz0Cjzu88z8Np}2 zea?u5YQ?Sx1XSPtNu^vl_z7;hvkjoSdAzuxy3K>AZa7{V&j3$>69h*;7$YQ9gmI?S z(knMjlYE6kY#a*UG2{46Jdjb`49D6)4N@upK|Ejv5)WuHXndQMk$j((2>`RQ%Bxu! zJ}@hT0%m1kz^trxSRUecvoaIz17KFhd_60J1kK7I;dO_vW@WdpW@RkEtV|7F@e{shnIwXqr94<6g-vN!Z+@L6Mv&o`GEl=z&*eVxCgLr z+ym`5?g7?KzyQ&c9+-_TZd6p%`j}$^F59|U zSK@x;l2ub7lyBf^cWDy(!h=~b_>+h&8!`6e;Duhrx?6mlMs8xwA-N%j>ycl4s6qs= z;)4T5>-^NLzH)`XLMfA1aT|y|kORZ%V`ryH@P5X?H|BCb0qs7YAr})R;*_>o+4~m1 zg)kVzHAxufy1g8nky1bv(WM^(gv~p7iH*1m@OI9=?1WcDH6^K9)IwlZHWlz5X^Vo{fXI8DTDrKHD+e48BI$V%mkoM?6*ednnszW2Jf2mW&2t_`i&@ouqxv zFB|Rn^)cHZFCjbz9 z$rO0%Jw=9h*1t^X=Sb18;q78}w7SH5G%}IeD6rCyU~0!U%FE5ZwjcPh_hRo{!`k{} z=1$aH>gnc&qmES(S#mA>?Dndfn>35j+}xY5SSafr9=QYwKLg%XIXUuVMlO+h73|KW0>aOV1YlRT zM-A-CTma#x>j1&-9Dyw);bsUfLiwqO5>>M?E5(J^$EgK}?_{7a02v7DiVS41+h+m~ zA_Kj>CIgKDWT38ZWT4>hWFW{vF5DkvpnPD85a(YUw3MU?ymh{Qh1Al{)El29C1?N( z(Tq|}7%}2+3J+%8ko&>`rr6b7v@PK~6FQ3YW@EQ2B|G0)O(9hQM299O5qJ691-uft7*v-vH;f%rv)Gx;OJk*p}I6}V#A-Z`lKc*U}%23WSo ze}`oYswOf2wwk11ySj5-P0Hm1s!3mdS4~nK@I8;PH~xJ!$=L1Hm2U}j&D(f#bIl8% z7?*aTDH-!{*(S(jL6g8%rdMG7LfquiX<^P*3Tg<<~vW@(O@3W(tt;G-J=p+2@BZ!iy1Vbt0$}o-ar%8`V zajy#5!=OU;J4r8pWT22ur2`bQi;Xu%s8^=3Tm`cjQFB`JcP=X_m7OaT_#qp^@n#uR zHZdI#_l&yheobf373Els^7EZ;LgEYmx;8n9vyb9Z=A@e08daZ|I^F!* zBuwUfyB%KdYFS2#rE@YqPkyy5bExBtsDvF_IaMo)^<`0rGSyy&Vi=mz1eRq1vz!qU zcBzg;m>!^I+41_|f-4%ZEW^59mQCJfUj!}7RzSIk#lm>&4u7kRrc#U3EuKkcb0`_CJoAyl^*4$#|TGEBF4k&vOKAH2;M~4 zCYML|Ga0D9l_?BF2Ev2uWRsTL9pG9y3Ak2AQZ0ehkuZIOmMf-YK}@uXK|vO75nqj6YApmI7Bb_RN!gpt6_)_ta_3o@GG8Yp{kVf*JqY?xJTP~ z>El0jjA;rTCu43vSrAkt6@##n*V++6&)PbAQF*5&qmknyw~qB23gv@X>iP(cj715> zSXue*TtgBIvt!cLgL6aMNp4{WQME{JKpkoGSJjctfBKkVTcHzleR+Au+#0SF)&pWC zd|EIK{-PU#m{=&vx@Q>nMqd6;nn~Z?<)mmc*pbW&!sJ@Fj2_{*tJOL0g|K%2A;Gyi zM@ecn4_e#GQ7!thz@F;IgK=X0aF^?Q?sSeluWI9b+pNbRQdcNo&5S(f$wYZOrC%!r zzTbhQ>owF^%@N1G$tPc zxs{hT3#iNgsXCG}!QNcN{$21S8RF-NrOLNKZaw{1EBSvbBucP50fVKrQQO2R~9GO6cPKVJ9id z-Z&xfk8ebPgkuzhEG zTn+na{aGAvjOt7}djPeR@>NrybxxOl*=!{iy{_|wF{~5fOi4>@Y-(zH&6gvR|_5UO| ze@t+&=MSDq{TsGr%syz`ekp2}%@zEGlLIQzzA~_S5s-c2?JAOGUec;=!xMOInB;W1+W-vfP;)IjJBRFfLj&IYxhjXcVYSl!7 zq!DSg0*amVENy6_LSn!#2Lstg;?(=FwT9H{(tfeRlY-YKumq|C>U`3BZsLO8!}W`k zSv&?(n%#k8)rH3DmWKYJ{U!D~($q)z14LeDBD=Nyi{3FK7+)s&O;Zw!S9zxCjHae? zGgDa6b-6y69x_MWu0~BfJOJFeDvRgs_zBzaTAo z?w3s)gWV|=%~fD|hlmoVw5YD1&8%Md%Tj&TumAM(bV!)!Tte~Y0+)S zxaGbP&y*}RHnq>%rVe=MU$K1&RS_+OxWqF>9yS%KhssKa3UZZZwFwDlmCfR;^)r^; z_dRh2%e%|3NVa(w$pUwkjgGE$oi;wgMC}w;WEb4H{NutP?*v_Uzsn=*k9PSsk|Dt? zmdnw*MP>L*hYwf^K7|I0O6kCph$OXo4#F>DHQ`mk4T>hjFr}6XQ#lg)Z?%@=`HQ;~ zY>&zumDusdk-JKDM!7>pFQSW!Dxc7}PO&<%pwGUP=`71H$O&>1dpp>x)=foiJSy0wNgfbUR6mxSXRh2Xu z;`G!K**JRil34LG5oh_^il`0%Xdo90yn;s_GIB(I6xlsHHmTUm>gHU*a<#a(}{t|3L;$Gjqu=mP6E%%^41q*Q) ziiJtORFRX|=I35!eoE<>SpT&!`MJOQNnAp!z`F)5pGT||6c2aX-1>v)+}*N0+)bs4 z({eLocN*ErO)Q^DLWGInG&y`aB_$0l11s-~PWAWa*F(2gj=f_ZAd0mpjX3X20aS2G z67O#Pn3Z(^E3?Dddj-~=U#!K^z}SWL@1-Tb>Min2*2zX=AK{0O9C>Qs0r~0Ey*YPC zV#+M_{uVSSC^5vtS#)u8{_0P?3bjT%nJ2u2ZKk|2;=~>+VsFz3=}Xe{KM&W}bU%XH zHF>d7TfW^1uAn>NAnql**Wex3Zf$2}72pV4GhWrTpyO{L^YCb{9z)E~<@^{nXX|_j za`g#A%TZaUAePwatmU?w9fJcZ{r+KsVKoNr7PI;{v~yYUs#3Nzx9Z}Fn~Es8{^g6q zilm9l`-FNsl%zN`(I*jlaI+t3lce+WE}8Dk1f(VnR_NlXkvTtA3ctwb$%$&PJPKt| zer{hbFXyyGAJ8=LFf0&@BCTvQ+TO+EQEb)qw#@JaF&o*;lXt=hLPeMP$kg{NUq8-% z*Mg2x#!n(IJP-R;h4}rCKOW0JB2k}gF=nA;5Ge?-Me_kU*GN!ag^K*5` zQG*-jwMvs`wNbGN5=a5+zB+1>YLXD^UO4S<0*e%UO+sn!rx!c>KdH_x@v6q^Gf~Q#NKJ$G7pIGM73mo+s;W7rmlxT6%L{^@8!jnMP6E)&)n>`pU}&~; z&HxL&SV6oiFf>C!OF_A!99=qxY?U%=`=^Wol1){&=zY2z#~W<}!MbX16>QrW%_BQX z&4((Kb!n*KUl$u1giJGxT75$9%J?c~OLVB9iwA{WNT){DUj_cnS!DS!PDW6!ieO~pwo{3} zey8;s5}_tF=3@~x&v^Mx1I;dBSl29uO>ahgdT%lzj@@x?qiR?OwJvKTNshz>f{fJ= zhxj3R&Y>&%HL9@P=K&fkD_x)w9AHOeVMvCTR=YF6eAf^?A;^LHMLrD-sV9XScwLpX z``g5bhe0UubEBH8kQPPDc?rJT^DnsY3HKx-vYAmH+yRo+OLtJ7fOX5-|> z33#UV$o0U2i<{?k52n+C>u7PhR1-uTbY<(I^h~D{9iE~;b0zBUQ@opI3B#3O;^eJ+7J<}7p>b-Cuu#D#rP!HXRmbJ4>U4R;E~s}BFfgOZN$gX+^L z!MzHIMaS+pfZrn_&j& zxHtyhjR@p*yrF3flRKa20R019g0fXsTBsE4F_A;HLL(2EVOC}gm$xN+e2cai8EsK! zdf*|}K3gMp$!*?bGL6}EOw$8lG$n(sEFIdSf6(BG%#X@hkQ}n>7~nQE+I!B&n!KTu ztJ7#xf+7-$A@!gj%JufEyX1^jD7*HL{o4RKq0Cs*$8cqAZ>`DXvROLs;mB}7>3xoP z5s&9`uB+gf+3y&r=?bRxVrCSfg4I$i$n0P`b3Gj-2V11-#zP}2B?YOucyKmv?HJ~D z=_6s|&$bp9l!zKO_FGB=wd55;&J5y~bnKVE?eBOr{f+*PYbF`8l=&cTLs7nzM`c&|`mwh5qknY5#~T1fwej{w%HW8FuRKV@v(}U&$eigF<0cu-0Rw_VVT~CON-m zm+ohnD+5kO65Fr&?7GRh8U7$AWNaq8b}}k@)+q~Wh-*vC?n)-vHkw>~Iw;P$ zk1(w*-3U;jBViK|=+zM0Z>Z44AS!e!dEE>*abwe4&>d@yKyUBiYhUV_T$AT+Kp8pe zEl@^Atdkbno(ep)Wm$`izCAk)J0Te+Sa5s8wr1|HTJnkJ{C|uRd%B5b}6AqXhKh@Hrhk zqBWcx7wPshk4qoeZuxEPpnzEOHHJi4NpPwxjDF2+IGoK26>Z5G$pk@kcNqe3 z(j{0KpB97%)<(6$eCu7W-{-MTQ8QjF;65XttVaKeYvML~SwlnLwH{n|uLH@4%e#lG zjn~S}MTvkVgp21O=puPT@$Psgq4o$@Z4Oy+;rTg2*Ry66egbcG7M&1t>Hgg4ck3uC zQ!Z+?w|RJ8COIHqa2cE$@$bevS40i$+Tbs&kRdrlJt$UszhNjSdJyG$;;JEb#YhYp z;ZBcj35I;Rda13nOY!vdvo|Y2nop%yWwww)jfMDWdbwkkmECGef)$PS00UBipr;vj z#&~B&#;1#gn5^@YRjl!S2gPyzYNfp+=5T?R@*zVK)wZ=|{yF$Z;Tsh>n-{-(N&IWV zALgG%?*9pY{uurcoT@>^2IV~uFHo_0W(TFNqo0>johTkq>5J(8 zQrOPlFJ2757Qbq?BVO(|TYQu2Eq;zzH%3)7oC)(t7vup_t{blb$V5$DJiO!Zz!TwF zN0hppmD{^Mw*h;HfDn%?L&{cy;o^|)^D3bPPF`{n7qfVCz^fM(1yA@H`F=6AN zD3fx0L~S}4w`ECBA5g0rBkqgaLde)9y*n&BvxDJQX>UcZ<#Oxoyc|P9q%N=fW0E{F zj7!$Px4r_VT9fI{rw_lCci;v~Z8zZt&ko=QkMLBT&7G;^929vPU`Myf=D`SzdSPus zI~mdsRAKb9X%3s|qBF1K8?c|_8xwVJn}0=nhxr#<>fgsVie_WNC@$SO$a{F~FA3Gw zJZ4+4O`QD`*R*Vrp{CVCQJ z+bV$cCQWm{l*v8OD*UT_gPs56;Gh&5I|wW^sBr}rYPbdqdNx9wm&((3RH97o_!aHs#7-M(6iI$FV;9Wa~1RZEv;M##+!(qX_A%+%K5pl`izL z+MxbxXRNs*Y>wXqhHquHdCBumRb1wYu}Yo`S>)t3f_i>nFO971x@7&ks2k90mOF3k-RX)}XMcXF&9;EZolnwx0ud0w_}EmX8u$ zEf}%Hp)k)&^gUQ!1xL#&&lIAa8T9aug{8MQqVk^y2Dh7R6E1NP##}r%2e{CCrW}B?(|u6_0^u$Rq4%xjJiGfM%2sQJlHgAF&b7^%_(CZNlQ+H250!I{JbgoxNrV9^1{uvc@=a<_@n$;Ai)$I}9UEk}UH(zA}xMqI`U!EFGp>pCW292!Pmvd1vT_vpv4eArbTS5_^cSJiJeQ|=rk29Bx#kuI!LFFCg1)tOF z^3KqzXEOXvd55nlv$P9T-U-L2af~4k8GpzeNY>$-FZ3REA^Y>JKe^IZi}gyN|J|4z z%PnX9X3^;&R)iK!)A3jel;14CNG5rH-YLhJs8R3is(31Sk#Yu97VDnIgmt)^w?Cjtrx+S)@#ZGGl}Vl^{;5Xvc74(1g^DS zP#~=r7@+lX>UsUUrwnv=OJbf7)6G)`7W9;HNSQ9=Vk7x%b@$}Wv%PUjqss6QD-c$Q zqF68g8Y{#GV1?MeVTAxjA{1!|{iOGbVy_4nqK61 z_&#H2g6Ae5dvR_QrE*jqEL`^N#;C;Z#ZgRN15|=!qzO^r-se1xmfv!-p1@n#7Q9&D!ng7Hq4QjoYc~+qIorEo%vT-U)Nv zZJ4H?icpNm0##5&Nn@2^ZYL>WJ&m||$w&)-MiH0$Q)J-)Oyhhj#)3 zORltDio7nPUnJ{IBpWnE4~(w&SN0W7V!Nql27?!4o(lrUE)OzRS2;Qc4oP%M^w#!C zxchxW@E%?Q`>Sx7NYci@`vp&A)|Y}IwO23afB;9O0<}T;8yI@+I~e-3bD$T!_y!C; z$-vl`83gUrdEXY-xISkf@uUarDeNc3do`bWa2}{KdOV2%; zZf+UOrYXNyWRTb5O3$mJA1D$`GRzN!f5i6S!y%*u(UE85uOe1Exp2 z%%!&?N6IG`keu<*>^x$3NVgby@pgj-&n0ryFF{5fYL^K8aSlhEfrM}_*rVM4Sq ztTCx8*JH*x9lq6SrrcKbnfKQT*+IXjwF6psM#}2`lyQ_&yqeIVN!c$rHnD!eq zwV|NO+;-VoQjf@Ei>eD8eX!**7?(5mBQQmXc$?l}Pg9{^vbnu|K33d;#QJts#CqJp zP=5%4YJ-|imees+Oh(|@8&8#6IaaOalCm~aXl!H@!FBQSH3fXx(_vc@A{N)CT)&9kvl=k}*st&T4oiY#UEFvM)p% z1`<=lD&_+jaRsaHTclaS`z<8$F`0z~v771i!tp_;w`vt%z8{H7&up;_RjDktU$5E6 zp+XkW^$3ri*D7r^pVBDZ6Q+zil&XsIA0V%k8I1`-U#N`?3PMdcmlpBFTD54v>Ay9B z$f+jvdR}hLQ3z)MwaxAJGVT}iR!5n3FAwCTPh)-Ai@L9=W%`v%640MN!#;|FE_U58 z6aj~z!Fpdxf|n9zzgc_c8}K>Y4%~eRD;s}X={{fTX6pNp2TH{5ivCVE%l9K(!7&Nm z#6%Kn$#o0WkL~3+BxLPJZ}NVoy$ol#1OBRM|NU5rnoiBe@r#*-zRszi|6dU4>H6+p zG0*V}VITB(-CVk}~i`}U2yMe$J25)W;F75nYs>{;>EN$&t zrZN)k(_n+{Ec&R^^+XQKzZ$NMwE`!jgrTbS_PEHaDdG;Q#pd1h zwaL=u4Yu71I5$%{^mW)as`i?VK@z)<$Yn&;!Ak|>;x$Ng_`&I6lz-`Z`&n>f{+W-W zT1+WS$PiseQOx|nB<8iJg>K`%0Apk+SGEiQ`t15E?{P8hudE}5EfnwMy+^R*;E;)A zlvn+H{n(nSw(vsqkzb>?ilWw~g!q-*`agyQdsWJDp6M+6ZEwvpU3stw z5QRL>yIMs`cvoVrc?fKf7Gh_czE1FEL%S~925d;n_KiS(>N?)q<2i-{4fNShr50s=v7~2T@CbV><+!} zTVC!i#SI->n?>Ues`q}3MR01aa<%cQ>I=r~wECD=aJlM`Glfs=bhziN@Syhzy#8W? zBw-SP-IIxDmrJbkCue6=MVc3x+zZXzYw$c%l&m<`hfqZ_0VAf*&hFH>cI>L9CgGz2 zw|(U#dYul2cN~n*%=#9>$FSmbQZ7*8vcT{1&-DGZf!?4Hsvhm4tw97JNpUE1i(n!N z(fy7Ne{m4?Zu{%Je?@5FvqxdO7A9xyTJ}i?cRrm~?xtnXd<=hYB(GXLsApaQFqX)E zVl0{cDP!qwNm#wB^AXI$JO_A2< zbD=Kg2~^bPr}93Bb({T?u>^U;SmLVzF_u=(AyU4E@sM8lW%GP8_^mG`UpZW#I@C%C z_ccOP&#O>10ft93bDkF3?jut2$6jmcYU|CPwMj!0HAdrOf0Wwcz`vK;ZFz5Ox8-XK zu{+vj69ju1hZ;rk_HDzCt)p-e^&1RVz7#3Im!kF_d_6JP5BuI-$cmxbl@0&Ti&~N< zB~ZJVY*43dJ9l^tiHiW-=B8$d*4$<;+H2@hEpgq5Z;+*70J2nZ4OwacAWKibLzZwr zkfjT7iK~duz4I^v4}BEE>Lh_C!Q;^X}(;?r$K?zoEh(kQY)5nlul@ht%n zAAaGm0|WDa5%H~Hfg(P}JHL+jVt%m0kHWZ7wQbwQ->{|e9FIz_;g>G)cqmiv%QJ!8H36UU#v6Fbp@Ol7@C8_%);(rIPbj4xqhn9e*3I%%|Hb$VK#UnQ~a&j)7(%0z=J>l`} z@W2cC%gZsEtB7yY;@Hl)DbxXDn@>bweat&lFm|A`*iv#JZDpcd5koZw-C+}UJk!o$ zVD9?I`S8W2PjyCIW#?_{a1^;fYO+%6+59tS`~DUa8(PKpOzwt^M(A?B$Z_c`iFIbk zQU2cTn8Z`TO+^`)1FWB2QVWz{2(PmY@tC&Ey;kpQ6O7eA)-6eTsXBWoD2aqyU>%H3 zP9E=wh_?5b>65-nZq;KO2E@um$*jpy{W9v!X88b-fwfSd$1_1<(`?|hs!{=PAEDtq zBR|9MK4yDgX!}_x=+m2esEw*VnNKG2mc%ukZ6q~y!KV?O8VP^3p>zH<{e}&=-$^ZC3a9^?BN zD-GO2+;kRAS+#)%FcGCIucNbm9_VGL0t3B(xZe)+D1SZBbH<3WaFeW7d-5Hzlr*7K zZd8{|Jv$+K4@U$=bHMQYE-2yK6YV&8^JAWOmGJ%LdERxx7Y0iB=KZOZW-Lr91%QN) z{Cb|p*){n>tLUBHb;7p-BPWo?;+dmp_fg(z2Qa^B@3gIL_pao$GbkiYK7o%`rJdn0 z;?Ci;+n2R7!dKr!SrOM2mK5qjvBxNrd5Yup=hw<6Nju7u)iJ_X37oaESTxUhq38h% ziWwa=4C9wXk0!ZC*SJWn_d*bQA#dF&-Y3>S3KLm=M?&nVY+Jmx%TT^`|FE(1;#27{ z;y7SIaWW=NUyOsp{oo_{N^_9xPVXZhFAn~@vB0r1K!*e}7c!A}3BBWqgv0K$I)k#3 zCvU<@_U`&G{pMI{Yb>s>e`P_*b;ZER_2@;0M{Z}+9FNdA5W25kg)rb-8EkJ184OrZ z9zVIm`0Rc#iMc4PZQGRvr9*2x@s)T!lN($@m9$UxH8|W?xdyrfu%J+C$%mrx zF8RGfWSD$I4CbVQQW*S z(NOnCaQFvQDdP_jd~ZA(E;NPIrAkWy9SS7+8=d-uNVZ;%8Bzc>fJg@e#lEaM`^fKY zSM+^sXmMIlLyMQ=*yC*pH*5(hdjT$v7+XSlY#R9Lk(eZ+rGzbf|7pXy&8Buq9-o1p`a0l46i`AJtfdr(Tgg3w@ZY5yQjh`OF_N!| zK&Q&`+t;T`Yz7pliYho#?VM_Pt+N5v4^JyysN3X6Zx>yUYXyrZp?38 z0tAbGF>;Bgfced?Gw_0NIEZTZ*|uHCh@A{~5Cy`?;AL~+f<(l@dn7YeDRc(}2UY)E zmpK6DV`HPdxWOX-WKp3hU6Vl17kY|Gt}I$!Uhuq=TlsW`e z-wTq?AW?WPrtD7u2I7XFIqjQAi6NP0cUtJ%={>pQc50ueM0T#hNGac%*S^bD_90GS z>NCuaH?pd5U=It1)`Puef_QPe(+~qi^CWCcCbH$&%sVIA+U7HtScui$wn6rpLBrNq zB-Ti>QgSezX~H70QjXE9LHR-NAEGl;ynfy|L{*j``WRGSOiv?#pfMOF2iZ)$**8#y*%^y5N0e%o`)ZFbO>QseSKkA>J?8q`|&b zPjXqieNwhu5ieYDP9B)+E*`9>*;7o(w@MFu#4qd*0)ENGr`&nyU+~{U-Pz|WD2eNa zkHRDw%0Y^{oZ_RGu=O`pxtN)5m`<_dG|r;DPxQhF-dr<{3Z7VrGrf=mwlR6+Hfk|T zwBstX{GbKc{db&==veD-m|=6XBhjNxS6^u&1kIA03cB3}Zy6m_F?KNty&EUU$2Jhe zA%-b7O>*LD=gH1FbXPy9b-a>|Nvc1@KLEBP?;V7NoCU&)HD+fIcOh1a74|upK8;^V z*Yy5Q@Jj-ItmvugZrZg3GxOU=Dx1VAfy9AD9xT(1eT`;6Ct8Xh1X}+ciPlfs(qAN6 z|E#l?+cY6NFfd@VDzUp5U1Fx}a5ui_u-N7Sv1j4rfWS~6$s@(trH@Ai1T7>NphNeD zZDQtlA&%Zmda_g$EODb&8>Zku#?s?EWg~@Y~k7$w|1JwzEqB z28&36kzNEOY}8add8iqdGCPq)5kM3o%lNvgs{Rr& z9>?F;{Gm_e=~<59O1<=s%f<$hDECG{dCWy`PRsuGv8-XknXFSspwlq6bEUK?^U?;A zVH#e+#D^nX7uV>;s^?;y_sQwnOGMRMA~cC?v_4EGcS@8nK5Bh-PJy8WL*iMV;>#`( zQ>9V<_6fZ`4f$7e-)NOUO~Fd|Oa~S(dzE=X*?C1aMIBuMv;k4WgvKun*VOR5_d)k{ ze^A5MK-BQo8*2FC+~`qCt~>a5UH%8**3sY;T`SkNA zB#~P$xSkb0J3{5sMazMPId%BMJ1{GD@3kZBl|W`;i&(QQ4lGA$iNrqI0^l^nO>#Xa zMMHiD@x`K(HjGSwf!8#H!YMws6bDq*pUoGI&w%u@>CsxO>(KyCy{r_#srL(PHk%qa zw;B}oWU_OJi!%4IueKlPB?#Yf^%6XM;!Y(^yY(!`;|n=MY+{OyyQj*S!B6It z51#ss(6SP%iM(r-N6}dc1&?$g<2%^l*SqTK+kEfp*CF8K1gU6m&q^kQqNitYK@0Ur zrlP&a)m!o+3U(i*^<~t~N%hb^LoaH2T1npCcDb8uyltK53GYMC{#mnhl70t|AK{Yc z4t{2Md=W%@o2->asP*7OY|J`;LU+3Mk*cbPD`aPuhh+-E4_}K&)&;_i(WITsg=2S2 zpAW&d0h7hOvQ}{{vGaK6U3Xxzh@L%gSY^nKX_Cu4(ewpLBfF+yQYI3=(e8c$rwsQy zYJ`)8%h5AEqg`Gf?@(vIH?TYDYuaXRAFN8PMObXq<%q_y!}{~Xy|-!VwZAag4Sm&> zj9Hk=VN>*|Z!fE4yKQH~LTgIo`k4)>Q(OLZyE6FBY$tc%G1$E?3dA8QP?)SF?}d%? z`cX%T-Xoxl{GE?O#?HD?5W9@J&+@=1+#s9nCWMj8sr$p&1Ye{uGl+4uuxhnErTQ+<YrADL6g#+RuL-P zR+uQ1WjP~#%a7sha%e;|si92l*Mj6xmW#^qHa4_fmQHj#_Rn3FG*Rh8Fyc6#`6tR0 zrRJ|a6)hqmSXbe@IU@Er{He>tUU*%#)_!DK+r9Qs@8o@Lc$`|6kR^&gRAG!-HV$8} z4p*?{($Mq`JO8l^PvP%D!)wZj?tK;xM#NDDgOL#o#ISuMh=U#>Ne&zPI0*KRB6*U4 zkLQy3$Vh&7!xB@}nosn(pv5k2sjAxi;joYlJs;-p#J(h4q`^&it=D4T9%_)-S5E?n zvoLH9f|f@e-bwl3 zwW^7=kT4XM+2?n!xanX=aMu?v&)uA{4H_UJ}Wrp8{ zS~{cU5nMQ<`TBE>8}KTX%dg%8sHuK{JawLYXTb)d=^p<7!`@p4#kKC+-Vq=;0fM`G zkl-31xVyV+<1WF1LvV-S?(Po3-QC^YZO92(C``5#pPD`4ENq~^m6Liv}VVUzx8Zd6Tv>E zH!!WlUziAp-2Y)(`w0yvEq9jX2>XMkiMxIjw z1m4A%$0SkY>r4ielmwWpjU@|h|)kx6=jo64v`vl zMrfi^)$4Fmd;jXs`@YA(d&K{U*dfE8 z5~6=6;Q-Ko#tt1|?Utyc2Va+3Mh63+u7R;bWMJ%&-6Xa&w)JQ1kOoW#gTz5WpgX^n z&hs#DH!r^Kg5=TV?9z;a8o%OlvC5p-`|D)A(~gre2&stb17hV&MY3|@w!=}lUW6N4+oj{w2JM{X5Df>~gIK>TNbK+dFK zke8;$t)zFewV=Fyh=`BUVUGaJBr-15(J*^Q2rd&Y(>~LQrAJDP7}FgE z$pbcy_U{Z3cM9%_6OeB}PIqZA+IL^P@-{X}vJ8Xou0cumI)zRv!B;ZEzm5z)LH>9P zB3=KTO3QOP(mTWcO{C;Rkvb@R(Z$f4_V!l8k|h?FolAj*h{3z`!&Y|b`I-jjuM}-5 zbxG-WvJ*B{9y!*W?2QtSSaIDC?MFaU((W%)5--q{WP6U|a`d;RB)4Z%((%g$pef1z zrzxqSKKOT2(h{C*x+Q!OKdp7jlbo2EgYe?e!UJBw{1Y?lwk_xU2kZ75ircYrt)rSN z6@zIuj+-Lq!w+_s?kqEmmMIdeNW4*{w;DO{O${TdhD z_m*&MUZM8A!C(sy+*DCIl?jOM?N@-aIuv80H=P}NH}V@$Xdgr~#Dm7>%5HWT-;mDa zK6UU=aC=K$?`RLoC{V|muZ}=r2l6>~t`bk_D8f@Cny3fkVWT)oAvq$8Bg@TLkk!z~ z)0NwZ_MR3r%zq5hOBCrj`eX5%irHd~iqIa2ZY|E#Bo|e*f-x6v-4NSSUGJ$QMy=h1 zo{8C;fd}Y`Ji|;<0s{#4Kvi;Ln09v)2=8}^7hM*G1>?A6fpp9$^wuNO?<)MEaV$;9 z?&s=pBqKO;_5E94Pn-qt=R&(}{#F5u3am;|x7?JZFp|~ek0W(J5F!s^jEjUNLr9y- zGzeTA@L`NMHF$mZX1XP%;)jBxVtUhn)&`cX{Z2F1_O}XP*@^ENw5@S|FAH88fp@{2 zjmW6ap(IADbOl{g)ZO|OUHrkTKWDy>Tqrw;X7&!&kmt3X)9;{75w#5~gP6EW0F zHr&(5*J<=)5Mplzc%pP5!CT2K|Mfx1UNK1K2U5funpw8YZL4@^@{kQX#%+{2peN}% z((*jq_8pDSnAY>QCpHtf?J39qx$P0ehZZmPm>9$GAKFNp9RxdrbAY|{Ie6nT&SMOm zdHneTfj<`!a6>0d*-r8T#XXeg_$jK2!Kr~HrT+X zlzYfzx&Q@l@3V&U_m;R_pSHvzp}n)B$vfHS@eEUuq75snKZJm+c^y_Pc^qofUt#xh+nV; zYESH9FKB=a5Vla5eBa5^B~~G?4Z0a-!ei8NN5CDGi|mj+#chO$Dls}%$L`EcCYV(S zByFBiITLs9NAvqBgzKJ05ym+Y31w*rg*P^Y{qQ6i{KJzp{nL{)4fG^!|MDb3|K>^R zdG;jf_fJ#AKSRUQpP}L9r%x`VDlT<@2MwPap7r=AX!zm30}bE&2@St#`2`J^s#FhT zbci=!`vDDK`3Vh=oWaV&{vHLUbLM{FR$h6Bv(YZhss9PoQSQ>R>f<1bY8_!h9W}+< zLueJ_nZ9-h{2K5y1z1NIJaC0sX1Tjh?bw{* z^Vm0!H8$dT+bl7CqvTOHxZv<~?`GiHRO;{v|1x_8_x0a#Bpu30F6zIm6M@a&5C-=Y zSB*}AgTrVlEFdD&=9i?>FKjV}ep67j12$o9eP9)g zO?^*L^dod=r!qg4`Pf9dtECfd4s3uRlpXT$Ocvxn4}z+S$${NR(>KDSw0xqZkdZ`3~+2 z7EV?sqj2=&o=R+xt8)U?VdR1tABJoCe5nr02-P9#$&ejgcg?@W*H0SNF*!Cq<^OCU z(U6Z!t*Hf$gj7C$|J?BJjD)6ORhZBf60q6oP$fdT#!k4O&TVMoVa4xoLf;g=CSl5F zoHr7>9{c9rAS4$PXyA8?5>YO8igV14hV}6EIdo{ljFl`7mcrlOL;VvOuIwV>YQqi; z9lkNh7HA*oftpQm#gqsg;&kP)#)Jcg4$Z-Vp~EAb_di32msh`q4)=`u=Qo!2Mt#ig zA8DCmar?Z6#@e->#=2-z!kV+V*ixV+zC0aDd@;!8q95AqdTbp+X&=!Hn0sS@5}}Xy zVBUVysmsb!MB=scjf?dW)ZW!iNmwWLX%hRuR^oPTh+-pat+lmgrI*%&DsaDqfrCLuUxtGICCita1}mRR)qS+j0z;8s`|-{0$&Xx z_<$!G*wP_KJ--i6Rxz5W(xYtZD(4vpSFil>_trP=ziWNd{jK#)*|3YK^;4;5dcl!j zzH{W{_tyT>mvtslNEE{h0l?6qH2V_dz(z>Q2t)@Kl3>;JOzP4zi-xEyZrM`Uj^>fRijx1UDM|Q; zxf=ARxtc*UqXkWW>+f3M9?CHPEa2#We#S<5|HMY^|Bj7fsj2~Dqk4Z}qhd4|9e!Y= zFd_~x|A39sd}KF%#zyho*wOx-fV1>xY}E0;#70T{9X1N$57?;de_*3#f5%2~+I^hv z>V~}+xmuXA|0{ssi2o`~N|}vHyPp&c7$%Kt2Aq+$dlVE%7tl z^N#|~@2N7*r}Q7GGIzTlsj{`_RGGz4Kg?F~w!|N6-wuD&zTNycwQrA+v$XCLIA=dn zWk6*3uTE*ZNiPD}O1ToaOU>Sa6IEbD5*(X1-~ddI7E2 zGF>AglTt13#C=E5@US$a5qMt>1`ryOB1e`>4ATBcrF!O;$znY$>A)GLxn6PIY zhzx%a4&UDh?tdO!G4DMOt{fsp{|~@XT>oEyquibx|7~y-*FPyZ&Hq;v9I{Fx&y{8z zZu~IRgbXFQx&KWG=N`xbkI{(ePra2EEuL)B3^uuH(i{LZYX(2pzI9i>ijUd&Q1A^) zg9m%L6uD4RVFo(gcqz(av)9UfzpSUXi)m@`9^%gm&ipO^#(zYrjNxwr;(t+al$0b0 zgb>`0G_-DNVH|M^E?Z%;y604N64VAll;yF*Bm;BH@_K0u$Ddr&ATbg)4S~0fmOn3yZbUsA!ecvl+YTsG14Ek_Jx%v^9D(k+Bd)9Eu zex}N5ex=F~fT=Rencq`o;fJ!4z)sdW5s3U1VF+=odP*(~YrvX=_)6(8QdKJmOy6&rBE6O?h*};r5wTjy;3VObA=84=@1(9<8tvnS zD-ML;uQLH_-==D;eP^I2A;I6~720<3@nf#3JHYmS@M{au&GcWE5$;lXv4bdhfM4wm zp)G?D&S^T;i@FvZSC*{1l2=+VDho~~Z#aA$ID4a1uYJ`ORbSqJ^ggMRYBx=RS1w)5 z9>addwd)7=d+HhcJ<@O8vy$8g8Pfh(FVZYB#)-Bkvj;hc>2kXN<+JGfJ5c{3;jQw_LSo($io}>5;`<+E=jq;5BPL8+=ZG8mcJ9Ll3hQ3Q- z!af6iFnJs+?>Tn3K6=!3a608ZGkw2*^yI?%lN;s4X@opN0yQ{DG6pZqfi3#xVPu55 zX18xo8NL0!`qHr4c@)Q8$s-};&D|P5c{hk#N*X8Sm@E_hC~SaZb|&CKOEHDoY9*e1ulx1iW>@RUw~tf%MCeh*tc*r614Cs( zG*SgX>^I`(E^H-(6OFa0=t7ldC;8Sw(I@FFqAV03_Z#GMDxGL?2fL|M`YC-1!$j4| z)<${|FD+n9)iQy1zrQan*C}Z@gPSzJ+13*(Mlco$MaiOk@VLTDsCcX7tw0duMkO z`wjImr?Cxm&Mx$Kdvwt9xmWP*PI4kOy~;y-gKDlb3$fkZ^`D*8{sfLHt_0zM9CXrQ zW^Me-G-@rU{@ugD8eGeW!H-y3GuU&i%>6e%@qZO7lld7d3nHFd)pwVgzejtHm9YV1 zWmErFtn69CsTA<5{)Y!A)UxZ@PW-Pm9MfRQCTEO1CP$K!X=q}ySUbNEFzrN1V!q`_ z<^4#He>B(pTEL`%QGcIUs*!cmD(o*;Vj_Lw}qclT*fuqWPf}^^U zJ7Us>%>seor~|Vl$=|?H`oIYs5FADM3mi4`431(1C1hqN$(UxQ^?D28d(a-#lxbl} z1oRb?zwPlB_+Xi4+lB{>m0_zQV6SJ``@g0oW(B0_;?h^jl5NghZf|1@Lul1@Rza>U z`g*^Vcnzc-f*$K`c=}y)SNyUQpK3L0{v>}p+w8;qEAsc+Z{+XLS!}p@=+I~KHwWuu z)}P4Vwf`0QJMcui8c6=OZTW5wB!8;`$=_2i&#GSO6s(FP?1HprallVRB0vhUnWT|` z&=|(EF`RP*c<8-H<6^ue}_`4ZtgAEA&J}U0>vUb*^ zfjE1>AHt)LIi<$@h7;opLj?C6E6e>oR%YT)Wd;Cv{fL$Mu1vw85vigun3AkJLR?VA5g-B#GMs7fE6wre6o1Zw`*B5Z= zPFHw?nB}_Oj`15es^A~osJV-Y5wz%E+^DOdXKs|9l)L@VWYZ%n&`T^2oJ18PbEpn% zNG>67Or%f2GFP9>dI0BJ*Ox1Bf8I$rWipAy17|=Jz^{P+(OD1^m8Gqbp0$pixy^fH zYk)ala4TdR(0Oz85D;D-7AbGyj0k#y3+~39ldtCIPn@7C{Irl!GY+4~qj6PI^TR0e!&vYXFzCI^#vwEQ4NUp=0Zv_CX8S{^vqB7@W{$ z*5TYQ1-@w1xq4)2fuCGCaK;j2n+Z5j#U^ti7#@=fn#+&CU(X9l=V|3^fK0E4D%H{6 z{lynUt9-Ia1oz@aVd0AxG5@1ykeLc#Wn^Gvu4PK4uVrLvYo$l^d^+`hoX!I^HGnEp zxa(Ch&8@99R+*x1*j==x-RBo*qITurG^ik6Oyfl~s5wVlme4-<>ib6#r$cAWyo3dnm4`g2T&>r1Qo z_6PM7R(GXlP6NJuU2_{J#;lT@iRzf5sgfP4uZZLCZI)BaY#yAHE*`*(qX)n&vX69Y zS1%H9;83;WyMuce)(6#C3^TuTSbp4r&GVYa^J3QB|Fkb`-KfQz%kAAGkZ|@u`Bbm6 zo3(4KzHiq!b~R?{dZX>ypi;Tjo!{i5RhYl9vSPHS@7%D7a(n-kDd9=6QN1WsLEz{M z^O#X#!rDUJ?p0s#LW0-YK}w7-YjrBcPCnd0-^8h*{rQ2@jq+22 z{KHtnfe^_N>DycA>(o7`>e=_9=hTd8lWTHpJJCsOM()ma;cl!wOA<#;&2LSN`v&?# zmEE%K7qJuf=lASYqR{KV^2&%o@CIw(^gS6am^UR?-^CsDO<}8AqK$pZtcJk&PKfG| zrf8fOk2rCEL7@&vx#^7aXY6}Q80aR!N8H&*V8p3XYDw3{F(JigbQ%n<)Sq*e!$GJ0 z4Am)6C_OVVYoa-VsU%Dezp!dET&?l&SjMGNRCTM}IN#*><)!OLmyAg8{(FBJ`EUpk zpI5WvUi(sV9t|2|{>*-ppI>^4Q4A|tN4*ci^>Ok19wH3WBbTBKJrcdC0*;_s)uo~! zBRcPy#Y&cK{wjuZ#)oy|#fT3$m9Gsi;t5M>-@u2OZEdQ4A1{S|PO@BCQM7vRmRPSD zfdI5rKjZhG#Mae&+`2vy+RjORfX@cqFB;CmS0sK7S$>uwmHC8>7&foZL2slc9jx|Q zyRh;m67q#$pxAUEq2O>oIPcWgw97*f8QaIpiBG940?eJs10%21DvKaunP5gBo(X``)bW}c=j{#=@yr<#l947x2&@#{DlT5Rktv+f0? z57j2=VqLr&i;G=`$|8ZxG5l5UI%~k}m@EjkJ}8D=G>wjed#ac(3ay?rnTb&-0LZ|dZ})$E zRNdRNpBlOfM+I4GH&c+(Hw#9+9Iq|fn8#{o+8_NKg5}7eu$~8J7?UNhR zD5_afo$24`$3&@XXv~P%*tMt~&3{)n?=lokU|u}B4UKizsa6mQClP)LKrU1*N@AcQ-wuc%|7X#KDD4hCWdjui_7v){MSet@78ca$xUwYJcPi&UV zB!+Z)?%ZpR(wc1!WXxYqSwLM|nU}g#a9ww1R^HbMw17?>guuD8!?57-S9m+8kRyZj zSP#^EQ`R;w4fjbzZ5a*34tq@mMTac8rhT-4!*gFPpHLK2Trhv%+S?N-x9d|cUr#V{ z!8hLYj!YuPHMeh$BxR-nPSO(Lb+PA{{BbKC@3(IP22xo{6@8+3lczEDtd@uG6iEY{ zX?aw$T`<6D->zQ1O^eF}EmTBP8i_KOYv7~UiH`IXMg>1n@-DFo3Aal2)P*{$yz3b! zjX3)*4;eN5YLVTyhTn+stu}7Q0iJi&ER~NY4!wx(mdtowtDc!8s0D6^0X#mgERIUN zm5#M`eoc1J`~lo;3y0&*#Vejf$E&l5Kn1{xcf0$hcl+V70;V|&ez|WRvM>)9e1!~Y z&a_Pa3>qUCfVxu{I~W8~ar!iwSUcCK2|$AR=#gYW*0hWWQ9L%Jr9TN)+{g;RBTxAzYAubLkNLK*?$E+n-)&UG&5 zH#hrDC{R#Db5L^wuP8gEKH5&#kV#Q5ex_nlNHBnd7mj(6WOeYs!&srPzL!k8fV=DX z=DNqvm*^dmwWANIAH#WV0I>uf6vXg?K!amXHPXRl?pjyappy98{PkS<_lRHgx!>z0 zQL4Q7{I+1Z(Xlcya06e;QhVE|?L9lKecZSFj>z!WsI|u*>y|#-&^lccj!N~3rs=w4 z+2p>r*I+=86td^MGJhc;j$5hj!7NE%Ly=j7=~@=j46)PX;`yYM)5ra~W6`=l`cq9t zaC$X8tW2Clw&TZz2_e^_Alhp($xeu{Ho_<)W!I z!uof^k!v>CGA!UD(R>Ddl!x@&QAV`pKJqMvH?GX9Vb1r?PZNssCIHn6SSY&rDnrq< zL^6ZJh%~&UK>jWNThB5*|Mq%6q0{M1y{wF2k4(QHK9xruA)bmhi?1qF(O_l=ZLe$i zgO<$82Q`_OGMYRFmL6;O8=n4|p$V2QC`z;g@gtp;>`XIP-v?+lzsF8@KqL z!3(_A3}tcdniR43qgo$_cY?-a>l3gReQP4L%L_*)j2GbbmKofGfS;X`N3-J#k>gw3 z4|SMoM?7!wzVcJhio|m9dybRZ98FlI>B`5ceH4ju64!eB>gW=Pwe6_qhQL(366mZO zl72<2TsK-SYz3cVfLqShM=EW^$x60ZUleBj@&F9B2~Wqig9@)E+EDTp4(&9I^zw`P z?oW`Bg7T?pNJj5cV?aU@$JL!B{KDvOZhQ|sue}$f*_?&){OH`FKs?*IAzeB&?R_gg z))ZH?EEj&U6y)4YuAd6&w3avcJ|z=1N=f!j-_eMvFF@CF0+X0k_MQv$hrR~v{3QiUf#a`A`@Mlp4X>-jnBwH-4_Y=&8? z5WI$3q-KA69|g|f6)gn?xhjK`J^3@r7oEc}I&jdYt&^4v!@Qf(jS-(fT?C`&9Z;mJ z0?dYAy`i!I9;P|Km2tJqz;lNTlmZ54<|tVPDuO|Xqz)?O$vg4Q4#tHVW?Z4pinCwF zghoblLt$35xNp28m{(B7t;i!b96p?GS$HFaKCBLExCzBrLnwj}@R_JFRj}(OHeG2| z6Uyp=6%H2G3mFxlCRQy7wclwl{=LvJ2Rr_S;J1CD`X1r47+s2rjD>lBFT5~CTcep* zXZ=X4ZYjF%(-kexS8q`?^%nf>49O~=QH;jJVEbDyS(p;|%z4iTDT+a)bKoSLAaVVN+Ra?Yftk!+|qBB|FH>VTm^-pG^wYuOZ>bK{9 zJ8jC#KQ7-9@O-c{JA8fhhUdN&UK{vO68I3>``SzD70!W|SGyYGs{z6~rPrl;l%$0! z{O@XNIG^IZ^Y9diTjIUT4Pri<6fq_#>+G^vpnA7(gJZ(4YQevvQtFs>;RXM&G_)%O z34#kDOo*IE3n?_d4B<4&_|zq80$pjRuW*J7vUt$l1+-4R-T@n3@P-9fr4taBN37?6 z&oCtuBwQL35)JX;V?5Us#EfRi3*y81dJ0L&%~1C@SjQtg3%IXgbDTwWThLqZ&h9^p zP)Xtmbz|o#Uw99C)&V{y&Q6fTMgS+{_Byg_^ZA(ZS_oGb)0X*k89*Yiyp zoRc-Hw29Hjwk0$PG@JOlyZ7^Z-pHA1CD$0%MT@&j`#e`@j@8j;00^H|)8kIhG{+o^rGRHBa%Rtf~S+dc)+&xphYI3>|R=kzp@C)__P?{l+{7+E?{Dc}~d zjewv)j8C6&#fw|0P;MYFa?`-7Y?;UF0GBDf9TgSR>$SVmRq!`?8UOl44O|t+j;_7+hZDet+oOe19?Lez%hFbbHwRWM@}X z&Ey*0JwqZ>9gxw3|9F3Fr_C4(`9Z13&oA?38#&mjdPxLJ+PNYY52IwrB@R2JPLszB zw9cbiFxn+zwj377J~JX*Lk;XM2s<+G>aG_>4|xWDSRJ|VVxzL-h8Jtlh%kcFyDk^6 z1Dvk=dUWe)K>vsRI@$YssuxDPVv4X}by=;$ST>~~waRFUTN(K;s(=%tjr)_=n?8yY(-jXkE;VeUDbcq> z14efdj^C>#9&!Gxjab*vxzW46cp)ha_agd#^v<5;pYQBV0S4ATZtV9PjjY%Cqls=G z9w1|N8D;dSId<;a7-vKkqQ(n%jD-mFuFBdJx;(Wjdjj(%L2G!`e3?druEfx-%&4QfN)gLrvEmj*^-x|~UJGN(Oo zx0xvy0=!me_n%c&5mthL^XXg|t@ZH6lrKHmZt(GIPpt70U);WmpSaTYF%RXiZQZNx z6tEB#KJ{cxjVaZTKef4W47tVM^b~M4d_Qy_3DPx4xnDOO9vYfWe{f;MFQ*YRU>E65 zIbd!wOHAtAyS_w`uU4^Cmx#+7?=xCn>HVMeu6D+HIDh1E}mTdFs1XBh*B zfP}}};m1?9$2*Ot>u#x_-ZDdZVKa=0r>h|@H~aP+x7Z+U5<)M?=3s#crO6ciQjt^c zj!+lQKK&R=-bI|-9^1>AZk(-$R?55i6t%9@0k@jupsY?q%(5MqMA&96>@~hXry)Yo zEiZl3u0W`(yk&zkq5biZf*4AM5TTi>$GHYBh>f()5X41}>2#%&wZUTt|=sM%I{tmte^kOJ;k%!z!_2Ga2Va7pLp#%I->z zTE~zYHvLyBcrM)^-%GeoO;ed)TG8J!FMM#z-u4a`pk{}rcv7`4w1kwNp_8T(u(~bf z!alMo)SJW78OLs9IL&9S*`B;8?-tL!84dr2wB}gcRJiCe-8CN!e-NBi$PK;KTnK<` zmw>^A+uBQb$fzl(e$uy<=yR z-N${sPl7jRR}}rRklE*>LwA#+Vr@X%oVo#6ge zwVUAE2G*&fl9?(Jv%9XCJ>E;^BQmR5LF3wTU)G~;M!dmj-qLas8RgRIxzWM8Vr|9- zUz20qk+D(tzq?8dZXNb@EQ4Mr0R+!zyx_eGezwJFhMttEGsU0v5Ws2Z#3HSwH*HS66j2qwYmp3byCS zUVR9O^$8WAEgJXlXnQ4pSvtL7V$T$9)tsnBpIhwYm+FEJvl+JZF5S^p`i78rc)kU@ zt+ihF6&);B=6wk&s;?&f^*h?ygB(*v^iZ($z`aQ2F0DH7c6?n+ndX|s{_CcbhgK%H z*$m4RSNBPiu!cnu-?8Y}LE@=KiYJv~=-2LQN96-WWxxXV+nWZ)jI&<2EUha+YQ82P zEb%KauPP%%xO>}%gz^n;=SUV}HiqjUWvuje?zBZ50Wad4A@l(Uh}V{eoZ*>7gdDca z@q@RW^nJWYcT5->>JoQ5U&rZAXX%30S}O|+=uhn;W8N@12BoOEU2x%CzFKx~>VxFM zgc*y%u?*RvL|@A;ek4&Som)LRiHU{0y4^U>Z-(Yl17&^J{3v60#(#r2PwKuMT$aSt zYrmM|;E90**C{O%fw3t(T^W#W|0-fAEUphd%(Xr`P%5=aT#>=YsCUS z9=`P3^bT%ILsp9NydYS0)i3X)b}3KT?@or69`CAgoR(al2CCg3SrbC?lo+=)E(}IK z^3!U&4r-J3zFRw)-cf@5^b|RO$T7fAiyCGb`}I7MP0*N)mCeDJRRnOcc!c~;km|^Z zuf`lS_n{B~P5?NHd`Y^KqX`XmtPj(|O6BA8vO(@OO%MeI67=!_`sdU(zjp6lz-}&A zxqh#7;e_tmh+#>gEMvWskpi2_c2mHHuwS0Us6=Bc`um#5#L=UTj3+Wt=QYBhFIyw2 zH6O4r+`G259-p#lOd|dAZ!{-KY`3cOKJ7p3ThS-!$*n(xB^yL+3sleW8PBUQI%O#B z(|c{MpVZt+AGlsfhs-3vJ!D@yw-u8h6a&gpxz)NU`B`<~BHqt2p>tKrNZ*Z_p1dLu z^IBkn%0x~{XqMS*j@K=<$J`|tWQ3iLXb3bH@(*h^NbEMlah_5!cVbBh zc7|$S>`!A_usL9uH~=Kx^7Y|-8VbF;*u1_vo$t#}PD!G@<$N%ar$#q+$yIYOa5mq4 zJ-z$+vQeNoDZOsDO0T(kh%PY%EVbIHxmmKhPm1r zR5uTzTD3Vxw0gFIxI=+4^8~|8Z}mz)1M1BPp@yx?_oI^5^!IaxySW--`8K>a4YN%XD$8tGSAuXQrcdWlL+L%; zcHs6HWh~Kn6nUUBkV0cBV)5@7JXNx^UI3}6>4hL|mXzxTuI-_s1b_>2L{(LI zWYn8Wd4cozsZBWh^$n9NdX@ZGj%w=mFXi2| zNa~s~yUF^=_17|J`s(O+RqLNNyWnwRi64y_$JqIl3ZzWAiC$MD933=<;c+$v-}2VNBQ;tQJEnendaBpl-Cy^AWq zO5DNx6Ve+V8GJ7aA$28j2RSR9K4~SSx(%`74=F)Kc!hL;n54CcFUevhqZO6 z!k4c-flvcvwzLr@;3*w{!EaDJ1srr17lpV()mIO(-asp#G zX5@+4NL?NKXCy}9D{Wtaz?y?Ixo|V-(uSpte}3PFwh}~r7$glr!X_Z@oNIUS(0HPR zIl?9-&;3R^ItsrG`pY{0)F;SHNHxXosE@052u-QA+li}hraqgnFET`GkK2Z|wQ?+D zORQi|GEX}89OZHChpNv~^oBz*tx#Rds+3@JC=drL9zMOj4{7s_apml^mu_n?{ z2CECb)-{qzTbDA$d4jtXWVhEENAVhqvAs*#E#p#9+ugG3(7tQGwDqhs|jxXa?F@~ zj!K%i=mlZS#AR6mfCKi64IUN~NNr(bf9NmpF(yJba<%j5oxvl5*%Iv(qW7;eu+?7{ zD?0Jb8|SuXQ&_;%bXA#H7#s|Ms%wyhq7TKKv8^H%-43l{rI0*StzWDh;)$-WWfuGO zL=lu(BrC44lr})JL`6br*tBtAPh6#+nAYjrFNMmi(hoeciGZ4Sb6(zEILa%iiP$#m zde&^s*pCz~X5x30Ult)J^wNNr4`WF1OgtU1n?RPDnVSV0)RwF9FoeunjeXPbS*=6$ zR(H<8fi_>5x!W^Lj*d}h6qfNvx$AS=&r`CTIXj!#FdThyPQrMf@MWpX7U^j*t?D=g zL%GVRu5JnknNuB5WjM)xZy^gedzvE~|E;K4LxFW)%oV9Bg*B4tR*N-dau1>CvLheH zX@bzZvrsIie$;9AgvCT4(LAYSv6?jUZIXj^_w5a1y_u^7dlUGBW!`u8Ah^r1ZSwm= zUej|i)3X$@2%H3Du1>HK<6a?l7rl=KXw6;e%1}w( zC#x*guKH#Q`Pb)Jyj;oZA^Iwu_6z`d&7%qMTK1vSwA?oJu~YmGFt;i#!Gx1^g-ku# z7&euB4adb+GsYvm2QA_iz!@rY%-ycf)|IIlg_T6R=~nLQ3)KuHx0qCikq-$|34Ag+ z2}_fd6KR;^*Y4>vRvm$l*L0%#E7dcXhH5vmAIBH?W7H}HV&Rwz)WcXcp7 z$NspwzUOQ{ZR&h(z@_TAwAz9mtF3U6zo)*KA~rdT!ijD`4x{w?BY^Q1^-Ey;BO^}d z)|noZN8rF7?YCe!qedUSD>LIfli0{;rz0XnMfrqclLnk`%lHTce^@xw2q|e@1hE;O zwyT*h2(bm(aif1NG! z8*@BV4W7f6*nyX|EX0;<)>QiX?S{dhDYw|XSpWFm7E-i&_no0u4#O%83b|qWof4WE zISz>QB}usnLReEGiWUz7{mtHoE>9}!ZY4g)4d}uOgaU-Ys5U8bNTm*{TA4uzkxb2a zxlC;z=q%ZJ6}NRb?fFngheO;Kt$TJwxOyPqSF zV%!``$8?cBr(qLjrYp<7XL?Vnr%iQL1aDWVXIfQHTDvtdkK8zp^4{^)7$J*|@vj)E zq*Mmmhc=EcpgW5BtZjZveUgctppHSvKJgMX&cex7Vu+_?7@8z5+Vx=nf;S$sA9dh5 zUO1M;Pbcnm=3@6vY-+#7s7M_NL{z4B_MDTb5xOQe1G2g?F2&77oVlbzM_-eEX@Z??2uedzvtY-)Y!@gcjw6@qo(o^V_)3k4BHJLnkYL@2BL_CnP7 zj#f20rFd`rI)#p_VenSZNd^VxI~-X0&`InJX}-o`XX8PAt)a@4lHdJ_6MNrnIc1## zeT9t$`%?WiZ|#+6ZucnO4s^_;j38G`DyIzTlED+*NxRbJiLuio^?=IeF$ve%6Ec7% ziQeoub_`=SzY`~`nvG}fn6he)*PBeCz7dy2&ov>?fCU${VPiSM z1~n6P>kaotN_u66@$TBWoohU)td)O)?Sw*lVzS$e(}|T64Z3S$sQt9WZV6V`*@^83UZ9$8>y^YGPMgwMB=onWueG%vSJyb}5CIpTKw8i_| z#u+-Tig573J@TuG45yT<(CyK21-X)1LeL1Juto4y#e0VAA}hTWa%5Q&wc9H7rny2K z$Et|LOQX1VlRZp0;w$dQ%_t{a=82A#`Lj4=FDA*+jI!o&rVyj@-klW?Cd)?(svvcI z+4e8cbu3Ais~u0e>S$beV2)`RGLjf)DojH!g)f4bVrQ6&9N&3Ak@?adn6P#g&xy2x z75BsS$8`*N%#BTl?u%G;g;Lq=?W%3xe4Wh$)=n=+PEovuco}HQzyYu(?z4If zptc``E*P>I28muVoXj5;9kO%g;Lc^bEbpuD((Ja0xh>n;ig8nfZ6h=n?5-TL+M|!X zZ7J`xG%)c!s4`8x5)G+cs7vnllNg1p%-p1*xHq?%tH?9eqZCL{tu#uIU~3fqa3-Iv zprluz>nm1qr$rYZ958~bZX7$9#3?PP5oV^+Qe!s7c|F^(I#8;?c|KXtPk+|H5Z#%( zi#1`|p!W`gsl=L2-!*S>m41v(U)Q{H5m88~IxAB}FUn%IGPdO%qXjgEUx8vXB;qyZ z>8QIWJZ396y?x@WSHUveSsrABY#f#CYDlq+{Fkp&V$Nn`jdmrz5T-ymX&;KQ%ps!I z6zPgV0~^{cR-!B2j{Fg^988Cm@t!B{wJ-3_)UZjupJ{(NBn$xOG``HW_ZDN2jj@9t z?E)80!zCudENFGHRGr>^T(MaEl+ugJYzk~nxmyE15#Vlm-M?1M*WI-wj0bmrJs%US zt?K^35x-Aht2b9tZFr5UY|rc}Lc+#X?0oxTwyLIw!%C`}T!=NbX2jQ3_Wt2(GdWejM$y0t98QMQ;PuiAC|GA3r=bA_A-3ZJ2Uqj-&DFcqRdK~u%o zz7f)iv_?S$84j7vhW8mR6lY7eaQ0}UleErdFUATfTY9cPsp6nAO247nQ!_B@){CHN z#70(9+pjZx-Ag#q`ADNOfmo9%FK+F*XoV}!06zPHqj$EdZ^u-Q!*Hm{C0KUKIk}** zLOHrakU*o)<>2AgSYF}U2M$tQ^L1#)b~{Gv(S2_$DK0kq$0t1>BFmD-Mx0uqHYin(5&MPVuLeN=u)JwNyI%ZZ9g}M94u)R$k7SVb1T#c8 zuGA5TgFK{ zW!>?2%CrrNuU1J15r@cTXUUG_pGja?UR2~jbRcFwJ7*u0dJiqQzZuJj(Gx#!^Jyt) zQl^6z_X`S3nEXZun3jK@Zf1dH+35tu<$w&mqmd5?uXkrNQlP>hS?|YY%}YSfK7+wH z^EhY-Bbx*Th0EbkKpzc<6;fwnRGFmuSLwcwx{@K&y(;*6U&Y@XSP8M!uI>~r+=KKb zPfku@gy=uyrI;ITsqEh^c0h>Vj3=t34SQ$P>7H`wLGNM)0H_s(h zinl~^kOVPD082)dg(&l;5ldDe8>~w0R>~OrLz0mRhZmO#yK&SvdI$s4*v&41C&v`m z1;UK-zG3XqDy?JjVzhhCVK1TqL@`R@(&wAj4)mSpDdE}bhwfhRcV^cCqxRR z)4N0VW*MSQYhTc-XOT6pyP@* z>MJXj&}g2Bet>#-S)yj%P%|9TDn=s0+rVn4FfCEID^=ufsB(m+eIRZPLr0P^D$o5G zcWK~J;2L8uud(;NBRak2zin1KK*;^wyQiGh_rwjTGGfCnZ$WFm(eBY6* z;WQf3_-4z5)WTvmL{M8b^lAMoae&DWSTg9|$RQBj>}1`#{38lEFKvB?+t@T}?Pu3m zNVfXX!7HfOVteI;r*Zu#FM3>anfaGHCx z--xdsW~S)f*iL>EzDw=q(%Xp$J(p52Me#MB9lG;+EWpXK zi3RcisNj1)mMW%W#_KF!&LP1GY*ZKiA)7?ONYp)E41WQI=-1!v;W1|TL9Iy zG>zI2+})i(a3{FC6FfKscXxMpcXtc!?ykXuYjAh?;k@U2&pzM%Z`G}*YE{v!wf9=n zGd(@sy=L?0SlF)Tm>{&SlLbfMR1ml}Qq@=7!J_gi@!Be{fOJw^_YZ~*e*{CWr%C zMYyuk*BgMP{;}8{Ne&9*a_2teX&L`%=^=v`Ap#?UAgSS58V0HV!dOAT);a zqHu=KYT7u6ltjQfh%}tcW=g$%Z}47Y{W-qgbxyW+nq*COepT{!lV(h{<#T5QW3ya& z`|-W&%zLu#rqZu!TlU!3RVa);y@P$Uy( zar&?4FJ&*}_2$cFX={<+aIPJg3gA`phHubCXn+qjQMP)29#)mTDtT<4u;JAXsY#h{ zQWYlTOn@~yU5^GkQf#KUhhA{~3YEfHS>OJSe#6qT0&2x|S#j%}cau32xat6Ng*)4W z7wAIs$V?kwpx(L$=Uk?F7wP5(T+UrjH%h`e_w;7#`FcdTT5nUcZ{);9n{fM9aGgHW z>CR0x@w(kh;8hVN8wr{h=QfgA^CsS|_j?Y339-k~f-szi ze(rg(lr+R-zLLPrFg3nS2y4V^2?vEv}$$_@=9K(g+;0M$g>4c z>X_9rcF@5y1y!o1w?nfPM;>D0jy2?w+|LfP5PpyBXAx$yo@5W}Es zm4>?-p&LlC6YQY7#@cRxbexBh)?9A^a$_0DxF%i`R%0QRZ#%VKo%>tU7YX6Ww(m#n zNfXiNhqMR`?LM7aZZ6h#)GSYeHq{l$9{1;h^IGgHZI?>PnvtnI4@p(L%vi$4ASP*y z$`@I7BI`w72ETK;(M+^|L6>C|xyT5ZpvElk^j&0!%(o%uhCg>--E6zw@mg{a)Oq|y zt+b5RJ2dCa{IQf#vLf!b`^3H8e#Z2BYAgHUeB1iE`R2E{>v*T$p!-Z!W5(AhhlU>_ zS0*jPVF4%4)_4TgVn15G9_$-!ac{F8e8b*hy3MTUGF(7g;D>B)m^SNdU?1f&iLBS@ zIa>OZX3p#|rg{A1m!VZkxF6yXd@{>fi4qG9(NqV1i&fNbARaf2^WVwlHlsqZFSV-h zDKJ+<>=g*Q=f?$t_W<3nr z8+~eoP#+h?8cv*Mf{}?J(#xOG{)T4W?K4%D)J$Qrl(*Xg&q?C>XlC_<`_4%+BD`I) z{@tWb)!(y>IQN_VOHCpxbSPsT7$yDpEd(V4NnYkUY^@wmv#nX{0ZtFCXEnJk!Gz3d z8gVJ*{W?HG_kJIJ3*A!5FvPc{9cXMfuFH7r4S(l4vLxH4WW(d$d8O_N)cciib#h)Tdd~P&! z%GYc@m!XG>oye3%jk4$N>SOYNveKDF`B~o~33B2H#JM>@_nO_O?Us~TVU%wGtzpzDgGe4azzoHXRXBmwfJ%pV>=31;SDtQ4fQf)Qm)4?o9 zj8`45R9+po8Yi69Y!-V)HwfQGRTyss9gcRFIrFynE0_nREIiq(NBwG#as3nOBMaIq ze_A|y4XA$DY{a^BOj$*^=q)Rc8I| zZ+%C$qi0u9+uJ@B1@mBnl0VpLg#y!2R1&~0n0HD;7mrGS7eAQ;H{gM&>uS?2@(}HU z&3`iUsqb3W`%!@eZPp894cr8z0d#5?+#1Z^aYsNp-pplhpW4Rx6<@*pNYb$rW_L=}0l3}Bl;d@$Ky zQ=g2ZAnKIO0;6!e8~D_-X{K(Q+s1z{VOQ6M4!Kv7w7+S;jgPgjR8v{F$(xx zppyeD|8rFWh{hY8Z{=&YbX9*g0g#bTZI|lC@wD~4^)B-ch&N3ax(>`wi!u@d7z?~v zcmf(Of*I%nms27Bd>PV27bQfpd}6dRT%~0OGVZ^g;skR>grFWg0Ry-WNbUWzZ-e>F zlF^YreVRfyhN%Nv#>cXtSwMg6u2oLfok%7uA*_iF9J>Ep9gf{o5F^03tAqvA0Hp41 zvEMaMd74cG^sa*UKTqVk6V;9ILJ5Ive46+)2v4V}<~jJmnC|1>n?27R`%9L9i}@jo zAzCc9Nc>&681`E*4z#>Ur4nhzj3SUFaHIE1WVc0W;SkXmAPb+57CMQUp2yV9{sLqR z-0ZEvcUsF=`73m5CD5o;=cw+E>zf2+;=a({NOT)D>M&I=i9;Y5bv8^q*oY&%0bZ>D zp9s)j8U$L(^DbAU{8If5!Ml!6jgY3$1J7nTX)Y8R{@2EX#6l@!vagzqtwI5qT>zPj z7R8Tp<#aS!Mc8jJ#b9%e@TPD|$fRiR+I+t<`Zids3{?rX76GWyTfK{B<93MDyke@7 zN*HP23T)=~BlCj*yy}tYc>78@=6-1O>aoVFIQqhMIN9TYF$k4gH(>q4cTcE>WN)we z3x6;ljHP)=StanrY~(010=Ao`Zg;|41?#PiROMvtdwth$wn#-fhq<1L?KM@m!7yn9 zOn=pb2(p|%Zw*5J5fv5F7QK?V)4PTIozVRbNc=h6GYH^~r^hc2%vrE6rPJjEU&3At zcpbOPn~vY#xv*tEeEVG_s=)%+8L@pL)JFN9>%Oj8`z!my(eyvrp9Yw{A71qyzp319 zuv`_i15S!MplaI>-8tZUZ7pQmma|o)1-`E?+c%rN*CItus&by&ANk*Oh!4GVA%q<~ zr*Y0tHZEBlxouqtABcLT-3DEBdtz$_ta}7Ubc9&E)|y=E&2AMf5~@kFU%cFWmJ(+*$Bm?G zv3XQBoo$qOzdQE{2g5z@Wtf9!YN2g4Q9CslcG|Z4t(dQkKK4rWtnV>w(cVRn<*irV znMdd-56C44E=?%(BQ=R#>tpp|aM!xDtJVdyJXnl_Z}uSk*|2_301f>s0VE`&JYdhA zt{E77-^ype5+YUC86o^4ZU(MR=cT1<+s;evGRJ}W=z_beHBP2n^n^O-ZV(%BvgW=c zgt|AYh394039fUbYq{jXdB+mf$@cJ=iNmsj|FVRja@xrDQ>1WF>E%yL(w$LU{cS+h z0G6MeA5iYoaZQqx4ia+VOD9A= zLKVd{h`}cST-itLbE~AwXC@r-40-Lj4x9-K0GYbT-u7Vx#6fXTZxw4;TbDGueKw0| zul`4-cQS+_G$0HJKk&&>0GJI6b`hlt7y`{(w(*HFJNv>b4O4^O9#y=Drl>$!_c>t3 zM~sYOxMDY-F$Xb#>JtrU;ofg<4#HwUrmkD^4oQ`7o%htTbpOOHEF?bn`4WA{0zfrM zUvQ#%<~|;ylt@wVT;U&z1HZxLVEZQc0Kos%TuE>@Ug_Stsdk1a$R08l8jwT88KQi_ zpN#*d#!@m!R2?+gq%=9N24|gu*SPs+pl>3V4y?v-!RV&`&##53#!FY)gxOHr+^@fc zW;R0TA_!gNZ0MXxv`@OLJhV1rx7YF-lQLQcj9F3A$T*HdI0_>(4mcQ4Fg>9|#TDyxiOPx=;pD z!G5#Ur5!YEo1d8Zdg8S1-l=zvilIL7Rz<+`99^$6!BT}=TRJK?OU3&<800$3)@R?F zwA%wwdM{fya(Ae$+Gs2*tj79H@j-$^k~wY~j=3tBDIA5C3xTD|ee?Lyru~V4H3MOJ zA`?T{y#`!9@fOR1$iRW&5Kn82_9)PPeY-+`iVZ)%z2J%azMX*Z_u*eoGQEJ5UP7FI z1*hW}*^BGSQpJod&0!zECEPJW-jY~)^__eh>z7u_$(canPhCl27It0(>$T?0++kms z-cBT6GE?rD1JNf!-k#i%XXHHPH9Q~EesR;(`VeTPM^x#N!FQsbVR_NXhg~@EE>9(S zJ?E^&XlA?|{+>MRX?_0fq$;PG-jC{LD~N`pg#L2PK_2lp2ygr6k>x{12D9dTF3sO~p zmigNi1fzaKtqE4Q#kkVZEkTWORrzn)lA!P*yPkkcr6(v9m6RsLK7g!w7>;2?R7IJ? zf3-d5j3>+O+l3>N6$aqpBFC-+aS925YLUawH#C9?+g2TXSl2WM>e;=3Of~-dbnSwU z{*r`nv-la`p?V@)@nDUOK0pwg|9!q}fyOf&+%`IVwYCF?P_1izd^Nj|anekK%C`i;8JUXpp!QYnkG)QgLQs@?NDcZfihg#Sb7Uw~1sH zA<%+gsQ`xZC=WvSKe&hHCO4G`We7Z{^Hr}{7?bBn_wA=QCcK~Cw9B@B7Tt2+dvSRn zd8_l>4vIPH<$Ja79-Xadv#+XLc>2Zyu?pR_u5Uw@_}koY8T5VofRO43xya)IL!HOn zS`1<-7+P@l2VF`+-@>nLJ|B!znE!^%6S1K{6h#Qn6P{>xjE`9@O&3$8*n*QxASi)N zNOIX6GwGl1KZ?C`dHX6n+p-J8|fWfCzc@mOKr*5CQ)WfFWaVlfA}V@mb=)C@vKB+2{|lnT(+nl>U} z_;9Q==x3}9*0L1lDFQJ=-+|T_hV2*ATU+-KEY?LwLj9mA6f!jCbf=BiZil=ySxG~0 zdEG~MPPv&eATD1=SeWE%BAW_*Zpu~W9#5kNFJpM6DFpuA&Dv>bSP{CI6+?*|Dx0$} zk01jPDvH2=53YLs{?V6hB`yrv8i3FVHEX$ok4-y>c1Pquw|ur2J%}Xi24o5^htt>A z2Y~jUr;7!Lk>bufmlZ>i$Tgq!MXhxJ8H$X?Bl_1x8JWk6<2@;{j1+Y}Ke8Y~qtvG5 zIfz2yzr4!bhdRr^4Z|E(IL(OCzNC{%Vk2;beM1^bUCI3A`1D#eJ6%?u>As)UIQ~~$^_!`I+$U{^TQ6$}IOadw$Vgh8}=c?-gL(yurx>`%#wA=-8 z-fGLKLX(A6VFJgXx*4Y(_@G{`{XEn@Fh47S6{)>hBVuPfWfJ_ALkj)8{nN2k=hNO3 zo^wUhqVK9ciK^SK`PB$#`m4@@C(E~V|3mkw+nYA>t+*VHBb>5{ma;I!g*C-r^AF6q zhSD%DaF=QPxoT?Y>jrQ>;@Cw@;0OQ=_MyxI$fr|)>D3mFrEu=?)!pfiw( zzaBV053Bz1utYP+s0xz-*)FgSOs>&qU>~Izrm6_u8?Y&H7!LgjGYZdRx&OT9A$#AH z!a5!`TdTi$m!@8u%)$Pl|T%63vMha8{aV7n<K}8ecFm9c4 z%sKH5@&iC{C7=Pu0t4WCkke~Mmn|`;ARQ5Mg)8vwGaP9&Y37{^7Yp*GM=(ySb_Q(Ci=n=6mD zRYyrj-KDR-!hMknGy2Pg;M^ZUYYl6dC{~apNDMv(mzZ71Hh2f|UC_&^#fBSH>ge$) zCO=$qCw+)L#*TPjXd)yXg`Lb+<{yCzBfC-1nw>=Inu94xh8RbTJ@KK?Oh`V8I9aUB zKLVFe;zq1y>k@`3Q@APnx+z7K&^W|PWF1qPLlWY#oOFJ=Q>mwIfXYsG@YLJ%_zN<5!35FOR5F3mw+CEKd2|(fpJd~Czcb2ndkm@vz zTQ@cV`Ovw*%D=nCab3d`V*=K3HJFz8((Gt!BV!TDv@4cE6b<4Kw90oJF>+}c5{KBu zn*98lj&tgyg1X}vQ>&AGh|zYp;$0SLiykIa&7W3)khgrZ&pGf_07PKg!&P_zS7|z7 zKsa`wxGVm5uh6OvG+8{27%XE)CZA zRW{nWdpiZ1&aD7%ElQha8OC*FoAnv+owSf+GW)+pahh^5^93DAz-RUn4u6hbR1FM9Y9IJ@O2)?o&NGOmp&a$xs&U$bCNd_ z2^jGmt;U@z*n>0BJn#dAYlugX7oS(M{3l$3XE=pKroUr!5cgTB_jdNXyge>bz=%Nb zFhBuvV>q8fn3WNSQAs0BL@wo3G(b(T_&oBx zG%^eiid^-YnsmAA@X!+^dvRb#ak89qV!iQAXMn_Lvy0b%i~YGODtgO7)*&A`R7&1- z;k$2e(=&t+K=6KRUN0Pz7cttBr1Z5YLkK$`Um`1V#Ljb$lL!3#ccZ*De7X7lo+e8T zYF}$diA&;kLx->j?x#9`xox%TSPy8I52ks3w!gCPk!qKnG#}eehbL^^lZ4uFsQdSsHS%FP~7Xbhcjch1dcP zR}DGBfY`EbLj!XzIV#P;JD4+DKd(KwfUNHCm$5`e**b|ZcVjk|e49{+Vy?*-tRcyQ z(cbYDqqHTCfa){Z7E?uwbh8RT&(Ic?1Xs8h;FL!d9z(s-KsL&&Fg-Lk0D&{ubOK)K z9;^^}P31pv>yf_KoZi%OHCUGf(H+cxO#sXM=h(hc{z@kxlFxY)bU{#&von!HL#)R=PSg6SmRO3R?Xs`}v#IME7P%zhho?_Q4+4 zF70Pazq6!Y@}2+Brnf`WrKh@5S1e-cdYJqwJms@A|JA`J#p@u9|3qgR{}Y{wiu(n} z+mLZV#Pay1;bF3HI7&J!>xZAQ`flIdg4GWjcZO61qpatytuYyUb!4Adnh+i?p?p+8ZCS5?y(?-f|kB*Y>6AU{#7P`o3uMI z&#L_DVx2bbrs|c9uAKBV5I1eL;MO2AM7>Qe^;bDAq=pd48BNMyt`SRBQj^W^=LW+N zU${SFZvSU6#_#W73?_~fk!(J9uZb7V&&RlV*bc4NHi1*ux7^WMukz1@S%Y$?zo_*c zT4Zg;RK6@No{o!J9&YbrA_>SVCk|Gb5ssd&G&j0D_YW`Jt^R!9GjGzf$+i>yzuwV= z!9;C;Cc;JI`Vu73^tH%Wk=IMfoy)6E>5kU>KV{c;CvW>p+Sje&Fb}kPf$gL5&-Lv; zE*}p+1(_0tf1a(XbaU}yQEhGS5!DVxssT=4Grwkl#fZ9Vd?0r$qEjEzG}l6qvY7hw zVB^KER;7;Lj&Z`zWCk!?`Ai5YmwEmj(5&(=;LbuW>)C}ROt{Ox+~b~HdU*dvEhf#18QLK;Q{&3dHlZ> zD52QrA>31mKKGZtbupN60r}83WGkRDbmBA69JLDx{Wu$%%u04fj**>yYi^xt?mNXS z*p&ENA3nS3|8gtK4hrsmD%e>WG+zfjQHIux@^|nqb6O)dsfy`fk;qKA3YmoPiGLYB zP8I5+-WN`e-r4;B3zJyKhYFzENS?;q5yPkdRopEsMJUnVzI*n@fd^-m6qrP58$sA; z+|t^2@@;^~K?Efyg>Xhq)>dvWADKCruQ`PEJAu34{W|KRQ|i(C_E#~XZ?xQAMdj`S zey-nQrw|o)AfIgg_cjTJUK<~Quivf@dBI*F-tiK6T*tj%b$|AZU1`7%_l^YM6&vKS z3gS@5Eoqi5vBRLee8JK^;!&AOUtrC*4U7dA5EmF=JOExQM$|Kjly%`G-G*!|VV9 z`F&8wr@Pg}+T8||?ytJhlIx|K-d-sjHREZ@_9-YaD7gGk-4I&lica4}tDb>UP}E{9 ztIiyX3u$$VcF=bpy2nqucyX017?uIz+7CB{@rL-@M>WzyI={cCEvQHSA0L&YqcdE> zZq7j4EIug~=Lp~bGHpoHEO&f7U;Lpy|NDAGp*zT!8J@fD%^|KhC6!gKi(NN%f0|tu z+_OV6H7|^Y?@i;n$JGmbxw3|Nw*~fVdAn_c%PyXb$Il}koM-7CdmdvqH!akQ;f8NW zd=S~K!1hmWhz!Vuc4>#y#2fC*RM^NPM_n~o4>QiE@GkrNaMoE_Rm)sFW52Oxn3-y5 z4A1jHBDfB`xx`R&r@ArazYzpemnB26pNV~=y;Fofx+*<#`SxCVH>x8pToHax!fg{i zur?*NJfh@MIilI*cy5zi5Xml_tl-U>4}cvX9dDgU!(9Yo1#JBGwx1NOa;W4~V1I^2yj> znD~-|JUiTf#`}~tAmpvz9CMBVFY%RkR9QOFUm`Y1)>|zus-k_tR)%peUo0!p=W-<2 zOtd97DG71PCDgi%?SK1PrjIcX?Vk;E3kL{4hA%Cs<~IpI6m{947#7&}2=@fW0+M0& z-~jnp>-K2?p8ic@PLs`mCP~2*Pl>$EI@w*FL3`0xT^>9WMHNg?gA|P-d1gS7nVw9h z?7ZibbVL1b6uRbUk>?!QCbbwAM}shhW$k%T>e?k{;%~dS4e(s-Y+Y5 zRyEz&lbn-@isE=zjQ(BKEm(iZytn-T$n*T9uQxORsNw8* z@<2Ct3l(}A+>dqYlp}v1ARp`0_h^I!6aXR1O;r-hXrVU1CK&&ZJwj#KBc%p*Z}GCy zeoW$F=kl~-?og;sYvB0vFD+@(?QonPnq8q8d6Y9D{6i?z$cv%~c2fZWMQNFmT}*wgdT z$J32zmLLrUtf-6;Quw=Eq08Y6;$)fcPJ%VZROz`Iw~@^In*-NM9kWLWWz(DD2pHXe zhkiHXlBLbb z<7Crd2x`)gR)kFFhS0k!A5y~!U>rC_{4)mCL z3&Er3S6cpJ!U?QB9~Ysg$|7%MiGv%oaX7Y`+#Y5w_}>gn=ZYlB!&PMD!t;at%{C-| zURa!x-p)_rNLHazkk1eXHN~NP;;NYZ`SM^FYDdAqk$VLjT8zs#bjatHz|%l+c3Q|a z5%kamF~BvzOWx*E3odZM7I<+}*FKAU=7cO?&fCxz9Vcc!mq$IL8>+Un701tzQj|z( zBiV|WliTHLRQg)iz%HkcU1F&Zz=ea02tu;ExTDA0QU!p#H-XP&mr*f(5%^>*`wPov zirog&Io3UF#CDw#Jf#er3PSpujd3;OMX6~*Me|MjXaXMIG`@T#Y}YT+4Oi*Ph!-La}8JCqG%$VXtL+bGath!y)D+lYyR3DL$S)s`ze zmC>LA6v>0p?qy7+ioBB?G|tq|o=FpLbMx?&}D z$}k+2R_PSZ3hi|R{@~F)ECa?d4j>N;fddd2 z4MT(>Kzl#}h$&5ad1P5LchAfy9!2SNa8G22q^BMD|M72D$Wky1glDRH0 z*-Fc4NRsM*+QSd`R@Q}O7MqU5J6ckR63Ppx2 zLy|7W7Gq7kDby3bUDgf(5nWF-3k{OA9iy$?f{u1RBW7BT%vd|y|*=vRPk z<$3;f4-!TE=jJWa>TcKGgkOqG<+_2UP}%o@6ej}pV}>#SKJ5X?$Qr)Nyu%!HMb44H zLO|vJiim1rw9wjX0`!abKBCy|K~qD^Iz#?<7XXNO`vnQk2b9C8V%Q@CP?J_;DN4oA zSK*8Ew6lC#v3e-vuXQ1o(-@oTLL=D&Z5X0BkA#s<*ifvY${2^E%q4$Nf%l?Q;Dk`W zT390O9+>>UJcR#D`2W7E;N~icF{Kf#((2?r+=hd&$=R20lSy+AD>zkN>m?qodlit6T-e~80+b9Uiv zycTx981!M;nR_?BkNKY=j%nGNW;x^I@mC@x=BFYS+0GT7bW%JSM8kT!(wJ7XD!op2 zd$m5D@Nr;n7aG;esc_(yNI*`P=67xefUE>Pjh1bg+SQJ9#_Yxoy#{NXYNbOo z48{d?3q((t*hE@Vt#4%WvdPkvIxsG)Uc14Li*qpR?f0n$o+#Ir3*Evn*Bv6GnF z1^Jg@*vtmtn?ueYfnQ)5pa2{fm#z#y3QfqHHpN)k5_Jv!XS5AG?ty{+$v+Y15)Kd=3^m$1jemC>R4r!{JQb)yFnf)IrYRf#Hb8Q=1qK5m zVK|@w2$6s(qTI%&LKQ3s=#5v;bIIRQL);|^tIN&rJFFoQQAnhsgdyuQQBwP!J={?4 zw)GW#CYbjcN=0&Fpwj#THiQUSkR(tHHU^WJLFh+tAL6^9SG2>34PNr(xsB*c)aA?n z-9!*?y7Toc6=0-GnU#TFO7El%vBp>tuM2gAxT8FL`A6WsNT7cX%@a2CM(41rJm|uR@jwJnLMB?3HuVkq!=C~h8GV}Y0!SBRSYp{ zk^OSA);%9{O7z=Js^xAb!66diRKzCIiXP|`$m!iac+602ofg}`FT$HcG=QwEGd6UF zGxi=-BdtME-Si@QqSTv_)-|Wtt{udW{~m(Ftg1f?7%yTXu_f?g7dT%c>QBrw1%pEVWcc`7zAJ&?8Ks2*bkX3Lmy)xo5;Y;Ke+e znC4JNqPv`h^$YUsuzP3Y?W#t5)VEz06bRJ8)%SW(WL1wr6eNDsvk1wd`R*$2DxQ35 z@QdoTw;vsK$aC8v~plh^1-Uh**!FCkhdx)DDO_}c02 zFMzbRKyOrQJJ}AT)S||r{WZV=Vu=t4K_5;-DeWQ&0Qx~nzu`T4d$5w`A>9igXow_f;s2(GW}w1~HwGR&X1l3(=*(PQZtN=ai<`M$B}d430v)e(wJ={yz-D73k$S zZTp(yh$(5bCGl^emJoZC!!I+^`DtRTG5;0pg%zgZRalnQ+hVit`>-2Pv?24@zAXwN zmk1L!)Sy9Wh%2S=T@eq~7?8&S1tI_t#RxSOmV@Etox^$u|91$skJzd{&kj8xk1Y=AIp8n;Z@|68}M#fKeIVBl`%x(gz) zCC70ve3vNX;IS96_<-C#m6+5RRq%%koN3IfDebIw~Cgw)u$0qU!4*qu8`D+ zOnkHLZU^Ke8vL$u7AQbS?{3JOLmBuZ`cQxO_N^?|)-YzGoVI{@sOuw}xnv`qXy+!P%WOJe2;i;oRJ+kU1 zh$fdn6-*5+YFAme ztgVEbK3kdgO3H1@i}aB-Y}uTZECp_%dYILnI{k3tJW1lwjo`A8*kl0(O`Djq+lL0e zla9dvTZ=V?JUMxMeQ{DKo+zWAJtDqzG&$)K=n#F2=@}w?y!Wb6Y2I_&?bc^jqgC*y zU2%dBWZ@5VxGTEYWX9NdSYSZR_fapxnUPUU3>;3kF6L~pwaAodZs-xq^M8Hluh=>%=vDhnAj0!J6QhGt zjVv{F{NyDV%~uLr#y(Bm61JYEP761yC{DhOpz}%XvL|c?xl;ccenc|%Xb3d^q8XjE zF?6?R5Y+X3-%pNocro=ywk@!dWYiKaQ(@{+7k3S^;u&E`;&m{`JMPo3vp#-VH|)s0 z4Vj~FK%rUk$3Y<*H+O4H?jXJ6 zvjKTYk7RsHm$O8`IU^~NF*AX0_lo0zGrOKiM3T98&FYO=*xhYUk9XQVyLpPXa|&xL z#bK~INv4NClzo=QuQCrKCm|wL0z(V}*Qbnx)&xKCUY=+fw!lR*QBq3Xgx|$Vl&Vv7gia0D3Nd=8PL}r_YCY?Nh3r^nVyq4z- z^w5tJQJhI?nOA0j90|KUgfRC;`W@IDnOjOp>Eil+Q zHE!KoHyIRSwp+gmLN>5gmIZ@$fT?7LfHy1LfzYeN>a`qNwbYj}cckk_Cx>WT66Kfa zfc=R<8I{vT%57n_hH+(;0_V8P;b?T@$WXT~f_2#1$7l?F z*Mel2JLbc=q^Pgan=3|JF32;mvLBYdY??UEaQV*(J}j>xMVnHp?ovL1ODNT!e3!Y@ zP`_gB49ZJJ7CL#7b(?nsV1&qz#O$B=+$SiYIOw;9EJJ&aKeayL-=!XQbx%GrqU0bx zY4hCn3Z1_`6+pLsH>_Yl<;VX3GLjGhg+JQs)M-mJ>O#i26mbjb9iMf`3wk8TTRhZb z8lyz<{qy0wyrK?nK_Ra3600tHDIdQ>9m-V-_e%@C!2V{rzZ?IKlDu-nVB@_UDM4sY z02$*c9#alY>Cjg6t5?o*3U_~alg?+ML@gwhJw_gVND?O(VSgh7Ef+bHQAeL$kEqrZ zj@ej@(jgiOXo5}U`04NUaQ@3(>=?c_BQkypxX8tI zjiH-Lw|UwIyn>Ki<0?x)bZtF>!yCVDx~nSB(B@8Ib>x{WPbTRs(bFyz(&+@h#BOnm zSMR&ftD)|^S)GlzsyA0(KRrY1;)DP0Dj4@*h=nhl6_?d3kq>go{=%~q` zwoPKbT;HF|)vS?hU>V&f!iWqlr@X|?JmZ8>^>wih4e+avz6`gh;2jO4OxOzd=W7t6gppvMi4TfR_E7{|t~$A{J}bbE2s z4ujg6EA{ONQ`;%_!-^s8O#6Nw$_v$TdN9;9G3xK2~=S%Dd94cxP3#u zgbbM5uq`S)ct7Pr1Sc@XN)d0&JBtm3r7Z-^3$Y%3~o}U*d!`& zCsrXln4!Fk!&sjrEkcryBvIXG5g%|#n$N@b=porXB&iT0^~?G+PA83<8%#KDioQs? zz9jP4{53jU;;Q)^tf?@W6Ml-!_(wKox!AIVk7LEyRpMW19q$~4semRDbdk3=A@?Eo zjcbu6onWP^8?w54f^On|V8PPEUJ7nlqdl}QQkYyBua`W$CivwE64E&{ z#k@{4xmzTmd(2^btbR#*E7gw1f8+6XN6m-!qLS8TL=Tk`fy77|BLFBsk{aZqSuZ@K zPiK{`V=Jv&Cl=NxKfmh?BQcJ;(f4Qih_Rncwq_djPUVYMjrOW@c~Oax0I`j0k2j^S z>T#Ts8;C-^vO<`Tm5_^&3-f01WifH^rKpT00sao$EII@J2XOC}tQ69)jn4M00a2MM zhrO^ZE8~*q6G^7g2u=iMVTYUpJ(wqYS|jijE>LPSH_if;6RKs+SKAdzKO%TByfuDW|v%B~*l^wEeYYpk#Ipf)*}G9+L>m@=kzD zluI-(!a1_^;_>|1*3lI>owEYf)BCR`|Es|V%zUqAcWO>@7&OJZ@eFISvW9uMfHF{Rwha<^;6AM(ti>)hUcyiibfnBBRx`; zR?qo?bbW*k{VJUt`@4;?Iti_iq+%u}s{k9$u?S4FUMI#0IxF6h$)A*Hv2UUlU)gN z_!{Qqb5GBnr}JRvLMW>`-Q;_bU)`YhSm|2u&R;6zPjAiZwsn4vXhD=?!KQJq}d^WvkJt(yG#E+mfoT_$n&7qQL>FbIv6UN0Wicyq=k54>_Q21otTBsNk-Tx;KMBZ) z;JOoGf!eCV#^o98aHT*dGu}5f_ucJjCdVxx9s=srmtiNPPVhBJ^vi%`^l9E$yDamH zd~6TO8?NoT8q_sP#}7uf3Jq!{cZufstVJ%WU)XJGzz{1{$R}Ny7vr+zLSyq2H*Ots zYms5-a*U+A5Vb3vJiX}Kli!80Zh<>1Ir&cUP1St++Z?_#;=z8}yfd4)l^WrmBf}_b zMQ68!^Q>>#wrkUD)(^@9O*eWem;G63s=MFQf2!(RCTSY&2u|hQcCP9;{-7@G=`v%%$$ zmPHsQiJWnQU>cclh)}UoSHUUl0b_t9XY9g+-Kl3b^ixgo=EGCZ%N0R@xUF=`c0M6t zd%#${YJGjkF}*;?_MY#{Iiv3AtMfXLbtSZ0emuF$W_#aoVU5P2b(FYdfzOS8xFzG8 zw4DLoO5PN~80ydT)u-1W`z%+x0PM%8lLD%fPYg+hcE}2?b6U}o&b+CsbYlnfbuhEw zwqRg|R%)bHA=%CC0-}F(yD+;L$h}+P1Uwu0cbu*WB==#KL1@A=E@Gce(P)$%xThd! zyJy{8_BGw+uKcDoz1nIHPRX`QRZZNC?ozmaITcZT68m22-v5wBEvUr_!IouaR&je( zu+TJq*-8yx_@%1WOlOP7W9^daXB4D7X2HdP`mYHmw*mY@rrPH=-?*f40#oI0U6vu# zX5)s$2pQd)*pahE7o~Nl9^JirBiGHW9g;v8Y2nc4m21VdrePetYH%GXFWim8yEljN zjh^bvfu5h_Et8}SJzEh7ubL&4TUvQ`QachiG9JJ{8iHeL4_BP)v0tzE!c^w%8%|%Q z@AgKmmaFTe=#?^_?KqFZgJ4elAa+6NOR3-Es9ssFvzBkHhrRdi0P!z%NN6!1X5(&;*oS2FbO10pLx4 zcHjpZ={o}KF%Riybn|<~f@0u5;|<)Sf8qe((JaC$0m7d?Pg}#?%p(i~y!jS$Cl+%@ zKqfpLlbyl#$4=i+NDPWz^uLQxJl^@XJ@vNN55m=wS!vTrdw10tIh|426qCyz>7(Ui-_8Ih`V zz)U_u!Arb=I)-iauq=>rB0U;9N>!Baow5k;_i5n|&`WSo2?6kA{pO$SqLQNpBzvAu zj~PZ&SSRS2xsjB9MMYVJ_7<3Si3;78vXy96lX!{XFS^%3vJs%8=w`|OMFQJa{RUVKp1Z`X_ObHc+}1)&u} z*oB%c9Bka@p%3!F-e(Ln{%BrXndc2my5u@+$yLN{r0r?%&jas%Qr|1_=UX+(f&}>> zYR|l?y!#W#Xy?p+E91VOknlrN3d*Mh9C{-cbVCc0=VLGn?9y#7yEyv=>Vhlh?;n;G1m_tH*Z>mOZ1m> zfU_q%l%=;_iGM%$y4&KFhx0k-Ecd!`1N~|QrJwGpf`2p3f`*ynl=S-e+$UC#Jl^ov z^NXB~gB7Qk?;1cpd=)MZr20V)1vD6m>O@~f2C}+9KaIClYNoUj2DNDT|(rK@ctu2S|5jV*!AujG2$6>&Cj9=5Zz|sKgkKx zzr1Xa)xtww_Fxv_#z9_wVBC$gBPWcw6P1p_%Qm3CTjvmb@quv{()RlmKfHX9+;mH( z<+l>gj21fZfDtEaZ{egd=eu(L=+LajnQb1gNo)d5BrD2?tkh1!+h;jBWYrEi?9~&o zpr6|1!mk>QLKTD>j_t;~-NwFezrZ5G37sUFfkT3ZA8V~+W?2CJ)YGTmU25Q8h>+06Q07@`E(_iBO=53-eL4d*HYp*lg zhjCD|0JT?)P=Po|vYRvU*$-u+gvYvLsrr2C0ea0j*~Gd`6Yi8ZqLcmvy8h4*irt+o z*GRSys?xN(lJlW}l9-KyG^ZF94InSRitehwOlirm12J4lSc~3LzF!f2`VRbyG7VH|=FrR<%{r6DAh4ru zm=nH1z7|1vDI!UEXBA7W%2iVTLg)Py`mxSN$K`)_`}tpMCio%37goidr&#}SmHO`g zKjzLcxUzL?)Jb=2+qR8P$F@4QZQHh;j%}l3+qOIE*mw0lXPDwfl)fip=H|h6 z>W6n*h37acU(yeH1C?pQ6^>fV&>saAnZ$q+ zAw&Lxc7e+PJFOQGlCn&&+4uxTk$K7krw2clE_6nsrg~ znfu!)#14527t3}*8d5Z|WrStC21&!nOo^Ba!y?j=veZxo8QiLV{c+i2Px99HNRnb8 zl8G@i>pIl`r;926b}{PTF4p!>7vnjiZO9ie4ZF7Ocj$Lu0Th=2=NE2@p1c#*mX2h| zCXz6nIQ{#G2n!$-Nt=476~{tXW-YcjygY^dBqrE)BdeQn*87C542HZ!&grOW`n}Wu zpkU^HU#_dZFs8UUGNtrOR{bDv!mQo2?ahKs5AV0k?tBclG0&V2GwHX$4V$MTPFski z81tG`Y8VNRBC{Xzre9^W=31PaXC_SG_&f+uFteY?1D%CCB3wO;SAUd~{&kt#ulZh2 zLZ1GP@BfPMgD)tUmjRzU=VdU~+d4xcA9fJ0rIQ4>eLpkd`+7TufzhLb?Zp`#}}UNOn65s>y}*! z_IA{;v#9jcSLPzCi;46-N*fkg3xc#%!j#g?+5Ns2&Rv@20Ej$Lj$tlNVObJPkl6{D z0Xj4e)}aTPhzTb0WE1+seG&AJO<{_%0mNZFe6tYJ6-U`6ktU1O6R!IIsl1f`QeLy4 z3Ai>zkp_!m0sm8Zf%gmRVc^LCko{B4C@&*;Dsgl(9E33s0nU5#hx1MXocHjz^MZS$ z4W)eMqJ;GC`t7`$UU*f;ty5;!BqJ@OVJK2aQ~AV+W^oiLh;-I>L8z{g;2CF>xF%q_d28mEenjzt zS;utpwC|2m5T_Y8uOXNrdTnd>*`0N5o%uR!+;>6HY<%w2H^$5GN*!?y1m9f01+P&FFBxvo-DMP) zYS$K=I&0fQ%Y$&v7Vz6znKSB@Cku%UGt}hMqWBbK50QsqmR+kr0%de5P`hfzS;;Oc z#)+JVF`76ZN@+rdl9wnb#mSsULCT>&njP3%%XGXmnoCnZSY8kcC6AZ}E{(&S%yIt7 zWUw@G2w53y#w*jK$2BocIZSMiHZEQS@4qN(5w7n}oD_v2g^nX3tiPx~*NeCAW0&hA9Q=MwdlQXzN+h<& z?nP6@&LxXKki=Si%h%Ty4I4!y*x`{>n)YX@yD?Fh_Mf#d{?L)y5!G9$T#&ML@URE2 z^b7ZnK5|rM?e-7rrZAxtGKmWB|3(@XNGhsnjS5UIp>UOqfqIxasldAT{oLG6xsf1C z`>IbH@@~qh#L>=!nukUP&fy2ex7=X|Bwa6c%E(_&gVmd?7wjP;MOBYd>|hBiU?VYE z*IMleo~bY)L5+A$-qNsH6Zv(fWG$Cq0~Jx~ALbh`vgdyI&VxBN>aCa2vl(cim0C16 z8H-zAJGUCPf3>I-oX9ew{w(ChDC)Q|TeLe_0_4l+N74H~i>=9JcN~t3oHyoN+Tpb= zKkE5_!=|vKxOKhH&T2m=@W8oPG4#5;0Yq4V*;sn67`Q_weQF~ZJ-x9sa9(@5rfy~J z-RoIPDmyKadeZQHMfv7QAA5*N3UI*=-lI?+8%6;})j$WwWbQD$~|oC(w3- zr{1;qm_xfO=yS+CCW`u!hmP!E%=EE_>CHpg2)uGc^E8knHJq2pHEGeCpwH~*f!xY1 z-E6dZ?jP-i*3HQptjNTVh-kL|kmRs@9KsHfV|Zf;b(wRo55YFs&emDjx1sylBxk+w zWU@ab*)~?<4@s8#Ey>uRb>O7M07+gIoxae{6J4O-fB|l?^Q*H$xNe92)D76C{uzH7 z?^xw$WJgG3sD#^ID3li;o$u~f z#W6bGYOh*V9~p~$I39DmNQ?AyaPRWn^D#8ZZ>HuU5`qx6IuHD;uj6eb^ zXvMr2-Ljo0kMB0KAy2opY^`%S=zgVOl}-4L#=GVyzI*DnLn@s$mp-fQ=uOB`oHgjq z)<&;<=MLoi-Dv&EUkVv0k7o@hRmaeV5#k7m1cib_VGuEinEF-65$Id{g>@(g3LQkO zUY8-k%o*h({L~9}#QNfbiD5;tqFHdvIR1ImF!VovefJ5>oZc`+1hZIqm5Gi~Cc;iV za*haE6fK$tM~$P-)^KOI_s^sMdPkw(e?vzi^PVL~k;k=W(X@QvJ)p6AXPsj(NtX`3 zU%f1pg_1a%Qm{ale_w^1_?Vfv`G9F_tj8{bLOY*X#3asx_(9|%Y8|_k-QH$@6C-Qj zmVC#muA)1l}$2XYNKya z@o$HdMUv5aR5IptT8Igl){fh=BoeLZ1*+`Dy^3PH0?V~JXy&&m1MR@b^$N`Wz7(vz zR;Hj7+{PDJTWzijsr>Qz{{BIO*2POtjv*Lvdhi$Aoc;^!Rt5+)Q~f@}K~K!=XXnfr zUN4tAmQ@A0z3gP&jMWXQ9!F4Tz{>CgrW^X&TRjC@eMF#au3n&Ui|y}s?URpffo?yr z7>e|Tl!glvh0 zBw7~aZLmB&X|~$xE-+)5>ZuUTI!Kvr(w9?puRY?Pjw|&$+Aa}PBykb49orY$oxP>c zgC5-3t?OT8RHXTNX>IvKXZ`56dHwd3-b{lU(^q$!tME~ty>trw8V?%t(u8x*FzMs7 zS)rG4Y_y@|*Vhx#GlYfnP9jTbQp&LUuxOJjYkz*GWI|TQS`kZl!9Fv%R6uuZf)~JT z8g%Jeq6%i9vX_9*L};fc)#STyc4+Vi{*H-uGm zB|iNdO^iV76mKZ!|DJsr;Pt=Cxjibvsaabm<(N;Dd+RD4ac=#B6ZrK*iRRf&kc0xU zcJ}Bh#<2G&-N?;#Js__<@@{JDXI3p6-kHj2r%)f1q@>2y+e26Bxau9q2dVdptF>xN zbx!09Nb(v62gJ#LWMQu~pMx!=iP(-7EeI}KL#Lp1@_?q)eUEI{7YLE>PzU7K)!PYI z2X2`uQRY-N75n_wPn4nVQn~?qv0ypAmNL8R6tPxC5G9?R`T_P4&V}z<3p0W^f8M}F zoTML#dSc!jc!5ktbXBqkEZ(E!UsM=KUeSyNVXRl7}(flFH((s;np@-&6H2N3f- zw1j6|a;?>iS4LiR=%0l0Yv~77#C} zZ=ti(%ARU71Z+S_Bgz&LfsB6ij#jKYrJ3okWj8^oxJottn3W<2!+{CI>}^7dG8j1x z#_WAWic-qrlzP3EAg2OFOP!7K(estaX6S2Rc=+Wr@C{{ocre@e+Jd74rJ0t`Ro|I4 zd3kXbP4Y(~(K5%S^4{XyRnZ+w#IpNpx)rZqj5Mpox0I`UYo_S0=VvR`(L2v_+yo9oCQ9rUsuQb{6UkmV zD>W&&_^u~At_qhmC8SzB-AM0jrRePKt*tkm%b<{p^w%}><$wjwZIU^bE_#I~!7?xUWSUD`_kGAd~SEUV8(3T9GCKuSM}UyRq-dJ2|+q^jI|FgQ%)$2P_xA^Jc6K z1l7KnXIkgcJ@R{{FGyJGml25S6fJ5d8*@+wnhoNQbuN7gWWi`|dwb92K_Y4F)vATI zRHhu|-BsRmo^jRlvfgm+sz5xTbO?C2?Vx%Nq47!Jjg46j$=^V$-v-%m=N@I z`gzyck@pUx^5|TOXX)22sj)4A-PQLA{!N=c=od2PL0E}Kq<3rAk=2^fnitLGFA>GD zi=D%P+NAoUM874$YKK&D=b|%BX6jqXemfr*A2I|wl+GH|s;JtD$PXVs)y=kt&fS{+ z#dshz6=$kR?aEb4;q&|%u#=??=R`-ldI7-he;EQoMzIA54o{*R?rU&XUV0##n4;KV z{Yyq2)*bM~yXU5zK9Cx2W*p~{rlE5>0h!^Szd}XgJq!_C@Yj;svTW)v_jP!+TW(dY z@y%3B6=-iAqKK?(r6an8n{Rmf#q?^0CtUPNdM=vEMtI75zi1sS)vIpM*u23Ejp0g} zhb#1874>&f3eBC~4P|vGik<7xsk|>_H+j>+b8{ZW-_~7WfK;dLM15Wj3bRbwm7zA5 zlc?ZNtyhT}+DNb4LHn-O#)?h~YP^s>lbJ)-a=1R3IXN)JSH;m-&vqWfCTnk=BFpzU zbo=<(s@l=6DYaeyIR)FuNvpNN{_6Rb;{K@->a~0-dj7J}HhcTE^JoL^jEveV+qzCa zor^TbFj-P9d9hW!`nx#9bO*=Q=}jD3X=jD-XhU*GTIuW5t|GxzrncRl)w|hUXRL&r zhTFv0S7C3xl3|!D!nI?dI*Iu^`+$R`(_5~+=t}^DY9E8es~;;7#tkg0ltRcXsw`Vr zw_VKq5d?VLCMtJ!{gT(OzrQHdNFq1X6{W2`zSY1ds3OdWddBoL?2Nf^)6vXB?>cMN zXoXWz?KQ(no-vmkQ7)jv7C-kER7TdM^A-jkPR6Lmq%QX@P8~}o1kU{`9kcFqKG!Y3 z00}kU2nt)3H@kBidp_^|j@C^vEd)B6WuH57^(G5*P?5{{QDb?`#aThGu#u z);11~W(KtOMh;Gvj(_a&?iF1t^O+Gr=;Z}!j1|_3iv!Lt5CV#Vc6B+SZ#Mg3ztN9k zwA&yfJlNjZ`EbIGVbEAD$mq*TR4x3{ec6U*E@$}RwuqXFh=@~6f4DU_*g#!V|J%gg zoXz2xAOUwy!4Fc__100g%LX>1hkj5+iFmFmE@&mQi^H4y zb1#nrOEKlwqeHpO}IN>>QR_ASw_66!K@c|dH7IpAr zrMfLJe1ARX%4p-?<~{+=IT#R7)c?10Zfxf2=wxq1&+z+;kyg*Z;*Zl^s3h&M#)i=O zqH601xDhgZJ?yi@69P133lG!%d<#dyL6kn(ryIU*L4QpYl_8&yD$BUl@AHN@(CG)d!i9^RX++-Q3J*o z^q|rT)un8%K(WaP&IYia0ER`<&s)axMUu5{e?+8A7o#Cvm+D@_;9uKGcA1isH2Juy zq3tpReTl*fCA7p+WzLeaR;vu$wk|y(D6hIcQJlcYpA^Mg&7C4Z^@gzYaC(Xuw$kl1 z$(f6*hf@07D>gYPWPMQ zbflxuJ zri;qc_Nz?k-b_qJ`1C855JSFWz#VJhQhw*GI#Q9(k4LP%gkz6lm4dBQU>i#LOsG3e z^Wk<)k*h_%VBADWu^V7C4%oSE{6#j8ZyV*7O5v~b`#+=1!gmqg<%H7ma+|Vh)oBaM z9;G=bb%fY6UVch)^{??{+rMJ?0CxlL{c7G3rpA1JaqKsJ%=R`1WA=L9QnS1&P380V z5%YQ*>>x43KZ|91+o{WIc$z3Gvec~qnQrdgx>k{mDw}c?T>P{rEx~ZE$OuF~Jr_aB z?tIi2g1Fw7Vi7Oui7|)YwQXd?%`C#PHf)pCoKKb0bCN7lAG5vq1ii0yei%MocgVrr z52^Smr>R-vm-l(cJSw3TU0vqVz^~+W#Xt|_y>rvx(l15Mp4X1o@PEDgOmfqpU;xoB z81NnUce(rQjqIF^9R3mTlw-MN`xy`dUtUm)8+7*{vmAM#D88bEqFMdqMbg>@S>w;0 zdvnPQ{4|>OX3VlWXWdPoZ)NAJ3R8>iC4aecup7L>*HRJwkbET}sYgYZNJeTlrcX-b zJ$_7UT%s9e*9&AuO=~V4YKGY0a3PCsY$#sHa(KX4GfLGdNoT8m&OwK*C;J5@2f@`B|2Z z{n6v#w@K(!^n4GTw^Wg>m0~06lz66rmWq-Y^--VWyS8j2-^4_Qdx7%wTJ_WNQ4Fnc zXbB66W_M`B9Kyjhyf)yJCA{327Vb-Lpuc`}*&!g9*MPw3{`U!-O#ge}JOLsAioY09 zjwb_e&K4VCtp`aaB1V#foHk3p_JRMZfJJGRkfwwl?Yy<-wOl==Dqd$_)`PdXv#oBS z(hS)iy{N}h?ey5Z%4()^=#OqRXp}wMkXH`5b5)HP)r(xXVCs3s2O5CcZViXbKiO^B z9;y-dXmFz6`Lr!C=&)V{OlBxF+E!*E!$1vP;>Is2;|i({Oh1NGqKi2}DllMA#)vGW zS~zKu=Sa#q$2qP?eO2ZbjqA>%TF6y?KV=1P<#`3gaujbuWkjer>O`wB43#7yuY@^a z7^nJ^LUD!vdsp+IyM_8)2sAJgAw+_doH~?zT`Sh8GcJ`)2&?19VBhvfga`$qk<6%# z8NJ3_+adQ!3kgq->%H&i$LSuY0CRkuJKN@!@iMCeTi48vo@>t3Nk~KwjJFqaC6C)n z{3CwT32P$oePFQTjWA4_Zs}<;9K1QOJjp_^c+@nJ#M*fM9k8G6iXDxtPVLwsl7z?< z^FuL=1E)cLWrCgRp0aZfX7c_b_?8w$t$8h^HRWtN+Z7YyY4i<>KP0Xt7D>|w zKSp5nJu;i})9uIWCB-Nfe#BPZ%9C-MT8gjSP^tKuafb|sv)9dA@X0qqK;l*iAo{VAqtOz2_>s^lc z<({6r&AwM>hc3d{=`0TTk8$}HQDMk z5*=PGFW|r4n`-M_R*Zm6Ad%y>^fdK5 zmMso;l-;@;%g%M*3)I!%%WqD>|N8lkOEhpCdimlh!Uk(&~C# z7#zjU@8W5Da5xjK5mR`Wm<9(yLpq33ija#-zW1T+TVOa+N^W%;VBGO?4*haIL@z2H z&sFc==nbA#zDmX`d2H9<>~*sk{4sj~Rm+2{c@Q}UgV;cH;%j?V8#4G6sJ2H=&D8S% z-#_o<%w6ht04_TXm9%>`Fs;k>w9VW7V%;^XG#pC9B>8oWr*D3lX$fJ}z?g{u$DUf4 zj21N60R{xR7bX}caDmgLtpF;5(JHS~O-le4@4lVM$-I`d{$dt)am;Hnw4->3bUR z5o$9wKNisN1$GFDTG-p>ESdwr$EG@4LGfocaMk^);u^4+2WWgFr^t(zecxb-pl2Bi zxuZ1rD9a8Z87#2zm+M@YVLGNWq_xuWvgQ_DuZExCZl5W)p*osY&pM<1olL)1e0l1^ z{-xQqCJbUT7kw+m*n48=y>6s=d{AeeJahRx+UF*9-WZt zK+y#?a7;nVz%9-Btu;giyy01st??-y87pPV@B4?#VH?63Eh6>1`N>yAkWAQ*7B-L| zn8m2stXl2+In}nnm0x#efCdXu4m_s4z?xzw-sdK5KL_4EGc#Q~YITbgF-H#me7M~^ z(@8lx@=zv1@5D2G`2uPoK^;j3qeRF;rr--4jQY(SYILhOB+ekKUH(j-Rq6)^tI)Tw z#05+!qVm|?HW+l zD5T#RzJ!sdTgEWEc{Pf;AjS;6++J<;1wGVkKDN-kCV9WsJx0?O)SC^}XRJU|%M{CA zulaOm1t?A*Hkn;o)QlZ9<3$%#zvW2Zif&Lko$Jb~bltV~w8~L!LFrer!~Nv(M8xLq zn1)2%k<1KfwF@0?K|*fG{MA(vrx{0clpum9T_2c@qiJzu+*q~P#~?T)8YAlS`LFTl zy&bl#1;CmvKtA|4N#mLS%A5Ex>oo=h;fHr9!3b-0GL$W*Sm6vBbri}rhB{i2{b6I` zh%zx$yMR{@O7#;#?oG}%W`~iZ3136C<_D?>d#YvK+j2ba)XeYxMBb!fvr);& z_*tvAGTP3QD?5hOxBej9SIk&_!&rE&vvA+@Witr*5s8R{FqsD844?~6fK3RJDa_Qh z9gG~u7G255A-}i73ySGo-H*ky{?>(p!>H+~?T@@m$!D6+1x10iyjISU=LN0$iITKi z3T@wS^=$}uRYR|vzc~JHEr(K!EJr2*`9R{B`s-yTV*s~x48UIV z-v@gv|2OQx>nG3oqk2HRf$OJFc}`t6!#sbvfeS&_NpsHMG!(i zAdhYOd36h1aE*Ne0y!YlM~$74Zs2-o+AyorA#c}vU7r>boe{~j2s9OGbX-iEq6((R zUtEDQn}38cr2#0-f;5*XGx3KqDF&Tk^o@WraYej#HGM8H{EODf;OQE4dz_P}EfuVR zzkHO~093!!fIQHWSYR8=_xxd0H%rB}N#EAG%Dbk@I}3CmilbBrDu$8P;bD%iRvBOo z%^R9*6TM@TKD<$hnL9kxb+2O$DNY*gcYKN8Gh;1>>otxG;IT8S`L_mDwQ}gu43<roP zauB?{MOa6GOSmDv{w%d{uJ~iU=}V1~TQ{yNVe?wiq98Z9ipOv6?H^L^Ky|Rpj{Sps zWqi+F*tT?=KbicS8;N-pBkLTQ_}H2Wrp@yoy?FXOQcB*O{LEi#u{~&9t*-KSq>(oKC!N_ArS(c$?7LMA1Dt0DFSx8nzc< z)DKZf#sKWSqg&wdpBH_Jr8zxm@tL#Yvu>)STc%pe(Ih(#E}W|g1wVA?*PYQ%CKYl z=ZB56g_RZUq+wxt@YOD9SbA2_i3UR?3_5D*G?b*WIsrtvTPyce&c#;jM!bbSXn*U& zPlkRaR8;+rNqN5C5hhbaov;4cMrPsiw_@YMu4emQsbX8srq zpF2_Xe7bCQu!U0CYPuKvY<%~+*cW#j^-&-7Uc#MZiSu)0^x+FTjptVQPk99FRwrGw zorvXhJptw(s8L}$*R*;94MknnJa0+DG0Z%YvHpTs(K$&0QMjr?hZ~W_j2%kBZc+IE zLB1d7u<~txFY>Yc7x|=Z))-)dFP~8T;M*45(!ekn_l1)Yg z>+?Un;&(6l*yCe#;_2@?R;)>C}vp@Krux1mz~@VP_2TQ9pDQs4PRhNj|;=-MpH>VkZ!U(JaZ z6rID^5Ra$=Ex13j7`h|yf_=}7rUJlU9vXVyMx(FiwIC){+ItaGm0AnyP6pdfYD=S( zf}J{xaL*m#z2YiZ#%D!ka5|^XD7yDTM zgMI(%Q~Xvw*xdk-_3NIb46>6JCtB9xamTN^lru#TNkCZ~m>&YL}TvD#re6d0UtrWi($j+1W@XH0s0NA+@k89x)^>bTs6Q<^dM88pUAA zB4(*lUYf3g5+|^>EfuQ~YOz{ciZA3O3SG@Mv1Mt8wwAzWfb`8qEx4pOSDtd%mcbaf zeHAUx|HPEi_K`)k@v#z;osfTqs?c8w*s z^W3=1>`>e3J*eSRh@LIkL5f-RX_Iq#%Gd$XPg49}Kf$;T`oK)6r(NvI&LW;Xf37CO zwY~IUca*AP-%!?Y>;B@I&2xCAf zk_Ja&VJD9`W3%Dm9&T6ur7C3P{LINUu(fHAXwVcT;x!iJR){o?dzy5$YN?SgDvBK6 ze9PZKR@2S2g6YN3?yfjgR8VopdgNkeW6lpwD4&P>6>~8Pt)ay_6p!jeRKu zTcM$HT!xgH9+X-d5xuyY;lXeEVgHkUZ_G~CA_JJhlf&_lpFb&oW9XMCiXCSlNH&pD zs-u#$zE8VbwPuE|h!}AC7InEn!$2ZIIXJam&7YRr0A2~QBwDbT6B}K8_RKn)RYa2i z&}?(F^{4b<0i=(%VZYZFcfNK%@oSQDCZ+nm=5D0L`xoAEoU?U6xA|1h9|>6(j2Wbi zB@~h>w%0G1AJ8|qQwwGCh-MnO3%tVDu~{1W5T5xSnhUmCr~@0~je!72U)*a?V~_>n zw;E)3!^qb5q`bS-3)&-x_PpF5FLru8zS($cqQ499d8>Q&q0O!|?5#}v3_>lQD}J2l z-kR*8r0`l=g@;W&xY2+Yon?QIVSE05p;Ts9Lr$&pvMxtfYpB6*IX~!^7DBIY5sc17 z*pch@h_Zc!6aR~{CS6WPMU1YG!BS!PyIK5nT!6*FdWHO*JBHEm!}0a8&0iyutf#hi z>)#7~tp9~RS(|kNgn`Q!N|i6Sv+=q=bgL5uGjP*^r6LWu=n8S^IA>)Dz&eE&qB2oWp*eE}4O+ijE|q`#Q6 znVlJ-msq@1JDsWzG%ktSTKHt$;-QMn8rmZ5&Yk+j1Tqn2svmhp7!Y=*NW`hn!Cz2P zT3y$9D;&!>eSS-y@;{~To6B%x&2lLF%Hf2$Ljn&QwT4;(SOa7APO=>h)}Q1nbH9DQ zp3M1?jL-uSS5IJxK#OQ>3UQ@^ywbp1-D;^kwdiUh1^K=8VN&$6;yu3pP+Ble3P-u2 zNZ>W3i~gh@B$kVI=)Pf%vB{#1T3c~)^$+qbTl|B3J=p;A0T23~O=OG0A6_niy-<@6 zD!LGIXzW3W>PFGfF62g?h1-@P!D!|n3E1$|)M7vZkf zya{m8rDbMZJ}{i7qI9hIZycy@=u|H`ENp^c;FH_fdrB1D_(nP@MtI*HdX4gelJ;xI zw80mL8a`dW&a$G^l98`&7 z85^v};<2;qE%$Ia($Hs`i{<7Xx<*&?WV_%}J8io`&u6b)RV3`+q8&fL)4`$d+}C?jM;sC+kurT_ zZr=FMl>|sFcCl)Pzx5%07LZr?N@x=Gc~n@i!h57HGd5^P=gWuE(1($DnN++@Q%Hn= z>@GQW1W)&XoqfdRCEEez?#ZrZ7l(I8G~%<@_<4Uhda8Ywq05l*JE!ePGtSkT&t|tt zM#;JL6+V|OJiQ6$#79_g5lej#O4 zRZYDxP{MGUY2%%@UfAu#U+%5IqTVHE(qFK&b7v@BFA|keO#+26I0o9B%&5Y4LT5}!_E!P1TztUHknj!@Zr%3dj zr{aoc33`kKC8-P-Vz!u}BfOAMlj+8naQ}GnvJuQY9@++xGAdi7E zvVub+a-kebj-IAL#~?QDws?r`#+kM6!O=q-d=017f4BbI$c9z-01>(A?^B0k|93=I zmbO{{V`d71>sDdzkX;!4OUqdiBvf;-3wG{TUw@)ft%3L!R7!H0?GJ5**{UU(gocw% zz_?n*&UnVm(07`#&dp_&&Zf?4t&|+2M-H`@JP)JYZqH6&At(`OaSC)GKiE;xz-AxV zra3nMe$$eua{=6o?|Ini#`)p|_}cT*y$1|$AgPt)HeTbdp&HgG#yo_mMhd$Mc2sr) z1wEiNoGGa%@#x9RbqqLYP{ct_plTy9s$vfMrG%r24pD_r?ham)IUy%f%RbYWe8Cd6^0)Z z`&gqEdhVbXSjKXnuUZ*;8AWOca)LWc+E;D5Vxt-Sjxmm!np6p8Y-?+u**XF3r`F6e z=fdWEZm*qSxM*OSp^#%m134UL8J#mk+UUwgSWR$|BRaF!M3K$5Z_wpgc>U@n!nS{JFm^V(;Us8=zhM+YcBJGX{qDK zM4jgsp5aQCU%2d>j^RnEen6aAYZcGt{KSwm0q-%Gq+Weyo%QNn62L2KxHVyT6nAnA zl(RNzskISxd6`*jxRLm!fj#v3T(g)#S7|SW%4FIr>1fd

DB*m--$lAbV>$k*A}m)6j*7O#KRbIn5uKq4F$BW)mn=pM-tk#V~y;)XxH z1{F{M#hml4QvePp{RCPIfq@ZI9@bytKc#l4$jkgXba8(6*iGZkMUY#&!DTsUGp6`8 z^SR`B!I)!`VbXh!Q1UG89F6(q$vo+j7%v+B0P*=H0>QE!)=cv%VeNYE6uT&v{ty$e zs9O%E-4~x=owt`09yQQH8}5gyTH`m*sl9j@>BmmVcSi@R1y>JSg|7QjLJ%e@=b;*T z4VwujZ&+?-=VK6}K;2iK1b;|PIr(vq998CPbhv`;I)ocPo^*WVh+p^WZr!Fz8`0I6 zl3(08p0WSRmWhVTlO_OH+WtPxC-%Rx<-eLwo|6C9=99Ski2TSbU6Q5$iINQm^5` z`*tQ~(sM)&x{zWi9%>Yu4DaQTrz>$SV}# zI^4EQ>W47A;E%h1RN%}4UXqXv631A}Y(c6VyQ-Qy!JtGS22W+aEr|CBFBzzl5ms&w zVrUc-$V9 zPB>|~`(`eacH;X(L0<|WB3yP%@0Xxr63JO@W{S8h-@QCvo}Kfieve#q`l)mVZ7wuP zP9fRj6uwP&GVZ9hZ4ywq#X*ZZ49W+ysd1@uJUaNw2r?jj&t;o_0`#8v*;yt)kfSnv zl-Xwhy(i*n8eX?cfqudE1Ccm?VadKchQjT5v7Uk-X;cNik#dzJ;88)P-wKt*F^UPO zVIt8nBo&n0f*Ny?0p+(%O+fh#t{zZ+YffoWcB0`>CYw@|VI@+gld~@SL`lV>A4Fj$ zlD18{Qdnwb^FwoG(tT~=Srr|~R9?5&e%y?Xk}{k!*a~ZU^Bw>_$!hnHcnw7AanhIB zbLEW%>l^qTA;d0GzV;yPCx)H@uv@E&KbTk7R2cAXJr6c;M8M@3+O*d5p*QwGcy)|8 z|C@Q|69M}}{9FDpi*LBAzUBzbP>TGpKoRHE##_2C&_ZYh$OYW6f>Sc1Z5}M&mM6iR z0L&w%|ATo_NAu?Jc$`~+={nr1AJpGFxP~djhFHp;zvDk69{oDsBsS&do?)JlOP<^f z;oX&}TykoTc4wc*nzff$Rv$i(o9i^^K}uQ` z^GhvKfISgH9*0Q~x#Y?=Sgoz*4E86$H(vf|6|GulS4k7BYk5}eA7dDKfOUB}sKOX= zTQ=w=8F8RvonltQBm$A6EFe%r2rHurQ|c(;SPE7~xaTHDr8K&CWZ`Y8JjcNq>`pWC zk^Kks;LhrxSO$lOX+#OJk<)Wh#7y-as{+x2@#{_-H{8C&RjDlL(RkZN{D^iHVe~HY zh8+#0ej!Q9eGXN6fjntxvNnyoU?oNDv`kN~o|=Ns7s7LARk+upq6jolRL)0`P)hZa z7bY`HVm1RYI@NlfcL;Z2-*8ikK?R8~?a0N<*9=*8kdZHK6_CRkr<#Nyy}~j2vcTNy z^K((9##lNe9X>Om?22Q5t~(VJKls1eKJ0s%Gu7LzH35WPamHh z-Ubc)p`h*-T)xUKJ*X4??CV|Q>)z10Crs~~%isE&IsT73Q2|l#rRi3|1W_=$^<96S)Y*wxj<>~o6+4EGpYPHf{F*){FD*2mv z)>xDcI|8f)WJ7IKLO<}KqtO7P7Te^RgxXSNW?@^NFSu46)5zntP^CASWlBEZei9E# zO#)*_)tX605YO;C5u#0@TNhB}TO$Rd>!&urBi6rUA+_??k zG6;ktdQBrSJ9ygU9WbK9bAhMMa{u04@ok}?D*UmI>wL%7c8~C(cb90>FoV5(Li*Qo zh2=pTOKyZ)lpSVp{?1(D;uFT+ZM8YL4gTdTq$VUH7oj~`;GF>S%w0UtQZhxUhPFdt zMsnr#WGd3&#U|j=LBtHDzZ^RC^MU6Wj8YMz zYNJY|w-_tqu&r3i6AE6MjKHu~*rU#sVwg5E?rRIp3Kc@^q z|C};RboO#E>eFe{BrLMe79U`WBlcGz!KK0J0Ibc55IIj!Lc`Ma3!Z=a)`_=Qi1T+>KtoGhh!hV&3f)V&gl3X!ng=swPvbghB@gvOZa5OP za5Rku5<-PSkCIMwS-_*dEf9<6r-{+&;<{Foh1P(-qE)?=;W&Z1F_KZ!z8JGv>~@`Q zAY>Z4V@8n~H}rj*mHHN2($fSe&>&|KQ6JTQ8I$-RhLu*pwH5oE6YoRYO^!o?rUV(r z+x)rkf|B)Jp{-C|?bA6J*Lfd^2{nUDZCM7O;WXYxi{tnZz((-}Y^F=FVdM6_y2%y! z6FeS&=S-G%dzM@5>v9J7)l6HvDD^Zepe8|<28vMY<@pP?>Cc7}gQLBI(nLZ}g_Wo7 z*sdP)m4FSKlTomvjK*FM@e%j*8F`4s)v9^*0~6FNVN{*_<%{)?R?o&8h}9xM#|cSj zPW*Sr>5nl(`9H=C|J8B2r(|)SQ<;$SnNufN6KS<;Ko1BG?ji6;EQGF@q~P5mNYwGOaf5kx4f+8l87ty@2x^3xLRu_=sGOWn z5{w$T#{=yzCu6&tN8cLX0-`bNKixOuxXos@r_1+#PV!>1x*ok`%ws$2-vBx%W}uKy z(~KEUq{$V{>06MBo_bk-YSezm6HAB2u1C}5QvWCMAXkH_Jpj*0c8a>c0jISzRp}7V zVTyn!E0;HSzuS3tp_BfaKJ%-qo|9q(6)u6JpsyRRQLr;}K!Vdle2njwf#)6Kc-YGl zU62EAnzvOHGmKQBFld5ll>Dm4k*F{TF|f}$>R3h|__5rnh4+Cz5D5;lFirDjUUD!^ zAmycdtG|Fxo=Db>ozs6O)fAET>e(sgg7=>zh8%RSBj^8zxpxe*ZCkd5i>quKt85#q zY}>YNTdP*twr$(CZQE6^&N=tpxO=}ZzPSI+nGs|D7%?K|96d67>n*co=Cj!ijDo1i zT%<`;-Bcu7M&?eL4{t_*()3}a+17$?($o_dm6*;OZ(4CQYtei|Ku6qdLy<6rG{`1y zpSlh9vYRUm7N_Ge2$H`TE)TY|COO~tI4bFG9kxG;fiPLH`!kA?qN6=g&eqCj=G=;m z`JVT#_pEQWEcnYgeXwL7BS23Ze-|j^amf z5nlbtBQK=vgwTR6zk(2Oau?-Ql`j7~*Ay@BywG~n;CVI_YOtPFv@QlZ$Hb()e}Xm@5yB02-+w&Moz1JJKKW-l|chS?IZtG z81DZ=VdzxChjxd)#{m7UFeHzOL^w)qVQBdRCz>AN$I-s|E8KCT{1n$Apro^g#iHK{%z%Gj%HRyHcpPT z26~p3M)tINhE`?{4rVsi|11f;&^WVM6Gr^X>>|7_;>3@9{GgM?R@4ylayp+^t~8C{ zUq-XiCXz=XZ{nNz`Y<+ACo~$PeSo!%X0Vi4U)dQy=kn&H{QNw9IO}M?*0cF~*j#`n zsDC2SW>7*gzJsj%gvHzGC48@iHew#}%AFIMlvX7-X!}CQH9Yn zZnncCgXultXpz9gbsfk#g_OtK^oYW(Uv;&hH{t6$cJAc!G+CXW1M;Nl!57CpLR0Do-qaMhnyzB#} zBBdNIolCqHZ^i;+$As7>H19TjjTH~YdY#fT98o)}$!_W@1Z#t4lr~~S|InnA>W@4( z?BdD7h2C&dq=rlKVsCt=)T7xnwkAiBXybpsn z8rru$v?5Fb-OeO8o_`&pXD;t&wMtKgPOD?&mjHHz@_Y=^ewEuAskvsp6rrkx`f#xs zJ-CsB<NX3$Qji={os52kRaSVtI&^IHuBSDcK{bE8D5-sz<)Aux>Dmw~=K|Mb zA2U$X(^!pX>At|5I>2_T!)-^fyD@5-96`dFY@|d4e~V(+KqTO~tuS6NLue=Oi%?A* zCdWWTK|Bf`EGO=jr1w*D|8c_g-OhJww`1WM4@-JlI((!5nJlY$i4u1YP39(Y!)s6P z$~DmEy^4FkYx^gh8^rU->Bp2+bQp0n>W=b3RGx%}x8FXKMMBHOu5!wNkb+{8Q9@!| zVg1Eb6N#ig^YM6VB+7QNcC6jqr_JkClbAH7Pn&V(yE+w*0Ip4(QJpMR+1YHUo)gR9 zuRb*`G;l(~9x?*M$a|igx6t;4YW`3jNya-r zd)kc?Q_WgPD9VJ4lWf!KzB`{m{HjMYDhhgg)SUHr%~%4}#SZef@kPlJ4a#N0)3>bE_1)B?Yoj~GWrRq%A!R!^<9|gUI?n^M=>s*FSllxV5s;Y7$Ffdmq3Y-#B0p- zMX8bO79&61#aCph=@%ei=bSKx%pZbz28n=bBG*E-Qg$L;C=5+C9H1sM-~RoQALzdw zQyG3QHIWIRL}@$Q4sWg$-QBd^jlA?+?;AR@U}Qjd@!;YtW+G`d;f%wau;%U9rO^b> zv()Ys-I#gw26*zOcX)uDys-Aw^^MGoxZZ!Bo4)t$dpvWtiX=+Cx(wybxZ2yg-(DV{ z_jRbJdwcP)^LTyXN;+`8E*-PQ8oQ~tyOCviDvVS@_()}m|+meQj4!+VjwNB9E z-n-v>KJJ@|?z@>e8TkplkjcUId^tC>VhRK;T$DBF#MfmoIwZ3tAy#(nD#Z2Gt z)4*by2xMdv_&V|xnBb!(nX3#2|D;E{IB5?2jc-tsKO*q09Wk~XzuWgbN_)V~5d{LR zQ-s*G_qrcHWl0hQr)EOl2kvt;?&;lPeEjUSdGZnGCc&@NWCha{L96&rQCQ$jBL>D(MpWi(8q+a!k=4d}Ps+U{TkjLk*`N?%1NZo#aE1N_Nxg?+m6pT-dsYSpBdJVB0_&l9WxB* zU3Hn&fR0IT)Sq?b!mp$yc4{^D4_X){gjWoHW_6O8)hEC1ZiWc!jo@1J#{2f`FMLW| zj@Prqwh4v$v;cgj^>_<)HpY&PaP{wepUt=_bHOEiJNsiAijphl+#9=cmsb^_ls0QRQI8kDaV05{*gQwU058%HO^G zJ!djnKOe@KTN#@dpE#O%=xC>QS(Q0V?3WnQbe2BGqh+_o^46u2c-Dn94~@08Q5Vd3 zK4#e|EGJKY2fTPLoosPz@+OaJ1&e^a&S{>S4>zP@8cRl&%xmGEX9=4Yb8sMp&aC96B9YT$qLTPEV{)d|Q#i%qT zLI0NAXurElYttRi@P;fa846LRHJ%ZZhY3T#XHQ$`#6qQLCZ;|^10oQoh>|fY zL;{5p87vZtOCFDFIk!mi+QxBac;Iv}6PWLL(!SNTGn%R=sTjB0b?-1*@v#<>j#|8t zxOyF|T5C#oP0yabPtOY*naw-^_B;t6PlEvmvGxg}kSdH^p;^3*T-Hyd014!QMKVo` zcj>wn9(2v)x#AdiN6(_0<6UO=A4i7Zr^Ce#lS(jvP*U+pW8%BgYSNUjrB&PuL8V33 z!3sr=42!6x8GlHW7c+ka&z%0yaqHS#2gJ5m7OU~6OVNNZ^JZ7XhW|7gCxw4jLAD=JLT8O7BWF>_B8SZhz?%6N zMhndJg=?yr8_p7&f$`1w&=O+_$1Wn`i?(sr<*giC)vZ`&T83CWhtogwOl?grr_p0J zeXb@{y_EiJUzATyg(`sfiIa9kdA)?H(0P>|YhGU7qUa>xX4FEjK8vX?;wM*t&6r(c z?f5g2Sv@b#YA(u^7!cn~WT4qZff`S6#$+58*INr`52q8LmAp)w_r zKQjtoUQBW$LpDerwF@n>cW`qrmp%3FZ%1%oIao5*bpi_F=7fmpcfqtd|HQNQlaIHJ z&1j3K-w~q>C&=`$ObDPNk-m*wAbZmki}jEAUf_>A6y-TjarUJB!_uY7=!BY-5Vi(k zv4uO4lzC~VLDLppP^k819$5s&*dJ4N5mOeO13Y^S<*dM;3V&F7NZYLt2RGs^^hAgD zoriVa1K^@R*&)x-F{dj&`Szb3>7jRSl=StZo1#0}<{#49(4CKU6jiT#o{auTh`NH2 zQ-d8ZRhfmWBv;X`EZf+ln@ki5(jx>z9fj{J`fo{;3u18N+0bLKq(P{+Q`_*84F90d zfpJ!$PlC;Z{d)8%4uKG)ZY`^i99`RZV1WQ0KvO` z{=yTX*A!PLpFl4wscdlL9*g`W`nAYE2C@BW81PhZcS4ey1Z)Z^d(zRf!P3TX@kKM{ zS5dwy6jQyq@5uGyd^~e>06%-e;zSJ1I5m1zWOm6H!einy`3dw_ClG5V#smL$f}-z4 zLEe9!6Z}`#32GO6Ge<|Gf5lM#Km5S${*ORv_Q{^?eKlcHBW8-D*eajw{g%WCA3SL_ z2>AT@q%D)g2Y1TE;Yy_~%ohCtPw>kdh6ih|8CIpHR~oHKV3RuJoP7?Hp5egZ8|-`C zfaaz0v?#N4qh@JiUVF_tI<*owDI8CNRv7%7B?66`keURr@_{ad2 z^1H{8SNMw$BTWH3k|XI*xeO+4bi(u`kmedeu{pcJgc&Ud-Fu4j+#va;4$41}zh9E- z2a>pX<#nzJD_Ma%&4$y^88#g;=B!KI&~F{=IOA=0c=|)GV?mt*e>je@M#kb!1rzYJ zQ%Ihk91_e2Dt{cZ4B!L@z(NrOcf#HRF>LZrq5PLOUY(lYkg&wV+lP$7vQBoCC{4*3`Bjy$+YPha{N?{UP|(6 zj}s#%RuOsSSxBHp!TLg;EVj;+xtN=-Gzc|%|D4Lhpdq%3H;!yjY+7y4H0rU9$JfX? z80m`p`l}(=N-z*OeH*gG_imo{A8W|}+{mL8*39>X;k}+zlHH_DUgf8KOs@s#N$?me;l8iM2=O#W^R@j zf!GnzqfWT+@5YFlgK%F{&SLPe-e?F@0RTn^sK^ zToXo!N$qW92MI6F!zf2iSS%iyUzl?$lXsY&{ozF=LfWe(87@bj`x0%hM6i(^wt=%C z0_*7@aTf)ZMxUco6049bRvMfy1`0dOo*UgxXVAyxRO|@g9lpV&{B!Lc*)z1#1+lCZ zmv0NDoky>}55rC+gLhovDMc+1R2*hait!KV8N-`3015To+iFzSIl#mbE!Ij0Hb=_O z5NF;-4Ii}XYfpmS-tI=%b6=QWtF#VJ{cR59nl4jT$GnRWt}{Jq7VF|Qcy~8_uDzGL zkciz3LDQiA02IpE5Nd2#v2uf;S@#Q7qZ15;A$I6TcE`6i5pIL(B^fHSzbvCWCmEKG zbY8vR-)7r&d2h~?R4Hn@hDx-WoJ~B=rESDmw`@NDMksa>%x?dY8?N})04M$Xy~W1* zdviDZj+XzugQC)i{Xf*eXOyyBD=2o68Tt0|`)tZB$Yo5mRpijq^V7QkNI>)YNuS&R zBP*8dm2B;jFs~26Ppvlx;5p-G#O3oM$DVtZ&i==C92f2$*1KKj#wm5B@a^U3NkY=M z#3MffL%KYwF$qJV!J+`MQm@hBEVmjBl0j~Fq~=I8gXctUPaGddh9cjZU*zQokZNNK z6`4iL(F83Fe)UJ&Ytkvti>K65K<3fI3!ItD6$@#ZYKv)I%g;m63|2#-+}|?S!|QfY zY;btJtyU<;VNS;HXxbM#sNXRh6{5E27fcqAp+(ZEp+dEiE3|zV<=ZZTnqYktMFS?o zC!^-)3K5g_S&r|lqB`1cq?oU{$!fW7!l(oyO1trOIeE-257K@S?DLu_mu zNXS~zMD>;%{`ggRh7@Ph-x-kaZwY;(L%e7bF?b1p1|KiScR?IlBrxCV2JXc=e1ILn z#YAcAHQZ1KSnfxYb}iF)HK%C9RH0=M@>4;f1d`Y~M_{!=S;{p5Qshahdi&$|*w7+*olKz(Pm!zQp!{$mN*7L7?LP5LJc{~Q z>wF-^bR>8#$j|!z{C!!mN|fr{ZD>gHNPrSsE%Cl=np$9cj%y1O?sJuEkENE&Kc98O z8sa?BILw$ls!m0z%GA4;Y@0bW`X;Bw0NS*Aro8@gdvaY4$vI=F@nG|O$G9dM6P6ks zsp9V1p(=YEn5@~7khCeIN@8x45S8{QP12Qi6XKJQIp&Qz+Zs(bTx=bkosq5J9a*c$ zd%4SuU7#or7jWVj*0ha($~b880+lcJ==g=xF76~t<_~zRX6It? zwvokmHe7asIxzMZE1l_)&UM1zSpqFrSzae-)?Kqn@2}9s8ACw#oI2djM z50gW^VHkV8&CE?Zi;CJpt9N*~^-%QLpl`c6Sruv5pK@P9w8`>CgUeeaiS`mynXSP7 zR=!leYh(TA;i;xiEP@`%Dlm$&t6)WeZRrN(Wc0J9RZ{hRYNC@zdT-XA$M`lu-_~(% zL%W{)=$iV_*<~@I8;9OHgB`sgH11Mm%ajps69j2AflLbv)EXBnOjhFTy}+b|914n* zql07g)gDCQqx`k}q8|iaKJhGExNr4uQ={nbEbv5(4o0~uU3OPA!67=yQwb1)xSX}A zOu~*P)@#;-lt4fEQlMP#3)(7PGn=eeSs>!nJqj)~~W2J}J2!`+W$B<&mC zDTH_4PqSC08Cz=CsYqy2JH~P~bCXZHcLg^Wz4(fk~l|KS}0lwKXopqiNO>NnP&_$(*4`v&p zkk2RcC23hjFi4`1yq#mOrY+VvvuD1EC$BqCOD9`h&*{Ndf9vDopxx&gmT0;(i2-xoGdaT0c3j*4!v6NvIaAQkuP$7WvpS;QtD9y3FU z(z+F?kxh~HbunU5`Q(W#D(k&4#C=ZTmYuk`wKG~dRRX@$jDV%no7-HHO}!+RLp*U&wi_yAZMh%Q`tVuov+HE#qmd8V&nN@+sGKy> ztIc;1guE00aeRXB$e}x{ff}5yHluU5>d7+2h>9JwwxpJV3e4oWIm?%bEImEb~GwfjK|3ZCS~`dNc-^l{o1sC zC!-Lfe?{NeoD?yK1sNIZ%t&_kel1rX>erjNml~G6vc;;#+WK9u2bP9BMa5Ks9Bwxp>@M!2McmYX^(>sCqqX zpHdm}N28^sr}NOUj2Uy7m-DiaoG#Cx(JN>D5?60R1LGN@d9OQ5)hrjKhtIw7D@mqB z@9mY^NtSD>LZSI7E z!w~Ly^0HPRqFQ?#SNp9_!Vqco6Nc`BQVN|h!j14kCDNvr#MIlI6LtEaYo%sUUKQGf z_^|F98PQ;OIr``+9!oXH)nTbE&#}-3KL=S4H*?G9sBe@j%Ck-R0Y#gG(FHtPes0H2 z42UUBc=;`JIhZw|bdp3tC09Xv`3Aa7#ojZ}Fmlt9x;YimnM9b6C^ro^q(5jqd=C{& z@`ZNSh`aEjkp}3iC1EM3A)gYtvpkz+DKu-bSZyD+vA${v=bqO?7B6-V?*-S?_g;c2 zFTc0!yd8i1pcoHI{UB&0*E}`rVD#x}gl&;JLL#GRF1Zx8D2yme>^QJemI7d%)QT#i za8xdIc~eqnGNV)f>NHzt_By!TJaX=vhl;e<%u9-DwupQoAx>&}D;O~~Y#`fnr(|ti z=v41E+Uv}iFLiTl&s`Lts3_pJ$8(JmK>BZY!I&|*bA5(%WO~Vf+ma;a8Qx=k7EvtdI zb2%>cHz&biW00m2*SkM5Swozo|~_pTz)1?JDdy*o-dzw%hgV(UDEKUl=gw8=p*}DbnfOj z%m--1@W;dHDnHx_&&k07yxr68p;NO=?uYI#AaStQCm}%Z1e){xhbZX1KTozH9oUmL z^yn)VjWWkPci4Apjx1T7Jg>4y?sYqB9nl`VS3Q|4oEIzm_Mt*b>#$sj-WPAbJ1!~| z)k>PIMCOFr5qN(O;r$7_U%H?2Xk#=>0JjBp-pN^Yzx#Mywc)Ro>8p(vs>8{=xV8*2 z;pAG|S%IB$c76PU$`m@Det$gsv6A~`fn1Hfs87-qYy_pSnTRs3{&Vr7$Gjk7?1w$o zC@H2L-h_CsQ zM|Xd24qjf)E*F2P6&<#o%8jmbDQ+8{X6x7usanqcErKuD7nk8Kflr*UWu^&pg#Q?-gvb+D1g3 zP4wn?nG{xXR@rs%xo&c3$h@cRj^#9w()1mbH2dvi96sHPCSCI-9n2`^ty_^~oplyU zfE1&-0k0?8R%S8Zt)8tI8?O!-8^&}nx1Ud0z~e%SDRO2CF-z0yag!6KLv^O~AGzHr zd*n7dLcoDx@}pl1-ulfe!yqE6mGqz=_C|G~sjY_J0Z?)}Ila*69Ud5VD-ty;MuyDz zLX#M7aZ)#2tawG4Hl!>T!c;l=xo$vX#enDB@&)DeDRbQOhj0e6jANyMb2!;~e9x2( z)Gh@F9&+**6=AIYv_vu=ltPrnl1-5nlj>%c$rS|~1r6K#jWrfIgMi?g_Bf3FwqWM* zi>L4N0ct1`F;Z06XT#l?elBILxF<+x!f#e~c3SWkVJwJXP&~LfGHF;YL{fTM_m4!z zpG-wu5GAqxAYA4s1cpmyF($a8?R~fF>x90(g5qXjAuYcYjGOh~F(H3bW|9%$pmPz5 z=K%zY{SsWIN6|Hq7wZKzTVPk*JUMt&e1G2?c-|25v~zTI=irDIDGITX7Pd+90#az^ zZnjqS!N$p~fAOqYA98-S^IX)vR%*GL+T!+q*d6M-Yr45PalP+*xp~&P;k`LJ2GqK| z?c&6U8oZfVIceLwdxBH9dpsoMVSn2JbDS7C#>+#RN;MYid}@$>_VRMyB){Wt2?Ty_euNJduP9|YYjSK4eP!>es$?fy(B)-mdM8CqH9!fjcq=1TCTIhmVzw( z6jz*N@+?8V1XRX043%d24IZ9Gy%VBRL2P; zHA3;JTpY~lgPF^T9R%-1R}pT8Z*qr&hzD;-jDbcVx&?>V5!59H39t2KAg}O@UECFV zzM!KL|Bm+;9icuAu-|?#Ja0By5&` z^gC@twEG|`*r1T6pdJux8<{dIt`d{A%OYPsAwRkEZ2$K(IYdz9y%GYu&R=AzXmqaX zYH>%Fqg4BtGM70pAO}CRM*YdeuVv^05xG{0TpnaP)7wL&*b==&f4%XzP8|2;IN|e`3F=_9I(+GIVRdF-4s)G7Y}@ zJ17IQ@Sy}J&nNC?mchTtI_5eh?qSH*&`tB(p)znSV{&cEbD?39j?Xa}PnAlLWXJ=( zaX3&xg4?-Lz{pd7y* zKrz1Gg}h6_G7bdm?RKROt4(e$cKAakfHaB>s5`rji+L+50=V7OCW4TV5fVMoe}U^_!!+C^ycYqulZ6(SK^tZ@MT>`lBgP-5<6|DVk7Q6^ z*Ua+7q2BpNCG+RC8LzX(Njb08VG5qD4{dZ3t?{|OHa9oVSYcB|h;La^@h#G{3SRg- zW-U&{nY+)`7NWlh+G|t6Vg&f?1{wt(^SUzRO~#*aUhC$lbMPixsEgpW4cj&>YnQ$Y z>*fuZrnVht*7eK(&CRdzoySF}4GhIk1qls2f*y;C=s(p9BBHOKNHRJg*`G3?vzC4u z3pwN;MK~@4J7st*?hEstUU3fdplJ;4W?s*x4=;s%`Tn^t?+SUtWm8q3e~UgeaITd0 ze@7i6zw;2@|DNcBfu)h2wUaH4rH#o(40lBDFZf????Fro=5?0P=yLjQx_Kd1&H5V| zfqoO0^lez}gIm^wOX2HE6bchdSh#Vl1BIH86V_jL&!bo?0$MhD#|7~g-HfNmBTn7R zZ=vu^b1vh?B8tWoSSSl(pdbVw$o5Lg#Px>szV<($=}(VS1~o~I9cMlPTHq5JO#8Vw ze=qM0$`CXH8p}u8VZ;|8NNrA#+CQc=rxn(_P&Kbp4 z0*J3sed``ES2nrU5C^c?AWtH%?+N-Q0OoeNgF~;nslifbWHDrm$I0$t&Vf8l18#2K zPTa0KsoSREk}ktsPNuAeJgqZeHq5P8ffn^NLkyqjt^J#g1-XgG3=;;>HF&g5So(vl zNbBo7?;Rb_%w0N@CMNiM*g|v&6(QC7;Zw<$*T1S#hyMVkp)0S<$Is_%Rm$wjVKxsD)mZKJxHhD(9Y2{^8(`4lEn80uTg8cGI^|P2|bu- zbPoqTii>=#~8P?lO(1rfig_^|;(_&U_c5;7&5X_2w{R zlaMuv6-kIDXtHb;Iw5KZaWH^QFWxr#yE!AEL8NdBIOr7oo^Sq2vcu(2iRc^~U(B#8 zwjN=C=qupwf_t&uB8&+CP^$G7#G&ohO5kgtNeXI6UuCWy9rBleZB3N1gDSk_o-?oYg0Hi5ip!yC}_aZa$Ia3VMH`KT$l3<*;jrq z_uMF3!8A=CRA|)I^SPpFF9(aLxJ9nBt~pgNtPygx8o5?v)}^5Rh&Wj3u5;J82CBuG zI%VHfV(w|E+q>Ms?LH2eQ(^U5u%XS)E)}1`vc9DgXMX1SW2n0-Ro&6VduUbEb*>TQ z8V*{x*sRPpR%WK)38wJNvWdObW(yzHcyb3~5-OgYMV|`O^2~1~jeZZp(9I7fRNj9) zp{7dF+PmmZVAJ-lmVU_DZl$%dx=Ncd}U6|>YYmA4e762^0)f{e7n*=KOW!T z_D&9t#tziZdX{E}dXC>kV$_W6tjtVo?2Jq-tkm?h_Dk3qt?^^hg8tW+04+$+Q8_4^Q_}PW}|)M9q``;XaCWq9@8)nMhihVm79G=mD_JLSB77&|w6*wt;th6RE4$7J4B9`88 z@7Rb1SFYGLcYA8)>d4qCKdeKVcJf1P4&(vig(y%N&-^*vMr^;|ZQ@t7OEG%LX|u%~ zR@}p^9*Z@aI-NLCB1^e^_A+fMm7ByLJF?-`^DrN9MUXa7b3)a%_ZfHGWolkk(fC+% zx;D+Z3OyNdfIf1Fa5Cq&MM%8zj7i5)y%mK|QKsTw-lKV|%J9RVSqRkLRe=*rDgTqU zTFQ5pY+qQuoFXWWWH7z?C46$3S`9l2Z3tg8mN;4Kw+UskByF&)jq@y*Ex6x>Lz)K* z@cb}Gp0cSL>|=n;Uyb)DS8)0t01*Fuh5eh1_g}S@)VB6U#+GI#rvEkNFN*zJd=3xb zp^eZ|X!|?LN}U&=v$~O0j~GwR67oB*=J(s3iIKMEqFzw@>LR#79zLn^0*=F_*8mo~ zI)<9uj?-5*JKf)VHo!qx>}V*5lu?wfE~eNx53mrSp6$N7`A{C~&Eoe|m*vPgy?Xh* z*J^0G_a3N0!PCj#uoCtpKsyHP+poz00I2_$OJr~K54kUmm0^YYfA;|*3CYZ6@si|h zDX^L~T;;%MJ81xprG!KQ&vso_CgJUD1Gu*}7*H(?=|>*W0(g6W%*%B25w8%&NMPP5 zC()_oVmsR#7?5H3<+{lIaTyW;HPhkc?e$Uq23e)eE-Y+^f|cDHa51D)D;ic)z#TKiI?g9^f53H@>}_g$8m|} z1+W#~ND`3u3&LAbpuddr51`8vU=|9$BiwuR7MsPxDSo+ek=_p^y?z-dp>_@)NEhi; zxoKXwj`>;WX&UrtO|p=)7c%VUZ2mxHn2-m-(8ZWkCI{k%R=^y~xOwn^3=1MChs@ZH60Xg5z^*-uW`ds@z!uI$~PI=Jiz%>X8**f1 zFO@wE4yWXKk?E~A5g$WlT`YT&7`G{J>{AA53C3Qgx@*CT-@5F^t?xh~rAUr#QU&v)Or2XRBqMddv?)u5{b^!?V9%lXe^+A7LF^d}H=_z0?*uKx)8I9i5yEm zj7LI28O=E0PpUVq@^?@td0;f~$>>FG#LI*|0}#t+t=_uyLJ5nXs%5-D{dTZ`K#{dX zY!wt)%9in6g+yTGuYmKtUJy*}=q0*9X@T6k1)1x|mOv#8LBWjsr94-<<5_gVDU1 zL*WiNK?kGMsHJ+P(8a~5t4-V&NEdNSgAux^W2Y*renku20-9h0D{4G!25Ay8zBN=^ zl*j%s=bf6n5J{f;MiQ)bUnQIz=Ns@`)*seP z@ZY4-xH>~7-D%%@02|#FUD}sC3PLK{_||HF(%o&R*tmNn&(R`mt)x3q>a^Z;k`%ogv@2zlyySmTK{_s8wRl2-I3No4C*zcWo@B%}(>Pk6A9 zA>?rOl7kX_k5U$tfkxmEWO{0{P9&D|RLgB{WvO&leUApg`%{Odo(<>`FhyB@E;#CT znsfM9Nao&is+ydO3Am!q$m6h;c&{zw%m%Y6`D_GaIjmOeQf#V5`FCqBPN|KYfhQCi zc1uUL5ce`C+7N2s*Wab%PFPs_JL$sCL(NKdKc(3Zexgmt+WwX|E`ElSXqUak)moS4 ztB|OoF(jYKB2^?0@FLGe@wpb0ME8DfyvIp5BwR~aokcb{NJQ#vz@JtK#p%mqqHWAo z(u-3l2;qi_0(aMnIL=IOFUmEW*yq-pTmO+(P|boUB(Qk^feV=@gQ*U3+@kEAk;$u{ zXGF@lNQE4zQ9l^=e6s%3tToymrFG?~Y=cr}z+uxTf0{`z{DU8Wf=2>4x22)A45M?nbk1H+UtXxI zubO9v5B1p1mvxU5xhQgA_9@oE?_H!0Qw4BcFVV=m#THN3{^A3V>0<2pHgXKRgIWIq_jqy7cIPKCsg|w`Wwn!OL$F>5otq4mAg2^& zVrrjSd!0sShIx{B`kkT0M=P&Cy@0|bp`PNr166ERbxdmkBpD%!RF*(y@yabMxJpzw ztsB07GVC5oVMyYEemR&U{`#cuy~9NmJ{b$(*afySfiFHR2{5T#%-l*$mMZ4>{GIK6 zInnwy2~bc|DGAS8vE=cq(#L>Eq2~^pMSG`O3&KN`2S8L?oeK{MiznhI-(SQ13|EO`q1C*LFcU9 z{k0p!rB>ToR^~!FK1t<358vNaWbBN#M!?40hSE@dZ!#z~4{%&yD^6Mq1hN+pEQpTu3|R)#7-4J*Xj7wP2wNLNf}Y?u zxoTetTAD4WKl)>*9e$P5ziIXCJYJ#~J)9tqj^3tb3r32wTqd>E1~}5sCwJkC;3voo zA=6mvI}}pJ?6V6#lqPpKiw*sKce03?8FIG}XTzD@L zdizyv>-83r_d=s5;R%k$~}Hgmo< zA$=hVde|UOR_=ly&vzEvM1z&cj-t&FzNl+i2+j0d%oVpTm|clc7ji{aLvrz~2rucL zRHDhBuY78VY%qd(fFusfVG1>igygcXnC%~O&UF&UVaNaEFfisIZ z7S0%X!JpbaX|OAlJ9}(ud3lu@!1#oClfL{=X%A@Zx~RzaFy8iz`U&OSz2qDSIWMZl zy#g=!c`2QHN-CNXtt{$r*tImo!PTM-|Jqp_6MgK_(yloM<2_WP!$u@AT&|a28H;o% zBQSgE6N&i<2@%6lNU-h3ob9!+7ei~RDSN-vVx?VEn5BF$p9~Z+L8YM0n1Ka;spkhu z>ds5{3(f$TW9~b%wv4r|Z6}Yu$-rh%If;!iqJaB~)aOHnV#mw%V4Wr3@(2P5 z>_&1yMD%i^Can5DNh$PD#6QG~iK8LEcfSE^bIsqM+A6Zw;zOjen>f(mX)UbbM~9C* zmj;uo+$kL^GZ{RIIL50xTv2y;21brycsp2Leu}=_)>$YkJc|y8o7BW& z#yhn-DokYXOr|~C2`}^t%E=SZV68fhh~V|6cGgPI-Uq}}ahVBnX6J@5{sL`oun2kO zl6tympik9S68BzVzx2?wAwlNxxPIyR+i<{PQ|aIF-^{6ponqzEB%~*cNX(dnDrBa-rm0NB|i~J9JfG~4%V}NgZ6o@FaIZ9S~;$%L9uF6t?HCextt2TN!uEn zl7uc>@)%Pu&Zo=0t+VB0XM*>t^RBn-0P6YaRmA(_^MM+AO*XdtCttWLu)toMR;H+j z1e*jnqO_0}vf@4PLGX65(Vy6P7?Z$RUtzx$&vENznECYn<&APkkxw)W&{ zYHH+PuO*AaLdaBC`h+NQ4gDzbZUZH`5L}UCgoh0v>d$yp%Q!qH_ zojErX>Wh*Mr%o9`M{d%`)Sg%OOMS5Io?uwl7dJoNSLITwJ+U(&)_*44J&^CvuLzixrrtGVf0dOeW>;*b-tZF!)hB&= z!g>}1`87MqZkx)>BBJ*@F~&KL(Yx>aJij@fd5&lPIP>{@ zf1mUH{LcAZzPn34G>FD>-l_#g+&#MLC${d6 zPll3zX#UWkcF-j}Uer!g=E`@f9v@Cy=6_a}K9MPUyZcDyvfx++YZsrEg3q+11r=@u z7u?#q9V#oX-;`faQKl8J!IJ6uZ*B6m^A@$YL*4sg8@nqs8e5)bM7Z_0&pPybNy+8V z&-I-xJ&}9OHvPOmpRw{OD`%4@DN877gL`@C#f!D=;YEEc8Ku-F#}2a$o$uI*o!`Ab zE#);WDM@LeO59&!iBCnmOE1iv9wTN=PTp#2V)fy#845Q`C$5>?+!oU>vZUtHc3bhs zuUPaz@yT!G{it%>Y~NFPPFfR zxa-R4TrFuy*a!D>om;d^r7UZ{l}1K~=hPGR&C~x5ihaEK{$Fvu;I7%c@Hn`7bXhhsu^`St#|EyfuBT7eV zWkv3~*5A^T^0v*=v7Y#_f9~2nHxuv2wl#(52LDoK98!H?^@>?jtxV$9$dxpjm1#-T zA2NSFrR8d5|HroddMce4$JgfMB<*_2KKApO#VfXUO}7o(&%Eky67Da2H!Z99p3$6N z)lDvMNjNp>(V@O_RaZ-es<_)%HKo+slngr4)S{DYz9k9&aqmXfPjP;;_Sg22yu;m` z9^dkbaPGUta@h0eiT$Ko7WK8P@z)X$n?_X4f9z!ym$LP#m+>ic*V?G~=YL9Uj6Hk# z>cyJe`0T$-WMua>>}z{jW}vuxmEOwbTD}fHH<`aEriV#Jw7nBK)^6Id>vC~r;eup^ z@TNnifme)T6Im?R!cO|ZA6zGV((U>D>h6l@Y6W9DqnXw37qYZ>`m)}tjI>7=Zt6D3 zl)Fvt5VKIB>FMe>F+z2ud**g8lIT&sr!c-R_ElTNZrkpYCMB;Fr+4(4E?d&XD6Np_ z**jH5qK6urtF@&2Q2x7eBg&GuS6{A9wTPPfz0*C<`Bl#49almhUFvi!&${UGq#^&3 z*Wr75F^3-nX_E>PCtEFa+UOurmVM*bH0JU8g{5hGuAXEB%*b9ut&IpsT3+8ZC(F>z zkSUrTOqTU^$*%7;IPzllIsN3Cobp>Q;w3f2WvOu!w@fRgHr*DAIWu8}QCM_YwT0*r zkygdK$4ZYT{(j@Pd6iO*dXg?)PDi$wU)7X1ce*c@dY$E{+v6V8D)+IU`C~5>pZk}1 zdDpL*)oL4%|1`U8Iu>!0 z;^%I7yifDQ%Gbt`Ss7X8E8jbIsZErXh`a1jc zHsvi{QQ2aBlj+GDr_YcztGTuDu~4hL)a)sG&N{-a6Y7*}++1Dt9hKe%mCcFv`{i1* z^NQ!jyY)h3GK_g}hDazRIyHz! z9@s?#{}YmyZ*G;q#g@U(XmGC&XEw70zWM5(_|m1oU;B2o#SL2+8YzP1#etY>{|VoY zb?vVl$02*rDfF)x#34i&IydiKE%2NK0p={`;2vz|S$JywKOy{YMw-mEzAWd|B6<2k z<8AjLMm_AQI*tn&&TOUzEKNm+TASnx`*6VK8&n#}GsxF(U<(esDwY*xR`)^7L!h`B zXR#6YU^81_*#tWDI=B%MJZ|~tUNbWQ_XN;N8W%L2*~~+eQR3Exs0UJfV6%r1lD8j) z5=ioZZ6D}?U&clW^`pDla^5Dm1C8O?fioz@J=n}9n6H4Y1vs?~=@i-zgL}8&8z6EX z***ZUa#pHwHgw`1ARIZAu=R(@J}AJC>Orjg?av>W2!noJ!v~vdBot7>Hz0vlXxt5R z@Ik+3lkTJs`P&dQ&Kct79&BdzG?b_fNnR9)bqK|a6i5xA!r=VCOes8mzlzT2_W*hd z=pCCYla*28=j`ep806u{AAPckfwKxkzX0ybByQ;8%x3DUAVjaFNuttxeSEhAeiKtG zk|#9?rdO=jAnUw4J8LHEQbIZmFi7MAaCQ57F&@2p7TON z>0I=1W-}+vMoB=MWTB}yG{I%4K?1gf(0yU53n3RBrLe`SgMv!IaFN$X$wh~l%P5i> z-XhQ_LkFAL3a`SD8JLzXhJ`2eO#OVfQ%KwvA$fQQQfQ=rAWuJEGI-r6M)CWZA3o@U zHR~zj^)}HOB|E(@>;@-M4Uq%O{_=D|C)Ar*j9+mj7+w{mUu**Fft-M@rSyk$A#b6O zw*^q48%4+=?ZBPxd2r8!KraJpfsF4_axfUqfdS78R0bh_=Zq+i>7b_>T9C7b?iEh{ zvfYFdzu|EFo)m8?jY8sW;0F(_SLE z`rdET25*nkn8u(7qr{K3_o1{4nF`v24x7mGyF+Z0c9r!28P3ZM>>aWWnh9Cn)gJKq zK{9+AB}cp^V?X3vFL)(@#oxn3iGRT1$HrfA-gBKCh-&j9At7u(OPhrfKgPqoXdb@X z`Kc&*qnt2jfYAc-)KSc>(HD!71;)JxSn$tL=c8m+_X2^DhG7XixuLrqzShF08T8~$ zA(YeKpe}UyF%~|ioLrJ_yy49B9Vw@xOq47zw*9}!Nh}K?5gm+w^U1C%yY4*vhEPs9 zzo5jAvFtJBR8hI0>@cC6!g5fu7%=WZkN6rnovGRTbTT|3F@X%$>?#XU68T!Qe)_>7 zjo9oC6rseAG3znqlsNy+GGepaa1A930?X!ac1oFUUc_d1{1!?g7}w^Log!-vEG3qc z^&OP>F=jocoT5|g%n}K+lfomEEC?+7pUR0`hmr`!wfV?N@pY$-J)zl2zeI>%7h~3A z%8C6kypOo=B;SmZ1%YM%Q#rY}p(KKFZ9a0E@yShcDxsVvwWGw3G3znqbRPWjR(SrAw@e>tfxN=cL=?1~F3pd^BEZ9Z~xN;9n_mQ#r$O8gkJ9#c-v#h+S< zv(rIUge>$3EStZas-xDa5)V{7R7Xh!sdX@YfR}?C|M9# zHh(#Vo-|l{1m222fDd*irsGp>>urxS0`wt|&&ul<3CpUx6hY=g*$j)=U(D}jKo z4f8X%5l*LZdRFYO1yR2{BJw921Oz(2$ltWgHGBgP%muJi0IT`PCV_yjGiv1fFH05X zL)42wI9ScgwF(Gyev!8!OV>3)>M>l zBVpu&I|Ty1-l&nc-RbIDKp6Rny#fNAU*zA!Dm+ny$iE+v=V`D8NB|sGf7Hk;s1sHAj}sOQ==>sIIOoR3R}lHU5!psmL?GZBj2ih{x$m3vAnJu9@_d4rfI#OL zdEcJ#BA$6-fCK2lmhp!18oOy~nDr$XmQ&A)~D6#|fLIIDh$ zdzCL0%nL6OMxG%jAkg_mej+11OdTcxKU;w&1P&rH~>PoJqA z69JEY@Ja!f?Jvy~5CX&N;^CjC%JbVv43Li_q|8H{N7%swmUPY%5cs@Ob$IapfAr+8 z0pRX1;Wv|u9L{W}qcuMW>8#AeE>3mluldm2+K9^X9p zD&agrH46ll5aB(JFW}50BEASY5q#y;(ToCL@Rdj6n2x}a;Aa{5qN6+_^?Im@@JqG$ z0-!wH(gvt;kDg=kWhr^6H5Q{rJ-kRIKKbAap73x-gJmG-(hVzrg2#$49>T-=)*J~| zqEaGw(D+gnJkUB;NI(x2uD~PZAcJ1Qmm_TO%sH`{;WkK+a?1kXf#S;oaIvzP-@vQ*YTxT^N-n9t{QQ*(aQb;J+hjY*T2iccG2LJ#7 literal 0 HcmV?d00001 From c5d1890467599e27aeffeb5dbf1ed4b37141fd06 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 17:49:49 +0800 Subject: [PATCH 292/343] docs(validation): record verified I05 scene isolation --- docs/validation-heal-operations.md | 13 +++++++++++++ scripts/validation_heal/fleet_node.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 5d8d7db4..b9d7b042 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -857,3 +857,16 @@ Missing census, unknown/successful mutation, extra request and physical drift refuse preservation. Drain journal windows follow the numbered first-drain label rather than a fixed count of preceding caller windows. This change permits preservation only; it performs no network change and grants no PASS. + +Isolation run [39757904633/1](https://github.com/rustfs/rustfs/actions/runs/39757904633) +completed with helper `f43151595648d83b184ab95fd3e908e7611d0dae`. +An independent artifact audit confirms all 101 failed-run files are byte exact, +preflight/move/post-audit baselines are identical, and all five archive parents +are root/root 0700 on their original devices. Original business-scene digest +`f921b69ea51622f89e9ad4a78de8c890caf0b7ffefa0b401457d6f9796fc7d31` +is unchanged. Only the owned active paths were isolated; no namespace was +deleted, no backup was restored and the original FAIL/cleanup FAIL remains. +The inventory now reports existing `bpftool` availability before selecting a +service-scoped network filter; this report installs no tool or filter. +An earlier dispatch `39757736435/1` was rejected before candidate resolution +because the submitted helper was abbreviated; no fleet action ran there. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 78ae2569..6a4e7f3f 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -413,7 +413,7 @@ def environment_capabilities(request): 'runtime': environment_service_state(), 'kernel': {'pidfd': hasattr(os, 'pidfd_open'), 'cgroupV2': Path('/sys/fs/cgroup/cgroup.controllers').is_file(), 'fuseDevice': Path('/dev/fuse').exists() and stat.S_ISCHR(Path('/dev/fuse').stat().st_mode)}, - 'commands': {name: shutil.which(name) is not None for name in ('docker', 'fusermount3', 'nft', 'az')}, + 'commands': {name: shutil.which(name) is not None for name in ('docker', 'fusermount3', 'nft', 'az', 'bpftool')}, 'installedBinarySha256': digest(binary) if binary.is_file() and not binary.is_symlink() else None, 'platformRestartAuthorityVerified': False, 'compatibleReleasePairVerified': False, 'canonicalPassClaimed': False} From a9725c88acd4977545b278eebe3653c916f907f8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 18:03:47 +0800 Subject: [PATCH 293/343] fix(validation): isolate I05 from observed platform connections --- docs/validation-heal-operations.md | 20 +++++++ docs/validation-migration-map.md | 32 +++++++---- scripts/validation_heal/fleet_node.py | 73 +++++++++++++++++++++++- test_validation_heal_environment.py | 82 +++++++++++++++++++++++++++ test_validation_heal_lock_observer.py | 3 + 5 files changed, 198 insertions(+), 12 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index b9d7b042..3624591e 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -870,3 +870,23 @@ The inventory now reports existing `bpftool` availability before selecting a service-scoped network filter; this report installs no tool or filter. An earlier dispatch `39757736435/1` was rejected before candidate resolution because the submitted helper was abbreviated; no fleet action ran there. + +Read-only inventory [39758842602/1](https://github.com/rustfs/rustfs/actions/runs/39758842602) +with helper `c5d1890467599e27aeffeb5dbf1ed4b37141fd06` confirms that all four +current fleet nodes already provide `bpftool` and cgroup v2, and none listens +on port 9000. This observation does not certify global idleness or historical +scene cleanup. Existing failed F07 units remain outside this operation. + +The I05 `lock-timeout` runtime now starts its private transient systemd unit +with `IPAddressDeny=168.63.129.16/32` and no allow override. Startup and every +resource sample require exact effective properties, the owned MainPID's +membership in that unit's cgroup, both actual `sd_fw_ingress`/`sd_fw_egress` +attachments, and root-owned `cgroup_skb` program identities/tags from bpftool. +Missing support, ignored settings, foreign programs, aliased cgroups and +process changes fail closed. This policy isolates the observed external +source from this test service in both directions. Other units and the host +firewall are untouched; the policy has the private service cgroup's lifetime. +It does not identify the platform's probe configuration. Resource evidence +retains the effective properties and kernel program IDs/tags as snapshots. +The total FD threshold, caller/request identity and 30-second budget remain +unchanged; complete I05 canonical acceptance is still required separately. diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index b591052c..030d01d6 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -9,24 +9,34 @@ live run proves each source assertion on the required topology. ## Current acceptance and remaining work -Count reconciliation (2026-10-22): the checked-in per-case audit below supports -the 104-case baseline. Later PR summaries state 108, but the four additional -case identities and complete acceptance records have not yet been reconciled -to this baseline. The table therefore describes the documented baseline; this -update adds or revokes no canonical acceptance. Partial I05, native and recovery -diagnostics cannot supply the missing four records. - -All 158 canonical IDs are retained. Independent complete live acceptance is **104/158**: the prior 101-case census plus B07 `heal.below-read-quorum` ([run 38727279347/1](https://github.com/rustfs/rustfs/actions/runs/38727279347)), A10 `heal.admin-option-contract` ([run 39197088183/1](https://github.com/rustfs/rustfs/actions/runs/39197088183)) and B06 `heal.read-quorum-boundary` ([run 39223475622/1](https://github.com/rustfs/rustfs/actions/runs/39223475622)). Their complete source matrices, immutable candidate/package/tools and cleanup were independently audited; earlier failures remain retained. +Count reconciliation (2026-10-22): the four additions after the 104-case +baseline are C02, C03, C05 and I04. Their original raw-evidence auditors were +rerun against the retained official archives; all results match the recorded +audits. Candidate/helper identities, complete H1/H2 branch coverage, cleanup, +acceptance review chains and unchanged earlier failures were reconciled to +`canonical-migration-ledger-20261020-i04.json`. Each acceptance retains its own +immutable candidate; this is migration coverage across those candidates, not +a fresh replay of all cases on the current I05 package. Partial I05, native +and recovery diagnostics add no canonical acceptance. + +All 158 canonical IDs are retained. Independent complete live acceptance is **108/158**: the prior 104-case baseline plus the four reconciled cases below. The baseline includes B07 `heal.below-read-quorum` ([run 38727279347/1](https://github.com/rustfs/rustfs/actions/runs/38727279347)), A10 `heal.admin-option-contract` ([run 39197088183/1](https://github.com/rustfs/rustfs/actions/runs/39197088183)) and B06 `heal.read-quorum-boundary` ([run 39223475622/1](https://github.com/rustfs/rustfs/actions/runs/39223475622)). Complete source matrices, immutable candidate/package/tools and cleanup were independently audited; earlier failures remain retained. + +| Reconciled case | Complete live run | Required coverage | Original result SHA-256 | +| --- | --- | --- | --- | +| C02 `heal.data-shard-bitrot` | [39281951263/1](https://github.com/rustfs/rustfs/actions/runs/39281951263) | Eight H1/H2 branch instances, 3,334 archive files | `009c96a4bbd1b8c711009c0223519f6f12d8762890d597db749f9e415ca23f1a` | +| C03 `heal.parity-shard-bitrot` | [39286997859/1](https://github.com/rustfs/rustfs/actions/runs/39286997859) | Six H1/H2 branch instances, 2,904 archive files | `da2df2e15aa973b6a04a0cb40491413db17426b63d6367f11682ef579b84235a` | +| C05 `heal.missing-or-corrupt-xlmeta` | [39322217398/1](https://github.com/rustfs/rustfs/actions/runs/39322217398) | Ten H1/H2 DB/DV instances, 3,132 archive files | `489b8179ee90a7412b223e669db8fd4f6520b9e69051cbf927b6f33299de423b` | +| I04 `heal.queue-pressure-and-admission` | [39354651483/1](https://github.com/rustfs/rustfs/actions/runs/39354651483) | Four branches per topology, six exact durable tasks, 5,565 archive files | `4dcde2f34bb7df52f03f5a3ec1e87663145a7134d3ef1a513fda2da52d1709a2` | | Acceptance state | Cases | Remaining requirement | | --- | ---: | --- | -| Independently accepted complete live PASS | 104 | Preserve original provenance and complete results. | -| Retained live failures with existing executors | 30 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | +| Independently accepted complete live PASS | 108 | Preserve original provenance and complete results. | +| Retained live failures with existing executors | 26 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | | Required native barriers | 7 | A09 and G01–G06 need exact candidate-bound server lock/version/commit barriers. | | Orchestration or evidence blocked | 4 | C11, F07, I05 and J02 need their remaining native or full live evidence closure. | | Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | -The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 41 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. +The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 37 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. [J02 diagnostic 38874900846/1](https://github.com/rustfs/rustfs/actions/runs/38874900846) completed all 20 H1/H2 × DB/DV branch instances on that exact candidate. Sixteen branches report PASS; the four original all-metadata-corruption probes report permanent Failed rather than Unknown and retain NOT_OBSERVED. Independent archive and provenance audits passed. Independent mixed-branch audits verified all four repaired shards against original complete bytes, exact receipts and counters, surviving below-quorum failures, original terminal preservation and post-rescue S3 inventories. All 50 node namespace finalization receipts passed audit. Real legacy writes separately produce Unknown with zero repairs and unchanged physical files, but do not replace the original source procedure. Full J02 canonical acceptance remains blocked; these partial results add no canonical acceptance. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 6a4e7f3f..1cc0a389 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -561,6 +561,7 @@ def start(request): outcome_legacy_phase({"runId": owner["runId"], "action": "outcome-legacy-phase"}) if owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf", "outcome-legacy", "frozen-race-legacy", "lock-timeout"}: runtime_configuration(state, owner) + filter_properties = i05_filter_properties(owner) command("systemd-run", "--unit=" + owner["unit"], "--property=Type=exec", "--property=KillMode=control-group", "--property=TimeoutStopSec=30", "--property=LogRateLimitIntervalSec=1s", "--property=LogRateLimitBurst=1000000", @@ -571,11 +572,80 @@ def start(request): *(["--property=RemainAfterExit=yes"] if owner.get("runtimeProfile") in {"root-restart", "journal-bounded", "durable-mrf", "mrf-crash", "replacement", "replacement-target", "schema-refusal", "incarnation-checkpoint", "incarnation-mrf"} else []), "--property=WorkingDirectory=" + str(state), "--property=EnvironmentFile=" + str(state / "service.env"), + *filter_properties, *gate_module().start_properties(gate_module().checked_gate(state, roots)), *quorum_gate_module().properties(state, roots), *(volume_module().properties(state, roots, owner) if owner.get("runtimeProfile") in {"bounded-capacity", "journal-bounded"} else []), *replacement_module().start_properties(state, roots), str(state / "rustfs"), *owner["volumes"]) - return {"unit": owner["unit"], "started": True, "journalPolicy": journal_policy(owner)} + result = {"unit": owner["unit"], "started": True, "journalPolicy": journal_policy(owner)} + if owner.get("runtimeProfile") == "lock-timeout": + result["platformNetworkPolicy"] = i05_filter_policy(owner) + return result + + +def i05_filter_properties(owner): + """Exclude the observed platform source only from the private I05 unit.""" + if owner.get("runtimeProfile") != "lock-timeout": + return [] + require(shutil.which('bpftool') and Path('/sys/fs/cgroup/cgroup.controllers').is_file(), + 'I05 requires bpftool and cgroup v2 to verify its service network filter') + require(owner.get('unit') == unit(owner['runId']), 'I05 filter requires its exact owned service') + return ['--property=IPAddressDeny=168.63.129.16/32', '--property=IPAddressAllow='] + + +def i05_filter_policy(owner, expected_pid=None): + """Check effective systemd policy and actual kernel attachments, failing closed. + + Unit properties alone do not establish cgroup BPF support. No host firewall, + platform agent or persistent network setting is changed. Stopping the unit + removes its service-scoped filters with the service's cgroup. + """ + require(owner.get('runtimeProfile') == 'lock-timeout' + and owner.get('unit') == unit(owner['runId']), 'I05 filter requires its exact owned service') + names = ('IPAddressDeny', 'IPAddressAllow', 'ControlGroup', 'MainPID') + raw = command('systemctl', 'show', owner['unit'], + *(part for name in names for part in ('-p', name))).stdout + values = dict(line.split('=', 1) for line in raw.splitlines() if '=' in line) + group = '/system.slice/' + owner['unit'] + require(values.get('IPAddressDeny') == '168.63.129.16/32' + and values.get('IPAddressAllow') == '' and values.get('ControlGroup') == group + and re.fullmatch(r'[1-9][0-9]*', values.get('MainPID', '')) + and int(values['MainPID']) > 1 + and (expected_pid is None or int(values['MainPID']) == expected_pid), + 'I05 effective service filter or bound process differs') + pid = int(values['MainPID']) + require((Path('/proc') / str(pid) / 'cgroup').read_text().strip() == '0::' + group, + 'I05 process is outside its filtered service cgroup') + cgroup = Path('/sys/fs/cgroup') / group.lstrip('/') + require(cgroup.is_dir() and cgroup.resolve(strict=True) == cgroup, + 'I05 filtered service cgroup is missing or aliased') + rows = json.loads(command('bpftool', '-j', 'cgroup', 'show', str(cgroup)).stdout) + expected = {'ingress': 'sd_fw_ingress', 'egress': 'sd_fw_egress'} + require(isinstance(rows, list) and len(rows) == 2 + and all(isinstance(row, dict) for row in rows) + and {row.get('attach_type') for row in rows} == set(expected) + and all(type(row.get('id')) is int and row['id'] > 0 + and row.get('name') == expected[row['attach_type']] + and row.get('attach_flags') == 'multi' for row in rows) + and len({row['id'] for row in rows}) == 2, + 'I05 kernel service BPF attachments are missing, foreign or incomplete') + programs = [] + for row in sorted(rows, key=lambda item: item['attach_type']): + program = json.loads(command('bpftool', '-j', 'prog', 'show', 'id', str(row['id'])).stdout) + if isinstance(program, list): + require(len(program) == 1, 'I05 kernel filter program is ambiguous') + program = program[0] + require(isinstance(program, dict) and program.get('id') == row['id'] + and program.get('type') == 'cgroup_skb' and program.get('name') == row['name'] + and type(program.get('uid')) is int and program['uid'] == 0 + and re.fullmatch(r'[0-9a-f]{16}', program.get('tag', '')), + 'I05 kernel filter program identity or type differs') + programs.append({**row, 'type': program['type'], 'tag': program['tag'], 'uid': program['uid']}) + return {'unit': owner['unit'], 'pid': pid, 'controlGroup': group, + 'deniedAddress': '168.63.129.16/32', 'effectiveProperties': values, + 'kernelPrograms': programs, 'scope': 'owned I05 service only', + 'hostFirewallChanged': False, 'persistentNetworkSettingsChanged': False, + 'snapshotOnly': True, 'canonicalPassClaimed': False} def journal_policy(owner): @@ -700,6 +770,7 @@ def process_metrics(request): result["fileDescriptors"] = result["descriptorCensus"]["fileDescriptors"] require(os.readlink(root / "ns/net") == namespace, "resource sample crossed a network namespace change") result["networkNamespace"] = namespace + result["platformNetworkPolicy"] = i05_filter_policy(owned(request["runId"])[2], before['pid']) after = process(request) require(all(after.get(name) == before[name] for name in PROCESS_FIELDS), "resource sample crossed a service identity change") return result diff --git a/test_validation_heal_environment.py b/test_validation_heal_environment.py index 1d9e73a1..009c235a 100644 --- a/test_validation_heal_environment.py +++ b/test_validation_heal_environment.py @@ -1,4 +1,5 @@ import copy +import json from pathlib import Path import tempfile import types @@ -143,4 +144,85 @@ def test_legacy_mount_only_inventory_does_not_imply_runtime_or_clean_proof(self) self.assertFalse(result['cleanStateCertified']) +class I05PlatformFilterTests(unittest.TestCase): + def owner(self): + return {'runtimeProfile': 'lock-timeout', 'runId': 'gh-123-1-h1', + 'unit': 'rustfs-validation-heal-gh-123-1-h1.service'} + + def test_only_i05_private_unit_receives_filter_and_missing_backend_refuses(self): + with patch.object(fleet_node.shutil, 'which', return_value='/usr/sbin/bpftool'), \ + patch.object(fleet_node, 'paths'), \ + patch.object(Path, 'is_file', return_value=True): + properties = fleet_node.i05_filter_properties(self.owner()) + self.assertEqual(properties, ['--property=IPAddressDeny=168.63.129.16/32', + '--property=IPAddressAllow=']) + with self.assertRaises(ValueError): + fleet_node.i05_filter_properties({**self.owner(), 'unit': 'rustfs.service'}) + with patch.object(fleet_node.shutil, 'which', return_value=None): + with self.assertRaises(ValueError): fleet_node.i05_filter_properties(self.owner()) + self.assertEqual(fleet_node.i05_filter_properties({'runtimeProfile': 'manual'}), []) + + def policy(self, variant=None): + owner = self.owner() + group = '/system.slice/' + owner['unit'] + values = {'IPAddressDeny': '168.63.129.16/32', 'IPAddressAllow': '', + 'ControlGroup': group, 'MainPID': '42'} + rows = [{'id': i, 'attach_type': direction, 'name': 'sd_fw_' + direction, 'attach_flags': 'multi'} + for i, direction in enumerate(('ingress', 'egress'), 1)] + programs = {row['id']: {'id': row['id'], 'name': row['name'], 'type': 'cgroup_skb', + 'uid': 0, 'tag': 'a' * 16} for row in rows} + if variant == 'ignored-property': values['IPAddressDeny'] = '' + elif variant == 'allow': values['IPAddressAllow'] = 'any' + elif variant == 'foreign-cgroup': values['ControlGroup'] = '/system.slice/rustfs.service' + elif variant == 'pid': values['MainPID'] = '43' + elif variant == 'unsupported-kernel': rows = [] + elif variant == 'missing-egress': rows.pop() + elif variant == 'foreign-program': rows[0]['name'] = 'foreign' + elif variant == 'duplicate-id': rows[1]['id'] = rows[0]['id'] + elif variant == 'override': rows[0]['attach_flags'] = 'override' + elif variant == 'program-id': programs[1]['id'] = 999 + elif variant == 'program-type': programs[1]['type'] = 'socket_filter' + elif variant == 'program-owner': programs[1]['uid'] = 1000 + elif variant == 'program-tag': programs[1]['tag'] = '' + elif variant == 'query-failed': pass + def invoke(*args, **kwargs): + if args[:2] == ('systemctl', 'show'): + raw = '\n'.join(k + '=' + v for k, v in values.items()) + elif args[:4] == ('bpftool', '-j', 'cgroup', 'show'): + if variant == 'query-failed': raise ValueError('kernel query refused') + raw = json.dumps(rows) + elif args[:5] == ('bpftool', '-j', 'prog', 'show', 'id'): + # Both bpftool versions' single-program JSON forms are accepted. + program = programs[int(args[5])] + raw = json.dumps([program] if int(args[5]) == 1 else program) + else: self.fail(f'Unexpected filter command: {args}') + return types.SimpleNamespace(stdout=raw) + membership = '0::' + (group if variant != 'process-cgroup' else '/foreign') + with patch.object(fleet_node, 'command', side_effect=invoke) as command, \ + patch.object(fleet_node, 'paths'), \ + patch.object(Path, 'read_text', return_value=membership + '\n'), \ + patch.object(Path, 'is_dir', return_value=variant != 'missing-cgroup'), \ + patch.object(Path, 'resolve', autospec=True, side_effect=lambda path, **kw: + Path('/foreign') if variant == 'aliased-cgroup' else path): + result = fleet_node.i05_filter_policy(owner, 42) + return result, command + + def test_effective_properties_require_actual_process_bound_kernel_programs(self): + result, calls = self.policy() + self.assertEqual(result['deniedAddress'], '168.63.129.16/32') + self.assertEqual({row['attach_type'] for row in result['kernelPrograms']}, {'ingress', 'egress'}) + self.assertFalse(result['hostFirewallChanged']) + self.assertFalse(result['persistentNetworkSettingsChanged']) + self.assertTrue(result['snapshotOnly']) + self.assertEqual([call.args[0] for call in calls.call_args_list], + ['systemctl', 'bpftool', 'bpftool', 'bpftool']) + + def test_configured_properties_do_not_hide_missing_foreign_or_changed_kernel_filter(self): + for variant in ('ignored-property', 'allow', 'foreign-cgroup', 'pid', 'unsupported-kernel', + 'missing-egress', 'foreign-program', 'duplicate-id', 'override', 'program-id', + 'program-type', 'program-owner', 'program-tag', 'query-failed', + 'process-cgroup', 'missing-cgroup', 'aliased-cgroup'): + with self.subTest(variant=variant), self.assertRaises(ValueError): self.policy(variant) + + if __name__ == '__main__': unittest.main() diff --git a/test_validation_heal_lock_observer.py b/test_validation_heal_lock_observer.py index 905af1f9..3430be50 100644 --- a/test_validation_heal_lock_observer.py +++ b/test_validation_heal_lock_observer.py @@ -355,6 +355,7 @@ def test_metrics_keep_same_process_fd_fingerprints_for_live_file_and_socket(self with tempfile.TemporaryFile() as file, socket.socket() as sock, \ patch.object(fleet_node, 'process', return_value=actual), \ patch.object(fleet_node, 'owned', return_value=(None, None, {'runtimeProfile': 'lock-timeout'})), \ + patch.object(fleet_node, 'i05_filter_policy', return_value={'snapshotOnly': True}) as policy, \ patch.object(fleet_node, 'caller_socket_backend', sockets, create=True): proof = fleet_node.process_metrics({'runId': 'gh-12345-1-h1', 'action': 'process-metrics'}) census = proof['descriptorCensus'] @@ -364,6 +365,8 @@ def test_metrics_keep_same_process_fd_fingerprints_for_live_file_and_socket(self self.assertEqual(proof['networkNamespace'], os.readlink('/proc/self/ns/net')) self.assertEqual(proof['fileDescriptors'], census['fileDescriptors']) self.assertTrue(census['snapshotOnly']) + policy.assert_called_once_with({'runtimeProfile': 'lock-timeout'}, actual['pid']) + self.assertEqual(proof['platformNetworkPolicy'], {'snapshotOnly': True}) with patch.object(fleet_node, 'process', side_effect=[actual, {**actual, 'startTicks': 1}]): with self.assertRaisesRegex(ValueError, 'identity change'): fleet_node.process_metrics({'runId': 'gh-12345-1-h1', 'action': 'process-metrics'}) From 31b787c51de23587da1f9a2ac0f910400ddf66b3 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 18:31:34 +0800 Subject: [PATCH 294/343] fix(validation): accept current bpftool cgroup attach names --- docs/validation-heal-operations.md | 18 +++++ scripts/validation_heal/fleet_node.py | 11 ++- scripts/validation_heal/timeout_retention.py | 67 +++++++++++++++++- test_validation_heal_environment.py | 17 ++++- test_validation_heal_timeout_retention.py | 46 ++++++++++++ .../i05-filter-startup-39761856012.zip | Bin 0 -> 28371 bytes 6 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 testdata/validation/i05-filter-startup-39761856012.zip diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 3624591e..a80ef91f 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -890,3 +890,21 @@ It does not identify the platform's probe configuration. Resource evidence retains the effective properties and kernel program IDs/tags as snapshots. The total FD threshold, caller/request identity and 30-second budget remain unchanged; complete I05 canonical acceptance is still required separately. + +Filter startup run [39761856012/1](https://github.com/rustfs/rustfs/actions/runs/39761856012) +used helper `a9725c88acd4977545b278eebe3653c916f907f8` and stopped at the +kernel attachment check, before seeding or issuing the diagnostic requests. +Its stopped H1 scene has four original format files and no object files; all +22 original artifact files and FAIL/cleanup FAIL remain retained. The parser +had accepted only the older bpftool `ingress`/`egress` attach type strings. +Newer bpftool uses libbpf's `cgroup_inet_ingress`/`cgroup_inet_egress` names for +the same kernel types. Both exact spellings now map to a checked direction, +while evidence preserves the original attach type. Unknown, duplicate or +missing directions and non-string flags still fail closed. + +The startup preservation route requires that exact run/helper and all 22 +original file hashes, original candidate/package/binary, stopped owner and +empty object scene. It reuses the existing same-device, private-parent +isolation backend and its live owner/physical/tree checks. No workload is +replayed and no namespace is deleted. A successful isolation or corrected +startup check cannot certify the I05 source case. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 1cc0a389..2131fc65 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -621,11 +621,15 @@ def i05_filter_policy(owner, expected_pid=None): 'I05 filtered service cgroup is missing or aliased') rows = json.loads(command('bpftool', '-j', 'cgroup', 'show', str(cgroup)).stdout) expected = {'ingress': 'sd_fw_ingress', 'egress': 'sd_fw_egress'} + # Newer bpftool uses libbpf's full enum names for the same attach types. + directions = {'ingress': 'ingress', 'egress': 'egress', + 'cgroup_inet_ingress': 'ingress', 'cgroup_inet_egress': 'egress'} require(isinstance(rows, list) and len(rows) == 2 and all(isinstance(row, dict) for row in rows) - and {row.get('attach_type') for row in rows} == set(expected) + and all(isinstance(row.get('attach_type'), str) for row in rows) + and {directions.get(row['attach_type']) for row in rows} == set(expected) and all(type(row.get('id')) is int and row['id'] > 0 - and row.get('name') == expected[row['attach_type']] + and row.get('name') == expected[directions[row['attach_type']]] and row.get('attach_flags') == 'multi' for row in rows) and len({row['id'] for row in rows}) == 2, 'I05 kernel service BPF attachments are missing, foreign or incomplete') @@ -640,7 +644,8 @@ def i05_filter_policy(owner, expected_pid=None): and type(program.get('uid')) is int and program['uid'] == 0 and re.fullmatch(r'[0-9a-f]{16}', program.get('tag', '')), 'I05 kernel filter program identity or type differs') - programs.append({**row, 'type': program['type'], 'tag': program['tag'], 'uid': program['uid']}) + programs.append({**row, 'direction': directions[row['attach_type']], + 'type': program['type'], 'tag': program['tag'], 'uid': program['uid']}) return {'unit': owner['unit'], 'pid': pid, 'controlGroup': group, 'deniedAddress': '168.63.129.16/32', 'effectiveProperties': values, 'kernelPrograms': programs, 'scope': 'owned I05 service only', diff --git a/scripts/validation_heal/timeout_retention.py b/scripts/validation_heal/timeout_retention.py index 6910b660..b84b33ad 100644 --- a/scripts/validation_heal/timeout_retention.py +++ b/scripts/validation_heal/timeout_retention.py @@ -9,7 +9,7 @@ try: from .heal_probe import canonical, payload, require from .heal_workload import verify_seed - from .healthy import physical_content + from .healthy import physical_content, validate_topology from .fleet_node import runtime_config from .lock_observer import ADMISSION_SECONDS, CANDIDATE, DRAIN_SECONDS, TARGET, same_process from .lock_timeout import (CALLER_RECEIVE_BUFFER_BYTES, CONTENDER_BYTES, @@ -18,7 +18,7 @@ except ImportError: from heal_probe import canonical, payload, require from heal_workload import verify_seed - from healthy import physical_content + from healthy import physical_content, validate_topology from fleet_node import runtime_config from lock_observer import ADMISSION_SECONDS, CANDIDATE, DRAIN_SECONDS, TARGET, same_process from lock_timeout import (CALLER_RECEIVE_BUFFER_BYTES, CONTENDER_BYTES, @@ -32,6 +32,8 @@ H2_CONTENDER_HELPER = '1b88cad3581e755d8e904697cb2ec471e04b5f8f' DRAIN_BUDGET_HELPER = '0442d4dd814ebfb490dcbb3cc266fb6fbebb0365' DESCRIPTOR_HELPER = 'dd7e54e6672cab76b5a6245ab3b4fbd051ddd39d' +FILTER_STARTUP_HELPER = 'a9725c88acd4977545b278eebe3653c916f907f8' +FILTER_STARTUP_ARCHIVE_SHA256 = 'c6760d0b4e657c306771a466e0a5aa4233f8e05db4aeae018ba037d289fca4aa' DRAIN_HELPERS = {DRAIN_BUDGET_HELPER, DESCRIPTOR_HELPER} FAILURES = { PRE_CONTENDER_HELPER: {'type': 'ValueError', @@ -155,6 +157,65 @@ def validate_census(value): ipaddress.ip_address(endpoint[0]) +def filter_startup_plan(result_path, failed_run, candidate): + """Preserve the exact pre-workload filter-check failure, without deleting it.""" + require(failed_run == 39761856012, 'I05 filter startup isolation requires its reviewed failed run') + archive = result_path.parents[3] + files = {} + for path in archive.rglob('*'): + require(not path.is_symlink() and (path.is_dir() or path.is_file()), + 'I05 filter startup archive contains a link or special file') + if path.is_file(): + require(path.stat().st_size <= 2 * 1024**2 and len(files) < 22, + 'I05 filter startup archive exceeds its reviewed bound') + files[path.relative_to(archive).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + require(len(files) == 22 and digest(files) == FILTER_STARTUP_ARCHIVE_SHA256, + 'I05 filter startup archive differs from its byte-exact original') + def read(path): return json.loads(path.read_text()) + result = read(result_path) + provenance = result['provenance'] + evidence = result_path.parents[1] / 'evidence' / CASE + folder = evidence / 'H1' + labels = ('binding', 'inspector-build', 'preflight', 'integrity-fleet-attestation', + 'reserved', 'inspector-upload', 'quiescent', 'original-failure-scene', + 'original-failure-logs', 'diagnostic-result') + require({path.name for path in folder.iterdir()} == { + f'{index:05d}-{label}.json' for index, label in enumerate(labels, 1)} + and not (evidence / 'H2').exists(), + 'I05 filter startup isolation cannot include a workload or another topology') + records = {label: read(folder / f'{index:05d}-{label}.json') + for index, label in enumerate(labels, 1)} + binding, reservation, scene = (records[label] for label in ('binding', 'reserved', 'original-failure-scene')) + run_id = 'gh-39761856012-1-h1' + owner = reservation['owner'] + original = scene['vm000'] + diagnostic = records['diagnostic-result'] + require(result.get('verdict') == result.get('cleanup_status') == 'FAIL' + and result.get('case_id') == CASE + and candidate == read(archive / 'candidate.json') + and provenance['source_commit'] == binding['candidateCommit'] == CANDIDATE + and provenance['candidate_sha256'] == binding['candidatePackageSha256'] == candidate['package_sha256'] + and provenance['testing_commit'] == binding['harnessCommit'] == FILTER_STARTUP_HELPER + and provenance['binary_sha256'] == binding['candidateBinarySha256'] == owner['binarySha256'] + and binding['runId'] == run_id and reservation['node'] == 'vm000' and reservation['reserved'] is True + and original.get('quiescent') is True and original.get('pid') == 0 and original.get('files') == [] + and original.get('unit') == owner['unit'] and original.get('hostname') == owner['hostname'] + and diagnostic.get('diagnosticComplete') is False and diagnostic.get('cleanup') == 'FAIL' + and 'I05 kernel service BPF attachments are missing, foreign or incomplete' in + diagnostic.get('failure', {}).get('message', '') + and diagnostic.get('namespacesRetained') is True + and diagnostic.get('canonicalCasePassClaimed') is False, + 'I05 filter startup isolation lost its original candidate, owner or stopped empty scene') + validate_topology(scene, 'H1') + plan = {'kind': 'unresolved-i05-timeout', 'runId': run_id, 'nodes': ['vm000'], + 'bucket': binding['bucketPrefix'], 'owners': {'vm000': owner}, + 'candidateCommit': CANDIDATE, 'harnessCommit': FILTER_STARTUP_HELPER, + 'binarySha256': binding['candidateBinarySha256'], 'ackFilesSha256': digest(files), + 'businessSceneSha256': {'vm000': digest(physical_content(scene)['vm000'])}} + validate(plan) + return [plan] + + def archived_plan(artifact, failed_run, candidate): """Refuse uncertain effects, later actions, mixed cases and physical drift. @@ -170,6 +231,8 @@ def read(path): return json.loads(path.read_text()) result = read(paths[0]) provenance = result.get('provenance') or {} + if provenance.get('testing_commit') == FILTER_STARTUP_HELPER: + return filter_startup_plan(paths[0], failed_run, candidate) require(result.get('case_id') == CASE and result.get('verdict') == result.get('cleanup_status') == 'FAIL' and provenance.get('source_commit') == candidate.get('source_sha') == CANDIDATE and provenance.get('candidate_sha256') == candidate.get('package_sha256') diff --git a/test_validation_heal_environment.py b/test_validation_heal_environment.py index 009c235a..15f517e3 100644 --- a/test_validation_heal_environment.py +++ b/test_validation_heal_environment.py @@ -171,7 +171,13 @@ def policy(self, variant=None): for i, direction in enumerate(('ingress', 'egress'), 1)] programs = {row['id']: {'id': row['id'], 'name': row['name'], 'type': 'cgroup_skb', 'uid': 0, 'tag': 'a' * 16} for row in rows} - if variant == 'ignored-property': values['IPAddressDeny'] = '' + if variant == 'libbpf-names': + for row in rows: row['attach_type'] = 'cgroup_inet_' + row['attach_type'] + elif variant == 'duplicate-direction': rows[1]['attach_type'] = 'cgroup_inet_ingress' + elif variant == 'foreign-direction': rows[0]['attach_type'] = 'cgroup_inet_sock_create' + elif variant == 'invalid-direction': rows[0]['attach_type'] = ['ingress'] + elif variant == 'flags-array': rows[0]['attach_flags'] = ['multi'] + elif variant == 'ignored-property': values['IPAddressDeny'] = '' elif variant == 'allow': values['IPAddressAllow'] = 'any' elif variant == 'foreign-cgroup': values['ControlGroup'] = '/system.slice/rustfs.service' elif variant == 'pid': values['MainPID'] = '43' @@ -217,11 +223,18 @@ def test_effective_properties_require_actual_process_bound_kernel_programs(self) self.assertEqual([call.args[0] for call in calls.call_args_list], ['systemctl', 'bpftool', 'bpftool', 'bpftool']) + def test_libbpf_attach_names_keep_raw_kernel_evidence_and_same_directions(self): + result, _ = self.policy('libbpf-names') + self.assertEqual({row['attach_type'] for row in result['kernelPrograms']}, + {'cgroup_inet_ingress', 'cgroup_inet_egress'}) + self.assertEqual({row['direction'] for row in result['kernelPrograms']}, {'ingress', 'egress'}) + def test_configured_properties_do_not_hide_missing_foreign_or_changed_kernel_filter(self): for variant in ('ignored-property', 'allow', 'foreign-cgroup', 'pid', 'unsupported-kernel', 'missing-egress', 'foreign-program', 'duplicate-id', 'override', 'program-id', 'program-type', 'program-owner', 'program-tag', 'query-failed', - 'process-cgroup', 'missing-cgroup', 'aliased-cgroup'): + 'process-cgroup', 'missing-cgroup', 'aliased-cgroup', + 'duplicate-direction', 'foreign-direction', 'invalid-direction', 'flags-array'): with self.subTest(variant=variant), self.assertRaises(ValueError): self.policy(variant) diff --git a/test_validation_heal_timeout_retention.py b/test_validation_heal_timeout_retention.py index c0b324b1..9d764212 100644 --- a/test_validation_heal_timeout_retention.py +++ b/test_validation_heal_timeout_retention.py @@ -21,9 +21,55 @@ FIXTURE = Path(__file__).parent / 'testdata/validation/i05-pre-contender-39711742431.zip' DRAIN_FIXTURE = Path(__file__).parent / 'testdata/validation/i05-drain-budget-39746002648.zip' DESCRIPTOR_FIXTURE = Path(__file__).parent / 'testdata/validation/i05-descriptor-drain-39754893467.zip' +FILTER_STARTUP_FIXTURE = Path(__file__).parent / 'testdata/validation/i05-filter-startup-39761856012.zip' class TimeoutRetentionTests(unittest.TestCase): + def startup_archive(self, directory): + root = Path(directory) / 'artifact' + archive_root = root / 'rustfs-validation-39761856012-1' + with zipfile.ZipFile(FILTER_STARTUP_FIXTURE) as archive: + archive.extractall(archive_root) + candidate = json.loads((archive_root / 'candidate.json').read_text()) + return root, candidate, archive_root + + def test_exact_filter_startup_failure_preserves_empty_stopped_scene_only(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, archive = self.startup_archive(directory) + before = {p: p.read_bytes() for p in archive.rglob('*') if p.is_file()} + with patch.object(recovery, 'Fleet') as fleet: + plan, = recovery.recovery_plan(root, 39761856012, candidate) + fleet.assert_not_called() + self.assertEqual(plan['runId'], 'gh-39761856012-1-h1') + self.assertEqual(plan['kind'], 'unresolved-i05-timeout') + self.assertEqual(plan['harnessCommit'], controller.FILTER_STARTUP_HELPER) + self.assertEqual(plan['ackFilesSha256'], controller.FILTER_STARTUP_ARCHIVE_SHA256) + self.assertEqual(before, {p: p.read_bytes() for p in archive.rglob('*') if p.is_file()}) + + def test_startup_isolation_refuses_changed_or_additional_effects_before_fleet(self): + for variant in ('run', 'candidate', 'missing', 'changed', 'request', 'H2', 'symlink'): + with self.subTest(variant=variant), tempfile.TemporaryDirectory() as directory: + root, candidate, archive = self.startup_archive(directory) + folder = archive / 'run' / controller.CASE / 'evidence' / controller.CASE / 'H1' + failed_run = 39761856012 + if variant == 'run': failed_run += 1 + elif variant == 'candidate': candidate['package_sha256'] = '0' * 64 + elif variant == 'missing': (folder / '00008-original-failure-scene.json').unlink() + elif variant == 'changed': (folder / '00010-diagnostic-result.json').write_text('{}') + elif variant == 'request': + (folder / 'fixture').mkdir() + (folder / 'fixture/000001-request.json').write_text('{}') + elif variant == 'H2': (folder.parent / 'H2').mkdir() + else: + target = folder / '00008-original-failure-scene.json' + copy = Path(directory) / 'scene.json' + copy.write_bytes(target.read_bytes()) + target.unlink() + target.symlink_to(copy) + with patch.object(recovery, 'Fleet') as fleet, self.assertRaises(ValueError): + recovery.recovery_plan(root, failed_run, candidate) + fleet.assert_not_called() + def archive(self, directory): root = Path(directory) / 'artifact' with zipfile.ZipFile(FIXTURE) as archive: diff --git a/testdata/validation/i05-filter-startup-39761856012.zip b/testdata/validation/i05-filter-startup-39761856012.zip new file mode 100644 index 0000000000000000000000000000000000000000..969d20b7a07df438c4ca85edca77862306a53270 GIT binary patch literal 28371 zcmcG$1CV9Gwk=vVyKJ+|Ho9!v*kxN?wr#u1wr$(hW!rk)=e+p$-1Fn#xbY*-+OZ>6 z?41iabL7l1XO2vHDNry}AgDhdWJOgSpno~g{+t^b*ch7|8#n=I9c`T*i~w{Njx7mmbxG~9YSqQT==vmoTV{O zk%rZ%8!s}NGLaK6JB^boz<>1)6t);4G&n7RXv)!GLeYVR>SKu};o8A~#QO{5f(m+? z;a?hAy#<4cBZzWeBFLOI+lLGNO8q_vTs_+(K)}fj@-`|=FFd_5e#yMYRIJrG&7Il| z=+QLqI;Z}!Z8=+J*$qryf}Xn1Gw;Vr2`c8@$B;D|;Hnz8`Pq9NT+Omkwe>YhN)@<6 zp7$dww8EIky^?jsmq=agIR7D7JCWRB(|qV>?MJN*=S(csMQ0S#79}06 z+ZEiMAP@zlH0K2-_CYh{I5I{FSA*)corw;BlJSs~VTjd-}$`*Jvv_==RCq z`yE_f3KELdDi_ig7zl_O6bMNE|Fgvq{mP?ziLkTZ&+3SQ^LX7hTaTdU`1zTYh+35WNrp1!zn3_EmTP_yDge{QvQuk7S`U>A%VzAzqKYu!tdwPRPTZHV2BSn*L4hzx_5 zzs*;35vRX`2%Y1~noT+B3|X*E1P6M6*OGYVQk2MIS}62O{-_SABs2Cs(>w+G{UF^r zTn@`ZdQazfngd+iJFC$c)qo#5P)MutSi6g)c}u?O7@wiej2r>KPb-8yA?DGE>@7Rn zfm1QEs5Yt|TV&;>C;_iB*Mz$|?-Ey1NbyZS-*i!M-LR7d;W8$WrFm7a35ZuP1@$CW za1g=#%Dy{I)OnGD4!xF5dJ_>&Vq#q*FiWby0#$S`mXUCX@5%#M=T&CW*7vg@;-k); z&K|f7BFeLT6f*&$ws@`C3zeI+>~13F6dlGCa0b;bs+- zob~dNG0H#Qv($IuFbtQ>5F5jV?Ps z@Lx<@kxNo4f&c+^g8eI|iT@4LX8(Ik16<6F0X9Z}{{^h+#Tn@t{v3?7hUR~KyN&5T za1MwY|Lf94o<3m&*HtI&o-q~Mi=kpGz7dK%2{~X=)dGYN`$c1Y=xH#_;%u(694`&G z*i}^IekI_2#oM1g32myLo{{0D>=G?ot8_a>A?znMBcQUCohWzecF?u=e|V=8gj}@R zl)GyiPaY&TjV}-!=(@ep?A6NM+IFL5?_W9{i=}li;IbjIw{ zA&-Y9+QqHXli1C`K&LV)KP#M)`^H_m-bh*n)I6+F5L>5t#$!HgmaAzs!i|$|FpUU9 zB?{tw^#fpg_Whfe%+9L)qlX2HPw1!oX2`EN0VtsrwaB9p!n0GSsEOZ!OP}T5;3>%ihVYFOC25`j5xQ=EhWJe))V}FuDugZ&(Fq#JX{xV|%%Xx49=-GRz z-r^bwhQ%O1^Ou%gAAAPV*rUM(W)*?VtAymg6*=5VBVBI85KE;IWD}O z9LE=AjN>Ri@rPle36Kkofz;0ny zM%v0#pE-?)u;}JQ64e))s$z#3=d*P^>pG$~vBR7w@KNC{Yq)b|5}SDN=zk zQXi}uZtln5*Rd9R&3HO@-jwMtqi?Xd3-`|X(cFg6h{u1vSIjiOEe~H?Kya+Osy%Jr zu@z~M8Ei5Mrxx=V%o&kpF#m9s{dhG-6|_Jz6vMq@&>DDTE!M9yzy)uFj}Q_bqHL*0 zu)MkfWrHUts>K*SJBv<1A-f&;n1` zaYJ};etonNYH1~ce%28-p|EfUuW(!;33RqtW%G{FwyPV~TAi{3wy5!M#vY?ye9V|t zQ-ML!6c@5*$IF5_vErEkllY>j>~TR-aDcX4YPVTtm_2G?Rk%1(22l(6x)^6cw% zOH{bdnH=n1X~qky!z^1vT%j;^XvKIqsyn&30ugEwl46pn81{25+uD?Bc9NyHrsH~U z)`=T$^42Y%KnxyKWaccz3w}b|CS0 ztOb8Z5M?N0m2&YQ+ISd?)h4E4o?%$Z74Cc_HHDE3`D;<%iXvjP?`UzvHs2qgKBzaJ9}I$C zzP@^dPPL=Z!7A0|pfDgtcBHw8Yc{HQKf2ePgcGSvMUs_HhVTkx?LiIji!&j&Tu|{M|r<^d|~|teqwwrjEZ*BWlF_f;Z+60@x=Sxc3 z2qcO^-u3RM?DuygQ2EfXuy7Pqt{Q>tK3Km*cTuvIYL_Qg`5Gz*d8E1D2V=QU1PtGC zWJ__Ut_sLrXzn2b01<7c=Gn(>8;(64Uf4BTuI1BR->oAM@C<|wYs8if<{zsnWHbnL zepbBI4KCp!sBf*#tu8fnn!n;BE_Ru>YX#P5qEoo700E)isV4(!fuCKnre@{#w(qh; z26eK9vy|KYW^7IGEmflpT=APGH}lG)->NE3tsC*mzS}jvhE(NZcwQh@()%q;@ip&&6c9R;rLaKkDCRf&gb>uABvXIsU=H)z zm)4hmWH3%kS*o-CzzE{Mu}_kJi+RldqcCEnbpSX599#hZq@9C_qkn^*;4AMxd93>4 zmU#jE+-4A7bY8eogNmVVN){6Hgjc$u(Kt9zb%enA5u{UJjxQ!0uH|KAZ>Tj}st+3q zl4lQ0i8dY+4^3@af1-~?_tWlR5rVrasvS|`Jitav(^ld*4MIIuMP`VDTX zcEt13!w5veGZ7NgsnG5ROy@ zdz-=5s_2QSd}xfA!WKUPSE!}uNb|O9Pa(BPg&7hO5wYF9<`3nbpom2GF=^vqPHF++ zO20J$BP2nN=`7Gqak_e^F~!U9tXQBXMo6S*bMN#dns zCF3+!#JCR?%?a0^W{{;C0kpJp8{~hr5|53oR{Y$vzY-eR_@K50L zP6#6O5h981@|F-*=)eW=2kG{s05M0AH?pE_rfzkf3|+j%KQS-00nnTaoVO{ETgTJz z$KQ%7tJg1dF>jP5yU6xqIYOYr(H$iQQ4)Cv8GJS29$?TF3@Zt%@Qm99?yV;l*hG*bLq+rU5H|4QVM{+k2%&ypK=AYz7BuWsb?{C_J zbCAK3B+PK}Qdm$M%AW#?bl(J!8h#XU8@y(1ctO>e37>5Rh~B&IX zRwFvHAxtGhwm=)6!@aJPlI+&Z9+Zb5Uni!Z{IXnEB2>4^Yf6fD3sL5F3iXnAgRbw7v zJC}Z5F3HpyF$@d{_aa3iEN(8Y4b?M`d0CN0etM>U++OAjoCm=Fa8AEednh}cG|J_p zFDuin2FmTK_)}mO_vQV)$V zwU)G#y=XIjEtSPza5-I2-9NVj&Z8s2vOAc!PIDb@mvHEb-I&5)bMi;ozK3yIj3$vG zaJg54U=#SSk9Qg97%|Vr(XC6JhRiiO(t9U*VR}GZW+)jOC9aWSC8G2c#`P!%j(8;} zQ~5EvB>_|q8!o#z&%D?4vW?YLcoOAVCeuWJ!5QiGv3!wNrLWs8y}n+& zxpsGdo_s74*nYjArC!?Gf8M*@09<=_XnZj5m!N6_}cpg6o*v>l=&jKMMk% zP6;nwE%Cjmu_hf9${;5st^4z?JJlV9e{AAbg4vPk=YsIob#5mMQe6yoJ~2g9Xxv$> z`xF~cAf{5{6gQS{tSOb1{B+;M*BNtH>PvfuyS=cc63$yldzwW%rFIkv{GQ1?rw(La zhqV0b4C{P2AL^^=%5Fs3t^E!C-!9#G4(derhX4WK|J4VQ{hJST`HyyK&i{9pwz4&K z{0ED^_^Fdn$X56G@d+ccU4dtM953KiiW|+L>uEAQu*MvHvidZD{{%)Ppj;~hEIPy7 zi178qP73w_zIpGY;;Pbk9&zHI2n68!rA(dlGOP#GGIATLBi%cA0rJD`MpC=1zVKVh zSh^Y`yN!9eaD(LSL@IMSc73Y>^0Hv|g@7^kNNR!9)Yv+yvCNy#k#pLtveOM;g|9gX zSjkA%4-6KaA1?$uQo6Mw_c*UC;A&{J<)j&YA^fPA`VLa#66s(li|I=$Hp-C^_!Rjc zg(aQD5DB5G${zyp9v3H_@FD?GCWp(c^<7n1_?zdqx3^rLn273y8Z~O@8~whA!?tt{ zJn!Z(pDkUb@!)%CI;hJS^JlOq>Ct|<2I&Or2y`-L?IV5JA3n<%)m8M0F5`+4?~j`q zPq%aa_OhnZy3%*C#e|cl%zpH`v*8lhF5`6;BD>#Vd_<5BBZ#Lp%XoHfsjCA-EnL>w zo-!Wi&mu^r$^wf=ebh;>z*ssAG#V_HXDC*qItI!qtG~O`R+r~jy>rlGG`PQZyJxfN zWlWn?+^Z2}8QXuBVTtAhb&N@|WgYf*YpB1N<8M7H{4m{CBZJSVmTb{WbXN}XxBc__ zWfiOCXA$Aa%u75{&dHEoT$So$=oQo{^CzcCd!zs?9Tq*T!84#!OPtH1Y?aGwEd?O19D~OV7z3@ z+;!Dsb)Z)+iB?rp(!tj5-yG-FrTQ&;76q z%^QliOjv12x|IkkZLoIMH8EUk%ZbiRQR*&?j1x>(*{}wSyBBxi%LP(R?U_KqX*oe7 zcnp2(6N45Jxy#V@Mth6{m zFNt032XICA&yhkSNTWFW;STB7{?psB3R*2>CzIo8)n`QwKHw-_ZVz5}gkqO=#PoPJzv2)$H|cs?Z{ z$9%Mhttk<_kx4Q^goq}xB7aOB?Xa$;n8x&`B*z#H1=fatdhS36!g{eIgu$nqRN8Du zMepZbaXvfhPf)0$ z`qG_af@KWL(2*+oPvs{tT%YyMw-e0a73ST`K7(_@aR6{`Uc;K7o4&4^k}o0u%=I2K zj<;cZ6YG@Z`MVW?g>GjnNS_}G1i;9|obFY7);o)+lq*j`nQ2R!wp*qorx@(VN=1K$ zZMy71wFw$WXV>xNc3ph483kdwQEV+c%dUoo z`j)hOpD91dH_5A^`KNPxTw+Ki=B*;1b7bzup?Rmicj|JTQU?aInXZ1%2E()@81AuV z74wuAE!OnB7CJDieC}8T&!nkTZA}N@AQC_XDw0jU>)_b=6ij}CGnOH;VVbVoHMf6D zH_^f*Rqk*?e~iIzNToC8sq6S%(oKb2kQUq=dp_|=lX2esXy5W67Eu-=yde$on5nv4 zV%e;RIpo@%k$?p12I12)%Y*;Wy#5|CYPv-URH4r{QubRQKkK-AQXnoE&6E^G_)9!R zepoJ;1bVGMATBW}DMs7A)pDc4(2Vu#vs$ki_RwmXTcE%RqN_LC+|fg%CSun%LSVA)eE)lxBRf++ub{m;jk);LCl zbobOTt&Cz`&*Oe)my|J$_>{D94diG`Cwya(eUYIe3c?y595N&~!(-G_>x&xAhZ-Y# zfLAOnISxF8>eZ6Q%eOe0*o#Owu{OnsCz~|bfiv1>eXi=VK3i~4N32fh`Np4R?ai6- z@>od8auLG1#UgVLo4n3kX@qW6pRQ`$@QU5yP)~CiRaU;L@C!pB)&9zA{Yz2pugQ@| zRCj7`Zs2f#b-4?e|O zy;+ri2l;btc`tG=G@Ct%gSnY;cpW(EZc(B?!q&Mm2tLvn8*0&K=$)61_Rly~W~n5g z`=*cpjRZwMW_0KSGeBdz(uv)D!M(^q^&;S7SmI7;xRG#Ba7YS{nXqWP&^0^?WT~SS z{n9fXGi}0mq(RW44cve#R}G|ii!fGg0v-vl=lt$n<|AxZ-~HJ0wzBr#xMX2g0XWY-DLayEN2vi1Ts4X zNFRrD1sfv|>kfeKA1Wdt9^}Y^G&==y(I?i5)j+4G9qB)RR=Q;(&qwDb=sRtuw`mRdA|9-u9mPO=EFNXli2-T6B=ft&LG1y^kAdSazIil8kVzk@nqTKpG z^-CyLM%=4CCyqUbGUjxI{MN!TQf{kN*_))xDs^6Qv}Ve;m<4tfkXzW{Y2dCG8UxI} zyE*y%awJ`+Heg}V9#|B#AzfThl|1qyYQymQgNMMBmJE1X&3%)oL2*RnXDDeBGM_UW&3uZwqo z`iT%NCu4x9Rcv{oo<{*fT!h9N%o3c9BE&Z2_C$CxYgEb^MUdp~+Mx_80eZ6PD~vnd z5Nh3J98tBJmaYu_J?ikPKcA}3`WrY0O9hY?giHTSZZR~xG-xm_yInpdA3aOfzC%0* zI(d_fOB1Xt7`A7sE(;S2(`(#rOLC{D^v7{rt=^ebzfivs5dOgZwXAGSLCCP6bs62o zjzNt9CneOxN>AJw7N*klHP1RDNcbkwP!)#zl7_yJ6Q4zBdz^CEsA#1g{`?yw$Smui zdn*d~Z-ZjWD50o+is8JY*zBy;g5llX3lVrx_#sH&Sx*|wPO0NG@;)d}3dkYP3MD<` zYHtqg9I()2&@jNz1QgyllO0kJM3m9E9Sw7M7)mf*eh*%Q6PNDP2No{q?cP$kQ_bnZk}wnNIcn#V*!Xqu^2864uVOvpAiPm zz>RhQas&i09A(f^Z5=15GLnNw!VoB)9X1L!3Sv*741;J%Gw(g?7DE(VR3s!}zaY}Z zp&q}7@%hiadkK9%!bVW~K!|Yb^&Tj}0Qq?)>6jLKVRi9|yHJW1mmT~GX0JY314#Gi zPDiF!;n?QxNi29C_>L0rY)g{g=b0Kj9y@r(mdXWk1y^CSN5sZaQc$-AWBO#%R{=ED z0-4JiU|lagW?*HUpud^hgw^8DwNzlSE0(vxu--Z(mb~6-&RGV2Zx2$|o;=rKiid<{ z7L1lOQ}BJ;S3kP);zfXmsl|w=yoF0?o#=sE69q~$7shw|g~C0-G^$}X!|V(0cX@(^ zL1DgVshkNz{U_mheGif=yS%?N!+~t3UTnFSkj2n+UYdUi!qewfE1nI8moqL5=)^~i z=^;j455?v{sP!WmW-3MdLnCU#e&L86OFvTJvp%{j+_1KK>V+nuK>7g1?KR~B6lqHY zOmtCxbr{QTP`Ra;o+ph6Y5!`rpckP5pxs5p500i#nZ?&y>#Fq4Zz=1dA?zGg>uENBrEu92p9l0tTPv$z4cT6ua+PE%-=Tbx{FK^Z!yV)40 zX6bp42{eTO^0zqS$zju0W6fpBLW)?bM?jNM&U~u14{LSA-ngEsCT?v{P!Nnkp!=zFQ&{kv15Myk)JYYP1(=^YKla z^&Yu52DOT8q8dCE1)sm3;8Vw+vnDbp*A5BQy`E3L^v8&pPh1wqzoOlW&RcbjzYZs! zKc(NGEgN6F_?1&rv74&cs-a~~3MXybP}_)^+_|HTW8b!q2=k38pMK%hMei9XPH1S< z(l4?#qbu*nYK_h@l^<3}_VHyACzMmHmr+tBVRl@8>i#lz@5XbuK5C;YbWT+6Vu4lN z3?n`M9%d!%`jjx}rRS(8TMMi+uXo0*!S2=RnlWw5t=C;oQeXC9nqE1x%$68_lAI*b zaUW1fy?iXBIU{czk_gjNeT$RaIO$z|7yinkNL|y}!3SCTBABRs9Be!PtF1UexYboJ zR3ODpx7eLvw{MZTHaqM=y28|u7v(HXeR?1yBP?caScqqudy%kRX1c`X#tb8dji%Ay z7rd0Ux<$VtPBQ&)%>o_%|1Ix9zV!DNjS;5(gL{O-_O6zl1f3D441E_OFYeM8N3%;s zO>~XUzbd7ybu7+q0a__*(*^&})Baw4WB=-YNjq|ZKQ(WpF%!fM*(6HJaLn|Lg+J&{ zG;(qC1TLFT!Ov4S3l+ zt{@o3T5e~JK(*w?xCHtfD_<7t>9D;&xeX&RmWs+uyrrV7O zb6nzQxkrnFY)#0VozkFhGJSbyajL?a$h_6W4%Tq4XKNilpAJ8w@n3ra=?OKTa(qp0 zrkj;#+Cqx*uE&dS0`6xt*&Vk;O72KpnAUJd8=7 zA)g_qM`?YJJNGg>Mea&pU@LBZ;7R(J%06khFylf)m?brCcbJW3qr(^BRT%h~7@kV2 z-Ezo6f$LfG6A+6hnkV_W=V|8m9Le zHeJZK6#9R5lhsz+5bQEhKn@gJr?qqX$bn_NjlY-V{mOH9nsKs1DNsP4m#c^u!~%(9 z0sj;4aD?8~t$$Z8Uie{niS(G@RY>sbDR$wvV*MhuReA|?4(`*$1O>)7H_c>b4EHS& zT>ll}oFrn_4m}0$yF}Fn$Z2Uf-07ddWUK@%D)eK|HOx)|GN_EhEE~Ro5Ka1KV5r0- zfxzJ;sP6c^ru;&-F}Him`WOZa(h?5u8)4ewol0x%@i~R za3SV<94n>q9VLZsip+>|!{%+(bd{Gc5Al1qf>669gK9EqWS7FbnIh*hryF;~8~pTt z45l62mikrM4a%bBPx=ixd-2;0%Orn2lrm?mV4klDbukLf@wXZpniLW6c$fu^0m;?p zU|K!kbddnwFkloJVMIrd@qumR1m4=?Z||Lj8sIkQUawE&TxlP|-(L}%60^e3_*n-C zr}xD+YRDkRJLnERrPAj^BjhjCew#pWl=p=A=>GI_Jows6`Tq6L&VI%J{=C-R=5g`0 z_t5>WxBd10IPCM2-oJ+4)0b!=19t;Y+pcl5&L8Y(qFcl6E133!ST4ctguP8q9^fEI zXIueZu#PnIbIa)CGKO>DQd{Y|+*qu>Q@#;P^jy6e7F z?m}fwx9I4e+Fv=iTWb)HgAELp9qBDtzcN-kM%#=@Hb(c_)?ws!s`(?nrRk}oUVfUM@ha%|ew*B2!pNt0bsn2@s(EGy<0UK(<9oU;g zngRPL_M>s4oM%Ewektu(6U;G8iFzGGo|5Zf({f!dqvEQ|ecY3t*eS81L1z@lUd3v3 zUjI$Wy$;_!?AR^B%Mh1;2eWN&ZT7HV%%O_BvIMIM@kj!i-Pj)Ip2&2a29Dgh zsAva8RTFxVosIIeV?%)FTD^A}(@SD(gqXJ1aKC-ZVNxVMNgiBCF=hrnDG>h}?j$dV zsD7GJ*&MrCg}(MN7j5D3b2c!xuzLv*SDqTqtMzc{9<<)xWc|%`qwONRXtFj!UkKrH zx5MI|@{pe@Lg%hyinzV$f^(`x=~4^I7PO)V0t3>0GoC+uxm-3oaCL>7*@Ty}cac#dsn@$Q}P^`GwxV zWtSR(AsXpM&WuDxOrkFUPajSKYG9^_`U7)!hvq#}XIL<+FfL~%h(6@27pJwXh~TKm z4kf$~WZh{!hC_6(SG38~pYHSLu(@Tfc`hEh8NyI>iBKu)8p~QXkTr40QPuZ{nAA_TSI z)(mIR`|FcFeu|c$iZ<1wyf_le{*t+>ev~lJNo-??aK?EMsh}1ZcQ$OH2K(;h0@s5(Jr+J>Eb&hzP+do432u;WRm|g4~C)P+=dCmE& zk8=-F)&fIe*o7D-3R9%mFAO}2FgIDy2zxME%-Vmtp7bWG-|LHMZ!5}<>Zf4yii2K_U$H{Df~S9JBd=ACzT$m=aqQyC;tDU>IoFJ+vFR8dWvwpR z%{LZ3_-|WDE*_0#FA2uXvEt5s)T_jNJaiZR*c{;^jfStSeq$js8yI`k;xUR%FBLGj1}oLVBj31Cmt{kd%PsmY#V*8l zfX0tF^WNPvxI=u`f|rJ$X!kwgEM~51I7TFv?RH(HCV3VNC|!Il)UYMj9vZ@+X#=ay zcrar%EekuPk(JcltoM#nL0r`ELvMeg*CYMe{T^FiMoY&b!Q)u+zJQl$bp}_hrmpjA z@gZ(YPU7U@h^H<89xsv=5*NN?Odax@?eg~xy-byC9^lNPS_MX3KYxcVEqYzW7k^14 z-sEb_>0^uRC9lTvkH(5Ek4Zlbplp>eH$%@6wo3cK{;!Z(R2nl;(+Vs3jjCg{j%;T8 z*s?sVz*@pngJz2*$h}8F-U+lUX!Po}N0~Q&Ne7M3yi!Q@^Vot411a2f<_n#it68l^ z&!^ApLc^i8){Zc*B|msA2Gd1&9(8@f z^#O>c${xbN^k5PFea3a?&;xVhE@k6%Hop?r*L|7^jlcFc(Em1F-zZo<`)k(S$&?K* zW@TW(bU7O`)gD)!4LZ(`7g?SF#Z3Mfxj!r~49_BH!jY9hkDLLzVHX;|m=D%bbx(26 z0V!pmK5M@J;6=S2I}~-?iDL>^H)hW*`PzZnOX~ftPMZ3B(2o;qTpxe4h((V?BUqrg03CDX z%W*~|>zEXY;Xaf=D3L&IfSV|tnGM8J5wGQfh%x3y139+x)oH?tvjglBodAh202RYM zH5=JpszMT+z8pb2!81t7cUU;MD>_>jYW^EreL7;HD$lh&;mtaG=f;xDlDK(VFnIY0$ z;|9uL3P?X~mt8=YMNex7>!eNY#`Kt|O$p4n;tPnz-R@-gQS&3q4RqOVQ4WZn&g!th z6n-bjQJ@?M!Q2fBZ{)y9Xg(PT{UC>W9USxfP*h0_D^)HXVu`p50YscxdMExc3%6h_ z2#hT@$DgC74@#FbuftqY#QRk(ezhcz({y%dm`;1^&{|Z$6?9*g_FH2qa+z3T1k!eT z-dz|R%vTanL#7-x2?}xX3(u>b%`z4=F2|!fqjSvVMdeTY0}rSucted~w=SmiNb0Tt| z^pIqU-Y=ZdWItbShUMxyDYfzU-3QC#*+?D*y@uAMp?cU2MQ$1CZ&`9^4<2m0ULMzN zpO~jz*bZkL-ZCB32n8eK-U#E`70V+613WieqEi28Pj+9^KeAB;{!D{l8e}Qa5spIr z=G;@=CrB(8R!{%-DnX9xLxw^wbdMLCH1VRnPm+8LS?+ffl{RM)B-=bc_(;}@b?2Ft z@-S8DU#nQPQ}*&$$z7Q1Im`HM1{slNIm?fyiDLApm4PvFhn8w<${Rl$Am49p3M9mt zNKs5o8Fg?=eu4MAG~$>Km5o#=P3Qb)&KXe*o}fb*pShowEqmABlw2E`>@tx{%fLXrP(|nJLquzyLY>GIVIY=2bY6ferikw4x~W|@Sfd5h2c9c_le zSOk2IhaOu<*Jfzag?+cI>>9^V4++s`;3g@;L5a95-b5Rxcn{YSg4>NZWr;|kWts;Mp9v`jrps7w7ET-zMjimx&I`Zv(gT4a25 zGVGY|TW0t&sH@&BJH*qq!-GG@UXmgvYq&Z+gD*lKS%#|$%(@fK3q2RPiJhaZ^CAgJ zlZ~SG)|%$5EapRrtY=ut91gvSoLD?3-VB37T&HR-}Gqwt^3K8Vu7d;fZ#sH;jlmuvKv%&nz!7tDNbd5qI z^e60kw=A9agX!*fpTA5DaK-Cpfxr(FtxFDbTT*k zD^7Q|`lm4UXMC>(E+bs<)g3B7Ut>+kYEipP2$ER}9rQhtFCq#?3G$`9WV*e;SLUj3=PZT{Cii zxSif7Q!hs0MLg|_>Zl+d4F9uvBtMKoCt!+{tE6aC3Y2C~rkHE|mJiwgp}Tuh$&>oC_(1oM9{azv#}xmjyVL#8 zG_{cxz`(}Y?$5Rarkk_8mi+>75IdVby%9skoKuVgd1P3yI&1NZ#w{D8qILu#CCnFd z+q$@I$+6NBsy{n^3p#J}G@s3McXf6Uoa#cZ*tjx)S3aQZV5||Nis{2iAk_h06D?}R z_73Zq4Wr3PD2sAuGny<72xG9MopwTJB@O%Fniz~wzu-~pF|THS0UC#6ivKM8B|eqyYbhnBCC ze1iNnXe0M(+29{@MgF(d9RJ>TO#d?m{kzG7lbJiMqqDWOfrI-$l#5Zyy|U{9NS#OD zMLjKwT1XXy%mZTySzxFkm5JzNf`T6+>J513nAJusxj2X!gED zPTR025=^MKAiu8{6TaW@4wHsaxz1D1fKGdvxo>`mW`PGB#Oaea7+0?dRzOPWuuiO= z3<8FKYJ$g5eXmk?K3z>3`7)^ovrqo;>>=aU8#Z}Ybt3W9VMTN7?;2{x6xxWQU?Nj) zH3N36p@8N*uEm9eC=4^oFQ&MK))8xJG4hln7|RVKrXh_3T#eQj$0P(HO=d#UPSh4} zjE?r*k8%UUcjLpq?FUAW$1dYjc-HRG(>57Xc-rk1D8O0*E_?B!`I_kICXeoZ@xgf)yOk~Uq>PtU`g2;tGAJZJ4TZm?UTUUXWki-=%QgT+jH^(B@ zCPF3VtgS}Y2zBS}GjZkH{iazP^eJFSaS^0cF$==Ro397I#8Q?E{qi71jMvuSGRkrn zx2o`HdcTSJsRkw^?8rnRQUcNaXDwOQ8X-sq(>^#Q4|IN&wg?TUH7(KZKEB{+5|lI7 z%m98vp!%^(NmDW!lCUhD`1($sXcW)F5b3%ze&_obx#ygF?pcff zzG1P}z*>9Hvu5_Y-*QhG?owxi~3{LV7=f+If3;XlpNwwWRs8;JXJTE`a0wiKOv#8A))X!k3x@3 z)tqo;!8n&xCA?YwrrwF(ECY#1>8)_!pn;RCZ~bolKx2@*gOj2Bv!XkeB`NA0MC8PZ zlv%7CwJCS3j~#K0jf$~oH?pgPsaO!YwuQzTyr`PqE+8xGXcZk#yYoPsE; zgY5hvq^gIus~c1)F;qz0U$8ocC$Ott_Lw{Szk%+Td^fJ=ShiXYxM%6?5J`}3klwVm zi)?Ep-!z?B&yS0)?Qtq&yPKH-T)zxLmnpNSmg@+{i_*gdg^_C_~``SWe(IMwi) zcj@|*v}!xAG8ATMm5ge>g0A%o`Bc27D#u=HCZWI+Ba{I=Gzk49urgxqh?8V_Z98D# z;|4VEvgB*z^jjhh7#X9g!e!cIRGY_ndM zERuCM_bVqD8j)ds7k@kKEUbREIn~Mujk>dMl_n_sarg`kKEA4F;w#2r<12^=akjlI znlQea@da-Sg-^ApzlJ*OHIQH=zaAG8|K)k=iq?34Du@u4c)v|q08IyKX;{nZ8A*c3 z`?)#E5j%Ss1}}S>Oxc@^X^m)J`ew18?FEY_i&?)lMbGx^70mtQ!`Sv8rv5Izk8~e1 zg^DXyNT_>&4oS?1XU_e#Rm@_K`aBu)ICyNo(D3zYTgEI2Cw7l$7Y3`-s=kP1D{#so zZ;W>FngPTD#!P&VR!7BHbH;eE%(F3hf(n z>_LOubmjA|n7zq-Q@JK;Zb=_^Tz>2yXvy~~tsCAEiwu!UmDEVX{(R$h>c)sWqhGCO zAewg1N-wCCKs#Iitu0C(tudZIm8H=1g)S2xA-47sF_d39Hk=aq+7%*q=$Vys+nI~RjOA7XYJ?iKCm42=9fmz663g84R_(9 z&7Q4RrWK+(He#-Q=L3SnIGN9idV7a4u>iunB@u;WO} zA5EW?mD8rY&c3NSQrRxsiASzD<~#6^w_UN1J*v%)Z3fRU4n3ClmLTX3cO^P`op4^@ zW|5i8>r-=j#-roP%@dSw;^P@39(BE6RX0iDVM-k8rnj#K|A*FjFB)wk2&d*I=(!Ma>1Hy8*Ts>!h5i@|zX^#yC4`PJ~|ZsQ8m zd2u^gR~V^#P&W`Z@7z~QA@D~bW}Ez!>eByE$lHdwso1xQ9e0q%@`-dOo)nSk$+j`x z=jZoWGTVF!yq^pMG4hAlqc<@UJuu;ob{k+EQ|Rf256$>C!k3K7V8MWCJqY z)Fd>UcfauX5PJOJt7p~yLjT~M9q9D=si*((eC^41`qqGhZLs8NO~8St$NqQ72ZFlu zoM5-(?@LRkeXXCO&w;H?z4T_m=li3pgWLAA1x=lD*p1h1)CxpBTT$$aEPYm-rO`cS zExcADuFv9}SK;RmqKvWz%!R#*A0lhR&g(voEXnQ^}8l%e7 z4%BPJOe1_Wn7oIEy&1X( z#oc$b_RNoTBShzOLk6Wg^5+Wuz+>SmVT*~9!ijXYLYKb1(ELtM>!((S4z4nwWp&i} zxi{CYZT1{J#G8kva1%d=?Or}{D8qBVb3ItNTsl3Zwm3r2vx`=^08{olzJTZf$#bq- znR{V6Yg6m@^^a(VsEcV#0hVirB1{=fa&`qYrHLK&{u-5Ga~@%XjmQ}bWS{^;%b;ko zQ4O?)(De!RrJc!fl13$+(M_YQH^iwtz+xR9y!WAp$e!MLSYm_hx&n#5oO<~9EvMQ& zL@YS$00%a9P!Uid|9Dvgd-5oWF)YXcSO8=NPFn06Krvk=(6(OX3wHs2TcbzPg2;11F{9D4SGyJ~2J2a`q zVyb5y$D&J4)L%D$@Tn>-+|6l2NMw5`U^(8+=ALR=G3hheVQfm~3<&|Kh2_U*be%Og67BpbtbK;sFq-{$?8f6|!W=^r@pv3r5GtiZv*_?y0 zr1-tmgKgZgTTQb2_{C6}L<$>BsTUm%MWWGfclaXkO)+4RZ%~d+*2e6bWb?7=c#`K# zQGeD%d^cbSNu60sm7Y%b=F&I7qp)>YvftD(5c0gRBk@$UK^TMjI^z>1_OVB*5R%1L z%oSQsP=>!PJhrtf@1V0LQRrMr934DbSPpep9QahSe$26-{dDi{H9Bch_eL64yfNdm z=^KfT{0aF9l-!4*#pu$uPCTQP1-qjqpS_H6-`Z+4Hdue|ZO7qyHZ{vTt zy-1kGwtA!Abd_TddBtq6M<2Kr7`k*2RkYE(yU0Isgf8}(YWQ=~Ot#<;(XGBw^1%U$ zXYcc*Jh~J2?4iCrj9%Kxg9v)eqz5W( zbauW+_Z}BI_xR7XsAPG6)5f$1zNzi{Ib2;KK88cX{B`-^Ph%(-rj?#m19j(JwCk_i zzx61q<$UQ*%+sTSMD;>f+g;>ZPSSO25i)aW>2T9619 zJvj4>PP$LfOF$|7E|#QSV4UC99ono#K_UQXP^Qbsj{qYOyH@Qc%J6BtN@B zJ$WS2zr_pwJcu{XKKU8a?cT6eq{r{0tc=4z{iAklX;F09#+0wqMmciFQt`2}iOwgI zT1@8ifJ1Np=JWYJu**4FPt`aq$B|YQL~?)g0rZKgx6~J>LR*lxwd@4cG|_%D3OiSa zCA+Ngmh>*qDs6Y&SZ~}zX|b0`s!A&y>Bq5;spN3-suu$?4jps3hM(6L7otDrad-Ci zT7bT2?2njn&>Wqz_DjW!8la#Ga=VYY07za_8EmSUR|uF=*GrqO<|fjzCcfs7)`piH z<)|b%Le@}fA-X{1R9YtB#}KO$5}I5e0RlWR@-PJ84GCD~3%DititY~C6Glm|H$^~0 zOkKSC)J2;^Td_L_6FT)szMk*Zqwsuns@ezVaJRKW?w-vHvT>KCFi=|1yr$i@->lI3_Paj9Tw9Zr29hGq;J*h%>;-5UJqjX`>v2~&4 zBI|meC1ivVB>dom^a>!Td27wvDp6gQ#5(V5D0E46=}b(C!L*A%j715udpE!QV?qrB zcA<5j!ShPUjib;g#yihogz`Y6sIvn~wobdXTmpxH^)S z+k9~ut@&6ug?f!@sj)UdH9O#mDVNyEDp5%Yl~Ntafb2?iui@N4#~l5cA4D{f6x%CXS_4nd+|Q3z zhZPH{?MwaM$W9Y@En8Md^BPj!jNh%@^E-2KKBFQ``}WG}EvmS8$a4+<@a8-ThE9hJ z5}6I@@6jbe*)!C)QFGG{ZN+$YdXgSu*vEqD1}!T4C}h`<<4mN;y@X#K-d zUafmkjwtUqQLo(y_Z0p38t-ckI+$RPUe6luwf@XZAFge<`w0}Yfj)=+-Mp@JiTFsU z#j7CdbAk;6O%36=WEn}jN4OZfKhxx>%WUxn+7b_OJ(TQoTR5Xw7bssVUE@f4m4Cw@ zy?;IA>C4S-EJgWAfxda=rlcXG^o&H5k(~(}uN2Nq2~wPVAfqVE}j7xvl(GiW(hdfEv!|+NvG&8XX@o zx#saEKN4TkF)d^WVzBi&?7h#_lOEwB+<}|^j&3~}V2{H2bTn~XmIpFliKhA@w-46{ zjV0C{IP2%D#{#E|X$xOR3ft{#((RRob~;H1J9SuVyM7V!`|vSSd$3G7wF1`XRHoU9 ztG`A;=9b+pB`axOkk*{p6w6#Nu zw>LUL#qLjyD9p5NXyhp;YCt%HQDmd2@wh@kz-`oN+RDC4y`D-8@tc?=$+7d{E*=40 zb}Se`uXc9c+eE%A9dtrEvw`^@LHEFQ(`&#c_l6Vo;F%kgU)^Sowg7 z=Vd<$T&8gH<=;G8ux*i`J&mV5?bbXie=MQ5gR#JK{j_~}pn-~t^Jo^RPIl^lz(aWY zY$aB%6>{KL2qq3HTk#RIl5Yi%6v?-u2GI`DzdndrX%)$cA6k%HXe8cDwlIi>#Pf~g zbI}?~i+rip$2D0t1y4bqAU*o9Gi&DpN(7PKhIrI{jutXYn_Dkw-T=a&LKw%N1jhN5tek z$2OpF0}0HiOEJG{4{qA|SbUv7@hR1zLCm3fqX)LjhB#9bvXPqk``fsQIvN>_II>)W zjZp)4spu`=u0YBhsy-6rJ$99p6VB#;B`}~G!v5HtzzHb9S|>@GAv4CDwGr&yy^_36 zrEU8(J^5sL#+Pi$*H6(nhj-d8m4TdTQ`>^rl8TfGgG24KB(r!u(@PFoWme+4t5o&9 z6pd>qb4j-z-IlFaB=FoebDJ>huH*4EZIQ2zCLWvOPwn|6yN9&x1b|48i5}Zir&nSi zNgQCE2wT#8i=R@=z?VYZ1hIHeCnRV~29302*UaK5aUCX8^AgVl5;VKAG*jnwom5pj z7@wA#o15wuK0&RyZc_if*qZke)7x2Cf&${hOF}(MO_Z6*p90xg)Y%LKq>cr0ZbiRz zunpE0%>CG*@fe$1irP>ka}(MBlYvTSP35F`=ubM}p~4OzR5jA|?e*n2D}ce0(@%Bt zPc-l9dph0g`XJW^mBR*^sjK>ybxI?ESrn)mZ4-oVasVD~!Q2}I-KP!plH~i#3uh)X zd9la;`=64d3ujxua-<(^HJ;_cX7={G|88c7#rgi2*_|w$?H<9^BK2C?x#-y7MBtO1#U~#>-rYRyDW|@#1YOA-_tle=A2sgoNjt z5vvX1g0~CE$JAE|`R{aZml>N8%H-fV@e4+D=__RXmsOC6eDGpD#HurRvEBvWw#=3A zA(ZYRvcNx2#3CYio!bSAjNFy5{IjYzL@0Ru5@O{Ayng8d>RjPUp#G|6LL~XK(&Nv< z3wWi+1<3=IDMSFZ@<9~)2oi>ePqK6ttxG3ovLR1~g$HE+~E z_ Date: Thu, 22 Oct 2026 18:52:19 +0800 Subject: [PATCH 295/343] fix(validation): preserve reviewed I05 H2 startup failure --- docs/validation-heal-operations.md | 21 ++++ scripts/validation_heal/fleet_node.py | 28 +++++ scripts/validation_heal/timeout_retention.py | 103 ++++++++++++++++++ test_validation_heal_environment.py | 26 +++++ test_validation_heal_timeout_retention.py | 62 +++++++++++ .../i05-h2-filter-startup-39764496504.zip | Bin 0 -> 254781 bytes 6 files changed, 240 insertions(+) create mode 100644 testdata/validation/i05-h2-filter-startup-39764496504.zip diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index a80ef91f..dddb9d33 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -776,6 +776,27 @@ A separate protected prefix is checked during the fault. The original key-level ### I05 retained-scene isolation +The reviewed H2 startup failure `39764496504/1` has a separate preservation +plan pinned to helper `31b787c51de23587da1f9a2ac0f910400ddf66b3` and the +SHA-256 inventory of all 102 official artifact files. H1 completed its timeout +diagnostic and actual root cleanup; H2 reserved all four peers, then failed +before fixture seeding because vm002's bpftool wrapper could not find its +kernel-specific executable. All four stopped original scenes contain no +business files and retain the same complete sixteen-disk format mapping. +The planner requires those exact original owners, candidate, inspector, +cleanup receipts and failure boundary before allowing the existing +audit/isolate/post-audit protocol. It issues no workload request and changes +neither the original FAIL nor canonical acceptance. Isolation remains pending +until its independent post-audit completes. + +`environment.inspect` now also records the selected bpftool path, installed +`/usr/lib/linux-tools/*/bpftool` executables, resolved root-owned executable +digests, kernel release, version exit status and a read-only root-cgroup JSON +query. A working installed executable is distinct from a broken PATH wrapper. +This inventory does not install tools, attach programs, change PATH or certify +the I05 service filter; the actual owned-service attachment checks remain +mandatory before another diagnostic. + `validation_cleanup_run_id=` selects the narrow I05 preservation route with the original package build and an immutable helper containing `timeout_retention.py`. The driver must remain on the original Functional Chain diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 2131fc65..6ff1d4f0 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -378,6 +378,33 @@ def listeners(): 'retainedSceneRecoveryCertified': False} +def bpf_tool_inventory(): + """Read the wrapper and installed tools without installing or attaching BPF.""" + selected = shutil.which('bpftool') + candidates = sorted(Path('/usr/lib/linux-tools').glob('*/bpftool')) + require(len(candidates) <= 64, 'BPF tool inventory exceeds its bounded installed paths') + tools = [] + for path in ([Path(selected)] if selected else []) + candidates: + require(path.is_absolute(), 'BPF tool inventory requires absolute installed paths') + resolved = path.resolve(strict=True) + require(resolved.is_file() and resolved.stat().st_uid == 0 + and not resolved.stat().st_mode & 0o022, 'BPF tool executable is not root-owned and protected') + version = command(str(path), '--version', check=False) + query = command(str(path), '-j', 'cgroup', 'show', '/sys/fs/cgroup', check=False) + try: + rows = json.loads(query.stdout) if query.returncode == 0 else None + except ValueError: + rows = None + tools.append({'path': str(path), 'resolvedPath': str(resolved), 'sha256': digest(resolved), + 'selectedByPath': str(path) == selected, + 'version': {'returncode': version.returncode, 'stdout': version.stdout[:4096], 'stderr': version.stderr[:4096]}, + 'rootCgroupQuery': {'returncode': query.returncode, 'jsonListObserved': isinstance(rows, list), + 'stderr': query.stderr[:4096]}, + 'serviceAttachmentsVerified': False}) + return {'kernelRelease': os.uname().release, 'selectedPath': selected, 'tools': tools, + 'inspectionOnly': True, 'mutated': False, 'canonicalPassClaimed': False} + + def environment_capabilities(request): """Read existing mounts and kernel interfaces without reserving a run.""" paths(request['runId']) @@ -414,6 +441,7 @@ def environment_capabilities(request): 'kernel': {'pidfd': hasattr(os, 'pidfd_open'), 'cgroupV2': Path('/sys/fs/cgroup/cgroup.controllers').is_file(), 'fuseDevice': Path('/dev/fuse').exists() and stat.S_ISCHR(Path('/dev/fuse').stat().st_mode)}, 'commands': {name: shutil.which(name) is not None for name in ('docker', 'fusermount3', 'nft', 'az', 'bpftool')}, + 'bpfToolInventory': bpf_tool_inventory(), 'installedBinarySha256': digest(binary) if binary.is_file() and not binary.is_symlink() else None, 'platformRestartAuthorityVerified': False, 'compatibleReleasePairVerified': False, 'canonicalPassClaimed': False} diff --git a/scripts/validation_heal/timeout_retention.py b/scripts/validation_heal/timeout_retention.py index b84b33ad..36f7815a 100644 --- a/scripts/validation_heal/timeout_retention.py +++ b/scripts/validation_heal/timeout_retention.py @@ -34,6 +34,8 @@ DESCRIPTOR_HELPER = 'dd7e54e6672cab76b5a6245ab3b4fbd051ddd39d' FILTER_STARTUP_HELPER = 'a9725c88acd4977545b278eebe3653c916f907f8' FILTER_STARTUP_ARCHIVE_SHA256 = 'c6760d0b4e657c306771a466e0a5aa4233f8e05db4aeae018ba037d289fca4aa' +H2_FILTER_STARTUP_HELPER = '31b787c51de23587da1f9a2ac0f910400ddf66b3' +H2_FILTER_STARTUP_ARCHIVE_SHA256 = '8a5399be0a35295703b079ed10cb55685ee4e347e9f9e276b29140afe6764e49' DRAIN_HELPERS = {DRAIN_BUDGET_HELPER, DESCRIPTOR_HELPER} FAILURES = { PRE_CONTENDER_HELPER: {'type': 'ValueError', @@ -216,6 +218,105 @@ def read(path): return json.loads(path.read_text()) return [plan] +def h2_filter_startup_plan(result_path, failed_run, candidate): + """Preserve only the reviewed four-peer startup failure after clean H1.""" + require(failed_run == 39764496504, 'I05 H2 startup isolation requires its reviewed failed run') + archive = result_path.parents[3] + files = {} + for path in archive.rglob('*'): + require(not path.is_symlink() and (path.is_dir() or path.is_file()), + 'I05 H2 startup archive contains a link or special file') + if path.is_file(): + require(path.stat().st_size <= 2 * 1024**2 and len(files) < 102, + 'I05 H2 startup archive exceeds its reviewed bound') + files[path.relative_to(archive).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + require(len(files) == 102 and digest(files) == H2_FILTER_STARTUP_ARCHIVE_SHA256, + 'I05 H2 startup archive differs from its byte-exact original') + def read(path): return json.loads(path.read_text()) + result = read(result_path) + provenance = result['provenance'] + evidence = result_path.parents[1] / 'evidence' / CASE + summary = read(evidence / 'healthy-summary.json') + h1 = {'topology': 'H1', 'diagnosticComplete': True, 'cleanup': 'PASS', + 'wholeCasePassClaimed': False, 'canonicalCasePassClaimed': False} + require(summary.get('caseId') == CASE and summary.get('cleanup') == 'FAIL' + and summary.get('requestedTopologies') == ['H1', 'H2'] + and summary.get('complete') is False and summary.get('diagnosticComplete') is False + and summary.get('canonicalCasePassClaimed') is False + and len(summary.get('diagnostics', [])) == 2 and summary['diagnostics'][0] == h1 + and read(evidence / 'H1/00042-diagnostic-result.json') == h1 + and read(evidence / 'H1/00041-recovery.json') == {'cleanup': 'PASS', 'errors': []} + and read(evidence / 'H1/00040-cleanup.json') == { + 'node': 'vm000', 'cleanup': 'PASS', 'unitStopped': True, 'rootsRemoved': True, + 'hostname': 'rustfs000000'}, + 'I05 H2 startup isolation requires completed H1 diagnostics and actual root cleanup') + folder = evidence / 'H2' + labels = ('binding', 'inspector-build', 'preflight', 'integrity-fleet-attestation', + *('reserved', 'inspector-upload') * 4, 'quiescent', 'original-failure-scene', + *('original-failure-logs' for _ in range(4)), 'diagnostic-result') + require({path.name for path in folder.iterdir()} == { + f'{index:05d}-{label}.json' for index, label in enumerate(labels, 1)}, + 'I05 H2 startup isolation cannot include workload or unreviewed actions') + records = [(label, read(folder / f'{index:05d}-{label}.json')) + for index, label in enumerate(labels, 1)] + def many(label): return [value for name, value in records if name == label] + def one(label): + values = many(label) + require(len(values) == 1, 'I05 H2 startup lacks one exact ' + label) + return values[0] + binding, scene, diagnostic = (one(label) for label in ('binding', 'original-failure-scene', 'diagnostic-result')) + run_id = 'gh-39764496504-1-h2' + nodes = ['vm000', 'vm001', 'vm002', 'vm003'] + owners = {row['node']: row['owner'] for row in many('reserved')} + require(result.get('case_id') == CASE and result.get('verdict') == result.get('cleanup_status') == 'FAIL' + and candidate == read(archive / 'candidate.json') + and provenance['source_commit'] == binding['candidateCommit'] == candidate['source_sha'] == CANDIDATE + and provenance['candidate_sha256'] == binding['candidatePackageSha256'] == candidate['package_sha256'] + and provenance['testing_commit'] == binding['harnessCommit'] == H2_FILTER_STARTUP_HELPER + and provenance['binary_sha256'] == binding['candidateBinarySha256'] + and binding['runId'] == run_id + and binding['bucketPrefix'] == 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] + and binding['endpoints'] == {f'node{i}': f'http://rustfs-node{i}:9000' for i in range(1, 5)} + and summary['diagnostics'][1] == diagnostic + and diagnostic.get('topology') == 'H2' and diagnostic.get('diagnosticComplete') is False + and diagnostic.get('cleanup') == 'FAIL' and diagnostic.get('namespacesRetained') is True + and diagnostic.get('wholeCasePassClaimed') is False and diagnostic.get('canonicalCasePassClaimed') is False + and diagnostic.get('failure', {}).get('type') == 'ValueError' + and 'bpftool not found for kernel 6.17.0-1022' in diagnostic.get('failure', {}).get('message', '') + and set(owners) == set(scene) == set(one('quiescent')) == set(nodes) + and all(row.get('reserved') is True for row in many('reserved')) + and one('integrity-fleet-attestation').get('nodes') == nodes + and one('integrity-fleet-attestation').get('allParticipatingNodesUseCandidate') is True + and one('inspector-build').get('built') is True + and one('inspector-build').get('candidateCommit') == CANDIDATE, + 'I05 H2 startup isolation lost the original failure, candidate or all four peers') + uploads = {row['node']: row for row in many('inspector-upload')} + logs = {row['node']: row for row in many('original-failure-logs')} + require(set(uploads) == set(logs) == set(one('preflight')) == set(nodes), + 'I05 H2 startup omits an original upload, preflight or service log') + for node in nodes: + owner, original = owners[node], scene[node] + require(original.get('quiescent') is True and original.get('pid') == 0 and original.get('files') == [] + and original.get('unit') == logs[node].get('unit') == owner['unit'] + and original.get('hostname') == one('preflight')[node].get('hostname') == owner['hostname'] + and one('preflight')[node].get('mounts') == owner['mounts'] + and original.get('binarySha256') == owner['binarySha256'] == binding['candidateBinarySha256'] + and uploads[node].get('installed') is True + and uploads[node].get('inspectorSha256') == owner.get('inspectorSha256') + == one('inspector-build').get('binarySha256') + and one('quiescent')[node].get('quiescent') is True + and one('quiescent')[node].get('service', {}).get('MainPID') == '0', + 'I05 H2 startup scene is not the stopped, empty original on ' + node) + validate_topology(scene, 'H2') + plan = {'kind': 'unresolved-i05-timeout', 'runId': run_id, 'nodes': nodes, + 'bucket': binding['bucketPrefix'], 'owners': owners, 'candidateCommit': CANDIDATE, + 'harnessCommit': H2_FILTER_STARTUP_HELPER, 'binarySha256': binding['candidateBinarySha256'], + 'ackFilesSha256': digest(files), 'businessSceneSha256': { + node: digest(value) for node, value in physical_content(scene).items()}} + validate(plan) + return [plan] + + def archived_plan(artifact, failed_run, candidate): """Refuse uncertain effects, later actions, mixed cases and physical drift. @@ -233,6 +334,8 @@ def read(path): provenance = result.get('provenance') or {} if provenance.get('testing_commit') == FILTER_STARTUP_HELPER: return filter_startup_plan(paths[0], failed_run, candidate) + if provenance.get('testing_commit') == H2_FILTER_STARTUP_HELPER: + return h2_filter_startup_plan(paths[0], failed_run, candidate) require(result.get('case_id') == CASE and result.get('verdict') == result.get('cleanup_status') == 'FAIL' and provenance.get('source_commit') == candidate.get('source_sha') == CANDIDATE and provenance.get('candidate_sha256') == candidate.get('package_sha256') diff --git a/test_validation_heal_environment.py b/test_validation_heal_environment.py index 15f517e3..006b8b63 100644 --- a/test_validation_heal_environment.py +++ b/test_validation_heal_environment.py @@ -145,6 +145,32 @@ def test_legacy_mount_only_inventory_does_not_imply_runtime_or_clean_proof(self) class I05PlatformFilterTests(unittest.TestCase): + def test_tool_inventory_keeps_broken_kernel_wrapper_separate_from_working_binary(self): + wrapper = Path('/usr/sbin/bpftool') + binary = Path('/usr/lib/linux-tools/6.17.0-1020-azure/bpftool') + calls = [] + def invoke(*args, **kwargs): + calls.append(args) + failed = args[0] == str(wrapper) + return types.SimpleNamespace(returncode=1 if failed else 0, + stdout='' if failed else ('bpftool v7.6.0' if args[1] == '--version' else '[]'), + stderr='bpftool not found for kernel 6.17.0-1022' if failed else '') + with patch.object(fleet_node.shutil, 'which', return_value=str(wrapper)), \ + patch.object(Path, 'glob', return_value=[binary]), \ + patch.object(Path, 'resolve', autospec=True, side_effect=lambda path, **kw: path), \ + patch.object(Path, 'is_file', return_value=True), \ + patch.object(Path, 'stat', return_value=types.SimpleNamespace(st_uid=0, st_mode=0o755)), \ + patch.object(fleet_node, 'digest', return_value='a' * 64), \ + patch.object(fleet_node, 'command', side_effect=invoke): + result = fleet_node.bpf_tool_inventory() + self.assertEqual(result['selectedPath'], str(wrapper)) + self.assertEqual([row['version']['returncode'] for row in result['tools']], [1, 0]) + self.assertEqual([row['rootCgroupQuery']['jsonListObserved'] for row in result['tools']], [False, True]) + self.assertTrue(all(row['serviceAttachmentsVerified'] is False for row in result['tools'])) + self.assertFalse(result['mutated']) + self.assertEqual(calls, [(str(path), *args) for path in (wrapper, binary) for args in ( + ('--version',), ('-j', 'cgroup', 'show', '/sys/fs/cgroup'))]) + def owner(self): return {'runtimeProfile': 'lock-timeout', 'runId': 'gh-123-1-h1', 'unit': 'rustfs-validation-heal-gh-123-1-h1.service'} diff --git a/test_validation_heal_timeout_retention.py b/test_validation_heal_timeout_retention.py index 9d764212..ad1dcf51 100644 --- a/test_validation_heal_timeout_retention.py +++ b/test_validation_heal_timeout_retention.py @@ -22,9 +22,71 @@ DRAIN_FIXTURE = Path(__file__).parent / 'testdata/validation/i05-drain-budget-39746002648.zip' DESCRIPTOR_FIXTURE = Path(__file__).parent / 'testdata/validation/i05-descriptor-drain-39754893467.zip' FILTER_STARTUP_FIXTURE = Path(__file__).parent / 'testdata/validation/i05-filter-startup-39761856012.zip' +H2_FILTER_STARTUP_FIXTURE = Path(__file__).parent / 'testdata/validation/i05-h2-filter-startup-39764496504.zip' class TimeoutRetentionTests(unittest.TestCase): + def h2_startup_archive(self, directory): + root = Path(directory) / 'artifact' + archive_root = root / 'rustfs-validation-39764496504-1' + with zipfile.ZipFile(H2_FILTER_STARTUP_FIXTURE) as archive: + archive.extractall(archive_root) + return root, json.loads((archive_root / 'candidate.json').read_text()), archive_root + + def test_exact_h2_startup_failure_retains_all_peers_after_actual_h1_cleanup(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, archive = self.h2_startup_archive(directory) + before = {p: p.read_bytes() for p in archive.rglob('*') if p.is_file()} + with patch.object(recovery, 'Fleet') as fleet: + plan, = recovery.recovery_plan(root, 39764496504, candidate) + fleet.assert_not_called() + self.assertEqual(plan['runId'], 'gh-39764496504-1-h2') + self.assertEqual(plan['nodes'], ['vm000', 'vm001', 'vm002', 'vm003']) + self.assertEqual(plan['harnessCommit'], controller.H2_FILTER_STARTUP_HELPER) + self.assertEqual(plan['ackFilesSha256'], controller.H2_FILTER_STARTUP_ARCHIVE_SHA256) + race_retention.validate(plan) + self.assertEqual(before, {p: p.read_bytes() for p in archive.rglob('*') if p.is_file()}) + + def test_h2_startup_refuses_unreviewed_run_effects_cleanup_peers_and_archive_drift(self): + variants = ('run', 'candidate', 'helper', 'h1-cleanup', 'h1-result', 'scene', + 'missing-peer', 'workload', 'extra-case', 'format', 'symlink') + for variant in variants: + with self.subTest(variant=variant), tempfile.TemporaryDirectory() as directory: + root, candidate, archive = self.h2_startup_archive(directory) + evidence = archive / 'run' / controller.CASE / 'evidence' / controller.CASE + failed_run = 39764496504 + if variant == 'run': failed_run += 1 + elif variant == 'candidate': candidate['package_sha256'] = '0' * 64 + elif variant == 'helper': + path = archive / 'run' / controller.CASE / 'cases' / (controller.CASE + '.json') + value = json.loads(path.read_text()) + value['provenance']['testing_commit'] = controller.FILTER_STARTUP_HELPER + path.write_text(json.dumps(value)) + elif variant in ('h1-cleanup', 'h1-result'): + name = '00040-cleanup.json' if variant == 'h1-cleanup' else '00042-diagnostic-result.json' + (evidence / 'H1' / name).write_text('{}') + elif variant == 'missing-peer': (evidence / 'H2/00009-reserved.json').unlink() + elif variant == 'workload': + (evidence / 'H2/fixture').mkdir() + (evidence / 'H2/fixture/000001-request.json').write_text('{}') + elif variant == 'extra-case': + (archive / 'run' / controller.CASE / 'cases/other.json').write_text('{}') + else: + path = evidence / 'H2/00014-original-failure-scene.json' + if variant == 'symlink': + copy = Path(directory) / 'scene.json' + copy.write_bytes(path.read_bytes()) + path.unlink() + path.symlink_to(copy) + else: + value = json.loads(path.read_text()) + if variant == 'scene': value['vm002']['files'].append({'path': 'unknown'}) + else: value['vm002']['formats'][0]['sha256'] = '0' * 64 + path.write_text(json.dumps(value)) + with patch.object(recovery, 'Fleet') as fleet, self.assertRaises(ValueError): + recovery.recovery_plan(root, failed_run, candidate) + fleet.assert_not_called() + def startup_archive(self, directory): root = Path(directory) / 'artifact' archive_root = root / 'rustfs-validation-39761856012-1' diff --git a/testdata/validation/i05-h2-filter-startup-39764496504.zip b/testdata/validation/i05-h2-filter-startup-39764496504.zip new file mode 100644 index 0000000000000000000000000000000000000000..a01de72c6db8f8d6be0dd87f356d6952dc773744 GIT binary patch literal 254781 zcmeFZQ@#Ty|n1q^}$00001@RX#g!(yEaZVd`mQ@ta)oW~~BVy|lY%Ckr}0;Ua0k;&7r|SL2h{0~l|eNqi_?o$A2AKD!qH7ikf);jwKg$!(H344r#a4=YvY znbPrs5VlB^ixH!eXz?qWL9iHsF2S^#cB3K|MB=v$f)-@N289N3 z2kj>`M@H(!k9u5fU4fS|NgG~+jOk8qwNeus*I=D%xIbRfS;CNI)Uhk#O!R;ssEX1z z$PZ%n=L4Un>Gj|y#{iXNxk8WPRVgSEZLTKg57g4GCtj0>k3J?g`Qi?>4*%SEMaIG6 zNd_@h?J&LGpv(WIdSE1vrh2}sh3C79;QnXz{HI=iPRS0?BLrV_iySdU`Yk_tzA-L& z+AM_6D%TBw-drJ+x&u}-rKic=JIl!O%fM zlTc2R99J)Mr=DKXj$}G41oUJ+d%0g}x0_J#z&+TW(M&N}f8^E7G4@&LjA^TjuJ#l8 z5O?{Qp3rumJipv%cYy_&BLlL^Z+`9Zk83Mi^saq$**QG%2Sx=pg;^#`Vagi=npSlr zl_ws+pa|I|Bz}0g2mNbNoede?8?7eLI8oRKy?h+h0#BQB2oKWyAH=V=PCEowmAL$NF4FbHS%;qHSi&PFk6VQ>>{5`d8xCxeiJEO6Hn+~2jfG^MW|(I@Z$nG( zOyXi^t_b7flr_^=q>=fuOUr161Rgxks#P9@{IiS5u1V8JwcGFBPi+um2FoPu|3?$p z0{2Fi-GG`6j5$fWG~>^PXrjWS!H1EiWdRAQNN;003S7Y|Bazk;(rmd+A&NFDTR=2L z*q(DqZ`oW-b3CUpXeqAz4iK=Ew^RhdAV*E&9$Scc9P;6vVAToA`F>OEBkkK&x`!2p zk|f4u5_K}fO>iWmY}B?DN+K|!B6I~dQD;lz9v)Uj2%9ctGgW%c`S>0Q`zG4eo9ZcA z*_W@hOFy?9s8hSk3*DiKA^uUf&Yak9XyTljH|8gwwUU7uUGtEz(jglv&8cH?IrqSOUgP+97^gG-zuWdbRl6ip1Icu71CrO-fKKvd|%+d3K#+^ zTW;&SokD+Gsrmn1z$UI1#wNB#CjSQw(}^?C(SLs!XbdgBZQ9oC-=a48^>0y&JiSEr zZm3TE;V5*IK_t*1;g7ruJCrUQOoruw!4~-JO4P{}fvR%GKT+I&TY7k~oTRb)jSg?r za;|o6_28lq*5*h4VDm7~N}Auo>IvSgAXA}wGI-v(wR1iEyKX1b(jLo{_uP9j!{Y+Z zd#2IjcyHF^b7hA<#@lVv4LEM6#6&RYF(y#-JV&yMu#(VsVcesa4#3KPLO1;mTTTvG z%1v8yAkX4luz+LZwQygOwDBhXxckCQhcAZZRX2q4p`+i@qIscFZ)X$cj(u!3^T;K* zUC9am#ML0rgK6s{fVxQP?ZWg-O3i#9zvHr*tK*bDPl@$3kw|b{lqwcW`yLU(2WDF1 zpdWO0e)f+2vkE(jUf;F6wwi*vkT|{=1_@Q&UvcownD>%T!M6b2*H(h7#~xz{%LoOE zwnK(OI z28c5VF8_Qbh=eK}q|P5j%lZSA0+G_JvS&B!J*|R-tg5R8a;QTS2tmFXkE;QqUL_O& zXFVFQQe3qUA|zThp1p={lLi<675HhvmYt_qZTe(_?UnEDk@?1u;{aGt`StAJjP0)y z^L#{^f&&ErkbnXJu>V^n#ztt_YktYCJEcXs|(7Y)K zB$eBBGdmRfG#t{Y$~jhW)TY{dvtgMyFi+<+;L!<6wK82w$HUN7n!|vaNy*QNFiOA}uQ;!GHkL&VLT)loi*{mJ#Rf<)} zusGip4|}AYJljGnc_WYBAAD}g$KA87hN|z)xMQ!S&UEikzs|zrR#}qbU$cqQiH3?S zb^6V9*3gYDtTn!PfN#j8*sH3pZi!fjRnKf0tn?E$+brNU9y~@f840M9{($T?t=&|p zJ`I(A)HIw`d^HcMr=`5Adg5SW=5QGBncOvN$!>B+-69lRVWW{1#(i8VyV%C7u7fv{ zbuHAer*j7jV08pUazVPs2JpYnOOxe1$Ejr10NFMIq)>r(2_wS4*2tx2K)^ z^X2~aaNoVImDkV1;q&(X^<}6cLC^F__8JdVdpgaq;_0JX zdu<4*U`ppL*-LTI<0eJ&N;iT;f%EtlTI^&FR9(i%Wqs>n80P0v?u1R@+-l5*uxsi% z(#vOM1ZmHdGHt;^)iYxhqcN-_WJx$qbLe?wWVb`*+`zC4?%xTV4uVfU-c!X~)}(D0 z=0y{XK~RN$KagaQNide2i?69=;>~}S!xPJ(8&D%lM@+Weha-B!U1T&a5_bfAl4+zo zIDWMmxw1-P<4#Z%{j5Kb1;0kUbHt%hR8*@2OiW5N8z>By^@71^h>3oW;Ooq4oKnp{ zNX>&(@S$!#DDqKUl+nAVqNXj=$)Sma+sHr~UN6}Ste>|tGsGD$?^$3pJwP)#Zbp{C z<9^}KTBb)GgFVQcN-ji9iW-eUhU+xt>a2&m9NAD$JK)!dmvv8Lk@3TNGE&9@m4UHf z0T$xdEDV;`Uz8pco&&JNI&@`gn`qzIO&x!egUh0Jk(gjYk0imXidQgdEQAWjNmS^= zK2N>n6$TL6kLHGq>gPgBS`U1ZQR<^id`Og#L`Qk=eFN6zUc5iESLUcEfK2BwbE3u9 zS=u0oiyV6{52aAMQ#n;-HhL0sPSkX|rs?tymYD=gWQO~d%}a{;cC@AS{p<)v*$io)yl=cRAwuikt~{0s5pJN$cq0RUM3t=`N; zWAA8UYHeX=?)-089!iq2%cMsLy?&!o&8(w;{Ii$jSXQ2?S!Nlh0uMGErPZrtZ9Y%_ z)ivCqaVa@xo39LoUTd~pcYG7|QL9-~&{mQtn(+{X|7>7YiESL`ONvEyo>;UOvG3E# zwy;|n=L>)EAb@Tt8>{6U#b{BOFs7S>S4cRNZc*uw6X?V^$ul32V@iI0hboZcx8adByF{JA$5(z1`Ip>137M0 zqQoouK_6?9@M^L-J+X^oAZ5w#CbU8CM&DO%cLq2QKC&u zp)^pr^rvz?SP{Vp_Xsrkyqj=eO}@Wco<3}fd=U80$UuM@MnyCWq0~xHvsrQ7eo&Nm zT!mg0l(h$dLWVlVMEIBQXWKAHKM5~X!6~!0*Gt;Fyukb-+!WnOJnWR{2^^5!h4(Bf5SDfcvW7Wj)WjVfJ=j8Jggk+qMebd+( zby^DCUA=qxdKKmIE5OsO<`d%H-xkert^v*>gMaPneq$ZYvWe+f!lh-%CE}~T^2vQG z`(i)Cv=bYIf1)KWYTN}aBndJ)JEa4<4w>mDS~fKo)Rg#rHu#~yN7K=@syH^iZ^ixHhkomiV;A~>%XyNQZV`^<;@;x+jb~bTxHgLAEv;EIt zOOYpTksjglNiBMQwpNn$hO|YV5TPB45J8?GGJ*>6{uSAEBBl%FM}*^>8s$&d*~Ez)Vp}_HdD_7 zpU(JDkJ_{Qh>2ug!Exic*&B(WC`3Q+u<}PJH^S-8SYY%76a#>SPe=s0fE+6+5Db;I zdE0 z7ir+|u?E)s9)9zK0{~e3t;%7caWruGa@6ysrv6y2-z*U--f7*{i@6)qV-RWoYmWqX-5Y*K3N zHf0%aS0}@p7sHQM>clBrq*ZeOs#I{5`W-~rrcUzlU)bu{`ped@^Y+hN40axM< zxUIJ%1v|gL&>{7po)jQLv}P!zSjXf@PN#&TvxO{O{`t?sD7Pvoe`ryN#6jSh{QI%M z6e(2GiRc5iB>@3VAn9^{=7a<3isWsF07Wy6b4}-i^yQ*t5~3z+@F62^uL)+L%lCeX z{o3KfrwXsoytwMVmM70jP@f6I{2?kRu}vP0MbbJNa*`jNj}i+k$PCs7%MAvx+vyA^ zlSh0iWa3!RvpsNG?9+yPc#`;swsd#SpNG6eISv*Vy>20}Y3izEzs;xE)^vbRv83cZ z$oEBnuXD?ztK&_ym| za-u(Fd|Xi-6P2QdCRyEHB!5>EYp5$3E)gQ2m7t{6K7ig`ahwu4CFvNsX_9Y8z0vq} zHpcnU{_%eaJcQgx@n2)6;P1oXZxuM}{}?;D*jw8f{72|KlLB!41PG#gJS7B`Ixs*`MULF>gQyx62qTRJ6yGagz zvWFo?qB%(nAt&+-(fez{JiwqS7*-Kf=ZmBRVGcZpd8Lsi;jgWmbg*sAqMgY=&f zbP-x3qK}}#<)7CLLJ)kSnDgs|0e`k`w!66AZ&w{~+%#Uhue+V_ruuNG`erk=RV*f* z-6YrxK+;gQfFQ1qMOt!T4*Ccpe)!G8wu|gq$V(*$dNm2milYFr5R_7y2R_v+2L35K zq}FM%rt34KM-fPcU^eK#CP)gj1jfWU+)|Aa>C|qq42Vdc+mWjc+W*{v$#G_x*x1?*vWvOJ*PEA z6|gi4Eo$fH30-yrUSJO%E`8{!jqqy)c?M!-cI}5FdZs?9$_u7F$R!@vU&wNAR$6cZ_c+0PXtUX^Tz_!Ve!s@RKK;h*)P^Uldt(*ctV88%Tl zG(Yd&G`8Blw1*<7O~z^coZ4hv?1|O*CgX02TYisThar>%DexDN=ZzUCG#S_m$*G-y zmfR4C|0c)zGW?b`ePJOn4JjwC;ze=|?OGm7H!yH0L=9 zNfn3NX|yAk^}l-Jd{yuHT;CL#9pRfI|E(>ZoyOSYyMcR{7}Gf0+1pv$nR)!X>vO5L zVY?xU;Pa%0;aR0|k);jnlS{I$P!L5mM?2q3@W^OYmnpI_5fmmN3)Wi?r^61v{9Fa3U-cGRy^e$PVxg1k5XX*TCdpSX5hiB85 zlzyVh;-LY;B!9Q-jV6x?HC(tG4$7Xor)O1K?Fjq?lhXD8LYS<)U3JZN4@}u7$kluF3fBg*+#D{u6an^AdOw#1jSGyy(#wEIGM*P-w4UY zQnSaK&4_-Y(nf9FLapulS?gt&ktw|}V8MDhT%f=>7O&tRQ9jx1~kM0RH=L3bIyXl$-9$9MGeK4&aS!@2+-Tq53d>*lnU57U?A z?WT_zJ321bn*(gjg3&y{V{g1m3)WcwthiF&uV>LO$fDlEf|fC(;%+Bh@*BGN%C(o3 zTy)QPemtAoWY7)z4aUVSeAa%cWj>A+*d?27FBoypMU;b!n2*&M8|)xXiKJkN zxTh&GJBkDq9?IVsrH!JvtzqNMl#H`w|{y2kc$C9AGz6g(2$17j)>}( z7QK~>0Yo4`=#bUHZbHJ`Cu9u$s|u(D_T8uPWBiU|K$wu_6yPJlXVz_5Hj^{WmWN3z z%5A#MU*0$Goz_-7m-z4_U3#$x%^lSXTNl=T9cx|+lg}CqR>l~?>g*uh!g)Z02)m6;75;TCM#|l2Bou6nZV)V9c3~0sr%u` zAJ!jR>gBO^f91h2j0p+G)POq$8U`jZ;DGF@_fq|y-0^$2Gt`aAAu)ZeOwBS7)jFtj z6*TX`MvK|<1@zab9SxGDcl~W) zNZ*nB?*+abGzLb_E(X?rHL|3iGE#%|2>jO{s6}Srw+u4aHCKH+6WN0U#0lbc%tNLk zh+iERMUf-AV7JGhx3?NgE(r^n!}srVo6A#O*0!pwByduIj$E+FVH}AJM+@hFnr-Yi zCMjZ0>fPytt&JgH_;w9E+O(%j9_EbzL0#4x?p6Kc6#-p!f$JLl%~$IZv+i2E3R=lW zEvz7|FEAr}ccYgO@}awzf+=UOTqOz4mjSGn{iX55%K+-wXwvebOk7ta2~8G3h=hU7 z=K^hLzUORR+hGx5Swwu+E3W2U#(VSxdG##3AztXt&G06;(A=qS_>IT3l`3k^u>bn< z=lhiRNx$Pr8xjD3>)#ni80cy294*W&Y`fRGf8p`FY3G^Y~Ne_~0hj2$-H_xR`& zHFSDAR0yue6M7vll9YcAtc{p=@~4nk6J=Y43If|7zw3)vscOD>C|I1vWd9^rvr*jm znGWxwIIp)irtL%{Ow*mKnb)T+t~?R5qTKGbS50!mY}{y%dYsp`N|u|L?5k_P%vdb; z7s&U?=V1_6dQ9L2(Tad=)`Td zoO(UE0?S4oF=$q8P|6yk1axJ5$)Bo!f@RUEuBE&_v-G-U$#E*#`SWP*&(+nQc3~$c z59SD0K|;b%=WH(3A8Qpb%=U*_=#TELb1BTyd44v_#q)`!dg-jxY1<-093rcTSD>N|9b}%#SY>L+f4*ry`t=1VUO>5ho-QB*}<%}15JvO7Q7Al_2jWbuZ z+60L@(JS>y2-cF=AoP#r^f-hxM=$2uxwQF2!xI}6{9Z7KBrS&5B6kA{qN=q(8qSa& z(j2U#MWkhAqCQ%ECrcU=et49Mv9*&#(4f0I1I9@`$kk)Wi?F`xpJnWES!Uw6M5o9d z5$B4-xw+36H@3B5zD{&2Mi+XeGQ{*nQ00u|n%EL4R*hK?q@;g}73||kIWvgXH2Etm z({l3`vf?rys!IC><#kjbrp3$Wj?ReBREg>u7x^F3-4J!;4N@$d+T3L~DfPvRk|1`| z@;#*Iwg{`RsqG?=KH>9&zmUNO{X zoCqhtgxphq_rM-WSrh#^jO86`;9ne3yee#5&O@!?rD%l2eOx?U@DqzDs0q5w5Xc&okq>INIA6(v;x7x{1S*9%Fv!s5L66gL3=_OMBIZrO)rg_lL-rrqXG7t zyS$Fw&v(0AWX^QDlBL%43k^CyYZTiuzJ*`!p3^6e4eFWjdx~r+eV1v`3nzObhHLj- z)$ckHwQd~K4P6agJpg}oF1;?|L8WiU()d1{{#NH=_^Wf-JKEWq{@b=HjuqwQC9BBA zWoRX29Hyu!#-(Xg>lNu&03*;#ii_Dv&A5(>T9b|sTT7DJi_wdUfr_LeCBMmkJaUv0 zFj7=hmJg<_Wvp)`ZO*q86p6He=Puj@84QRMj8ovmjBK``ERGT zH}`O|_|8T7k4x)VIAL?de{A>k=hZA6$!_wCO@cR3do}*R|54^u;8q|MAWckMO)@_) zu&@pJ^)!`Ft5`^_QEx1%wIr4rQs6JVZ^FCC^m2`#&7<|^4g+At$M@Fa{WC5sXs$IU zPUET)maM=5x9?u_?WWLs{yK(kJU#BERPDBv?UXGnGcG-7;JfEvC9@r1qwG|g9q0aB zC+5985T^SJE>My*Yr9XE51#K5&-BuJqvcNCy1LDgRl4u!tf5X_0QpqiWf6jP14}Se zITuZfI$mH>fWzoxE#Z^n_f2@~4qWST0b&jh-NIYYp8h-?c98ESZsv^cxU28hmRDOx zPe)Tu-OMr;JeiuZ=9kO$rS*f=t9RGk?RsQ~ zpVp2?UC)E5-QCjk^7i%H7sc03rz1*=RnO-3=H|wZ``hu+^7J`f^yejA$J>UTUKf&+ zwcTfP+m`CZ!};zAGXAO=Rt&h%=gSG;C;Wo@mpT9SlN25Py9@VZV`b0R>gH+!E!$`J zm!Z|i(j;?@`=`i9%k^WkuHDDV>-O>n+e*4w^WN()pYPWC+GfgehMnJ*6?Rz&*tPH0 z#?CKYJ-aKr>TByS+N#gbqmstZ7fmne4L{z8p=L|o-IxMyESpEZ$Izh9n}`wG*2C?M ztOvD>H|&k`j*j~14J|v?CqD}^vZS|!f`X8TY^&Oime1o5_FjvlHUSt9=2V!77O#=6 z8}rAtk(Ze*-%ol@oM^qr*jAc`pYJaPv0fkBH;2zdwNJnF;L`kFn6p(C5u|G=ExmMI z*nh`)jl64m8e`p0G-`sRVoUJ&jQZs?b7F_1ByFE|CHBUlx9}d`?4d;SX6MG4*5AUT zsgZ+c_;P9l%*oLN@}Q@_NUUf|iMQWSt`C=MXIv1mTDJH1BA4{Sd#I1PbGLanxHD;iZm!FtB?1vrK)ro+%OsLVUGrQxJ1Q;812 z0E+i+)|pOdaFkb@>qXuciAo|WTwFHLvL;yzP7-`=ccdZ60NS`5AT19wrjVjZGy0`K zDLoGN(&&IVLvo}VHWn9#3+)SQV-RRCm{2yABgMoR!j}Zz6}KiD56=0^a8}K4nrR2a zMu49$8f&Z#8;;A=?=$R^NrD?&S$<-nBd)}=+X;|Wtfg9_N}fI~J*!uen6OB?=1Pf7 zH?{sm#fTwo7Yp*+Q=qTMvETqMYc)db6r?aN9?GFs?k|;M`6FIdh4WRL)rwrEbdV+L;5GAX_s%xF@%J@A~a5BQJ zd;BQM(cvtMRVc)|o{VA>+eap#7J-VUA$mngNeF zY~ArZ+Q^sikp;rWw%>vb@OS2nhf=1zOpkK{R_6l;%k)+$8_lfQ#i1p>fKrsBIKMy4 z9l5hc@TDjl3wCl`ncDNAZ;o$j!xC;H(vlzN&(S~cA|LCMhICA`9m>W}Kr`-N#kzR) zOlG&uxujy*yT|isoT+oJ*6rhh2cx^J% z-O_?&Dze+>R69PcM2e(`*M{>VW166_#%`sN6COx7$;6$q2Wgt7(7o+0Nq@OH#G-Gv zdB3x{%@|A^4Z((v#HZgiO_%qIqQ7tHsP6Qdpj&clEnlp0Cui!ouE{jnUrTKmu3^yG zL4RB}58W1V@1R2)X=tyzI#_UziFK3^j<6GAnU>8hgT%uZ?~%PmBXw5HqDHcaN3U>L z5HfJM(?)S!IIETreuBoCuf7pDKW5F2AwzXo2gzwYkd2HEOqBZQa;M`z3MS2@Txr9i zE?~H$GHaV8YyA*r#(*KUBo^!#_N0Qy^Rej!$F6j-X4=6bv=!Xp3})})3~p`hMig|f z3C=PH9)$a`oL?EIOhJA`A2#&t%vk|PIE@&%W)v$z{!@*ZL-?39DHr>aK=^yk>sK3# z;AU{T3xtF3_u=64Be+-HN_w0#DQI@-P*K{t%OP(aA{3D48{g7pibag z{ZqLDno?gjMPqofgwMCt`w35$eUXI>XIqQK>*+mM=$$|q02g-zBp$^hMz^zwyhgIv zl1HnN((ysFVGvDBd`bK$Iyp$A2^sETRF-`ig&sO6CMK@0k)zP2-g=5u_);OrH6Vrl zH?82LZXB^7wMOKRD|~6UHiG5&Fxb1EZ6ujcp(|uN7*eHY9iZxk$fnq0uMr_Asq)Fp zMJxm1(x?K#%3d>U5=g_+a?2ku6pFJ;KOnLAY4xKeBVEk(BJ4%*SE_SvW#kfV5g$-a zi=@2pZulXOpzDtURGJ9GJMz6{6SaN?KEdPPtP}32HENm0YKd_kkZ7=cr#U7SR*2^P zF%2Nm3nAxPWHoAYp}o+Kj-+SNjv`)HF)XVL7go7sT-7|1z}4z*>$nxZ^9Aj0N>qrtlp?I;=29G9b4Q|Ohl@a+-uvq|2}h&qqn z>{0NmX*r&EHCHksB#EXhsC}*cu8~gzztUKX6B55AQ3km$dAP)|pgu{te`Db@kx-o9 zCR4E=8N((Bu#AG&12j*5GHS3QsJ-p`&?s$?@Z%_plFh4-L{@<9p7LHTwq~yVrLT)? zoReRJGm4TR9TDoo*cbWcRRmMbxyYd-{U8!jZj5Xo8P5!o_li@`AYJ$GXVAc{kZ=4o z7^TVZ({zQvYay3kpX!N;n%pVA`lNc1gS0rF^$O}>PCbrZh@<&UjtyBGz*kZQr;w7;rkQ8~1II6dK5&lS{57(7%olNyKiR}$1Rz&5dlzSP zYw30bU=OsYvtuvZkLihQJW*~5j@_UWM0hfY&P$GwD$oEpY6AYj#~4OUokl1W8KTi8 zq2kOs9~rz}F+vT*R(0&3MziIdI=AInPR@b779Qw7ROv-4BMA_K3s%hu17=i0s3~Nb zw}Y)YB<7LFRAwWxLdXjxT=fN{5#r_QkzER95!|=W1axurkgqu<^ut`f^y;oOB?uC& zN!+3kKmeuXjiJjVwPDcYSw+xE>c|TdZszJ}NruDwyz&;dC4KK$)K?^c@X=RY@_XSf zrYBVKFRIZ&4INVpAagCLmJn;ukPMoP3KTq20v|XHL25Rxg^+Wc8CrJi3qk3%A1dHn zNoZzIQmiU=|;pyQ!pq18?*Ou_c}Qc;)3Q_ClKDbbwVRRB3K*Le5^ z7cXHGDdAb4MrfcrGt#aU&`uLm%B&7}C{z#=9Vx*qS^VP-LA9dwDO-R_4A%xxjEKM> zs(Fl6u1F1w!86_tl0mnpq@T#4iDOuUQkqgC>`Mi4Ed5Ij6;s*6>BO1qQKTkODt|p7 zl~ZYrt1+hC$`}Xy_uXs>p(__D`9c5igVXH&fc&aVG;$U-%knPeF!IW%IMkh&L=8B; zC{>ndjMQp6xRpg)9RuRh*qvM&1InRky)e?hpAdgs@XFfxF7WvG2?>KRL?*Q`#4Kw2 z;?4xnY!p`v1}7JcxqqF;51{_je1g;RmEBv|8K~VyB|LpvX{H0yvMSt+HHw$SstPWL zH(JmbHBT-q*_U+X1%#eeoB-;|VA$P)ngE)7m?TBo&<_&KiZ&{_2}{}j7|r)vc+Ff7 zC$J$iO0oP@KWSc*j693_Pv_Etm&fM!gI01;Y57!kd}aWRDa)|>4=SL$D9AUF;yM9Z z?p%;&1?>-xg$HrT=o5y&LGvkXM+#N9D}LW`JLKK1{$BzZ|Ca!gKuv+GOv#jDLaQS7 zH)BKEp?dVQx_+_?ljz1Wc$$ElmJH zzRV%f$duFmswh^y$7E`t6tJHaDpW;JnN@(qGTtH9X{csXDF3$uQ2*NjGzbK%@4ddiC{FS_RHX4h*|DgCCeV<^Lnx80$0r%Zx)yqKI{_{z>tn|yju-zl=|bdeGaNa}qOSd}k^0`WiUd$z zE&H{R-qRF^laeRSFNtxW4t%&P1jFMqg#CO^GN^Yrh&c%$u3&aLFHeXYVv+#4$FLwf zuYLzB0Rnk1>x0gHYI(J!ki}0F$x;fDgtfKGh(soYsN??3&Z9|+)J*Gh!3x*@IY^+A zBb{LNK8QNXKz3CsMc+oKBUknF}(m)uNxNf9SJKE&T|n zq&Hyg=abKO4oP%jZifIJ&cqNX?$rmfs;q<3$bOqIihD74^x(?YFc|kZo&k{s+d_Ny zVAh+@NSBUqQ)s^YWn&)aooUG^y26j@7i1a2qjRS4+6DvGFvFIT>}lh@iY=prA|#>^ zC`wg?jXlcT<}+bkMW>z^Uu!_uf-0`wJcIVMW#gIC+(5uh&{STB6*bwD2YSN z>K;{H8ig?k^%GNNakyy*pVXqnex_UtG*MY${k*)@y^p@o{5`?`SgTd$WOCu$i=#pJ zY%&&?>>#swml(U6nvj)z5(TL)%9wifFp!n_v33c&C`9*{QB4*?30dHzUe9|LlU&X9 z(WGe^)5`BR;BwH4r%Uu6L1{RpIhI8v4oON02`;H>RXPk>NXS~3X;Mo$o~7^mVbd6$ zOu}DHI+Zmd^zW6x+o=7U2~@^ubtlQC-3>t>v!E=ix4yHor{mn&@qDhFHok{7Pt5SW z|F3K{vfzjhvi}0??td%7Vfz0^IRA~;yK1=0M2T2~vi^ z74tJ~nmev8tiKVv8=cwvI&*9W);D5b13ebu5&tdr`#WcU_m6#F$E)P<&`Ma;UXuq! zBcm+kS&o)@5>oQeoU-ilL2Y6c+PjDyJp9hI7)xh-L)l}fUlOM+zQ%{o(8Pnb7409* z&`ZaL&u*Ea`jf*pWg#B$AjB#PT36(Pps zKiM~l)2i6%?H!Ja__is2;k-AzAi~$NkJNNyA?zG#SQ=&B?mzdrqTH+Jp!rRt1$d4P zZCVa{1u7LP@7BO4K)uIlj)zPNPS~npGL$ffqXIVWjyuep86Tbv_Z*2skvNq~+sF)0 zO4&~$6cd+qsnNW;eH%35IFaRyeHgntc`_%6W~OCJ+|ivq+cj0OT)^;b&d3YaVuNPw z-l}+$eqqJj$Hu*S59iVCM0Xh2o^PCZH7#~;SDstf4%1Y_o?OTDi<7bKUJNj4`LzO^ zmgl&6y31B6RL&8U32+vz0G>^(Un_4uWVmbh{x%kA{*IhqMFt!g0kv z^sN7Y?v4;H-sEq9DSs6;l$4ro)mIAD=MwqSmpE}GTA?ge>q2n8Z@>HHmZvsVi@{QZ zC2U91awv5d?@aYe+3PCzMe{1bSpm6&wPcCJ{&*X}sJe^Pzc=y1EZ+GK$?wV#uHMHw zWG~_bkbD97%0LPa752>OOLpywY0oW2xrzrs={IAt)NxQY0g2Mx7bIdxCE%+`&}%^xV5M z%mNB06d4q~*iMN{Wh1w1ZHazXsWf5p9SMF2V* zp&eCovpaq+!+HCTuy}qs+zAeXVc@cJ4!~4F7|?6Mj$U7{AAmUz-pr0AEv9d$ExNJH zbje!3S>&D^qY&uwFow@$gNTv$<~qT~r6Z4AMQ0kAFGsr66&){d*8G?w&&=?HJ{{PL zPl>(M+b`L}6KLL*j-@lA;EK*R#;Z8v^qG+q&~sZ1=VLo62yiurZ2V08cgVjo_W#MB zfBvoJ#rzlZGW=ffXl-F@Li7LlGmU|%^Y?N{dly4%3#0Gc%Ku6?yZnbp|3jqzA=3X4 z>3@jyKScT;BK;4M{)b4H-~B_R{~^-<5b1x2^gl%UA0qt^k^YBB|8F84fM|AB1_1z& zhxd0U6|(#r>ipMMR{za(zE{Tnd)qaqeb2*;V;tAHNKoaOCEsQXnX?aqp29HSSs#B8dnjYK9&l_qOc)M|Rd zt2B|o`ho9AbN@yY#ETzps!CZ>8w{vfz&01ZWZv+#9Xlx6m>oCCE_qSuDf0P!;U=OG zht+alfXFGF7fVgfj-(Lv!={TPlKEym(22YBppyS)M2DNU4jEY}n5lzs&Ok{A+|d33~a zVpK2(O=AQpv97}6V9XOFTWH51IPT9aEec}xGzio&(~jk7jO*yc2jYhn6>Mxttg_|h z+=G-R7Y(k1=i%fLjkDMe{kn;E_VT>)xsnl?R-9|dlvs}{YhCKQ*U9NT%~e+O*Sk*N zj|u{530W4ioR!S()EOJRX1rWRPJ=%+KqHxZ%ZjrBO5wVal8~G#0{MXl5hvUZkgWhX zxCcnMoktt~RoGD#Tj5>r%-NmTawB9x4Gnuo&hD)4PMp@j?CX{5kR`DV zx1UVhX}O~b@Cz*keQ#MNc(Xl6d_Aerys@8rsnOFjcrG(ngMDK&Iie{86tei)e(8+{ z_xk-qC1dwUMZP$)VL=lMdiKj_-vk!8R%-6>Hx2n+DG!y2-jf>|yOI4YXtj;2U*nkt|Z)lZzr_s*Zzy?&oNJiOo z-LgLYG(jkCG0_}Ybm^B!886CNtN$fIycA(WlM4E(RAZ*>R$8yHTvdYIZj(~r z!RFlYYXY`<-g@|2qYr{b)Tx?vWAT({rlEMBm0_#O>>^48+GWTFF0X|=s`&Q78#qK> z_3O;z+)R8lvSM+gl8y74@XtfKeZAVOcXBQHIF<&b`YIA2j!I+yz;F=RPqhh};wpLE z{t=!xx;kC*0gaZ@)ME=4uHJM6Qv@Bh$~U&EHQ$bS$DGbbhgP;%s}v}!=dO|3tR%iId0qy!Yu^IGS~rohW` zaYF$Hv=3Z!db+%ZI9o>AA(+4qEtAOVAE*e1?nw0Jsj}9@5qt5-dmKzq#`M}hOY`jq zfy`(bh}L^k=>KCpmM;P`&{mQp4@GaSPj9R9m-r?eqme^$2w#yMf%uTHq5ts*a z%SfgW1t-E!fJbJP>9Gn)F;FK##cs3_8(Sj09fiZMn`T3H6Ax4|FEa}>p%0AZL+QzI zMvPLMIOilKXpm;(Nq1Qg2G|X>t!i?t^SmEv$vL=rh#5b(dM(2mm}``)T5G!3B)A(rqKSd*eEF(Tx+JnS*| zoVv+AXk{tV*N?5wtZ)6A3ql z4eh!J2?Z~cA;wz&4>~~NRfS;O3G{FhTp*>GQSQ7Y*y5fE{+FCTb1@Znxm6z!B;>A5 zHblM#fE}h_sq#C2x4cNU^>g?By*HAno`EMXWJ3)(PfqZfnrF65f31)HU1uzgHU2(# zjT5%lXP90yyngn8ME=K2O^ZE0bPR+FR7*avk}JSp_D#Nb+meQ#`&O zuE+EK{v4gjuP(Z%qx@e!+FL3tlX@zpSg}2NO8(p-wK4g;Dz8cVHRMb+rJTpscY_E& z8smshyFe3Npq@1M!tm{6Kk+6MZcpDPH zPs5XVG>}QBZ(*67ufMJ}c55}Ay{j~gTSOW#g;Y2`6&gy}+q z7+~JUL)$%L*J;2bK)-6Su%cU&^W%rMXcKHjGsQRmvVRKRyJQHk4<)0_v-mj= z(lAl4ZSgMZqu!CiGu37BS@6k2y=Uk42X9I4mVUsnHy>J39-mpNdvDL{S##@}Brg2I zWo?n_wwfy1x*@-OCoijobwbvZwZ{{nD)mK0lLcbSV5ij!ZDYn4Bf`fC&*F6%y}4YL zF=M(V0bAJj=4dGAuQOxAo|vIH-y6`Ne0QY&-gR@V|FaWiWNu<)WpDSrEzN&cWT;5R zZLlJAU)LZQ=b8hai%2XE&_g-DiTfL81j$7J2N8ueV?{W=*e^pq_M{YET!NVcUF4Oh za^7Tsn{{JOdv`+y%1guavqH!jr5Eqb3y9vY3(S*GsW=k}&4gc(A>#VO%zYvp$%I7Z z_nBx4YdDV@g5EAFrCxreMxz+p)AHI0amI{D_|ITQ3#zQH^I zx%NkAMdi~d*w!H?61}Y*L`@yb*tc;!-yS308yM=Ix9G9raRQCv8c2IK+O&PFQ^#!S zxyM9Rl;*%LAgQM(bLY$S z6E+rkm}zXD4_v%k{pPv7I78GC^wiy-k?j@nWMSM7O67%$vyvcxPv{-pHj3b$f{pY? zd%RX3BQJ|Q9?5F~ojv(cIp;=jI_O!KGb0Mu0Dm;UrnhljA|dAyKO};urb1so2{^(iDi3_2qS!zY^0$`o%)tt$l;1P z_GH_2b^kF29mMF?)uZDg*-c9Q6Gy&Vj@aBIR`g9A9ryy02Yf}odhg}g=7j1k5szyX zs=?;@4=g9{)%I4$m)1Vt64EtpPP%!)g|lE_lMNUA zdh>_Y2|++@iW4Sl%D~eUyj-xG9^A$00WqKUvj5fP^O7;QCx;|rbD{`@O?~6kIcvoM z>43=AAx76nO2sGrbq!oA$=$Lzi$up@FeX?Wa~gOZ;bDK$L|Hq z*N=|{WK>FG8chw(Yb~k6xGL3+Yo~DQJXvTbw-|tu0ubyz%_XVdRu{Ge)J|e-$5>RC za5J5?8j`5`suk-rS#0e>R30tED0EcH{bTV2eoW@*pvFgvN&?|1y1Xm6abMP}#wn;g zU38!>n;LxIe?hx>jj7F$k`TQ3&gi#)Pa-s{57iVD`_jir(7JTnVo ztMvvmuw4FC>@fFKxe+83`M_ydTIeG~YJIiO%2Q`b3^@Ca6sYIUDrRKQx|GA|xbgP0 z{qJgz)hVaD0OEWKk2K#E74qY44`^guUIiBgI0OBe8fW3T?9$7%pJDT5zZOjj z2q`wn)xUQFpK~SK+d6^=RiiwBY0goU{}h&@908FK7eF9}&xzQDEU7gr(St~!g8ESj zOxP&cQzal$x7uogl&Kl`tOE{bg%%vwCO60w=0(`Don8E;CE;_AzsA5=YtO}&T$SzY`SLIXM zDc^U|sR8U>2>j7pD&-PQFa$v+az-T*=rvvCeueZzf7Nf@4SfkIy047$8m=yot}Kl% zsB*bA&n}BF)x12Kn=FiIE(l`H5TR>Hu zXF0^)<~*&EtzELCO-5Eh8#3{@LIr3gY?GHkD4CnVPZ>q^hk}HQyS=PzFAtfI`%F*r zg*;9#BXtRI#g9|>E?c%|92_fJH?Mw2VnT?4ghE3YqVDPYz_iVlvJvR+=MwAnru!U} zi|Q>C>24~OReBOd@lbP%L|W;3UX)sJ_k+HynAgfxy{Ikv0e60Q&x&eNha-ecdV@IJ za{2hD*+M`!wFtQUvNw*TY?HblS0{lRbcntL=st3F_J_;88M0bwqPkJOF$dC{5giu= zP6-(;ub;mmeHETlX1kt_rPDVs2UWH=#Sx!w@N-_-(@k!;aE~Y_FAZdBT_4c*-cg05 z#rzEsrx!n;f7k?Pok$d?Cv%@S9M^w#Oq9nP3H{iTkl;rWW+0(IXeU6Qv;!SB?m)Dc zD0`~aQcgE1%?8=nvx~USI zPeYslh$IAF9Gos>%BRhJDL`QeR*r z9Qr@`PX9x8`2QZgOwac3zhDN&Cbka7)JEoxhPE~~U$ERSXuhGP@jvSi4NY65VML!p z)oiZag3Bba!*1Z-gI`c-$=c`qL!}p$B#zal1d~Km8rPe$k-L)VOGLr9dMY@9hOVlr zs>+KMyyKg)nLO*1Wr3crvX7Ob5j7pA=TrO5VQPy^FRbrYU|~YMRF5;fZRjPKXsqu9 zKBc5cBA(WTiaDjJy@@`wp8EsCsV6 z1)D8OWdfWP<8iZ^sIsc`*oiakp@kuKX%A8H>doCPdSm^e(fJMPF>1e+2$&EsY}eWm z(1B2BLXD9o#z{~Jp#!Q_7NnkXlva)$0E?*U3Z#p1Qlz~JyDE4k%MVyE`7=-p zZE=9@AJ6K+cS{09(TGQ}Ta2^kNyBKw`Pa)!U{VDKG8mzkOCs#8sqH({SB@Ygj+Yty z42PKlHQ~QbuZYv4p!8RqFwL@JN{>(A5uxlLjiPT6+C!+HL916LsapnRW~nGal@T&D zy_|ThngBZlTaSy|Sk88x@wT7yWWPp6Tc``rg8x`K#aS=37*MtR*?Rd>)HIB{Iaaz~=krlvuRM(^u);3oa<O1OPk?m{9V94k31DvcZ4%6g$(0ah;26pB>XnniuOC|61tS8h)=PBI8&r^Hb zsnbmCck!|fFVCa}I$n$DVf)9nLtB#`m=UZKuds7Xc-iE=~Fd(3Ng z8YFtR&-F|nJ{;}vudvhS4RIy1!tCUPNmE*9)Z@Qeack6xD$<=|2^1xy6fko#vOhN% zBiQ}nCd4h<8;e`SH~@tx@#Gc4tIuQ$@47fqw#`)l$bS#FI;(IXXh9&!Jhbidz$o=mrgfoV3o99N?!p+YURiKDjXZhcNoPJR{|EQ(t zk_)ZCmFpFyBPVz25p+q$I@n{YZfxerC~*MgH^_4ppuyH^r-RM6)Zg2 zUab&O9)~fSf-Wfi?NLfW3>wlgo_Ns_pmV6$MJ|B@LVx?KPC;*Q&C$oV!>z8mw`}=T zQDgLDd;*D!jsz+*@?v38i67#HRPnF`0kTmO0z#t+2oa=W16CS7olQ~_lj%(;b#>zY z%IXqHQ$l(vrmZq;gIwrplx7u@)Vv;B=fXW8mdzVXkyrE1H~Q?%GZiCHSq!Qa;HA`J zHx^SXfQ_mwu@PesSVt@RW_Xz0-XbFb118{UBQ32cJOVN}ba9Iq9MTrd#dsP&IWS}$ z;)qlDIn0VuMtMB4(+Widv57%&iTp5&j5}L5xJd?vKD89qfHRR6w}5Z>P!(P#elNj9Cs6&uc*-D(w@mSd_34F!%Y(ZwsHqG&^%?;8GK^{zzT`Nnx<)6(CVD z-8etn>N~k>%c!V%O1Z!?aIRd>EVOz#@5N#$=1EF^xxUG!kc^;ydxbgDYk{V=R%4x# z`k}m_ooJ*cjJ5%yeNgxMDmmx)Wc=Wzw!=IiKfkq`QgJxvXFHr0hK8Svb=Aw# z0efJEN2_wv4m6JS`UWAvo*lKKhH0~J##OI!`IhjZGoW!i6aA!d0Dg49d5z8EL-xTH zbM;e)kLpQLj7g2<9$f&lAF16*h9!6s3i*P1e(t3@qPHyYQT!B<<}|#NojOFKZvvTGp z!3KZDWB(d|#cEFP~qk)6@-JMa{_Dc<6ov*5cjSumXdExU67dWXq~ zI|%>?Lsc?h8FzxpH;h?MdMZNVtr&)pb2*W2c{FmO1g(eO$0c-WK7<)c1FnXP7znTK zyDoy|X0LIz33iE$*N zJL_%tGwz=H8%iAKd4-Bfmr^fPZv-=AuyEPiV;Q=YRmivY;fg-QAfwjOveL4$lah;% z-FZDasuA$srhx7H@AyVft=J<3$kGdIJO5D3ZmD)c3KLfsT?xYpELY6ub8QqlT+eT) zT1hCwJD}{IH=g@(nlvEv6Nn6=FSMsvnUxmRe{3uCoH!N$hw!wNv!#6fpS!dsmW#0m zdBnIvo(hA@En3bzxq_ef1?&q93C`5pkLP}bu|Z0^z)+W@9kfm|ySH9F)?3IUtK0LO zndv4zyw+NgRA>edLn6h=dBP%}g&Or`+IL?Nmj>8xaYtJ;SY&;#Y|YSNBT?p5r9cdl zr19H@uXhq>r|}}7f;T~A97Ea`npr`>6b~bS#BF|*Q5^2?&?2j>E2XNmJLEB9FaQ4I zL5P!IaVA)ihUr!WEzN3<|4T8m22a%-?SP#YGJ=zVgwV0bL2TKqOkga;Bvf`x3>s|> zhas*|qO_d44j*`KPpk1gFMa2_eI3=a4=-G^%(j^-dbMsodkh?nbuTz8)j}SRY4i8E zXw62~`98j?3#>2UV8Qn4+>bs|MycA@09GueqxM$uo6)w#5m{;vS>ac#g1?M%w(^}8 z*hItP7NMO}#3Bjk;NrK{dX&X91kV?Dj|Y8@G@OX2sDYxo>*=HQ?PjR0y&Dh0^u!3> z2+@!R^pq?aUanELcYUW99~O2FOkb{y^mndTxDOm&i!aFx7j|&B zZn^tO+2Q$cIc$K zRHFvY1Nx0^`n`=h6Ni}O0xtD z>Pb7)qk&4XcS^!jS4SV=O)&xTwt_6JcBp5Xp^bSK)KS`e;(}-rwQQ})rJ@Xz)6W7j zhic{I=kvULS*wq0xOlW+vXK;gu8`>O*7+}>qbzc;f=E3Bc0)_?@BTzQGuB_#tsNm8 zO3PU53y5f4f)x&ts@>ne-C*sU6^>veKNr~5O6^lPf|@~nFyEI z${UwOICRoG&8ka{IUJb~CozcR&si~d8iNqdY4op!6q0t!G*co;UT zs95}p7e-HqN^SZ1gbt5|b}(Z{{xW)M8j;66FrQ36#bunycc;qtbDzUtiY`x^V8vlL z6g*uWgkh)gSAOhO-#1eoO@sicE-3x#a#`CZr6sicj@y3ZkM)PDw~>w;w+|h z&>{n5y=3QMKBix}IZ^f0fiV!TK$BBS)u8$KU$U8~Y)#cq5;khrDDdkFlozy)n>BLF z_9V5Kz~}NYH><%E2Q?HaUndH~Hty273|lDnxMAdBvtn4>G8QGCqx2xO!*aa;5YNaCC*`CHUYKP5DgoGjhyym!uwi6 zv;7G}Nw|*C1gTmTAQxW{CO!4At)tlnEQR+kgnvBXFFG!W|5hwFW+9;n91Q;PC7EHf z2*a-A;j9EyI?ip2D{k{CM63X-wZ+R!T-s>k@^D*X-~XaR`F?>4$n78R!VhXZt#OAT z|3lN9h<+IK^h%3@Uw=2ke2*Z2SDw!t7m_=R7EP4|vXo0YM#2h&?Lb!MxnVviT$x-oSTOhPE zOalTHj1h#-=H9!hB}Dgxauf{*{%_UeM!@{=)r>hSQ`y#2z^p4lOIdURp&8$T@0Ev> z<|%7B6`pG`WFBtXTozQ(bp@d2-d|5|w$`-K(}SM*)sf*>!k|j9?;T=AoPlF>MD@QEq!>q7Y3)&@- zD!aqEPa48dwGD^7KTTa9qrz5aVX4MVy2qscN%&Io={p1DuXE$<0)za7!$R^eOa z)}OK2>9WK?r7;;lUw^k|vsGmurL!`z)3N5!txQq8Xl*Lko+q%al@10JtO815lVpvC zx{^+kobCCFLANE4NBQ6BpA4rmif?tqHig$av;zh%jmrm*@7FC_O9Li`B zpivvy=>gD+`|i!#R?qiCI0;G?JxhWL~H3xC@K63j8gjXA1;>}=>NS$`?qrWJj2prfT^Aqg2R@}em; zBX^vn4T=esZGz_4*ZzK3s(eKZa%?R({oCf_uXrX@-IK-^C(D6Q`d&eWz3uJoUe}1H zn;8AGd1;vklYx!pna1;uP_mPg*l>asdL6o3Xl3Ilx8=D5|Ll=;S}EF4flb+--w)D* zT5O6UN9v)W7N3^0(-_LR;iGZffxOPd#K*B2XX9tX!#WGwiap!o!y8K61|uqZz(_WU z){1c&>JTRFIYJsk%W^A5U5I0|7}OA!2qfn7TZ-yMWRwJ9#>`R(q+=(q)oA@HI=Sxa zbUFqm_Z>eQ6-#kC%}TwvM<6rRhBKq0eLNnt7K%qw^PQW`X-mDMd>`tF|JiO)@fR_rJw6SgAryg0Vg$=6qnbD zfN7F=rH>w96hP-f;$lTnHs-&AR3$ps%q?x5O@2soF7J~R0Q1!Gk7L)#FWwCgPRuek ziclUFhU&$)++3&2u8GpfcOvXDYG@jd<%)7>8Y0!6tRd z!zB?}A9`j;z0ANWS_3e($00C>gp*V9X*p2ud+q z2#cI3h&XezdMn&{+y7R@3VvBXAtwtyT_Li--vIe0$C&NxF zw6A+$=5B1T?;o-?HV?bO3BIKsr?ji4bx->$Ao;cj?|rOppPtXC%I# zVq8LoH8=#}41^6&xkudHQQ}13BdZm3E%b(G7q&m`P{G%vpqP4ao@2S~B!@431eXRx zYN4kD-tj)2c2E91I5Uln+oLW<;m zozOs4`Nxzko~R`K%(ud$(LEPAJj*ux>g40r!6@$Aos?5q!_{aYR)^R}y7@Q_$tn$x z`I#~jOX-Ewa%OeQ+6AME;eQ?d_b*rc?fO-`isG5rSsKz z>w%(wEkSdWGJ0<|jRT{t3Aq4VIOfq^$MTZ*k3@RfQa_+5t@X<|Z& zqSpXW_ODa=^RwCof2{wS_}Ki+^5ZM6(nsSAWh|}ol`satHr$nc|-S2>5Q$2^RfUK_%&fTjq)t%juC|!!RB6kxL0RRn6ug!?u&@L7;@K}Wj`mY* zon=Yk3dv9B*i~K(U)M#MHXCZOqz9#l^OTJvw!6^}37qx^zJlI2K6;u1PQ^9{MaOrT zqXkxaZ=_Wi$OLapNXN}cKKHjbK=Hc}%~^X}GY}9uKK|w2E2Qg1hC3@zbt9?CEr$rS zX93*EcM}2`0nWR{P&cuH+EB(IC5#Yz_~$bZlzNmca3~fWxisdW@TvBigD)0?J#+{l zkyv!fH<~vhQ5=XjBELIh5c#7$cm4Y;Gk)w8vDeLZw@=xso}~{SbBUH)a2)_Mge|y! zAx6R(&l=gD{d#Sc6auG$nfJ8=9u`ZT*dtHy3VusQoX1k+N zM=g#tt4Bkvm>WWpARPd#M0eYEQBAnopRbR*9T5U#;WGKrcsqR1aMWh(t`st;WTR30 zj*|MwQl={`Wg=O>`vEZT2^E4ZOzr(vh&~z)qr#_$nyHj%jv&9HuLNGP+BeCt!R2e6 zCp{C?yop`~!!~+eE}tabJTii^-n=KY1F4=*bfKG&6jiW<87bybQYRz8fJv#yG!+xcW8Ijg2!$+S>~m#J_m5 zWLE%bG_c+&8hZcM(L|PDXi`^m-o$2vNP3m`sLKOTId_>oDfZ3BdqI9d?mGNvSw9L0 z6(N2PjgWV2o$Ys298nM^DSQr*mjVs}EoAzexQiN*Pdcj4PcgAKf(OIExSmkE2(d(W z_a`IwqorcJs7v4{2wrF)L=$;5c6oSrSsK`Nsv3DJ8b22mHa1nmSl1+8x_p3jSFx1y zttH3y`(N-A1gPD=1V|w}im!IGl)|}~^SABRPI*xfjV16-Y)QMABK)p{ zmDO!A*%5svtH9r|q}^~qANcL#BV4_Vtc+3X`0WeaUORH9(5+Z9WV>RA+E>kAeGGOxsU?*gpu>I_eYl(!V15O#+aGFKJ_OWXdUl$Q*}y1hpO$iVJM{ll*nMjK$+r8$*Pz`351RC( zy*>DPR3AY{2E9jF(HWyWE%)uN*7(^0WR(0m!jMb?&6m3iAV+#s5%UYRHw2EDH~$Mp z>^lQe3`9~8k{`L;vW6Z{O}df-{s{bU2kLpPMR{js7E-#N7sMT*75U{eUpXl>>Xx{j z-3SJiJcM|2e85F^GTAK90B{{ZJ1236F?q`gbZd*LROI}xZcUG#C?0=QPdg0naDbvJ z++I1V+G+z?P|NBE3msJ0p;gA7f;vLF1JA2b-gbWZ9lgdew|&GgF+p^8!a#NfL&`Wy zagDijhaCY1r>F0RuOkl|?@?y@-2`Yy+(lX5#%XBL@9IqKJ^!mkoyQ{rG8_T6LfPA9vcHS5H^G>mS?4!+8fS9o3#!udlsz8$S{sWi}s4eLQTt z+aBM$(e1e}U_93A@_5bWZQ(9wFb+h|>De!Q9^aZ$>;i5#y4^>W6&lmvee))5#n$5oSj}&HrJ?N}BwuBv3ltRCL^gS37V4Lw22U53jxl7#-D`kF zk_5HjCxI+uBk8eKjpvn&5o8P1mnkmO>-}pd*XUZv!e0zvYEWeyEo~9tu9+9?wX|ZF zH3e$6oM2ef6Cswoc&HT(hvE_A z=0wa$kGdYA4WqUMErE6$gvW(v5b!j+=aU`%d^{NI>TvT<_9{qbW`|>egDNfN zK{FbYOlx=TQ6C!fC?QsqhqRR=rTsoiQuZ650DPeg*_mW=SAo`%wy;4{gF1dq3hbJ7 ziobI1+eqn~1aYH}5Dad2e*P2$+_Jtya@_9km7No(!Vz@f(8!G(mxwo!Kx8)MwLJp0 zj-1J2+!*G3WT{_z?NI!Ver{7^F zx_wQb`SX~fM;K-Vh2qydDb=SGIPekM{9qRIY>SgwEbvQ3H4 zWVfX@F^vN*tBt403H{A5>6Bp?<<3y8GNqBE;dA$UH9Qb%!LGv`W4r8hZ`Z*hUo!!f z$GB1nUP<%U-h6fgh*gyp5qb&|+f+mE6d#NKYeYO)AhXf>#7$B?6sF!nxr$j`|XG?xF1Q%&#H^iPF zP5xa5zGVX4+&Bj@q9n>!J&No|pbyL+s*c&hGSIA#Nm0`ZZ=?IGg2sOp)u;?=>j7l> zgrLcr|KU9-f+IOTWid*!v;~nYpQ^#U+sHOyUcp|-;wMO6Pxu_6{i$0$kA{d10DT1% zgYjH&kW~3Z`|5Ic==`xOr>5aG<2rlaiB=o8^zz;6AiI&AH#zXv-EboaXi%IMSRRzMz(BYjDg-w|Mr9 z;JL}aYBCA!tZDeuc$dQli~E!GwJq}8mlhA@jhqOBD(wY2Cvpcuhl3QeHwgstG0MWi zTWw%lx#uwx97H{mP6p_%|=Vx(A{>Mh$T8F6!j@T+pD$A>KI?@ zLq>4NzE~NmwS;$LF2Xu{I}d2x^Vr8Sq}HzMt?@>giuflr!r+2}Y?*E}mk^3rw?$Qm~3 z#3Zp+4aMaNv>I-7hz+X7t11z+<_s|GcvD{XH%1^Mvw+F%au%Nk;3Mbtqw~Bze60@G5JwYT7Ne&yCk03ZxeKKUhzI^m7|$tpFu1 z^Jn%h$bd_FW@!`_Y-xpOUj3mf%b-Os?-+hEp4kT!r2nBS3wf;@Le@+%ajgESD+iq_ z*^g2Z7)#ZPl3bzheuf$KzBK63Tn^a}h8NUHhOXx zqC?>&W#X+*q{^P_i>tk>{I_di@zec2=F5_73G<)cv-}gT^?$z>=c?AWUzX&L4PE@F zSWSt?*AInvokM3g&Hp%w8(E#Im5TRqBFLO@N)cxpMG4 zu7&gR{LEG6CF`Kbn^#61ygtc68uvELFiVZBfN9;a&TF;~#yMxUowkXdAt(w&W(tSs zF^isz$gIRD33y-C&-B>fK*epMTyD(bi@X{7Dfq_#FX%XeSAYZHS$5zH^t$7%=*tPBws zeR&6$xLP~$F@4;CbNh;tubpkgXiuYPE=h;0A_}p=dI=!F6{Tgaqr;7u&qxAQ^u;rA zFA2DC`VKEqPjrPqp%{CZf8_xdm)D>Uv~!(e=Hbh8%xelzY7xI!LM)4zuunEw3y7cs zPGlz|M=e(2BoW4B+8y->#c;pYh`(?^tG)iy(v)(InSihQHz1gKs&Qa`Xq-zVj6foT zh~a?|HEyJevT1u<#IzlPSRo9p7JOCehW>&DDJaNj(fT(8*_>Kn?x7Bo9EK^uwhN3%C|VS{qLO%i!Y~bL{V#AX2o6 zA-HyLyDsTQO}(=q+>doI|5#RhoD^04b>iZHL}_I}EQm%6^!; zrulD`V<*{t0jSO1)ozQP$lG01P*e8`s1weOw@={NE$;qsC%XA9Gc#&r9fF0FuJ;nm zUXRAD`@lpDr~N%e++I+Hk3`%B&#vMeNxl=~`MqCLML%9#Y}Il<%NwYYco}~8+93kM zmio{&#+Y~m^#GwQs@pBM2=|ApM&t0dK-HAnF(iS(f^0+%e#G9y&*g8C$$kYoPSYEJ zRK_3@jJk`2jxC|?`M8s|RP}a-*ob=5;)LV~UyF}*w9#CcT+Z-Lt3FQJp0{fDITVmSLSarQ-{L2>P2~bj~{`?rL4-!bUpn*^S@i}@cyKWk-;#l&WEmp8Q+B23MrzYQ3sU!y>mleexT3l!zF8yyyj!W zB4Glt*e<8wAfVjC5NaZ~6vz+D zbj8F9N0Ut4xm&Zrt5MpEmy98OdwUPwNiTHIVA!9#Ve0F%EGx1F4>c|diFc5nyLT3g zwTvVW%!l&(MxfCbOHX13eq22vz~z$Kkc9rEOn84w2jhz!;n%xp`-bq#$)X-9|HFN9 z-0=r`rsW34`_^ng*SEheR%Y(4sOw)z^K%pc0RK^RD#Kp})r}nV&29eLkDmTvxcSWN zBG_?kzkBs_!o}J#YPlNnZ<*BzEI+w8p$p*-;4-Y8ruclFM>E{j6HI>>c(Cdlx~QzI zEUi@iWb*BFdv+))N4z-eSZhJ2A8^rW5M11)FwmYo%DSt=CPRF$UX%PhamlyQTe}N) zQ10-MqkH7??S&Al|T$peDsC#hr9WG4T^<(xSuUCGHx zM}^2Ftn5OA5S$^Kgne8gKY}@~6yi$rOv@Z)8}`!`>k2L+4z$Ovbm9Ug!Zk#ADT6*x z{GaV;{c7!+cDI9w*C0JIaYR?Tw*OS&ehG{uB!dxf=(NQUXg*|0=uqHjPv~#tVk>I8 zEHxR55_n^9y_Q6a%G2W1_;LD(>ds)c1Q#V1Pu%ZFB9a&R&CPic^Jc)t8b5zK!;Z}U zk4(v|(l3+q;d|Wcp6nGDhxaSd_HxzwENei(uFvKAG5_adch~WzUtE%_n)X*K|_ zdi`o@+a*i&xCys`<+Qo3=s5lX^%4j79*`IC>GTRlIw+fZ_|bFmVS5Nx)bWk_G9&%5 z$md0`JLL9bE4~@$9<<})c>A*c94HGE+edePiMM(m3@MyK)Njj~=uH7awqvAR~ z9#vA$iMtQvnyb2S8EsZxSw&hI<=QPVq2gM?tRS$-v3D<*>SUC;ua@}m2Im@nQxAKv zpE;$27}5gDMei3qt@X_YgegAbX+fftFnK$F zM!OPKS~(z4Jb_x3!dwbhv}DIvNAkRld4rq-BX-{=A6Sb57+Kj3RKn5=1hW@#T&w^o zmr=;DzDyL5Wq8?a3WX^=5sOF23jGZxg2I-PeO$QSeK1enM_4(2fu29`t~Pk8WQo_* zup5a#WUK})fN2K(3VFDUWB4@wtpEdlM!1cj`n{}?>TR!&R4mA41hA31Um^+@l!!=d zdTS{fGsA=+V3mBh8Iy?}gDv+;Ez4~8M>TBs0f3(XjieAmsbVnxy)isv zbf!8+D18iUtWy|`T++`y8VUkb`LVW3WezK}@Qbf3*$=Q$IE(^!rNkH?@$9U`sj;8W>Ya9wTEtVBh^_0sk$a>oMy>=l2aD$ojQJx^o8;i2^3 zewEn4ptT5VrukfER>9RJ>1CGNu0z8>CN~;_MdB5p--V#$MR&z*YKa)dz|NT*mqp>6 zX6MX`^YXU>i0pRj$5a>FE<@>Wc$1J>{SNz>+5EVsg5oTh)t%42B7`ACnDa zh$8UF=q(8s2hdop1c_Bpdk(+)(D1F>KP=8y&wq5Ik7xc;DA2VBsFLrI?hHCkp$}a$@Q71?iKF9!WN<2tQt?AeVWL} zImurS?nhUQ2?f+)^};a=$JN3ynS<9yxl_ZVCT8HGV~NaG*D60C;BNeZGiHu*1R}`p58JU9S-*@`Z#y2? zN(cropY|nO4jQ$S*Y$n>=YI>FlOM4d{udj@^FO*H{FkxjKbPG9ZgbVN#Ti1f)(3~ z&+A~~<2iqNAI>1~XKUkp0_(VQmgbGy@um5E zEm`eO>E^W0*SPR#Eqa5^F)5{#lmIrnoE=dtg6IH3FHPO7n>gm!njv z%NH01GD!ECT ztJVL)L2z)7ggINamB-X9#RQX0v8^kls2wK2}tX+LkD`Mm2~-E6Fg!z{;oG`q{9 z74_wEO?A(+(xC}I7#q*gX{Pz&!sNQY#{I#CfidN^WfNr2Y9p}W<}<$O+0g~bbRW&g z(<3PQz8NP&O*=D3A0q#(7X}3G&;FE9CGdC#lXgisUs6& z8sJ*gOYu zSLQRsHw3KWDfYGZ*p4kFGd;^2P21-s?Edg_V0QL48h0AIsP_|AR&$_SZ2xpVSO$Mi zTD^OAyb8*-RM`sHy=wYzp_=nyK&3ZC8Gy15fgzkgr+!{b`^an$$+ z7p9Beh65)v;>+Xu%?m`nYJqMZuA=^NAB+D=7XSN!raX&rOi-P(3!w+yKz(jR1rcpx z+Ly((tPHd5hXr&Fm4;K_+exvqc2`?({vdu-OJxCFAt9a(vt-gHQqY&7XF_GpM8z*0 zI|nGqN5IJw&bhc2q_u*$9A;$1x&yKj{MR9TQIqn*@42d=IA;n~{R4_O5vIL#NlTX9 z;?qp^Y~FDb5$_NRO=G2_jGWbp9nZD4^`U|z^6`?j2;ORc1d@Wi#C2Q6vd8wJ)vzyz zs|WY491TY?r7hIY4q@fSgep`GvcS?0RoZ4cK>-~d>AX?Hlks>R^AHvfk@e_?gz zA%)%^W$>-yTqK-JZZ|;Tjx~c7$a#F9mAcO(IP)O*Q{jU0p_65N6-hK^&4u^01)DBS zad)weW^Nv|ck>KL%qQ>PUtk!7$}fNG3)JG+pI{iX#`%BvTh%ZKzQ8cOSBn~e_#mbo z^jegZ%wi+C&=VlzT756*)ogTQ1s@uBmsMGGQ5W=;0}*})mIn~!ACTZRTB4RPj4Q;s z)Jf?-f2Szp43KOe}QZEXG27Xtgc#&e?Ph+Nj3&V>cq+ zt4cR@m!N^5nP@Y>!{nP46Iw&*8})qrW@P$a{>>a*1mt#F@r*JTLT1!dH1J6U$~!{G z{UV-uLvp00T1O;^{4nTnDVW{#YxWN#r(3KP9}O?jzNeYKn`>3zdB?lBlbmB3$2M{Y zRt!R=@i!TE>_|%r|lUCGG5i!~Kxn z9TAN05j_W7Z}mbTw;M5%5MQW1ZI{FUdTARlH~^eaUGK5z*Psh8XW1C<+~xzO1P~d% zIecpNITb+Bf`v?bpU99V-%uDA2}!=`@Uq#|A(a^gG9goI@!YLsZ_^hQCbXLeGIlzK zlN#~xtAdImO1;W9gswS~AlEt-<+}89uIJuCi`Gu};sKxXPB z4=7BMn*na0T7&DdW+?Xg`6~8=&LpD11+VW$jC^;y)t0-!gQS_yyFP@Z+vAmo5%C+H z`Daf1dyZFE>bSHW5{l+8->stzsL)!oY_k(dJ8LPYCbP6tbl0{Hl4ic4Qc>ZdB!o_* za;Tg1NrS>w_Q3IOq`z?|=Iu@QuUznFCY@_*`NExZBfEMo&fD8$ zI#%XT;0+dNE8F;x4gcXpy;vYyWTF`>9A7^vrO|3AOE*DdQEV=C(JgHpBX?bA6!7gF zOS4MK`8{>|gHC4lT>o3A_$jpvhk|R$d;MLV2d+nxe%<>^eF)#SfX8B=#1M28phPbv|6sG^kFXHj zEz9>S8h0Th+YITBTD46`!qxKfG&r-viqYG+6v|2*{(m@8sRgQPY}>EH=(5Af!?zeVOE055kh+`GZ})SvgY9oV`EpiG z6^XrK8Jfr4gomE+>-JJd6BI8#VwQ!-yXyRm^32JOBj_QyW99{r!KW{R9$1Tzhw`SF z6?b@fPAwZ5B!NU-l37C2qqkS3*9f?+B0A6;1dJy$|$j$HNbH(hD- zP04%C%avAD@2!|3p1)R(^)bAjOK7JM!S%DBPrQA6Z!WxrXBNY zI3i4vO|w37d<jgf4V@A-&~Bei1&jK-{D1Vi6P~TvaI>VN)@|CG*hLczfe( z2xv>Gdeak>oAW?rnTfr3c%b(57!bT2TyI$a>O&)n8hP&fGU{UB{ilt(|8Fd=bM;l5 zwJ&z?q$>Dn5=ym7er0}y@N}U$W_?Tcm13~DVAd|G*nW1m=BIKSswI0`JE3VNX#_7(wOMf4(Aa#z}yh}pf?s*v84s^ZyQsv(B z5d$zn8XRq$LN!(4YC;W?x98FS#ok-T#ND=QyMyyGuiHldM zh@&QEKpnp^kv?@U1ui zOWf;k)_Pki?b4-v?(de#O~$8)UW3Q%BR~0JnH%+f=%@)i&kSjDq*g^_rG|&Kyr?8G zQM@EH%8Fo@|MZ0|l_Ji;R9M!M8{T{4rCWlX+jO-{fSZ7BnYAQ&#e_K1e%j^OMOZ|t ze&~_wQHW(}YR2a`#akXk600cw!0 zDD*~Y{X-94QB^RU(G2j7jw|}$3SkbL5s$}Pgs76?1|S@zcdGr$DF@uCiV`K>C{MP5 zv6O^l;uXJ!LK;`CLgM`FcI&Bc-rBAgpy~E%eA|dEO}8A{ks-@70{0HPgkf`0af70953!Q!DIII=!yB5?23ctId?QMtb?gbEx7F^e-!<1VY+gm%~YIItPW)h0E$f^q7@AbfKXku9{ zXqA4KP?MdsII^~}Rh=kC4J=Q>c8-oX_uy|KiKuts6ZG^Oub-)r*)&fIr1~rP zi}_FDxM;t*Ft7?6sH$i3*wvD`NKxpo6FN!1b>C$4)1w`8JH{VgPYC4nNzTk-%Jj(&=cFT41lonyNvi8KDFuZy3 zB%j)b`yDU3S+YC@GN}L&Yo&zW4ZIkk5Rv>KYyR4E`JUBa@~5=Loh%e)iFxo@ELh|@ zZp~b-HH1G06aivzGN0z}OH~GGblU9cCUL6}>A!(b(+jDoMm8Gt!+enrNp7C~^O@Q> zmlj#{;7<7QAffv9pNz@a|Cy=xherJ2Hf3gPW}xTz-@)}BTSt$rqyMX0M+Cx2xq+{8 zRibf*qz)9w>Ia)TPz!or4w5-5PW-{u3sQ^tKp3O-Sx-{*-7oWargrM*ie!u?^yciv zKYPN*vOWVyTmHc6!QVb$^)%v2?KLWV>EYLSySJ_4I9Z?;%Z6z~N{Nln+bQe6eVa}m z;n`_2!QPUsPLf;78dS!sbvN0nI|7%jKhNT!matSMQSU&Gd8Jeh^D>8~ZZBry4-bp)p@)fndssXkb$afX2< z20xti?jLt-$?R&*{yKabxL*=(o947UqtjqKy@c1duJyi}5rcNHU#)ZgWuu}beqj6b zD`Z-p-LPeKwZixE*haQYrW<6~V_!gD_1K^@KYV;{{p}nh>^O~IaOkkh=wAKF)k|gj zN3NbalbF1?>&h4Q^3rebsfH~F*UR=S!FzU$Nxy&(T)hwb8y;M}pK~ zxi5OtluyUs1qbp?oWJ};W%*LzgF**>WT37h3yEM)FIn=h8c*ks-)w5f`nKXWrPWqj zA9Pf^MvCz)%mN!}_M@iRu^hsFQHmc}-Z8BD+Zp(})digNz+}A?9DfKx@Oy|NDN@(w z)mGkMm@?M|k&3;Iz1}K881RjbOvNOI+2s*-`p%K04LU~NF*HND=k}dWMh#9V(7zn8 z3uFFl0|$Kcqr?{#{$|SE=Y+6Y+4JlL66Tkxp9c}75&PcG%1J&24`97mL|Ng9Kf!v^ zrJz*T0lakbC#3WziHVWvlqT8lGpaKSWkbide#irv(U#;J8YmHAG-97b4{F3xQ;_L+ zqD?zv!dbh2(C6Bgk{tg;njSw7HE2;*6e2;`Y8!;cI=wo=wMYs>(Le;gyn-u@m6q7g z6x_`c+~N}@BIbO9PGQMp3E$0>8*>W$Lq;{hw?nG;HyKq8DjQA6^pBZu@QV`U-^IC9 zgOC(Zp7WbXlH=grpghTi6^%_^`Pv!%$Q+z}UFaAH2X&qAyT^K4y8Yb4iCmRJ?kVx+ z_qTx}$l;{^5?9DIn@(TJ?D^pq zpR;+;S7x_*HFwSJstNbsZa&DU=n@mq1Y{m$R3qg820fHRV(o|$e-f&FKx{oA_E4p| z@ws)UKI#buVXFhF95&i(DKC_=uRt|!z*7Y+8ILKU2OOW%GB&itvaLa3 zyY`SM|5q7v(FtPkNRCR1MIwCiLRFKuEilx8|6gKE z)_*A)_|>6xxjd5+)81zvE6}96AlQFe)%u5yDrn_FN5wZf;xWqk2LZ2maO6Qp)l4lp z55`|l?hTl$Y1GgH1r$5SO}^+dey##KPyogAu9ta;ZUGbAV)ofy@{l=WcbN# z)}$x+j_H$#(JxfU-zjail)rBh9yBd0iDREAxaQiJz zZr*GTJs@(^p5gN9CXp@9^RSsieb^iZxt%F2>Kt<234)3rR9!jShxC_w`#_QY`93w# zp}0cz@Qm8{>d6zQ|LA?{PXOLO*Nzm}|5V&0O4$%_zYILQdlNVc3H*TG?v6T798C3Y z9I=r?XgKnNXf0i1vC>QKBcH>)>(6C?=;vsJHSu5KoK5P7&|1^0MB^o z0C;@?=Fbu_$<1tQyjG{tPbUg>t0KH^Ea5y3OFuaDNe##6iJz>52K&UYm1N*~d3Bwx zPnk=>*wlso$yg)vWRGjpk;@dj`QtC0LniAtRFiz@ccg=F@4lU}CH3OlD6ypoBnqm>z@)+^uj%}DjuYEX(--SY zCvL0q@LZyCzi)u7FYa#iX&`$;qPbXgh%~V5^R^201jZ&-z0wFq`Ic$R>Ozv#SKKNa z$+jHq@1?nxrR2fy&As?jtU%XbbKWh9mdN$-ql1HG&72lKLA*Zx?3kFIma$CwBTFSj zye8-v>TMf%_cLC>#5->Dsofz^m1J7tsP(#qA$;x#;6;kJ#N{1ykIvKC0vnO93DJZg zo%VW?_gz9OxB+OPZfU*r>sLqkbjp>j4z~fgo?dWqVaHjeB1XLO(}C(RGFN`MAI|M` za>UCddk>C^@uZ1Z{Y|UaBtH?6<1|>Wb6xdl$x7#Z8TkU*sWR}_r(R7fl65fYI+!yW zy~B82JTSKz`^@jOQjcv|2W55xWiP0sEV7U%BF#jMh-?e1`TMSXZ)WBt?y+@6khcT# zjKQ8xffNNx?(-rx8g&Bk1dEz9C>7OFo}6t26<-S3ocgSZXEKjq#tgQ<6zQK0tTD!ukJ8@ z_#ypt&cfu+VS4V5PE>)Px79Q@Me0^3rg>kvC6r;of_-S1tR?=2R7pI5vG$el2ab0; z%dZ{FX*?{RWT{ep=wH;QndsNGW1QAX}V}qXrTJtRG<7oPR(@In{pxAhHxjWuZE#b;3G;A`qZwVC?_{)DQfn0QbvZ8BbJO0Q?{0F6In zo-jVQ{@6LXkvEzhiBINNi`u9UZ~2T3@O&)qhk_aB`s|!y%4_uDm0f@4HB9~xH3>NA zEnldjeNkTYojPl7pw^e(9N;aT@T(~&Rg14L$)}272nFH9s9Qj1!lMjZLU)1GyW&RO z%d4vN7*1@GeB=ze!1jKGH@H~Bwxm*+M60Lj9%|*ALuP90T|CF9S!6P zK@G`!YRo>CudNFKlTkPBqnq~Cs;L;u^4tj>`$?gbo692lb7!q(H(K8-TzE&TTdP~! z|A)S0*njlk`ez2wzw{;b{&+BatV=%DB_HdOk9En%y5wVB^06-YSeJaPOFq^mAM28j zb;-xNC{x9m1Fh7xq;=iJO&;RH|=TFAuKaZmS4I-jI(D%ywqFi=+ zF0oO^b4Q?G)!8xhK}wxEDBh^(u>BRSSPOLUTsZe+X|g#wlI7LQe0dSqC(cdz&H4GT z-K6v-ZK(V0kI*rngvhj)aqd^cO_rsk^QC(v+;03EVY3$}X_>72)+>)r#d=Tin97dL`L9 z;f@G|wgiXtotBMQmkwzSR)jJtDTerJFK5U#!OB~XInJ6K>*QS_?zQ92rST#WIl~^2 z_UzlXs-*-Yc60EAjB;Y4Kc8*puXhyW=%J|RZ{>(WpB}m??kd4bY(e{%6Rd^vjq*`6 zn)WD!L7BOq2kn-z^uDshBXIzMbg(9Dm!@RI+LZ>2T(6LO zGLN;7ASF&9-8eQ(pl4Ynha@70RZnVa&3M805i3fNhnB*%FhF9>x_NpO*CtGHgaTE? z35P&Jy>NOua}_7-)Qx-fXkIDEZ39E4V+C8vZR{O5QwC$fmUj`8iGHKDO0AJ58H?NR zw<>QN7`(9$^kt#sLNg= zUz#3hLhzz}u2%ZCo>&!97Iv*fm76-fF&L}cm!`J%YyuhRGKh!aL0V6!6J|7)YxHmQ zcz7Ti=0!ckc^wG@+gva#Hb91tYqZ@>*RPvfM@Ke1Zvhf1@8mlSAB+q2W5$c1)n!{Y zK|aAN7QXwNZ_xY)h{)>XZ-_|W14Ja^Pl(9+zd=MeA0Q%^zab)@A0Q&44-kVo4qtTT*)V6z~CI zV(Wc_HF|y45I@c810n%C%OI}GSghu)E{`|F4PyRGcnLLsrf~b)lQ;Sj;r_L}UfUZZ z1#C4QHKY2hPjq2d@6Y5fbz9|IG7o9`bw^ z=HGdKV)I{lei~9lW&sX?tVSnaH;^Cx9ZKG7vTbuYTKcMMdyHa3-{kdCfhx28^XA-G z*jnwvYU_B0N@GRulE;e_HKJv@-Oh4=%ZQPW1z~~<+(@244h^{pip7UDdDicrJ{vH8 zjr6UC;FiV3;ZqpNA#VB_GwH*jsS@E*Dfm!cU(Cqhin`#c81%k7O?1QkC=Ev%ziV2m z_o?|~&3ZHzCQl$!ziH!JamwZIPe)to$PklQs%aAzs;WVH5f+O?f8&UB{pN@;rc>^S z{wqghAFs3D9|_?fMR~`%m{PXtLi}Xi7wTIEXM+y;CCA$GqX7@1yw8a^>8rf|BFe++ z4MW+O`%{#s_3xrQSAiGr;j^YhO(?~%WBu1ck6ejO21ywb!Q+LJNZY_F4W9(CHua$Z zmCVYMEg|yPa02zsk){mb6+ahD`)5~P zz1PhjAQ4yPKV5lF>B#!3{=aaA3lu$N+GIu2tuD(}9xQprDtfYnaUx3Y+LE0W zTDqrI;e?hrD=j4s1RhsGI(4YJ+MVJ*U@#}A`#oz&3=4oR1YUUI7sLUhHKIyG;s8J2 zu%eWo`M6i;Tcvr*9KjzT5yn43B0}xf0|mOuTt@<=&6FJNcMV^jwKOjgt36+mnfIc# z@u68NH3mNf`kqfuE(lveku)KQ!FfVRGrb>YK55jOERN(!*#<@`MFdAOlX(uQ;#h0z z#E)XczR`Xt(z}&p(3wk>{uUW1jzE^Z@c#p3fchkmqYNd*1Jkj4v7RN)V@& z?h&uK&7H4Y^P3)KHGmt-!X0kY9iAz{@iOiIu&?$Kha8~9I5&T4%(GjzpI>Csa9aMM zwa(F1k<=+td#hNhC*8OU!Mv^(ffz2lcowLQsF6gN^cy()~fgQ9ZwF(2j{) zq8_X6fGDNux4>;t0Z$B_U0CZHW>l*Z7j#e`^yX9DP?fb%X4(TsEP_eP?)Zs1>*;QZ zHKu!j*D9!a*82QbjBx%`Y@uGr!O1JO*|IKak?f>+N$2)o*r)%95efc}UZOY{^(-wJ zZHEH$)|RT% z6f)NWzHp#oCbkAc^!wrWyau;(F?YP%$KM*3&>J?gF*5|2Ct-fex)aO@gF2%)}luvBym8F%x^t#2z!T$4u-o6MM|W9y783MJA@pl_AyF0Wn3%Zq=mbSMrtpQvwiE=yfV zn1a-3wc9m6FuwMJ%*1#T##e!Yfzad2OR%R>mj!Q{difO&?do~fI8dc!Z&TSJfSb)S zVfPNkr}Q~IdsUOqoYs-Jb-Su(niQX;6=Ox5DTS#%!Yp=wO*z3DmKcJQ~9;r6w2njVE6JGS}{Y-CZ8Ks zc3H5_j&613yolzYvn0fC>Lj2yl`yBv*X(M;1JlL+#Mg+h`7(X(x{Ek7#I7v#C{(#Q zB#Rlz)z(6E0c_t=m_qvlL(t33acp0!Lz=1wztMmdYc`dy_pSSk`nQK2jb`qOR`y)l z9;R6uxDGPib>&b<>{|wr98L|-5b5P|+MG|kwQCl1QNRA&(Kv#mQI2uwrwtLs(LWtv zS)!gr`juRqqv>~Jg6Ti#Mw0=zoz@`no7k7x?ZkU_v*3QHAq>eA>wTW$ z`|20L*PkZV@vTh!zN7e>`5!={NAtw$uw~2DsZ1f;1EQmW${bxUIm_uX*`v)J8~MJw z$BHp9a*lb9JQ%>Hx|u-c4k)P&N=Jzj^q6m1L{3R~rG7bx@ijcN3<{fd+xhXt83I?V z8(h|v_nfu@+6}d#2KPCj^*iCI={Cx&er%-Qs^Rt>Ar=Q z6K(8ZT`IplZGB=@SYyA=mEk!X=AdJUbJmN59meL2qqKT>_=D-xQR4R8eiN-dO<)^)F+F zWxuk1+hQ5P4m*}B^|P!W^43;%q@qS-mx6M!d4hxBYw@CB3#O zm~m@eJ0D~l#f{V3l@xk7Z4 z^!(!b`}y_$!)HcYdm9_$KjzL-;jHMLEH4pYEo2g5sB{!f$Z&67tsvoeJf#|)n@vFW zzvQQ)6>hGcpL3Cme>00`fZQcUZOX!>=o`eR7!w(%gAt~nTFiSFkhD&`gT zuJ;v^B$^b8iGmL{Uhl(kJ=EME&%JbD< zL@){71ooCs*0(7S!+ zX4ry_PU|ZNOPjm8MzHc8boW;u7Rjs$ei+h+yQ@vOJB&%!d((G^7oV9)Zkz;$Yf9Sh zXBTHdb|Hg=GuiTb%>W10QGi zkF)#7+5O|}{&9ByIJU0f2OJl(8wwXb^}5fI9Zw@*BNu*8ZD%>~a7)7e&%J8n(LAln>f zp)1wBYp+GSK#4h<2{KO62&RxR@P4ZTqM{U#ojd5A04Lz=Y(7HZ*c+)}BGrlQH z<0+SdDy6(wFkX|@74GV8)JO6z8!H$ddFs&Nb zJ|eu(dd8-I6P*)%)D35%t%2z$nK&0aq%}8pUC~!-=nI&!KF6p{IjmfGV`LX(^8LCv zZ{e3>DEy0RCcJNm@*Go9=Pw@oxKDjeOzBjgPP`0#zLA~pTJepFhcSZ)b47#kjMaU; zQDNGZnRc_&9WRb@7!`O-xRE;kiPp>?k-RWa_ls&T%4#pdsU_-q$VJw5-quHt7S3S) zx8ghlwbkA9#ADh5#DQ zfTboc=(hN7L6CoCfo*%mm_gJ8Q-3CddWbgpb6ff_3aU9>B5g@Z8Ufz_L6E!fmmv4Q z*=xCZ_vcf2UqVmUHi`SofEhK;6Yr*()ujPSMd0XB5H&UgQAMB^# z8R_5wO`}a}bVu1iOE&@RZEum4u??kjO}&i^kB8uA6|IL?|byoHtdEJwS3iQ-v;HODZF#gP@ciK9_Fc=G}$N?a`1_2riugI zSaPoZ7_P23)jgTX5G4jQBAk7GSY;}G*BbLrrh zU{S`pUSQ7`N|LxtQvAI3m1U$$b->==IhAEBRU?B<7=|AUTCZyXTT(M7{Y-zlDV)P%q zOz|>W+L-(qd+oJ)IFR=x9bDae2S4XuA>^;CPP#y)CN$}*;>ae7$cdY;_OPUXsiveR z)$smFZ?49fe>==r%N9v25}}{&D9X6RdP$Gh^W0Xyl?3qq&_V}s%91vjO%$g6i{z_*SI5Qx_4oHFqk;L@raF7mwOaP7e zXOqPnrhtsFeg?@QcI~)6oqgW6Io`(Lnd1>1)!K`B&;_w7afKbO&%EA>ShR6Ng ztxixxv0r5Nb7}Qw7>e(16WVb%{6n0MmV9nOB|ZcpBTM1;*TuW-cfkr}DB1p}7$I0K zoAPve{z^V~yXBBusEXea?~9_e_S=J$!{Uq;yu3Su0&l`lHh>snb_B`L-5a0)DoE(K zVA--(w0thRex95F94^|`qn+)je#@7Z?^HY|SO?K2A*qLEN1hpA-jc6@;-^e9I$3mx zR1TL&yoJv_e}WrA)HL z8`_9-eNg8ZAS&mD2^Y70OCwZ5#Ti9_{V?im&-6GbHjpJ{_`?k^*Ih(bo3caI<rJ+8*)Hf^x|IEjB&duaw+O7~`(c28@0Ud*1k+fSw1dziSEx!)bjS>WjchvbT! z@WM$ncg(NTLu_hu_tDbB85#A5Y>%lNV?@Wh%Lg{2b9*WoHb~by;+%c$a}Vu1bjAl^ z#KCGNs*p`V=rPzMEBg&USS)FqEVV>lAK(!4=x2o+w^rr$@(rZ4-U}Vd2)E4| z08vQF_Qn|m^2&z57b>;JWnoJDH1#Jmx8E>QJPAW5Xvmaiq^k%3H?y%&iH|GRt?u=r zbYfm(@s6IUFmZ^+^L}9s^r?k;+JoXU_xf8r)PCSJ_{pzc{qoYgh9bvY##aX*Pg%YUlO&__dP{YRtl& z_89UslcfRW80SEvZTT`OSNwN}sV+VCZc z1#Yo}&X)E0HYZF`%SK4B?I3wIIeJk*bdgZuYbkD9K3PJ(cfkQRw+B1ES2A%yyYjg( zlfRHO!@X+9bVI280GlGCh5dCQfcS70hkFENG-qLqn?w3!kCpWY6PaZxH5KSpQ;_yw#(6g zdq0s=|04ir=dB*#02UUdj|Uqh<)C|H`7)zwEq3M#z8KQn}-6@tF<cQ*V;L5H<+Eerl7h0t-3GYqmDy!D2ss40hefMsY}?M;Zd=;VG(+3ZPFRjy zKwKI&gRsfDoQ%=pKu4|14oBu*O&w;bHWB_*;unY`WljP|F2~ko@km6r8;LCigDR=w zRN?HXI>hn^c#T{btFE>n zWo2SFg`pw8U5;L$wf6-e;~f5%PupiU03q5CJz$k^Qq3|n!{-9fXg?!xCwNC`+}Y|a zbPLuA%&>V%6r6ESIoCSAs>4gzHZGcGaQkTvLa^>(b9R|P7-VxMx2f7LS9$Si9YP>@ zS39^&;5EfNZKHV5DhEAT*c zmbH!*Zp_N9U7Y&-t+|3BY{racPS=^cOy&mKEmM4fRQ@t3r*Iyb5O8mixY9o;P(OVhhr zIa$p1^7bkLp#ixZ*`+Lh%Ck(A1k9H5m%*E3z760AR%*A^MxBrBu zGTDzX%uymFGSeqGvrk@_3^%5+qMAOj$%8YsC|PjB2t+(m#{ZJ5-0c(L+ip{o1*mrK zmtO}_td5&j>?=;ExUc6K*Nhnu4aH$9rRC_vBNnUKn(@Z@0I6?*dh-Mcy{4{sB*FX+ zdeFlLp0fLU(qVJz6=$JDsJPN)7(M1^KqH=%i8y8>V8xB?`|Vbnv)sX=#=+UJab-Q; z6m>+OQiAzGvC2kb>(!V~xDPXQ_u~fCi}91JiyC8KkHM8)`C&3QK7jLsJZ#eN7(Sa2 z(Ia!u;@k3UJpM4S|+G4W;cxektxO+YD%D6;AtCHv46|VSlX$~{F z_wTbksd^~uEeLG%?Gg;~V3lUv&8E5j+WVNm^bzV9SKx?NtkOuKrSu7{za0=mf2%ks z#gj?YEwBV)W|FYc1JuV*-pBkL%4nTaa<|`F6r}^zUPsG4)rTn`Mjf6sn4CWuLdVd# zVoUZkFS7OdP-kM%Pn9Pnx&MtRI*&V=gEreg&Kc?J*b|8K(+WwNSt^@)IMXcWcF?=q z_PFbMx;*GXLks=1$9Z8%W7Q`wHe?bn=X%F6?4`*#wQ=3C6bynPORr(Ot5T5FoV7#| zV-WR)HH?Hh1g;+`XuV{o`bvhkp|Re%BdDnMZgO5Oh%GMUq&pTQOyITHM+)EsB?761 z_A#!jo3F7ldNNB(DU3kRJc`NPp%$n47i;Sl7xp#ETprty5suaZ#LUL!_{r5zhuD+N zdhu(dhbCvXL8Dt^x^D{8ql6nB;wnHB z@>wntmt7I}*Pt1GksB}QvaLL}u%)P)5ucAb!ATvcFZnPup}D*_k7S{Bx=m5rRbqrY z68i13XE#*t_8Tal4@3$ohnjeT;xEE%UIVpA#`ct^-;3S;#HcIwf_i z_FyKz+Zw7Uf?WE{5NOZkYvb`*_f^jm-S3v;=JMgjM+waM_A?-?*sp&dm5(t$^iwb+ z^4zJsD}K+%o!xOAnbwQfPzru3acI-Au;flM7oD*BvV2-$^U{ynHdN5k%4W>?$*%kv`B^qQ zG~4e`dhm6gq_B!oHIy$e^_<+f!<;Gdy{c#U*-zB3U)wccuPo=N+wYG14t*9CE`v{8 zJhX{96QsGAiBOM>6>$dJy>7Fv%?Oj3(~`L~yM4-1aH62xQzM09lc7ZQoq=T3c-7Hj z%Z$g$cG2sjvODrUBC3G0i_6t`g<=j8{eS_(sg^Qzt}4NZDu$`5BXNhTKEbtYo4S^) zp7*EA`FgQ_O>prsu)PJAjI-jUA{%&PZk6JMu~Rd*7!n-Dt)S-1FCW-4UJUYqZZ#>Eu00IiO+(AAlQRc!%NYrUdDX}}i}Xkns<#c&o6ypR8w{{E^Z zG>SG2m5$d=6OYU(IYu$X*L6_Iye5XpD4_>8?QC{1o%99x7QT*LYZYNEEd6VGWxa{* z4hPt)arQdhZa(!j_$k=H=R!ABt?hKD77AW1k*rM(hBA0{HI($X{_0jI`o=+($H~X} z(IgNR^S-E|@mrf*`b`0ucDy0G#;RP|sKY2^FuWfR1U{7pira1lZ3QtyFdB0#O==`| zo0sS8FugFnj(cD~WR(}nn(&d9YgDAS?k^#iWJFPP){I7>hJ_>FYs}sX(R|j-3X{y8 z>D&$3`j#fMDA5IU0?ssK;EZ#Qai%YE=KPr0AMT#IF0(`E%JRwdNzL>zr5;(wz-imZ zfMw4dyulmcb$O39P)N$Q=^GpKBC@|WfJGf?h|*nz-aTNP$W!J_>>kYv-fIQZOE{sL z#nGmFL}8KT?8RAV-`NYw+E&j%b5ZkU33^#oar;e*D3JpLWm3Ydubax|JxUFs%LogI z2Xm>*=qa7*3T@5bOhnxw+uEa zh_M=TJ(4l9@pbeDF5~XFomo?YybHTsCh5i`RSs(g*Rv>yF$H5R`TJ*ss2zS?{NL*w zQHPvW;19*>w{TT5RUC6wRH@VUAzL>GWZX@kM4LX{%&<;_j!!fQIny8N=-u>-=Q=qq zo=ImZaZiX@x4#{k9-&hxx5F6PuH+Is=f7%vJ#nr29hg|uzzW=p*#)rl2lm?&=sQE_q@K|Ayv@Y56m5B2`6?UIW$b}{~ z$JM5dL6=%gq`h-Whi8ZgF9J)0QBx7?JZ> z$DKf<2=9Pf!XbSItLVH4Pd)#1q4rgpVXJXe((IS*=A2|B@(4y3)w-w+))jV1Va{<8 zX62JUK_&0%>&oPp)CVNZ+K>zoROO-|FF?A}^*D9G%h*c6;OZ>z10Wp|YOLY)aBV1d zDSmD5mcA}B^j&mU5D?hlmtn<`GjcgjKM3+oAV^Ve-*FQ?tafAG#h4tfW6e9h-_^Z_QcJO4x?gR^QK&57h@{^*XrbN;Oxu@t`hvCx z14xw_tE`)vBNm6@6#{0b%C~3SDKxN`Af?b%X6d#zqx=yrQJ>>3r9%xuRTKq=B5%8y z&#>x&`MacuuEOCi=s;l4(-0(GlnMk@cm}xmCqwm5<11WW z{=>`ljxALM)cJ}{F9MGV3760FJc;2lkQ`B(5VU-m-X~-Z)k`HDx&7li7Pe1I%HiB> zpAE3{3Q9PTyR;bz=)?Q1hay0Aa4g0kfV7Sw8AV+tiy!c%sj=sbGKym#FN*E;HXL7y zK{AF9#1yYR(*sawa|f2RSg|eB`9_})qMDdy#pwpvGU0^FM-4FCMf0tvRw(w`8;O*u zQuz(MxrooW@|#w?Tr|{-3@uLWe<{Bb7uUU^mgmqR-+LqSVpCljdV&wCD@mi4lAes_ zx6Vd3D+=9(?0$`rLWM)~MSSn4t>PD|*W}?hCc}?}k|ghDGjv^a zTpz6g2yK^&rdy$y3nTGfTrkECoK62!U1bEE{pao6&XsRTI!oLe6@|k{J{Bc;CsXy9R-Fl? zaM@`nt?}XuI}K<<7Da1at`w~B65>wn_ur*_wm@AW$L{p)3tnfhtWuvHJ*$50S}@_5cKwJa%K&1cbcDrC-AU7LZLW> z#faDvBydc!8Y5`}s%W!cl36ahIn|<^xsSeF)|vngVWwBYU3KW5Q4+HgP=YeY8fx(- z@fN1FUIosrSqpr=t~L}JHuyk~5x(hxb00|}prEF|PP&d~eX}Wh$BO|<6223lx$9FbN*x5aOuCP7^;;JoTDq4#ObrbLedtD&j%gnElyCil7wRd*7-Zf~&y5~q zmPiS>+s?;Wx|s@*S%C35Cz7;ZH{pCPx**ZHKZIZ`-5)QM-(4IIzMnn3I180uD(VDp z<4$gP4He0B>$o4EyB~YIugWQ=&hpb8Ghg5wt>BsnIDLNFD#B|F+W>C9UO9Ka5{8mt$ z2dMczuW?^1?HJQ#Zt$QaX{;$5eC5bX4Ypg=ZYKbn{G=@IwRa}4uC_{_{x-Sy^V>AHZnt<|sKCMg z!}N6AGBynHSib9Iah+xBxG5~$+w)nPQ*5eQ8kz3pLHymXsLNU@`gA!L$+0~v$Di|mnlpqCToGuqX=9nQ8-o^eG$iq4;fGeG(oE`k?! z5~ln}He!wo+$=owEm?5RNkd~biLMwIW%OiyUdCD1_#v6M@ouTb-TzRsybSO=%#_5*Shax$uKPv-AaT%bG=oeLITnt23dsfSdr8U z2G+$78EgOQ;i!SRA0KemxjR@oe71;k_9ayD%=B}-gEx;>W!1t?B6`Up)s zH#(yoOABU+J}$U70axU` zg~18b(wn!gtCe3ol#T({rJ@do%*ryfqw5@t3ot94A8CRkrVvoNGhd#t=m$}L2T^vm zJBJ)Oxw^%5GIa4S_yG#Z!$39p(qECB*u6O%G_Wp1PVrG?=0#1Er#ULtzE0-*m`wKksuq@_hETKI z46QrWE)0uHJ+yWlovy=Q5?-p^7$KK>nC;F?D;Myun=9>h3=HV&OCd4`ZnSRSij?zb zc?i3o=s#iog8S~pTf~=L#f&gma9%9-weUvwSL$an&;3z45%hdn-Y32t<`zpl%K77E zPrt$a@h->ln3Z8uQLBqKQyw0M;}JXs!rs>`ScxdZ2*&X2Uz2`H;wn6R)xW*`3vRlR zWGjTNDV10J%^`%rI6q}N4`Sh7TQ>xPtr=U-$zGfUUu(7UM^X266d^s){EzvT8u@F}C$8@{ z`PR*-6W#$o0xjoim{a-9AHH7=vq|9WwP=+wcUrm4jxf!LgLteyZ`5#3rH%Rf#H?pZ zjbpTj_Dhy}F(&azX`K6f6KlXXL1Qb-^*yMAG%hU;jCaqDP)F__q?lJ#=vk>_O&+_L z8J>{sS<}t+CTgAQ>OCq+pg?{ajr~F60P;vIHQNZoF;_OlU9v(h5ofe>LlTL$`Y(vS zH_wpkk)SK++P!PMMCWXXU>=4KtK=T-jJxoITpFYKQC7Y_n>Nr}q=bH;zTLK;t92=U<3rX=<#)FkskojXX>tr_9kcBz zKrCfE@Q@04WZb+q9*FR{-tOWCK-*woq~IXf-OA@qDP)2|SyIqom_7j4y9;uit3) zu9h&ro!hGP(-E~(ggPaSQzxDaLtk&--Ko%3leCx#cNT9BuiAKqqzMKE3+R>HNIiT= z8_f1je!so_mZuruwY`B~X(Ywxvezreom;f~%qw{vGE;uK2Z9_Hs(Al2 z-7;@gwzwy5jMRcIi?gBm`F^xYqci}d9VlAex=EBT7B6l$LYth!)~{_(`NaA+6`|8s z#K`c9n#xxE(qVjM1os|CerwYht4q{Q%-V)$#Z7G}>n@5=A*5!z#>w;@YxISgFdUqD zg~s)PK{{=sX6Bs)52aeo+pIa*yEhgyz0kXcjh!%`)+ybw8UU){esN3jj=D*&x{W>> zLJ!$=-m@qH`fnJquYeC(fd7K#Nl5p+fdoviNpM_E9 zYtJ6Le)s;&PcHf;f_prTD_G@^l)8DFL%MGB--CWV$teTV-|SBmHRP=jZ&UQ zrX|K#EGCz4GU*58he;bKy)btJ%>I?h*JPY^oE5`)7?;@yX|uACFjM4k&-GBipl&;M z@--G~i*(-_@642kguKn0C@v+6S~AKcE;gs(&99#whq>ojYkT{s{`zSG#hFqMG}~JU zlSMZ7TsH+4TJD8SrEBN6Q-hFrVON%l-qt5(m2HD33o)f-YG^J@7YA|tLe4)O+yb;6 z0!qCb?OJj0pqBntW?(!$VmTLrr&AXyv*LaEG8a2>;3xn2eT zNYz;Z_}ox^U+4awlF-T^ei*GTpNt`JM@b8S(zLG!!+MgF&nNFP0lPtFH(58XWXE$Q z_q+nKs4*+E_=&Gu+kssIgauCvHPn%&pS%A*yu4*lT}!krdviS`VxeOtXFA;uW=Hk0W?U+2F zB5y36o|v0asrBD8$?0+O-z-(1s8LC_)Z*Qm`(kPONfX59(Q=(`ohI!tCs&XQo3k`q z*Tzy|kvG&=w7iGdjmXm5;||?OdHql_S{bI7v%V|>qfTO=HpeoI_Gc12CQ~a${~;;cEFzVX!VdG))Xp^2y8wl*63sM1M>J4gj^7 z4yqZSdW4CJ=WuMxYe(-DhOi>f&usdB%xW@X+_>c#PsG5Uqj4P=zac^39zp4jQP~NT zJ=1k7we%2wPqdpiYlLz{R&bOvSFh?@F?aLC*q<$2u~oU=DE72IRP2cE_RixAvI0x8 z-)fadt|v}HQ60)-iv`~edCKKhOaoH?JKnr$W#zD-w;5}N3m*3QIW&0tpNOcQX?8L5 z;&Nekj7R+^9r@gjFXwf$YnM1n{lerdT&G+fd>El$gx*VuC%wOZJkW_#JDf9hXx7eT zU9o3ML8#5@TZ|W`f!NW+S?egLEt`Kdw_iPA$6@O2|7f;Bl^dJc4_?c5ewOc`U&C~R zz3zP9d>`z2a;1`>dNHq6bad+iiaIin;Bv)F9ExmwC;u zPvvU`w6@nsG*4cTk#ZQ#AVs`(uxBcG!irFlfX7P&Gg|)&{*Hjwgc6^CBCat54ktfg zj*@~OuzOSSIX5Rr8Ab4sPOLioSxZU{uiL%vD_qE9$)Og~p*W~$G=ETN4!yZ^V@^6U zdmk`!HIQxO0BGJx7cOra)qVfN1_Zv-z)K%yPDZ6|Op;`M$?o%5VDhm zHoZ1LQo&~DB4a<;b0e5mzO*^gNxR!A9ORV$7hA`aUDxzLQb_y}&NOvFU~kCeqU$DR zzkdChZt~KP<&tLBnUav9(@z>^lf$i^*T;fm8XU7tr{;-^p}UMXwPk!0Z$28W8O1Q5 zon)8_FDE=rm8qr3M|CB~b2vgMM(2NEA}kpSL{ADe7x$e&^RgM3j@aXQtlM!LiM8iZ zv@gLP0M>vg;=5GQxQt8h>p-sO?(u9@>_)uz%-HV6w};to&8>zVZewL0jfFXTCJ@4Z ztNrGnpK3n}poE{V8RZCbIWw~|x@^X^1|`K^Sp%IQc22n9VIiCPY|N(k9z`n*WyW{3 z@-wb?$RsaRKoNG13Sw`S6CDPs0_NQDjjn{5$n}O`do5G%S~?hUfJg;46XaHsE)6S` z=RQ_Fz>>MU1&q93HR{wRjXg;o<*5~)m0m_JX?Ee}NAugp^i>8yFNc6GY;1l2D~4f( z%hCoDg=x%jWZ$)AauN5La0AcY$|2qzz!MG_UxVJQ223O4j0qMUhCAV&78-Es^PUT( zY%aUPb6-dYqnB}NUx)1ITpmErYM!a&1zHq7>^Qu=UzH(KZWoE`8jn5xBzqv=wH#qU zgv=rneAP`{d!u+Gh;}zwhTb~#@Xm8Mdi0#qA^L`{b?PX|Stje+O8TB}E9MjKCfV#z znP){qo<8GoM}+daxVfih|Gum2Aw$ca$i(%9ay1-Tt+J;_Ld)HurqfjbF%_>cab^`vnHT^H_Sr|b zW{Re9wq-V9-GE!&;-~YB_M;-77Q6&|da1`o6VH%{xU!hrnENh~*A?dbW`!+}d-@ex z{lhTHczdn!%l4XN%Uw!0w3y+AP{x;{kM-~1@5}E^j#-yvEKlCf>*O}d)Ly8hdBiuf zT!bd|UY4%95@N;+&mESm*;MXuWww9P$){JiwpW}n$r8F!xUPz+76>Njiu@vaBG2c+ z>-VXA7FUWtGzCH9VOb!{X_Z`d}!6^Cn8 zX_~q_^jMy-HSyfHK5^v=^Jk^CirJw_gy=$;8OQ~pM8;@TAH`&?3`)o~V3N$WL4JcZ z$Eutp0FBW_7Jp7cK%SvmKD7cG<$RFgPINz|L5_Ty{48J&DpF09Cc6f8cr2;yv#c=* zx@OlbQ#~Yz#VCXuv?L+?I*TGo(k8l+kVqOTJt-1hK_6-E(q0?d95y3ILI6uFSoa*v z7fk>*^I!pBCu(GW?^z3YtG(xUxxWN9fo1|*eb5Q86Ejb%ZgMqoJ*2|`l2r*{Y;h=d zVps%bKYvgrB}dk%xl-6v%?L5mT>Hvzcul-?X>}jK$=dpFY~w0Bbj-hV|UeA(|8DmI)yfeCS*epbk0FCwL4bM63!`j7AXO3 ze%Cek-xxJ_{}(+KF@Q_q1hKK*Zgs` zBF?7$*G%n|go+a})PQ8s5^n4Ep4~U%r%0Ph9Pkz9+UWi>&%JFRt{GIDYL>C#;&COc zs!8F&P@8Xm?%5RQ04MuY(H($Hz^ZXJjG&IjEgnINyyvK8g~&*<72>f9YjHa%xkHN~M)tU26R11k5vQ7G2_L z3ur*?Vxi(+w*+i6>v25drHxCiB->=23U*9O2K;K{>lQ#loIU-PJvM6vm!zFE+X=Xkd@wg9+EH7eVz)pWz=+ zU~1ZG`qQ1a<|MNVDBvpO*a;PT0}Y$-g%g>l6*egxKh0fUXp{ep3Gt#F2F(YT;uv{f zK_jkNcY>;y|9Dff;qUOtsIdhfe4-}C9mC{#ia4t0lFtfCGAn(?rw=a>yU+YaF4x{! z+{uj23&J1lKA0!oEDYb@Re2Ui-06>H_=)V6izN-4Z@p2b4C5>&si0oZ?v+0XJt>fx z&Dl9N-qWXmvUk-KYb$xmVY{r5J}ixy)x&kQE-uhg{S=~wQwvPy(z=fg;7{~hazY7y zq?6#Exvn<0o8Lt-wYwSrrtP~=8n+GB;aQV7$z7+M5wX}MmhO7fd-7xl2NuTb` z&Dc%HP>*QfAX5|>!&-$Qw!=U>-!dkeAp5@pcd^2x_=iMDraM zoTf}humNW4%n3;SYDn~K(-L)!pUKso1D0=u$0vw@=leOuKkw(vDorFEJ7%Q< zgIi@33}T?aKbX_($|M^cUz`q8NtM>n1#{nk8Zd61i#N5pP>aUh!|hv4qV4Aw-8WF) zZjEcb^OSET{k*4Rq3Y&ZI~CE&esde2#yXGv(1obHXlea9N3tUSwo8-lV3^eO&XAWC zh!JW6u{|W5(Qaiuvkc0*cH^~VwmLa;K{9&vk_`n#ZYLGna&E}hT=|AtpuOrT;_H59 z+HfV`FgPPv(pSk3r1+?jZ+g#NdMEmWvXfT(#}1pBXRjRIv*Ma@(^DkAm4qRMi18L{ z%Pv1@xTY>Y`M1hzJrgX(>ap$^HLE_l?G9rn=dgsH)lD!-u-w1ozf$~o`x%DbFJU^z z&?^B-@6|tz$?M(*}bE6A7kZWUvt> zuP2;xXf67>Z6^?}-@3u@LS%T$wB@bHtGJ}O#{0x;iRllMP4B|VYhT_Ll4Q)#y^izh z|DgYk@e^h~oPvW`X=X*YX1Cn575wV^zbhV7{QMt3Q|M`=QiA6nK~jSMtdo{isnp;v z%-B(Km}+`0z6|dOA}<+Z8^S>$bojTs41BIZoM!j z`UDU(v|m22qS)Zc&J74?3eF7;ZV4Hhn+|V;7Yty)hK2`)J9FU3Q+0FI%}upUn{TIW zr_s%GWn9v_0bmto72KoTv_PC8!3*1Rimw*+{Bw}$X`^_TOB!!o6w@a2THaqb(yJ;> zRAzbJEwU{i6>xQRDJ?%;A>w(6CR356D2F>w4&~Fv;ys?fr15-dUp;dhg$TZuaf`p& zY^|Kj;&^B5!mm@jx#0YrsUYxt5NX&}sw!6X{T;t>it<26faksS_{}$pedqWtvZ8nT zv)sD!vw~ODRwBanScLC`b6$wgR)Qd`Bn)~!S&1Fschvw?)t)^ak4w7YZ_sdkKXg__iml@Do z3BW7scC0`OXhl21^rY+O>2`dW@%MO`M4a^BxZE8+8gpE3hCOf!+hYiD;6(_ok>u+%Q)=!t$?`d)TnK!IK<)f~a-w&y84=fJpPD&V);E)e z^4fUl+Yb+Y$*T7g^TT>P46gWmHes#wNsC<&w&XRHuWfzcYw|B)UD|Sasi^VdZOl|Q zU5uELrFFIU4Bu|5y6|9PAe9Aj`-7;NiIxUZ%bnOHZR73;TcO9y40F~V2$$%7f2_%i zxD$LM`pq0kztx_p3z{abF1p@0?_&0UwrpE^1@Gg}-|gb4G} zJ&8)g=ZvgSNC$wq%X*wUI9!_8N@>@%?BSrdYRE^`BqSZLGn?|BuZkUOxkS47Lw1<~ z##35LTXxuYX{87Ce+4s7ONzFq(FJC6B1u%^a~O_6q!_5BG%qXSxh^#0JuuN~tzjMa zEbJM#f<0*$&Faiwaa?Ce^FaOJS-#E6rOV1i&v-n#yS{sLM2M(vx)Nc#3&aF-ki3Fg zTHn5k74I%C-yep~d@&>?-%k_gu)ZT(i+B1$p9-px0>8c3h@#MQxiiuBAyQ2ObXG)C z@}ZcHf{F;H2nG*rC}md~GiBrldH!rE66?%w#HjT^Neh@Z^Tzh=HS@l^s^y%&ZUYtb z9MX^LRQ(AS(5V(8G-~!|AkXJ=H)eIcLdCrKsXzQOCo}50?5Yoxeb|IxzpQOEYQbjSK>?#scI2?3AW<9+|m&!^Za6bsW4K>A8T(QlK+^zA|tEHL`Rq7OT(8HG5 zx!UuZ&&MT`{2CaBw=dQoj1ipNQqJH>ha`Zbt3c#S_-q0y$O5wC3GCZP_tsUZ>e7Vo zco@UGkB#ND$lK*`10~~#d2Pd?ld1Xy14l3FeL%`gQ0Z3?%a1o`^Jrp6#&QvfODnJS z{R@2QEMyKE}bHGFjTALAKtBx!EZAlbY~DuaBal|;L3mfN>%z1zCu z@h+*G8C$4~BOZ#28KOGcxe&QJc>cbsSRxQgN_7}O%{v>fA}~d(JcW>e^J+fD2rR^i zIYW4|xGqweuocsH~Pmj_b<$NHTxoRF~`@I2BU)9d=cN_RAdp zm(4Eie(v5k61v{ms!>$&eV=W*nm1FV>J#z}v12sm(mu#T@fKj=SW(8D?Y z4rvc-;4V~)G&I<{#jclAhxs;68f52nZ@FoI12nJB<2s}f)d&H+`;Q$)}j1twHxHHc25BUc7Drl@wM?&wy#I zX{+^Lla{Z-ClJ_fvC-g0Rj>PQ zAi5v^SPq9+n)I7-O7;vUgZi6;nAs{w&u_+#BMM+rUb!E}QNJ7KP;0YzhRHO{jHN8b zJ-AnF3f|9Kn#D%WoD0VZKVFnY6&XG4wR+w!9`*AnKRj{Xz^-p;ez|V+4LbFNd#JBB zm_5paB{5lLq!%D3$ytH7oSGky%kSBUP?k3i`?8Q6wL{%Me*inkbVh#6alp-_>micB-{gWY;WJ3BTu0)i!GK*zjaEvzN_zcpUkGHza6ydkG+UzzM@u#mAhUK)@_U%R}u87uIH zM0w37G`E062P+<15sN-CP7N5gb5#cT;ObuCei$&9)J$P{7f!p4bq;?E>F(3Qzx?e8 zZIlgLlfA*p?&nDU>E0Jg|9}Mz!!+*{ST=ME+Fq;;_)|z{pCIL*7MBxx}2|X7azqP~$`V^w``S!vbkQ>|! zP6ZXCs*+Vo>!tN_M%feY3H>SnUh1VKoEMH$(vWM}6YdC&1!bd(kwr@rr-^b#+5J;s zh!qW-PI{V3=Qyj1eNza}IK0j%LsaQ)Eh0uxO~e2@NxUf#tzbA&|3ed5vXN1S+6uB8 zYTf6g?fSDDJugNpAQW*KqlLju_YW`PWFTDKZ+w z$V}He#3G~<;`Mqm0K8G=mpuF<|aHJPW#_0}XE9QD`%kU{a92QQ*72SI8 z7p~3avp+ORJM%6ft%k;r8Gj_v+3;6{iWMP!FlKxoPjdEYrze`4;Yi+Wb$#c;ezpZ2 zd=alOs4G+Xq_=462Y4Kz3K!K!+rkLbZC5J4EIFVo#dj@(^`JPyPZXg1w_PP3E;*tK z{e4t^SQc@zffNUYZ`ztD6#3(Z-^L@+-&Rv@U0Z{}Nf1NZeVF^knS|L8ikW;rU zg=Fq=+g2ak5|kAV2cAs}Xv?jPXMT4UVo9_M0?pe0hu{jqG17HHy7lJ$ZE1Cd3|1tf zdq9iea^?|w_EFjTSfXu6lkFU(WRd;;r^a>wcQ)pzv=bo<3-M4I;gUWnog`(_Ng2WE z5omLA$#d)IiS-~i6j7MBOnXKlVvLG3YhG(dFhB=^Kj2%&F3o)f{9*NCiY;twcCu-J z-w44yk_R-O?~~SQEB;Etgi!&t9bNJEmPw{b z5i^Pcy_1tFC@x8&K+<*~czXKrSyyE^$Gb&*II3B-^*C^p9m0Ipx)DzTt_APa^=?x& z?>Oug+V%r31z!a3$6CXCD~UA!GuwR?V5%HiHi3Y%c=j-L5X}2PD!GJQ@q)Fmy#R2Xx?5v7*O^N#g^d%?%Xqw}V(=Ujalr0ZVig0}<@k&ja| zJRoHcu`Ii`jet|Zo6P%9j}FefhK_eRSwP+c+a)L#XVpsO3V{UJ$AePix}0XYK$?ep z-QxU6Wu^v81T*n~dTGgl3c062H@OmKP#%EECJfs#2RH>@gMTW!boWXTru2a>q7#M| zRcbTOCj|;a5aiOcG>jRR<8E)g<2t87KhL`yb3ICK^WC)S;(y;)Wo0-(R18PgTLiBtB{XF86%cz(4_Q+vAI-Mx7Z zbZ^OAr7MuIZY-@iDjFRw8g0YRB;!L|4@m8ukoaPI06hScCe~$}H)u*MMpqvi4{Xkp ze48y=LU%zy(n9aOF=ZXac*J~lXLqo@u7vDa!vDPUOI>J=T9vu!J|_V3K5Kb7H8OWk z(DR$M%16TMr|?geYf2I+%eUJ7IrVpd#!Z-oin+p%GYNZTcYVdiq9GZ{T{S&1pUypJ zwL+U8tLs6V-8@tkWlFM^;paYW%~uEXX;{t6rT<^D;(Y&qvLd@WHAL%F6Trz*hkmBG*7%;~awYL? zF09LrVg!4Ua-cSAU!|F)NJfV_Myc9jr{|8j*-o5Vm2C1Ew?fGHRi#12-Opryx$G~S zJxbEcB=1-^Dx47q{1y28gNKngT0>KsReJEQCa#Bednglv??gBU?f_x8rgJC6)C>pz zOHEArTTS#FIHNAxg$l7i#U!8TTi%ss?NBU(uF#6dN@=Os&G;wNzjxPW-ih&c^YBNR zEaRC9>^S|rYoQ7_o-X(iO`onTW6CtuXx-dH%cA%zEQKfs!&?0~!-o^8#KCqtz`dECgZ{^O4|pa0GDxA`a2pW$bw|M$qr9o`;JZI-@V*nhoT_FwNKa_|%)E2uzKbe}o)4yQR!@&m zPk9locbYe#!Y6GqPuW||b_~x1D}!Z{_@@Pp%Kgi6KO{wQjG|oag^RIAvq}fgq=+;E z|6Vm~7rt|AFL_bpv zu?ZH}P4fo88lnSf>9H{o;Gq1$JG@qZ94qOtr)sh{S~>h|gydJ<^od#bG}VxW=TzUe z93+_(mM31uwTlHtH{0VE5izAzAw@lLX=c6Uo8r5UQ%CPi*< z8;%eC82%d44ZH-y)B{L{!Q3NGF^9XHJyStXOlHmjTM0}xK& zO<+rZS$<{vfwclphpt82r19I&y<*dZ`<2t}SND}lv%PHw8?3Tj)n8NPgqfAvZ3V)` zBV*|a!yKkjNa{I0Q0|H0rQ7Qekj_t841meRwn(e#N7MG>N~2|0vp0ymKQ z2%-gv16~T!SKK?CMK9KR5Gi)+Vt!|U|AP_&Xv9Dy&D#pzoDR2^N|CmNAR%B-Dk=w= zjr2xpFGtkB1)wb3XXbc4mCAL(>`f7ZTq8o7-mWh09)&_lLo&8&=8imt#Ki zu@Ob4*c7r8Y7P9v>-vKmJ}**hQ6StfAcy#Iv* zwyl<|S*bLZJLnwz3Yx#CdjGPj4&>-Sf}Y$q>-Q1^|M}nQvr`d0Hjwru2L5o3%i00XyPLo14m*?1K#0w-&oCs|MTl3$OO!f(cNmc?Q#@}2LHZ3()u)S zSCwq2SH{tF9!2%0zGrgL&~i`VZL~qC$}n7%7J2lpH)?XX>$zXd3F2fM*VI#PS4(X@ zZ)FGf_}MaO`-ml8U7Zzyuip1IV zI(MzPAjWT!9u8P@2-pKC%dOl;!X$NId2PySQP#7Ffl=pK3^WnU@%~K>eDB}v_uH}C z;{>cZ>=F$WEkDX(YwDPfi(QJ!sR4t4{wuBa?*v$O4e0IPEzhwI!#%poe)+ZCmGt3A z^z3%_6jW-hE+}g}*}BJVY?5fAgi85F%nP20+Li7swp&j`{ zdui>jxL#Yqf>9Zi8>!V&v(a0F{-~R{A{9>w@*)3tWu@!yqX{7Lp-MO9!97?dLamHD zla-@(y&LltxuTz+g1Hi{Ui%_pJ}@WqQ<`5I&x<&$r_1$-p-)3h9!ob=b*rS2^#$^S z9PAFQ-Vh3X-&vTpLj$woJ!dsscHux=r7=CnKg{4bd5G_oJX5<9=+SvFvfoCdDCx&Y#o&6>KIUSj} zx-BE$U;`jNFnDbkV#27s`ifVl>h_JhvhKZ%qeJlbti5N@vN6 zK>s-6+(EmV*D$M#eDd9Ah(66@A@QN0v*?)O730JDHQ=4$`{oiA3M2#C5v*=Xaz#l?7`ZTsJu2@*H zrE0L(TiN{4nU3;{=n9ju5;t*Q<{bUoc^k0Yw!vNKmw$Y46@O)%GxWMi(n zicw!p#r$ayTK{{`UDQ&g%&?EM`It6>8~(lUQCNg*LKcAwNc99S1;d{6R=A&{5XCB%StvX^tkF z>>WiwqBCi9`62GDbmH8^c+ou^>rj*L*zmDT6p1i0x3TKF9pm`1j!` zyxh7@-N&JSjGQ-wT>B0y9)Q*8k7X|18jJb2tDbjftIzDeHtUWVzY17g^#3A797(?@ zbW!4cxoPaWbo;bl8LT$?TLc&=Kc9B~)o)gxoKncKTWH-huN!`IxHq}X&Ysq`8Qj#H zRcSt~j^>RCl)MCn8_BYS9OkPwnCQlppx9(o5DA%T+xUWQSK8I^xl56UN7bzBUo^E0 z)d<7Kw$Jj9WbDiVdBDj0*-9B>C!e@b$oQZsMWYpapNRWR5iSG-L2)_VHz<)r(H^_rAR$MI?$d}QQyS&tU4l?*CoHMn7Nyed^ zEZUDLrmBTx>N}=%#%!5>dl~E_LzlyY8Pc}T!zLT>r@Ko+b zpzfRV44JM7o}DVNyl$4RE-M=h9eDonKtV8PThh!Sh9poX}2+jZUvSv zcS9j4n}|52-bINstc@*n^7(1ZDL{0bOYEL@Sh}LGw;~ofP(d^npXBkb?n=o~_Zz(C zB*sN)rQ6TQD`>7cv$@RI@nwB43;xU2-DKCJ<>0fjpCt?%-)~;O%-VCLu6MMM!H!+F zMf$@wJFVR{z8x|)nPmL>hA(AcwIA_>(>rQqKN`tKfu5aKu^$5vzCNr0KGQXUS|0!< z@o;sBL;ePW06HZA-5UT)PVm?`V!p4QU?st9%nmpYP{l^zrA5gWwpn{ zHpKvt)>tN~60YA~tp8$Tr)LC?HXn08MO^Z*GuuIm1kERGrN4fxg2SF?y=#}DNX?as zM4qBT;0>WX?o%f<5&sA?-5B(%`!(%iHBfWEfSVho93l8nU~Lk_ zx(u+!(sApnl2Kl?hdM>#wfUxkn;@R7^VGM8R_1#ANvie!;AvJPedO5jjr^7-q4~X! z-hoQS@#+Hg7Fs32(4{2o1NjDrHfhgX((^aLOs2nyjSw55FuzHDE#*zBE*yi-TRg1x zk)rmR94D7nQuANvc{wMPv|A-M(A(?mdXqlCXY@}+?vesmj^$TsMS9e>)MkIlpr_N4 zWB&(2$4av7+?q}e$9BKfc`cRRKklA(|nSHo67 zNYnm1nCIsD4yEg@JkY0Aav(G`>guel|K4}%uzqes9bJAQu;}Q&#{RQ+dLqWBaUd1; z#vs$AtXF4SxI6Qdx=m}`B&4&yo~~?>oV@` zlV8d!$gwyCwE~-voS%~gH~e~GqtFTIgq{nIoh>ne;05S>M!lS5Z%?H*QhM2TmzPdN~ySv+Y=-`gctD6~M~>F;m1_Ej-&hYD4=w==45$u>aHRRxb`;-T5O>%)Ihk zGnwe=i?R*K*c^^X^5W~7){l3bhg0$1!uQ~D4WK*P=) zLZY!`@9cDurt&GdG(SQ*IN|N|!D;uelTrHwv#!#Wj;r#!>_9(#lnm_;3j~Kqj)8X& z?rP}On$|`1n(wP{<`B$1$LAh7yF)o@e@IZ|kI0~0OO2MAjoccXI-AQ)ITEocG%>cI zOSNzd8i#E*GHPj;Mjzb{Dua8Xlfn*Z6`cQkSsgb@NyBW%O!Z34c2n=H-__-O^x?A0 z&Jrc}j9-50j5t>S2KHypOfy8&cngvmL7ULBq&F5lXa%-uk``KY}(GS+|)TCRLQ@DEe`-W^>4R zjrnN|A&3@Y3A@&xt~Fk_7p$@l#C|dTBs&|5UGRg#9ahak5&2+=p&tb+>&I z;%{lT0X<7de@A)Uw9-ElaNWsAaD>ViGWd1 z&%YNg>bbyeE5!fo)2?X*b|pxfJ;`n=)nf$-MVdHOls(Fda8;;1$QShu*?$FY0_`{6 zmh8hBmhVCN!v7Oqe#4ux6w4MMckR}KSYBj5Ikk?1-B7;s~U&tJ7-D{_eNP)f2mr? zo8Y9cZoc|kl44uIdRC|`Fx@%^{1Ra4{%D7|Y2mJE;X2Do&IWFr2iPJ7h}18j6HQM} z?QIHl!-5g?9&~@*$&8`+Iq|TscoE77X9stz8I#sdJ}bS+{UV|U`LAy^002`h5S-6E7;!MVOPRWawKvGlR^v9W&U%KC(7$0se#$Y6RiUZ!Zy4G6e+?tDKqHRk zvw~kZGU$VBOb%^3hg1+25fmXRMj)vpNm^_jBI;TZ6amc%9mUbkWS+&VDv`Q{*=Lhn zpzyA}v0P?4oEO1ol)(i$D}esr2aw%md->@M@VGm$wM}5LOp>Yu9`J?Cpcuy|fHS$D z2gW(FUx%9WkDXgW6a09Pj*BF&Tf7(yS{^<3&!en^4!fl0%VzZt?El=w9ULHj09+d= z9|zJe!C-%}j+b)~`!7b)G4A)WSN*Z&?-^CXGodEgVl}G05tHFFv_qeIaxERelAvX4 zA}6c>MT=%Kt%bcqp4X=hy-XH=luW37xA)ZcC;xzdp&qFkzi}!4n1kHD>2#abd)iW6 zN%a@@vAX8{>*V{Bf%w0$586KoOZeCZ)o{hLo;^)(0nvdC+jCRHu}N7j`c4(sOTb1O zDepA(HOLH_b+{OCkgWS6Z01@F7%Kk~HjRD?oBz>{-*^b`5lx3P{>2&;Jf=kb$NwK( zFz;of+r7xOg_|XUefELp-Bg zwgxgLIEFREz#H38F53EXZc}AokmA6HYkW5(-d{&x$Jpal^bR_IZMZ*nZQDg$X8W9S z;a^PS4{0)m9{YugCN}&_IbXOc!^Hu=j4~6+KjiXvH`fNWukN(J`1~u(i&1%k#<{RHEX}B3>JpwCV z@SouPzoeWNT=k*pv;A@Y-SCN$SE5(X??!Jq8&AW~1f4UJ0dvz;)Kz2v3To^0!dH60 zL-49_JH(wO+p*^9bOjK^gr+Ge{tNrydPWWp)eVarMYLZCvRql$8qTIWOmi%qOh#MM zSTGxhthOw{TV~#WYp~%Ts62J`y(__xe5k@rxodAgaaS|r=BIK-FiREn`16)OYcx27 zCW)qq*SIv*Ov_m&fVW}3C2?P>-Hz9&-z8oDq_1WI=@pdMJW^c;Nnl-|fo=XQ_4g&W zxdD^y4MpbJPzQbYLqyx@>FUqg+GUr{4o0rzp#>@oN^yX%sSV$U(a=@%*4CcCbvIF$ zpBQVL9<6a2+w1KT&AAuIFcq9gdm;d6UVOM-$HhZ`PI=|Z(Zad=D{s2!sT$+mDPLjj z_lH#xIm+v!tr-6K*uA2B%`&{>5nS2|o+cGy*I4+g59CaP(3^?Q7UYe!z~v=$#bNu) zS9B_ra?&s~7*PsuzL@t0+IOe?uxv9<`0 zsWSt7MR&S}tJ<5RaM~YeefNDnk5o{O%wcG(Vbi$ItH8ZQwY+zmhv!6eNW;j_Mlcr$ zGJ0=r^1h6FX6t@+LC$0-ru`-=9q{dF$#>`e)o58-E|Xo6UxU4jHPJ+7UsIv|L*h3F z&%ee8F-?fkhwV!p7QR^H#g=TWfB4Wo2BDFm@P-Cp>{+TFM`bpc-4FIv-pu zgB~2jOxKx}y9s)Uo=vcg?o%7`#;DNTgWUZB!;%~&4&TEmF!1EXkdCL(tva2UqYzXs z4H3^Zj6^cx2k|&=55nL)T)hHkfNho|-W~xGX*C-C9zt%WS6@ll_c{ddxHCpT4N%=lkEQ7jA zn$fuKto>LZLtaF3^iIQ(i>u-4FKHBL`%bt>VZX5%$6?p~By%rVBKit=Zm8$)#L6?6 zA7hW@1Dcz`Q(gP&F#-Dr%QzorU7^m40Q>OEg(J1BaBQET`u3{_50**?ahj(1gV;2k zT)7mnwK}kXLhGW1cY8eLi?*u5kog5X>sbk zSKU4*^v4wM*)b4?u*D8?rr8OHd&+B$*{$E~$vx$&+g^Kaz*b3IF!}c{ENE z#Vz3MVuTaO%pcyr6W3kSv?L21Ly@2VVS9|7%kyp=a+n}2%ghQDnD*9WqO#WA@vZ(5 z&W|!+C7W51!2BHj0zM8gQ&f&r+L@edp0LWCT|Q+v9o7`W4*l4wd(7u6E>=7iVp0^o z4=joNmo)`d=u1whuVL2Jgj+qKkn?rtq#vkB!=knloWZ+Y>^8r|U} zI#$G(0w1v#{M720f|DH|F=%2D1ux@-!04dOi5@pJz)NQ2N1K!A#$S||f>Zv>Fz5VO zt&5lXPHV0`ABydEv8ao+VCU)gzu{$%2Y5hLJp?6qg{&+fP6mR{o{c>3c{q|v>7(p& zmu5W9-g#}+{BM_h-?AvX_NRBLO9oQ)8PU|~ioNfua!hiz4C;Rph)4aw0&!3 zYl2)?a)kCFwLeNR^X12gIem-f{B-et={o+j`L;*s_#UD0&+Xm zW^>8OyB(UnO`{>0yQ+H;Bg)v({FX$P*OCVvh zNE3Ws?vZC}OP~hW_uLr(n+1h@qR(Gn*2M(YteUN9&a6d&{3qO4s7;+!dv$j5=iqmA zOu@&?M*PbC?4yphyzP~*+{E7TwNr8shczkP{_$7gK`Mp=Shz|R0ndwJ<#{pqvNi5c;nA*MECAP6{2LOq?iF=u3)8HAz!V)&GJ{1o1OQ?~)|@ zqbwS3zmBCAV8lA+z?)obrYF9*ArHu5v}ZRV=|4ulDAdJ0O#{fB)Y={kjNA>RaLL%S z00~{-{t$bW8J-9>mjCJf-JwfFQ(}RvxH>^?3MN9j4`_u$vXS%Wy5PK-_3;c zDow8lC#1og=LiTu-ov)pwhl-|zv7*K7&PRxpqxOa|3LUoLT}l8KdN@OQp3qS=i?&Q zGc2vHpxRk|z(px&tmWZc&pCz+g<5ZO!nh`e#|{k;8ab}fP_-<6UJXW0M~Q*lRapJRhr$@Up$b^M`dh8t+*r1G*;)}u@drSVRwq9?NkZ- zyj!bqboADriN;-LjtnS(mSmk(6&p6dab|D8abCw#l&-(3R`6;-DpMwMwmi1vOHeFz zuccyWO?cu@G~x<*K_mPLWTwB-h{uxbIi234(RuYH%F{clikMzD2loC;)>RFczR@K} zpqH)Fr4Tn3djK=#p(gu0ph(s^L$F;I2n@TMhwWujWU*P$b^|f+qk>5oFNAB}EX{otX8kR| zZ59^I2kO~W60ifI)>fD0&n@wBQ8>PaUu~zN12I~0wnvYb7?AhOoH{xvFU#RO5bpUo z-H+K^z}=wNwKkGAgsC0`S$6+Y(hg8O%Ay=&8FV>HlwPX*zz-WEfI;>}g15p!EEcS1 zGH7y%?D%r&`d}xO_QwS%<7!n=pW2?ve z842YF%!PXiE7zTYyDJUKtZaHdZm=fiBRH@ngdKhoV<|T|=5)agXhJ zH>W_)PspCF*jZQSWa(%YWywws!oG)f8PZIqpoHm+*{)4z*YORB%4paxS5O3%mh#WR z8%M&fRV@2GA4%;UIdtZLSgTl@?>FvgaTQrLqW5WXJ$o5#76489}f6BsZtm1u)|-;&TN8_4A?ZcZtA_OQ5V`^rkJ;EF4nDn-gm!L9Jf zCY)u5=}OCR7buoB?T|Mn`#dw;f?Ee^2{lKCwbL}Tmo^;9?2-%@$vy2)p&C$^(ngA= zi^iKD6jK_vEgYC(6+Xl$#p&r#Hg1H5anSodMWeIb6Wf4JU0ir{DJGU{ZT3aLSlsQS?&yQZ$~fG#i|;Ml*t-aNpANH^&i*wcd4~B+__slY{uOi>a|+TSDPI(Y_F8pcMfp) z8C0}#gRho17)}lRj6BMw;`Wa$o0pE9?=^AL2PqwzUBjpk6Na^~X&~@uH2sR)nlH1I z>aX@`o$7FsdOXWu`_L_y&ueI3acf3y&dwjwIy1FiC(@Oj(M`*gP36t_>0Eueb#r)N zK&72Zg7MQdR6fhoVs6#wTXA&c`~P@*%b>cpt!tMAC&Ar=ySoQ>g1fuBYw+Oi?!n#N z-GjSBu;31NLH2pi&c5GQ_x^czRaT{_f?`g_8ol+_`ZLE`b35O>TEgSGd*EjXWwDPl z3K7|T-g*|Bd(?~;Kj!AyQMhdS>J$w7@d%ILG{EY(czC+~=~rMm=k(Ff{fw=Jr(-9# z^{yo2@4Gt}l%wS|xGn8Lgi=0}Q|}Dt_IP%xm%5)1cT>k`>L_4ap04;IN`UA>P%oM{ z+)_QcX!A1m=5USDKrExInO@X9R}nz~EdUVq=hsdE=n2mI8MpPPiI%#wbIEx;y!tJ0 zsa2^e2dPz7i$b@cAY-U8+9YkZjzIUf0C?z}7SK%KeIWL)Fo6GJz+K~Zs7^y;@vzig z4(~2BzT33wgbio*`A4GR(edf}^*{u{BM=Y@g?;$_yWW1~Wqr8~70N8G=JwCK$5v-2 zr17<8d07g~(O3uQX+9*BQhABKPdU>2H+9`6Y_%{?&fu;yZBG5e}~ z0#CC72f1Bd3^^)pZxru3!e*XlB37-McDu72k3y>Q)m4`kdoXf9r^yI+^TH|fOMFMO zYKY9|(p;~yOCG5lvEoW813nDO8o$p8Aw^3=G!hD!xDAl1Ryb6**tZ#<)4L*P6BMpi zZqZpZ9rLO2R~fFIUew1v^{^=yIQ1yoDr4@;I4bzz21HSHS6JE&Wg<@+Ms}LR7sXfK zl%=%7&eFjK5cBS8lJ$iSwp<%`j(kd!lQ=t5iqsoQwk0QF8Qax&k3T8dy6ebmN};=< zr+DdO$kZ{1OYI(K{14}bH*cqOj-vn`bd?1PM@`WP@o~G^YoQriA`Sr&@>6DV&Qk11 zM2c9Tr8qsotR*e?o%DKb+!-^*3b~O=x63eN+9&ZAlSI2pNXbDuaTzT_TI-w1089I( zap0{%(|iZYkq=Fx)%Be$iX`wO?A4e64GIJXXb@`B#>GlZ_)_noHV>g}r9l45lOrEE zqF>!#jTXsUqh&k7Wkd3DZxIa|;77DMKf3#X!SwOtgG*Q9VqeO-4(-*y=ep)t14vQo zBN%M)d^ht)j$hyhmz`^!M;UF~Nf%;{F^v>5;OBc;cXlxNb$QMx&;>SE^z^u#Pjbn* zEik?D%39CMJPwXw3WZcb1_nz~EZ#&kY+I5jEf7{QyN4ryc4b^{FV!Nr4cWfJn7w)Q zfBYfB@l$M8YabHeX6mbR++f!Uy`_amfL_*F^Bfw`g3%raB{Dn1 znEY5+#lhWM1stTod6lkUb#p>mw94^SD^0;_UeUh}Ao{%Q0MS=C`Q@l!nb$fsmRMhI z3#-t3X3!1x>Q7o@wD-nOa9IBGCp_16Ni`RXYKL_xJdXQW9cBdHaB_6f`0oxe0q5=c z5EU~pZXNb+{pX7sWy7JJ(uz3Yf?N-jhl|8_% zqXoVb7=L9XkL~MeUgd>4@D&imdUNfFZ?2tn2#jT88Lg~x^yYB*Z7B@?SPFv^JPEfT zEpF#hRI~Ua73BT|=)5QA_5yCj8|*^>CU)_M#o0{U~k>Jt%0 zfdRtE&sloxuspS-ONwe+#ag6w8->>mQslgepA=tDc^F{%mZ?gUu3GU_TQ_ZDmw9&uuverM?j!buIqPN; z0`>PYaE&1LNFLbH3ae@2X6yR zw&^^3j^1ex8Vs{!^-!BqEu(n;2)A^03N>9>cKQU+uo8qI?oKkpkikX5LOs*NHk&3$#h4~@OEKRmz|ePK6IRcyUcu$J4$y4n*;_3(uS)%Pi9~$> zWF!%(QHR~FTbB6Cw>)SIwNR?k?=(a>ta}{FMt)=dq6Hw+$w){B6_g8bdef|(8tQwI zuhB_M10Zbh0UO^d!D2!F3Gvn|SDgx74_vgiUa2cLuDDwPw#2JXap3sv+Lv)M&Q#sD zI{tL+7^`~Yx);oTne~KR?5i9?x@p)}iORjibvpB%JR$|KFEA#?Jp13LVRZ8mU=wD9CX>3ziH?}>X&2Fx{!A2cv97TJA*0ls&MO{Vo9o1&97 zueE<#^KY)aPlmYxa*voa9E|0+wrsv?i;N8oN9d^iJ*?7_1x~QybvUH55e4R|LY9*S zn&mo7^n%%;E5x+sX2On;{{$6xm65_HBAjFd5{eo{eA;4J?Zx`!yB~&z78T@8XabFS zc7kMQ6Krck>IgDvaJAHQk{^z%;tM}bdJU_nP2LZUnb*yxa zG$Y`@!V=sS;`}A;npH~&Ci}OrgyZjFi2%+Du_s#&DDKtdv1id|F-}IJ-31$DdqWEo zZXbI<hyE)DZO$j1Iqv@>FcSUs*HXf?~WJ+TGDMswl; zNUcIPzZXgicK`*+Q$~5zR|=;eQz3z(_iapC{1%1D*9FKuS-qYQowI-P>!{lPLlziY zgjbzXL}*GTIn4!K?OU8ed)PTvX)A*z0oBO&wHiHRPqi(9Fy7bB-Upg)zbk7CFJ9h` zx06#wv=_g#N1#;AQ?XtULJpy^Jx(SC4lj-=ZvC=;JbJb?&8G47a2)jx^RC!ksGf(+ z>+UpVczTiYv|hMdTtpW(nzbiMyUP?eHmEmUBP5INP}>3kup;ej5=jZqD@1 zUik!NCPz<=QH;5DBagFdb+#{Ok{w%WGlD){s*Zsm-esZ~O>LpbPP#XzvINLulmc4! zR%^qan{!4GyW;^L8PuI}$%_GjAh z4v|~6f6A(65qeD%&pV>-qsw2K(HGnd*S8mVXUJ9&CZmZDGssOj4MvlXkm#xWDn-q= z$uCKNS(T}(!8$9Z8Qy$4$2Rwz?HR5H{i&5Aeb&b~XgsPbUQjM48cT~Y)_^?8(OZpb zcChf!;3jb`*?U@JV`Il6e!X{h)vw~Tk&-ezJXTu1>ov?3GkRDIj(XA@V)>&?>iJiP4zrxPEIR=J=3Lou_{>BZTe`eKn4+FZY@MZJHx{ZOfP z;eWDP-6B#A4yD#|9I=;i&eIg1z)VbUtAI3(S4#<+lKANbvJ$yV{N~V!#Aq+%;mT9~ z!ohsYA(g4}F40J-a=Sn;=HLs!jdP-v6sHu2xX?^*}gCDbUkfK`z&B}&(*rOmSA<~ zc^A$zg(;$TY>>+Hh*00pvd3aX_|fy+38Mh~+;YEpXL@?ZK`p!o`6=3ReW6#9J&k2p zypomw9%J;bH%$bU|KZ@V{7#O$8Fj_r(dBwSPQ^$t0_@H4b{!P`<#;mzj`xDJ>6*|M z;COi-;(vI|9vHizL`!+=ZiQDdH1+V5HLl~HdNmI4>^~t{)?OS6Cs+|}4ZGX8Wn*e) zw|gq?@{E;@ZT8OeUmTLK>=vXP?ku64INh*0c`>E0t}-DtZ5uS%zewUtn<~<4#5bcV z)%QIeFFbHXv=4X3s5JguF+)w*Yj=O{vKpjQYl2d`b3GO=-5ne`*^FI^a(TRIQg%AK zmb5x-sxA01!R@xRLS(gzihHr+@Vs+>FmsR8oOz%rkuhGzW5~C~zLoXq;K=m(NBM37 z1^VO3$)4MmW{W4sHi?&)apGA(*||Bq-?G=T)T#Z#^)g2Z48AZ>~lPyPCM+Ez`=jQ|19wJ z8xZ`v%`>oX59#aln{v;x?KkRf1Zm1QN4P^Zq|@0FA*5I`tl8EA8{fL%{RyCilTZv) zV@>}1v-s}?K)xw}e4Ew?aZt&F#)ptMk;xN`L*Zrhu?E?I=z{knfDne$RDcU7rz)!X zKJ}m9`Sx=Va4hmBH`OV9k)Vq8WHT5#jVxB0gTU!V0UtH_1LT>GsTK7;^8m@-@$tCCgsOfRHYrbMt3rC0SIf6f5iW8bNTta1nK zCb805O~2Y^nKtFZl`?Wnns~P=uGOmOHd`g&(S(G(g!-knrybm95vP-uK`-tQuRNWI zgG&>-n|CoNdP`*K)SDlc)TWe_dz*3hfh`YfE-n_%$6pwE)HMC2@Iw+jhceo+Dp%&5 zUQS;gpJC00nq^5Jas?i&Smuh!k2lRSC@r^QgteZpYf!wml!mdkSI$GpTvZK;l`@)? zDW(fFPx6xQwS7l=C#bw!CP&iM`J$7v3hGsvyIi(dEx9u8?#Z+`lRQ0q2iSJ<3E)-xQ>9np(Y+ZydKZcDZd zS^cneo2>EOQM58ztgmanbv{cspHs6+ZF4f7DfTc3`oiSLIuzu!Dw_Z$2Ycz3Z9;`Kazn$-T(U*CRr@06zLb#Y}S zdfx5=GX|KLYu?^#B_Q38W^nJu%n-390rvfJ`^D2>dzvA_#o(sEs@K!TbtK>$UwX)O zhcgCxnjxO;r9@ZhGuiLlQ0ckOZImzIl^KlP%@ZDZaQLItJsVHYTNaQ(spe!kYLP*4 z3kZjwvRU+robLLv$?c09`Z*T8$QG16Zv&@q*|e?$Gd9jehbF?F27i9Z;C!AoJNdqP zGsBCTMEw=X zZ}KFt9<_s^qcH*KX6nY_G>s*;TxHAaRu=gX&*PbF4YkDhJGabo`+$LzA_Y`xu6h&S ztcBjm_H~(u-}plwlhJTQl0!yimdI@{XRFomS)Suz%tITa%+M_BZYZS0L!>f-RCTny zN%a(OBl76Jr3$j2mDHDnHHYVVPPbpJo-OG?UoL8}d}b;pK8T^yqs(k@$*tBr<|(bJ zoiwkd^@T{SWyEV}ZylctoKQ>@!=JjQ7n5pxwBE`1Zx)xM&|FglPpO^a$MqvBq#?GR z#Mj;=uealf55-d)mYxvpT+~kM@$AYW9J6W6496TsoE_zI1QwSS)R+$Q!2e1q`gO-w ze`qwkTK+si)_vC8+2(p1+~Dyrfzl(6n3ljp1^eXlIo5##bs(#Jds1W2sa$P@UobBO zCW_6ZnJJN>z*!J!HeX+zQIVyOrx{*I-PdLM@|VS|(#h%Y(*4~zYt_s{{^=&*6U=|N zezX=bMuW)`JaM|I)?@+awlLipeSo8Jn^_`M+%gVPeNTnGm5pj&vox1OAoXNNygxXLdCk@iv36xFqv={vYh)xI*$Lh)LNJ7a1 zKZ@+gA=WZe`WYul zRHV8Qz0MMQ&+@t#32=qDTdWYUB!F!&A`?X(fo6{ALRP<&FGZw?Vji|e>#p=kiM80a zWg?vBDQLPGc;3ohKSV~oxOrYpZ^F01uK0Y!jraQEuKew~{LFrpy6OU!$5Wr2wb-E| zz>IiaxRO=->}d7z=k{X>!JT-*dho#f6mD*a4KJOx6PYCk4vo~98uq<+_+tgT^%Egi zo+e$;bZJudjpAGnx>e6Ls^DgJsI?5|UY~`R3=;Ja8mgcuS$#Vx2I|g+V3gHW!@3QU zQJQHE^HgP`h5n**fAlc4S#Dhm(GW|P3aT*-E1+*OKSgwY5L>dibh|2EasX1UADCic zNhqilY-;C99-_FcFylHZDPbqyeJ899TjOs29?zK z&Xft1>an8$9H)Ve&w#j3P!n*Dz?qWuSu!BH1UOX_mi9bZO|b&Xzu*eUH@E`JPPj#{ zMPbSZ5GVf;T?SHO);7DV`KG-0gT-077YC5nc)#W~&oG82 zBcS7clip68#RAv^8G*awRVEtYyj4?D%kf{EntnH)9mas7;k*HDLV zfXbyu>UkZllyO1Nflc6r0Y>{A>x~R!g>;9j!si1wF#AeIE0}ExSyq5`$$U*`(%c4e zOHpt0pwC2DFRmR~Ck(KH~xQ-=itQ(c{6&RzQ!yCiwT=eZ$DG$B}xcSyE1z zIt{|e)Scz{4gnLwmGNu?ZiBH<5o$QioL_BiBHHWnXR*%bRrYy@c+&rJ**LYL4=Ks!Z@v{`0%%}v#iq>cl$nylT`3cZ|6@q&s@AOV4HYR zz@+mLV=g%$tyReTX9KUm#s6zf4x+3!RVwsG{FW54L^1;N$7LYUQ>~Qs^bHqxEPEO8 zlFSSU{Kv*bWVG&=dF+^n4$xYRikN~A_USL;Kg>d)>&3IxW|v$Hy#h#-zX7D77PXvD zF7YKmzD%or~{qRXNhBvmP@mDUDMQP9?~pVm_ei>yv=SNU$o ziIXX}selBb>sQlo%b)WD!}(--vG!9NI;R$V-gUXN%!~AUvf?a^Unm8RlV+z5vr8!O zW>KL&rV_5sXExAMnWcO#h$lM_S`}5at)hlBskNGITzO4(Z~#G++v3&ccZ(L7TmCot zq7;yl*~>7aQ8mt&slotYq_3P$6ryih#ewXu>2#w77lBud&&GbFOVtIm^PXM{HD zgP=I@-&boccvNZWO>NMvj>^uv2=SMo8SIK?2@28$gy z+$>?SPisoGAOD@K!BC%d$aPm;CYgv1R@ z>wQW7s%K4aK|f694j1=9Sc2?+sYpcnO?Z^D(KzNGWZtCPy6>kO{D~6jvDO1k;_?`9 z`u!~Q8VNksF@|bQWDF$>m2)w=GQJu*=DvApEwB|HmtFf_=d*DCk-@8I3QE_w0}^q~ z{ZNG9@>f8Q>r1-b#O%TMq>p(zO+mpfWom}}#jO*0@EfaEHyHAO9kMbO{FCcz3?Uyj zFL16*7UT8G4UUVYb0i2hP6Z9*8x%vL`z9wwfo|@p99D!lYkyKP1`%WSMm}=!4^nD(+|3 zX%6FO7ts*luu@rgUMJev77M1ZxBiDFA%Agi+hVJ0p>%^*SA?@@b6jV#3>H;rM*Y(yrmdB#wrJ~ zZW^@JpMqh$WwDvGHRMDQC65o2JI+6CL;@PY=Y}zWmMEnncYlOzx03ox2pwlh&8&#j z<0OOa0ZyGF#DJWm|5A7F+8BRW5^k_Kvmx}b1u%^?Q{)=<;pPQp&qc~NV5T%ER>mj# zFy0tEIs3ajSs6|KS)O*V`$nVlYUxP>J?cB(7)kiwELig^3uc>BuU3u(*aZM8$69bGjwb9H05KD_g1H2Yw2SL@72B92G$0&b{Lu-MDK>Z*3{Eycb(h0 zFx2@F+#-AM#(?F9$cB51_q%1iE$J@;n)_;~ zSyxIJd;s7RWc?}$txrtp$wl#2Rb|s47?%W!nN{4%!Of!cNgPD^YNF{D5$;%xo|W@Y zT|p5(1{sk}RnbmeJt7KfQaOQKrJLFh3|=uBi3TYyW*-^oKG>3q!aKnPBi~yP@K0D^ z&L%AalIk1wRxzo(i3;VGtS=L(pe7%x@PImAT7a|y)c(Q zf{E|#9dYftqo0G(T>@_(4ZWZgDn>`;F`fKrUtsv}s6o)ig)riOQG@H1XB<0hGoKdu zDX7C)%Yl!-(-vXe}!^j7gBULw10CXGp zc1Gnoz(T=T4X~AS0b98eg5^vc-DnxJL=kXSI+hl*c_a)0_9_RY!JZdabr*_e;V$|0iK{3Ib>PmkTOH~weGv!L6>2|(KnIEW-A5+Q8Tr6AU zlOVI2*tn@wJ|n8lG7mu}@mb@SGBjuPW81P$JgDC=!2WE|Xn_H+(6998%4uamkQ=>| z)n^v~ejDijWvYw!b*7Yl3-ie;)jDwt>~DZH*5^@A_bKty9$i=~&a0<=Y5>h)0S;J> z`#+Zh{OU(Nte8rrazzQkD{GL<>jov#LXF@DVFw+o)|brB5IZ2}CJT5_ zV7`9SJ#4Gs*UnJXkKO05&hV7$Jb<_p;-F#qgpTenCpDAwn*-SZa3Hg!<^7ayubtub z8>Hj*H%Ld0%ubEE)yB0FY9Zag#{J+|as9mzcc}aA4C(w+{;05IP!I^5^ys7{9wtj% zovK^(*UvicA)h~Cz&F3pu0f5~d`RvUxe2x~|HWrRp;*i+j|>4=nG<>2%CI1|)ngwN z4_&(2a3Yz80b*s66bDTffEj%TsdjtFlv2a!7-k^G8Bx872%xIPnCvygZTQ|E831%7aa43<*h#}(^rawo7B;~P zx>d}grnYNuBX4$U^qv?pi9e>^ZOyA&me8i9@IKV~0SQD^|S^&Ffy18y@Fw?yLU4OcPcdoOs>@xd7 z12fe{dVqXET?JvZwhVI;#Y{Y*n1|?Dhsv6Y?6)|~aogx5lWwMvFoqEk#I%7y=Ho>& z%UASZ6T@e32C zH1;L}B5^%P9sop2F8gJKg7j@V^FTF0ed6ReVDzkxR4&Gp!+ zG3@J;7Qdg>nEd^$2F5DE>E})U3Rz{&rllYQ^Ngyc}! zUs*z!L$)z>QbIC=MT-%gNM}hfGbVOubXD79qt{P!wepxojW5AABp$X-V?KYy0%|6~u9}29Z?+^GGPe9Ca()jmi95?sPoI8gs)-0^Fe8?F zOd?dBG6g2p(Bgr`{}4DPB*8iVvjW#Ey*N%)^*L*hBuIb;@NdiT^TZ-z+mQ)t z#4i+g^C`ED*5&`plLDBn@3_C&&|HitTQuc~kVk4d54<%Z_>%+3q9{cMJs&-3I z=+y^PLRYIFQawc!#}<41f58X+ukgW5G;0U41Qv@=5B#GU-R+bDNyAh3%k9IOQoZ0Wou7k;Ao<6V5w*D|L1!}=xAtZX&Gs$^vn%(EFG+=P3^2Ki+5$ErH6*U zh$>XcN|O&W(b7s!y@Mp$%O&28L-{HIjOFW#=ac7>MAS)i2MSt9_BkjzdrH2-TE^2; z)JEFW(i6SNS;*5yR8yCFem{F^GrC3>m{xkTXy-l9pEqa$bhAYaxG_?|%jAFc2I;75 z4fL!W4QyThdxv6UqLL#NRMLQ(gtGn!qX@Xid=%szRAk`ZY`sR)#(!LFFqG?vA>bk- zUoZ53^}&GEV@=mn)7o6M}# z?|WRX5A9T?prP}lTi~oVVI|f+W=$5XmTt#MfKCQ+l)LjBY*2mXLH__2*Td0PP=AX7 zxO&mZHU@^ALRDFuA`_L$%D!wRiWp32a?y+|Sti_Ba=lr;^1}ORsZ+u~SR^ z)B~yMXv}<|$bve&(`H(K&AMX1_+CqIUpcbDkPqT^&eez+t%{Vsj<$0j7a8#IL42Za=b;cYSj~6Tqf znX>Waq+A7;pY9h5i4I^5JCev9$iBOB7DY6t3RxSRE`!A&cGYyF!#vF|E#(DqjjRlt zN5kBeep~oJD%)*C9*Wt|-6G|WdgD>C|CE&r`82B%(f{7txOu=lK(Wbn(tV!X5wI1AM-aEo3z-h*9erfHYTcW!Km*|i0vVyF- zV`T9!$2)s+Uy@vH!O&iwb}0K6B9nPCTcI^P+0qeoaW5hjS59iyTW-7k@GaDB=4&^f zu1wFqJF{|GIw!WN`+ePF&IOx5cXb9(YRt1Mu=C5@S~0AfD{$Nkeg$zBI=`H2qIjyD zRAwd4d=tRtUCraqK&vFnc0(&wja3SR$z>RZE+s7XT48($RN~8 zO~lnmIlNGsCK_lhN2S5=P*u%CCYP6;>c4)FR9a&(A|~O&uA+p^?}yZ!4D4KJ8N1zC zyzk|%A=^3SBC{{_n-G)bMLSSwgMY_^QUiwAQ`a@Rk9Q($y9JNm#uUP)yNqi1i`Ji! zGcWdhlv}*wg^;UDgcT!7b9!j@GnsvQ$3-1Sjqw&-JEBE`%`*FA@?_5g+FpApdK%km zBV@p6&K5Rc`%EF_5%VQRGpd_r=WZatyMW#n@6n;_KJ1r-jKS=hI8fyx85EM~HGNMc zf6kgq;VMvv7lPcQ37r*GV6e#pi4oJ3R+VeYV1fIE&ewpglW6_42-Ej9@L~h}&$R*F zuFX9IR6dt~ukz9VA1hz!#`{k|2q%Lw$;99+=^~>{!0*MQkRiW=>@yDDwu65n3yYnN zNfJO6Z9YBbKAKA?rc`rT`CPu*y{=}YRFB>&HLFu&CGl9l$V8gpuPp%;j*kBwuNZP; zBLlu905OkAn;(-~&=#|cZ+GWpW4P{mT~3kyk%0qY=;0WcvY^r4j5V#B7r$K6eI8?@5KB#9BRW&! z-m}&KXK&CcZL&oC0hgW=)UuwK(CcMVuDW!;duJ~|Keo5o4E&M22c`^j;^DF=uzA!_ z@t7R8duux^oMzHqN@TtpjaxovmH+k^D=eRx6CxWvtf_P>4_(eqFNkE%20T+Hy8v(h z>i&>*jzZ^lSJ?bbfwP&>p(6`S0w1w-$rm)a;F(iJ1?3=|^m(au`OA@j6% zwz2zr_pAOc_TN8)5Qx=-tnKblZLrAhmPEE|yN8@bUFOI|eqTE_@g{-?<;pYNvdoQd ztBkOOHL32R?b?K~z&)&6|9v{DI8FQjCG3y^4Tv+6Dvb-u2IMrq*@!|1?G zy|P{{a0)rb@u4UGKAhd2;$*3L(Nq@tiH%>r2!oCiC{B5MK)Q=CHV!45m5C`@tC`@U zt7(OMq*cBDI(PW%8OzCBrUX{d2eQel(=*b{YvD@?c=?yRf*fNbesWK6N7Up4Lyh;k z*agHEDQE0CjBdsJ@x;_>hLZYT%T`JxebZMio!!3reESy;th> z5Y#vwH)v8GP4vxXTv$^EPSZl1^&zMuntv4ff>#HjjwXwaLK87&%L*61{+KdsEHH*UR|UIbXtF_ zi%~M--Ly#lYc~`^T1anyi0|ea?$LnSi3l=AyJ8+tWbWv<&a^0A-TBLAoYm? z-P?}6xv=|={k=_e4}z>65NTHvgGz0l5c8B65xGLfko==hWMVR=0}W8@Uo2^)QdZ?h zJ)1_=BuCauL3>3=6P7{qq7E{%JIKpOgEJ_Bq3DioY zF`z~g=U5B^6T!Lse!dZ0j9`4y%xFaj7+QCM+Q#=yIMY3!_Oz57Ld9(!^^L32-L11W zJ(u0{jK}l{M@bbIjVbL^AxtQ9qO}V_WlBbB6r?fq#59ZzO6yc@%|NJ?6-7$F#_Oq; zwy0?ELE~IE(>s`*P%qUMZFwuq(vM-G;d^GmBIm$$Q*=SoftKL&&BqJ)y2+Hh-n9`cLEV~u zeG%GzC=hZ?QTt@G(2{!hqViCYWB&x)%Gw<|-{W)W%H?ypq8EfexPY~le8H?GDh_4) z^)8Hi=*h_>lzbQ|8cJK!hW@eT5nOMJ`Z@wPCR4DxBtEn%_tB z%SI-~Ow2f)x81=_pFkSG%EfkOG=)3aYjldo+fD;G6HXn!ty?TcrIQJl0 zrQ##fm}lSZ$1SRTVMw&BwVas9SU64nW?6-X{ek@A-L_WWh97N*I<^m{*wi|fi#Bk2 zjKPAQqI}l^L{S_QMEM%!6kZsZU$_<8-tDC~oJ-diEt7;4w|(q_OVLGN(P&ztVjZri z?AfP0V%biWmU1pEg=7(tfWi3qwe|>~X9Yc!qwwvNpj`9a%-4nXDb0)tQDv#QxNK~$q1qc z5&MJmRku_~XqT=P1P)Y=J$=9Wjv@641VvEAIQQM&Hydo@0XvBHlF<(cAc~87D#VTH z^pE{ZOIK%J)rm3!($J$*>GT|2B56H69AK6B&^t*!6b9JdxzBY+U)H`< z2R(o1P@I9e*57d(hwkR1Z9(o)wZMwK_=V5BW6e(zi;6WIN;@LzJ*en?6nkwDG~&!F z!Y#TzX&*Zg?TfjyT%tJR$~rvpYjSIH?ay*UZV5Pf3ZNBjfPU>?>Z2L{)C<{R%OzU4 zmIsQD*sdl?((a=4k^CuGsz_w6w6;`=Kc)-`Vv0nNZG0cyL{$&?IggKw9UTYuMjh2a zJ$ev1*Re{RW-NzS4eT|S3-AHwV0*jPa6S$t&Nd8{Txxgr0c z`Ddw{#hNSXWbc#9kQ8}h{^U>Ht;NG{QGTohshS5jXgXgZyiy5ELRqfKsO=^j+>{pz z_Z|&;gAr7be;Ft&c5i5};N5iT60GVcu@sL;PR*Ar55Wv(hPgo3hw63yG7&p;54E{R zoynPtaC!!*^D|D==};DTU9}g0Scyy@3@0s25)hbuUv@SIP?U{T4&2u2d-1VkakAui zs7dKDgW~6$Hkt#5A*4bw9doT|Y%8bQrtC1;;I=$_;R8WU{*H3Eu49O6zdH>yG~hB8 zHyo~yUgga5T01h^2R_wIW!50X3OHn}Gx@Vz2_?LRrun8!mqH{Ywej;gLW}_>Iv8VS zSgH#c`>NBySu@Z-h~`I!cUk^?8?A~~;OqiatN!0pEylm8*8hYZ&I5S?VKnb^Ik+2a z<=p5kIpd?{4B0a$t@E7;{ds2Mr~i+ zV(F4FD{yl0>NpRweQF1)k`HR^LzGT9gld%iXV%;1Kt+C_vrcJDV~^ug->l;l!`vW% zX3a|Wf^vS^Vd2YoA%kJ2FyQKwg@Puc+^?DHd+i>vC1>wz;*f)5ar zAtxQWl}kuYDVhSsHWA;s`iX_{t3oZ#vZRfx-b|sMpITK$&?KBM`@KyW5xI%trzn%; zcR-`aobiPgMjh%4v#v|C_g_#Ul;drZ5Xa~>{iuWJ0xd8mRvj5;812g0U#@6Vt9>d< z*6$K@Xb$Tfi%)YuctkmWtH#`qg+)v){k+zvq>(YYr#Dk)Kd}P6x_*l#(fkg+`QS;b z&K(y47QVApCQY@Oic{J1@QUj8+sVlUl)NvgG0yWaEm$cbP5%5>4^SD<1CfL9tsfCX zO&;$@%nyB4QEpOrQBA8&Ou1t1%xl3TQcpjZX1p)!pvY-iEF@J9q%XS9Ic@gddpu^w zcUI#VV(9&r8*`^K+#o${utJ@)c2yfVEpn}k&jBp~&T)&RT3zYD(b8TEHlUZY%(__J z$xny;O1fqRP5lmRPX_NWs5mis# zwFYd=(%j*MUt`_YCn3gQQBTSDZrRb_vG)-y<_;He`@3RbYppsPNlKSl6q7V68}8+{ z^(sln_UjHTo7Wi=f-xi;e2S!VHzlfM>5>&NB&hx5d*C{0w}q;@Xd}_7d4H0ZLv%OI z<9-3R)Kdd*W?Gi0D;M80Y+>DsLKGsCQ1&mPSNe9?B2X0lF{vf9gs=!s{+Rr*i(BYK z1vTXGO|qm#IP=hW?6khpZ%m!UA*ka5B#V#zRkB2!k|zhA?>E(3RnpT>^n+9{O(Tz+ zdrm|?)m|)K%@Ai!)J1?`aoD_OixuH z@!X)AyYwxvTf^n{Bbojv(`p?cOzszIg4LdC2A*jyD=Bf zM<1~wS-IT-ZwDTQbMMX`4>mu+FBnUmV$U$FXs%l>z!eq;=K0;91qZlqr|njRXVPF)KJ{(^`JkH#|{^;m!vXKq9BJP5LQISUylGy>jN3YzlAjV zh8kn4y2-NUC5%mT3Br)dSHxK$$L^(kNzf-T#xn@7Lx@!?VPzFNIWbV?o-hE=t@&O` z_32CXtJ&6gtJaYH&1cRX(Kp?q@2j1ICxEmOVr|{LLF@eN#}h+XT7!Eer@p&sb5k)@ zxcjDCipXaM4!?D4?oGE+Sc*p_&*w|b4J0up`dR|i`YMfY8S@;t`&gffe@-6ub9lX1 z%zr;=VR-l zw%1ONzzH>|))tiTdJ#3%5?I5e#CsU*3f^F$%TEm$jLCInnA!m|KDuTYljz=I*<8%7 z#^A)?PsA!x5=O25d>ffan0N01bgS{-(=DdI>6Z9wNZ$qs>7Dlhx&{6&`s8+fNOEhi z;YzuvAaKO~m$FzW86VMS5~;qIRv^(qSy>hf)3lZdE7ayS&qsTJ+nT0O9_DxxF3PO_ zSK*3-e90R|)-|B6DnY_B`j&|-8VCXIbvulgU5?Jt;w#L>GswkZu>3 zfPv^OtW4CZslf*fa9dMhGdBs2REM|gHQ8ARd7!eCG&1oASY$)ugbn#vK*o4FOZjza z@9Zqp4gvqP)Pb~9ja+!ou@Y(qs?AGuQow5JS0tE6mxssNBl}c3Luj?m3>J%_g3 z4(Ujp+YSJvD;E~?Rl53>R@6uK^yU=V_Hn=fb_**1$qx-6UAT&^AN&JD*H;Im%2ucW z(sgvjb&Iybm5Dq6Hd*2QOD+JD>Y^momj4Lz8>NV_h!aM=XZFu)Hf=*S-1#c!<&Q0q zDUx!BS!r^`T(5S^_9@jGS`*`7=gn@(%;37OL9-?}HHWTS9E*%25hP3P7GQ7kQ71Sw zhkDM<;fxW?J0Z@yd}dz^`I5zisaD_ppl0^f9DJv5M)M<1aN%dd*qjeOblG!ieB&NJJoA zCH9Us4-}ZMX2_Phm@f`ZNm-UJXyQGs0qB-;oZpT=eD8)UyEWz#`~2yKW7QLvgQajU zx?q`Zv@`*OekLvHmH@tg9&gcCKAUVh;$H4*%_WOnZ->eT$l|ls6wm+xvff*U8?#7a z(qmqV&Z~EWSHihl2Th(~_66u+`|VCChs`^8cbahnf;8ZZZIUKVk#MlF73&0@S$cpywSuHqWo-a8a*8g^!1EeJftp=k-c z>g#vdpt;;C=w&g-OOxbCUm4R>wqf@OE`NSb!Own#L#ktn?GJ?N-M~H)4`zh4;J#As z5Y$D5d=E~BV(AOdq)Im%9V`O6iuQaH8Jij?e>OImmL$$~di~IK_-_2y*J_qAlt3=8 zgdZ-I8BNaQQo!(%3StYf!D3wl9l;XL0K+wdh8G9Qvhh)ei7cPU-R{`whfW?z;>MBQ z4qN)>U3WGv9pIU1Q82$K>mT%O76t6kapvbvQIDYsex-)b$$fi8OxLyUhvuI#!xP9P zWjh$dg!bsl(qPcO!7_B+A-DGfVT0TSwlQIo2 zMWXkma2FNv7x6;Ng1KdOzop)X>E3seB4+ajSk(bvaG|m6;?FVUo_ZR@-`t|S4SeXAST=Gk=F`>ttlJ%`>_d8y7h?_wN;;@3PS;Rt* zXL{KIpw0OkH@d7^z=4K?OX6 zh;DLZ=o;@4st*@*v0UrE|IAl_lquVTR#eJu{y`QPvKg(%A`G7codt;1e#L-!R(E+w z53x6pQ7zkwU4r*D;_jv^24l)@RRv)PNdW>fQ8j!^VlEMgB6Y!+Wj@IlGbNQRQzFzG z(n%j3DH4Gu@*Aekg2JS}1&ZSfraQQ;PSQozMvrsA$xOhnU}j+ig}ioTMEP=%a)^Xb zCTVnlScyK3V&?)thr>#!N$2OUcwZ7D<>gV7)%7O9jU8YYA94e*Y?L-_0x$#Z z3rXIn&ZSMcHoXaOlGKoqrf~F>2q;}={WnTf*L7`L9OkR<>3L-~jvha!0tW`NC`NcY z-MyrG5Q=;kh%&z-6K(W#nezY-qvy>>Rf+)@7AdWUVd9h^MYI0?4qyg|!Q+s;#{46%!V>T(o3$!qCkXWYUAHbECkoDi7gOW6X8>pX4+I`$GX>Qs)xokUZYPW3R&>dMZykZcLQR zT>5cbN~)OD?pHAtaSE!s3$+H|_kM(}R254Opp1QnI&!#&(U%Mzr-9q_vb~Af1E#<4 zE+;GZY4JYAwW5E{jA8xzvp%*%=KEjIdS#8P9jVvQ*=ursaACi1al7O3wP830V16nu zm+Y`0OY>b02j(Wx8Rx9OiU$RHw)VJ88_b~&Be*8+tHUTgEX4B)^`szVFh;~POvA1o z846g?3F4sHb_xeD<5`vrdsS6?3Lq>7O38R0)wo4b?&d9F(NSHWiD^!<0WeX0)~|-X zjVPT&?D2ThEnyzeee>*SZ&0tyR&ZR*h0#_c=X&tExw447wF2MOUA&#dwqDEaU_QXT z&jplFKvO^%iX>*S6#-`JML>{<>Y0^AN9#T4Q%B1@AK$H)=;KH*rbQ{qO)UKQIo60R z2TxoAyv0$XEU(hG#-QgP!6ZQ?vFKXB!)ta~o*sm0}A}DVa>ARv{V-zB&NZF)6%RU=OO0eknMix_!b(Cn8mX; z$!vJNIM=;6KV9RCE0`wN&e-HRG+E6i1mcQXEViH-Khkxn6BZY4X@x;M2>*;P{LlF02*4nFe@cosF0XISpvx) z?~kkiU_v6MEv_$SjBc1lk0BO^RW2eUCf-UcagaZ!Q6AU)bAx&+4-l?HIc+Gt!=Yow z$qHT}NXUgjf)=x!_YGn+Q)p@hLQnt;&4rsbgPOLdG z%B%eXE_|*5yN2}Vpu%VE4?Zh9ti-s4{vK6aUG|!;wXaJVTvrI|>jfl7pz`$Gui=bT zQOR1;b$~BW!YGAAMO?5#WM4(rdB2!6-wDt}4BzEKb?LA*VT^J#4+VUkbjekn0!-O* zD(l+#mxR^zHwlXbyd>etY^?Ps8=I2{(pz;6Po`-7C29R^uVZb1Y#z;GvvtTrOEYl2 z*>Bf{_0x3$`pb3c|6~C%t$R}SpFE(?73sDLC^AnsDPbIipny79$?2P;(yWMw%_TnV zzj#0p>39?l7(>3i)Qu405>l4%Ie!XzhsOLnJ8CxfTsO#JcGJz0n)XHb;5;57USFaO z)rlCIb>^28B4oECwA)pfd)%NHL&dFJfpPX$i>`aREtOQT$86ac`B? zsUF+MozaWwvR2HZ3Ab(A3@1}B{BR{dcC7#>8?ig_=HMi%3>j6*Y0&g^wk3E;ShUWnC%rf)t^4%E|kxa?d)nHG~CORo$oDEZ^+`$n{2)b1)GzJU%d$ zao!C19BAs=OQCjIzMkAOcEa2tOR7A7Eqo%wBIp?P84k2JP$mF+Pn|ZCily|&v-59F zRocbwd3xtMY5hqgWeZ#OE3;h5C-v@Azmnk_fzZdtZW7bADV)v~R-GidVbWo2P7Sh< zp!g5p#dthHi9R}D-Z;{thDa1~C(zb=@RGsb58Q`TT%+w>FK34vC8i@JNrvYz1%*oV zoAURRc{8}-0;l0MsD>!l1gnioR<;qthWaEHGk&aJnAW(#`P;Lp3s2~~k2U3BHF%e= z7?u#%y>Qs63t&8I$p8PfVqNBTu zaa*@tD^KdipFw>umB3$x7>64BbzP`}06k3LkMDx#pZn>%1cr%D_EctkOWAou_!ght z=|Hg4z5baM&=HmtXVfpRCv1#`h9b1zDxPy2u_J{dB&GtX8D%Vy(l1H_NZ${VV?Pvj zkomf#;MUb12+DnCT7*?`Nqep_U-v0rp?rn~luWb;hnT;TOps98OFIDM3ih!|TXB6k zSZQ8ZZC>c<0Epe^*d-7VCn4Y~sd8Soya$v}bQD2?;M6GWDJBskpU8M=I|gB{VmLG)8R}UPoS%d@anPAu@}tHS(%AWFGlnYRsRzfD7%ZD zEStyWw(tJZ%Iid9Ao2($^ETfCE~}+>BA9mEppq|6Ri>sCeE_s{KhI*sN{JEE@@NEVjhHp)v!S10y*dt2gFgB1fff= zWTSSCg%2i@?8f46)M~x?sb+jXvU3uH5TiHh4CB(VKNM_Cf}AvJEIN=LKpRmXxTXgy zjARC0L!OY!;)lr3m{&x{N^Q;T-?QWMw^U!Gzmd-|rtEjz3Bu z6d8{_;VeLS>rOnl-Ffw-li15s{bIaXECoIQMx8_07Cq)OZcA7j>6_pg2AZ(aH2zMRk0fGXf% z2A07^>Qp}O2F~okohP(rk2o_VY*wQkPwxIbDA3c2(?jd62YGCnetqVApM+2{%j9`? zS>jL&DY4mf@Abvx$#d#6C=m705cTZpKsnF8#6_{`VGewtW^Dt>Z2nugviNPekfj=B zHbjn64{Se?fTUSgKs;iB#m~qcI^xF^8we`ES|>U*h|Ztpr4#HI3>QrO9kH%}@;!bF zSN|YlvHurhN&iNy;L8`Pe<7An<$oX+4E;vYwZDaJl2zh|j$W4cj}o`T>#=dDzYr^w z%(a)}--y-O`Wvyf#ZJ1d0*|o7LG<^2lnee9709gy{~Z+={1X+ZxGdcb%GL{36w5kP zDti8{T$y|-SJB`^+odAUiE*MrRL?iMgnYXkhG-48(iFz3qwS0LRW1nHYdIy{;vlok z8rnmx_nZdA1e4$;s;{_2=%H7p@r6mdU@yoiZLVuPpVMYIeAMd<0+mHP)wAwx+0|Tr zbk#0}vMnD>m^&nJvr=iQCxA9G)NCi)QDgE-n`Ol?$b6K$-#%YY=JY1RgYnvKbrO|Z zdk_X%hwappMa^ju5?%rDvesUI1ZCo&J30I08V zGX6~;UPNz5q3 z1d!Y2Z_R+_cf$^fUnilx(bZP>3Iof-yF1jlRmP*ZFVAlGM+$o>cGSh4WDU z4Ot}Gv_eF|>>YkGPxO7ukmK5(^Am%fyed{r0T?Ou64E!MK zWPa{^rT5gv=CVc9-tm=64klmE_3`+VXvKG?8hp8j`HN`52$7MsTju~Wtv=@_5AK@$ z3|g^FUxD7w)Y$DKI^>c&J@mD>Dp%4uS4OTB!cczedsy#vcydNV=&ZuoPeT-*9eHgu z)GFPtw@#V9d|nxFBYx2*=`S*c+ZYyZTs2{u{DF3@G9ILT5nwOsV;ZZiu0N z5$=8c4OyUGH2{W3Z2QDT-xCwG%Q=}j%a5J@lV^qe56@EkwY!QOA`T{>qvvwelL)6Xx01N#I0xFxo09qJ1gxT(PE}Z(ff*14+al*zRfX2 zr=xlNWfuxk4Mq%|5I-fYQK@izJY{qdmIJ{~NjRcm?uXs1uXm5m@)Q%Tf{3B~nFvuz zBp8Uwpxm>GRNCD}X)<@o+dWP;s>2_~47-2=FpZKunB6oXzVvqj(-F!(nR4l`)}Dxq zOv|(;H>Y=m>muAZmtzAiF=!hD?z_9KSEenyDT6LjpJofwr`gg?zvK7}-Shr1%b^Bh ze4+93lfTWg?NDF*dFW$W)Tfgp$sGGEE78Ww{97DUU(0ZsTZ+ema!=T0S zU!e8hG|N(DG*23~s$7Yu(!OfwH)vrlZc(n1Z+7)^{-Ie@F==3bKH=$0xeDoa@ZG2=&(D#QKWKsG4(4K2hx5f8_-F(mrzn zZ2yrH5d4!9z~1aAX&oQjz0faGw7LDwvt0k;Ss?*faBXO)XxF5xo$6NB@xvy@RHn1O zve&d7p z?R3nf=ZN~ELU^%p)q;ds+3Cm62NExtHo{26$%n{`6X)S{su7mO9hxF83A%ewAnzb} zYcN}~@yq-Wf-U!k!@htwkEXu2fAcJ$6iU3s`Qj%3CREK?yQ3(DF8D-1RURaK%~dg( zG`(OKegQ}AP4WO9IsKFFI+D5nJ>Eve62N2#2>`Gr3IO2nf2`W*Uv;T;f7PVY=^I&@ zIXIZvSpTW$JJ&q5SrtY6$nNRCD&fM9e0Zam!&cH1_i{R$Q>p$IA-GgB5g}Yoyw6Jh z+_On7EtzQbOA6e7Qu^CyKy?Hlqt;G z8_s(mmhz)9?ukj}i!eqO!FPu=@kytl#OAcYc;o5%qc6tk67!Hilj6s`QCw~~Tr7zq z#nPltw}Ee|Q_COP8=tF~HJvG`mnw+WG@bM784K1haL(}$){_kFWEWCvGIZk=ORl4jM_dkqnVY@HNU^7uB&V}~SOrnb_r`+wS{kMJ6glMzIb z8mN+FxNy30Iqf@pT2F@8TidH%I@ZSrvMH~l+3U!>N+*!!4ib;xWI^8tdp(Qm1llp= ztT-#vFTT~8(l}88SyoM`Tg=#1{95RLuOjxgFq?aW z^{qf}`#yt-^J1@e6E1?P=4w0b8x!64GMWC1xXH%3BXq)TnfGGdwvp3uzvfVyX^fLZ zmBNHT$#B<;tpb6!1HDU}&ob#oooQ})fnh2_7KD2L#8t7Vv?D-IXWz`yRD0=C>!G~I zDZ*~6$9+$*uw>rMV6y#8??5_D;0Igmd_DFxPIOzm)X&JTuyI3WlhCvHN#cWXs1mKU zl(mYa;x8u5&$R)YTjzt1c#E02Imojmmk4lIVO(811m4%^Q?A`MYnM21S1}*WA+KPb zd%SdjeCk>sYY(ZweN~Wbv20P3R&N;bvyEU5i+xoL^Xg#bzuLp@S_7Kb*Kr5f^ZFnc z-2{CDoW(`g?{w*(Po>ifCbwyeOnZ{&$;9@&>OC<>-5kQA(33x>)TFXo`T5}!GDb2A zilUz@G)Z1z!5~R9kn!nA&l6^{f(#btE+Hdl_!}t7Swe8(7Pn)wWcWX7zQd8@U*aBNm^vV(H9M5$RcVy*T48(109j+zK^j=Nh(|u{`AQ*@W!@VY7+MYAin2 zN;Nl)U(J#+;!Aao|Cyy3==@zW4SuHh+0)$`98H`O?FSQ*G;}a1Sz8zoB{3F)U1+Ho za$P6}Y7w)T2QtsEaZdB9xKzc323L*Ah=WqyO{oZz2Qtu$9~3HyubqWK=?>k5(lq~huzJm1Z7w^CmJqh zvn!yC1N+_3HC5~vVRFsoScz^^t#ifq9=Hcle8-WAb?WwgUPHK*5zDU>&yjON~ z9Be!uz5M6OfUZkk+)U_Ud_C7!ySe8}8frY==pi!vr#D+}>`d%;0ME^hheBqNshqS0 zK+%9%n;IqT9M}h$)pIzbVHEJA(Q>Pl!iyBe1hg0_=iYRoT9{necB$rw=BFr)Q2G3y zkT;FsidBb6P#cYD>c$|Q8G$Sl_pBvgAKlcW#nkpLbNkK^ZNdLmM>A=N8Y?Y@+VvoM(Ko{rm~a+vL7VMZ6whX{7K0V z?`Mi|N(1Wn0NIT7Dy}n_UD0&60Vx^vb5tXm!m@?>L6FEGtw`<_qo5GcpNceKM-8+D zXeN+QFv?kQ4vdK#M{vhxMz8KIf|&} zi@A!D40=0aHZHaqJhI_>cVwD)Z_uGo{&r`n!g@${eg;(}BFTzCSm6N2bBcHnyUD@@ zUH&YDG6tmJ{Tr?X*`Rh)fX3x;Z()wJ-{w0+I8iRJy}bDX^98 z3IGKa$k|a&@inKCcjypz?}i-GIg!aYB7Y4)y2C&lGl3P4z}HlR02s>*;I&YHJ)AFl zN(2RUva_x@7rbJImO{WEsI?1nX8?~9yRAr=)V|9iguYnh(MoeNSW8^L@N;j1Vp*~S z$w)mqVb$?aQ?^P*nW(9waUBZ8dQqEKY@>iA4!}!$P>68dF!Z1z&=tOenSvL3&>rE~ zg@L%o3!L&l?1LU~_IkDXwZU5ITI*E@%%)(aPJNLoE%Fpjr;-9ZEc2sJl;hMuXfldd z3>Na8$-)PC0hq^3hU6;C65b*ndb;x)0ro)Di+5UnHC{|I(w6X97U6zkbdfUBzV|jW z-q~sQaZHeDeETtRPYVZpbJJbWqukoz9q#M|9|*d5m!3E&W!;>Qy4%sQ$nGv(ripCh zR!&coamAkJB5CRUVvT#SeL9u0Dc@zS>-2E-JgNAs=l)Po6HSQ8=0Q(cbI{sU;fL3O z$%YxInsZn)lFqc|{b8SVW>Ha4D|hwOQXS{MgSn2WHD-FQjhnhic8ofrO1Swm6{#Z)mhcF>yYxp9Gq(p(_ohCW`hD)X_j&APFfF_aYo2KKzG zbsC{ocyZE9=UruEJy7in)IANx+Ty1r#87$-`BdEY1|P&Wah_r|m7-p5mA;yBa9`Zk zWoF$IiM#UWU~|rOcI`a}?BIXSyb+xC|D4&RuUZ6Jr>|=M`&)FET3&e_;L`5rX8S0J zt&1K(fjV7+Gu^5lhJY(6?>7dpaxAq@fYU#<=e03#Ft&Hrcl>{Ly*yT*5!so|UsUWTDIIA)2qC%; ziho5KP?gn0pEo2ruXMnI0rQ8Ug)50uhSkh|yi=2h6H>&x*Ts`lteFjwqV@u*ozd3P zJ?^Fv-x0FuUwdRe%e!JspaDv%Uj`)(N#19ob+GIE;njmB3y-DN0z-cI0+GA(r5Lw8 zBJl^uGPKRlh{HP$~Vi6N1DY`>!Lq({M8o?_EY$;-JR-7$`L7ysaa^-SMh$YH>~ zEotDbmlm3}6iR98p2?v=!}iLl)hSE5E3pr`dV;whcsZro@LsSfgG}rPPZ3yft0$)a zS*XGN`~%!(ixIDIO^aK`u3ZWfAei`WlSwC2d3ZTTIre&nW^~sW{dv1lHSt3oGY3oL zm?ft>+tdQ}j=3Nvt?xPA)XWPCspFD@w2t0ETR7@wywy8&cla8*qP_3*G+2t|Og~2d zduW;ARIiu=k2*=cxm$Trdwnp=6my-W;&K7zyq3QfT1I$uqr@t;=jE_Gd&s=^KA7b! z1e5mi_w*gN;vnS0v~mlxK^n*{;M4R;t@Sx**=p)@cmJWL z8avMGkq+Vl3ZWLKf=CSNNyb6z*m>;E;cNV>bpaV8*EG?$L=uYe2j7I{T)%PqE&GyR z-!O_>Tqol}CPS;(;AYW27L4Y9#u*UYwJoT zAZV*1W00=u6$@w^AqGk<8qpeqla8^_NYIxfdXCn~o*;i*Idi+QI1i-^tgcQ6Fty`! zcVZZ0^$rLq5cOa`hD=`-LyxEnaBW?k?^SCV$24zpcdkIWz7AY%kG6GZMe6Wr$$Hy) z2m5@@LZ7$jY{!Tqo!nnPuQ&~1;}Ugvus>T{>+*(~b!2)ra~$w{`x%j4gSw$lEMAb> z<%-buVNC4gAKK|q8dI`a!a!%pwS$dUy(G0Dl9+Vh zI>tgc>-xxkTvHTZnDvbAqe?G;r(d%t{Gitn?jW$%Wq>?NFlX%md!SaY#Uj}N>!dhj z(0)SEvS7vL{w;gZP&7Z*!CQ83kYq!gDBu(+ z2iu7xT$gi_4EnIrH2F@ngGU_Z12DzkDS%ojJ2#vq=8+rLmc6gIynMEr&R`Y&)yML@n2PccAx)TeUpL=+n zu|{y7X8H)VHUmaos%3?kN}MsmDS72xjeT_#nLPK_(Mqx(eF6{6>5uQtXPBYA===qw z!lw24t;nmKv&sfX;{(YD{nMf>^tB`vSvxReDlr%_Plea5#fJuHtmPr-0cFk1LMFD< z{ofTsKM^_z9&%TJp4XtwRzr_p22(#=Vs6yT#)JFox!x(3Cf*AKVKhd1HT*=Pz7(Z? z?$i=#yf#jmZJ=SV3`Kr+`Em7$euhS^C?$gc3cz}>fjRncSZfKmvC&wvRr+{?RCsNf zTW*fL2|Tq&I2f4hSiDuLv-W`JB5PSE+z_WwxF^BuXXIqAy%_9%b{eZwY2`$oek5bwY=Ser46d3jjK<7vur(6u1A=ZAa zr&{&jd$O5iu=A`&l{GW;{8Aa1V}*6y;H-Ie!lj>MHF2@lP`-Y#De4G1Yc>`{O_4`u z>_2DXxx-R9T{SG{G1{lonKKuQrx~-~pS5XXHDrxZa8@f= zrzyt!Xfl$#w|_~un%8%+dl7;8ke72=w5Sl3KvgDKXtgbZsv?(CPbGzec4!~7b{kXm zeH=J2KgLzFvd4+n6IVyFMcff#jF<`L;tb0Gu|fBC!TWlt-ugLk(z^OlFxfkF#|6a! z(vvDp@#j$Hj3zMqUB0V#N4XOo2yfXknyjI?WwQceY6m-{&hWH--wQf%jQHpQ8bod% zsO)K6{ZCUzp)8)kT3Tqwywdw11L#-{hBi0cG3YS*4k*+T-c?3GCoCR1zICzOyVDm> z>;e4H$93D9qSx@Cm?%S<+_8fx7iw5_0HN$5*Y1ext2;%f4%!5}bOT0*nc?|6hO`?D zTTgtB5b53aI~T=1C|eVg(}@&;lSMy4{LRQP#o?Eq>#-T`lbD@fi0wa4jSNd2q7bDF zIqr5*OeepK5Ro?ZkzKA?)iF)Xkz&bGo>cqEqPuCH9yOy&Y zTOT*jN)Ma4REhMRic<#HDOfIZIIs5EzU#`UVLK^Fh2itGGsVr4>*_Qrwn;+OT zJc@OGj5^yb!|e+=(ZsCf8k@4vsRd=iOLCQ~7bItW*waLoj=66Q;3Av=gqyzV+PD7t zZ(73gKj~0RD3y{>?&+Vrj)&L6_T1ixq&6Ec32v7>s$rGVOTptiidL|L;$TYwE$XyJI_Z3uI!bn26zfGZ8=<)PdIz4 zlX>IN>qYg}AhJyYI>u^K!rNk9!DdW6o5DHT9Cpo`3M1kfedn32iuo`ANJi1B@fNli zvjAa9WpoNORzyeD&AP{)zn<0GzZp84wec2AdQ4X5)7y;H8;uo`&Y9M4EJ81ybl7_P zZ*t4Bq0A08t6D)6GoE#oN^;cC)ZpY!m~oz3aReGmW=!T+&Mp>+XDp=`9p~o|X@;oN zqjI4umCjRpT>TP^qkHtxz_}0ml@!BN(+#~DRNJZPALila7(i{vH_Nn&KA0o~4s&i^Tr}3KkHM0Knm&%O zbN)u3wg#67Rdff5pV4@3sGX#8pP!6#TaP!0Ey`4qo9=aFLO1&K+W1Sn)RgTlalUcI z&BsA;hIrNOX+|4oOCsMvQ?Hsffi+~;xu;5_Yn;Js-<5O3=OcsgqYL7Mm2Bu0e58pV z9#olMZa3e!J-#iRdQ7v3e(lMi=%id!H}0jJQ+^NahkG6+8mknqg5%xdkWjvycHjx2 z5?WrnV{Y9&X`w1uLo?f6QNsAVJw@%c80Z&Yydwo-y z*$VcAZzw;bT)n!!GwkC>oUgJ0NX{&|r2Jg|0);N4Hm{-xFsa5La-5VCo3vchSD4@s z+l1oaVykvhx4ufmYWX#G>RLd)gaO4X9lt;pqt;0jP%4MaSxQ(ujGO|2q{$}{8NzcZ zRyrH83$#m8sJy9})oM?Tv}&iCbu_pl{+DPO#Q1JJmlTqfA3Lcn!C0-Mc1>$nQ^?L0 zWOJEjy}=@-CCIMQTC{V>uGqRauBL{Qs$BHkF)*2x2UHd#eTlQgQc(L$xO+i?;+GpT zShF}ze+^jdjAxh_n5O8eiJEFNSF+nmE_k1Fs2fnv6GL2KQ0)0HMJnMP*+_Q7r6mj}F*cyo-`ZE_UPxXN(@Q z3$jT5KZo6yI=ptVv2zHdjz?Q7Ag@n02hQ8x?-^~{wK*Rq)@W;oJ!v_#sR9)R*_Nlt z{`@Wc=*`s-CzTei&1GqvGsr_*v3aApdh|cBP-YcE&5#s95&Hw3txP6XASoGVl9$HQ5)s;YftnyjmmA5Nrg zJ@A@P8j%P5b7X6y7ZMigoC+?mB*}14rOD1igJaW)PMY0*U}eT*aD}>I%Y^H06QcIC zv_UmzAZ=9!@GsP2JX<0&8*Z! zahO#dj%~nZ?^fVA4o4)^_~xrlLdgXZxNVD(#HNwh{U?I6XN~8I-2#)GRK{GIy>6l% zAFg9%A&tm$+u4moFdhWB z8WU2D4k5BvL|GLlM;6S3hYXS<4-_OohRl=0u8>oz|N16!8zwPS+LO-(g`4iT|@gNTl<0}t9Q zsP@HJus;pb$hZRKP~F->b9rT7Ai+>&w;ys#jam9yOP&ti80UD6Ic7#ldo)87gwVpt z=}6-DFAG!_J7E$#pX7zA!A+3Hu1?D$40h`H7_W>Pm;Zf#Wzw!GBe7G+;i1N?k{J-s zcb`4e{s&`p+n(&}$NX)?8`G#M7On0@hsc3B$s57M^;c6reIC5UNHKq9XLeaHg7tn_ zcUl3v^x2(fBoNeOMpFc)sSh=!O3#U5Ib`d`qaDR9k4Wx#Uk*2EY~ZY*NuL5^>>U`n zlYqpDF~0%YR;17`{m!U^?;rKVZ6M??tpJWoCbU@alzJ)_hmQn(KXuzdvA#Oovn3o> z4bp;J?&WoxbTnSB1h1;{lhiUOBTKS;ylX)gM8a8`Bo(FZAaemHU^pP2)jhA#r$k$}ZAp;imUKESp+#;|;c^2ETXOoE`?u z+n?=%D#>;=Vossr**BsSOhq0qdyXAQB0JM@jNoHSNdmjbN}14BI6gv-trFBJnfz-e zaOnH?IP}6hk?ER3@tl0_MeOXfIWi?Qfk|5VRxhF~7OH3S(+_2vZUO#IMZPL(U6~;U z%}THpdEl;)Y(A%jNyVlp4&7p^+DLuWbL6fay)FxWA!CMW1BIlZ=;1E+@P1DV%jUxk zdS$KAfRQvC`R34qm>3#ezXS!fpunOi={`s)zwRtS3au%rCO+@z(cX?;Al5+v(zce~ za1ye2gQ1;7FR9ZCmSL^1b@*As)Jb&kIntpoV5#)ir>~n<<|&rLGfqpB>am z_isk~pnBBGkZ?bgzmz$z%hox9*?!zJZ31GKvY`-tye5ZS{CXZEkAlapWpUI`pUH%M z1OuY{a5=S+*iIeqe))zGohOd(S`siw zL@oV@<0JG4WEVPEz^4nmsl8zeGUIu5>hSQide}0x{EI5gbr$ff>9KAKJ`MT~kN_mZ(SI`NSl5XBJ!W!D>oO z&eQ1H>$vIxXAdJAv}9cp>Y=yH7svXOZ4wXQ;Cp*<{Z^MY51)ByzJgt>3)r6MJ*}Fv za!*y%3hc;B;O<={IkRE;>b0t*Zk=qu$_HcKr@TjvB|Ka2B zVD*!bFzd8E1RD+o!bg&ah996jNQ6MNnXb8O-)kDVpOgVcS{(F&fs*0 zf!=p4t`b(Mk!@a9U8z68Bu%}l1vXoHfVkJbq~utLG0)|`Ws@ZJ?N@Ckg!Ee#^u!$m z_6!VV_(fa#1d`9vnnL^uC2h$D%c){g24r;hdI*L%E1`-&4dh;k1(vL~_A(5e{S~}If}D5C zy1s{ymfewxI;L*yjg4sB%<1#1{fQ&CCZ?Rn5K3}Wr!WQ$spUQn6)@%!4TyrM6;z<1Y*z-}VA??HeLtThc&bO0&qhrd@D*n$?ZG6Y z@otaXh{6XyrCovTeo34|&yH=`^Ts>f1E~42>(NR2d7Ogx<)zfK$Ka16xHo%;l0gAY z80ue2%CB_o$Y+z^mw1PM!5vHLgc*SjnRTDr#yzTT|K%##0Au zKvot(_>n>ooKG8In_xR2hu-yyDG-(w3>Fujwbk_JRv$qC7%((0CatSy?bfjn2pk-; zr(o(9f`I?mVDM~w{;S-pUOpzotb8I@3vYPQb3VsV@(o$pyU<0- zep6b-+hrYT-|XhD)9zfS?Edgw3A}i+9%uFQ=*DW0Udp~4vP3E|?O-@9-l`FXslf8> zLCk8gzJJp&>4;j+%1}{$)tL-vUG07OYVo2uv<^FaLCd6{u$UGu8Lf6)F#%nzc1Ud2tLPC}Ep*)s zgTJq?_N%S6hGDg=fBU?cFZbs-7^}sUWDm}c*v6S?b1LkWKj-W(yC_Q6@*=wHWf?8! zrX_A>cmwycJM+LT`L1MT70;#?Z$sDAyHdReW#zK(Uv5vDgs7Mc}$A_QON9psz8j-`+*`a!!NfDCw1AMt@ll*ZMm|) zw5Bk4%7}D25Ngp-AKx6*9ViuHQRDUU2O1_WZLlP$6hR_DDNE+43IO%ddmCzSKf zZg+^)E%?D7L(42OYMXJdemI5&0P!~F9zrl430CbdTz@$i+8F1#+wiKtTX-XJZFvqf+8I(GT%5eT{1v9+P8gF zs-|qKkFpM?uaClWdF`2%g4$0>S(*Rmu2ZDxk6cY@l?EFTs5-SELzQ~S{92aSDaT$54u-nj zONgy@;@j^h!d~kM1xuX_t9f-N-0CPSqcUPkHn}XGCOJ$(&$lZYpW>GTup&uyWLk0*WVwo|N9?XhaL(Jb-Ry{ zTP+Z5%nf}s2rS}CZ+S?~wbt`GisH{z)S>k}(WewO1>OLIELs!rW3Z2ECG)_k z!!R*ZW*q)bOm;QZ+awC+%9OTozS@rHbmdS*1<*al=MeTO(+{}6*IT^ps~fm}mhGs0 z7Mj}pQ#QVXp^dHapV+GQXM0a92I$ZBo{>B*nq7EZ&q}!+^}gNu#(`=aqft*azNS2X zsTqKNoW>M7AHXRPp=BhqN+LTgw4c$ZRKFt7lt>JdkUvXW{A~Y@1^Hdhzb}^O$5+h^ zlQ+rAQkI_J=@U?0x1NnPKx~UuQeK5@GRyJAo#Z}{6EdYu`FF5iN=(pT-)^xqxS-{} z_D?l9?HaFlvd`QoFny$!&CMG7CDphco>zDH{CyGFAQ_;Eel7q#U;qH$e`*mp+Ur|8 z*#4fYV`TQZAgry84IPb*{;Ucw#lM=q9#TQ)jHTv3Qp+KS%?iPq`M1XiO@9%oqGn+{ zj&BUe{cIpbf+33QR0P$;#l*Fu{mz>)aWK+UTc=As(x(}I+WE@eQnWgEpC>$ zw3OWL37? zX^;yOAP8hE5h1>?+#TX-xXbfhVE)1$9|OwCTbh#+^|O55P91R?T&9&Arf%(vB5wB1$I&ZU$d9B zqoz$TQ>J8XV=N}QFlh0dR*n2aitql}39;;S5qk1*? zx&rofs|WPs?bt|5!mGM-EE+QCuh9&VsJEW3PfNASkttG&+;8X__*uZ5EiJDKV`&ImP$Uq($&W+!BlG*GbSihet~y&{e=YOB&%7#^r-=V^cTn`F#h zA9Qqo$w~1N`=BU*DLI4iWRYL+pa>Sp&6^XV^Lv&jpN+_8D$SuE66wRERQ6uDVG9Q4`sJF3VYO zVBpB`rsdEKW?*lZySCT24)kyZk)V?_T3j;mR*I02>B{qGM5`pOxrK)!fokS!ja!5) z1K*9@HFXsmQ8!ScbT~+jR$u4pB2eUhT`SIMi!0Www&y4iUyJI!=D%{nyeqVeYZSnZa5;xf;&7f@V$tNp#N1@51G<8Ob!T4fKTP4!= zXL?)G>G9Ug|5%G_x6=)rzUQgr_gd`n|JywMXBm!Cl(t3Whwgk*Rbpo};^ERwf+sle zm;348hwNs;H4;Bzle&Q-y zN4ol8+DmLfqk>h+%@Uwv1}jw$vzAR*=v*Peu3*i(*>Z?{(E_sqW!ngR^AC{S_$KjN-&7h*RN@DYWL6qH4^< zRVg+jMzX@diIN#30x+&ay~sEkgUCt5*L{ED>>d&bB!&3(vD74xG2QLzbl9;QplTRd zUv$RYJTqpK07!vD28WZnigR{6Bg?qA0Z0EI%Hi5 ziGRr1%!8PJGLSfkyW_}H&?Y$@Gvq{0k}=mx^UOd?YOg*?yUy01A%=BlcPF{rS`FFz z)N3X8(;fO{?%Hr2nI3**K}O%PP%Scj9{0GEL$gp>vVXRfU7z#VZg)6O-f;L}s^L|kr*5m!rJw3fwuK{1 z9|=!*aKw~Zm#*utD5TZh&^=le=vM&M2cH$9IA`{d71e`+YPbb&QtyCyjz}v4v`qHo~L0 z-?Bj|i4C_^XDoB`s8c|2c)m{nUfcp?_GNxi@7V538VOj0uOu38y%{Ihl&ve4_9J} zyWzw+XYOevg)0Ij(I|^R@&gDZ2W?dpWQj|Ax%7pY4lE8;1Y`pKG8o+#hJ za$MKwCY@HaxtX%oG4E+M6vfXnRMrU;M3xd%lA)SUA|C+oNkX(UBwa8cJ6cvW7b$nQ z#^UW|AOj^J)YDOLNPiRi03TTUP_)}z7+>i^Q-X{*YU#ZzEWa%_RK27eiKRi%53Nw& z&w_!TgOZr&|(3ksL1jejw*T#*-I%V5;jXSGoaZ1f*!Vw?6UJM5~`YZo++# zr7(^wZrSv}@+dF zDzZ`r*D-nqi;SV)1o5ioGdONOhSh>1VAIL6=wB6Tni{X{^lNeI!qsyLdDL|5RPn1) z8wr%zBjnAC#TG|u4ZTUOszFbGcoc5t`Tw8T1By>{GM z`;7gZ>(TV|#OAO-_GGwKY09kNPzz*~7-o0l2CBwDTZ7Y^=V0OjshAc9n*}`WeS#|& zW|S;XaTKS8x^dtTodX#NFArx-mdIQnFF&QA2KFJJ45JEg${{-N_QO4=x$SXxcK+$K z1IF4sV*fb)0cbSc%t>aPp<{-h)EL569e8dGv-!5cEWLJB>N1YUs{84hY|iVo>eu=> zp+;sH5=(cK*0ckYhvcP%GZ-awN8@lm_M%)T#C4fKb0ymmGO^e$8M_nV+(N==t}nx> zWVeS37o@~GwCS8vJ&e_Oj>R?Nbo@?=2zZzVmo>tC12-q~{r#|)y2&pciszGpM%DhI zV7})zyFa~p^^Us4pd+htUUu0JsG4H6aSAX~6O={)ySgwHn%#@aMBvB?matiPcyCluVhfHEM70o38CHw5Y3iWeEL zQlaiFO0H;3)A1quL0b3+787Cypy$u`=|`~G<`)UeahM~pN!l9}La!mTE$UQPYF^s` z;#uUcJA~$d3oZN%7G;-i7RBm;MXyt8MPM zCNleZsq1IZb*_>|aM`a}+H4l)(hx^nNk-bZBApA$2n=ZnuLm$)=R*1QaRJDWz}h27 ze-No4PBiZ2{N%*Dao4TtPltz-4x*p&>zONq)-!#&7ZX1$`JP#F@XwRE)7}8nYa{|K zG~SI*EP(jVqoK6;pY5E~R~^19Gb|cHsXb|Rv3D+1VfrUP9uRzeW+U2^0((0hik;y*lGtAqZSlIK7;eYj7#g5&0=y%AVk)&8jfK?Ty4=6w2=W$?{ zRh7a@2Ug}%qAv^0+au+n9*y1rA#qFN##WqEfJ&o=g=_6A6v$bb{aJ|1@ATPt` zEV9t35{9mjoX#@Pw<1d}WCJB3@jFK|FB8F@kaJe0qJDNSD@wYOcevV-wQt*UnwY&e z_|j`lhH!XqxIbnL7wI~Rtb++n7H5+p%c#|3G0?PAst2hb9}GU)VGygD$ryr4v>E8- z65D*2L^jq|BvOp?p*|em@r(wl%fuCT#9r|KI-QThmgE=9BatUMHIPmq`bnZw94v!1 z*b{ER2I>s1thb8QPlQH(0326CKd^vk|77p}Wn&>bHN3Qx!0F+@?d`!V4i_5n-cmYwjIxIN}=I z4e4+B^Q#2#okNW;9`^2#@F|}LTAJZ2$9~(3v8MTni;shlG2Nl36xJnfQd(V4&(D~c z6jww{inMjl4p~+M!P>Q$r$_9DYkdRRAy?I}z&hSX^El}>dlSkHqNlx+n6t~(xsjK| zg-QU-QN*!{EA~3}o{FRhJJuYBbO((<>e+!9Znj=khs-^qk*|ExE z@@A%V1uW1=@BY@wgRtG_3&r?4UEw&PiFl|HKJ^6RTUPZPYjG5qjqw9<9D;CgNtMEN zQ&h7XkSP0oYQ0_cx{Rv^)MR!R5x^omdH6T_TLcSE^?U=raiM2G;RBuJlSm_TD{;31&iF`Ix_X-QZn}o6FskXi1%WD zyu^t?q>zZjz>pX~;ywb%G$TAckzI47FX7_k?XqO}#GDahd^%eNR_Mp zDIMENh=^NI56dD=AD;B1LZLY z6zQL~nUsMtGmqpf7vvFB=2fJUu)rO6KlUBz@;+Pf*xX(qAKFNqG#inGT7TfM}!uHoj1@H>lg0gg98Ghfh zkraPc9kj3$lW;H;8OW{_((Y*4;UAj1PzXy>L*N0p*a~ohHg9lDZlxb`R5K0G0c}P6 zNJb?zf5HP#(ZZ_s`#@HigrQIb6@O|%ED_8LAhdJ*j?R@_#V?SuNCtd;(9RD(RjU)$ z5?97XeRtDQ0p=1O+x3g+m#_%(YUwIBec_tBJPz;nL??mEEJ=y8Tx0jjmd5(8{ zV*KMC&XLiO_oIJ(yhZktZfrv6-WWX4fsxn?9ijDowpz%6afC~Hh309g8bn&6%(QJd z;*JiS%;d2e2P28#sMN{G5dY`!g3c?x$Y*M7sCvgJbs_s^M`=-UqGS7Vr*;qq99__5T?CD6ZrEu8Z}0>S?V=v3MpY!Qwy3@-CN@hM6n;B>Cw znPPtksICJkHHpnYX0>NB`+hU;P6}7dSDdP4uNdPk>pEP+)CsctEzuo^-c|HLS$lya zQ+lW1Xjto{>7IO}_5`qao2--3G$KvYcpioQ_>(zjbBooW6`g6a8?&gF$}v2z&kREU zS6bWpn@BHSBaTOY!)PcBu7zlhly0-+PoaF@SGsrgz(X%FG)-mo`m#Cz34}o0^L_Ke z&|yXv+$J#0*`oRqi{dR5d#}WFZm_$;>|YC3fL7Ou!tiTe=VjyKS{~Xt%!EX`_io=^GrD~)U3}RDd)TBWrbbhX3NW95PIp&^sW{=CBv zjc}sP5Z->=fgz1!puxyBEJvVUjgoyb1)wz`pYF<}il>hUyF~0huZwPaYR)7}c8-4W zt`z|b9qM2m*~kHc4ePEN-`J0EkD zNp6ROx#o8?u!>PHs&}d*Ln_ub7ATP=JHYz758bU^}xb^TAv6*v)#A7qLaAX(Kl;@-ZLVKN+>6#s;C5TSI3os zPBJ7lqNjXy7>L?2EA8NhnEep)Z0y^!W77=;WcoeZj*OI(L3*Lm7NQF7HxXE%=I3Bz zGNpyJmCcycfwYj>Nftfv!4AbTEeE(GVL;`6Qj=A}R@I``;YkKOfH z0^cBM4sMgioS6=cKE1HNLwp$D>QGou8hkBICEP?b0KIF>{Gy$DqAtDZ!v7oWR=b8N#Yw3sW4rli?(RYp;*mM#3GS`V4rTMUrg5&S}5AgfwrykDqbhiHzT zkgpB-4QT)6U0-Gk&r_Wl)sE&slZ;mgclIhQn21tFz(EhGWwEv^n4Xvb4IkJj-17 zVMSe-zcz1X`84db5G>j7q=F`w-9>9SP<6EN!nqs$K$Sere^JHv@a_DmTWnYN>`vPh zlerz5Q5^|Qf}JuneuqK*#z7ou>^!5efBGmQ34rEIP#kOYwv(t!ew?+=$)wkxES}_R z_SeRN00L@!OYA@p5e<<&erZMwl=xO%tPth+&J-n~dL7Y+!j)&7DkHr!ww;T@43NoY zSYl*->BmX{V~Z3`UZQ9+gRT108*w=JSVPGix4;lv7?&x8^rn1vUMh;UUTX#1Jn7zk zU&PJmyM`sz+H_W0!>3=!>`mZ%-A& zI$Z1<;N>n@s23cpHFuY>9OYhVxRs5yt6Np7Lsv;>RUUf{KrgvcP^zg<2!8QM?Kr@z z@o7~X7coDYU3~)WsbkCQwH*Q;WU`DI_rG?F3l5~IEZKKl~^>C--aO( z^8IUsb^>IwSGA&(lU&6#WWCc_-nmup;JY59U1|~%XzASloELUJ5D^XW!L1rf5B(k8hSSo)xL4N-v zXvMzZAef&lXnn?pqN_v603~hiH2`uCpq{l1zZPh(2X!LLaQtaql~>uu$s=Ziy2ngL zc)SA|9p^4N$r?Q1Ai3YcfJczo2zfP@8M-clamRI`XAz_xC!11*{MwjmH7?G6ngWLjld<03DJ6>OI1K&0@V5 zGTuvW_0V4E>#hcLVPV`JOv57%Lp>?ro)?OSMlW7tc?me4BP8#E13GZ@+uwr^-iU_m z{0fH0(%xb9uC|KC$<24u!^1%{yPHFRY~^m#?rx!e6R|!)tuJ&PNPhf9?IGMShRsdW zW;YWQL8Ilg5}ok(vXeJf8cI``>gdBUQix<4ElE>@-Fn>@gi6zKj^9DI+7@Y@alxgI z6!aUnHL#yJV^56Ds|gh~&;=zh{4tfcl*uRqy@%7*O0BE!blV*NNJ{e)wJxp2fn5oxKUQ`q@3G`5fXe<{n6ae$eGU z6694q87tOYGh+^u*i$ap^1Nh8-XK1fW+n-BZKa!ik!pzmu9`@2ZSSaaOdo*++%ETw z;`aS{v&FdgNJ#Ql=W{)F;j!ds7^&CbUxT9ae~fo-GG{CHp;v>p+4p73X&IGk#A0xw z+CZ1Ub5IKvizmnQ@uL2rFG1c^FM%i|=Wi8x4cutBKYP0_7hjt(VKg}Qiyv5^Sv!#C zr3q8M?b4@1Je=1JDmC&RY4Ggz*8Hb)Yzz=IfebyquCBe!9l0EiqHV8>!8Q5sNRL1F zu($8u;#|$6^g+-y&0>Kao}`Nh^bzu~i3-oyhvYt=xwi99_0F0TLeRK=K+E4-Mvq^O zNR%A<#aBKyE=0dq+37S;EIii;+#8_CSrW7%6IzMpWn9@RAU~04k6Iz3Y9_$3Dn3v} zmV=axh-%t_n_^V*WA(v*N(aGUCjU~Y--cU@ks3Dz`ym9pOB{4C)?_b)dgj+e#DC^@ zHgV1jUFAe~G-i}UO`QGPou!h%$IF86v+yUyW5FL<>lFQJmavtXWt^!^w{Wi!I1*bya(q=R=LgkRj2u2u@5sz8DZ&C7N9*MgSJY)n z@UNZn^rzc4?^|4{gk)%+C^TK=V7-iPVX@4q{{9Q5xb>7Uo-!%Km9OFUM%QPb`V5S# zQk+dm*#bXH^msm_C6R+)cGs66_~M@4>VBik^!obf20K{+vsY5t_SG1&?491xeORhs zcG=xyhLAAFs|Y{hS&2>&YOT`$aRiXM7rwkK{u~B+miy5Jtxv%UcVESOy5n8KZ&qlg3}f4Tqr4d%_1cLLi14OV1m|{5xEQoof34l zxRq>1lr1iR7Q@$}>-{3bn-IzP^RlUqOXy2mBr!?gr}y(^MCn=^#W$8Y3PrX?uGjVa z;e|2iK#j}oF6e9H#X22tT(rBxMaRd}>m$9Z3%-UEvll%wD_BbED>{z+ov5O{?sAi7 zG;HaD)3#GzMK&K`YyJ9i<8(!xZ&O!jQ&{ZKfB`k3HFbQ@h=#o%NeVdM!+SrMa2n2-b&rn2W12?(OhV*? z(S!HS(IEmf=k4G{c(Xop(U!S^@l9V-2pRN_)`$i92-YE$ci`vkZw5?clbod&BKNVw zWzdbUqiRVqhn1fS3OREqX)9uME)Tf3Sk;LUA3Ixh%$OqediA0kFC1bAi&X5&OAo3a8g$~#=N*%y zcB%h$LB{6A3e@J}tBOLd_Y0v%)SZkLorh8=X!rBw4HB{Z9~Y$i3I)J@4ZN2v&VK&Y zNH@wIBg797qc!Zm?mq)S+fqW#WdB$p={&Ld!E}<|BG)h(m`paR^yYuL|FBf;p%H07 zCCm{5rgV@%rS#muOnFiB7XM%Um~@_leamuGP^aHem{@E=>4lPf^s>w3Na!BH6@f%n zyn=PFjLq)ft8;fZ4-fm3Sz0k$I?^;M1K>^G>R{Q%U-}GPqesSIRo^m@dmqwWa@DWA z>z!w5Ua$SnmwjCwUD+Mo?HymmAB^v=voqBWX|AdXBb{E}jvtq|mx4znHk&dR<-eb< z^hc{mc%`emJr46w*O_xpS{44@ z_vZSr-_Z8pV9%~=L(b`PLsrggb@f%JMBTx%q49uMAiM0%t&H)~#Jy`mJ7d8`Vlvi} z_s|wL7Q+{^&866`_9oV{+2<*lk-ileX&UpW5 zNkNIr?#6X>x9r=B*}EF<9C+Bj8qS=&lmBj;bK?o`#(l~I2k_y&bo!e{|1*-)j%;hv zI;k6XX`bYtx)kEOr#+cl^IDNgHXmLa&TLw*I7c7kXm?%UXp%DoJ1;?MZ2=8vwu;a~ zv9>DJa64dF*6ni^8JiwAa{~HrBWyGMpQ?`UyMLQuGg}Vdw%W1nx3wkw6X36<2GsM{ z3i~$Az+ey7S}PrZWfXt^Q&wmA9$jFQjcNC31e!5%2Zw~N7H(pE!qCTx}ZB#gBLW@o@BAYq=ZqfE6W?pp2dX|lWC364h! z)qXtZ+DWbnTSt0$&{olozCXX+0B_T_4BzQ{;K)*yJ6%1Ty*4VwPRrR2Sy~PjISv-v z_g&cbFR!9KuuRvF><7JMMAtXg?Xnjjp5W|ztsQzcMn0OF?=EnvnmO4wFE3p@Lgy|V zTh=f4>^x%IFLWlGJla=r24`GES=X4G*BIgiv8*)LEzkBxI*?kUkY>(RG)X@X12?sh znJ*iHfY(C@rm2+|m6V$xF)C27K#}Cs;_{kWV1wji%U|{AG=y^{%<5h4h1(x5(&A9H zJgNIaRR34_4x$BE8ia%AAl-};YS|s{2Uv=qyI%;03 z(Rf^VFhy12n#jRiF*FCU>h6}Kqm#}0CC-_eMw!1bHJTwA3`AMnwepW`tnTH0ZnqRo zZ@g|&0pFj!*~8?tKXj%IKb^nHPh|bghCFePiHdSI=wrKJYB2`<;~ZO;{i0)KeEjP( zi+tOMb6hC->if}GM*bBfzOM}YDXHma67QB?r+ry%@GXy1^g625@cTqF>V9Kr-F?k1 z3ZmIy^;EyBvbv-FkS4*i_@iphU6p=l$tiEeW~rm+W1@}|!?<0lE)#SmoZZFJ%F2DH zN}kjFPhAOGvQjwZT1BQ22$lGfQMr<@B`GVf7UxJ~0_jwEt4q8kscT$d%ipW4dTwuH z!aq$~oG%M-j7u^O7yCNE+p8nhok0K84;Hn$+@`LMs21j)AO4ux2bx&|G=RhP=fnsZ z1E1s4;dI13e%ekEU2b<9naeqf&>eP9wEU>Ooty*P2Wd_r@8huUbg}`Z+bzs?!PnWH zr_M|Kk5eWuwHlT&G_hGz4sArF#834}CT))2SThXZ9eRuJoA>x|x!hNRhCcISI%lQn zDqYRI_)l6>&6&%~*p-2wPPUqg!lw%})A0Px0 zpMJF1$ojjnCg$xL$(@%{jiwiow@M0mGcvuUxbGx#2aPp}QLI6v^%ul;kxS)2+X}q) zvHyIx8ycY8xfKCWpsAsbGD-YEpAr5;H7y{8N(Q6?0&|f5C#;?+GzIyoMdXbcJd}W{ zwG5@ohWgP`1%ReMUY(}rV(B-oP50sBS%T`K-5YbWE^9W7O}7mOX0vW{WaL`)(PK$B zsy%O)el#?Jj1lqZ?;gV?Lbi+G2OsmQeuX83jArz)b z(3T8+YAY;RZe0qr+~^K;(zXy$x;db#btP>?@@zq%@I-N@@oBaNwjpukxutDOmzaK z2UJ%5NA1RN)o8{*;}VDl<2Q=?+=)p!Jz;DiHsIGx_-Qd!X|8BtX(%td^SO0$xN{ys zzLb- z0OC@$Dq(Y2#ACUnJR*YyULb!j0!SKeADbH(p}SC*PZxxUhA|W0i~+4l2W9^A9JIl- zN=XE#HJ-%8jJhNKv))!( z^-9KnzUnUPF402FS16rojzIVP62;-f8X%L{dfxWAvsPthAAZ=));xX~^#xksSX z6ALdaRVW1zrM32Ea}Uc;k26lrD> zm@z5UlSb7DaPiC*p=_8M-c|%aR2;wN^J0*p;Yw{&`cV!A4g!Ts8h1p6C~;XL zt1ea2X2t)!35UJkO(^YG%>gWFyuz0VW3}(E^V*Wd5V>mc9H3zcvO7jUc`h_^!%ish>l(t#pt4#o;I2I% zPY!shc!#n~y=b#winWF7$y@)F6)VtNf4MQP0BdkWA+VGKTQLOCVJPUC|4QiPF_4>l z8`cYV)2W%@brs}m<%#&U#=d6%@9kl?wsxGmMH*eF?G27;n@qK49IU{18;$;%Da^57s=yy;(Kfzfog(9BE>lvHq*RXYy1$^wKAsm z8H-A_MwAl|ZAs0a4R|wn?Ih*#>OI&lJqVogX4H_K z@tcJ(K8q&IZYt;MJEI$m`lY%3jqaC6BF4;yb?z~upF>0WOp(PSv9}c_iJP4&a2F5s zm8zW{m7RZD7vWZKM4J?uHA~%NhS|)xM28}n^9R54??a2j9KM6th!V5egV=qEF%Wns zTYDr`>&3&vfl>EY_J~^-3wwnA!mm#)VfBxkDWXFj%=w+Ka3x_US(m+51}()NbiY$C zPH@IaT-@ux>La(@@eS0_8y^IWHW^gT3dda#YdM)qE$%!fV^@Kb9=eBKDMKqzxKvOv z%3*1g`3;hMB{In>+9cL2tweR5^{P2yRf*S?3r|SKl|T9(gf%Y$8!eiRn|CUjD7rRu zXir7dutyTrY>PZatQKVcl;O0L!ImG@8rkpv6R^;Teu1M zNGb6O&ExzkfrXs~hDyTE(W#WD{Ao-gCgE*DQdF_=`_FHBI;a5BffpHQ2H)v$QUr`B zdSHQ<#6xnfy}0D?MSR{G)8^!t$NmwA6&>9FwBfxaRlFl=-gWi+ z$guohXd-iCkHm=-WWRofO!S*cz!2IN(1}!UeVs#F|EVRV5SXz8)r>=qO*TRJUexT> zz`mxT!f6uII9;Jd$!xq?QNzqD{a&{n9)Fu5%{ILdD4w2s|2dz+ExSt*vg}zlW?X*t zW^e0;KSYkRDsBxj6r|eXi6FdngMsLAV#b0xmSM!peDJ^^iUN;#wsk6#ufgLbjg3QQ zjfJ$Hx-6~?(1j=}-e_-KRi~J?UbiagISS!A1rSzCbKMB-;tjZ*jYZ_ z6sYvCVkDfzjO^5<1v$%9!y37IWEA1h5~795zI^}SsqT#=sD2e*oXNc@`uJA$pWK|z zZ=}I|nkK@q)gnG1U&#ouS%;FoJMIc^pH|eXw8!ud=b9YD((MtB-{l6Z+U8i3w~t-Y z{%6cpS!{RIF$@5Jm>~dw#ee<%|2y{TUjbKI)_(;SKC^W3$#mU)jqP1W4yydT?XZX+ z7HS=v974{t`On9uchw1$;~maXUanXrqYZK0ZTP|HktWaWHjj41Ms2IzJH27M zsdC6-tiN)Wo}~1gw%>q_)w!;nE5HOmH+%Mk!eZUSJph5> zfhR+8p1ZDKh_W!qXs^kN@l&K|Vob6X|7^qs1OT77ZQfdxnaGVN0g9%Xt~kjdZez^W z^^|}0`8>FHD|g?tF?3VKW96mgyLpt4x$9TuI0@XI_5PT5;_H2W7G3MXKz@~^rqD^4 zI8+bc#pX=S8+R}(S%|_Wp8En+6tRRMH1+I9B~`Rz8n$py7ND;%hKX2*w$7uOs~|&E zgtipFeD~I2A|P3r-()rlf6C@a+q>(}m+$A4RtTLPlv0%;^0Oi%V^J8(5gKV0E=2Hs zL_V!!*sqvvXl`mf3Vg!Q)CghF%zVxuH6HV5-&Oya;%(-s2~7{z;gwlHo#p22ATQFm zM5j?-U`1~=mLINgbdUGa^u6(n%5p%tkiMCx5bmbNw9MY1e|1!lAxObCi91S-k&ktE3p}mKniwOJjM%j8Olu1}gd}wJn2uOz@a@DIVx&kw$GVrPWqEWLqdD`` z)UKloLdbK2wRfoJX|Zoy^pWle?&$Hz(_Bg^5{Euql)^8QEjzjAH@MHXiYqSDr$GFp z>hwG3Ns=*aJ}<+_?Y7*3rAmFrB>I$ZZ3eilPOrJe|%ew9g`Z8)G}FD2nGkP-F2sBFJ479^f)T_n(l@CP-{o?Uj$?k z>!Anh;Dmxk4$p`xwK&JB^g+!NoLXP8tmURb8&AfAf=Wj1BEwrLQ0kKf9T8iHf@-_% zqgtEE%A6uHl_@F7(qn-Us_^^Zb!{M?3tHL8$66`j6AD?#S^HFr?P2eda^YgHFU>Qw zM1;!@2Z<=JkPfZnX;#WmT9fO$%G0aL4iRGctu5-+V8a=BIn8D--@~pZazq+u6T-JbhiNHpaxpF;K~&Up@##`+Q`d zwYP@Ts!*4l`k^HebwxQT98xbt_Hp#=v{;|(PL`uwJTp>I4m#Xq7$7B!vEWA+39 zY2B*|A{@{LdSg{jtHob=TJc7EG;ukmLHlu8t-6F_A@(FrOBoE_j>=m7!UOTt33x+> zU80N1-l9msny|#`FS>rUT6|q>EisKs|QG#kLigDJR>Tz%~^@$m_xsOpWl9DxO|z*`Sb3 z=jMjA7mXjk?~=Pf{^t ztaUYPLqB=XE0r09f1I&dMtl`>UOardJ9&G%bUFG+uUxfsSFLwjNpqX>EZf9vT-0)I zZvlS6KD+jG1Ak&)U-|kp4_`hmZ{WOLcD8Z5{C%Lc-v9lKK00h~+utpt*REOyA>C^T zoOY5{lOSe z7@&+f`*Ro~@p#*E3ugGBW(xT-AO& zLI#HmGpfU$?!Y2Vtw9b0W7wV`;2ArOX&9bGWqEwz0}S!Qh4R5mj0;E&6d{{VSKnmw zj3cS!gR7}S)J(cUFwv0#l?q;eB8KDg;R8jiyUIvTeLU?FXpoSloC{u}5bdZ6X%q{T zTge$%)~Ceg$nB>VusjqY@s~;3(vd=fX-UAan|h@XDac1lrIVvqh0b>4vf!dHpeT9t zJQ&t5;X08^vh}h~U8EoaymRv~jQ%)n=)}#||9VY{AxEYoK>KjpqmD?Eu+9gCFu0I> zpWs)dPE+_1X#B$f@y8EDmL@C#8)hgN!(3Q?O+5}14kYk-^)d4k=XldnV1%n0p5K~$ zC7p6FMA3r1;CF&$L9|9x)@E${3296uMhbaShC#wE`9Qn?*nvDTni0P;EGJJc2Kvg6 zvT25{X!0WR`{(9n+?Y7HJsla`+pUmN`z|`2CtmxHuL~xQ!i`bac8YJ*-+ELJyxDx7 zz!xKMVW+l4hKu)R-tP8pc=LU>uV#ST)(?DK88D*q^XunahmUY^8g|bogq<87Ke^iu zjqaaj!VINn+1B1PDBru>-8U(2s`K;dA-P}a-90>?yNoYou+tGuafgR-qkTBAaI-Uc z`s6S2LqvVvoIH8L<$8|K=UPU$s6~0aF+`8ZUi!%}u(Pt?5vM+$8#&p(nT;5gfg<5( zy|qfwv?HTwGPHs>>>JC>yNOp=Wlqpn!{wt|qL9dVW zRQwCpI*cj(5OuAG0mIO-k!qU>9_dTs#4ju=e)T1A3@^}E4il|VN-Syc(5$9YK`PK& z@;#+fltRjdOp7Yy8}9gMe`$BkH6lP+m_wjcizi$7!INaSc@#=W)mBM zH$+}Zp8m}}fX6jXRaQyDwoyq(0iBho{OQ=ZxM)$8wyR)<4L{^r3OrUQ+E|V5&s!)& z#s(LN>tF6g2gD_wCYlB>!XPsSrI9U-H4Pyer=!&LZvi2mx0~@c#}yF_$$z? z+z0@WnafuVLYMHkKucc}p`9_yP}Mt}d#C<9e5yobYMV?c7Sq7 ztsx}WOT=uT%J*a+6K{TcajuueBTK1wR~7~LCGscOI$Y?DAe?m`C|VqYQ5G!PAQ)ti zi$z*gDXtj?r8&q@XnO2(hWh};*vp*Ax??ayLe}>&axFNuVlFNt36))bN9w>c8c`tW z%!OZd#y8a}NCCkmpxAM2!nS!-=z02R%nTcVCrhHzYU*=W5nCq!Gn^89Amn;J>1im9 ziqMYXcbElzvCxn&cT!00?6H%LM%*xQpF$@aT_V$p(PVGy5JUct27D z^MXFpBLFr6T`F~Ba<1By1nqjle zQMIY3AB}oQgD$vsk9^t)m)s*c+Ja99D2G*kZw^q7OEq4tL9Lb|8Q#)h(> z{&X34Qu^q@;MmMD#>XLmcMt#quBSUSU_zEeNCSbhfjA^#3@)zvuj@XuJq7J;FJnVV zPO6eVD|lI-^x3K3Dytu}db_U=QxkEgz%wpZhob7~{-v859>2`%*e@^FJwGm^Tz{!n zo14tve^OXT^Rl%#<~1g+=KXRe2~k(+YliSrGjO|W2BF%Xu!eA5lo-S0k=pEZbKni+ z_B>nA{8^aB>rP?&*pfMew}lbq0oAuKb!S-jAZTz|yrO#At7`uy@CQOy98!kb=}%feNTCj#b!aG%LKJsY&mTr z`c=EF?#rrPPDzeRpjeE5&Ar~voW4ni*h9E|d=zWMsp{%{p#JB6^Q!}K;O^Vs5c~~7 z{crgj82*8q(o-9l+ZdVKnEtcdv`!e4A*6>Fe)1Gfq@ELeu{LFmjMW1{$X`jhplHKC z5&8&4!2k4G&lHOUl3e0;wq2p<)}=-n<5_6v?Vf*_Xwt4i*&SB87<$A{nY%5lQnFJy z(}OvdP-@hg95`!P*|2@vsny1XVsj##aONm@SZ4KTBh}(nY_>)CsIG?aXyZIjkABtd zN5bb{mEa>jz{OYDRvrC>9@R9h22=qk?qDlo#mM5=J*0DXek-GaNRnHzn^n+A^ONS9 znLpfUG7ss>xhj;CxL`?0j`W9Nb0Ls{R@_?jB{K+g0K_!6pt^I#GUwD|}$bYs)Lx ze@;C2N2Cc@5C8!2Z&^vZ|B|EQUlWgk+Web`<(quP)`8l<+1%=16fB)8Yc@!t@Lorh zB5yx_T0VUkZ1I7`!s^sp#hmhI-ZZZ)aH$`D0-(Y8 z)vgATI{}WMe7kCDi)5FEMKVz_!vcofSaWYWBs~vsTxcgjS_kxFFCbV54e=SW9HuG4 z#0=23R@n%)E{+5($!&7gp$eodN626lbf*J;mD9g@_3S)RvJWjBKc9}?wss3fnzKSS zz0DRl($6Pv;R~Pe#|#0}Slc@kV&>%nyAm4#X`NK+;Q;DkZ61<~N9>dJ z${ufpXt^}A(_PV!TiVgH4fuj5;-CAy&vm(&d)DP(wVkP7m@BDMU0YPI({R|879_Y= ztRl1`-<)I}J~JKFv?FsX4KHqBYtkupDk{qx!d4+wQycnAefUk*b2trqkCBXq{AwhG z5IrU}>+)5{!BUSJ2Ga_!W&w3H6ql7x?2Jt8_WfSt+osJ~jZVlL_yS9;)G|WYk4vTJ zn>baqaE3D9?96twzd-$18~_oV5$-VoeD8D9WH`>SDp=Hi>>_|7S@den{$Xx=1KbM{ zN1IT+MOP-NyC#a{s!&`>VEv$H&e7^Y;GrWw1O!*W^j&3hGBq1x~a)c642;)+r9RS1>!G zk6VDN7In047n#Uh@zfteiu}0*Yw)+Tt!%g2D1F&lnzInZsOOWj=c`$sPxrT(^R>Zy zT8)sR$%9)vEpW*|3a1U33o($xMg_78SG+|2v-oBj%w%?C9ft5lJ*y&Ursoo_gms~u zD)jo0E2>(Oi)STxDfg68Er9}+QzIn95sU*w30Mv@s98ir*L|fN|B!R8lLQWXfhRA| zi6TxblGbyx!ZC&b$O0ci1ZhNKv<0W4D=O)Dv)D2?B5728DnzNU@z(p$pPsPizZ>R> z+WkIB)l(iEzFG}kSR^p9$H)s~>-J>8u8{8>u&5OjRBHhflM+q)3qoZ)pt0(sBHzP! zJ2D$4RPy#xb0OrtsG9Z)y%gr9b?+&uXiBxSsl#E{ej^O67H|01&Dxq8V2ze_&oP+n zp%@=FA&TR0z3^o&(jkw)>}58rqy0SddSnAW-Um@%D{TvNc<_W!tt-*|u@2>XdEUwtdR3I%V6oZQHi>>btl5 zzP{bBf4#mvB6E*Dc1Fe?5wUaTTq{=0r8_U(FFXTajV9{8PUsymbmKwB%{znmGF=tA&HLi-u(uw z%{_mAYOBamO8}Y1Vd6lGueGp75EDM~TpCQNa;J2x%xv%^=9r-Fa7EMM87MOjmcR@b zo5f3p`F60p{1kJ!t+P;8coq{5KdFh$f`4jtRG7r*nL>BA6JF>Ql#?f#bqYQnS%$?_zSeT!6M|9TjuGafgw#>Nz!|Tz}b% zz^!S$?3)lkWBu!#(0|8fCK_9NBV$W56H~{(JE2o#Y_Qo7g0DVNgD0^<@%!k2TTn!P z(#X{hedjjjsBc`jEL=@YV;=UQYfy&4rX(qAJOzlQou%c_yx!P(8$aG#czCc}gCaC$ zTf#pR>Ck-aTb3Vtn6&J`9ApeVVM^bMYWgU|4&i(~bM)$nNgotd-@0v%&@{v{vqPtQ z%9%^Z*(yiglG(Y)U2CM~{IC_p9E86r#Jn22w$-gOyS-%np;bE!;THrZP3Yew8W1DR zYaT_*?QSU5wN)?E1g%82UL0cZ(N=TPrnh>obH`F;YN?*Zrj@<2>$$KsD1*8Ae6lqC zK4DY;YOwj>WBnRy{N}_vkxdoE`TdfBLM9=tK>0nb1f_x_ObZi>l;odOaRB^lSS3La zKM_baPS8jur5cJL0C;}T*M%`47p1;edUX$w^8SaUpI8Asxr84rXMC=2t~{VSsjn&+ zLyFPPa@;9uKgQVd>SjV{0S@?(8+?KrILrOpOQ!#pT`SqtP>1;hcJqfX^$X}8OGmHaGkf#~cn z0u3*~fk^>TR7ZCUC^>u-nB%C(gjpL1$IX)oA@wD5P8Et3S@OA-NwX(Bb)rNCp%$s3 zgq0P|@|9%Z5_3$G3P@^ukC(ddf<)1=Hs+2`=N9<>88S4@xbAPh?TAEh007T_DV<~f zrgM%)Cf_n+ZZyW0Mn;Y_df$R$4vzmfX<|d7Ow1hpH;8iqB}fmo8a_a$NF^u4+$1ISzg z7WE(E_M-MJKhJ1H1RB_XNC+786vS_s#uc@S!OzUUW|`)y|0OkPsD00qiNCa(i{Hhp zYT2leIl&r_N~IV`8nywpfHqdDtVUS7D}V|wlBuLxvH@4uF`iIvs1#9KX_ss>YVKjy z^OF?wsUI*zDBWn~dR62j+vfiLxV)YxhA- zv#p7kjJLHUMz*Pimi2q*r~kbqso=Ye5Q?4$=v;-E0lNE!$VmjxLU{x^9Zg?gE`UI^ zrqs*{p2>TBwCwSE>BMd0LwiaTCXQ5}IA60jv+n}yK7zWcJd^}q2_Phfvgh$@IsG{m z$cHXUbUq2ukMn*#9dnD{#z)tZb|7KRt?N z(m#uGzL(Ftp#cEQ{-uv13yr%!P1SKA>2IozB2O<+eHL|v&nU3_z^y-~jb;XZ z;|!Wp&n)zrVeN1Im~r0Y7r+5#%#9MB%#^wjPn(&Wuw<>u9eOcm-AYWDebG+|heuuf zSXbaX4slZPdI;FJmT8`mgvcX7B_Xy*LUNZ;LA@X-X)A<byOWHtoYN9j-D26&>pF^!0L4`hBXD}lB`cN?~<~p^H|nvpPh~SD?UF=>%x|< zXQOsCQT&H@DTK^hvrIT&4hO8GR%~NTB)Dt`0T?BDFQ$||DaQznpfv~jYP))EpiWtB zaj+uUM0n=luCwo4{sb?z`TBjeq$Wz=oSc45m`fw{gjH5$+AiIuy>nOb?iFn#J>Pc( zI&ZjtrbrL1AfXIcG?S?BEnW~@nktMqH9MleTro;#0MVN;$~!fb9U{~r2L!o0K{m4HyElC=i99i~_oV21Wz~xv&|0@5$wn-DZZxhf++On!SGQRjz1BXJ z{-%jPlakl!l$TZL_3_8QHBz}fP`O2of3E#;gZuWg&*hv8cB|%!+4CjwIohjehwCvj z_-fe&t8-V=HdC;z~o@j1`7whHR zia+FkDNdkiVV@~LxeN=!bNd$T%qEfwhaVuYp6uhH5A`ov4B^0$@{?`in6%2%x5g!J!y zl}o?WD~d@XEuU4Vj~ta|;3%Dz>T3u+meBxLiIfZjs!6L9z0ImO(eYxS(&y_@6Q0Wj zLjrL3j801%PxAN%Z+1^E2UVdTV585fUX}EmiusbFDP>O0V?t2Yt4h7X>f?$qYs{`( z3YRP!#IMS8^^Cz;Yu~xZ z^bU>1<_o>VX6oRgoCEuG>+h7i7EIbK13K+!@6KukTUzFX99DWo5$lEiJ zpI}fgVHEpq-%vr3fu4Xx9JV-%C_#yUok7gHx{iUIF>-=U@%&CPcsE53ZYKiz@bLs| zTL7+AVD=k)9D?h)@#+7G%HQD(QIwQ{o(!_DYzONJFYtDYZr=NElbN(-W{o* zWv_)=-vwKD1zYXWw7dyB6*kxrQa4lqpR5c@Pm0WUr7Pa18XP!>>sc45@DZ1eO zTb-%xw_JvMzGfkZ7v&;V%OS8Pna?B`hZ2jJL#@3Wk*s_c8b!cUTWQKtcbW`EZjgMO%-nU*z-@WTvRn4`ljWI^hc`ZB98N0@ z7uN^ZhyNr}*gxoq_!mI@>w$v(e+DRq+vZ{~@6Jjp=*}LLF>aa@Jf(&S?cj(}Xb$7T z$tgU-^#B?$PBE!-`_%L2X+kcrty-Qo36B^EDxvvgH2^vMN9Iy4uafGZ!4KX0K3PkdrD6O zL}Tib5DZ=bvxo;i#nTk6wub4atBHp@)_Cb1!+!C|DxQMgV<$;^aoaybbJ*5w_Jf_J z>4o#?!rYA=ZT>2{-0>*H1gr^1QD@xwC434F@{7Y;JXu>JSxX{WHdm^u-=+L7LUGCL zIn+9LZKSfxwfIdaRKE#@!G_uM?KkN-`z9TKuLJ5ig(c*2bD=Tbgd)^}x%wBWn4cP8 z#h>8n>6Fh3WciL^Bk_TuJTr!-9lD=2v=z`KWd2cXEZ}JM%Ul7&A5_ktQ~`)3#Pdg<+TLk8>BJ6!>qZHny+;Gh3h!SAK#|5r!_1O5LDR)lW+ zPm+qZHjBHW{|%^MVEFH#BAovlDio0Z-Sz;*{~M`5Ibeppy&^Sgv-?9328cN$O2pwH zfd}RC?CRvD<+v$W(@N`1Ggz*=PwTTKAnp}Y5}T%X+_{Ql-zF-pOnOv{l0xji=qN4D zAhgtvgA~l$igz(zSVP6yDP$pseJ^>KjMrPqV5_M5#_L=fWP+N3?~G&fF+IQ=^@*{z z6tWOyARnZGaimkIwqTzv3s+(weifptix1Rh)MNqZa?jlRaHGd1`fm4e(oN62_sDS| zsG^H7KTdF2FVZmQ+byo4ENMFOV5VK>U9nm?R-!n(5OMHq6+9P{-k}4sOrlCdtr+P? zo90PIs$=#UYg0O2&l+j=VkdqA;*a01w4H_tC#SZRQ-@yDu2xewrxVx7i#Vmd>%I1453*Ng0 zN;C=N4+#dgObDw;XpJeu2i`EuKro4^q~Eg%&`S$y==sIX9%0Riiq~0%ifW{cPTHth zfDAm)tti;?i=h&?5NvLBCe3^t)kUeGx_(*Bx`kERLJ6_w`6dvWC#8*Nw|Z@jKu?*y zkCMi06j`i^!r7at-qzfDfOGI~V3FV=rp)-V10A3dT{>ml<~d+86!?Y1moG(^KSrBB zW-?QNzWW!S_y>8Yg}(WVPq>tF!T*h5yafG&b6hDt!F19=3AH{SBHhurE?EvTHr*V; z9q{t?=edC|qmRZ{^OjDQ+P1%=$Sm3P2;*l(z*L5n1R1gqW;I_0KOPK^-*TYa88xBF-KM`iOn5eW zYcjbwGFjAT(HtJC9}jA3rTcOK>V~65GA(_^{7~DRP<6)Q9Z#dGh8J^UEjk!-S=u*9 zyB{>fmMfw~33SNMqTs(+ta~VU+bsk2&tg)uY-BT)DE7P@PLTWf`ElvL<^x+E(F#vV?sPibc$iqS*vXX=-jG`w~A( zv%=!+Y{k5dKP&>+#@&EXjuOL?BNcA4VOrgiVCvh&)4$a=b+^YxZr1e_LsDB4A;DV> zsNr8S9bg8>g7#Y@>>(wCT=JI`^Y4Jy2|>o=4h$)VK&1LKdffQ*Q597eLR$SldTgYn znY+YNbG?-x;FVtEf(k@e>c^lZI(&XO1B8gs?^aA0xXLXI0Cmv9D` zrEGZdz&bgSdc+c~fJ%B4@~lw`XK>w6I}9zsyEOx*#RKMl?sM&qEfZL1jF2qO?qF6% z7sl5sm^ZpTfDKpvz#0V(88z+yG5-+wYoF?&xxII|47XDdD$*!PDb`bO6MyogMZ zexO2U!OUM1PmEbBaf_lg9R`-RTSE9=`~^pE4MGZZi?%yLF&K>eNX1*9uh^~M@JL2M?epe=7`n|!6fmcw!2Lq{Dme$h>Zw`uXf~ zjg9OKoaZr-_^}$ZB3(VLR;CBTBYLr7@#U?2na0q*6b+*G-Z+ z3EZfo{RrIX(E_;kR}oA6GU(Ou9RU3?DN&Y+F#I;S?KK|#hhn$?2cmciGXbu!9wNV< zysvqw3E&#?s7tpq9h)xKcX=zmJ%HpBlMi>pcK6LS5)mIzcaCFE)&x2www!lR<3aod zBdLQeC^*8BfBHhL>mi52zh4p5IP$azEyZyxXPzuw&hHB9EnOa`C+Y z*;`y9Fd+VCiCIEI$%0v~M%iUC67^!Fh()*54W%1?xV=iemQJcB_0bKy0(ru~zs=P5 zT*)7dA$wO=uvJlFlqn^Y-VEr;gB6FMcJH>d9_G66=F-tZ9;P)Q7(~;Gg#Fe1S&x72 zewPc92h8X^)ZCQ3zT}8eF$O}hLGZ8CvZ?7jG|O9*JYq2p6=}Q|Zf^=#!L*D;nj8S(Ocx4X}}95pFdQHpUJQ*6@^ZpfrQtgPUf;? z0~F1qJ=y26#4NsZYTPCNeAbeyGG(!tG}q4mjdYqG?EtE#3AOUGi!*8!K36%8Jh=f2 z*uyIPd*A5eU-PgE^KbJ2Zi%nd$-R>dH$th0s9wRVM|v|+R;O;5zATl9L5XB9{jJCZ z`I?bwVV^deG^W=570scMr;Fj>~gP(Wbf$Nw4cELY-|SPCm-3d0077!0s}bz zD?Qi@EdP}aSlWCCaF{#TSYLRZ#pkmoT|IsLfbgloZ7P|C2IOEsJQMR+s@ZJJYdf+& zZ@>)%kRMPa%l9k4ybb(%@&F-u@hvCf8ZjT9l^%CQb0={0KFZ0<%k%k2*Yk|c&w1B3 z_O$Ebhw!?QY_xvNqmLMb*aXj~~~sr181?5(xdP6K6Z<)O7C4W8RU&vWV8Tb^&wh)yH#^jwbR zcmWr)k&DyP^nq(dV{nC3O* z1GQ*l>eLoBo~$+RfpV3f&kKdLH3GC>XH9hBu7J6U)Ja`l3~v|E{t#>ojF4_!m)lTn zC`@CQwrG=`UVgfDK{f+OAxug?Ko0Q4sZpIF&psLjD-tafN8JS7l{8teS6o!iJ zQBY0ui$`qOpVw0MrzAD3N+1!omFTUNR#LuRQ) zOoH6gFY?j<-t=QTs~2LHB-|#lVA$S_n%@nR!3>T^lo46W9-%Dn|A$xV%e4>OgfG86kIcnWROH_7&Z-!eOndAP68mnIB`Ta;C{7F9U_%Wb?A}_WFMq^llsDo>P&dme=-n{*`*Z5vJOK@OJV5E1K99%14-&!kwY(EBnw>MLSP5E{&$`wql8OEbT~$|l ztaVpTpFAZm9ucIPYELG+3mKwb&BY3-_2C=*6}Bz&Eg4)_X|wq7Lz^}oM|YD!y`g_6 zdQLV3=x%etnlX=1#K9u0W0?2!3T3()CtFs}SRM$@%2U zAj2@(ujaNSUXvCE1~I|8Ek_%oED$r2o^S5H4m#a?RBOcqx*f3vsD+tq}ksWL4A zcjFh&{cY=IQL4WaIT4KBhcacdj!E5?l!;^mQdOrJ>yNL}MZfDd%8(*51DSQ_vAZs7-FiXaLs}qsBG~OL#I;lR|D>5Z5&Mc*fyd#SE zP^z&4azbdeyQ{EE!B+%xT9CvK1;?0hnmhKjfvX$K?GUome_~SX^0}KVKFP-PgRm~R z`TUUmLkz|5CxAE_RkiuKxABI*=_a@EcA0+ldDj5W((Y2#iwpgp zCTWxyCn|Rq8?-|zA0$6S*in3o8(Anvoji%%lY!l2rj4OvZA0U2#@r4)AD-NDLl;>& z?{m==Nio6t>tjsMMX!rGq#UfCr^%@5p|;T7Mve2l?1#9K*@aYKPzhErIvqSF)KMaC zfzDT)+$e*=$Gxaef~cfy_i9rT{Sa~8*Wvc|@cGxNmSIx%s~f$WSz}}$X44yua~5L! zhf50hkl~k#g>)8d{CeZdGW`mlw4SZ>qff)&ehatig{1NLqdT5jz@y}l<>2RHXKV<1d0~)%*cFnRjA!5o z40;JmnMXT>pdY!eMzj&qwTFPdLuM$U8J~)h5Y;U;vmo7c+y7{D)W(fuDUF1J9q(oD zVoE14OSB>YjO-V%enz^6pzrh=Hdqi|04Wc)hm0DdKZU&*d$MAo-D*J@xY+7?7Y?{X zy%vrmJDniE2MpGyxcd4ety#zP>)MH7|E-O4=<~(Wq9dL#z4c{W5AGiLVQ2eC?4gE7 zZk-|UEA?TA0-w<0*WyCVJ?uu&#(gZCAhP`WTqq9#WSbMJab z`5#+Rc4xURt4+_B^g8 zrug2U=PZb2^LkT;B2;lV#RV+9C?<%ZkxkQeei+@e85Bg@U{X?$I8ZVvcp1GkZU||z zIE}__IB1rl+r(oYAxiu)=S>tg!3Hh;%;RuO@lpeSpr#jLKS}lx1Kvgg++L0jXaI4o z0oVqiPHX&J0&ryj(5;MczjS?ki23)4M4{#_}yDE!pZeJJaA3um5| zD{R4XdlYXSXGA$-iMlhnrAaXic%LbAj;zv44WJrf;9?^1C**YpHI+}lJ`0_GUB>4s zTMpcL(q^rAzXSbr6b$-wbSE>66-lz}=~zn|aAe1(F@S&ksarQ%VK` zV+wmZ7$ywFb(@7}XC<-^SCu3CXa|GLJ)wgbUiv2`@8DCN1AygyLN4);+JT3k0l*fn z?rKFBR+dbGubC-cDs($+c-AcKAw0ZFS%x;S=Nm!ywAt21_HNviQaJNuAP@bFwD^5p z+R0z;Uf!PZ5qT)|q|h#lWMmhv}}zg+mPd>=s1#yBeY|y)tI4hvo=Ssd|O>#0~CzU?Ti~| zzYMQYgT`+;_eh3sKVafXZPUp%3r851tyWdi=|I>44~=%bLgVQ_0F*GCF-6L$0&7Up ziNr)Eh>NIxL5dNv0MW&KDQ+1eie^;wYNS{vso!U zE@ZAtwA}aOIO-(EUT7BxCJHE(C>2ab)mHkAVbL47@*s$OeyAHEMV2OE{q72X40#QG zR=)VT4*-rodHqN@acPd^47|dN|641>eiNzHedGSuq6}QE6|8SE41i^W|roa%RdcRH&N#lpa z3W2QBKEWlkH9;Mf5l~d&trB|wCJTakG>zsCJFv$CX92HhX|}kYU@kpjJ@dnPQUedE z7;K`|Bz2K^%?VCw-Dxe;O&kDkZuz;Eskqdc()eV3lrpwfc%Jb@(^ z#Hr;PRY~8rYxMGHq!m8Dv=3#)UTC4YmXTlddSCL{kyKaqy|I=1Lkz*3GM)^oST=up zhd(PI*P08aI2o>RSpQ-0g5yYOYzc7xN1f){0|Qq-kH>*RY!z2u7TTPwOy^J6(jrQo zk|^rP2G7hu&j!tHc>T7JW|JyUs#_PWd=^A}FY7pxV+ z__%PP^*R+^)R$JKb2EmI`TeuNBKO?eh2H3(h5%hfpd@^x>Q12)n%=(4hK<`;NYzb? z<+Pg$%#1)v9$yZ}a8XYlO9Z1Lk`bhu(N1JhA)DWtcdV}@8nSs6{ zjeXQ4aJ5s~FNz@Nlhs874+_ZD$XTb9m$r$wi3&72MT9y*KS~aWzYw0z6JJr6*Tck2Sm zx;k!4hLr1F19}cxDplA4%E>GH8(@Ib6Pe_TRTfFNe&%s66c@P5}SfhRwNgREOHn* z%ihSsRmVF-JmVMGsM%D)x#O&IKgoaY_O7|_D43Cz>S$EJZ4lZ^JSicj6B=tcceRCV zE*L6~F2x+#-RKr;m<#M3oV<5Bc@2|sG&Y&-C_SxqzV73!O>{K7#SRF}OdK7JwMB8X zQUI`{=$)=s*N^w)$xKzzG+cw{vzjb zOlPhWb!~w>NLXF2RlmLaShMN$*ht&W{_v5+@J|EjmY3TEcWH7W|GJh>YV z*0DZ@0uO>KIgdQWJ}}0#OP_42TH5>>5-4wG!R>!g(Q=P6;;!#5v-a*@79ezb)V<(x zrx>WsG=-NEyDMRKe|iEZN?z@oi2=OVkUdy>D}rf3H5Nman$T(_pGvOb+p#Hd+%<(7 zo25vTX;y)9V$N{~8CFrc2reEYx$2Ya>M$h^9}>IR0h@-C9fp&kAY6{dLzZi0ezyZk zfnAz70g@0UgV410^eqgXZmmoN-9ayvEH&{ftzdBAS0K^6O^+HjOlpQy>=P5RG4-{V zo}D@8JSrJ9CZpC*ocJ{)Xfd$r-2#I;Qt=Z-j7%CmWN@zLMA8rDwV#j?QY>eZO*g5g z0g2ge1#&fX2h|e|T%*c8|pt zB|%w6Aq?YRB#=V@vQt=1s-_4R-ya@Zjm>ZL`S#Y@-a8qwy>9`v*8sWo=@i|DiVw%92QT8obqES3RR`?T13 zl%52TE}pZy*1PY$D4+E8IXtNoXGIE(-HtZkOhWKAh)Cmg24aHovKC?fBj=15C*!nf zQJc>3P&%vK!&h|AE6HHK!fI8K2d1-ATGG-_ddWc}U|r)Ubg6@&H+k$K_f++ZnFsqL zqZ7PhpdRflQhl5Vj{YDO4OoN7K3E7#cdSMR<&{b@O=8Z4f+bc8v8dCijx+2bbpKd9 zU;xJ-^U&3O%&+c^vz~sT_x=u2!Jj5F7OfB&x$qAlN7gB3e?TWH-Ot>6SF_h~e??*5 z?i7x~dQg^B;u1Yma`K?3PUxm!J;E@Cx!bjbS>|(P8OkRG%sgQ zDhlwPB+M?V8oqp`1sUkWc{={o;K6}UP^Kw>9iTw(>^W!GzkWrY;vb8+y&bO{ldjdo zqDHWX&D{($;qNEf@+U}Oi;zViuzE$j6kZGIAt#GPUzxkHLb(|mn8Vwxwg_xPtLbOa zb^Ga~w^)Yuq0&W>namTzFiUtyRW@y234~!b5l3SZlR@o#se*v{5>pS|6R$y9?Mopx z8xSw4F7W;yGmB^lEB@F_FH%3VsQ98h@A=(FbDF_B5 z{%ah@9|0~fC>66MBRvvHAF9VvK@<)(t`)5=H$wOK$8kzs(1tYIZm^!9EJtxk#ifZ> zGo$5NU^e-W?w~5oDd)YAJ_vivnH=0bBB(rC(K)FDo_h&MD-F}NU56F4$g%*#Y)g&% z)secIn9x0iczZ%6Ai*KWP~=oZ6Y1`3;n^`seplYTUZQ^IBJ@FzBql_4OY@qS)c!(| z6d|_=_^+e%CL49jXRR09hl^p+I5Ywd6$w+%2Q<-d35{8~Y(0>*h0ZL@D*)u69P6&g z0q#$Qjg#bI__l>Clr$rwhpXJ|-f z_Y-#BM>Y;lnvgcW8@^|#kv`;#HGp1<^V`hD34$G}8JI$e@Dz8R5bvj6iU}R@(phoT zGvo{fCA%e-z;J3wNN&)v|M|U$0B(G@?tYB4Fo}sPvwoNX>!~z1MlgzbVUE61xv{FA z5lTa^HkHi^TBr}zoKoiuHlw$}QE35vN^Ay2uK2X2zvf81#gZuw0tnT-y^fX6%_Z@W zZDZAkdHm^I+<$zqLTYJZn!}2iGsrQR0+Twb4&6{!X=0flN257hpggAJ5R9`~9hxmt z?KQkB=v`~LXT1-BZ8v8p2owkFXBf}Ud@&q14xc#x6igccy($pAMrx^*V2 z($N?xjA)n+-9)2qHtiwbG|mM@4?83XSDl(lWiQXTBw-768bd-sc?dd$o{D>W#F3Ax zBs~kfTcyqfiDo4rn{y};#UWPWZ3*hy8%rKyOe6*{);j>cT#sh z+8(V=b;}7;G*?M9KrO)TnPb;=*<-f79Q^AeVr z>fSnm{>Ul&l&^lXJU7ZOjRmZ5@S9s@(N~l|_yO1j(3OC?f6Q{FmZyKzNbbA| zt44G`>{;Q;oq@|z?-{>h5gTrGzaXGDW%+lj6Yt+-*QyxCFZQc{Co(Q)srmbfC$9{* z;8R9a*x2e)$8pr`)LB3uO3mZIZ5ma!!`td0t23?_goCEA-tghY`zrE{@|AU$nMs4Q z@)>EZHqrk*boGH_S-P z7wUg`N2ZN*RANli(7Yo&ROYJW&)D*T3<$sN9W}GmWV&M!r=7GNp;p~L+;ROC zpN?i>_R3x9^=ObQfl^aPpV8o#5<>j0)X|(TAef+;nQ8=$<)P)ra#HBgR(qSQb_p(< zSw%Bm0Dt$octML1x;3uLO3V3qhNO3fj+Rkxql@K?Xqjou8!IB%AqpH#L?LGf_`+uL zowS$b&^ zpv}%4-FtJhW$poWHnWXs#qPJrL!)e9ok4C#G_mDR-}Wz`u!~Ta!b`m!df4lv|7@iBXzJ& zIK)1Zc(=xKBMTIRD^mgr(zg!2b1XSC)KEaW!4V`>-S@iV7+F0OhyVW7@5YN``_^l_ zt$%*s24+c3M4-ZEQg$tX`sN&8GS=83xbHGW35VmTMqiC6^bs8u4GJ1X^c$RdgbDF8 zmo^))*C073$!-Oc`r&fJ;;!rSEmJ<^{V;gg&~593QyO0}u-T7pB!W>@a-kz0{`5)c zg!Uz!Fb!>7`dsaBT3gBUtqMrVKE&abQ2wXR8TIi-UZFM~A7Pf!6e{^lq@qOLzxqQ93 zpmKUI^k9yX(U3O{V7OVa+PDH`xzcmP4+4`>sg`x)7NiR=Nb+NYhr=5yU0ac}&D#Xv zzwAZIsk8^Xhd3vPE=+)iQ;z9Xb`^v$ZO0Dg8DjKmZp!L?f|vsG7*Q)xyH!$C3?Oms z1b8Sa>uk+yhv-ku-CH-DcgLhv!(Fe_-14Y#RDs!x|788Z5ezJyh6Nqts3viT6xU5? zn{&W|_qJ;ynrG6n7Hj@Pn7o`%gg{pJKJ6F=+7$}nnXQiw`d;hVCOhuLR4~-6hi|ZW zp-;-&K9Sxro2me8vy|=eP9Zk330_WmsL8>vUia5e+1P?N-GmwT7(Wf%s?!2%R9*?aO zO1!BpiJ!L^j3>VNqdNvu-EV~xgdg;X27V*d#_5#?dcL{W#L=FYVEUG5Q>p zAB`U1s(hNcJnlaq{TDX6#X?xDz;a=!PgS{!hWT@^y6ruT^Fmbvdv*OGQX|EZPTpc| z;zD(WA;JYG2SvMQ!g)n$+HkR05xMM}7qZ}NrgvbIQ8slM#s~wj3O4XgQx5`_gIm^^S{;PK^$jx$}-Da#Cbjg_hj7ZQLcQlJBRodj{=sF;BF>40xVy#3VC!mD}CcZaw?pKz~6@6 z>t4>-+u*rf#Bl`n_;Utfx1s>f)Fj(VYPKVCD(w2v&OLHWBVzZ40?Ro$kXV&DVJ(6* zX#@tZ*!HTidcDa>r^4E<-PE>ez8GaW3ZIedful7Ayepskds&&QKLg_P^DcYtC29-s1Q7aKtzGfYGU=n~+)+$-c?acaC3~;v8i$8vMOj4(Zk~rgGOosso z;v5<*%1@78_elo(tA*iO;5DatiGm#*ih9WYZQ#wPEu1K|1RHvR#CgAF1)ztn%-v}r zfSp|z>b!B2_-z$O4V;&?;ZOOfJmZ&9BF!A9*-*D}fS1}yIeXC8V?5y1e*38T;Z^rG zWbabc&)ko?U(004iFF9aXOBt`9x$Chw++taZU`!t7G!fD3gb-(SsiF2msx98mqa2} z)2>WNJ&a!++|BM?FwTywUh%*bIAX?_W6hk0i=ofLH(PfpjkSF4WSm&j6BRc4EH&V7 zeY=ipt$p`lG$P^M`bwk4Ip1jE>o|HV0#4{0lEp$uPpb2$u#-T@YIrQ=ir}Q&(OOI9 z(j14!VjdX6OvnfTDa)r?q?3F{BhA3UoM*3&`e6BfMq9H4KU2UBOT2LtfCVg>|*)@X!pEOTVh9yxAltdS1;Ug0=& zQwYMOX{|^|a-9igL@@eghCZo+{rVI=NC~4i*Q+$#`6-Nc&~6QCbZov*^Y%I5nvcMqRII{*-@mj0@cA zGBvy6jBI@hj+(~>YY?n>)80zVt=5d|#H$Jozr8Gat>dd>w9SxhLY#C-SvhRnYg9;s zv!C^pmDb*+ymP>^X_ShFMq3JWOPbqbb%DqOjxZ$UR@(nW5NtWIkgI{Y(wML?xUja3 z#aWb(&`26JL6{AUvx72QI>k&~*FZVfMk)?^@=Fp-7xGd1r@k2n6GxT*_u;GBiMMAuO+15KOm=NP|0^bu1=`lZz0^;bR4g zc7$IL5Y}SJg=AL)(l?xQ9={Rx^y<6tu~5l&e&C5Oo9amcoHI-1SZTM1BUm>UGL}1e zoF4~iB<>Hj;LNcAoM8`z2x~c!5}@vv#|Q$?ePEtZ7v38;Hgtz9o||>7OE_9sAM4r>3os_M2jV-q2s<^b2V2 z30Ds+Lt{KvnV`i7@%GkGGJlL{OK^8e$_V~wuq!g?`IrV&Zb#2J;SC;adQbtGk!cfP zRvS}JeV$l@uaxnJ+nBWI%}}bmmRRRoeXxRpYZW6+KBQcw8*>aXCO3$~$pX!Q=cj4T z)0AT(slGxQg4`(I`t-`~u@2QY~Y5TH$!@-ij- z?0nLlH03iZl>gQSlqWlIH+54;*cv#k79UA_(GD?EBlBWFmjJNr(5T+nx~#WKF4m&J z1lBpQ1Hyl#)f%qa6e_^~Gx=3f+FoJa9suQ=Mh~pi!V>EAW?yAtOz?LrP!M#G?C4G_ zZvUED-lSaFyDnZu0-p|k)(E)U-X8s=Lvz>=xLDfr+F6pD^w!1L0kt%r-dFpNlx=bq zng&xEXjYsUzsmoi?X9DtYQL~y6-8;01__H!DFuOtmhLX;7>1Hg1p$cxl#&wZoFQju zkXAYen4tv8p;P)h>hpV*=l$0A-?v(^ds`Oo+>w8GT)fCT zOY*}@=wni)maFkR#mTUOQhpr_czWJs-9w&-AMNN`-$q9e5WB5G~GM) zm?xiFvSCU};-+Z_>mH!SGW=-&d~gB( z^^mRn?w*7OwUgT6Vluxnp*rtyC!Uh|cvavfXmSgcKlp!zM zjj{NdnJWFBE-g1~v>LgpkD^`=>}$mepCoZ!tqf~VWyoYSZQ-Z7A#_AesAv}2lRIf; z%FKB`p-S)dVLI@1@AO8zt!!l#?k9a;(-AK%yl3-r7Gj69Etj47T%N$AYR^@Mmum<- zu&wHPZ}Lo{J31ntAcCm~l)Bx^zB(-@i$;H7c_2~B@MWoojwYwW4*F7&7FABfWcCg} zCS9H8A&P+tL)#uMLM3e<)0*t3HHoig#e*hPGO3=l@SjcLT#RCozWt-e`M$lde(k~i zbm_ZMD3(|JM~sr9+|2rkN|DA-2~(Q&K+mD~R7PX04efMEiM!M2Vn0sgX^ZexA$HS) zl-|v&cjCNRc|U-?@PI5OWaPVi+s*50T}N*wTqBn|$vd%54t=hz-toI{&Bw^>J~_Vw zvyt~>%q{;Aj27fnmdZ)({yt1?`HXtOcf+mQrP#Dwl9nmO3i0CcXLpN`afG4_X~sLu z=k$?9@@8@fCuquB=4!{g8?{#6##7e1)pjfbbS&ClT#v*JpTxl=5+2&5<2Wwve=^_u8msn*u;C%c9J4mo>a%gzH3pdvt<8wn0V|)U z!(?-1eI749Ghp3ppyF|>LK`f@i&Usf!F zX^;UPVblT=Dss>zaHe3SlifJKE1yvfm1 zl;`a`p@NXmdL7l5lQlRppMtGw!eoq?uj#w@Pl}4s?uGI>kWjT$Fual(v=a^RbmzLc zwTXA=az&bBwSKumF?}MjxA?77iYo6N{JjW}`Ecjml+wNNwE^jV&`d63-8IPt8nyM}DfBJIt9t7g91jOJ>gZ)v6U;$%p2V$-8d%>Ca6 zcfX_3YviUP3R}^KS?c#RXRNE+<2)586GB=(vm$J+hVu$}*LbV(aO+pweLJR%s2zt_ zI;DMbg5U0N7t1>kTulz_`;xtL4J+$?I8A^7nZ|Xp(a|dmSS68TT0LscY#2s2Yqivc zc9QcaKi(JTm&ogO&*V)xTb?;2S1VTu!WKtlk31uv+1!gL^|v)=lW)n(kAk>Zfym%$ z5H#=ls6I4@BBf+XE8H@=XyK#8q<_Eog^a8ntFAw{4Lm= zX4~KN)dfrYB*5&P_pve>3+D@FH^#40Cl&UIb85#6}%iYUJ_J(Ifmm+8O%#Wx?2@3LVAJWbM*YZEC=J+{yL(Pe&MKgGcS| zi`0?H^W5hYjW$0CR$GZI&c;S6lb`&$Hu?S#I{bOg?!r3POtW2yu4NcLN zZTfpHr7qI>5eHUdV+M3M>fR4wT62BJJZ$h_FQSNu(#(ZkXzf5a@1hGH4@pW@wteZk zgRLR2aAW99<0nvnEaR?g{EbqSAqV+=pK*`&yAcof!%7WHrZ_L2k4(j{p7Yzl5&qq* z!X((Sm2saswJzo|>L2$4lKoXNM`@P|hx|ZgP0b*RFwaJHx&l47uc!~V*Q$N9&9Kd@ z8l;410t;le6r{GEN^`qE@fW?qOYnkVHsh+I7bv=(*4&+gcUeDZ$Rh z7G+K9f|i17s2lQHx{eiVh$mO1Qldu487dn%?_Pf^g{6j7#&sL#L$=87lZ10o2CqVQ z;|-)BR1nQ=^ifO;tw9#Oe^wyqxleO&_m%Lb5BbecN@>Kbyn#3^*?H!T>qYB~+`&-XtVrT)md|_t@TNdE&3y|slWP|5ZsM9gB8qvn{qFr04ZQs&^sf7It4;eo+dOdKgFqGfYiwenH7zA(D^WuGr+u`J+4 z5Jj}Jvt*pKTogmp79R$P1J1YZ6M$`VZ)-0@jC;fvx6T!QXC7c)v7ihy#n5?-QDpb zn|8Yu(|O~KJ>&VSvuG0|)lXMG=>{7f;7uMs<&i9vzS0))BlocZKEDw+@FaeVnCWhN zb7Un$S_qz_(UgIX7}dy>OF`60(R+fQczR-_krc+L^93mTi+6fRdgEI9;`%G zyK;~cbvz?8#e1MtomK@Kxr$>+{_JX)CH6?keA-Nl_1RWm{(Q|0nJxA_PhnYH{I?lj zj)LX~G<#u=CXr4Kq z&gyKDjTN3A-yTRN5`M#@l|iF**Dc5{>Gc@~y2KYYG#`cK!B77P8@y4;<-Bq|<3qoO zs#1Z=iyJSAxd}sA@go>DsKF`&Ohb7AKn?!i3HF}zuw#t{g z=8U@FCnS^`5^Yel-FqtnU;VlvUe@|@0jhsNP>J7Vo%U^~LJt#5-)>%X@Opl=ZCz$b zQ<_ftL)AQm&^K*`^zdMO{>o5?*HtV+`d8^lU$~9gnGl9(CR_-moHY)*!&~;Gw%eBO zgr3vCvAT73Tzd<4v3*`Q(|u5OCHP~mw6R5Ia#__XufHPcl}2???BUAev^+tgh24_I z<6}%os9FW2%*$rVPi-*u&5H;PB-im;&|58CORA3SXZtlMr!UzoEz*XR?cn12i;T%o zQ<1VlX`Bw})Fq5C@cPvxN7tq4$8^b8sdju-Mn)eLT5)p7^JvB`xf>T4nNucQGd)w3 z%dKrb$kMvj(8@Zf6Z%vDD*Q^5Km`mzW&sZ_7F!K_b&-3t?~jp`d&-I9LcW9mIaisK zBQ~}5a_jBk57lM)UziphP+YmjYc9*|rfLtzA2SYACsg%})O{mta zw(gA$TJ6Ci;m7tKuHzfWoiVPiV^O%zY&TF-X?z~*8_LOb8WIe`wV=+d|NZUm!PGm& z((FMy1((bKw*UOwsID${aU-s}q^PLq!S|i*IZrEJu5J!~vMzYy0Oq;hm36q#!0JCm zSWv=Bp(lw#KRAx&X3meR{`?gKvAS!-!M9@7y;K{;=7Yq$OQ`|BFr8XcAw# z)i^f$_Q=8r*ZecD9jUgkiywJeSzOBf$KN6ZWlc`EjBg#aH}wV7icT5!2GmwJ)>Ctt zoDHPfA1~UybU$958c418KG}$mnx-O2BU_C3-BS+nMvbRm zxvk0O%#@)?T0<>0=aP!ANYlkW*5*vpkL9V+(W8_xLC3WH7{|HqvzV2Wl(Y1O0}aA@ z?=7q)!Dp-UBjxmYiDyrCS?Kc;MmNqo{ocDDifzPJ)?GNKXQvr!*PYNG%Auw%s#S)5 zZ?)1}-w2pwvh}!_7nS?D)pNb^bf>#1)&Fi{Xro!pzyG}W*)Oigd0gyRKZtadP_j6W zZt``8dxDllePGT?G^rSPk0`~N&&{Lli_rA6~l}2hrNz$~Nx7%U({25!+rtx4}gP53b zgV(5@Pt6%-!L6fdxWyr31} z?gu|Z?_Dg7>2%Idw4YcVu3z|jx!0b+hjj=jIz7a^UkZIZ!zh25�>xs-v8)e+s(r zTZS<<)owQE?EooJi+iEv+Q{a3A*@J8#XUKV9q{ zb$VP+599Rr-CgWkaDMqXRbbPjsl4uk|KrcPC^H)ed1#{pu)Aq8lJL+&#_89W{#8CKHvbu5Ke~X;CRvvP-?~3r>6fwV|{|%JY?VwI1!LbxgEJ ziP&`Gxu^e`^Um?!ooV@Z#;x5)Npr`(OGA6absuOrryV)D1jR@LH+oaf(2Ng>=D#kk z*A<;4IvozJFVr2bhL&XO^zTZ@XIXlV1fL#nAfNiptekGkG7ayJt+6_ZJRaM1kkk`@K;^}GgR;}MUmX7Vk#u{p7>bBz>k<&f71n{bBN|D>rSf@c%REf%= z;LP`Ax^$zB(6*KR8KMi()bmoT-Ie`s=_wH#QwN&?L>uEtL(9(tbZp;FHm-g%HJ#eM zORi6Yq336BOlh*fU0n%1KkLjoOlcIOI!7Af%X+<9e_2p>-EH^Ww~t@BB-w9HD6emK z=0I=nGS-UZ-^)lbAnqO#|wPC$A-72A}usrYI(ZX8C--$Cy?jx2g=1Kd>Lbq~TJg?AX+5H7CUXziP(7SEzdx;m5 z=*3T`yM7px^qd|N=BfF)Z&%~1h;U?;s(QagFK(uMq?}gdG=^n29i8qGIhQ|My*Al5 z=*R0f;n*aSjMq@X{*}Y(uyelq3>09jU40RgyP20)GbOUv?>K$xW?(!alCzdUz`k%ffDrK7i0AM2k~OYOOPD)uid zSr4yUo@hGrVsFTNfgE`{PDA3OeO^7O8~4YUhjbBHGmU9v4yPG8ifPtL%QW`|Nvvjw zR;hhb+G=@bN2;jvZS8v~POncv6s)hWU%7Ak{E3*Ivd-$d6N*4~I9*^WdWB>*ARVUK zM2{T+642@N`RE9Fyq4t8lc*XsBN!mI5gNPL3FTQ+JX_pZH5)cIp@vz%?~RmQ2s&Xgt$cpjr~U__FTU-@C?hGOI_Y>Eggf>>6iOZLW{tUBAPr&0OL3 zQ9i<`%G-&u7yBszCkJ&i{yxtd_I`{t9q(WK=-u86pLYz{#W-H%I1M*8qEZzT*Ihkl z!vX@*5L^?+IV)Q9p30~7FojYcl&n#riORWQuc?Q-OTa z80e+H*B-sP@6(I1i={zNcjwLn6Lw*-lL%;cg=FZegP7YzWW~ue)NP_A zuyWeelj+pNJ0WgIWzfR_?Rmai9N*^)-$&Eu&HGk;pbI{XJNi*}^dqdby>oHCNr*ND zoqilm6`yXROwtxNAzCGV^kZeI{pP}aQ!QcFK6XwjezZbmx|61;KUwQs*3Pzxk?ewZ zmb$GudBf`>mrUTX4TGm~?e+`Ag><|UVhcr}%$nmT5N&o5WwG}k)5{;bY?tYg^e%C; z!Xw=WeU9&Bxr8b1T1Ryy+lof0*~SGJnqJ_@F5F34?7th&KkQ4G<#Gqj)g^D#$2X;7 zot$32*Yr$c-&|%n&)2P&PlrB_hx0@#TPztEI@$sln)m0>(8qRz$&3FPx~Dwp(pRBs zE5!#9}m*x1D`T=)``{8{_`J!5d1I#j4b5 zq;ijVWi5O8(%bVj)kGXJ=18Kt9zE_&f65BVJ&LA}m#4BGZO|VK5?%=Kb1z3L`c#72 zorDLDdlwE%9CXzC!MMtKYUL}O(I`{ucfjW`FEdo*;XF39_=lz>tpJQn$8-7 z1Ra}p>ubCZyG>4e+ICIy@}GzZr!^gsh?>h&o_q^6IhUCDA)|63de)sw6D4@Iz;)q! z+6CpXibDT(ngb zEq?zoySgUuZS8=P{@knXrP+tWF~c*@#u?Spn)>5j>6bO&-fg04`lPLispvq`)pM(l zH#cPyb{&iS=|-etIHO<{=A#pPf|7v z1JI0e+J!3(i6u)kLjl6IA3eNj)06~rbe3A%i;g@TnmDF5;Z33j3bhNM^SJpsgb@9w z=F_h9)z2@V5H5}dlwOVAPDjH;!I3yS)AI(j*SKtMN#(t%*SRQ`oL#`bYMx3&c8$yH zdRq?wE*h%r%Wt?IPO@2|0uv}fP`oh72kvchh_E)V zc$(z0pB2H9EXpT#fbS5UL0A;rThMslHy&kJL+Yg;^&i||ui8{iQPh;#|N^t8T2yJ;#7j_!G z3A~zgaI}37s|sS?pAxa+VMsucc+wG1GPo-fkIwKpNtmyz^6gn}@Vju3kx?q+X4yUe z;g~XfzUDZr5E&_hSHY{?m^z&lNZ1@!%G(Q(@0bXh@9w=w3K*`J;<`&YqZHV$6fQVC zJKfZ#S2Q)nmaDP}VWtn)#Gz)5 zSMX$`nY!ZilgC7_PPNcTsydxsLim{Z#_G-6qD`A5<_uq4BC4CxjC`A#-VhNjRL`kK zY2OG4psN>^Y)G}vU(T!#z(60&-hjMXX3QT8Jnp8w=I&jBI5wu~7P>APeNR=9`UDfP zj(<31Xq^z+*NLZ?W$m;0`K#0L_bsN!89^E1w7~Eit7yG)qIlBG#S`Q{8NBV=8Tc4{sD~ zk)OOIFVCRD7DMjKejq>)IE)7FG zNCbMlhQYAK={kMfnA~1Ai)mql8MJYyS;XlvD`i$ zu>MoLynjB-`iW-z@H7C|o@LLc_;p3IF1j)eN2kSz`c1mOU-UfA*rbEFleG$v&!iij zdS2X|VsSnU?{zTga-bPWvos(Ob_htFOqt2g(Gl?8ET`#Q?-xrx6+xYy`jxFO`b57^ zzfg5m0@1JJ}vKKzTC`f0&X-xiFTY6<= zS22)&8*6e;=MlY^z-o2PhpM-IyV0_Ba4*@!%|4ild#4CJQ62O(n!>R0X>z~saRJTv z`31tU=_&s?vEJ53I^ndIevyYuIi(p$Dm^{Z;P^xsKt-MZS z=;nI*%30~113FWqw?c0R;^&wuG;XC-_u~3cn>MSHbZ;mdJcvhJZ($tWq_uIZy2=^w zqRYwI89`H1rx6mZ+z)dnXmwvS$lR3Qu6zA#Q|go@c@sf09Y5))S@-B<1zod0AV}wXh;@ixf*wS%J1$tzq$Z+g!m7xj4YCjLkM-XBCyTr#J{98XkfBi{#16TnTvfzLgYX|WN%2+gCM z(=RKpy&KNA$1vv{1ENlNljnWw0`=RA%KM)ioI1a@etnB>$W?K+-jff0;Yb)bjsy4G zPt)z6%{_QGm5NbX4|A|fyK7nE=1kycb#d3x?yHKgOibK`&2!P=6#WL{{=Hnpcl=l# zq31`Fn|AaQ&UMqHAD3ad_HXaU(T_epS@P5Y&W<-T`&?FICuw?gb>BPJ11?QJIw?-P zTQfZ(T5cN9P#`j>1SJ+HlqXO8YI)GVottplW_?|5H#+1yZ$rT-X8*o}-EMI7}UP#f>LaalPMuRKhb3U0kq7^yD%LHt}VR^b4bdm~A`w{9e=4 zno6Sgy$_Z}kn2#27QH5`uw7*onOc#_Z9B0->@IPRnn+nJ4n2^?;AaIUzc~K%oPUsb zS;B)9!sgb$4&_WmPEcVFPx>^f&I-4>xzF^~+T@O%wAgc&y1Bx{9%XUxoB-z!+kDHS zUUpht505t<((AkuU}v1dm6Y*_9sli~^Bz@$ddY_l6-a4OjXwV3Y)`pR+)8R5rF zepBz-Hf!;5m&Y_gaY#}M8o(Md<1XHA%L2tOekc;3=-ELOErQ+~ukPDn!=Kdc;eT)i zG=XPW2`H8rrO6JF)*_m8tg}gSyC)9kA-hkg`w+z#wfv}FOehOSxBGZQ8Zicfao1QXG@O7rL(sU~xVu-; zA?J?z^8PVWdy+HSfC{pilA)^r%ybf%>0Elq&zV}{@PeT);Vuf9(Ea)RT12&n;NO-Y zF&ONrByeBQ=8H#k54m$sjaOz?x8D1E7Yzj*PAMp>Lpr@f_ZgWbuRsuQ&Ocpi{rFW2 zxdEXL70Kr!8Y4g5LgAM7^QxV`ko2izjM~!6bR*GoCkJy$nfC3S&f$%G`eUzBqBAvs zUh->t>gVI?@9yrx=jY>79{OD}UW4fb#$T>WB3vof%D3||Km~PewP>+W?+Un|)4-zW zcr0`vj=t8BII;oRKCPW}mXv|`(Uwr%>uu(fn|gLPN2U*W+LHP0wbq-) z>W|sHhvnDZSo|NK@IF|(+2=0gM3ZPNqVdH)*N!(yH|Pp(DNJThwJL_cDCM@RdvW%nfmK@V*W^;F z@IpKE0dHt|P3~`uw6w#3?l^R+)egA%@z~0(|K%^A7hf5mgvwU$qvX-;M1$o+KL*Hk z2Cdw2!6a2)-&C?V3>_>UHTTnyo8@qXArQ1QV!odYwEp&et05#VnH%(Bj15~5SxfhQ zFt2bhA`GoUd2T|m%kM4sX@Ql1W6ZlG&7eef6atGc>9(loYI_nsCy(k;{sx*8U|qHK zU104k?mwPoQrE11yoHK(iJ1r929-R);;A$#b3=rn#4zXeIss#S$ZU4O1l#oz7>y=K zv#d@W2w{F_8nQjUdY5>sIewvgI)+aTwriP<{m=-9n05l(q_n`XHt_T$9;D2v=AC8W zr%FDDDLSNEAxIC^k-<|t+^!BR=bu0h8FJrfZZK9u47q!u>;$8f`P!u`lF}s5ZD8Xc z!!~${DK%{j%#Krq1a+(q6f>_dke6i~j!?zFP5LDnJTNG2q=54f6=LOT1d8abd;H5J z^;>y^0A5wZ-Im(Ar-=%*0o}BPuGzChUE1s-t9xsRMvo2nIU!I-WoXSSQW48A*$A4 ztAQq*X&Oj!f82!`+yKuUN{~XA4G*#C7+y=5V5tP|ZwwZLAb=&vxz3UuS2#x=Y(Umf zMn7JkGQm;}Y;6}KsB)?aJE6(`;H!z~i+yK<<})7)g;is-{|!wf3OE`LG=8dOL*!Bh z#A`OSy3e*5|K34T0Y~0*0pz040wqQy(aD8F?_A0i&bu&DWf?7^e(6DN<_AhNp=PM} zsK4fylyg0=fzYiT@O5n(CW)EE%Xx&?78AUiiMT2`h^T-%i2j2fiON{dK-X_|4kmC2 zRyeUKl@1~4svb@fB0cF-S#GHRonvHRS7Qf{b%BK)U)A1|!Cx45Q5U`egi8H#UNhhq zXLy|)prZ02`i?|X0gw_U#{Uw@?KREYe!@LdACusHR*m$q9u2p{E38(o#j z1d{}ZnQN@Y#r$pgq#EexbT^6_$Z+4J?G2eX94=+JFLIA9({^EWGyi0+$HjXtB%6hz zd;U&6Jz%Ed2~~<$z{+5hyK*@?0HpaVZWa-0onJO%KN;T!H>k@H2C=pO-hLw28gPEk zgkPsuA&|oWY+(g5;uDAJ$yUd@OOLElyg|8F3hC+@#sG882kL_xVgzt|_&eZ1E4tQ5 z_WPP6oDls%K4fHjIM}N5)4>4PaBE&QLOUf)e^V*accd}f8;a@Bjs%xMJgxFdf1>gk zAF_Ouu^edwsx*N=ZHX=YiQH~V%RYLB7J#z>2LVK%0bL-HpYU5nYB%3J9~qCx1aGc$ zhr=jk3uq)c!@MmhxtW#nQ3+A?bDX^Gd#xlinDDa;>UZGU;ivVj%`bWyCFSx-~m*% zyulst{FB-;aCm&sB?rj{r#d>vBLGxzUkePU*KLa*|0}T4BN6KC;b`U&>r(o)NAN9b zv_Pb}34S(kBES(~gE$Zbu2?N7l#!$C}$&M9sG2l7FwLW7@Ny znJVmzfYGq}h1~;3a(g2*u7N#c?8MVj5vLT2vuER(3sw#aswg%vn}b|+8Z5c0BTStE zfC2{~V{fap0vVgx0fIXO)sY5DO>>$Lhful*ACgNjNLHmRq|yK4;Wq*Ol6an9DJ~F5 zaU-q-kV?axj2Jf|Y$NLq244c?SB7LW9blpVBx;DE=ciPH*jcg?{CfA?cuskv|G1-} z!>q3p+s8cBpNaScwyi`N?%BIWS`U~e@M8#InlG>$B*tlweN}^7#Yx#9FibJe3^%v&Sq3U$=Ss_Nh$0BiCxm-Gh{F|z z7^0p?&X;$N`w!ezwkR0c;$?y2}QglvWulVcuy~XHbX2w}(YiV7 z)WL5uh&y~g#l-v_PDvtGk^4j4$&>G$XhL{AM=8c00!jFf2YhEwN)pbWsGSD#a3d(x z;cYu7{L%#F41hZpPA9dS7*HMKglaDM=R8}3bQQX{W%L|x2}Sy##T zHHs^#wLQ)6oWq6vjsQpKGlFs;JdFrbD+h&)3|`0DfRtudOqm_7cmG{QwOJYpqe5!e ze_CI#!Pedx43w&WvL%oi@Ic`8S?Nq@mG;-$-L&=UzY4_s<9h@^{3Y#UiP1Z1Ftspv zY8mAPfg^H&0Q`gZ1ZlT+S#hn|OsIVMGsrsHb|hdUf5j&O2eINrQQXh)v>V=^e-WO? z0=R}TNtK$oN-LX=ty$;?suKCaHyrQt(Vc-gkO zc|HpQm8ajW99Ox+adL_}mKKFHfcI1hzrlJ8*3;pToU9PrB8Qhapa*(S8>WQp! zY1HMGAjpMKnD2Juz#6EvMfm#N8EW{IXZPa)Pa4Y34owK z+k0A|8uvWQrj0|w(PV_Mu)&T#vcrf=Ao{)VdLDeB5V3QA5RpzNyEoAGpMuNc8&Uz9 zZRq$wFJ~RMlD2p`l64W0^mhSa#nP+*k|@r(e#gObxl?p??mbXb{mge<=eE^AMonef z9l}$q3(T^?WFNF7n>}d%+fESnN##noe|uIZ8(7Dth#8KMWd1nw;qR45ssYC|7kkE- z12f25a1YXyNFb6_TL1e*ZOcDFhJF>aexONa^xQze;* zMqJe-xb=_qs~>&SrWAQD9Cz{x@RdI>W8osqVy?7a`!;jkS2LBy+vu_<0DJ|LZDx@X zg2@0^YS(x^##lr;h&dexIX zH*WUHfHB%Ucfj%cK~$1u20&r@C-Os--*^$n6=3R~S3`_=sh1*N16k`&1C&usps%NP z82-+QEz%CSm3zGP^7evig29sNYd~)ntkGks%9?D(1N3(Q$3&1)kx`aV%BkJ}72Z2XYfL zFiM0kwdGfR5QyJtT9ldee|Bq0!`YV@p4%-cOg&D@XQdYE#{_SF+@Zfv{Fvk+y3B zb7uhNj^|5l@l~I9jDq<=FcgEw9kXsf{MDG^@B!d2e<;|+<7*3*CS^t9!D}rswDE$2 zkoH|l-oMQQXQqjV?O(fXDNJ?TvDbcEex0(5t`;!OUCU;DUd#=BsXRN@h}ZU%ZuIsB z&oIA_-itAjmELqNhurL6 zHcIE-3h+SsUA(`dFiNGzqB@PC977W!NZ^5PNdj-PJo>8Vj7xizuL7+H^Q`TQoYfw~ zUjBEZKgfi=bjmrC%ENPqO)mNP^yp=)*)f#moncS4Gr?92lR zF-lHa^_<~j$~P#oSaeJ@{x0lYCW-{^wUG(C>HORy<1lkKW88!7S4T#W;*@a}h@d-V zGM*1wy+_A{s^HufCxf*xxa5D%s9f>IB6c}j0uHMMMTosqn&$AX?0LsU(0_xrg1pRs$5DH zV4RLyEBoPTrmPN$VtA&vq7iWg6|bfXMm+L>8X!T?-(x z?qT2r8CK<#U+hM)ExVxRmk`Hg7fL20x5;L~5<_4#Ry!Ccc=p?6|5a8j7(s5N(_@O{ zSb9&pUcgrshyh&v14v0p#qWlG=*PDIgNSYUig1*_y!oAQ3*(pYWxG%;e2dO9oZfM= zL?@sY_=6f&HJ@!QL2~4+x{fHDHQm%hpxEdD5NYmXhi*CYixhH#0RkPphHn9;yh*HC zRsvLee^yc^!6FTk%Nct^TwBU=iR@m^E#TLH;?sI99s4JqrNR4fj0}8&m_gv-=Yt8m zaR46tpPU6L{widh$pdGOc)iArZ+7?COzkq>MU?l+(X6fl$8I3az)>;@{{%GcKYzYu z8v~>~_AVi7Xh|oB8G|q)ZCI2%qWo_xpw{QSC9zUFkB2ouX;30oIuH2yUklcKLrp1A z=ZiW6+7@#Y^WUI->GhiE<@|Z&)gpS}9WNlw553c_@&B z=w z*+qH2&J1kF85jr)=iFs~@D`72X46xV{j7?_+@5^$=6_~Q=id2^&bI(`t{G$VeqgTa z7`CQ7f{T>?jEX9Z=v9qO{e{j4B-p2d8b5%S2qu^intXF1;SqL}iqVG&F2&mz*w)yUv2a~rWAk~kBLrMTvY>4fy#e|EbTu~OjSpzQ_NT}|aLr|3x*Pbr! zzckP9vX`J8s@74oukEHa2`eQorf40I()-^YHms=~dH~>m_FiZQ2qt|&-I!u|MIw4X zonjim#(F>(Gb%xCP$N%r`0zS$mTPW;C5z{J+pMHX)W?yiW(>f^-qg8kKH)znN16M9>Nitcd)Xxg1`3!){pC+R$qge_)3o$9l5C{5fP;E%#E|le8-tvFKo;Go| zHuB9+(u?M4vNLnl%B41=XHpxKWByYaNu!J0GV*E{|D}x3@R2CDvTIq}6u57=l$r^) z2|YqE==tyq$VW?;13~cfG>Va|f?|?7q0(2P>jIR-(TM#mQ5>hmnyOd!F&egJ&lsQ|V zV)`>lnxqlx46UtNKy*MmRJz}iG`QU6K-Rb`9;KAX&b+rl@1?1UQ;c~4Gx(#3*f?Oc zM)<294QlHS5|p!TH2-czvvQkl0dWkVfW2vo^&A6qjT}pvjk8q2!$oFTCBHj<8@g0% zox;8zxS;`}7BzQ89C>@{No1jJ956&VuuAto%4iq%DVFOKH&g<5wEly%x@%6l(0&7$ zK2U%eOUGdAR!^$-Q7`%FMpwYE)wLvW4cd&t+ogac@<*C}uvAyPjg&F~*fOYAmfE_i zqQfNzb6Essf^WlhC?XI%u12ROXP@=KNP%$>_FksUJHcTz|0 zzpYhyErrglxYxz=U#>0hYbsyI2-uW)GD=>!s5`~!l6#iNaStND>Q_q-BJC(z^268b zFRNwAJ#vnbu$yl)Y|y%5gb2ldC7FC^#*+cF_zT;y}XVPnYLtv`toXg zII_TOE8894h-hk&?1uAOfQl?`J`S^IMaEL7RT7Be4^{sUIswTm3SnRKP7g{k{oyCD zc4kBvl|O8PBP^r}$HTt_QZAxB5zO#{6DPS!NSQ|vdF^F6 zkS2Lj|B=yp`twXENuPnn1`hxoe=t2M7>$D%8iSZVL=n1BqLJizUoTm&;{6M)fpYCT zs+C~j!4ELD4mG>Z0YGIj{xNUUosd#tTffdS<`uQ=R2CtCI^I1ge&E>0yI z?O@4?*9{OTDWuG|NDa0P(33bsraSS7I|SovbD~ZPM8O|^&y4sUE?1Ba8JuCHRy$&# z&Wr%44*xLn57gLl!nNnsK*qQ2>KMSo@oK--n*SpSX~~3@+Lay1sh+&I>Np+nUhks$ z0$2(TF!CYHd_4e`!ipiRw=iQM1p}Xc_V6()eAYLJ>Yjf~ce@4IAT}*)X8kN~(}rPt zR$U5_tTK3zY8mNMpdv!Htux{OC=sig4WulOyyXe~_+SV~Zg`#=p90T~5VP=jS zDZ+hGVh~A{6KNaacl%Os#v-haRVw2V3@CSqU8ne)|FBdxWX-x%_)U5}fFxRpaIEJ_ zu)7prFO_>hHFkJGxab^H^x^b@@mF<9VBPG26y+0Rlll{%=-Vif#Yp6oal{;)xC?ha z{C|9;4aFS-m}c6$iJ~MO*Q9;<3bJCnz)|@n%e;uX2EY7sclJkgVwGT*VUL50Vth6v_(ez8}1C~Pg!4vsMp4*%;8e)_|C_T38nvt z6)aat+cFSBv_|)v4$sn_82+bQEBULgu!vN#C%O4lKak>}bRps%?Y01v0FC*9#=$&nJQ*OOO0}s(Ucsr2G1yf3jtmL`dwG)Pp5?RSf<4Y zBztpGUN1qijNxapSg{oRy4J(kMf#V7#EE%B+y~S70R6g#`?Z+&zPFm;eEctLl*!TM z8*qsaVXWxGDKpMOCHDY*em+XM(ywe#@wX0@Xf*h<@JjCSex$v2%Q$v||6oprrC9Sn zEOQS5o`x|f)nHrm=Vf2cMP^yjCa@8}Zvl!ko%_5gEzn(1%H$4s!tOy8+Y?NKk+Z~_ zDk>Z$t;*I=T1W|IBEgSROlXOUi0>nE}nD)jRW^? zSE>e-m@!b6TVIMxg8=+55S1)~E3;6rg2apAnPZt;9*JMqNRhxrbqf}~cm|8o0IBn7 ze@LEb43wao=*dyrR%EM~6@hNP6sjb}@-$pJhL%7&d~q^iW62xGi~OAp?V6tvKn-)= z-OzQcH>zM(tzu+1xV@##hoq=t1PUrZV(_9pvWWz)Q|mD^Ds^s`z5%~Fhf8qV{1#-> z=s(2r%kplYj7*af3bXcg+-lmmm7rf$Ep`>qNX*fP0`j`%q&`Noodc%|O5K2CoOtiYWS|`xAk@}eptl$jHf{thU5NLcc-uuz84*93#iFbcD8eaWSzRy5$c))W@)bkNY z?dPR`%w&e>_{&nEWCe-=stIRRXaO%t^0n%&g#ZlwpSVfdpLn6jUHG$fEDJ+L8kzj3 zbTo9n9SAw#)m1J6uAB&_Fd_xNL;S*}btZy~qfp;bLw+tCKpKBt%>Vz;_U6%0w}1Tq z-Cao{3K0pn?0X6kZiz~kkg@Muma#7xTUk<;R+25*6*Kn1*g`1#I>t=M9%JnLHot39 zAKc&H`JUfDztcHRhcV{;zOL)FK3|Xb4Or^HCo7z^Ex%oiJp6o!6UsbuuI0Hr5TKO) zXiPo_7tebtk}R%2*$SHs(e4qFlSk{$Me@&n19k!^D}VRKTXMuvSvrR51{&o`h3BQS zvcXv!vGe>YSkqrnLvscWr%P?N-HPf7l)(Qcd&b^`OAB)L#g_ySp0z=vxY z?@7h(cXZD;X)~3{LhQ5AGZ(cMP!6k13VqXymO2N;W>2hhjlC<)sgNY!zgqJ}{M?Jb zn`am+E~0>H~(H|7Ou^^#RnCC|Z zePpJ3d%YGJfAO-E-gPa}LaonWe^NYT()}nUR>M?O;$L~X;Fn)rPNWDbN&gBi2zbs^ zB($poL?L=KPfb&da_jHb<5M0Mv%}8#2`H-X z*}1-FNev)D`1;RdjW)%S!B2g#mKKV;CfYqax@y*kq6{&(w(%2suO)cFi|uF*{odmM!oEhS0Qh?3t6#;xC;En2R6PyIXv+L3V; zeXh@KFH4fgM?823YRO~SC6`NA3i_oukZii~^3wdlyow;#d_>AMbvl8EXD%zScVx`= zP_+O)-2A@m*O7=rwQ(h`Vu;kJRpJ?rPDqn+qZ!XE;?PG_!SdNA>Mp$`Ngeyp&yB>A zMD!0I$JU%nJe=uzxrLx0Krz|QBOIbtkJ^jZnT!3Nk?8~R0(~+1=9HpL*115P&Jw9* zu&#SCpE~36klOHOX5kzBn((JJb^|Dvb>bV%9GwgqQdAskWG?Tn6hBf0(p!1Z0{pGP z{$6aF7})^t+xk%8x$)^So&U^D7bW28_7yK;{6%edq3C#O&|I%NN?^2P#*Vmy>;AB> zK^2m59+WCHuKNNUN6Cn|d<#+Vw-A)M$o3f-D5!i@K{qpQhn?u@(aYnEaktI(6^J>t zXZEu`Ix*>&DRbCVSI~EUTWLDx@u~)-O3Z>$3-#wN?`@^QvudOWMob=mKLRmZ$=NA> zo55Y~+#8T4C6n5jd+KCE+aHOd;^{09_0`*(ay1)uV!EwGP$~=?diaDvkY+H$2V0QkYpVHkxyRnB_oC_+@0* z#>%-iNP|!%D!aGyYS-JpfhA-FYTWo&$!0*1=jx&J&I#&T*+Ay;w?l}GshzWG-^!m5 zAZV4VQv?*^V!x5uTax3UT3PXQZ?Chp=nVunME?WER}LNHiU@=wLh;enV4)`TIsDOI z83%OmD1T%w&_tfHOoeoA=(DnQqP^{l3yl_zh!h`)3puj$HSAfIVcgvJu|nzwV%QHp z+B4vJp9k|iAL!&58jIQr4>m9}l(?emvPLpj|C6^XK^!v2AbRD4tHH{+aa(4i>`F?7 zKOQ;D*pCzsS--VUjp{-pp3+pD_Opp#3j3uQ+7~<(nbQ=zyHrl)HT5ZO*hP4$&ez9+ zX3#Kb20chV6AAKw*zv~?B46R7Pk`hqartazp+&yX%YIQ)AViB+8}10xj?-m$+0B}+ zw{WjMt^@>KYz@{9PZnYQdmS1lbjUAbMDdtTUtOKbqekFl2_z%~DsA+qI=ELqY;CbG z)`ozL8g$SyTr%)YI3BHo`r0VuU>3biH6OI^D|B*KwxIM2kbdol*|ZWtcZPkRro2C< zltOS1r|c{>=iC#??nKcJ$2Vu^iZcj5D>D7~DryBhjvkG3^~tv9oVmPJU7gL{X)o$So_wWDE!nJ&VE;vy+`-0L75^ff4D7tvx(pkU16g!>>xVAHrw7ZnF zD}4CD=jfw4uRU-5^_S$>|KH3TiT^9}#_t#NX6rZe=C2Z|K@*tScRWkK7u7oU$yn?W zwD?nB?Y^h2)9m2|#X%dh@p{_6&)8DyAvRsUaqQpTqSUm}A59v(7ssAGeM8)!=H@m@ zaM}XU8*UR{q8j;1efwk#qD`WFBRxF3dEnv3^wxw!w?ujC>UJ+K`I^rP0jDDcvr%z} zywh3iHCP!RcV5`ZU_PBhTqE?>d%KCVj;syL-uy_*JNR?H0rRGgy?#4Js%(4MRiXk< zyhL1_7>hS@%f}@I_<(7C*y8lq%35nZzEE{*bA4*UuEejHS!j}@>b)Fs^Km=tknf^r zi4SpNW#yKQ%vikSwROOepI9&W&N}{Fhx(%P?&s9gnQvvlUU8n#0{;WVn_&{gCX}=Z zSONB?yV=f*=$w=`uv14O;ZOti4QZuLk{^%bB}bt@~YdF>CSBPDw&g% zsTy5gWf@o<*dlDKlp_h_yS>fXtIoV-#Qvb2`HgSL*jY*OH%dgCpKEWCQ&r4uTexyfKPQLCecAiol6y4$eaBj*OR-vr275%tjsfy=KQPf61~6m|Vd2;QGUWo#&O~=A{)<(g_J`eI~wAr(0*I z%{4vBr4vJ^-8DBy%Za0QZlcS5a}R$!lJiHFln+~|t!}S|wimxy(0h;PNsoB(&(~#p z^(q-0l2&Q8#;>aq+Ey(S<`@2I3Yr;O;&(~mG;7n31%kF)XVM_sSe#G@t=A+CtCUG- ziHz@i66@x=Dz_a9a8Iu9v}g1BdvXt~`hNHIS*$86y`TmVJ$Y}MZQps8`*7zBR1=ry zce{7C030ugYfioWN95Tl-iib#9>gH4VccmK^0MKPfr&-eB|r8Fz%BB3_XS>!__YO@%*M5u z)Vb-!1@5`^vec+4QLd$4Z-4fHA1Z?vHXGFjiwm7faC1VZxaYmeQWlM4ZzX;nn$<$A zEu~M7O~5?d_=Kx*6#+z3=Yfi~$8X%X-94wBa-Z~}24yzWnq;A|pn1cb$%9_Bf zkoSx8jYdjD$$&<>uI{;Q95wfxxL<(yc5xY_+Ty&_ghol()WpEnwRXv&tBnaBZE=%* z&rdFICv~U0O0*o`>?UmVQ?oYNd!3M*5CBvBW*PH@{EMiE7l(+`>$?h9=HxWVhiqbT zO*Fv|-PirS1Y#1uCxo9!PnZ)9L$x+k>e|F3U%F;_tC&TnR=hW|#zc^vL>5qCLZD_i?Jke^hY8724xG^kI>0Lp`*-y8#QBVO1|aaU)!h*a8ki&Eni*&NvS`e|!-6^XIAlI~)w z067<#V~!`j7H=#+kfn@%QSN~;o~d3D0^!G-Mx#O~Tcye%O_y*u^?{ zX5M-?7kT-@V}Q9)QyaP(AWaA@oCvKHy#O#bu3q`Jxt?B2A5OF_tJcrK*o@ERo@gw& zD0H%w`i(@clk9z-?mJ7diN37qnrnQ%)bF=*{nuK*x%_3A;I6vnDY-Gq8X#@N<=HJF z*?Gh8{q|ZeaXembcQd|w3Z6;aZVuQrsg_C{blGgp-Q9=};4{q2^^7l^%1NZA{yxR3 z-n}ch_2xu(&g8FB5fobs;+yNSAI|2o|(Nl+OFo9;w5Di zzZSy{_^dHYL%Y>#4TO3@OaO8wVWxdmsQvT+y;nfb=s@eR&l=jQdQ&7r2-#RL;vOVb z?t=0*Q9g%GZ>;)C)T7c$t>-A*YNYOVsk*PS_W4$W?{CNc$#`Kj-%ygNc-#=L2X4*kfWEF5R~>4o)Vra-Wv?*sIurE8SRqKnwe7Z;*X305f9_RDfs zu>szXrW+|c)HkN%tPs~#MuXc!ZcW`vS<>>15RkoW|9ay>Lt6f(%iXB=&fu*qNpJP0 zWm1GE@kRLHb74`QnA?{FMn2@+nHUPVr*tMn&%LcN>P^c$>4VRN3Qz;s&He|!0_#78 z+IZmU-!zD=54CQ+oK*M0eNYI7FZ}Ys@So=GXF&zlvF&E}+E~3w*CO!iTC8>lPK+;? z&`k^mZv^3niQYb*n9Dw{4@#WH+5qUrE7$R@YCD{~EmIzVZqxzj=4H2}|5n|&_3qd0 zHru?_x~-|Yt&rXl?f{~nmuv;GZLM{dAnd2VmUmrZTFx>yz5510o)Cu`p1e1#s9nhHPM|IIPuiO{J3^)-*ooG47v#3(D(LB!48b78w zy`^2(pXx)caZfiG9KBlmrxM?|VVvAB%D!vkJ3`OQ29*o9hId@nK2Nx=Wrt6=4*8GH zIbXdz>{iZne6*#qXs0H;a~vUee#-(aW&~OF;U7xy-N6lQ6RS5yr{?^L_@V7p9qt&M zir9@-whhupJ>GOK*9~#>M?K|V9BI$j3~GAL z%)2T*ie7w-Rb@FMY;)}9c!jUwosGV&S?_X%zUrORlYX6JFYyDj^K$RhY`XJ@2ODRX zs|Ttp<-S_KRk)~Gi%Y~VQO(p(Yz;3i?|i>!`Y?rnG$M$W^3R+Ank`hIZc&Xp@PS{KKD7#~I7_jRjyRSkD<-`#nsIBlpQTX_*XMQ~^O z=oMGtTWZprA)Rq%v&g%;zxw*l)^f7c>WY2~PKd?l%S_OE%^H6-@lnOj=X~3*Q~CXP zxPwG3Ye4hH_^N|sCG_Q0ahn)7o2ULbqxZ%}2?M_Ej zc7ai6K1SAyF)rJ=a=Q!E+~wg@YXpPZDkq_gqX4@A<_*c zZwB4ghFh;7UpTp-?Yy?0PXRJzef`){XZVI8zuEH>8;*>FN#Zu$<8vK(gsn1z=?<1b z_c_5v1aIHm%PlXJuy6LaCYF+_?T1)K=MnwMt9C6$JAZwiZ1*!KKc?sS*IFN;-m6h- zeg5`y{^w&^sTwP@Q{#g&WsK&fCi1x)Kk6g?lJnvzDh}Ixee02gK05nzILi^O8#XCb zLr3On&djgOJC-w_winAN^vX{gc-cTTKu}8VtW2O>+ICqQEh@3gjjz^O zbAwrZ8I`>H{PmGT*F`25KjBwf-QN@rLEF9DB1f!i)RuO4`ZJ`Tba*hzjcrFp6n@Wr z{@nL_J0D7B%L`f|MV$5B>KnzxB@HgW9G*7UBA*;>fG^c8sjMDv6%LTt8P@Id-&wgL z8{i#5$ZqamaLR4$bTq!Ortiz!8grs#b~Wj}`ilKZdMQff9hp~~` z<4TkAjNnqQt7W{#G8vy&L~_&CGo{hg(YgNHd^1sNohEY2R;p&87{caoKmV$#$t5sf zyO3tm8tVt*6Xu)BL0cX1?6ENT6m=T-C9wGDi7diDDzzId{$ykU8T>X z{CwKALo*IRCkskum@iOBPxoJ^-ZJTPJT*J31gVwfuiZ3{WW)M9-=LrSOACzf=L7&^ zNM1ljFlo*oO`b!kY$Ck*>5<>+qpQ|=d`0#1cL2%krbx3!)JU?nboy2%f4}r~a4sE7{LFir%dsxpmLCq;h?Dml~x#vdR))S5T{PqQ;=v z-)O^jt8Q#?TfTDcC|zn3f>}b2Xt}sHHox?u8J%0bc4=dS8ZPVxS)Vr-recp{E=N@eU4SJBRk)p6=ZzAoA#MCL`sc< z^0cZx1?AhgYpN5e@r6HZVvlA9t7>*0ec-f-6(F|6kBYyU`Z46xju@okp2Q6#FRaw~ zmfy8WVqhg({H{OQ_`v_?3%7Wpak!_#>XOsssUN#2m+%b}znzu2YX3(|Z79||deon* zNBa{=+#L_{5rX7J?D19;SfTb5W0Y!u6!D!!Ux>m32_J?`ld?JcmGNqmiw3;vbEOq) zHPS;nru}Iw=}WrQVZACRx27mu;w0|14hjBPJ%Tze!$8 zX5A&WBrk^dYA+#2S0EIXq|GZdAzH9%=T9 zUus=#eg-3!I%WCVL~W0nW$oMmUq29UT#xw0+%c?tpIc4bKn0vEqBi_{t=er!-@2me zo{Y2Lb?3#|#2VGDw2^3+nvs`%`eEQHU>Du7)fg9uT1w{Y#_g9#rqyjO_SPy(R$mb}D&vlrl&x=m zQB1#FqrLtjZPFQz-l6zc?aOhH_CB%73_)1DIR8v-VO56TnAqg>Rl+lbtrD5os;9t_ zON9Z?`-;Z6WjU1MswFs*rmn)@=i0zj4HjivZzpRP;f=dzg{aFb)4s3PT~CY=Es8Jn z4(S`t_uNPycTp7%a7YhyueIrW<@8)#yKLph*ud^5SFiNm*3{Juv+^lV&IW1})bK&m z*5x^D5_syE&#p{o@Orh8gJ+!$swXUGYNr??R~bUs>@Pd1p)4_X#=z-u^ybDXebZjm zHR)8rRraql?~X&f2A&)rJ8|9r?L3FQ>UVB@Wy0#lmr16bPzhQ( zKzc1V)zq{Rx(##l{rvq@vVZ|}OcOFUi41XUeXn1FMTF|_1m4@ykeIyC^LkF$_5-JV zq>4>YSl8DTampTdPrZg|4N3^=9}Cdq%72C{Jr=H{hs3tHi%a4|ObSnUsoKOSLi6}M zBPG&#fF2TbPwO-Vm>zu7R}jTVcO<{H;eFXlpow5mB#Bgjr1^i)ox~`OCV2nLE#Afc zA{q2#Bo^|lLD9&Nbx+ImWE?0DUKc+xhp096EDD6=r!pEcc!)j%nymd#@zpl(L8Nx= z=feJ&o7gO=K3#fcc_3y_Q~mBg^4yyUW9bk+Q4D4fbEy828rHEQut518Vje;J8XCL` zN~b~T8K@*$!LYOgP@J41m#1J>%*h-7?3q{d5fjsQJL8wM$GcNK2+RqO&^`!^s)^I? zd#H^OGE(|^Lj*2}kUJy`QNi$~Tjf|Cqj4ZS(5Kf1P=&<5N$SnzZc`wT&{uLceqe$p zXTv@>ZWh919M-1A$6}=z5_Ah!)ih8;-^Xi1{aM4@Z1>DV?W!wZ5= zrU)W#fvevi6TFWlGwYjK(Kg%aHjOWmyX`r{WQEHYCcUbi5?PNp4dzcS>?GOb=92hv zK@PgF^Oa!ieMX`8r2PcRbm>FUWLh&BKoGF+-C~dHTikq!e@n?{%5XI2Vm(HDH1wjm zYoFQ?Alm1jwtaN`E|nlHhGrYpOy$pKO3UU+s}(afzmjXj)v+BU-^NK z<;-;9b>XSxV0J7fGDSE<-8#Vfks!B@*FXcOPJ2{7VEx=~ft{)7wMS;b!tSqfzCv%v zs+^_wdJB%CEjWt#>KtI;n9Fvc;#C0Q1e9pmB|E}>(^<@^Ek83+jFIa|&K=M=$}c<5Q01_SSRhZI;i!b z%2YV}E=@Yru1PbLFKy9|#jXg~=kFr=69U-OdgS1e6AAN!6Be@*uM={W^;o5H{ zMYQ&h&o8}cE&=Twe>nlQ(C~>xJ0^}LW}z2z4;g9HJ&=}96AD*co5Si2RirrGwcdaJ zHWOti;1XXV@#1Yn%{ZCb@Pjis#hSfBtZbN1Xx(9>{4Zel<^hqNq8t_Ax5VZp&MN0* zw|5_{L5C*4vf!~kY!Gy963p;cPmA?<$dQ-rBp&F^WC7cC;P+$UF*leE%$grVhxCRh zWq;8ffWg`c?j-h#B&C9BCPdd~e`Q+p)I?3}3SV(w0R$x^-=}taygdKc0ErP0CW@-3 zfMyVZYs1;PNHcIG@js%m{z`YhDV9*gj5&AB(6XfG`O1Y+^M=AXUYatW=w0*E5+Dsq1M%i>3vbC{mIBvl=#%>U zDKf|@KA#Z&@sRwO%aP{I4P^@z!ocxBcu@v4Qt0p>5DQ0&^D)~?pFiuM=x}1RIZu^v z(0taS%V;cH9pv)}mW(*|x8==g2md&iL#6u-ZA_r4edSEUhwwb$x%chzJ&ZooV?co{ z5Sne#9ftn15&n2C?yqT-YK?LIfu_=AqF>pte?+N&uFIM~I-%%e-tih_A?(2|lNm*j zLLAy=mz$u@=f0%hx)*(7jsT;!%gQsVVu-}2&Uh^7Ngni#@p{FPH?~kA zlwb7cZV983OGEZl9jyRBVh@OPdiIHH#nz_+;Ve*_9bLocpr610`CsMr)-v+i;e=@% zLY4TljdUo$g#BaEuKR^zX-#13uH9q(@a{L_DHeYw;S9nN!&!qSvm!Hk0`}YI|B)@8)QV%+Gg4X0fdep=MkB=3&Y zl?Cp-3uxiYlav4_Mx{&NZ08ey)%(CAVOvN}uf=?;t^*U8_y)n&glc!kJrX5|-#TZJ zZxdU8?ig;v*2Mxxr_*=nh;ca-cQy>iuvJ{bctTlIT*hr)01L2x1V^|A&LcagC=6&E zBJH%L+m6l*i-4FCoCGji~ET9NsiP4=e`Bnq{Br zq5&aWQ~!@(LpOe1!8#!$o~Lz~b+ETjfW1u;e4jiQ3G{uhaL~x&PM*p^bZ}g=P+T(* zCG+V}?1`Kz#==M%4V4bm)T(@FTua~soM+`!r6AWKNE&mQK3b8*jzgcuF45@*cY`?% zPOC^s=e$;I1HOx~lIo>}i znNDeM{~9{c{(=B4=}T+0^4^_$z7AeeEN>_C)RkSRSnDllVXwyUPD+9(ItReM?tRKy)NDNpT~$-vrs^aR-HLlI+bkFR25fG7{Ws z!q%OH4x~CZ1d)H`R_rnT>?vBtQgw?tR1zojrDA&9R%_2m_=D}X ze-S(m@aRv7x4>0QZ6fhSeE+!vwc=#Meet`W@{IgXtHc@OA^&mFVrJRJ3|$b6tEsd* z_PAi!Dbo1RfegkhW*wBL@%8 z9mMP1nm0bL?KKc+%B(u{Rgx|SeaO0uoqKW;DTHo$d(42)P`0k+Z>nl{VEvS=k?4jW z_Hw{K`(&nc`MD$?GPk53pX4wndFZQkk`%q}>Ay|SZ_)qPadPyXT>JBnj~TF#byG}+ zNj&{{Fu#lI*H3SOL2cfXBR7xNpzZlX7!*lhA8>P%u53=SE@~Ir8=oa%x%iQ458;t| zZb$p!Mfg|f3brnkF{MV_2p_QT>?E7LYMTa-vzMTBN}*RL8~AKvIHAioUcdz%NuvS> zkEODhT%JoS+#1@EBg11PO z(&^>_(*~uppFf_oZiBB$e#idfdk3Y=*$842(tX_z9(jx^sZd?f@_n9H^u5L21_Hs} z{@E3q`jkeQAv^d)zVAb14_vql@PYP!sxb8_wGc03lT6yPc1}K`StD-sdk43ocj3wx zfY5P8D`s0jZ=MQ-G8^_`lgam-+#isd!%ch?0~h+1z#~&;6naH1#_81Iqt4qalI|6S(vXx&^Lm2|@5M>u1s zHR`wkm*BzKQRZvR8SqzTt(jf-(L#y+YF~TI1qy((1Jc=@;2I(hx;Ah(5KQ4(Mx&48 zYhmpLG7t3gQO6)2xJD)MdPN|c3+x4z3^m16enUs=n$P_F4y|T)@}i;3P-(h4fEzRQ z=>vt1QPbwP%-85AisOwfc z(RP&!;=x$|bH@zWagf+W^zXQbR*6`hP+jy*1ku&vi#A9_^Y<8M>kp&8#tAXF+mxI4 zgO&RM1mSVSjAWnybfu4>9%Rlp)<(K|)T9-&dJHYsuYf@&AOM683i4BtlSzWLuA`067xACi^_Ubn;M2!EvgP)**GVIG&OmnHmZG zsaQ5=bnr^7Ya#I;Am8aBX62F&k%bWIf_2}xL&2#4$4xW$tXM(t{vIx<+vgZA0>8Cy zj;*bkL#S7-m0ZG{HWRTEkVrq27PKHw&_bad#dAlVBC9+q#f)6)0I0cx6hQtWEkDUX zHjtT=j0BqEN)V?~cqP&S?g+$EsjC8=B!^U1;rWk4M1~`^MMv*P$Z2zdbHH4aQo;|= zyH}8!9!gxfqU)yHFlxG7L!-Q#KZ-7XF%6dp0F!;BJ1Fz(4nDg}Rq^u16J49?fAc(Q zANGGTmpOckY^u4i%Ir750Z)MwKw4E*8U|kQ0My-n0vU$8=EsL2 zm?jJ-BMw#bQ#=YsO0xDj2xAGDFS@)dFn(%2ix{h%d~yBp`@Fv^ig~4(qTqe5yjW?= z)s2To7`@H+n^O`@G-0gHkSH~H%>>Mg>Kk(f%Q7@Q_PPv1nH*)Mnkcz5gTKBZlyyj{O(Jdgr()^%9VDUmB_+HNI5Q$pI@*+9{ZBn!D(@ z*ann$+2ibp5E1nG;l>|fZeW1uz($H~&`-V}QGEYjvl+}omS}$D0vJ=;UsCl3OdgL# z@fxf$`gnAH^JTQ!$Z{(t>i5(DRk#n6#W#Hp`{NKsqwBDj`0VqRqafR7aSE??xfk48# zP`T_U`mPPY`hX%oyodb@hTf<3P=q)^_bzVs5+sM^K1S%YL3+r=U}2$mhivLSjM_6o z7*ME#BMRonpp8|fNB5>Fe^*-+_>uaOqkZHRy?!YgWjgM20SDqTaI2F{T@gYMr$H?|4G=-z4}8o?4V z6Vz>yKLMTwu)k)>SM?@SIy@Q$j~8g(>HdVoSONpXwomk8$46W)cB6cfFX~=JpIrJL zBw}Dl+oUouV!i+VuTGufYWOroqy7{e^$-~<(ot`ws0c?-OHC?1o{0*+nO?O1?`u%)e@y!Q zUx=Q_oJLq~^vCqSDa>z)k@btC4)0l}FkPo$xB(4Mgoi@6?f{)MiA=O_{wJbhyl4gN zF5Wep=_t)IG!dW&&UXLD4`e)zwP{FK4wcH3wg`^^8{@#TSsky@ox-F3rm#qVi|9-g zIBaT^&=v~rm!IA{f6B}uf_)jr)8>8XZ(GWUe=&u_i@~q&xZ~OHoA|y57*pU^{+KGX z1^3rYQ@eleKAy=k>VB$gTmfjFV5k44?7TYOf9IT&r60}TA^FBInNJAGDKMdXVBwDU z^Q3noXO~h$kaKZ7<^q?&#@Jtau_fJ)kv;VJG+&U)5|QHdNMOzmoOQhyky=d5e82(| z2yB(K^?qQ&@4#6vLzpUCROY^r2YjIwYsPv^%;9rPd-u5fQ??33GX=QwNFCRX`o{lU zJ>2w3PNxUq99co~)QKU?KbB$#$B*)LMMTA68WP>+jJOD{D-G1?e;wL%Uy2RnZ?gw| zVy*=5pWms`Jc8ZZR;#oPJ5D77Uc%3E!AOk*)n&$#LCZe*o~B;TQoGQfaW(V|v|x>O zNF&CB)$da#O>z}B#(&TYNxUx5)SDcGX$RiKQQ#C!agjmyR2OQ74q5Q8uV;?oeimhe zmLRfI`Cb0@&o2jqw@AaUoY652xg`xBYCS_*kg-B9RP(99EYW^AV2$`0_W+-wJ11%a z9w9MT0Khm%#2;+V*2y}eNt+4Qol(cu`UYO%KEG#l<=f&ekCUiPd2gtBS*8OpD14_$?V*BSLByvg zM?M*Lm+DX=QK0)Fst5uBRp#u_sT^C0|N0zKtj1S9qCgM0Xqg~gc=@RU4Pxs3V2vZY zSou8d3XD4aW1f+jSxj##y8cASUKvD#_Q#br(rA>O(L1*CK|~&^$5aE=BqEjDjw<&fZp`=K?Dg9oKY`jMx_W|V`Ne9SX&TtG4u333NBP+@dUf9#3=NY83(pu;3^ zM<+eHFi|x9ciI+Y{n^$5%wS&1mT3C2GVz&DC&u?FT;h!^>%*r|^Gs*3V$pXyzk;Hb z)Hr?ODaT4H!>drEv$+-$%kXJg5&HzpFPgaaF*!|KJf2;p4e^Vks z^H9kI*~|l1HM)8?5IzHwju^iPDkA&YEi(m<#$Mtkor8is6TDfl5}d02Y6~qA`^^-P zXK5Tl@==i)jKAxG9dweB1>m%3>*pr)!LC-I%kqxLGd-0oowC!V08A{>ezN;JK?wr5 z+{lIpsYWm09_ckHdf6;6nWhm4DLe-ir@8mhPe{|ShyuuypDKQjKn4?Z559~lwL28? ziuJYYpAR~XmuUu1iP)%pjMg}A;BfvBcjXo5vFsGjl-0?9YCPQw@{zirVSJ!(q9k-R zk4sq?uL)x6@ec`_k-WcaQlw*%N&-iXvpwap@O$+4ksPJd7Tqzpda%v_SR{W${U|s! zmajhZo=Grv%7l(j2SQ7GFwfDR>7bQIhX9EX3UG7QCawTDh3a zk9Y~pKPgEMd;F_RF_FhbMnC;tpT>b%;uF9e-#m2Cm4x1TK&08xxOewRxt{b;q$cJ+ znUGj!Wy0029NqWuA|rp{cS}J_YX2FJ7t=D}wxsb8W+X?Vr`e0D*4O6`ZF&`e_r>;R z3IKG{H8@H<>eR_YDSj*#mYyjchGL*RW+3ee@j(2ilw$beN3rfe>p)?^vYzesc-kkwGxB-S7WXuKj1D(LS`EEv(iUoZzlnfWOgFjUjfL}v z*->#=7|V2=K8?MfuzR0-eP}qaEcU=C)Tmp4Qn;NpbMLZFx?m_W2(4=GzHyE7(B+pc zvPm*Sd~aA-5f<;-DKbPw)FZ@z8v}J}lVA$NNoUCLt=pT>eH3X#aam_0QB(rYn zQ6!I9!TR+Td32`y+ihU$APNOX1Tb6V%sTdZGbGkOZLZRZeDkANdp&+520iBN_zYId zSBQfJGbTMo5h;qw!Rf~oeVoJCO_9@;FVZpKM3#aRdE-49iF!|e)H)^GE2E2EUZ>N? zp2^7JXJS9EJLoh)r_)G%X4mq^qH3G;HYIj-f%RrMcjb39;o+boKr(+>$HE&xXT0OS zIQghOU-Jk2cd?I!NKEqYx?UV6gVgVKf8PA%XDtkVa{6qbBL8t!234D;wMTngltDN@ z>B$1I-A3qZoObDd9WxkO?yA={J`HZke#;>C>QUxIT0A@)kXNdP-%f9f+}8Ndh7X2! z|FgLDwFZM&y~LZKKksBfex`)x^=W=IAyT-hFFh!L8tZ6qIjC4uo#* z3IY^hga+cuJ`rGoCVV;!EjrQhYUUeL&%mAr^_P1U-KEOCv1KWrnK-w@q|5nqV;t_7 zs@OakaRsr_5n!V2&YU43NMh?BeTe$FW6bExV!&efGt3z*8o1Lz1-wl90Gp6~q=wwq zKK{nvI?~skW8)Vu1_3}iZ~=VLDX&O4FWX4IxV*v9Q+8<3nhAA`fZK7r@xf?`1>ax4`$pQ zmv2xa#9wmOAd6kx(G2Atdx6RCHuwY32Jaztk)cEVrsf;5;6Cm<()ieZ<@DaL_6)F= z{!=k__vOg6{wZc&nGPi1qC_3S7O{vwjirO8z+_cxV==GX80s zPz1(k{46zo1>R+&1%0Jq6)G?xaIkX{e~nW-9Vv=kImZTu_SFnBBV23g@c_(m14kl_ z5uGyL$wYMgf~Va>qu3&hog8QO^ivSzKUlW=|0M&N;sa0d%hH#;_FPCq_S#9=)KgjP znMA{x?%Tj-#`VA^xh?PanBS8o?<|H3$+d|d1Oti1_#PKdGjjT-!%65$qf(>Rz?f@- zJe8y>YQfb@6_ip%}-?@o<0_e;3byPVjo6!p6S(~756Q~*mgaW;@9hYqi8Y_EeDPrBc|+~idye7JFEt>$(- zvn8xO2ZZhZUW6Y{B;CgciVrRVa~TzT@B{N4QYqRoqFct);QIndF!!xpvQ;n-S7~n# z5btp6OY1_z#!L5D+z%d~p(+Gmvu-mDX{&)Szp`mdjO!^&MefNieO_naX0m}vKIzP) z2NXov1)V1?zQc@4r!|8(r}$9Dw};Lp#etFz$+Qm?{c5`Fy^QRXrbXLB?%h4cpA)C8 zPn9b0pQ|}%$&H0okNg5OXyIIXELg*iAI574u&V@I)6)Vv0*jaC?gbU`mx~XCI!7mK zThhWeD4WQqkLN?a@C3c6Zv}W}5GnwY&@PE2G!qvS3`7r~i_qkP5ngt!ibf?}kS-y4 ztlIH`uSne+holi-6CnJ&nWqdn1)mgEGCHJ&2ptEF<3nAz+q?_cfDccDGJQ0%$7zy! z{QJ?s$l2hN#gI6*KEuFyx-7ptZ_K;mxI8U*6v4ZE5GPTBh=3C0Reg=MMEGQpCH!%- zU8G3U!+hiMACSH<3?~gfC*np){S(mmI*`-Heq*vqEK0s{?r%LSDfU4AwNz03-{%P3 z`)ZdQLgMbCnPt9+(6L&99Zo8&HGO8Ca9e&&1A2pcJRSOWg~r!b_0h_Gia({0DYi}2 z6zwy=ayj}XNt8L@2u5GUJ{(8m-rgg%?o%)LT*LnJTUeGC4`DI^GT%0M+(!wO}5d?^g{9(VSCu zDmaWI5Bk_%v{wCUb02WM3NBX=%Stg+SWYk7T- z*=T55p3W->YP8Q<~BBOeiYDsqvnQj^@Ev zxE~Mn8Vq0Iss|6c3P?$@Lv_3}PX@^<3FieEDs@1pVtE-K(kGO23xjeqyMy|~W+{yt zp{3tzSAX#P@GDa0%9X#y3Q;lU*Ja45f6_Sx!I{Sj7Ly1ztp)}+M?n_EV_gZs_R5&C zVUYw=iy`Vi6rA7BNB!z5Kd*FLE2a$;Y2q6^)ZQA5WgaX{jJ-zufakeX+6X}~h!RshaC3D@3&HAUCW z!1JQ>?}|Wzxpcu3klMiZ->-afSp?B5X*XyxN?(b*?81ZGI#kMUiOn-Lr@;$KXOWs> zZB&Db8Z!!qTQsz$HTq>E6T!clQpL^#JmD;$@T|vemy)I9&)>@mIx;RX8GrDcrTXiU z2=JOWdbCEVuKbRPaFXf^Ve4T3WCRV)DaqqH*_`=xJoIuj&@WlYLh7vI`I7 zcWUwb>4Rb)6HWO;5TsFoWhf07oeycWVZXFAHrA(N;FR#GXuJc0y!=JzfBehe4+$MP zi;!`s`~(Dv6V*tQ$sO6@S<{L_(#w{}{6gEA8L4F{WIj zoI9W=jeXbgLIHFp50xBi(aMCIA;67Co^qu8Zy=!b)`lCSY=OTF)RK`B8qQ7U>j6JaN zzM!n=?7Pm|>&eg*0ihmHWCm3rIjaiwHh+)&4gYm3qNCeJ7uw$M;wQ0KdrJ}5O3vfd zceX8#$!7Hu@2D;BSeN(_{YO?8SaPeJ+QfJKgR5ZDS8Q^t{I-;D%|$$GqgYa|f@N1- z-*nkSy##lw`Vd^)ig8s~)fAOmzA^#3D@T*$sy4=UUqpwpO8W)4`RdKSaBa*!K`j~Z zLnBnH`J$SzkK__B_}~A*a+&%M%LQOxS_-fOa^Jaa|6pHajLU_qw--4|w-+S+-4=U0 z+{;_nTB9P~Us(FQFg-I9`eI{nXT@~WxAH#w}g6MuRB_19mn9*)8ftjrvQ9Good1>J3-R!$!7f@Y4Ef)-{D4py#$X68<= z?!s1{ww6|o7FPdXUVHnpu!x9=sO0~R1eECaVzeQ8mN=x&&WIT!yWQTibyqmTr_jZ> z?@=x-W>6q6pPla8NW^8kk2=B@?LvzKP8#>UvXwubFOGLu>o)SFY5)1Mr?XByc)s0#$gjw#MSS2^HUZX+5FRYw$Z?79cXe4nmv7X^moR*{ z;|$h*{qz)U;ccb;Byge!nzh5ZML6@ud;XVaK zT}Z;=ww8jR=LRLMYvTby$~uj2Lo`#qP3>(c%3JZr6)xo75{+4Iz#x)&{tjBoVX zuCTGijn28(-S!yW$?ol`J8~n{q=+M03e zYo~uzHS8sg26T7x$4w!p*`}2)m^PGt)oqL~d>!9VIy*k_9j{r^DR3EGE%QrdMX1E+ z_?1<1%&63BIgh*9*`>Kh)!3Gu#~rK|xzyEzaX_9Wb7$X?LK0?53U?%ACUei1aOXVi zMsm+f)zJv6iB>%8wF`NokS=$)KCgn803wn)S1o>hOk|n<*0M}@@4Palmp3ZaD5+v$ z_VxLm9$a(0+gwP^s;Fi-eDcfBUVQ%UuA++l^Ht3`x83remStL~N^wc1#ub%_T$&d^yc>W;X`%#6RsV&oal$=5#P<}tX8`jbq()LM;#Yv)FOl3 z1KwLtlnZTsz*?7;n%_0rQ11OfM8#|0IbG@V%5Br*XFy85tYR0Pb1K)T=v0T)a|hBz zW6sZU!LS1(+bv~tp(+gA;!ZxUVHZ35mfV@$;qzbX+rLa5Pr3L6>wPMuS}Uw)>g1qB zu5rz@#}WH^c}N!l*VK1a-Q8!V*E;G``e$or$mW{P2WDpY`Hp{tj%>c1m82=pHlF!X zwEZDzVS{L9g>L`*HXQwMe`Md)v&Y`EhZQuZ?pw{C*VaYva(g@1yr5X7+g{$v5^~tL zOm?r)tk*EMoFB##?|!0ZJkwa@$T043n3FgySY&Cnay+wnd~9ev^T2iS(H0nqayB#k zU1Gc?@0JqGblBp9vX0d!{)+6Ww)0NXnW1&tIxGC(t^R=2leUu&UBO!mUo4dxX57!q z$!UL#NjAwZ^915wag^g#0B}+8t4k*KOG^zL(U(hEE0|O+x){k_$EFDK1Xi z;SCcMbCUbp(bB&JJEsw@Z2W99w+Bxwby$+7XM>`oU`6-TkbFz8^N#Yn49W7%hc?W| zXNJD><9JiM?S$GyqI0bJ|uzA+hp4zA=uIqH1 z%-K6vooHO}+&_0S$2BgpDn9fqABl-SAw>OFR)2wm3FNCY@mGrt z^rEHl?(c@kI+lo#JLsiC1rQv7PPQwM?vqwB>#L`cw zHaahk59S(9&bxQA7Y5cTF1~;9JQ?O9V?W!L^*r5nOt^Ib<4g*uL42p1-9M}_b?n%F zKQ&uBRd$iJ7c^|{m}e<_m>~O6h9kXm;H#T;1{rndsb6R*__s89 z4)@iFbV+Em*u#Hr{qXYE4{lOKM^mR7Cd$$!;3nWCSbbdl_tq~jZ*9VDGM8D&3ziLs z{j+Sk0eVzg`+@pSDw01}9r?zVYxmc<riK zPbiqvnhs0aR;E_93dUP(V$y`@R>FgVon;>aqYJdW#B;1r^KvYxNzd0x?W!X!(y`NcDV zjNc`k;NiyCx_b&1HkR z48L|QV|{nBhoFL*JI1&@V@W~<(=HXi+KawFzO$Mj5c1FylN9DM3@L0UfAM~>zv^_O6U7o@^&M`ki7*Lchbgq}l6 zw^r?CVi(@o+ia=ao9Ul+*LYTD?&%7Pb+T1)j9H&4eM%uB;<)@tdEzM)ZDyB3OPB6h zU8)-9V}A_Tk40tm#$eNHL9ax@0@`DUzT2w5#g8NlUtnRGZ!1Tc$+~t>dsL&dt=vEL zBkz1`^~p}|BX@)&@0SK%8UA69rn-~9pAAV(bu}INQ9cJR7iKiKwy|N{t~=wy8#Sc_U>*vd*7>Mz`{_4Iw7he-43!{;trpi>Rn(x0 z)K3*1KGTr39J}~B7T*^*$2K!?QIU6Jn&nMw9exd`wj`aQ_R7(IVt_1z{H8dBz2T0e zQ%nPsgX6jT3u-tMgRE5|>{OIo@>>%J8--2@M0sZzUC?2?Z05^C!|)rw%_NV#8V5dD zAQE-Ab17jqMR@1+1RE*G4BLSV-BiK%letn=wlxoz3Ss4q)eTFFKqIxHPgA3(2d-lk zga;}jUgIfeKfiR^OZb)PjtGPj<^7aCv8oIUEi$$6{2LS)DC=YpEa)QMPhwB`&sexbINO5XhbZ1A== zC-uQS=e4t+BfTw4MLNY>?iK2}MJGOCILM+TTf3X!VuB3bF#AIx0kaB02~Wi}Ik?3e z6`||jw`*_tzfC(ro%c{LoZYfoSdWEzs=eyti8iX494+1XmiTgNg9sBxq*z15Y1?Xy z^|-YtwFdXrh6MSo=hcCDLWJ}uRv%jD$kZ>UAZ%Sa=fIlX_m&U4ska2{YoRSdal9MO zn_&|=8~S6Z#OE$eECm{B-C`$PRt;LZ&UFPFGkdS_zlfI2+FKkWS~m)d*2-MwWQ(f{ z%lhWJCOX|q9G5+Wm^NT*bD=ghP8F)><(~{ICJz}}VBGv}1muTJtj9mC9B~^}=6oAA z_dkx$`5znwIA0u;YdMBBRGCU974oh)IDLGlVrDF)S(0BfT7+b>y>;=%gItgxXaqOz z_>17@3FtUgv11*1NJPe3JIiL@wMA3y^*QqULJ~Ec58yptzb7kPJgqK7Efzx=bM36P z5xgh(jvTf4{gkSFIGU>NXA10MqZf_pqA>%g#rNMTjWAl4clycdOJ0L6^C*vGTq9Eb zCV3;DoS;~flREXGZvSrT8-p0W#BF6hRUN5YOe@lNY^Iyk)f+L*u1O)6oF}qSp9K?H z8zsu<$R%!}e^ONa@GAWdzf>Z5RcOpfS{AZVK}bDOrt_^cjQ85?3$G;OR}kc`_QMt8 z6Xp*kYvm}fDj$6@f07KnHg}x2(tgOf+&r_9I=t|?5?M%8pTK;YjhPyJv)jG84_ z8kA3_2+hpYlQMpA?Z@rgD&pJgDy{h1tL5rAvgfjsYz|uG8W0 z_*>6<)_}$7hwigzp7W5&%B&+O9Bzw&tW6P8MmdJ0-- zVswet?@dqs^5HF~ufmWZ^{f&TI4P{W>psBotnx`#Z<>2zg;0s0i9_s^DtQKl&9CuI zpUde2I6n!JURSJmxZHB5ZzuAL}e?MqP%P_z*_WEMaNjCqSa-l_f!&Kk6S# zDD#0xkH%YqDN$91*!(4{$DNbAG|+=4ysA*b1KH@}etEN2#{6jMfhnq0)M8ok;%STm zk|Y_qJNy{BG!-T353q|H6=+zA2~qN%EFe&bZWL!aB#$#DH9dk6G=gV67a-B~%0ffU zA03LuBDz_O?a-qkYPS$IA&)`|9`V!UEov-DDf(2w5nF=r{TIbk_<{JdxmcgFx;Wjs-N*F#4lrM?+d4Lwr#Jd>(tll?f5u&eOdasEIOw?W<9Sd-7bd`j zj2iTwQ%qW}>HmtUVQJDnbFZ88=OZf-bq|&sB7ste7g=!n~LH1UBi7Bij#$l8qAn}eyfXwi-lBV+=6F7(;dOhW?HvN z_=N@yuBVjuBR5-W=mFh%Hf~J6j#e*G*-D;p-P|TmhM);;!+O%>+A^=Z*hO-(1f%=& zaLLs;FcxEHe3wS*O}b|x zYnKsIucp5HPH<$qs!!TEM_@=S6zM?wu+@T`*#zF&RXeOU+kK5?!6Nn?mBZx?=Kh8 z8Sx#6?rzc=)Wy&Q&ASUf$pfc-K*{ZK?9}gL)e3^xDfG@%(q(VKy!E_ZT7^T@l z4+d*mp5L4N)Rdy3wHb>Z{zHPAfincw;!SZnYZExd?=ic@&9Q}+q*UfC*^@W(W(Q0C z!-U-Oo50MVufFf1rk)sT%s}(VycHiyDe8qkg1eaVW&5O zN^|$M*BtQyAJqf>v=2q`wb3T zQf5k_#oZ`hiqf$_PVDmcGbaa5!T8g+HSY7@G}wt@UwlmTbX~-NxpXWd3j4Yt_Fes2 zZa9WvfFJaEh*3Sw^Sf3{-jVy2yq^l^Q>#Cr((#AwM$!it@PL=Ba^#kifT5QB zrkJJylm+I=iywHaD5W;(Gef8SgWb7?^z%Y16`_PFjanWguw|lBT4dBXY`p75mDiV~vRLNP+{}1Sq+uaFWNSX+M6QQg zF6M(!!dCGu(Zr&K5e!IJ3hgCkd)B>MixX3~uELi$4TxT8d3!%wZ%jMaH*l}g|3x!`jU)d zuqC!%gQ}RUyhi$Y*!a1PMCgbQs=W;T2jO6`6B3q7&xJ^!O)LJIN0b9&80xv`!o{>o zEUH?@ht}ZOey<=j0ZAgkMQZ13;k7l8P;lop{bJeES5QeWa3b12GMb1qEnD9ikV6Sq zvO+*VRVS1srY9C3Ksc19VmF-v8!>Nd5Q`sA={v~2#zioRBR!S)O*#G%e;+vbfy-5% z4w`^fE}PtZ6k?R0i4ZPU3Q3e0hLTg2WHerWemtlPJ~t$%YE)+6SiR1LiV(YJ0iku$ zS_wv6JqXv1cmjp*pZLiZ^0BevHufFa2!Xf2~{F(^9budIU zgtHf!Py4g^{pN-&^r{*$HX(D0I{eC8A`!6UmwkzvctA2k5|%A(f+VITqwQoPBXyH+ zb`9lS$7CPJN;RExE-ZPTE=*Y+upEZ@X^GIDIuEg`s916y6>ZtV>k^q`lTco$G|U+6 zhE1N?44+q;ZW!_Ub^anQE*klP&09TJ)BmZgEk8WC&}-8w?xLp1Dix)sNKKZvJ9MM= z)3^DSjmGZtVEe;1%fX&R>zMVVQCrs;RQ4^8%oFEhkE-2`Gd(S*Ugi3#vw;t0c8;{! zXP$hLpU&=42qnHI@Hr78Pj)y-4tgM!7W4eAkkcefSa8t0pI@LF4Yp-%lGpvtENgj; zJL@08juXg~YK-$5tF*AKhN_PyFFZ;;YNq>=w94X-V>GP9NY`;xcfy~@1b(U37*S2G zaWLY@)|TUd6+O6b5iyX;?O%C*vT-)E{9&ZFHxs zEzOmT^^zv!bwT)W@64naHIB3-2+xGc4?kTxcKy9)JJi9X0xhMfF_|cera{b;jQcC9q|OnveKkyV5|id@h&Y0~6J~qG z*3mRWs(cElCZq9DUleCy?@tdv@3+D3Pk{F|-8-``Zi^)?JM6YK&n)&sJSlfzHK8R< z4F+%}dI?wGB{MGJ%Hbvn!Ka|;5r8IRstg-^H6CXAe?mdW2+u&Qni-~rm&ue%I@vyQ zPzxaTC0H$lO8~IK&qG{8_}iV>^O7(W3tZC>(a?q*GRbT{DHgB>roO9-$-?^xN~{EJ zwb4OEw2b4EH%5(A$QyF_aH&J_;I_8 z#pE7Z>cfv;J9%hLPW*}85fEP#4O$1{FaUcpfj>fwK5>H}_&2n+Z&GFS+%f2Ox=A_7 z6FR_xk8CpmxM%^?vza3F5*M=(k$BI$&v^jm_0gv*!Jdd-In05PF+5grWHo*uqLv*_ z!^#igy^5JmH_h(TWQDy2V-eobcpDpU0(JQPdVKwvKY5R9*ee;HC%mzrP<& zayEl|)NC;gX6yj07Hf#+o(`*4Ye^|zdLP?B3Et9{Q_lEm5~%%ThehRtB+cbjnb<)Z z7VTt)8ZY>nO_SPr7gpG6<{*4Gk&5Ry0VIr?L#lDonf(YJ| zxUzF9b9666@-BxR);k0Xq)UnM)zm*Wx+RHq!RF=|u6@VECPsq!!Od2w^i8MRTcP=W zg<=CSyj-0dp(_=GaL{qTjS4p?NcnjjV&?4fvk>kWG6|rzLOfKW2a(qnlY;JLSHh?9C&|AKYvy0Dv|C|{}=6D3UJ|ac)e-- zfdgyUzgnKg!OS(V$cUZT&5`wm2Ko{b;F~w(%QtLztM6af67zdTq6yY2ql-FhRLgw_ zQ#+uQX~ALUm@laYTu}>@p{_ce{Sw#)%=G;-vlLLoHHc>Sna7Ww8|q)t);<~ZVs2&R zbq%r!sbT9P2N<>L6>aO-)lB3t?PkRk&bO%4k6Aj3xx9fnFhy^QQ24y97|h>vGBm>{ z)&3-MEb{hTm&x-|Zg8%bp<3mr%z)};Y}lr(Kub&m@e?N-87s92{D`atG0exy;PKa> z9JoNg3PHmH_bV69w8-LC*xxLy1czPze#*KeTtXcRXMbrfE^Kj)10*AEH~|;K;k6=e)7H*JNKlRQQpG~Lg2WZE#spA<+m@7Zb?|FY z9+x3`>>K?m<~)+O14_K5dPlQ`tvF)fO}oIygcTM`(S$`}+h_SF?ee^$w0X!nD9uMmeO8EV`)CdcjBP#E{0=Oo8VHPZX^P8xR#KD32=as=w9#&n?8-o{?zT#GuLO zX5oSPu|=ZeRO?;w#_6V|Dcb)kzZ`qGgon{O{2OUa+^kR3F9W14osf`yA$l9SqR$NZHZeEeKM8xpJl zM@=utC>R(s%0pc7rtEJtx&s$z;i%pg&($oRCy{Sm^>SswJsg%2Gq>^^VY(AiVmG56 zX-K~KGq}gTt?@ZDD=Pr&Ho{@aS3bAb59B(3MD`sVqHhZnvO<7|8ULn?8K)HsH2#n6 z8Q~_3#J*JihAv!ASZW#HUlbkxPiz!=@7$yuh>;V{9b|F}G1VE$UTlBzS#8ViKR1hA z(aTL@UNY)04JCXZjmh`OoWek57qsT~2AU?GuJQoHg9F293ik(wgCF-%UYwy_VrDWs z0;W2|@!FIo@6kQ+E5;=fr4I2MFgOSmKr?H@6fIETUXY0jO>YhJzO=A;NwG@{6RRi4 zN>oL|nT#7@Qnnk$V=-a%2lizG>?`)ob06lyp$MfL?N8p*;OsMEjspXd3f;}tubO|k zNpBDG$RKB~>P5y-H3nVjJ2YRWc=G2RUeu~Dm76|?X?+Jz0ak=d7Ka#?ytaSM{Acw?DuC-f82O6g zlN@fC=m_kBDyR%B6M9AedLq3>T05?5P)q2Zglf^&2#ZKP_Z= zfQ_c|ky(#LBL?)Ae*s5MoyPO!;Tw=q9Y7zGjW=;L;2roY`niBG;y0avZ`1VF6rDx)ytmaA}S zLhi-Mx0ua~ituSs=FnlEG2-v|H>{+v`|=ifJfji}JBoY($RVR^BHhC9T?}-`W}zj+fOQ4m4SJa{kk{NozCH(SZ6yv197u?~WbGwG1 zH$leK0Wzj>iW^_lpjGS}r4kCJ-KsYpPME!H*3g0Js^|xJf%knn4dUL?fL2jqmPfzH z=x)9xoImQzt!^6|3^1t~U{dY8q`_slXJ>Q6JO)u3vT@zJz)H0{u;X<*h-ClZU(|}T zLgdA|PajiGl1!#aju-nhJLxg`3w=T zzyaI*k?8nlRlN4ZYTBU6)7c8(Cjrr?YZKnCYFHqGSsKKsy{8JkYA{ybQwbFTZs;)x zyqLEKFPTEK5@DO>t_c=`4&@lGpvcXTOCUr7Ava3;Tnf2PJF%>k=N^_p0e&c@OzeLo z;(zyM$1IJMQo0}9GGXc1BY}jX00Fmyl$_|$Is_Hy5G7LDRD&GET)>Z6fY6e;KCrIk1Sh;>CsnR#~|0?YNxUc3h+YG3hA;>ck@=}r2o%JLj zO!=Lpq4^aUO@{Dx1B&oh&sdH>am*IZt3TzBjiF(V%2o{8b_cc1P0P;o|4jyT-MLLN zfr^*vgpeA1CbnYO`A7On7SW$%VCf)_{6qR3?B!l1K_#Vp_b?1hTQVz-QcSOkkYfzP z^f~byYRg1or9`CL-zT7~%N+%=VZGlTiMW%QjT8Wm#zNF*v;E zui&XCsNk{h(C(v01QR$y^}+aa7zrs!0JZ_kfIX(>BcTpX#7>uk9K zZt-uLq8CP@SuEu5oBQHPpFj&Jub7SDgS<>-sW@6(@VP}xr7qIJ;OcR&mPnC^^*9+A zVH(^r%C}LtpJEJ!2Iy_30lUQgqN-cKPWR|$hm;auoi62u&vMMCBX1}mYJQtF_%4!> ziCQuyTB*0xOQk4sN8fO2!cI7{B4k=TWJ@8SSeI zpmH&v+Q*K9E(46;3+xHXPgCwHg|J{5BovaVAh|0+kYs>61CZ-U`*rhBHRi{fx7AjM z!SEGsxP?g+3lYdufc5*d36K~7>`mUt3XL6$ea%MDJ|@B4vpmiWqO?DdFY${RjW{t? zaI-3-`b(uG2zdIRn|sanDwDF78O!IoT4!y^>?-f`z%BX+0Gya^Tpa*76FSh^NxHhe zvJkW@NU2W-F(zfTUU@#5s_r{#@0cbH`j8lzmYnW;|0(?czGMozFr?U zGT|}<`VFh>#N9WeAX_Z=+ZE_w1|p?w-dEL&zb<|&N0}1yx*Dtbq(aoz4*=_btnR7k z7={HmOCYL9el8iebdu&NIGjJUS3Ez{KbIqc$giXo;muv$M>GX;?%$%J31fQfdpj4B z@YSzb)qs4dw#zE{{uTdS($Dl*8B<2;L2&Z~!qViqA#9=GiiStHYm*y3W#fO_B8*rJ zRkn!vM?M`uKB$5uhVS7L%-{gkn<09@MBwI(Llc`QKz)G!-;xYDu@bv^_VZD)4vrCs zZyvG`Dsr%EcTx`hQ>^LAhD{=nYVa$}3%mI(MU%o6rzHr8fj$*jX|WXax64BV(k|Wo z0+GbRi;14Qo} ztLlYGn~(n^RQ{`d#TUnD+Prxbajhx#&gc`_GFR5C3m`kD&CwH&yQFAEM_=Dfnt9HA zCHXWoVu~LrWfu3jsqu)ph^zm#+11<@L&M;qJSqazwYWLejaAO1Hvi;5L15nWIhuym zNJE$ZRIo~&G|vRab(Md8F(K6#vkbW$P*NAy+`)%=b^RZUM~6Cg4<4}rhB&!^r+|>- zZ!sA5Dq}Qvi*m^o7l4U26-Z*kEBZtg;L%waCdbziq5JCj3b#D+<^MR!`)4|A8({cR zX%3$l5e@-WYo#AJDX)G_YxtsM*A{Pa8_Jx0I9N z7V=gT)#7-KBla>8pcqK^Z{oR=MHmXwN*`}!Vi~pd^XusaMk8ZG#%lo}n)%JT`sa6` zO<=Tl(W_HVAh^@o_ub4$uSm2^FfRcP!DGQQ|7>P%;RL%3R~s`6^%V5XxN<8{I>68xeY#?DC#`{A#jR8ic2t7s}vc;-#C%oaMi& zk^nOAKSJTBBqEp$((9CB=>_PrU(^|6{wh!8lygr{avWd=BC>a}R~T!4fYn__$^)&m zTs`UEu)4H%?#BsYpzmX0O2e5*U9 zYNRMu#(5uH@Vl5P)~EZ`XyuMJnhRisMl6;l-5jq{jXl@B7e*A`C}>blaF%ldVIZ#L*N!d85jII#jM`jXd6*p1zV3#OIH;D=15h0s{DR^_`f%57DSsne@3L zu_ETpsY#4)jB(_vS5MK7%Nx-iT~#HVzhbeH1vzrJ1%~^Z>hw7XC#J?eFI?WF9aL6T zvXT5>?c^ZC-}6wc-kS60CLIUFQlpL$z@$*2d!9{W6d5_j6X*(BaY*&3sW>FW9fpr;p5@ zmQ2$B$UmU{8C)U}B%z4(C~3v<0oi&Q%+{}kIe#dEP~8%{`pFqvzqk%;{PQ0bR6PP}9Wpm8FyuBGL0(Zr%*Xa)3YJ}?X{vT{bA*jO4vQZl#u`~ITX5=W%Eob*_5%tMz5IUS<@m;dF&sd4oHc6@Rz z>!BV(5yJlc+f#F;D4nnT{~9b?g{T2@<#fcGdS8TiS1CRXm`~KCK9mEU0YTg!6r72& zdF5EPBF}T5ni<3G!D{Orhd$%3Pj|*d04yEyMF)%0=gW<&v868U$jM zfNhFXPFo<C|WI1XPW;C&0<9CWlk*4R@~-Q0875TqWM=S zZ}3`IZrWE;Q_To15TVS`jM{q;aRvB5xrsizi;q*TSgD{CVNv$^>i!2tl-b3^6UG1q z)P83H-qqK8rP>s#1|LxgvPNv5O%(ktE*)S9YpUxZccI043I2{#8ys#j*-fAtbF&yQ8K$>F^3K^@hX zp}J4~Rp}Bg(CwN2V|VluPyZht(WCY21*gvw(wr_c;?MRPqqX=FiSAvZBgRbf#B!FjY?uSg((+0j zZ9u<(R{iVlDjUHFPJ>*fCFw%xOCSsp0OOk9Gtxi~#D#hz(tqUvHdMD{{}Rrpxk&>$(5duf^&1K>uHx?U)TzWfPXtAY_-S5 z;&RztiRGXQ3XhxR<RC(Y{@m%wW##i;ua{NlZh-I-|tbg0__V$y*7Vu&NO_MomBl zcjpd#$6)1s1Sxng5u*itt7{Hr>69Ehetl(`PxgfEBbZJYg-Z})F@t@Ebkl&~`AzOg zY+_C|DRV+FENo1sa@1{_%G~w9LHw@TU+YFEhbC}g)rT}{gQ0W7GFnG&akl?Nlm{6S z|2d7w%Np5y@ePqDoWA?J$89)SEScoq*Jn7(AxYSOly7-5=vm=G|C@aA}o?hl>G1 zpP~I3jm)r7g$basFPpy9H8Da5xaG?8>0mqaP5D-KvaF zhaF;6S*M^RoUbnMAN+8iUJ+Mie$+P7r1ne@ujkv2C0I07nOm#x6$?Iq7C zOqnxiUu#p6CH$D#;IpZwT(S`p^xs@g?j0eCo=uE@vLmIhl4ck$3Bs@#6djz*%)g1V zI{oX-{%bEWh}EH8fVILQm|6juc^~VIuzM{}Kx}bYmZ%^hxGWgAGq*~mB`iQbi&0LF z`IoCJBdIRgwBM>iWnQWf#$Wdj&?927p1$i0oBGRUL?aaNl@Y2p^;rhn)u{KlD<}RW zL?cr4yx_{Ays8M0{%k=>CH)>F@yoz7Kt0H*DZCxD*0H}r1TpC;c0#>OHn4$tmnr#P z=~9WH)o6puRYyZX7?~@Pn`>0{_^&0Dg64%%s4Zrz`YhvObZ;LPpxp8=n+H6&#y||H zE)gM4tS;ekZE%b$bF9F^)&?|({MYiM6^LwLU7O0g#~pP|!$NLMUE+&I-i93V`XrSw zQ@&k{h5P?%N(-VC*yR&u2Fm6v1_rN5nJys@QIOB81mi5e(q0gsrQXeKO#7*gsvtX; z-Pg$OukHf_^OzNvRtF;tlUsA8?aD#rT>R%C2zU$ZsVE99v>bGjjG2b&sV8u}2i+Xr z2x;YJ!#rys{y)U>3jG>FI+COOl80nhsNt(wy6WLN7`@qt?mX451L?2zL}~1sB}14- z`;~f9LJ5(DaFaYAH=HEw39a#bX~w)y?BcY|zAzi$)!3U$&K zVE4Z{mv&|%`#PQIdmZV*V~XV#NsJorfqep6qJ1MwWC?^!f$Pk)&8kLSayKX^?zCYa z9x-5Ei6!ti%J?Uwl$Cb@;||w$rP_b>p!LtpSAtBORM$K!2C@d;+n|hHR%=m~v;vm! zM{dkaBbQaVSlUQ*3&B!C==dlJdG+%i=;u}OSO&}&#RVkBCGC9xUh#fhuyex_pR z4y4uG-Bo^FONhXOp*J%sb4N!&vj8070rg_9MRs<#HPCbyaH@0_O8aB2hk$*JyPC zVY55iYusDtr(OHg(qngZ3$=pp6fN%|{=FZIEUxfF16y|Ivsu*SzFgAq8CY5E#7+5T zLLJAo20@$i^ZCx*@T{|N?75u-is2b0oLWBTrfuIc)-V2bWE2Osi&Sw2e|?+&mg$#( z(d}QhxI@2xyvE9#*%`M)21!0;+zJh#Il)gL3?ABfi3qW$x0a`3@=I`S#9q z3A$A)?PL^^?#-3)#ZCB0u!(+3nMp_@7mz^->JZS--U>n`;8kKLuQ&B!`a(qlUoNa5 zc$v#GDO{D$oc8K$z{cI-#Cfhvt1OEa?bw@&nI$poM}yYa31UAO)<}9uC$7D$E!pzc z@q(h8Skl*w!%fZFr%*>>$YkgbMdSVHjLMCHb#}wLhWJmZola)cO^c|{4lE22MGrh0 zsvBzRPVDKQ5cxcP$|8$Px);VgLbqQQWy`~sm)nVUI{(mUvk~7kklEhWxJ01v<#GN> zfeq7F2yba;gk&51HlB+@Hutxk`C7{RhM(rjyaao~a*Te_VISk-L7uKxczx-5=!~Ti z{fw;Bpd?_bLJ`ASY((r^fJ*55^9(D+;k+=>^wVUY9l1Sb0)lM@DYrdOeE0Ou{bkgv z$^_>}=C%SQ3IWQb{ePoO22hBerGqsX6kC$bRq~r3x2CXQ_*XSIoU&qX;>=>_U}2$W>->)e z7AR`TPCX`YUsKUsF^wWIp^;bhi>T&M=JLg(%r;GlD6t~N{L*(SvW|D*K5m+%80ass zNgK3jw=E!exa53`5FoxOUi(N3Uy zOp&}R-!uM;elzp==$C|0=>{Tl@GVHE$Y~D3@4@i)vGsL?@Y+QF^^o@r0}_8c5{=W0VtU{Gc{#69$&0cJ zN7ht&XDC#W;pA@mIa+hfZGC?0H^dHUUQaVTCL)YG9&pZNN4~`kr1j-E$5`FPb}vXY zBk#;j(fKAa5OSQTkX9oqYoXXHHBHgol)Y0FD5F0*;7q{Vjf|aO{%l0mW?ICsHGIA; zd@=kus&L|TStSE4_5LAc=8}Z@vfy)e8qYN_0vvLCB)de5>G{WmxN!6j*OTTC^#yX! zb2=k_Cgqmd`?t~+4=SP9(tLcOHScRbA{pJI!T5ky?e50IrPq-RG zIA-^189I$$-L|zK93QW5P;cUA1K(r3YBy`1_7P3MZanUwplJTz*o_^;{$JGDikq<- zevH>RNAr_yqPvdL0@q>lNw4H!1RnZr5#q@JU#{Kx_Ba-58R{=v^L`EFU)G08MK!v( zECzU_So>yn1ffUEKCcdvZ-KW2Q_Lp2307v_eIn>_t?YJ|J^C-gsLy>tOma^cxJ6TP z@`w7GtU@PPC1zPAPGv*PLJ8w144vOBXGMH0!e@N%hOdGSxklZeMo&9tP9`|xGHo>D zRy9_g8&Jq(m!1uOZ5|nYSA{+$i+T_~g;TvhIkGLioIt?Ggt`AYN9)0UW4L~w$vnrs zmXWpOvd4rs@G}G7%SPFg-SoOgOZ_ta{&VIGo4}GEV(UjwVuvzH;xeE85aV-{d>H11 zx93ZZ>V&qIMf6md=7YXB5-ST|C@be-V>fZGkwc1-iT|~}vfIr=cC59DjDCJF-QgX$ zR)w`ZyuYt%;a;`8Q}}29v6SH(AIE}V6@)hHgFCoeKQWsaH$Sb&zwY*o0S(iFU8Yrz zWSErbFQ0K(Nf<_pXQWy)Q@f4GuMhlybj8tE&?pz(diKp);)i#YO~6u!ucOT@MxXgl z7rX%+WARkP;`KGxye9~6g92?_(buYUt?Qorzs!WlO~Va zA?C3W%h>2s{%ZmI$k}Mi3}a>6^0)VG=I6ASrs>Xkp5;|Mef9tvS2$`ny#=;=*vc>_I3S=Z8MHiJ$Ezoc0d!7`0Cz)p{#c* zB80DBe=23W?53!Cdo;q{l#IuNWF*naf~KZ8kw@+18V&>d#ZEBH#qejJ+*E${C|X-e z-%DIIbs|(Fit`GonpLhU9k{2;B)9Q)0IS81hp$d{QN(d#AOe$ zv%mD$H&CuVI;3CqjX!uc1@LzNN1^BcCW-wKd0usVJFiGZ@Xoe%*rz`c*(Q?Rx4@%n zoqX|JyiFmIdSm;c+VoT%m`WFYx|1>!UnBPoiF;;CR%5kV=ge8g#*dN^0nHBGkF0^B z2z^DLr*&T+O6sEL)EJWC*@mR_I^X5ufMgBOP}O3q(o~9OixJ*n3m6}l`KoV85EkF^ zYIXVryIMNsNHjwQVQ=|hBP9xzH|Q(^8LQ(x22b3*J!y#cZT&EZYZ?Kn~KXhO$Tk=yRE_TS;P7y zz+Cmx@!d23?fAkY*);Tne6)iWvY8WzEIQ7sf#dW3&+!#y|Ks>9{&9R+|38ke*L5a6 z{^5=KBw-_F5gLr|{$F#9N@x1tFGh1D%(0N)IwbdrM?)!t4B*A55@5P#qSm7!o|r}vQ;jwy|d*H zSHO!a+`{E^Y&O6lD}K+yV8j1b?qjmI$B>Vm#LKlWX>^<~9+$n(G0M1>jbHxcM`Pux z0_ll)SbqbwbCX>?|A+fzV6w*&szKJ_`d^rYv+kpveG7mBJqil;e@9>ou?7S!fdO__$Tk zMU>F#&eoR2@%OzNPc1F?Zzk~VH;(4%q~dEga0i$6t(rq^(JQO*rwtrazcU-;yRLe= z8HCE``4lG?+$kFn3Hha{+&a1|zpGpIfcVTVeJyB1WCYVC2h&!yAJxcjf9RHcu(fLV zkuGynYn75hC#bG{jwET*T~%|Lv@b&;H4$h-qpdVYZqP*3<&s&>-mWm0tfZkOWs!J9 z#qpGw=xvWYs;McB4$4rv3!S(9#J=ti_190%q1%tu1YrrF%41OS@#)N|{9A7%|PW_c?{PN!otD6Tw2CF~xpQ zWBgHr!gO|^HfdZsW~`-v(DR2P!m97wK1!f}-l4Br>=dG&u9r{zQq2C^Lo?mOBAOV{ zvk!j}Hj^osdjpO{}SAHQ7r+h^_mWeOL4hVY`SJ3le^8-pEOh?74b+Wx=d&IBH+?~nT< zM7FFEvQ=cyzGg{^7z!m4jWL)kV_&ikiij3Oi%N!&lzlBFi9&>g$QqGk7s_)~s+opg z|L1vK&+T=)y}X*~-RGS1UC#O5b23huX6efE#L;qIkEvTYC?EVHq{-K+m;Pp*CdZrL zE1F%+lnM#4d~GLorkhiloz3q;S}f#RrJvT-P|}mV?|Yl(Ez7{}^R!n_knKr6A1%gv zotb^=@Kv95GY*LsVXMBA25vm%JL?<^9pTsfIP-&2D;_1(C9^NAXN0GCZ& z+SRj7QK&h+EEscP*M(Si7jb2EdYk7YR5siwTZ47+W$h-}>Ycg|tXh4A+PlJc>=Ml+ z^`m73XS*L0E4!p`P*7yy+!H*fcg7iI%3;{LVpn2UK>^n9Iw58kSo4P#MR2{6$mdUbnsxo0lD#j4>LF0`|=B z!Vq=`kGeYW-P=K4Qhg)Qt0@opLgxW|JY6;tB7Hst2rM#@1sw~5%ZFy1 z@!#5HqjTRlE}hOpYR>@4`D4m5q7H}N1kFh*gtxL^IsKYd)$QwBO1CAzKaf- zN`qFra|B}(#VS7wv9^_9v?(W<3zN#^-)jn*_M=@ydJ+5JTea@pW$4%_z^!g{cXPs4 z$1Bf|=P|o%jkK*Qb_fcs7`#)#gd`gUD8f$zwnr-e< z7MbqZcBdsgR`!9zlYM-zZRbgKgpv?fB%4~ZzdkiZBlX!{3mN-(NFwB$rabOnH2lVO z|J2w#4Wrjr=eEWS9{K~*nv};~?YLz0>K&9As9d`oeWlXSj*W8{-eAsQnT>ke^dioK2!VDqgpe5J{8#4=id zNAoGV&cC7ch8Ant$NsRlyXx=0aO>VsrOoo@9Lp7n7_$fj#g$TS5i%)wTw0xi`;lO! zlKytVlY+q$vaJ`(xC)L(W>(v5>SpQZjOY_!HYi~ZR5mb}QNV$hMmwo`4kg*gDbob8 z)4skGfBk5FEOJ*cNpV8eVB8SLwjD+cFGfBZcT(?_ir;hX2tv>1sQ;B$JO(ct8*(o8 z&|Ur9*O{F<`!PwmeDcnC#5w74mS>$4*XXZ#9O^kdD6MkSFhH-o%31BILbdODak`E< zg$;|f=>z6;p)KqziX|t-g26zF4s*j`>_r_Aj%e$p)L>~6_`%_JN9KL>egndL=gAoCX1-c>J!VtN z?osPGrEHR2eCLj*Kuw*9s|0;T*==hN)^=WXpYkmi}$><_>H6A)}P~ zUhN$7xUU~m#d+jBn}Lb3dzBm^PpqY6JT%^B zgJE{M%*kJy(I88~l6Qa7rV`!+8%%cEFOa>CtT*`d9yX>~synp#5!SXOn5;uqblDg9 z*hW?de4Cz^Hz^&#* z-mPQ}e9|w97pR#MKDCpSWrR-7@9|MElIIi1){TE(!TLVy7PE)X`5A}SUe%0!ci=K^ z6DgJ6mj`AYDVu$nswR8Cy*+SlQtr*^Bi%2(I9lUhK71!KSHE$~-4p!WqjG!2_ny{l zE)8fXy(eDX*qZyro1z^a=L8Ps>(I0h&3xH`XesB-cWC6w?VUa=g;YPnzjebS>-%pH zW)7a#oi??sH<+^x&K48;bazHUN?O7BiC;~ZO~Lg%H)YfWBj&|u*c~l!57qVG?kmVA`vw~ksHjqXfD?(YAKdy4Di}0z0%#56SZwzgZzzp z&gD;E9N=-v++QLPefLV<=Gy5_YCiw4|COGjV>`b7+AnF+c^kecHxQui6P?sP{~N_-eMpkj%lzZ^s^*l&Tyx z6E`m_R{r*7a;~q(E2~f?z}xxT_~hJ|r+1<)tY`E_smbnVa77*55|tfqHpI2d@3aC_ zfOoK`nP`;7!yS7YZ@;YCDwNb1?Lav{{O*Q#RwP0{OKdolDRTD1@a*U-m2W5K8$Px_ zofJ0>O{wc2cQT{Y2@!JZ&>PVrf1hWtBg2y?Mt%5Vff&an1r~G3@Yl!fQ)r}TQQveMUEfLO&{hh*oso zJ>%zFo1$dTU-Z#7`GsUK6)s>aBb;eFTeDlrs0jR`?fLYL+u*DS5hmR;HARzQ48C+9 zujV#wE5UvfFo~OTSu8?&$nNLZ@1Bie41Acf~)S*QZMmao2t!#Tvey z>+Z#a$Xpy%7-!7E#~j1)<0^Y#8a-!~H$;>xZHjFRnt>ZW7K)jQ(caI>yzpqA)Lgk~ za>&U@A)9Bz6$KktG45l{?L6ngt7p?s3mQ)x8aynGoZ06VW9m98`oK2?Ub8+%n(o4N z75>fh{cqmPocq*e(R?Rm_+0qLIx`L#?jeVPluMGn@#4eX;VFKn5|cEJ%H11^h*0$x z^*DCzW*-T+v1h|64~1PDtQW}(^Xx+Bl;JZYc@kI55AFAFRrus%hT&&c@%jAa@jHC$ zS!JsibrqyTj}?EUL*K0@n5X$}I7dwFFzxc2z6uAaj#%~%B24mWM6AQaiirbgG=)}drw!JdQ2K$`Pan}oS-^P{{ z#-p|+=hS}m6KuY^vGS@UHOmXB^2ipY^6bqtRa=WS-+a7*>BL|h1H5KtO3DGa5sGx=B^lfvM{?hf8aT#>*vO|lIGw`wGX+ORJBoSMK)m;F5T?zDZR z+1aeGT^L$Fz2ZwO_-rX`R`H^W#7Lca+?-!`ByZc#IM-I|JL#OP=rAJrvKiRkcDgC45!+%BUA zO0Lu1o5l^+iI$#6u^9+J!P9nXG&FCovZXTZyPMgjy5|n8tbE_7ZWmWt z4CdvAd8>2Dg8S#BciCk3ZvFPW`BV3XgJs%{ zq||#~-gq?knVi}VK_P8ASCNd}@?|&=emJxLS-xG#V?~+~+Yz5aI%b;7&i-$3$z|i5 zc7>HT33)Q_&OM7TQQNjlJK=R`)`j|UUmJ!4jEz+;8t+^e>P;{5S$E|^s%~TPgU=-h2ec%<^choQ)A5|%~QhEXQuZsIFE+B-iXSnFfbt5 zl=diW>V!^C|DyGxF{?ad!~MZC53SU+1ZwpkrEZYMr|`$z9)&<>9mY5tr)qB=x`@1;>4!?dMj6A6hTToKgP} zNr~X5P%hHas#Lj{s2ms_*+GN-FqXqFPebxL@VN+Cd-P|gCfCrI5%q-N>QB5_oh?+k zZ8wrVoaDVqY$iS0mBpV2=Fm4sjfLzHG_}wz#643SXGpd+4gAoy!&sNRc6Zlkd4@+P zoJPg7e&b#%?I#91-Q$Hne`AX(+d}_|Vt^uVK13gF$vJ2=b#yVHl&$z8SLkzBu^UsC z%t<|8FlR#4b~Dbc+ms!Tq|0a*_3vUnVRx&n5A6)3SZb zG1t@GPd9ztRmi+jP?%aPC5iR^`E}*<*E(6YMOB#LzE*`+fzBMA#*Zgcmw%8EfHLN^a-lt)jYHg^m(FFBV-4OFrofdQ! zQF6JhRd3-Uyz<#^dB7E zJ4Agm`GJR@XV#~e-}GL7n-drw;oHnHq0}Ql_EqoIGc#IR+PTQiy%b-i77T;@H70R4 z-#Pg{xcQDzMP^V=StHorCzXQY>-t&uAb)|Ur0R>u^f`iq7Xyd&N)1ywRBDPjQ2ewP zH+!q+6xHZ3yrS;aXm3N3+^&q;UgD3bt_rqe7fSkI#Xio%dxK3dP1fUc@t`9AM*GCr*15j9+kD6>Q>*3${ZY2f(Z{w@ zX?7=F4DfO|VT*dcC9U^tTU0bz5xw``Q}8fnqp**uJEu5=FV_3Krf)7gx%o*mUu}9~ z&o-U?PCi=;6>XU=F^!K!pesowWsZ#KOKv}MAj(}pU{bK0qPId3JGz}ZsVJO($LqB5 z)G2gVYd0tS^uUk`uRr%df~nQJ#%8))Z>hFm|L~&fJNvj8_FQSUxKGk1!oD9T@$@9E zwD7`6{n+?;c6ra|RBBm?=P4(yWllZW(r5f+bNZ&V36A864mQnjp2dEJGkdj~)!TP% z*+Mt(lNxYP_L|m{Ic34#0}bz6cldd}Q9)i3-XvH1i2-Nxd5EK$=TIrnHH+iCb-Pb% zTV87Fl)H&^JJ73UwZn5*`MRN`5KnH^^Nfk2uuaOUXD3N^?erjXyKN=(>2{C*fBkmD z&N*N~^?(k^I%ZO4knd2!W?~N?ld-=^VGl}egg|u!3;5~sg~KlW2?wPoNh?*d;#xBPkV#9?oMpZxj?qMvoMl=RdG%PkMcU@#-R&xV7a z{^Pn265)z+{fFaMh!RjzUDxjboYX)7TRZ|CkR1La!T)pyjKtCzFgqngEzv++M;oG4 zeV!hNVO&Hl-O=_|MCnGhU10bMH1h_FWGC?G_JgT$ z=;)+G@mB{3dm{Z;ZK0chYdvVorR9p{3x|EV5lSv;pqd-X#s%%>EoyC#LV;6)+?Ey? z5pHPAkBFF%RKd@l)D-k^HPD(%3o6SO4!Z%&ML?&Z%uz1%t6B3dmZwg;sz81`E6o>W%OKi?=}uC@HGF zQ|8yvXc!l?4cZYjxRb57D;gYcMr#XScW(K_Y2Egbrl&rFUvLbC!NINGCaN|T7ryr%-N`gM? zv*c#*dIOm$l#FtJGJ1k|5#vde4_VQ=P8T>uE{Pw$dl?ajtpp{{kU2(hkm1t!Uv6)d zHO2)63hG^v7)M7CLxQ9MX@?@Ns3D#Hkyo#c7n5nt-8(^?} z9vCcOSx)f8Vbwt70hywbJOAWESUI3wT|o@F6t%4~go+-v)8ZYkx$6E*HMlzcxt)PM8YdKa^HfCRBg2nNIV^Mw&m^5OX)D;EUX zakW-H`20lmvZwUHQ&Bh?N=iKM^Mi8rG5%*j5`_oqa1Bb(A1-LQX4M)mz7w_lVempFg(7P0toq}@IBM_Fcip$wg!Rz(lp0^xTrtVJa;_mzDH0} z|8P5Xd_K#^no#txktZHfyw(8_3PRuabdfGLWtj zjngP2i%e+2?7?Yp7s$QBBL9mZD)(WGQw6gypapeRatr6O4%As}bp9}!yivCj0v3yPN zK%BRNaw%GBImDIgs$3(q)s*{?5HiOOl>7XbofD{D6Fd-yuAp39M!y(07_G^F*}0mU zH9}iWIVn+9qaL7K{$G9WWZjzJ$*riI)rG8^1wggpzwF$=t2IJfO}R~Z0sP%Wl@o1V z6Fd;Euh8c{ue;^5G+O^w?sDrIp{=GIVw>_yrvGf9i;|MS-nB`OgJaO0qwCfSSU>Iy|iUg3S0C;;tkFqnh!Xca)64>U`D1XBK{{W)$# zJwXMW8;ba}dREsJOE&8R9C5t%-nrylGJ#w6#8dVYE99UUP)F!! zg>w0?qd^N2&*Bo*+vk6}Ful>LWgKbG5AxP#TxS$68y}=66hPHHS zN;nBLJP0(j#cSwAb*Rx+Hbd#>%OVG9VK6NuVQuZA2GmqP2RKo+7E75bCV*O}!O*~0 z3u6Ecm*oFbo%Em)lvMv#C#~*X@b(VSi<+IVEki)@FBC@lhi3m(CvApSnj8ad84K29 z@t2DV%%SA^-|M9B=<3~01mp7&HQ}`FZeaSLQ}F+~PWprulvw{*C;irjJST~$YG_bC z4V`Lot5=s-RJt)N&6ED&^ z)(Euy_tHE7Njy^v#>EXd%fMi822z5GVG$oFA!YuCTyYh&vcfjAuZ!X2=G^Q{kogl~Bv9w&^FCod&zYLt|_iL$$V)?>hiy|N)S>6Yg5QK0a)ZZY|U?X)1Ldc)F z=bv>6f9{G($Va#%@~ literal 0 HcmV?d00001 From 8cbdd51ad34c9a108f14c2dbbab8ccd37a9d9834 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 19:00:04 +0800 Subject: [PATCH 296/343] fix(validation): pin installed I05 bpftool before fleet reservation --- docs/validation-heal-operations.md | 25 +++++++++++++-- scripts/validation_heal/fleet_node.py | 38 ++++++++++++++++++---- scripts/validation_heal/healthy.py | 4 ++- test_validation_heal_environment.py | 45 ++++++++++++++++++++++++--- 4 files changed, 98 insertions(+), 14 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index dddb9d33..625f7fe6 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -786,8 +786,15 @@ business files and retain the same complete sixteen-disk format mapping. The planner requires those exact original owners, candidate, inspector, cleanup receipts and failure boundary before allowing the existing audit/isolate/post-audit protocol. It issues no workload request and changes -neither the original FAIL nor canonical acceptance. Isolation remains pending -until its independent post-audit completes. +neither the original FAIL nor canonical acceptance. + +Isolation [39766457981/1](https://github.com/rustfs/rustfs/actions/runs/39766457981) +completed `ISOLATED` with helper `fa0b659`. Independent audit byte-matched all +102 original files and all four preflight/isolation/post-audit baselines. +All twenty data/state archive parents are root-owned mode 0700 and on their +original devices; original tree, owner and inode identities are unchanged. +No S3 request, backup restoration, namespace deletion or original verdict +rewrite occurred. `environment.inspect` now also records the selected bpftool path, installed `/usr/lib/linux-tools/*/bpftool` executables, resolved root-owned executable @@ -797,6 +804,20 @@ This inventory does not install tools, attach programs, change PATH or certify the I05 service filter; the actual owned-service attachment checks remain mandatory before another diagnostic. +Read-only inventory +[39766570688/1](https://github.com/rustfs/rustfs/actions/runs/39766570688) +found broken PATH wrappers on both vm002 and vm003, which run kernel +`6.17.0-1022-azure`. All four peers already have +`/usr/lib/linux-tools/7.0.0-1017-azure/bpftool` with executable SHA-256 +`aabc337b58169d62c42b3c8f1fccf605e5fc1e445a23feb0156638e9d9606991`. +The executable reports bpftool 7.7.0/libbpf 1.7 and its root-cgroup query exits +zero on each peer. That empty root query alone does not verify service BPF +support. I05 now uses this exact installed executable with protected root-owned +path, digest and version checks, records its identity beside the actual +service attachments, and checks every peer before reserving any namespace. +It changes no installed tool, global PATH or kernel. Service-bound ingress and +egress program identity checks remain mandatory on the running kernel. + `validation_cleanup_run_id=` selects the narrow I05 preservation route with the original package build and an immutable helper containing `timeout_retention.py`. The driver must remain on the original Functional Chain diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 6ff1d4f0..71580f66 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -30,6 +30,8 @@ "RUSTFS_HEAL_AUTO_HEAL_ENABLE": "false", "RUSTFS_HEAL_MRF_ENABLE": "false", "RUSTFS_SHARD_INTEGRITY_WRITE": "true", "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED": "true", "RUSTFS_OBS_LOGGER_LEVEL": "info"} +I05_BPFTOOL = Path('/usr/lib/linux-tools/7.0.0-1017-azure/bpftool') +I05_BPFTOOL_SHA256 = 'aabc337b58169d62c42b3c8f1fccf605e5fc1e445a23feb0156638e9d9606991' def runtime_config(profile): @@ -399,6 +401,7 @@ def bpf_tool_inventory(): 'selectedByPath': str(path) == selected, 'version': {'returncode': version.returncode, 'stdout': version.stdout[:4096], 'stderr': version.stderr[:4096]}, 'rootCgroupQuery': {'returncode': query.returncode, 'jsonListObserved': isinstance(rows, list), + 'stdout': query.stdout[:4096], 'stderr': query.stderr[:4096]}, 'serviceAttachmentsVerified': False}) return {'kernelRelease': os.uname().release, 'selectedPath': selected, 'tools': tools, @@ -467,8 +470,11 @@ def preflight(request): limits = resource_limits(request.get("resourceProfile", "standard")) require(all(shutil.disk_usage(mount).free >= limits["minimumDriveFreeBytes"] for mount in MOUNTS), "insufficient free space for the explicit Heal resource profile") - return {"hostname": socket.gethostname(), "mounts": observed, "statePath": str(state), - "dataRoots": [str(root / "data") for root in roots], "resourceLimits": limits, "mutated": False} + result = {"hostname": socket.gethostname(), "mounts": observed, "statePath": str(state), + "dataRoots": [str(root / "data") for root in roots], "resourceLimits": limits, "mutated": False} + if request.get('runtimeProfile') == 'lock-timeout': + result['bpfTool'] = i05_bpf_tool() + return result def reserve(request): @@ -611,13 +617,31 @@ def start(request): return result +def i05_bpf_tool(): + """Use the installed executable reviewed on all current auto-testing peers.""" + resolved = I05_BPFTOOL.resolve(strict=True) + require(resolved.is_file() and os.access(resolved, os.X_OK), 'I05 reviewed bpftool is not executable') + for path in (resolved, *resolved.parents, I05_BPFTOOL, *I05_BPFTOOL.parents): + info = path.stat() + require(info.st_uid == 0 and not info.st_mode & 0o022, + 'I05 reviewed bpftool path is not root-owned and protected') + require(digest(resolved) == I05_BPFTOOL_SHA256, 'I05 installed bpftool differs from its reviewed digest') + version = command(str(resolved), '--version') + require(version.stdout.startswith('bpftool v7.7.0\nusing libbpf v1.7\n'), + 'I05 installed bpftool has an unexpected version') + return {'path': str(I05_BPFTOOL), 'resolvedPath': str(resolved), 'sha256': I05_BPFTOOL_SHA256, + 'version': version.stdout.strip(), 'kernelRelease': os.uname().release, + 'serviceAttachmentsVerified': False} + + def i05_filter_properties(owner): """Exclude the observed platform source only from the private I05 unit.""" if owner.get("runtimeProfile") != "lock-timeout": return [] - require(shutil.which('bpftool') and Path('/sys/fs/cgroup/cgroup.controllers').is_file(), - 'I05 requires bpftool and cgroup v2 to verify its service network filter') + require(Path('/sys/fs/cgroup/cgroup.controllers').is_file(), + 'I05 requires cgroup v2 to verify its service network filter') require(owner.get('unit') == unit(owner['runId']), 'I05 filter requires its exact owned service') + i05_bpf_tool() return ['--property=IPAddressDeny=168.63.129.16/32', '--property=IPAddressAllow='] @@ -630,6 +654,7 @@ def i05_filter_policy(owner, expected_pid=None): """ require(owner.get('runtimeProfile') == 'lock-timeout' and owner.get('unit') == unit(owner['runId']), 'I05 filter requires its exact owned service') + tool = i05_bpf_tool() names = ('IPAddressDeny', 'IPAddressAllow', 'ControlGroup', 'MainPID') raw = command('systemctl', 'show', owner['unit'], *(part for name in names for part in ('-p', name))).stdout @@ -647,7 +672,7 @@ def i05_filter_policy(owner, expected_pid=None): cgroup = Path('/sys/fs/cgroup') / group.lstrip('/') require(cgroup.is_dir() and cgroup.resolve(strict=True) == cgroup, 'I05 filtered service cgroup is missing or aliased') - rows = json.loads(command('bpftool', '-j', 'cgroup', 'show', str(cgroup)).stdout) + rows = json.loads(command(tool['resolvedPath'], '-j', 'cgroup', 'show', str(cgroup)).stdout) expected = {'ingress': 'sd_fw_ingress', 'egress': 'sd_fw_egress'} # Newer bpftool uses libbpf's full enum names for the same attach types. directions = {'ingress': 'ingress', 'egress': 'egress', @@ -663,7 +688,7 @@ def i05_filter_policy(owner, expected_pid=None): 'I05 kernel service BPF attachments are missing, foreign or incomplete') programs = [] for row in sorted(rows, key=lambda item: item['attach_type']): - program = json.loads(command('bpftool', '-j', 'prog', 'show', 'id', str(row['id'])).stdout) + program = json.loads(command(tool['resolvedPath'], '-j', 'prog', 'show', 'id', str(row['id'])).stdout) if isinstance(program, list): require(len(program) == 1, 'I05 kernel filter program is ambiguous') program = program[0] @@ -676,6 +701,7 @@ def i05_filter_policy(owner, expected_pid=None): 'type': program['type'], 'tag': program['tag'], 'uid': program['uid']}) return {'unit': owner['unit'], 'pid': pid, 'controlGroup': group, 'deniedAddress': '168.63.129.16/32', 'effectiveProperties': values, + 'bpfTool': tool, 'kernelPrograms': programs, 'scope': 'owned I05 service only', 'hostFirewallChanged': False, 'persistentNetworkSettingsChanged': False, 'snapshotOnly': True, 'canonicalPassClaimed': False} diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 7622f39a..d24a9c3e 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -294,7 +294,9 @@ def start(self): def deploy(self, binary_sha, bucket, access, secret, *, before_start=None): self.secret_values.update({access, secret}) - checked = {node: self.remote(node, "preflight", resourceProfile=self.resource_profile) for node in self.nodes} + checked = {node: self.remote(node, "preflight", resourceProfile=self.resource_profile, + **({'runtimeProfile': 'lock-timeout'} if self.runtime_profile == 'lock-timeout' else {})) + for node in self.nodes} self.record("preflight", checked) require(len({value["hostname"] for value in checked.values()}) == len(self.nodes), "Heal node aliases must resolve to distinct actual hosts") diff --git a/test_validation_heal_environment.py b/test_validation_heal_environment.py index 006b8b63..adb3f035 100644 --- a/test_validation_heal_environment.py +++ b/test_validation_heal_environment.py @@ -145,6 +145,21 @@ def test_legacy_mount_only_inventory_does_not_imply_runtime_or_clean_proof(self) class I05PlatformFilterTests(unittest.TestCase): + def test_i05_checks_every_peer_tool_before_reserving_any_namespace(self): + with tempfile.TemporaryDirectory() as folder: + fleet = Fleet(['vm000', 'vm001', 'vm002', 'vm003'], Path(folder), 'gh-123-1-h2', + runtime_profile='lock-timeout') + def remote(node, action, **params): + self.assertEqual(action, 'preflight') + self.assertEqual(params['runtimeProfile'], 'lock-timeout') + if node == 'vm002': raise ValueError('reviewed tool unavailable') + return {'hostname': node} + with patch.object(fleet, 'remote', side_effect=remote) as calls, \ + self.assertRaisesRegex(ValueError, 'tool unavailable'): + fleet.deploy('a' * 64, 'rv-heal-' + 'a' * 16, 'access', 'secret') + self.assertEqual([call.args[0] for call in calls.call_args_list], ['vm000', 'vm001', 'vm002']) + self.assertEqual(fleet.reserved, []) + def test_tool_inventory_keeps_broken_kernel_wrapper_separate_from_working_binary(self): wrapper = Path('/usr/sbin/bpftool') binary = Path('/usr/lib/linux-tools/6.17.0-1020-azure/bpftool') @@ -176,7 +191,7 @@ def owner(self): 'unit': 'rustfs-validation-heal-gh-123-1-h1.service'} def test_only_i05_private_unit_receives_filter_and_missing_backend_refuses(self): - with patch.object(fleet_node.shutil, 'which', return_value='/usr/sbin/bpftool'), \ + with patch.object(fleet_node, 'i05_bpf_tool', return_value={'path': 'reviewed'}), \ patch.object(fleet_node, 'paths'), \ patch.object(Path, 'is_file', return_value=True): properties = fleet_node.i05_filter_properties(self.owner()) @@ -184,10 +199,29 @@ def test_only_i05_private_unit_receives_filter_and_missing_backend_refuses(self) '--property=IPAddressAllow=']) with self.assertRaises(ValueError): fleet_node.i05_filter_properties({**self.owner(), 'unit': 'rustfs.service'}) - with patch.object(fleet_node.shutil, 'which', return_value=None): + with patch.object(fleet_node, 'i05_bpf_tool', side_effect=ValueError('missing reviewed tool')): with self.assertRaises(ValueError): fleet_node.i05_filter_properties(self.owner()) self.assertEqual(fleet_node.i05_filter_properties({'runtimeProfile': 'manual'}), []) + def test_reviewed_tool_requires_digest_version_and_protected_executable(self): + for variant in ('valid', 'digest', 'version', 'owner', 'writable', 'executable'): + with self.subTest(variant=variant), \ + patch.object(Path, 'resolve', autospec=True, side_effect=lambda path, **kw: path), \ + patch.object(Path, 'is_file', return_value=True), \ + patch.object(Path, 'stat', return_value=types.SimpleNamespace( + st_uid=1000 if variant == 'owner' else 0, st_mode=0o777 if variant == 'writable' else 0o755)), \ + patch.object(fleet_node.os, 'access', return_value=variant != 'executable'), \ + patch.object(fleet_node, 'digest', return_value='0' * 64 if variant == 'digest' + else fleet_node.I05_BPFTOOL_SHA256), \ + patch.object(fleet_node, 'command', return_value=types.SimpleNamespace( + stdout='unexpected' if variant == 'version' else 'bpftool v7.7.0\nusing libbpf v1.7\nfeatures:\n')): + if variant == 'valid': + result = fleet_node.i05_bpf_tool() + self.assertEqual(result['sha256'], fleet_node.I05_BPFTOOL_SHA256) + self.assertFalse(result['serviceAttachmentsVerified']) + else: + with self.assertRaises(ValueError): fleet_node.i05_bpf_tool() + def policy(self, variant=None): owner = self.owner() group = '/system.slice/' + owner['unit'] @@ -220,10 +254,10 @@ def policy(self, variant=None): def invoke(*args, **kwargs): if args[:2] == ('systemctl', 'show'): raw = '\n'.join(k + '=' + v for k, v in values.items()) - elif args[:4] == ('bpftool', '-j', 'cgroup', 'show'): + elif args[:4] == ('/reviewed/bpftool', '-j', 'cgroup', 'show'): if variant == 'query-failed': raise ValueError('kernel query refused') raw = json.dumps(rows) - elif args[:5] == ('bpftool', '-j', 'prog', 'show', 'id'): + elif args[:5] == ('/reviewed/bpftool', '-j', 'prog', 'show', 'id'): # Both bpftool versions' single-program JSON forms are accepted. program = programs[int(args[5])] raw = json.dumps([program] if int(args[5]) == 1 else program) @@ -231,6 +265,7 @@ def invoke(*args, **kwargs): return types.SimpleNamespace(stdout=raw) membership = '0::' + (group if variant != 'process-cgroup' else '/foreign') with patch.object(fleet_node, 'command', side_effect=invoke) as command, \ + patch.object(fleet_node, 'i05_bpf_tool', return_value={'resolvedPath': '/reviewed/bpftool'}), \ patch.object(fleet_node, 'paths'), \ patch.object(Path, 'read_text', return_value=membership + '\n'), \ patch.object(Path, 'is_dir', return_value=variant != 'missing-cgroup'), \ @@ -247,7 +282,7 @@ def test_effective_properties_require_actual_process_bound_kernel_programs(self) self.assertFalse(result['persistentNetworkSettingsChanged']) self.assertTrue(result['snapshotOnly']) self.assertEqual([call.args[0] for call in calls.call_args_list], - ['systemctl', 'bpftool', 'bpftool', 'bpftool']) + ['systemctl', '/reviewed/bpftool', '/reviewed/bpftool', '/reviewed/bpftool']) def test_libbpf_attach_names_keep_raw_kernel_evidence_and_same_directions(self): result, _ = self.policy('libbpf-names') From 33d3bb20cd3cf105c38ca77d0be251d5a5d24b42 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 19:13:37 +0800 Subject: [PATCH 297/343] docs(validation): record audited I05 H1 H2 diagnostic --- docs/validation-heal-cases.md | 2 +- docs/validation-heal-environment-gates.md | 2 +- docs/validation-heal-operations.md | 24 +++++++++++++++++++++ docs/validation-migration-map.md | 16 ++++++++++++++ scripts/validation_heal/execution_policy.py | 2 +- scripts/validation_heal/recipes.json | 2 +- 6 files changed, 44 insertions(+), 4 deletions(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index e7a7f1f7..a1d047bc 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1770,7 +1770,7 @@ Topologies: H1,H2. Branches: object-lock-timeout, caller-disconnect, stale-read- Execution policy: `evidence-blocked`; disabled by default: `false`. -Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集、单连接历史 GET 断开和服务 admission/owner/socket 观测编排；新增 timeout 诊断控制器将一次真实 PUT 的持久化 ACK、服务端 request ID/精确写锁 timeout、仍被反压的原 GET socket 及物理/S3/FD 收尾关联起来，尚无 live 验收。仍缺实际断连瞬间的精确所有权证明、stale-generation controller、queued successor 锁闭环及 H1/H2 DL/DV 完整对象/ACK/物理/FD 验收。精确源码 cfg(test) 原生证据独立保留；诊断观测、客户端断开或原生子集不能充作完整 canonical PASS。 +Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 单连接历史 GET、真实争用 PUT 超时、服务 admission/owner/socket 和有界 FD 收尾诊断已通过独立 live 审计。仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation 与 queued-successor controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。 1. LOCK_WINDOW用本例慢IO/受控原生锁屏障持有目标对象锁，先证明真实争用超过预注册预算。 2. 同时heal/覆盖或删除；调用方断开实例仅断本次TCP连接，不杀服务器或关闭跳板机；记录请求开始/断开/服务在途结束。 diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index 742a4edb..9db5c9ab 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -40,7 +40,7 @@ These observations do not certify a clean environment. Independently audit retai | --- | --- | --- | --- | | C11 | `heal.bucket-incarnation-recreation` | H1/H2 旧元数据盘重返、原始 checkpoint/resume 与 committed MRF 重放编排已实现；精确候选的 9 项存储原生回归和 3 项 manager incarnation 前置证据已独立核验。完整现场分支尚无独立审计结果 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | | F07 | `heal.checkpoint-and-mrf-corruption` | 已实现 H1/H2 checkpoint 截断、未来 schema、committed MRF CRC 故障及独立安全重扫；10 项原生回归作为前置证据。真实较早代次回放与私有限额卷的 journal ENOSPC 已实现；仍缺完整 native cut 审核，完整 live 尚未审计 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | -| I05 | `heal.lock-timeout-and-caller-abort` | 已有同源码 native 子集、单连接签名历史 GET 和服务 admission/owner/socket 观测编排；缺完整 controller 与 live 验收 | 实际断连边界所有权、timeout/stale-generation 编排、H1/H2 最终对象/锁/FD 闭环；服务观测尚未 live 验收 | +| I05 | `heal.lock-timeout-and-caller-abort` | 同源码 native 子集已保留；H1/H2 单连接历史 GET、真实争用 PUT timeout、服务 admission/owner/socket、有界 FD 收尾诊断通过独立 live 审计（39767336741/1），完整用例仍 BLOCKED | 实际断连边界所有权、完整 native timeout/stale 屏障、stale-generation/queued-successor controller 和 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环 | | J02 | `heal.outcome-counters-and-error-truth` | 全部夹具与 manifest 对账均已实现。候选 `7951bb629821246868ee3761b321dd07c06db45c` 修复整桶重试覆盖 repaired 记录，210 项 manager 与 5 项独立进程集成测试通过。新实机 `38874900846/1` 完成 20 个分支：16 个报告通过，4 个原始全元数据损坏探针返回永久 Failed，保留 NOT_OBSERVED；四组混合任务与全部 50 条节点清理记录已独立审计。真实 legacy Unknown 已观察，不能代替原始探针 | 原始元数据损坏分支的 Unknown 仍未观察；完整 source-procedure 验收尚未闭合。原始 Failed、NOT_OBSERVED 和完整用例门控保持不变 | I05 与 A09、G01–G06 的精确屏障证据允许使用同一不可变候选源码单独构建的 `cfg(test)` 原生测试执行文件。不得给优化服务候选加入隐藏 hook 冒充原包，也不能把缺少 release hook 本身归为外部环境阻塞。原生子集、完整服务调用和现场最终 ACK/物理检查分别验收；已验收 native 证据不能代替尚缺的 controller 或完整 live 结果。第一批 G07/G08 的 native 分支与 H04 容量锁窗口保留各自未完成条件；外部请求重试或最终对象正确均不能证明精确锁或提交窗口。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 625f7fe6..d0af50a8 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -818,6 +818,30 @@ service attachments, and checks every peer before reserving any namespace. It changes no installed tool, global PATH or kernel. Service-bound ingress and egress program identity checks remain mandatory on the running kernel. +Diagnostic [39767336741/1](https://github.com/rustfs/rustfs/actions/runs/39767336741) +completed both H1 and H2 with helper `8cbdd51` on the unchanged candidate +`757b751482bb970bc321e92059690a7698554358`, package `39364893106/1` and +inspector `39566504743/1`. Independent audit of all 211 official artifact +files verified nine seed request/ACK pairs and seven versions/markers per +topology, candidate/helper bindings, actual process-bound ingress/egress BPF +programs on all peers, complete journal cursor chains and exact server +request/lock attempt identities. Each contender returned a known non-retried +503 after a 1001 ms write-lock timeout while the original GET remained +backpressured. The caller connection disappeared and its diagnostic read +guard dropped; original physical files remained identical before public +verification and all one/four peer inventories agreed with seed ACKs. + +Final FD counts were H1 vm000 `22 -> 22`; H2 vm000 `106 -> 94`, vm001 +`108 -> 104`, vm002 `105 -> 101` and vm003 `101 -> 97`, with unchanged process +birth and network namespace. These are bounded snapshots, not a global +absence-of-leaks proof. All five node cleanup receipts show stopped services +and removed owned roots. The original result is `BLOCKED/cleanup PASS`: +the workflow fails its deliberately retained canonical gate despite successful +diagnostic execution. Exact disconnect-edge ownership, distributed release, +complete native timeout/stale fences, stale-generation/queued-successor +controllers and full source H1/H2 DL/DV coverage remain unaccepted. No new +canonical case is counted. + `validation_cleanup_run_id=` selects the narrow I05 preservation route with the original package build and an immutable helper containing `timeout_retention.py`. The driver must remain on the original Functional Chain diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 030d01d6..a53b69de 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -38,6 +38,22 @@ All 158 canonical IDs are retained. Independent complete live acceptance is **10 The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 37 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. +I05 progress (2026-10-22): four-node startup scene `39764496504/1` was +independently verified and isolated by +[39766457981/1](https://github.com/rustfs/rustfs/actions/runs/39766457981), +preserving all 102 original artifact files and twenty same-device private +archive parents. Read-only inspection identified broken kernel-selecting +bpftool wrappers on vm002/vm003; the exact already-installed executable is +now digest-bound and checked before reservations. Fresh diagnostic +[39767336741/1](https://github.com/rustfs/rustfs/actions/runs/39767336741) +completed H1 and H2 on helper `8cbdd51`, with all five node cleanups passing. +Independent audit of 211 artifact files verified exact 1001 ms write-lock +timeouts, closed caller sockets, unchanged physical content, full diagnostic +seed ACK/peer inventories and final FD counts at or below baseline. The +original canonical result remains `BLOCKED/cleanup PASS`; source DL/DV, +exact disconnect/native fences, stale-generation and queued-successor +coverage remain outstanding. Acceptance therefore stays **108/158**. + [J02 diagnostic 38874900846/1](https://github.com/rustfs/rustfs/actions/runs/38874900846) completed all 20 H1/H2 × DB/DV branch instances on that exact candidate. Sixteen branches report PASS; the four original all-metadata-corruption probes report permanent Failed rather than Unknown and retain NOT_OBSERVED. Independent archive and provenance audits passed. Independent mixed-branch audits verified all four repaired shards against original complete bytes, exact receipts and counters, surviving below-quorum failures, original terminal preservation and post-rescue S3 inventories. All 50 node namespace finalization receipts passed audit. Real legacy writes separately produce Unknown with zero repairs and unchanged physical files, but do not replace the original source procedure. Full J02 canonical acceptance remains blocked; these partial results add no canonical acceptance. A10 run `39197088183/1` uses immutable helper `b16e6072679d6a60a044b9d4520b30b4a79b5557`, candidate `072b57b37559fbe626aeda8abf61617fdcbdd926`, package `38043283428/1` and inspector `38715612825/1`. All nine source variants across H1/H2 and DB/DV passed their 68 required execution instances. Independent audit checked original SDK request/ACK hashes and version/marker identities, physical set selection, exact task options/receipts/counter partitions, 27,096 complete shard-streaming proofs, physical closure before target GET or control rescue, complete peer inventories and all 170 node cleanup receipts. Full inventories are checked against ACK manifests; their endpoint iteration is established by the immutable executed helper source, since the inventory documents omit endpoint labels. The remove=true branch replays one complete below-quorum original object after exact DELETE and requires whole-orphan removal with unchanged live controls. Rescue restores only post-DELETE authority. The original failed A10 result remains byte-identical. diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py index 65b426ae..723088e5 100644 --- a/scripts/validation_heal/execution_policy.py +++ b/scripts/validation_heal/execution_policy.py @@ -139,7 +139,7 @@ def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, . ), "heal.lock-timeout-and-caller-abort": _native( "I05", "NATIVE_LOCK_OWNER_ABORT_BARRIER", - "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集、单连接历史 GET 断开和服务 admission/owner/socket 观测编排；观测编排尚无 live 验收。仍缺实际断连边界的精确所有权、完整 timeout/stale-generation controller 和 H1/H2 对象/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断观测、客户端断开或原生子集不能充作完整 canonical PASS。", + "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 单连接历史 GET、真实争用 PUT 超时、服务 admission/owner/socket 和有界 FD 收尾诊断已通过独立 live 审计。仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation 与 queued-successor controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。", ("server_lock_owner", "lock_timeout_fence", "caller_abort_sequence", "stale_read_repair_fence", "exact_source_native_evidence", "complete_live_orchestrator", "final_object_lock_fd_closure"), ), "heal.outcome-counters-and-error-truth": _native( diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index 42261087..e44a15a1 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -3253,7 +3253,7 @@ "dispatchable": false, "code": "I05", "reasonCode": "NATIVE_LOCK_OWNER_ABORT_BARRIER", - "reason": "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集、单连接历史 GET 断开和服务 admission/owner/socket 观测编排；观测编排尚无 live 验收。仍缺实际断连边界的精确所有权、完整 timeout/stale-generation controller 和 H1/H2 对象/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断观测、客户端断开或原生子集不能充作完整 canonical PASS。", + "reason": "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 单连接历史 GET、真实争用 PUT 超时、服务 admission/owner/socket 和有界 FD 收尾诊断已通过独立 live 审计。仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation 与 queued-successor controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。", "requiredCapabilities": [ "server_lock_owner", "lock_timeout_fence", From 42ddcf6ea543e6d8d3b83fcad43e6164255706fd Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 21:05:37 +0800 Subject: [PATCH 298/343] test(validation): add I05 queued-successor diagnostics --- .../workflows/validation-catalog-tests.yml | 4 +- docs/validation-heal-cases.md | 2 +- docs/validation-heal-environment-gates.md | 2 +- docs/validation-heal-operations.md | 38 ++ docs/validation-migration-map.md | 6 +- scripts/validation_heal/caller_socket.py | 3 +- scripts/validation_heal/execution_policy.py | 2 +- scripts/validation_heal/fleet_node.py | 15 +- scripts/validation_heal/healthy.py | 5 +- scripts/validation_heal/lock_observer.py | 8 +- scripts/validation_heal/lock_successor.py | 345 ++++++++++++++++++ .../validation_heal/lock_timeout_runner.py | 69 +++- scripts/validation_heal/race_retention.py | 6 +- scripts/validation_heal/recipes.json | 2 +- .../validation_heal/recover_environment.py | 2 +- scripts/validation_heal/retained_scene.py | 10 +- scripts/validation_heal/run_case.sh | 18 +- scripts/validation_heal/timeout_retention.py | 117 ++++++ test_validation_heal_lock_successor.py | 270 ++++++++++++++ test_validation_heal_lock_timeout_runner.py | 82 +++++ test_validation_heal_timeout_retention.py | 98 +++++ 21 files changed, 1066 insertions(+), 38 deletions(-) create mode 100644 scripts/validation_heal/lock_successor.py create mode 100644 test_validation_heal_lock_successor.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 7b6f2776..cd95e18c 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -34,6 +34,7 @@ on: - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py + - test_validation_heal_lock_successor.py - test_validation_heal_race_native.py - test_validation_heal_incarnation_native.py - test_validation_heal_manager_native.py @@ -103,6 +104,7 @@ on: - test_validation_heal_encrypted.py - test_validation_heal_schema.py - test_validation_heal_native.py + - test_validation_heal_lock_successor.py - test_validation_heal_race_native.py - test_validation_heal_incarnation_native.py - test_validation_heal_manager_native.py @@ -163,7 +165,7 @@ jobs: sudo apt-get install --no-install-recommends -y nftables fi - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - - run: python3 -B -m unittest -v test_validation_heal_timeout_retention.py test_validation_heal_race_retention.py test_validation_heal_retained_scene.py test_validation_heal_lock_timeout_runner.py test_validation_heal_lock_timeout.py test_validation_heal_lock_observer.py test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py + - run: python3 -B -m unittest -v test_validation_heal_timeout_retention.py test_validation_heal_race_retention.py test_validation_heal_retained_scene.py test_validation_heal_lock_timeout_runner.py test_validation_heal_lock_timeout.py test_validation_heal_lock_observer.py test_validation_heal_lock_successor.py test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index a1d047bc..a8b6a05b 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1770,7 +1770,7 @@ Topologies: H1,H2. Branches: object-lock-timeout, caller-disconnect, stale-read- Execution policy: `evidence-blocked`; disabled by default: `false`. -Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 单连接历史 GET、真实争用 PUT 超时、服务 admission/owner/socket 和有界 FD 收尾诊断已通过独立 live 审计。仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation 与 queued-successor controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。 +Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 单连接历史 GET、真实争用 PUT 超时、服务 admission/owner/socket 和有界 FD 收尾诊断已通过独立 live 审计。已实现独立 queued-successor controller、候选绑定的 native 前置及失败现场隔离，尚缺该分支的独立 live 验收；仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。 1. LOCK_WINDOW用本例慢IO/受控原生锁屏障持有目标对象锁，先证明真实争用超过预注册预算。 2. 同时heal/覆盖或删除；调用方断开实例仅断本次TCP连接，不杀服务器或关闭跳板机；记录请求开始/断开/服务在途结束。 diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index 9db5c9ab..982f239f 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -40,7 +40,7 @@ These observations do not certify a clean environment. Independently audit retai | --- | --- | --- | --- | | C11 | `heal.bucket-incarnation-recreation` | H1/H2 旧元数据盘重返、原始 checkpoint/resume 与 committed MRF 重放编排已实现；精确候选的 9 项存储原生回归和 3 项 manager incarnation 前置证据已独立核验。完整现场分支尚无独立审计结果 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | | F07 | `heal.checkpoint-and-mrf-corruption` | 已实现 H1/H2 checkpoint 截断、未来 schema、committed MRF CRC 故障及独立安全重扫；10 项原生回归作为前置证据。真实较早代次回放与私有限额卷的 journal ENOSPC 已实现；仍缺完整 native cut 审核，完整 live 尚未审计 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | -| I05 | `heal.lock-timeout-and-caller-abort` | 同源码 native 子集已保留；H1/H2 单连接历史 GET、真实争用 PUT timeout、服务 admission/owner/socket、有界 FD 收尾诊断通过独立 live 审计（39767336741/1），完整用例仍 BLOCKED | 实际断连边界所有权、完整 native timeout/stale 屏障、stale-generation/queued-successor controller 和 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环 | +| I05 | `heal.lock-timeout-and-caller-abort` | 同源码 native 子集已保留；H1/H2 单连接历史 GET、真实争用 PUT timeout、服务 admission/owner/socket、有界 FD 收尾诊断通过独立 live 审计（39767336741/1）；queued-successor controller、native 前置与现场隔离已实现，完整用例仍 BLOCKED | queued-successor 独立 live 验收、实际断连边界所有权、完整 native timeout/stale 屏障、stale-generation controller 和 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环 | | J02 | `heal.outcome-counters-and-error-truth` | 全部夹具与 manifest 对账均已实现。候选 `7951bb629821246868ee3761b321dd07c06db45c` 修复整桶重试覆盖 repaired 记录，210 项 manager 与 5 项独立进程集成测试通过。新实机 `38874900846/1` 完成 20 个分支：16 个报告通过，4 个原始全元数据损坏探针返回永久 Failed，保留 NOT_OBSERVED；四组混合任务与全部 50 条节点清理记录已独立审计。真实 legacy Unknown 已观察，不能代替原始探针 | 原始元数据损坏分支的 Unknown 仍未观察；完整 source-procedure 验收尚未闭合。原始 Failed、NOT_OBSERVED 和完整用例门控保持不变 | I05 与 A09、G01–G06 的精确屏障证据允许使用同一不可变候选源码单独构建的 `cfg(test)` 原生测试执行文件。不得给优化服务候选加入隐藏 hook 冒充原包，也不能把缺少 release hook 本身归为外部环境阻塞。原生子集、完整服务调用和现场最终 ACK/物理检查分别验收；已验收 native 证据不能代替尚缺的 controller 或完整 live 结果。第一批 G07/G08 的 native 分支与 H04 容量锁窗口保留各自未完成条件；外部请求重试或最终对象正确均不能证明精确锁或提交窗口。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index d0af50a8..ad813684 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -974,3 +974,41 @@ empty object scene. It reuses the existing same-device, private-parent isolation backend and its live owner/physical/tree checks. No workload is replayed and no namespace is deleted. A successful isolation or corrected startup check cannot certify the I05 source case. + +### I05 queued-successor diagnostic + +Select `I05_DIAGNOSTIC_BRANCH=queued-successor` for the explicit I05 route. +`run_case.sh` passes this branch to both preflight and the runtime controller. +The Functional Chain input `i05_diagnostic_branch=queued-successor` downloads +the pinned native artifact and verifies its original hosted run, candidate, +helper, binary, exact regression logs and required I05 labels before fleet +mutation. `VALIDATION_HEAL_NATIVE_DIRECTORY` supplies that bundle locally. +Native success remains a prerequisite and never certifies the live case. + +`lock_successor.py` uses a separate `lock-successor` runtime: the GET lock +budget remains one second, while the successor PUT commit acquisition budget +is sixty seconds. It starts exactly one marked PUT while the original +historical GET is admitted, observes the PUT acquiring the same namespace, +then closes only the original caller socket. It requires the original GET +guard-drop event before successor acquisition, an unretried server-request-ID +bound PUT ACK, exact ETag/version/payload, preserved historical generations, +a distinct new physical generation, all-peer S3 agreement and bounded FD +drain. Observation windows remain ten seconds for queue admission and thirty +seconds for completion. The fixture is a diagnostic subset of the source DV +workload; it does not complete the source DL/DV matrix. + +Failure stops the run-owned service and retains its original requests, ACKs, +logs and physical scene. `successor-retention.json` binds all diagnostic files +and stopped per-node physical digests to the original owner. Recovery accepts +only an unchanged sealed archive and the same stopped live scene, then uses +the existing same-device private-parent isolation backend. An unknown PUT +outcome is preserved without retry, deletion or old-generation restoration. +Incomplete capture, missing reservations, changed evidence or scene drift +refuse automated isolation. + +Local controller, admission and isolation tests pass; independent live +acceptance of this branch is still pending. Every successful diagnostic +remains `BLOCKED/cleanup PASS` with `canonicalCasePassClaimed=false`. Exact +disconnect-edge ownership, complete native timeout/stale fences, the +stale-generation controller and full H1/H2 source DL/DV ACK/physical/lock/FD +closure remain required. Migration acceptance stays 108/158. diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index a53b69de..95daaf4d 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -51,8 +51,8 @@ Independent audit of 211 artifact files verified exact 1001 ms write-lock timeouts, closed caller sockets, unchanged physical content, full diagnostic seed ACK/peer inventories and final FD counts at or below baseline. The original canonical result remains `BLOCKED/cleanup PASS`; source DL/DV, -exact disconnect/native fences, stale-generation and queued-successor -coverage remain outstanding. Acceptance therefore stays **108/158**. +exact disconnect/native fences, stale-generation coverage and independent +queued-successor live acceptance remain outstanding. Acceptance therefore stays **108/158**. [J02 diagnostic 38874900846/1](https://github.com/rustfs/rustfs/actions/runs/38874900846) completed all 20 H1/H2 × DB/DV branch instances on that exact candidate. Sixteen branches report PASS; the four original all-metadata-corruption probes report permanent Failed rather than Unknown and retain NOT_OBSERVED. Independent archive and provenance audits passed. Independent mixed-branch audits verified all four repaired shards against original complete bytes, exact receipts and counters, surviving below-quorum failures, original terminal preservation and post-rescue S3 inventories. All 50 node namespace finalization receipts passed audit. Real legacy writes separately produce Unknown with zero repairs and unchanged physical files, but do not replace the original source procedure. Full J02 canonical acceptance remains blocked; these partial results add no canonical acceptance. @@ -252,7 +252,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -Four Heal cases remain outside canonical dispatch: C11, F07, I05 and J02. All four have explicit diagnostic routes that preserve their blocking canonical verdicts; I05 currently executes only the timeout diagnostic subset and still needs exact server barriers and its other source branches. The 13 environment-gated cases have disabled-by-default control flows and oracles in `environment_cases.py`; actual execution requires an assigned and reviewed auto-testing resource adapter, and canonical results remain review-gated. See [the environment gate reference](validation-heal-environment-gates.md) for each missing resource, capability and native barrier. +Four Heal cases remain outside canonical dispatch: C11, F07, I05 and J02. All four have explicit diagnostic routes that preserve their blocking canonical verdicts; I05 has a live-audited timeout diagnostic and an implemented queued-successor diagnostic with exact-source native prerequisites and stopped-scene isolation. The successor branch still needs independent live acceptance; exact server barriers and other source branches remain required. The 13 environment-gated cases have disabled-by-default control flows and oracles in `environment_cases.py`; actual execution requires an assigned and reviewed auto-testing resource adapter, and canonical results remain review-gated. See [the environment gate reference](validation-heal-environment-gates.md) for each missing resource, capability and native barrier. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. diff --git a/scripts/validation_heal/caller_socket.py b/scripts/validation_heal/caller_socket.py index 808e6028..e477d20a 100644 --- a/scripts/validation_heal/caller_socket.py +++ b/scripts/validation_heal/caller_socket.py @@ -110,7 +110,8 @@ def run(request, node): client, server = address(request['clientAddress']), address(request['serverAddress']) require(server[1] == 9000, 'caller socket requires the bound service port') _, _, owner = node['owned'](request['runId']) - require(owner.get('runtimeProfile') == 'lock-timeout', 'caller socket requires the isolated I05 runtime') + require(owner.get('runtimeProfile') in {'lock-timeout', 'lock-successor'}, + 'caller socket requires the isolated I05 runtime') expected = request['expectedProcess'] require(isinstance(expected, dict) and all(name in expected for name in node['PROCESS_FIELDS']), 'caller socket requires the original process birth') diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py index 723088e5..88c7dd19 100644 --- a/scripts/validation_heal/execution_policy.py +++ b/scripts/validation_heal/execution_policy.py @@ -139,7 +139,7 @@ def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, . ), "heal.lock-timeout-and-caller-abort": _native( "I05", "NATIVE_LOCK_OWNER_ABORT_BARRIER", - "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 单连接历史 GET、真实争用 PUT 超时、服务 admission/owner/socket 和有界 FD 收尾诊断已通过独立 live 审计。仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation 与 queued-successor controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。", + "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 单连接历史 GET、真实争用 PUT 超时、服务 admission/owner/socket 和有界 FD 收尾诊断已通过独立 live 审计。已实现独立 queued-successor controller、候选绑定的 native 前置及失败现场隔离，尚缺该分支的独立 live 验收；仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。", ("server_lock_owner", "lock_timeout_fence", "caller_abort_sequence", "stale_read_repair_fence", "exact_source_native_evidence", "complete_live_orchestrator", "final_object_lock_fd_closure"), ), "heal.outcome-counters-and-error-truth": _native( diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 71580f66..68271446 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -35,6 +35,9 @@ def runtime_config(profile): + if profile == "lock-successor": + return {**runtime_config("lock-timeout"), + "RUSTFS_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS": "60000"} if profile == "lock-timeout": logging_filter = "info,rustfs_ecstore::object_lock_diag=trace" return {**BACKGROUND_CONFIG, "RUSTFS_OBJECT_LOCK_DIAG_ENABLE": "true", @@ -472,7 +475,7 @@ def preflight(request): "insufficient free space for the explicit Heal resource profile") result = {"hostname": socket.gethostname(), "mounts": observed, "statePath": str(state), "dataRoots": [str(root / "data") for root in roots], "resourceLimits": limits, "mutated": False} - if request.get('runtimeProfile') == 'lock-timeout': + if request.get('runtimeProfile') in {'lock-timeout', 'lock-successor'}: result['bpfTool'] = i05_bpf_tool() return result @@ -593,7 +596,7 @@ def start(request): "J02 phase commit is incomplete; service cannot start") if owner.get("outcomePhase") == "legacy": outcome_legacy_phase({"runId": owner["runId"], "action": "outcome-legacy-phase"}) - if owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf", "outcome-legacy", "frozen-race-legacy", "lock-timeout"}: + if owner.get("runtimeProfile") in {"incarnation-checkpoint", "incarnation-mrf", "outcome-legacy", "frozen-race-legacy", "lock-timeout", "lock-successor"}: runtime_configuration(state, owner) filter_properties = i05_filter_properties(owner) command("systemd-run", "--unit=" + owner["unit"], "--property=Type=exec", @@ -612,7 +615,7 @@ def start(request): *(volume_module().properties(state, roots, owner) if owner.get("runtimeProfile") in {"bounded-capacity", "journal-bounded"} else []), *replacement_module().start_properties(state, roots), str(state / "rustfs"), *owner["volumes"]) result = {"unit": owner["unit"], "started": True, "journalPolicy": journal_policy(owner)} - if owner.get("runtimeProfile") == "lock-timeout": + if owner.get("runtimeProfile") in {"lock-timeout", "lock-successor"}: result["platformNetworkPolicy"] = i05_filter_policy(owner) return result @@ -636,7 +639,7 @@ def i05_bpf_tool(): def i05_filter_properties(owner): """Exclude the observed platform source only from the private I05 unit.""" - if owner.get("runtimeProfile") != "lock-timeout": + if owner.get("runtimeProfile") not in {"lock-timeout", "lock-successor"}: return [] require(Path('/sys/fs/cgroup/cgroup.controllers').is_file(), 'I05 requires cgroup v2 to verify its service network filter') @@ -652,7 +655,7 @@ def i05_filter_policy(owner, expected_pid=None): platform agent or persistent network setting is changed. Stopping the unit removes its service-scoped filters with the service's cgroup. """ - require(owner.get('runtimeProfile') == 'lock-timeout' + require(owner.get('runtimeProfile') in {'lock-timeout', 'lock-successor'} and owner.get('unit') == unit(owner['runId']), 'I05 filter requires its exact owned service') tool = i05_bpf_tool() names = ('IPAddressDeny', 'IPAddressAllow', 'ControlGroup', 'MainPID') @@ -823,7 +826,7 @@ def process_metrics(request): "threads": int(fields[17]), "fileDescriptors": sum(1 for _ in (root / "fd").iterdir()), "io": {name: io[name] for name in sorted(required_io)}, "readOnly": True, "ioScope": "owned process counters; cached logical IO and storage IO reported separately"} - if owned(request["runId"])[2].get("runtimeProfile") == "lock-timeout": + if owned(request["runId"])[2].get("runtimeProfile") in {"lock-timeout", "lock-successor"}: namespace = os.readlink(root / "ns/net") result["descriptorCensus"] = caller_socket_backend.descriptor_census(root) result["fileDescriptors"] = result["descriptorCensus"]["fileDescriptors"] diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index d24a9c3e..1050b921 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -137,7 +137,7 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): - require(runtime_profile in {"lock-timeout", "frozen-race-legacy", "outcome-legacy", "outcome-grace", "queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation", "incarnation-checkpoint", "incarnation-mrf"}, "unsupported Heal runtime profile") + require(runtime_profile in {"lock-timeout", "lock-successor", "frozen-race-legacy", "outcome-legacy", "outcome-grace", "queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation", "incarnation-checkpoint", "incarnation-mrf"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory @@ -295,7 +295,8 @@ def start(self): def deploy(self, binary_sha, bucket, access, secret, *, before_start=None): self.secret_values.update({access, secret}) checked = {node: self.remote(node, "preflight", resourceProfile=self.resource_profile, - **({'runtimeProfile': 'lock-timeout'} if self.runtime_profile == 'lock-timeout' else {})) + **({'runtimeProfile': self.runtime_profile} + if self.runtime_profile in {'lock-timeout', 'lock-successor'} else {})) for node in self.nodes} self.record("preflight", checked) require(len({value["hostname"] for value in checked.values()}) == len(self.nodes), diff --git a/scripts/validation_heal/lock_observer.py b/scripts/validation_heal/lock_observer.py index 191f9382..1e1fa11f 100644 --- a/scripts/validation_heal/lock_observer.py +++ b/scripts/validation_heal/lock_observer.py @@ -103,7 +103,7 @@ def same_process(observed, expected): def validate_scope(probe, fleet, bucket, entry): """Refuse another service, source or target before any remote observation.""" validate_binding(probe.binding) - require(fleet.runtime_profile == 'lock-timeout' and fleet.run_id == probe.binding['runId'] + require(fleet.runtime_profile in {'lock-timeout', 'lock-successor'} and fleet.run_id == probe.binding['runId'] and probe.binding['candidateCommit'] == CANDIDATE, 'caller observation requires its reviewed candidate and isolated runtime') endpoints = {f'node{index}': f'http://rustfs-node{index}:9000' for index in range(1, len(fleet.nodes) + 1)} @@ -130,7 +130,7 @@ def validate_scope(probe, fleet, bucket, entry): def observe_disconnect(probe, fleet, bucket, entry, output, *, while_admitted=None, - receive_buffer_bytes=None, before_verification=None): + receive_buffer_bytes=None, before_verification=None, after_disconnect=None): """Use an already deployed exclusive I05 service; do not alter the fleet.""" validate_scope(probe, fleet, bucket, entry) output = Path(output) @@ -146,7 +146,7 @@ def metrics(label): value = fleet.record(label, {'node': node, **fleet.remote(node, 'process-metrics')}) identity = value['identity'] require(identity.get('binarySha256') == probe.binding['candidateBinarySha256'] - and all(identity.get('settings', {}).get(k) == v for k, v in runtime_config('lock-timeout').items()), + and all(identity.get('settings', {}).get(k) == v for k, v in runtime_config(fleet.runtime_profile).items()), 'I05 resource sample has another candidate or process configuration') if node in expected: same_process(identity, expected[node]) @@ -218,6 +218,7 @@ def before_disconnect(sent): transport_options['receive_buffer_bytes'] = receive_buffer_bytes sent = disconnect_get(probe, bucket, entry['key'], entry['versionId'], output / 'transport', **transport_options) + disconnected_work = after_disconnect(sent, admitted, window) if after_disconnect is not None else None deadline = time.monotonic() + DRAIN_SECONDS while True: observed = capture(sent) @@ -251,6 +252,7 @@ def before_disconnect(sent): result = {'observationComplete': True, 'baselineResources': baseline, 'admission': admitted, 'finalAttempt': observed, 'admittedWork': admitted_work, + 'disconnectedWork': disconnected_work, 'verificationPrecondition': verification_precondition, 'finalSocket': connection, 'finalResources': resources, 'afterVerificationResources': verified_resources, 'exactHistoricalContentPreserved': True, diff --git a/scripts/validation_heal/lock_successor.py b/scripts/validation_heal/lock_successor.py new file mode 100644 index 00000000..c849bae1 --- /dev/null +++ b/scripts/validation_heal/lock_successor.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +"""I05 queued PUT after one caller disconnect; diagnostic subset only.""" + +import copy +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +from pathlib import Path +import time +from urllib.parse import quote +import uuid + +try: + from . import lock_observer + from .heal_probe import CLOCK_DOMAIN, Probe, canonical, payload, require, save, validate_binding + from .heal_workload import Recorder, verify_seed + from .healthy import physical_content + from .lock_timeout import CALLER_RECEIVE_BUFFER_BYTES, MIN_BACKPRESSURE_BYTES, require_exact_inventory_entry + from .shards import files, map_target, verify_cohort +except ImportError: + import lock_observer + from heal_probe import CLOCK_DOMAIN, Probe, canonical, payload, require, save, validate_binding + from heal_workload import Recorder, verify_seed + from healthy import physical_content + from lock_timeout import CALLER_RECEIVE_BUFFER_BYTES, MIN_BACKPRESSURE_BYTES, require_exact_inventory_entry + from shards import files, map_target, verify_cohort + + +SUCCESSOR_BYTES = 1024 * 1024 + 197 +QUEUE_SECONDS = 10 +COMPLETE_SECONDS = 30 +PUT_TIMEOUT_MS = 60000 + + +class SuccessorProbe(Probe): + def __init__(self, original, marker): + super().__init__(original.binding, original.endpoint_name, credential_pair=original.credential_pair) + self.marker = marker + + def s3_client(self): + client = super().s3_client() + client.meta.events.register('before-sign.s3.PutObject', self.mark) + return client + + def mark(self, request, **kwargs): + value = request.headers.get('User-Agent', '') + if isinstance(value, bytes): + value = value.decode('ascii') + request.headers['User-Agent'] = value + ' ' + self.marker + + +def successor_attempt(rows, marker, holder, bucket, key): + """The caller marker selects one server request, never supplies its ID.""" + require(marker.startswith('auto-testing-i05-successor/') + and str(uuid.UUID(marker.split('/')[-1])) == marker.split('/')[-1], + 'I05 successor marker differs') + selected, request_ids = [], set() + for row in rows: + if row.get('target') != lock_observer.TARGET: + continue + spans = [span for span in [row.get('span'), *row.get('spans', [])] + if isinstance(span, dict) and span.get('name') == 'http-request' + and marker in str(span.get('user_agent', '')).split()] + if not spans: + continue + require(len({canonical(span) for span in spans}) == 1, + 'I05 successor has ambiguous HTTP ancestry') + span = spans[0] + request_id = span.get('request_id') + require(isinstance(request_id, str) and str(uuid.UUID(request_id)) == request_id + and uuid.UUID(request_id).int and request_id != holder['request_id'] + and row.get('request_id') == request_id and span.get('method') == 'PUT' + and span.get('uri') == '/' + quote(bucket, safe='') + '/' + quote(key, safe='/~'), + 'I05 successor server request, method or exact URI differs') + request_ids.add(request_id) + # The same S3 call may first read its bucket incarnation. Those events + # are retained, but cannot establish target-object write admission. + incarnation = (row.get('mode') == 'read' and row.get('op') == 'get_object' + and row.get('bucket') == '.rustfs.sys' + and row.get('object') == f'buckets/{bucket}/.bucket-incarnation') + if incarnation: + continue + require(row.get('event') == 'object_namespace_lock' and row.get('component') == 'ecstore' + and row.get('subsystem') == 'object_lock' and row.get('op') == 'put_object_commit' + and row.get('bucket') == bucket and row.get('object') == key and row.get('mode') == 'write' + and row.get('requested_version_id') is None and row.get('owner') == holder['owner'] + and row.get('namespace') == holder['namespace'] and type(row.get('timeout_ms')) is int + and row['timeout_ms'] == PUT_TIMEOUT_MS and isinstance(row.get('lock_attempt_id'), str) + and str(uuid.UUID(row['lock_attempt_id'])) == row['lock_attempt_id'] + and uuid.UUID(row['lock_attempt_id']).int + and row['lock_attempt_id'] != holder['lock_attempt_id'] + and type(row.get('acquire_ms')) is int and row['acquire_ms'] >= 0, + 'I05 successor write attempt escaped its exact namespace or budget') + require(row.get('state') in {'acquiring', 'acquired', 'guard_dropped'} + and row.get('level') == 'TRACE' + and type(row.get('hold_ms')) is int and row['hold_ms'] >= 0, + 'I05 successor failed, cancelled or lost its diagnostic lifecycle') + selected.append(row) + require(len(request_ids) <= 1, 'I05 successor marker was retried or reused') + if not selected: + return None + require(len({row['lock_attempt_id'] for row in selected}) == 1, + 'I05 successor made multiple target write attempts') + states = [row['state'] for row in selected] + require(states in (['acquiring'], ['acquiring', 'acquired'], + ['acquiring', 'acquired', 'guard_dropped']), + 'I05 successor write lifecycle is missing, duplicated or out of order') + return {'requestId': selected[0]['request_id'], 'lockAttemptId': selected[0]['lock_attempt_id'], + 'states': states, 'events': selected, 'canonicalCasePassClaimed': False} + + +def validate_successor_ack(directory, probe, bucket, key, body, attempt): + request_path = directory / '000001-request.json' + ack_path = directory / '000001-ack.json' + require({path.name for path in directory.iterdir()} == {request_path.name, ack_path.name}, + 'I05 successor lacks exactly one original request and receipt') + request, ack = json.loads(request_path.read_text()), json.loads(ack_path.read_text()) + params = {'Bucket': bucket, 'Key': key, 'ContentType': 'application/octet-stream', + 'Metadata': {'heal-fixture': 'I05-queued-successor'}} + digest = {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()} + require(request.get('schemaVersion') == ack.get('schemaVersion') == 2 + and request.get('runId') == ack.get('runId') == probe.binding['runId'] + and request.get('sequence') == ack.get('sequence') == 1 + and request.get('operation') == 'put_object' and request.get('params') == params + and request.get('payload') == digest and request.get('expectHttp') is None + and request.get('identity') is None and ack.get('identity') is None + and ack.get('requestFile') == request_path.name + and ack.get('requestSha256') == hashlib.sha256(request_path.read_bytes()).hexdigest() + and ack.get('operationCheck') == 'PASS' + and request.get('casePassClaimed') is False and ack.get('casePassClaimed') is False, + 'I05 successor receipt differs from its exact durable PUT') + observation = ack.get('observation', {}) + response = observation.get('response') or {} + metadata = response.get('ResponseMetadata', {}) + version = response.get('VersionId') + require(observation.get('operation') == 'put_object' and observation.get('params') == params + and observation.get('endpoint') == probe.endpoint_name and observation.get('payload') == digest + and observation.get('clientCallInvoked') is True and observation.get('acknowledged') is True + and observation.get('resultUnknown') is False and not observation.get('clientCleanupError') + and metadata.get('HTTPStatusCode') == 200 and type(metadata.get('RetryAttempts')) is int + and metadata['RetryAttempts'] == 0 and metadata.get('RequestId') == attempt['requestId'] + and isinstance(version, str) and str(uuid.UUID(version)) == version and uuid.UUID(version).int + and response.get('ETag') == '"' + hashlib.md5(body, usedforsecurity=False).hexdigest() + '"', + 'I05 successor lacks one known, unretried, content-bound version ACK') + times = [request.get('preparedMonotonicNs'), ack.get('collectorCallStartedMonotonicNs'), + observation.get('callStartedMonotonicNs'), observation.get('callFinishedMonotonicNs'), + ack.get('collectorCallFinishedMonotonicNs'), ack.get('recordedMonotonicNs')] + require(request.get('clockDomain') == ack.get('clockDomain') == observation.get('clockDomain') == CLOCK_DOMAIN + and all(type(value) is int and value > 0 for value in times) and times == sorted(times), + 'I05 successor receipt lost its collector clock ordering') + return {'key': key, 'versionId': version, 'isLatest': True, 'deleteMarker': False, + 'payload': digest, 'properties': {'ETag': response['ETag'], 'ContentLength': len(body), + 'ContentType': params['ContentType'], 'Metadata': params['Metadata']}, 'tags': []} + + +def physical_append(before, after, bucket, baseline, added): + """Bind all old shards and the exact new version before any public read.""" + require(set(before) == set(after) and all(before[node]['formats'] == after[node]['formats'] for node in before), + 'I05 successor changed physical topology or formats') + old, new = files(before), files(after) + metadata_path = f"{bucket}/{added['key']}/xl.meta" + for identity, entry in old.items(): + require(identity in new, 'I05 successor removed an original physical file') + if identity[2] != metadata_path: + require(new[identity] == entry, 'I05 successor changed an original shard or control') + target = map_target(after, bucket, added['key'], 'data', 1, added['versionId']) + old_dirs = set() + for entry in baseline['entries']: + if entry['key'] == added['key'] and not entry['deleteMarker']: + original = map_target(before, bucket, entry['key'], 'data', 1, entry['versionId'], latest_required=False) + old_dirs.add(original['dataDir']) + retained = map_target(after, bucket, entry['key'], 'data', 1, entry['versionId'], latest_required=False) + require(original['dataDir'] == retained['dataDir'] + and {name: value for name, value in original['logical'].items() if name != 'isLatest'} + == {name: value for name, value in retained['logical'].items() if name != 'isLatest'} + and [(p['index'], p['sha256'], p['size']) for p in original['cohort']] + == [(p['index'], p['sha256'], p['size']) for p in retained['cohort']], + 'I05 successor rewrote an acknowledged historical generation') + require(target['dataDir'] not in old_dirs and target['logical']['size'] == added['payload']['size'], + 'I05 successor reused an old physical generation or wrong logical size') + expected = sorted((entry['versionId'], entry['deleteMarker'], False if entry['key'] == added['key'] else entry['isLatest']) + for entry in baseline['entries'] if entry['key'] == added['key']) + expected.append((added['versionId'], False, True)) + for identity, entry in new.items(): + if identity[2] == metadata_path: + versions = entry['metadata']['versions'] + require(sorted((v['versionId'], v['deleted'], v['isLatest']) for v in versions) == sorted(expected), + 'I05 physical version population differs from original and successor ACKs') + if identity not in old: + prefix = f"{bucket}/{added['key']}/{target['dataDir']}/" + leaf = identity[2].removeprefix(prefix) + require(identity[2].startswith(prefix) + and (leaf == 'part.1' or leaf.startswith('part.1.integrity.') + and str(uuid.UUID(leaf.removeprefix('part.1.integrity.'))) + == leaf.removeprefix('part.1.integrity.')), + 'I05 successor left an unacknowledged physical key or generation') + return target + + +def observe_successor(probe, fleet, bucket, entry, output): + lock_observer.validate_scope(probe, fleet, bucket, entry) + require(fleet.runtime_profile == 'lock-successor', 'I05 queued successor requires its separate runtime budget') + output = Path(output) + output.mkdir(exist_ok=False) + bound = canonical(probe.binding) + marker = 'auto-testing-i05-successor/' + str(uuid.uuid4()) + save(output / 'scope.json', {'runId': fleet.run_id, 'entry': entry, 'marker': marker, + 'bindingSha256': hashlib.sha256(bound.encode()).hexdigest(), 'putTimeoutMs': PUT_TIMEOUT_MS, + 'queueSeconds': QUEUE_SECONDS, 'completeSeconds': COMPLETE_SECONDS, + 'canonicalCasePassClaimed': False}) + def binding(): + validate_binding(probe.binding) + require(canonical(probe.binding) == bound, 'I05 successor binding changed') + physical = fleet.snapshot('I05-successor-original-physical') + baseline = fleet.record('I05-successor-original-S3', probe.inventory(bucket)) + require_exact_inventory_entry(baseline, bucket, entry) + require(entry['payload']['size'] >= MIN_BACKPRESSURE_BYTES, + 'NOT_OBSERVED: I05 successor target cannot maintain response backpressure') + require(physical_content(physical) == physical_content(fleet.snapshot('I05-successor-baseline-after-reads')), + 'I05 successor baseline reads changed original physical content') + body = payload(f"{fleet.run_id}/I05-queued-successor", SUCCESSOR_BYTES) + successor = SuccessorProbe(probe, marker) + recorder = Recorder(successor, output / 'successor') + future, final, target, after_ack = None, None, None, None + def capture(window, sent, admitted): + binding() + rows = window.capture() + node, holder = admitted['node'], admitted['attempt'] + attempts = {host: successor_attempt(events, marker, holder, bucket, entry['key']) + for host, events in rows.items()} + require(all(value is None for host, value in attempts.items() if host != node), + 'I05 successor request appeared on another endpoint') + return rows[node], lock_observer.caller_attempt(rows[node], sent, bucket, entry['key']), attempts[node] + with ThreadPoolExecutor(max_workers=1) as pool: + def queued(sent, admitted, window): + nonlocal future + binding() + future = pool.submit(recorder.call, 'put_object', {'Bucket': bucket, 'Key': entry['key'], + 'ContentType': 'application/octet-stream', 'Metadata': {'heal-fixture': 'I05-queued-successor'}}, body) + deadline = time.monotonic() + QUEUE_SECONDS + while True: + _, holder, attempt = capture(window, sent, admitted) + require(holder is not None and holder['observedAcquiredWithoutDrop'], + 'NOT_OBSERVED: I05 original GET ended before successor queue observation') + if future.done(): + future.result() + raise ValueError('NOT_OBSERVED: I05 successor completed before caller disconnect') + require(time.monotonic() <= deadline, 'NOT_OBSERVED: I05 successor queue observation exceeded its budget') + if attempt is not None: + require(attempt['states'] == ['acquiring'], + 'I05 successor acquired before caller disconnect') + save(output / 'queued.json', attempt) + return attempt + time.sleep(.02) + def complete(sent, admitted, window): + nonlocal final, target, after_ack + require(sent.get('clientSocketClosed') is True and sent.get('shutdownCompleted') is True + and sent.get('resultUnknown') is True, 'I05 successor lost the original caller disconnect') + deadline = time.monotonic() + COMPLETE_SECONDS + while True: + rows, holder, attempt = capture(window, sent, admitted) + require(holder is not None, 'I05 successor lost its original GET request') + require(time.monotonic() <= deadline, 'NOT_OBSERVED: I05 successor completion exceeded its budget') + if future.done(): + future.result() + if attempt is not None and attempt['states'] == ['acquiring', 'acquired', 'guard_dropped']: + require(holder['diagnosticGuardDropped'], 'I05 successor acquired without original GET guard drop') + drop = next(i for i, row in enumerate(rows) if row is holder['events'][-1]) + acquire = next(i for i, row in enumerate(rows) if row is attempt['events'][1]) + require(drop < acquire, 'I05 successor acquired before original GET diagnostic release') + added = validate_successor_ack(recorder.output, probe, bucket, entry['key'], body, attempt) + require(not any(e['versionId'] == added['versionId'] for e in baseline['entries']), + 'I05 successor ACK reused an original version') + final = copy.deepcopy(baseline) + for item in final['entries']: + if item['key'] == entry['key']: + item['isLatest'] = False + final['entries'].append(added) + final['entries'].sort(key=lambda item: (item['key'], item['versionId'])) + after = fleet.snapshot('I05-successor-after-ACK-before-public-read') + after_ack = physical_content(after) + target = physical_append(physical, after, bucket, baseline, added) + fleet.record('I05-successor-new-shard-integrity', verify_cohort(fleet, target, 'I05-successor-shard')) + save(output / 'successor-complete.json', {**attempt, 'added': added, + 'queuedBeforeDisconnectObserved': True, 'originalDiagnosticDropBeforeAcquire': True, + 'heldAtDisconnectProven': False, 'canonicalCasePassClaimed': False}) + return attempt + time.sleep(.05) + try: + caller = lock_observer.observe_disconnect(probe, fleet, bucket, entry, output / 'caller', + while_admitted=queued, after_disconnect=complete, receive_buffer_bytes=CALLER_RECEIVE_BUFFER_BYTES) + before_reads = fleet.snapshot('I05-successor-before-all-peer-reads') + require(after_ack == physical_content(before_reads), + 'I05 successor first historical verification read changed physical content') + inventories = {} + for endpoint in probe.binding['endpoints']: + binding() + peer = Probe(probe.binding, endpoint, credential_pair=probe.credential_pair) + observed = fleet.record('I05-successor-all-peer-S3', {'endpoint': endpoint, **peer.inventory(bucket)}) + retained = [item for item in final['entries'] if item['versionId'] != target['versionId']] + actual_retained = {**observed, 'entries': [item for item in observed['entries'] + if item['versionId'] != target['versionId']]} + verify_seed(retained, actual_retained) + wanted = next(item for item in final['entries'] if item['versionId'] == target['versionId']) + actual = require_exact_inventory_entry(observed, bucket, wanted) + require(actual.get('isLatest') is True and actual.get('tags') == [] + and all(actual.get('properties', {}).get(name) == value + for name, value in wanted['properties'].items()), + 'I05 successor public content, metadata, tags or current version differs from its ACK') + inventories[endpoint] = observed + require(physical_content(before_reads) == physical_content(fleet.snapshot('I05-successor-after-all-peer-reads')), + 'I05 successor verification reads changed physical content') + deadline = time.monotonic() + lock_observer.DRAIN_SECONDS + while True: + binding() + resources = {node: fleet.record('I05-successor-final-resources', { + 'node': node, **fleet.remote(node, 'process-metrics')}) for node in fleet.nodes} + for node, resource in resources.items(): + lock_observer.same_process(resource.get('identity'), caller['baselineResources'][node]['identity']) + require(type(resource.get('fileDescriptors')) is int and resource['fileDescriptors'] >= 0, + 'I05 successor final descriptor census is incomplete') + require(time.monotonic() <= deadline, 'NOT_OBSERVED: I05 successor final resource budget expired') + if all(resources[node]['fileDescriptors'] <= caller['baselineResources'][node]['fileDescriptors'] + for node in fleet.nodes): + break + time.sleep(.2) + result = {'diagnosticComplete': True, 'callerObservation': caller, 'expectedEntries': final['entries'], + 'verifiedEndpoints': list(inventories), 'finalResources': resources, + 'heldAtDisconnectProven': False, 'distributedReleaseProven': False, + 'noGlobalFdLeakProven': False, 'wholeCasePassClaimed': False, 'canonicalCasePassClaimed': False} + save(output / 'diagnostic.json', result) + return result + except BaseException as error: + save(output / 'failure.json', {'diagnosticComplete': False, 'errorType': type(error).__name__, + 'state': 'NOT_OBSERVED' if str(error).startswith('NOT_OBSERVED:') else 'INCOMPLETE', + 'canonicalCasePassClaimed': False, 'originalRequestsAndResponsesRetained': True}) + raise + finally: + # The original caller is closed by observe_disconnect even on an + # observation failure. Drain the single SDK call before capture/stop. + if future is not None: + try: + future.result() + except BaseException: + pass diff --git a/scripts/validation_heal/lock_timeout_runner.py b/scripts/validation_heal/lock_timeout_runner.py index ccf1bd20..3df828d5 100644 --- a/scripts/validation_heal/lock_timeout_runner.py +++ b/scripts/validation_heal/lock_timeout_runner.py @@ -21,6 +21,9 @@ from .inspector import verify as verify_inspector from .lock_observer import CANDIDATE, DRAIN_SECONDS, same_process from .lock_timeout import MIN_BACKPRESSURE_BYTES, observe_timeout + from .lock_successor import observe_successor + from .race_native_evidence import verify as verify_native + from .timeout_retention import seal_successor_scene except ImportError: from heal_probe import Probe, compare_inventories, require, save, validate_binding from heal_workload import Recorder, seed, verify_seed @@ -28,10 +31,15 @@ from inspector import verify as verify_inspector from lock_observer import CANDIDATE, DRAIN_SECONDS, same_process from lock_timeout import MIN_BACKPRESSURE_BYTES, observe_timeout + from lock_successor import observe_successor + from race_native_evidence import verify as verify_native + from timeout_retention import seal_successor_scene NODES = ('vm000', 'vm001', 'vm002', 'vm003') CASE_ID = 'heal.lock-timeout-and-caller-abort' +NATIVE_I05_LABELS = ('versioned-read-repair-lock-owner', 'cancelled-physical-unlink-owner', + 'stale-read-repair-final-rename') def interrupted(signum, frame): @@ -62,20 +70,40 @@ def preflight(args): require(nodes[:4] == list(NODES), 'I05 requires the current vm000-vm003 Heal node binding') inspector = verify_inspector(args.tool_directory, candidate) require(args.rc_bin.is_file(), 'I05 requires the pinned client binary') + if getattr(args, 'diagnostic_branch', 'timeout') == 'queued-successor': + verify_native_prerequisite(args, candidate) return candidate, inspector +def verify_native_prerequisite(args, candidate): + directory = getattr(args, 'native_directory', None) + require(directory is not None, 'I05 queued-successor requires immutable exact-source native evidence') + proof = verify_native(directory, candidate, args.testing_commit, CASE_ID) + require(proof.get('requiredRegressionLabels') == list(NATIVE_I05_LABELS) + and proof.get('requiredNativeSubsetPassed') is True + and proof.get('wholeCasePassClaimed') is False + and proof.get('fleetExecutionClaimed') is False + and proof.get('canonicalCasePassClaimed') is False, + 'I05 queued-successor exact-source native prerequisite is incomplete') + return proof + + def run_topology(args, candidate, inspector, topology): require(topology in ('H1', 'H2'), 'I05 timeout supports only H1/H2') # Repeat the source/tool guard for callers which bypass main(). require(candidate.get('source_sha') == inspector.get('candidateCommit') == CANDIDATE, 'I05 timeout candidate and inspector differ') + branch = getattr(args, 'diagnostic_branch', 'timeout') + require(branch in {'timeout', 'queued-successor'}, 'I05 diagnostic branch is not implemented') + if branch == 'queued-successor': + verify_native_prerequisite(args, candidate) directory = args.evidence / topology directory.mkdir(parents=True, exist_ok=False) run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" nodes = list(NODES[:1] if topology == 'H1' else NODES) bucket = 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] - fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile='lock-timeout') + fleet = Fleet(nodes, directory, run_id, args.tool_directory, + runtime_profile='lock-successor' if branch == 'queued-successor' else 'lock-timeout') access, secret = secrets.token_hex(16), secrets.token_hex(24) fleet.secret_values.update((access, secret)) def error_info(error): @@ -95,8 +123,9 @@ def error_info(error): fleet.record('binding', binding) fleet.record('inspector-build', inspector) probe = Probe(binding, credential_pair=(access, secret)) - result = {'topology': topology, 'diagnosticComplete': False, 'cleanup': 'PENDING', + result = {'topology': topology, 'branch': branch, 'diagnosticComplete': False, 'cleanup': 'PENDING', 'wholeCasePassClaimed': False, 'canonicalCasePassClaimed': False} + retained_scene = None try: fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record('actual-topology', validate_topology(fleet.snapshot('deployed-topology'), topology)) @@ -107,7 +136,8 @@ def error_info(error): and row['payload']['size'] >= MIN_BACKPRESSURE_BYTES) baseline = fleet.record('original-ACK-bound-S3', probe.inventory(bucket)) fleet.record('original-ACK-proof', verify_seed(expected, baseline)) - diagnostic = observe_timeout(probe, fleet, bucket, target, directory / 'timeout') + controller = observe_successor if branch == 'queued-successor' else observe_timeout + diagnostic = controller(probe, fleet, bucket, target, directory / branch) require(diagnostic.get('diagnosticComplete') is True and diagnostic.get('canonicalCasePassClaimed') is False and diagnostic.get('wholeCasePassClaimed') is False, @@ -116,8 +146,17 @@ def error_info(error): for endpoint in binding['endpoints']: peer = Probe(binding, endpoint, credential_pair=(access, secret)) observed = fleet.record('all-peer-S3', {'endpoint': endpoint, **peer.inventory(bucket)}) - fleet.record('all-peer-ACK-proof', {'endpoint': endpoint, - **verify_seed(expected, observed), **compare_inventories(baseline, observed)}) + if branch == 'timeout': + fleet.record('all-peer-ACK-proof', {'endpoint': endpoint, + **verify_seed(expected, observed), **compare_inventories(baseline, observed)}) + else: + # The successor controller retains exact public properties and + # separately verifies its new ACK before these outer reads. + require(endpoint in diagnostic['verifiedEndpoints'], 'I05 successor omitted a peer') + wanted = diagnostic['expectedEntries'] + wanted = [{name: value for name, value in item.items() if name not in {'properties', 'tags'}} + for item in wanted] + fleet.record('all-peer-ACK-proof', {'endpoint': endpoint, **verify_seed(wanted, observed)}) require(physical_content(physical) == physical_content(fleet.snapshot('physical-after-all-peer-reads')), 'I05 peer verification changed the diagnostic physical scene') baseline_resources = diagnostic['callerObservation']['baselineResources'] @@ -153,12 +192,18 @@ def error_info(error): try: if fleet.reserved: fleet.stop() - fleet.snapshot('original-failure-scene', quiescent=True) + retained_scene = fleet.snapshot('original-failure-scene', quiescent=True) for node in fleet.reserved: fleet.record('original-failure-logs', {'node': node, **fleet.remote(node, 'logs')}) except (Exception, KeyboardInterrupt) as error: result['failureCaptureError'] = error_info(error) fleet.record('diagnostic-result', result) + if branch == 'queued-successor' and retained_scene is not None and 'failureCaptureError' not in result: + try: + save(directory / 'successor-retention.json', seal_successor_scene(directory, binding, retained_scene)) + except (Exception, KeyboardInterrupt) as error: + result['retentionCaptureError'] = error_info(error) + fleet.record('retention-capture-error', result['retentionCaptureError']) return result @@ -167,6 +212,8 @@ def run(args): require(isinstance(args.binary_sha, str) and re.fullmatch(r'[0-9a-f]{64}', args.binary_sha), 'I05 deployment requires the installed binary digest') candidate, inspector = preflight(args) + branch = getattr(args, 'diagnostic_branch', 'timeout') + native_proof = verify_native_prerequisite(args, candidate) if branch == 'queued-successor' else None # run_case.sh creates this parent and opens test.log before invoking us. # Reuse only those wrapper files; a prior topology or summary is a replay. require(not args.evidence.is_symlink(), 'I05 evidence directory must not be a symlink') @@ -174,6 +221,8 @@ def run(args): require(all(path.name in {'timeout-preflight.json', 'setup.log', 'test.log'} and path.is_file() and not path.is_symlink() for path in args.evidence.iterdir()), 'I05 requires fresh topology and summary evidence before deployment') + if native_proof is not None: + save(args.evidence / 'native-prerequisite.json', native_proof) results = [] for topology in args.topologies: result = run_topology(args, candidate, inspector, topology) @@ -181,11 +230,12 @@ def run(args): if not result['diagnosticComplete'] or result['cleanup'] != 'PASS': break summary = {'diagnostics': results, 'requestedTopologies': args.topologies, + 'diagnosticBranch': getattr(args, 'diagnostic_branch', 'timeout'), 'diagnosticComplete': len(results) == len(args.topologies) and all(row['diagnosticComplete'] and row['cleanup'] == 'PASS' for row in results), 'wholeCasePassClaimed': False, 'canonicalCasePassClaimed': False, 'remaining': ['exact disconnect-edge ownership and complete native timeout/stale fences', - 'stale-generation and queued-successor controllers', + 'stale-generation controller and independent queued-successor live acceptance', 'complete H1/H2 source DL/DV ACK, physical, lock and FD acceptance']} save(args.evidence / 'timeout-diagnostic-summary.json', summary) save(args.evidence / 'healthy-summary.json', {**summary, 'caseId': CASE_ID, 'complete': False, @@ -208,11 +258,16 @@ def main(): parser.add_argument('--rc-bin', type=Path, required=True) parser.add_argument('--evidence', type=Path, required=not preflight_only) parser.add_argument('--topologies', nargs='+', choices=('H1', 'H2'), default=['H1', 'H2']) + parser.add_argument('--diagnostic-branch', choices=('timeout', 'queued-successor'), + default=os.environ.get('I05_DIAGNOSTIC_BRANCH', 'timeout')) + parser.add_argument('--native-directory', type=Path, default=os.environ.get('VALIDATION_HEAL_NATIVE_DIRECTORY')) args = parser.parse_args(argv) require(args.topologies in (['H1'], ['H2'], ['H1', 'H2']), 'I05 topology order must be H1 then H2') if preflight_only: preflight(args) print(json.dumps({'preflightPassed': True, 'candidateCommit': CANDIDATE, + 'diagnosticBranch': args.diagnostic_branch, + 'nativePrerequisitePassed': args.diagnostic_branch == 'queued-successor', 'diagnosticOnly': True, 'canonicalCasePassClaimed': False})) return 0 signal.signal(signal.SIGTERM, interrupted) diff --git a/scripts/validation_heal/race_retention.py b/scripts/validation_heal/race_retention.py index e0769acb..d14a27c4 100644 --- a/scripts/validation_heal/race_retention.py +++ b/scripts/validation_heal/race_retention.py @@ -30,7 +30,7 @@ def validate(plan): fields = {'kind', 'runId', 'nodes', 'bucket', 'owners', 'businessSceneSha256', 'candidateCommit', 'harnessCommit', 'binarySha256', 'ackFilesSha256'} require(set(plan) == fields | ({'mpuKeys', 'mpuSceneSha256'} if multipart else set()) - and plan['kind'] in ({kind for kind, _ in CASES.values()} | {'unresolved-i05-timeout'}) + and plan['kind'] in ({kind for kind, _ in CASES.values()} | {'unresolved-i05-timeout', 'unresolved-i05-successor'}) and re.fullmatch(r'gh-[1-9][0-9]*-[1-9][0-9]*-h[12]', plan['runId']) and len(plan['nodes']) == len(set(plan['nodes'])) == (1 if plan['runId'].endswith('-h1') else 4) and all(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]*', node) for node in plan['nodes']) @@ -43,7 +43,9 @@ def validate(plan): owner = plan['owners'][node] require(owner.get('runId') == plan['runId'] and owner.get('bucket') == plan['bucket'] and (owner.get('runtimeProfile') == 'lock-timeout' and plan['kind'] == 'unresolved-i05-timeout' or - owner.get('runtimeProfile') == 'manual' and plan['kind'] != 'unresolved-i05-timeout' or + owner.get('runtimeProfile') == 'lock-successor' and plan['kind'] == 'unresolved-i05-successor' or + owner.get('runtimeProfile') == 'manual' + and plan['kind'] not in {'unresolved-i05-timeout', 'unresolved-i05-successor'} or owner.get('runtimeProfile') == 'frozen-race-legacy' and plan['kind'] in {'unresolved-g04-race', 'unresolved-g05-race'}) and owner.get('binarySha256') == plan['binarySha256'] diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index e44a15a1..f278f4f2 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -3253,7 +3253,7 @@ "dispatchable": false, "code": "I05", "reasonCode": "NATIVE_LOCK_OWNER_ABORT_BARRIER", - "reason": "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 单连接历史 GET、真实争用 PUT 超时、服务 admission/owner/socket 和有界 FD 收尾诊断已通过独立 live 审计。仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation 与 queued-successor controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。", + "reason": "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 单连接历史 GET、真实争用 PUT 超时、服务 admission/owner/socket 和有界 FD 收尾诊断已通过独立 live 审计。已实现独立 queued-successor controller、候选绑定的 native 前置及失败现场隔离，尚缺该分支的独立 live 验收；仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。", "requiredCapabilities": [ "server_lock_owner", "lock_timeout_fence", diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index ed497842..08666547 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -656,7 +656,7 @@ def main(): if plan.get("buckets"): fleet.snapshot_buckets = plan["buckets"] fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) - if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race", "unresolved-a09-race", "unresolved-g04-race", "unresolved-g05-race", "unresolved-g06-race", "unresolved-i05-timeout"}: + if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race", "unresolved-a09-race", "unresolved-g04-race", "unresolved-g05-race", "unresolved-g06-race", "unresolved-i05-timeout", "unresolved-i05-successor"}: complete.append({"failedRunId": args.failed_run_id, **isolate_race_scene(fleet, plan)}) continue if plan.get("kind") == "unresolved-network-load": diff --git a/scripts/validation_heal/retained_scene.py b/scripts/validation_heal/retained_scene.py index bbbb71e6..d034054c 100644 --- a/scripts/validation_heal/retained_scene.py +++ b/scripts/validation_heal/retained_scene.py @@ -180,7 +180,7 @@ def mounted_paths(): def inspect(request, node): state, roots = node["paths"](request["runId"]) expected = request["expectedOwner"] - per_run = expected.get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout"} + per_run = expected.get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout", "lock-successor"} retained_state, retained_roots = destinations(state, roots, per_run_parent=per_run) if per_run: old_state, old_roots = destinations(state, roots) @@ -190,7 +190,7 @@ def inspect(request, node): require(isinstance(expected, dict) and expected.get("runId") == request["runId"] and expected.get("unit") == node["unit"](request["runId"]) and expected.get("hostname") == node["socket"].gethostname() - and expected.get("runtimeProfile") in {"mrf-crash", "root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout"}, "retained scene archived owner differs") + and expected.get("runtimeProfile") in {"mrf-crash", "root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout", "lock-successor"}, "retained scene archived owner differs") current = node["mounts"]() require(len(current) == len(expected.get("mounts", [])) == 4 and all(all(actual[key] == old[key] for key in ("target", "source", "fstype", "uuid", "maj:min")) @@ -234,7 +234,7 @@ def inspect(request, node): intent = retained_state.parent / (request["runId"] + ".intent.json") node["checked"](intent, state.parent) network = expected["runtimeProfile"] == "lock-quorum" - if expected["runtimeProfile"] in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout"}: + if expected["runtimeProfile"] in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout", "lock-successor"}: original_sha = request.get("expectedOriginalSceneSha256", "") require(re.fullmatch(r"[0-9a-f]{64}", original_sha) is not None, "restored fixture requires its original physical scene digest") @@ -270,7 +270,7 @@ def inspect(request, node): def run(request, node): fields = {"runId", "action", "expectedOwner"} - if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout"}: + if request.get("expectedOwner", {}).get("runtimeProfile") in {"root-restart", "outcome-grace", "outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout", "lock-successor"}: fields.add("expectedOriginalSceneSha256") if request.get("expectedOwner", {}).get("runtimeProfile") == "lock-quorum" or ( request.get('expectedOwner', {}).get('runtimeProfile') in {'manual', 'frozen-race-legacy'} @@ -309,7 +309,7 @@ def run(request, node): "originalSceneRestored": False, "canonicalPassClaimed": False, "backupRestorationUsed": False, "credentialContentsRead": False, "inspectionOnly": request["action"] == "retained-scene-audit", "mutated": mutated, "activeNamespaceIsolated": "active" not in locations} - if request["expectedOwner"].get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout"}: + if request["expectedOwner"].get("runtimeProfile") in {"outcome-legacy", "lock-quorum", "manual", "frozen-race-legacy", "lock-timeout", "lock-successor"}: result["privateArchiveParents"] = [] for target in (*retained_roots, retained_state): info = target.parent.lstat() if target.parent.exists() else None diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 0d5c4539..9a7f3cc0 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -7,6 +7,7 @@ ASSUME_YES=1 candidate="" testing_commit="" artifacts="" case_id="" driver=healthy.py environment_gate=0 +i05_args=() while [ "$#" -gt 0 ]; do case "$1" in --validation-case) case_id="$2"; shift 2 ;; @@ -94,6 +95,17 @@ tool_args=() if [ -n "${VALIDATION_HEAL_TOOL_DIRECTORY:-}" ]; then tool_args=(--tool-directory "${VALIDATION_HEAL_TOOL_DIRECTORY}") fi +if [ "${case_id}" = heal.lock-timeout-and-caller-abort ]; then + diagnostic_branch="${I05_DIAGNOSTIC_BRANCH:-timeout}" + case "${diagnostic_branch}" in + timeout|queued-successor) ;; + *) die "unsupported I05 diagnostic branch: ${diagnostic_branch}" ;; + esac + i05_args+=(--diagnostic-branch "${diagnostic_branch}") + if [ -n "${VALIDATION_HEAL_NATIVE_DIRECTORY:-}" ]; then + i05_args+=(--native-directory "${VALIDATION_HEAL_NATIVE_DIRECTORY}") + fi +fi [ "${driver}" = healthy.py ] || [ "${#tool_args[@]}" -gt 0 ] || die "physical Heal cases require candidate-bound bitrot tools" evidence="${artifacts}/evidence/${case_id}" mkdir -p "${artifacts}/cases" "${evidence}" @@ -105,7 +117,7 @@ fi if [ "${driver}" = lock_timeout_runner.py ]; then python3 -B "${ROOT}/scripts/validation_heal/${driver}" preflight \ --candidate "${candidate}" --case-id "${case_id}" --testing-commit "${testing_commit}" \ - "${tool_args[@]}" --rc-bin "$(command -v rc)" > "${evidence}/timeout-preflight.json" + "${tool_args[@]}" "${i05_args[@]}" --rc-bin "$(command -v rc)" > "${evidence}/timeout-preflight.json" fi python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ --result-file "${artifacts}/cases/${case_id}.json" --verdict RUNNING --cleanup-status PENDING @@ -184,7 +196,7 @@ if [ "${bootstrap_rc}" -eq 0 ]; then done python3 -B "${ROOT}/scripts/validation_heal/${driver}" --candidate "${candidate}" \ --case-id "${case_id}" \ - "${tool_args[@]}" \ + "${tool_args[@]}" "${i05_args[@]}" \ --testing-commit "${testing_commit}" --binary-sha "${binary_sha}" --rc-bin "$(command -v rc)" \ --evidence "${evidence}" > "${evidence}/test.log" 2>&1 || case_rc=$? cat "${evidence}/test.log" @@ -231,7 +243,7 @@ if [ "${case_id}" = heal.lock-timeout-and-caller-abort ]; then if [ "${cleanup_status}" = PASS ] && [ "${case_rc}" -eq 0 ] && \ jq -e '.diagnosticComplete == true and .complete == false and .canonicalCasePassClaimed == false' \ "${evidence}/healthy-summary.json" >/dev/null 2>&1; then - verdict=BLOCKED; reason="I05 timeout diagnostic completed; exact native fences, stale-generation/queued-successor controllers and complete source H1/H2 DL/DV acceptance remain required" + verdict=BLOCKED; reason="I05 ${diagnostic_branch:-timeout} diagnostic completed; exact disconnect ownership, remaining native fences, stale-generation controller and complete source H1/H2 DL/DV acceptance remain required" elif [ "${case_rc}" -eq 0 ]; then verdict=INFRA_ERROR; reason="I05 diagnostic summary or cleanup is incomplete; canonical PASS is prohibited" fi diff --git a/scripts/validation_heal/timeout_retention.py b/scripts/validation_heal/timeout_retention.py index 36f7815a..9ae7705f 100644 --- a/scripts/validation_heal/timeout_retention.py +++ b/scripts/validation_heal/timeout_retention.py @@ -317,6 +317,120 @@ def one(label): return [plan] +def successor_files(folder): + """Fingerprint the complete stopped diagnostic, including unknown receipts.""" + require(folder.is_dir() and not folder.is_symlink(), 'I05 successor evidence root differs') + files, total = {}, 0 + for path in sorted(folder.rglob('*')): + require(not path.is_symlink() and (path.is_dir() or path.is_file()), + 'I05 successor evidence contains a link or special file') + if not path.is_file() or path == folder / 'successor-retention.json': + continue + size = path.stat().st_size + total += size + require(size <= 32 * 1024**2 and total <= 256 * 1024**2 and len(files) < 4096, + 'I05 successor evidence exceeds its preservation budget') + files[path.relative_to(folder).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + require(files, 'I05 successor has no original evidence') + return files + + +def successor_plan(folder, binding, scene): + """Preservation needs exact stopped bytes, never a successful PUT outcome.""" + topology = 'H1' if binding['runId'].endswith('-h1') else 'H2' + nodes = ['vm000'] if topology == 'H1' else ['vm000', 'vm001', 'vm002', 'vm003'] + reservations = [json.loads(path.read_text()) for path in folder.glob('*-reserved.json')] + owners = {row.get('node'): row.get('owner') for row in reservations} + require(len(reservations) == len(nodes) and set(owners) == set(scene) == set(nodes) + and all(row.get('reserved') is True for row in reservations) + and binding.get('candidateCommit') == CANDIDATE + and binding.get('endpoints') == {f'node{i}': f'http://rustfs-node{i}:9000' + for i in range(1, len(nodes) + 1)} + and binding.get('bucketPrefix') == 'rv-heal-' + hashlib.sha256(binding['runId'].encode()).hexdigest()[:16], + 'I05 successor preservation requires the complete original fleet binding') + for node, owner in owners.items(): + original = scene[node] + require(owner.get('runtimeProfile') == 'lock-successor' + and original.get('quiescent') is True and type(original.get('pid')) is int and original['pid'] == 0 + and original.get('hostname') == owner.get('hostname') + and original.get('unit') == owner.get('unit') + and original.get('binarySha256') == owner.get('binarySha256') == binding['candidateBinarySha256'] + and len(original.get('formats', [])) == 4 + and {part.get('disk') for part in original['formats']} == set(range(4)), + 'I05 successor requires the stopped original owner and all four disks') + plan = {'kind': 'unresolved-i05-successor', 'runId': binding['runId'], 'nodes': nodes, + 'bucket': binding['bucketPrefix'], 'owners': owners, 'candidateCommit': CANDIDATE, + 'harnessCommit': binding['harnessCommit'], 'binarySha256': binding['candidateBinarySha256'], + 'ackFilesSha256': digest(successor_files(folder)), + 'businessSceneSha256': {node: digest(value) for node, value in physical_content(scene).items()}} + validate(plan) + return plan + + +def seal_successor_scene(folder, binding, scene): + files = successor_files(folder) + plan = successor_plan(folder, binding, scene) + require(digest(files) == plan['ackFilesSha256'], 'I05 successor evidence changed while sealing') + return {'schemaVersion': 1, 'plan': plan, 'originalFilesSha256': files, + 'candidatePackageSha256': binding['candidatePackageSha256'], + 'canonicalCasePassClaimed': False, 'unknownMutationMayBePresent': True, + 'namespaceDeletionPermitted': False, 'backupRestorationPermitted': False} + + +def successor_archived_plan(result_path, failed_run, candidate): + result = json.loads(result_path.read_text()) + provenance = result.get('provenance') or {} + evidence = result_path.parents[1] / 'evidence' / CASE + summary = json.loads((evidence / 'healthy-summary.json').read_text()) + rows = summary.get('diagnostics', []) + require(result.get('case_id') == CASE and result.get('verdict') == result.get('cleanup_status') == 'FAIL' + and summary.get('caseId') == CASE and summary.get('diagnosticBranch') == 'queued-successor' + and summary.get('cleanup') == 'FAIL' and summary.get('complete') is False + and summary.get('diagnosticComplete') is False + and summary.get('canonicalCasePassClaimed') is False and summary.get('wholeCasePassClaimed') is False + and isinstance(rows, list) and 1 <= len(rows) <= 2 + and [row.get('topology') for row in rows] in (['H1'], ['H2'], ['H1', 'H2']) + and [row['topology'] for row in rows] == summary.get('requestedTopologies', [])[:len(rows)] + and all(row.get('diagnosticComplete') is True and row.get('cleanup') == 'PASS' for row in rows[:-1]) + and rows[-1].get('diagnosticComplete') is False and rows[-1].get('cleanup') == 'FAIL' + and rows[-1].get('namespacesRetained') is True + and 'failureCaptureError' not in rows[-1] and 'retentionCaptureError' not in rows[-1] + and all(row.get('branch') == 'queued-successor' and row.get('canonicalCasePassClaimed') is False + and row.get('wholeCasePassClaimed') is False for row in rows), + 'I05 successor recovery requires one retained diagnostic failure after clean predecessors') + require({p.name for p in evidence.iterdir() if p.is_dir()} == {row['topology'] for row in rows}, + 'I05 successor recovery cannot omit another topology') + for row in rows: + found = list((evidence / row['topology']).glob('*-diagnostic-result.json')) + require(len(found) == 1 and json.loads(found[0].read_text()) == row, + 'I05 successor summary differs from its original diagnostic result') + folder = evidence / rows[-1]['topology'] + def one(label): + paths = list(folder.glob('*-' + label + '.json')) + require(len(paths) == 1, 'I05 successor lacks one original ' + label) + return json.loads(paths[0].read_text()) + manifest_path = folder / 'successor-retention.json' + require(manifest_path.is_file() and not manifest_path.is_symlink() + and manifest_path.stat().st_size <= 2 * 1024**2, 'I05 successor retention seal is missing or unsafe') + seal = json.loads(manifest_path.read_text()) + require(seal.get('schemaVersion') == 1 and seal.get('canonicalCasePassClaimed') is False + and seal.get('unknownMutationMayBePresent') is True + and seal.get('namespaceDeletionPermitted') is False and seal.get('backupRestorationPermitted') is False + and seal.get('originalFilesSha256') == successor_files(folder), + 'I05 successor original evidence differs from its retention seal') + binding, scene = one('binding'), one('original-failure-scene') + require(re.fullmatch(rf'gh-{failed_run}-[1-9][0-9]*-h[12]', binding.get('runId', '')) + and candidate.get('source_sha') == binding.get('candidateCommit') == provenance.get('source_commit') == CANDIDATE + and candidate.get('package_sha256') == binding.get('candidatePackageSha256') + == provenance.get('candidate_sha256') == seal.get('candidatePackageSha256') + and binding.get('harnessCommit') == provenance.get('testing_commit') + and binding.get('candidateBinarySha256') == provenance.get('binary_sha256'), + 'I05 successor archive changed run, candidate, helper or binary') + plan = successor_plan(folder, binding, scene) + require(seal.get('plan') == plan, 'I05 successor preservation plan differs from original evidence') + return [plan] + + def archived_plan(artifact, failed_run, candidate): """Refuse uncertain effects, later actions, mixed cases and physical drift. @@ -332,6 +446,9 @@ def read(path): return json.loads(path.read_text()) result = read(paths[0]) provenance = result.get('provenance') or {} + summary_path = paths[0].parents[1] / 'evidence' / CASE / 'healthy-summary.json' + if read(summary_path).get('diagnosticBranch') == 'queued-successor': + return successor_archived_plan(paths[0], failed_run, candidate) if provenance.get('testing_commit') == FILTER_STARTUP_HELPER: return filter_startup_plan(paths[0], failed_run, candidate) if provenance.get('testing_commit') == H2_FILTER_STARTUP_HELPER: diff --git a/test_validation_heal_lock_successor.py b/test_validation_heal_lock_successor.py new file mode 100644 index 00000000..91b97516 --- /dev/null +++ b/test_validation_heal_lock_successor.py @@ -0,0 +1,270 @@ +"""I05 successor admission, one ACK, physical generations and failure retention.""" + +import copy +from datetime import datetime, timedelta, timezone +import hashlib +import json +from pathlib import Path +import tempfile +import threading +import time +import unittest +from unittest.mock import patch +import uuid + +from scripts.validation_heal import fleet_node, lock_successor as successor +from scripts.validation_heal.heal_workload import Recorder +import test_validation_heal_lock_observer as fixture +import test_validation_heal as physical_fixture + + +REQUEST = '30000000-0000-4000-8000-000000000001' +ATTEMPT = '40000000-0000-4000-8000-000000000001' +VERSION = '50000000-0000-4000-8000-000000000001' +MARKER = 'auto-testing-i05-successor/' + REQUEST +HOLDER = {'request_id': fixture.REQUEST, 'lock_attempt_id': fixture.ATTEMPT, + 'owner': 'owner-one', 'namespace': 'namespace-one'} + + +def events(marker=MARKER, states=('acquiring',)): + return [{'target': fixture.observer.TARGET, 'event': 'object_namespace_lock', + 'component': 'ecstore', 'subsystem': 'object_lock', 'state': state, 'level': 'TRACE', + 'op': 'put_object_commit', 'mode': 'write', 'bucket': fixture.BUCKET, 'object': 'target.bin', + 'namespace': 'namespace-one', 'owner': 'owner-one', 'lock_attempt_id': ATTEMPT, + 'request_id': REQUEST, 'timeout_ms': successor.PUT_TIMEOUT_MS, + 'acquire_ms': 0 if state == 'acquiring' else 15, 'hold_ms': 0, + 'span': {'name': 'http-request', 'user_agent': 'SDK ' + marker, + 'request_id': REQUEST, 'method': 'PUT', 'uri': f'/{fixture.BUCKET}/target.bin'}} + for state in states] + + +def observation(probe, params, body): + return {'operation': 'put_object', 'params': params, 'endpoint': probe.endpoint_name, + 'payload': {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}, + 'clientCallInvoked': True, 'acknowledged': True, 'resultUnknown': False, + 'clockDomain': successor.CLOCK_DOMAIN, 'callStartedMonotonicNs': time.monotonic_ns(), + 'callFinishedMonotonicNs': time.monotonic_ns(), 'response': { + 'ResponseMetadata': {'HTTPStatusCode': 200, 'RetryAttempts': 0, 'RequestId': REQUEST}, + 'VersionId': VERSION, 'ETag': '"' + hashlib.md5(body, usedforsecurity=False).hexdigest() + '"'}} + + +class SuccessorEvidenceTests(unittest.TestCase): + def test_exact_request_queue_and_complete_lifecycle(self): + for states in (('acquiring',), ('acquiring', 'acquired'), ('acquiring', 'acquired', 'guard_dropped')): + proof = successor.successor_attempt(events(states=states), MARKER, HOLDER, fixture.BUCKET, 'target.bin') + self.assertEqual(proof['states'], list(states)) + self.assertFalse(proof['canonicalCasePassClaimed']) + self.assertIsNone(successor.successor_attempt([], MARKER, HOLDER, fixture.BUCKET, 'target.bin')) + + def test_forged_request_owner_version_budget_and_duplicate_attempt_refuse(self): + mutations = [lambda r: r.update(request_id=fixture.REQUEST), lambda r: r.update(owner='foreign'), + lambda r: r.update(namespace='foreign'), lambda r: r.update(timeout_ms=1000), + lambda r: r.update(requested_version_id=VERSION), lambda r: r.update(object='other.bin'), + lambda r: r.update(state='failed'), lambda r: r.update(acquire_ms=True), + lambda r: r['span'].update(method='GET'), lambda r: r['span'].update(uri='/other'), + lambda r: r.update(lock_attempt_id=fixture.ATTEMPT)] + for mutate in mutations: + rows = events(); mutate(rows[0]) + with self.subTest(mutate=mutate), self.assertRaises(ValueError): + successor.successor_attempt(rows, MARKER, HOLDER, fixture.BUCKET, 'target.bin') + for states in (('acquired',), ('acquiring', 'acquiring'), ('acquiring', 'guard_dropped')): + with self.subTest(states=states), self.assertRaises(ValueError): + successor.successor_attempt(events(states=states), MARKER, HOLDER, fixture.BUCKET, 'target.bin') + rows = events(); other = copy.deepcopy(rows[0]); other['request_id'] = VERSION + other['span']['request_id'] = VERSION; rows.append(other) + with self.assertRaisesRegex(ValueError, 'retried or reused'): + successor.successor_attempt(rows, MARKER, HOLDER, fixture.BUCKET, 'target.bin') + + def test_real_recorder_ack_binding_rejects_unknown_retry_wrong_content_and_time(self): + with tempfile.TemporaryDirectory() as folder: + probe = type('Probe', (), {'binding': { + 'schemaVersion': 1, 'runId': 'gh-12345-1-h1', 'jumpHostOnly': True, 'exclusiveHealService': True, + 'leaseExpiresAt': (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(), + 'candidateCommit': 'a' * 40, 'harnessCommit': 'b' * 40, + 'candidatePackageSha256': 'c' * 64, 'candidateBinarySha256': 'd' * 64, 'rcBinarySha256': 'e' * 64, + 'bucketPrefix': fixture.BUCKET, 'defaultEndpoint': 'node1', + 'endpoints': {'node1': 'http://127.0.0.1:9000'}, + 'maxInventoryObjects': 32, 'maxInventoryBytes': 512 * 1024**2}, 'endpoint_name': 'node1'})() + probe.s3 = lambda op, params, body: observation(probe, params, body) + recorder = Recorder(probe, Path(folder) / 'receipt') + body = b'one-successor' + recorder.call('put_object', {'Bucket': fixture.BUCKET, 'Key': 'target.bin', + 'ContentType': 'application/octet-stream', 'Metadata': {'heal-fixture': 'I05-queued-successor'}}, body) + attempt = {'requestId': REQUEST} + proof = successor.validate_successor_ack(recorder.output, probe, fixture.BUCKET, 'target.bin', body, attempt) + self.assertEqual(proof['versionId'], VERSION) + ack_path = recorder.output / '000001-ack.json'; original = json.loads(ack_path.read_text()) + for mutate in (lambda a: a['observation'].update(resultUnknown=True), + lambda a: a['observation']['response']['ResponseMetadata'].update(RetryAttempts=1), + lambda a: a['observation']['response'].update(ETag='"wrong"'), + lambda a: a['observation']['response'].update(VersionId='null'), + lambda a: a['observation']['response']['ResponseMetadata'].update(RequestId=fixture.REQUEST), + lambda a: a.update(requestSha256='0' * 64), + lambda a: a.update(recordedMonotonicNs=1)): + value = copy.deepcopy(original); mutate(value); ack_path.write_text(json.dumps(value)) + with self.subTest(mutate=mutate), self.assertRaises(ValueError): + successor.validate_successor_ack(recorder.output, probe, fixture.BUCKET, 'target.bin', body, attempt) + + def scene(self): + before = physical_fixture.HealShardOracleTest().snapshot() + original = {'key': 'target.bin', 'versionId': fixture.VERSION, 'deleteMarker': False, 'isLatest': True} + added = {'key': 'target.bin', 'versionId': VERSION, 'deleteMarker': False, 'isLatest': True, + 'payload': {'size': 1024, 'sha256': 'a' * 64}} + for value in before.values(): + keep = [] + for metadata, part in zip(value['files'][::3], value['files'][1::3], strict=True): + metadata = copy.deepcopy(metadata) + metadata['path'] = f'{fixture.BUCKET}/target.bin/xl.meta' + metadata['metadata']['key'] = 'target.bin' + version = metadata['metadata']['versions'][0] + version.update(versionId=fixture.VERSION, isLatest=True) + keep += [metadata, {**part, 'path': f"{fixture.BUCKET}/target.bin/{version['dataDir']}/part.1"}] + value['files'] = keep + after = copy.deepcopy(before) + for value in after.values(): + new_parts = [] + for metadata, part in zip(value['files'][::2], value['files'][1::2], strict=True): + old = metadata['metadata']['versions'][0]; old['isLatest'] = False + new = copy.deepcopy(old); new.update(versionId=VERSION, isLatest=True, size=1024, + dataDir=str(uuid.UUID(int=777))) + new['parts'] = [{**p, 'size': 1024, 'actualSize': 1024} for p in new['parts']] + metadata['metadata']['versions'].append(new) + new_parts.append({**part, 'path': f"{fixture.BUCKET}/target.bin/{new['dataDir']}/part.1", + 'sha256': part['sha256'] + '-new'}) + value['files'] += new_parts + return before, after, {'entries': [original]}, added + + def test_real_physical_map_preserves_old_generation_and_exact_version_population(self): + before, after, baseline, added = self.scene() + target = successor.physical_append(before, after, fixture.BUCKET, baseline, added) + self.assertEqual(target['versionId'], VERSION) + for mutate in (lambda s: s['files'].pop(), + lambda s: s['files'][1].update(sha256='wrong'), + lambda s: s['files'][0]['metadata']['versions'].pop(0), + lambda s: s['files'].append({'disk': 0, 'path': 'unacknowledged', 'sha256': 'wrong'})): + changed = copy.deepcopy(after); mutate(next(iter(changed.values()))) + with self.subTest(mutate=mutate), self.assertRaises(ValueError): + successor.physical_append(before, changed, fixture.BUCKET, baseline, added) + + def test_runtime_keeps_one_second_timeout_separate_from_sixty_second_put_queue(self): + timeout = fleet_node.runtime_config('lock-timeout') + queued = fleet_node.runtime_config('lock-successor') + self.assertNotIn('RUSTFS_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS', timeout) + self.assertEqual(queued.pop('RUSTFS_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS'), '60000') + self.assertEqual(queued, timeout) + + +class SuccessorFlowTests(unittest.TestCase): + def setUp(self): + self.scene = fixture.ObserverFlowTests('test_complete_component_keeps_original_unknown_and_all_missing_case_proofs') + self.scene.setUp(); self.addCleanup(self.scene.doCleanups) + self.probe, self.fleet, self.entry = self.scene.probe, self.scene.fleet, self.scene.entry + self.fleet.runtime_profile = 'lock-successor' + self.entry['payload']['size'] = successor.MIN_BACKPRESSURE_BYTES + self.probe.credential_pair = ('test-access', 'test-secret') + self.change = None + self.done = threading.Event(); self.calls = 0; self.labels = [] + self.final_entry = None; self.original = self.inventory(fixture.BUCKET) + self.trace = [] + self.output = self.scene.root / 'successor' + owner = self + class Writer(successor.Probe): + def __init__(self, probe, marker): + self.binding = probe.binding; self.endpoint_name = probe.endpoint_name + self.marker = marker + def s3(self, operation, params, body): + owner.calls += 1 + owner.scene.contender_rows = events(self.marker, + ('acquiring', 'acquired') if owner.change == 'early-acquire' else ('acquiring',)) + if not owner.scene.wire.server.disconnected.wait(3): + raise ValueError('caller was never disconnected') + if owner.change == 'unknown': + return {'acknowledged': False, 'resultUnknown': True, 'response': None} + owner.scene.contender_rows = events(self.marker, ('acquiring', 'acquired', 'guard_dropped')) + response = observation(self, params, body) + owner.final_entry = {'key': params['Key'], 'versionId': VERSION, + 'deleteMarker': False, 'isLatest': True, 'payload': response['payload'], + 'properties': {'ETag': response['response']['ETag'], 'ContentLength': len(body), + 'ContentType': params['ContentType'], 'Metadata': params['Metadata']}, 'tags': []} + owner.done.set() + return response + peer = type('Peer', (), {'inventory': lambda _, bucket: owner.inventory(bucket)}) + for name, value in [('SuccessorProbe', Writer), ('Probe', lambda *a, **k: peer()), + ('physical_append', self.physical), ('verify_cohort', lambda *a: {'allShardBitrotVerified': True})]: + p = patch.object(successor, name, value); p.start(); self.addCleanup(p.stop) + self.probe.inventory = self.inventory + def snapshot(label): + self.trace.append(label) + changed = self.change == 'historical-read-mutation' and label == 'I05-successor-before-all-peer-reads' + return {'vm000': {'files': ['unexpected-read-repair'] if changed else [], 'formats': []}} + self.fleet.snapshot = snapshot + original_remote = self.fleet.remote + def remote(node, action, **params): + value = original_remote(node, action, **params) + if action in {'process-metrics', 'caller-socket'}: + if action == 'caller-socket': + value['process']['settings'] = fleet_node.runtime_config('lock-successor') + else: + value['identity']['settings'] = fleet_node.runtime_config('lock-successor') + return value + self.fleet.remote = remote + + def inventory(self, bucket): + entries = [{**copy.deepcopy(self.entry), 'isLatest': self.final_entry is None}] + if self.final_entry is not None: + entries.append(copy.deepcopy(self.final_entry)) + return {'bucket': bucket, 'entries': entries, 'objectsAndMarkers': len(entries), + 'bytesRead': sum(row['payload']['size'] for row in entries), + 'pagination': {'versionPageSizes': [len(entries)], 'currentPageSizes': [1]}} + + def physical(self, before, after, bucket, baseline, added): + self.assertTrue(self.done.is_set()) + self.trace.append('physical-append') + return {'versionId': added['versionId']} + + def invoke(self): + return successor.observe_successor(self.probe, self.fleet, fixture.BUCKET, self.entry, self.output) + + def test_one_queued_put_drains_after_real_caller_socket_closes_and_retains_exact_ack(self): + result = self.invoke() + self.assertEqual(self.calls, 1) + self.assertTrue(result['diagnosticComplete']) + self.assertFalse(result['canonicalCasePassClaimed'] or result['heldAtDisconnectProven']) + self.assertEqual(result['verifiedEndpoints'], ['node1']) + self.assertEqual(len(result['expectedEntries']), 2) + self.assertTrue((self.output / 'queued.json').is_file()) + self.assertTrue((self.output / 'successor-complete.json').is_file()) + self.assertLess(self.trace.index('physical-append'), self.trace.index('I05-successor-before-all-peer-reads')) + transport = json.loads((self.output / 'caller/transport/disconnected.json').read_text()) + self.assertTrue(transport['resultUnknown'] and transport['clientSocketClosed']) + + def test_unknown_ack_retains_original_call_and_stops_before_public_verification(self): + self.change = 'unknown' + with self.assertRaisesRegex(ValueError, 'unexpected put_object'): + self.invoke() + self.assertEqual(self.calls, 1) + ack = json.loads((self.output / 'successor/000001-ack.json').read_text()) + self.assertTrue(ack['observation']['resultUnknown']) + self.assertNotIn('physical-append', self.trace) + self.assertFalse((self.output / 'diagnostic.json').exists()) + + def test_first_historical_verification_read_cannot_hide_physical_mutation(self): + self.change = 'historical-read-mutation' + with self.assertRaisesRegex(ValueError, 'first historical verification read changed physical content'): + self.invoke() + self.assertEqual(self.calls, 1) + self.assertTrue((self.output / 'successor/000001-ack.json').is_file()) + self.assertFalse((self.output / 'diagnostic.json').exists()) + + def test_early_write_acquisition_is_rejected_and_original_tcp_socket_still_closes(self): + self.change = 'early-acquire' + with self.assertRaisesRegex(ValueError, 'acquired before caller disconnect'): + self.invoke() + self.assertTrue(self.scene.wire.server.disconnected.wait(3)) + self.assertEqual(self.calls, 1) + self.assertTrue((self.output / 'successor/000001-ack.json').is_file()) + self.assertFalse((self.output / 'diagnostic.json').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/test_validation_heal_lock_timeout_runner.py b/test_validation_heal_lock_timeout_runner.py index c7b464cb..e179329b 100644 --- a/test_validation_heal_lock_timeout_runner.py +++ b/test_validation_heal_lock_timeout_runner.py @@ -63,6 +63,7 @@ def inventory(self, bucket): class Fleet: def __init__(self, nodes, evidence, run_id, *args, **kwargs): self.nodes = nodes; self.evidence = evidence; self.run_id = run_id + self.runtime_profile = kwargs.get('runtime_profile') self.secret_values = set(); self.reserved = []; self.sequence = 0 owner.fleets.append(self) def record(self, label, value): @@ -179,6 +180,87 @@ def test_preflight_checks_exact_source_helper_tools_and_workflow_before_remote_u with self.assertRaisesRegex(ValueError, 'exact reviewed'): runner.preflight(self.args) self.assertFalse(self.fleets or self.events) + def test_queued_successor_preflight_requires_exact_candidate_native_group(self): + self.args.diagnostic_branch = 'queued-successor' + self.args.native_directory = self.root / 'native' + proof = {'requiredRegressionLabels': list(runner.NATIVE_I05_LABELS), + 'requiredNativeSubsetPassed': True, 'wholeCasePassClaimed': False, + 'fleetExecutionClaimed': False, 'canonicalCasePassClaimed': False} + with patch.object(runner.subprocess, 'run', side_effect=[SimpleNamespace(stdout='c' * 40 + '\n'), + SimpleNamespace(stdout='')]), patch.object(runner, 'verify_inspector', return_value=self.inspector), \ + patch.object(runner, 'verify_native', return_value=proof) as verify: + self.assertEqual(runner.preflight(self.args), (self.candidate, self.inspector)) + verify.assert_called_once_with(self.args.native_directory, self.candidate, + self.args.testing_commit, runner.CASE_ID) + self.assertFalse(self.fleets or self.events) + + def test_queued_successor_without_native_artifact_refuses_before_fleet_or_evidence(self): + self.args.diagnostic_branch = 'queued-successor' + with patch.object(runner, 'preflight', return_value=(self.candidate, self.inspector)): + with self.assertRaisesRegex(ValueError, 'immutable exact-source native evidence'): + runner.run(self.args) + self.assertFalse(self.fleets or self.events) + self.assertFalse(self.args.evidence.exists()) + + def test_queued_successor_driver_binds_runtime_new_ack_and_every_peer(self): + self.args.diagnostic_branch = 'queued-successor' + def successor(probe, fleet, bucket, entry, output): + self.events.append('successor') + probe.s3('put_object', {'Bucket': bucket, 'Key': entry['key']}, b'new-generation') + return {'diagnosticComplete': True, 'canonicalCasePassClaimed': False, + 'wholeCasePassClaimed': False, 'expectedEntries': copy.deepcopy(self.entries), + 'verifiedEndpoints': list(probe.binding['endpoints']), + 'callerObservation': {'baselineResources': {node: { + 'identity': identity(), 'fileDescriptors': 10} for node in fleet.nodes}}} + with patch.object(runner, 'preflight', return_value=(self.candidate, self.inspector)), \ + patch.object(runner, 'verify_native_prerequisite', return_value={'native': True}) as native, \ + patch.object(runner, 'observe_successor', side_effect=successor): + summary = runner.run(self.args) + self.assertTrue(summary['diagnosticComplete']) + self.assertFalse(summary['canonicalCasePassClaimed'] or summary['wholeCasePassClaimed']) + self.assertEqual(native.call_count, 3) + self.assertEqual([f.runtime_profile for f in self.fleets], ['lock-successor'] * 2) + self.assertEqual(self.events.count('successor'), 2) + self.assertNotIn('diagnostic', self.events) + proof = json.loads((self.args.evidence / 'native-prerequisite.json').read_text()) + self.assertEqual(proof, {'native': True}) + for fleet in self.fleets: + peers = list(fleet.evidence.glob('*-all-peer-ACK-proof.json')) + self.assertEqual(len(peers), len(fleet.nodes)) + self.assertTrue(all(json.loads(p.read_text())['seedAckAndFullGetAgree'] for p in peers)) + + def test_successor_unknown_outcome_seals_only_after_stop_scene_logs_and_result(self): + self.args.diagnostic_branch = 'queued-successor' + def seal(folder, binding, scene): + self.assertIn('stop', self.events) + self.assertIn('original-failure-scene', self.events) + self.assertIn('logs-vm000', self.events) + result = list(folder.glob('*-diagnostic-result.json')) + self.assertEqual(len(result), 1) + self.assertTrue(json.loads(result[0].read_text())['namespacesRetained']) + return {'sealed': True} + with patch.object(runner, 'verify_native_prerequisite'), \ + patch.object(runner, 'observe_successor', side_effect=ValueError('unknown PUT result')), \ + patch.object(runner, 'seal_successor_scene', side_effect=seal) as sealed: + result = self.invoke() + self.assertFalse(result['diagnosticComplete']) + self.assertEqual(result['cleanup'], 'FAIL') + self.assertNotIn('cleanup', self.events) + sealed.assert_called_once() + self.assertEqual(json.loads((self.args.evidence / 'H1/successor-retention.json').read_text()), + {'sealed': True}) + + def test_successor_failed_stop_never_seals_incomplete_scene(self): + self.args.diagnostic_branch = 'queued-successor' + with patch.object(runner, 'verify_native_prerequisite'), \ + patch.object(runner, 'observe_successor', side_effect=ValueError('unknown PUT result')), \ + patch.object(runner.Fleet, 'stop', side_effect=ValueError('stop failed')), \ + patch.object(runner, 'seal_successor_scene') as sealed: + result = self.invoke() + self.assertIn('failureCaptureError', result) + sealed.assert_not_called() + self.assertNotIn('cleanup', self.events) + def test_uncommitted_or_untracked_controller_refuses_before_inspector_and_fleet(self): with patch.object(runner.subprocess, 'run', side_effect=[SimpleNamespace(stdout='c' * 40 + '\n'), SimpleNamespace(stdout=' M scripts/validation_heal/lock_timeout.py')]), \ diff --git a/test_validation_heal_timeout_retention.py b/test_validation_heal_timeout_retention.py index ad1dcf51..e4b98769 100644 --- a/test_validation_heal_timeout_retention.py +++ b/test_validation_heal_timeout_retention.py @@ -835,6 +835,104 @@ def test_plan_profile_cannot_be_reused_for_another_isolation_kind(self): plan['kind'] = 'unresolved-g01-race' with self.assertRaises(ValueError): race_retention.validate(plan) + def successor_archive(self, directory): + root, candidate, folder = self.archive(directory) + for path in folder.glob('*-reserved.json'): + value = json.loads(path.read_text()) + value['owner']['runtimeProfile'] = 'lock-successor' + path.write_text(json.dumps(value)) + result_path = next(folder.glob('*-diagnostic-result.json')) + result = json.loads(result_path.read_text()) + result['branch'] = 'queued-successor' + result_path.write_text(json.dumps(result)) + summary_path = folder.parent / 'healthy-summary.json' + summary = json.loads(summary_path.read_text()) + summary.update(diagnosticBranch='queued-successor', diagnostics=[result], + wholeCasePassClaimed=False) + summary_path.write_text(json.dumps(summary)) + receipt = folder / 'queued-successor/successor' + receipt.mkdir(parents=True) + (receipt / '000001-request.json').write_text('{"operation":"put_object"}') + (receipt / '000001-ack.json').write_text('{"resultUnknown":true,"acknowledged":false}') + binding = json.loads(next(folder.glob('*-binding.json')).read_text()) + scene = json.loads(next(folder.glob('*-original-failure-scene.json')).read_text()) + seal = controller.seal_successor_scene(folder, binding, scene) + (folder / 'successor-retention.json').write_text(json.dumps(seal)) + return root, candidate, folder + + def test_successor_unknown_receipt_selects_only_stopped_scene_isolation(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, folder = self.successor_archive(directory) + original = {p: p.read_bytes() for p in root.rglob('*') if p.is_file()} + with patch.object(recovery, 'Fleet') as fleet: + plan, = recovery.recovery_plan(root, 39711742431, candidate) + fleet.assert_not_called() + self.assertEqual(plan['kind'], 'unresolved-i05-successor') + self.assertEqual(plan['owners']['vm000']['runtimeProfile'], 'lock-successor') + race_retention.validate(plan) + self.assertEqual(original, {p: p.read_bytes() for p in root.rglob('*') if p.is_file()}) + candidate_path = Path(directory) / 'candidate.json' + candidate_path.write_text(json.dumps(candidate)) + evidence = Path(directory) / 'recovery' + fake = SimpleNamespace(nodes=plan['nodes'], run_id=plan['runId'], record=lambda label, value: value) + proof = {'runId': plan['runId'], 'cleanup': 'ISOLATED', 'canonicalPassClaimed': False} + with patch('sys.argv', ['recover', '--failed-run-id', '39711742431', '--artifact', str(root), + '--candidate', str(candidate_path), '--evidence', str(evidence)]), \ + patch.object(recovery, 'Fleet', return_value=fake), \ + patch.object(recovery, 'isolate_race_scene', return_value=proof) as isolate: + recovery.main() + isolate.assert_called_once_with(fake, plan) + + def test_successor_seal_refuses_changed_or_missing_original_evidence_and_foreign_profile(self): + for variant in ('receipt', 'missing', 'extra', 'symlink', 'scene', 'candidate', 'kind', 'profile', 'dirty-predecessor'): + with self.subTest(variant=variant), tempfile.TemporaryDirectory() as directory: + root, candidate, folder = self.successor_archive(directory) + receipt = folder / 'queued-successor/successor/000001-ack.json' + if variant == 'receipt': receipt.write_text('{}') + elif variant == 'missing': receipt.unlink() + elif variant == 'extra': (folder / 'unrecorded-write.json').write_text('{}') + elif variant == 'symlink': + target = Path(directory) / 'receipt.json'; target.write_bytes(receipt.read_bytes()) + receipt.unlink(); receipt.symlink_to(target) + elif variant == 'candidate': candidate['package_sha256'] = '0' * 64 + elif variant == 'dirty-predecessor': + path = folder.parent / 'healthy-summary.json' + summary = json.loads(path.read_text()); summary['diagnostics'] *= 2 + path.write_text(json.dumps(summary)) + else: + path = folder / 'successor-retention.json' + seal = json.loads(path.read_text()) + if variant == 'scene': seal['plan']['businessSceneSha256']['vm000'] = '0' * 64 + elif variant == 'kind': seal['plan']['kind'] = 'unresolved-i05-timeout' + else: seal['plan']['owners']['vm000']['runtimeProfile'] = 'manual' + path.write_text(json.dumps(seal)) + with patch.object(recovery, 'Fleet') as fleet, self.assertRaises(ValueError): + recovery.recovery_plan(root, 39711742431, candidate) + fleet.assert_not_called() + + def test_successor_isolation_preserves_all_five_original_trees(self): + with tempfile.TemporaryDirectory() as directory: + helper, state, roots, node, request, _ = self.scene(directory) + request['expectedOwner']['runtimeProfile'] = 'lock-successor' + for folder in (*roots, state): + (folder / 'owner.json').write_text(json.dumps(request['expectedOwner'])) + original = {p: (p.stat().st_ino, p.read_bytes()) for folder in (*roots, state) + for p in folder.rglob('*') if p.is_file()} + with helper.root_stats(): + before = retained.run(request, node) + change = {**request, 'action': 'retained-scene-isolate', 'expectedBaseline': before['baseline']} + with patch.object(retained, 'rename_new', side_effect=helper.local_rename): + proof = retained.run(change, node) + self.assertFalse(retained.run(change, node)['mutated']) + self.assertEqual(proof['locations'], ['retained'] * 5) + self.assertFalse(proof['originalSceneRestored'] or proof['backupRestorationUsed'] or proof['canonicalPassClaimed']) + target_state, target_roots = retained.destinations(state, roots, per_run_parent=True) + for path, expected in original.items(): + source = next(folder for folder in (*roots, state) if path.is_relative_to(folder)) + target = target_state if source == state else target_roots[roots.index(source)] + moved = target / path.relative_to(source) + self.assertEqual((moved.stat().st_ino, moved.read_bytes()), expected) + if __name__ == '__main__': unittest.main() From 7005b6a15e2eafaf3ca12ed5501dc4470effdf6c Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 21:57:52 +0800 Subject: [PATCH 299/343] fix(validation): classify I05 successor metadata reads --- docs/validation-heal-operations.md | 23 ++++++++++- docs/validation-migration-map.md | 11 +++++- scripts/validation_heal/lock_successor.py | 36 ++++++++++------- test_validation_heal_lock_successor.py | 39 +++++++++++++++++++ .../i05-successor-metadata-39788321908.ndjson | 15 +++++++ 5 files changed, 108 insertions(+), 16 deletions(-) create mode 100644 testdata/i05-successor-metadata-39788321908.ndjson diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index ad813684..33a30e54 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -1006,8 +1006,27 @@ outcome is preserved without retry, deletion or old-generation restoration. Incomplete capture, missing reservations, changed evidence or scene drift refuse automated isolation. -Local controller, admission and isolation tests pass; independent live -acceptance of this branch is still pending. Every successful diagnostic +The exact-helper native/tools build +[39783997956/1](https://github.com/rustfs/rustfs/actions/runs/39783997956) +passed independent archive, source, executable and ten-regression audits. +The first successor live run +[39788321908/1](https://github.com/rustfs/rustfs/actions/runs/39788321908) +retains FAIL/cleanup FAIL: the same PUT read its bucket's internal +`.metadata.bin` after target write-lock acquisition; the controller +incorrectly classified that read as a target write. The request returned +one unretried 200 ACK, which does not complete the failed diagnostic. + +[39789732916/1](https://github.com/rustfs/rustfs/actions/runs/39789732916) +isolated the unchanged stopped H1 scene without deleting it or restoring +old generations. The parser recognizes only that bucket's exact +`.metadata.bin` and `.bucket-incarnation` read paths, checks their original +HTTP identity, namespace, one-second budget and ordered per-attempt +lifecycle, and excludes them from the unique target write attempt. The +original fifteen-event trace is a regression fixture; foreign paths, +write-mode substitutions and reused or malformed lifecycles still refuse. +A new helper requires a newly bound native/tools build before another live +attempt. Independent live acceptance of this branch is still pending. +Every successful diagnostic remains `BLOCKED/cleanup PASS` with `canonicalCasePassClaimed=false`. Exact disconnect-edge ownership, complete native timeout/stale fences, the stale-generation controller and full H1/H2 source DL/DV ACK/physical/lock/FD diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 95daaf4d..f5fe7db0 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -52,7 +52,16 @@ timeouts, closed caller sockets, unchanged physical content, full diagnostic seed ACK/peer inventories and final FD counts at or below baseline. The original canonical result remains `BLOCKED/cleanup PASS`; source DL/DV, exact disconnect/native fences, stale-generation coverage and independent -queued-successor live acceptance remain outstanding. Acceptance therefore stays **108/158**. +queued-successor live acceptance remain outstanding. Exact-helper native/tools +[39783997956/1](https://github.com/rustfs/rustfs/actions/runs/39783997956) +passed independent audit. The initial successor live +[39788321908/1](https://github.com/rustfs/rustfs/actions/runs/39788321908) +failed at internal bucket metadata-read classification despite one +unretried 200 PUT ACK. Its stopped H1 scene was preserved by +[39789732916/1](https://github.com/rustfs/rustfs/actions/runs/39789732916). +The parser fix retains exact request/owner/budget/lifecycle checks and has +a regression over the original fifteen events; a fresh helper-bound tools +build and live run are still required. Acceptance stays **108/158**. [J02 diagnostic 38874900846/1](https://github.com/rustfs/rustfs/actions/runs/38874900846) completed all 20 H1/H2 × DB/DV branch instances on that exact candidate. Sixteen branches report PASS; the four original all-metadata-corruption probes report permanent Failed rather than Unknown and retain NOT_OBSERVED. Independent archive and provenance audits passed. Independent mixed-branch audits verified all four repaired shards against original complete bytes, exact receipts and counters, surviving below-quorum failures, original terminal preservation and post-rescue S3 inventories. All 50 node namespace finalization receipts passed audit. Real legacy writes separately produce Unknown with zero repairs and unchanged physical files, but do not replace the original source procedure. Full J02 canonical acceptance remains blocked; these partial results add no canonical acceptance. diff --git a/scripts/validation_heal/lock_successor.py b/scripts/validation_heal/lock_successor.py index c849bae1..fde9e98b 100644 --- a/scripts/validation_heal/lock_successor.py +++ b/scripts/validation_heal/lock_successor.py @@ -54,7 +54,7 @@ def successor_attempt(rows, marker, holder, bucket, key): require(marker.startswith('auto-testing-i05-successor/') and str(uuid.UUID(marker.split('/')[-1])) == marker.split('/')[-1], 'I05 successor marker differs') - selected, request_ids = [], set() + selected, request_ids, metadata_attempts = [], set(), {} for row in rows: if row.get('target') != lock_observer.TARGET: continue @@ -73,29 +73,39 @@ def successor_attempt(rows, marker, holder, bucket, key): and span.get('uri') == '/' + quote(bucket, safe='') + '/' + quote(key, safe='/~'), 'I05 successor server request, method or exact URI differs') request_ids.add(request_id) - # The same S3 call may first read its bucket incarnation. Those events - # are retained, but cannot establish target-object write admission. - incarnation = (row.get('mode') == 'read' and row.get('op') == 'get_object' - and row.get('bucket') == '.rustfs.sys' - and row.get('object') == f'buckets/{bucket}/.bucket-incarnation') - if incarnation: - continue + # Bucket fencing reads both persisted metadata objects within this PUT. + # They share its HTTP identity, but cannot establish write admission. + metadata_read = (row.get('mode') == 'read' and row.get('op') == 'get_object' + and row.get('bucket') == '.rustfs.sys' + and row.get('object') in {f'buckets/{bucket}/.bucket-incarnation', + f'buckets/{bucket}/.metadata.bin'}) require(row.get('event') == 'object_namespace_lock' and row.get('component') == 'ecstore' - and row.get('subsystem') == 'object_lock' and row.get('op') == 'put_object_commit' - and row.get('bucket') == bucket and row.get('object') == key and row.get('mode') == 'write' + and row.get('subsystem') == 'object_lock' + and (metadata_read or row.get('op') == 'put_object_commit' + and row.get('bucket') == bucket and row.get('object') == key and row.get('mode') == 'write') and row.get('requested_version_id') is None and row.get('owner') == holder['owner'] and row.get('namespace') == holder['namespace'] and type(row.get('timeout_ms')) is int - and row['timeout_ms'] == PUT_TIMEOUT_MS and isinstance(row.get('lock_attempt_id'), str) + and row['timeout_ms'] == (1000 if metadata_read else PUT_TIMEOUT_MS) + and isinstance(row.get('lock_attempt_id'), str) and str(uuid.UUID(row['lock_attempt_id'])) == row['lock_attempt_id'] and uuid.UUID(row['lock_attempt_id']).int and row['lock_attempt_id'] != holder['lock_attempt_id'] and type(row.get('acquire_ms')) is int and row['acquire_ms'] >= 0, - 'I05 successor write attempt escaped its exact namespace or budget') + 'I05 successor lock attempt escaped its exact namespace or budget') require(row.get('state') in {'acquiring', 'acquired', 'guard_dropped'} and row.get('level') == 'TRACE' and type(row.get('hold_ms')) is int and row['hold_ms'] >= 0, 'I05 successor failed, cancelled or lost its diagnostic lifecycle') - selected.append(row) + if metadata_read: + metadata_attempts.setdefault(row['lock_attempt_id'], []).append(row) + else: + selected.append(row) + for attempt_id, events in metadata_attempts.items(): + require(len({row['object'] for row in events}) == 1 + and not any(row['lock_attempt_id'] == attempt_id for row in selected) + and [row['state'] for row in events] in (['acquiring'], ['acquiring', 'acquired'], + ['acquiring', 'acquired', 'guard_dropped']), + 'I05 successor metadata read reused an attempt or lost its lifecycle') require(len(request_ids) <= 1, 'I05 successor marker was retried or reused') if not selected: return None diff --git a/test_validation_heal_lock_successor.py b/test_validation_heal_lock_successor.py index 91b97516..6f27b834 100644 --- a/test_validation_heal_lock_successor.py +++ b/test_validation_heal_lock_successor.py @@ -75,6 +75,45 @@ def test_forged_request_owner_version_budget_and_duplicate_attempt_refuse(self): with self.assertRaisesRegex(ValueError, 'retried or reused'): successor.successor_attempt(rows, MARKER, HOLDER, fixture.BUCKET, 'target.bin') + def test_original_live_metadata_reads_keep_one_exact_target_write_attempt(self): + path = Path(__file__).with_name('testdata') / 'i05-successor-metadata-39788321908.ndjson' + rows = [json.loads(line) for line in path.read_text().splitlines()] + marker = 'auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891' + holder = {'request_id': 'c10cb980-5924-4e9f-bbba-14d180d56436', + 'lock_attempt_id': 'd3e2e124-e6cc-46c4-897a-cbbc7b2dba66', + 'owner': ':::9000', 'namespace': 'pool-0'} + bucket = 'rv-heal-5dd2430e083c6140' + for length in range(1, len(rows) + 1): + proof = successor.successor_attempt(rows[:length], marker, holder, bucket, 'objects/0000.bin') + writes = [row for row in rows[:length] if row['mode'] == 'write'] + if not writes: + self.assertIsNone(proof) + else: + self.assertEqual(proof['events'], writes) + self.assertEqual(proof['lockAttemptId'], 'b31b1a30-9431-425e-9a92-909b47ac6bfc') + self.assertFalse(proof['canonicalCasePassClaimed']) + metadata = next(i for i, row in enumerate(rows) if row['object'].endswith('/.metadata.bin')) + mutations = [lambda r: r.update(object='buckets/foreign/.metadata.bin'), + lambda r: r.update(object=f'buckets/{bucket}/unreviewed'), + lambda r: r.update(mode='write'), lambda r: r.update(op='put_object_commit'), + lambda r: r.update(owner='foreign'), lambda r: r.update(namespace='foreign'), + lambda r: r.update(timeout_ms=successor.PUT_TIMEOUT_MS), + lambda r: r.update(requested_version_id=VERSION), + lambda r: r.update(component='foreign'), lambda r: r.update(event='foreign'), + lambda r: r.update(state='failed'), lambda r: r.update(hold_ms=True), + lambda r: r.update(lock_attempt_id=holder['lock_attempt_id']), + lambda r: r.update(lock_attempt_id='b31b1a30-9431-425e-9a92-909b47ac6bfc')] + for mutate in mutations: + changed = copy.deepcopy(rows); mutate(changed[metadata]) + with self.subTest(mutate=mutate), self.assertRaises(ValueError): + successor.successor_attempt(changed, marker, holder, bucket, 'objects/0000.bin') + for changed in [rows[:metadata] + rows[metadata + 1:], + rows[:metadata] + [rows[metadata]] + rows[metadata:], + rows[:metadata + 1] + [dict(rows[metadata + 1], object=f'buckets/{bucket}/.bucket-incarnation')] + + rows[metadata + 2:]]: + with self.subTest(rows=changed), self.assertRaises(ValueError): + successor.successor_attempt(changed, marker, holder, bucket, 'objects/0000.bin') + def test_real_recorder_ack_binding_rejects_unknown_retry_wrong_content_and_time(self): with tempfile.TemporaryDirectory() as folder: probe = type('Probe', (), {'binding': { diff --git a/testdata/i05-successor-metadata-39788321908.ndjson b/testdata/i05-successor-metadata-39788321908.ndjson new file mode 100644 index 00000000..8313e3a6 --- /dev/null +++ b/testdata/i05-successor-metadata-39788321908.ndjson @@ -0,0 +1,15 @@ +{"acquire_ms": 0, "bucket": ".rustfs.sys", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "010f7a1e-4727-4e75-837b-5495a2e08917", "message": "Object namespace lock diagnostic", "mode": "read", "namespace": "pool-0", "object": "buckets/rv-heal-5dd2430e083c6140/.bucket-incarnation", "op": "get_object", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "acquiring", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(9)", "threadName": "rustfs-worker", "timeout_ms": 1000, "timestamp": "2026-10-22T13:37:19.815580878Z"} +{"acquire_ms": 0, "bucket": ".rustfs.sys", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "010f7a1e-4727-4e75-837b-5495a2e08917", "message": "Object namespace lock diagnostic", "mode": "read", "namespace": "pool-0", "object": "buckets/rv-heal-5dd2430e083c6140/.bucket-incarnation", "op": "get_object", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "acquired", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(9)", "threadName": "rustfs-worker", "timeout_ms": 1000, "timestamp": "2026-10-22T13:37:19.81565574Z"} +{"acquire_ms": 0, "bucket": ".rustfs.sys", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "010f7a1e-4727-4e75-837b-5495a2e08917", "message": "Object namespace lock diagnostic", "mode": "read", "namespace": "pool-0", "object": "buckets/rv-heal-5dd2430e083c6140/.bucket-incarnation", "op": "get_object", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "guard_dropped", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(9)", "threadName": "rustfs-worker", "timeout_ms": 1000, "timestamp": "2026-10-22T13:37:19.815935188Z"} +{"acquire_ms": 0, "bucket": "rv-heal-5dd2430e083c6140", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "b31b1a30-9431-425e-9a92-909b47ac6bfc", "message": "Object namespace lock diagnostic", "mode": "write", "namespace": "pool-0", "object": "objects/0000.bin", "op": "put_object_commit", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "acquiring", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(9)", "threadName": "rustfs-worker", "timeout_ms": 60000, "timestamp": "2026-10-22T13:37:19.815977441Z"} +{"acquire_ms": 260, "bucket": "rv-heal-5dd2430e083c6140", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "b31b1a30-9431-425e-9a92-909b47ac6bfc", "message": "Object namespace lock diagnostic", "mode": "write", "namespace": "pool-0", "object": "objects/0000.bin", "op": "put_object_commit", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "acquired", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(8)", "threadName": "rustfs-worker", "timeout_ms": 60000, "timestamp": "2026-10-22T13:37:20.076585778Z"} +{"acquire_ms": 0, "bucket": ".rustfs.sys", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "cab752a7-f3c2-4f03-a127-c474bb0828f0", "message": "Object namespace lock diagnostic", "mode": "read", "namespace": "pool-0", "object": "buckets/rv-heal-5dd2430e083c6140/.metadata.bin", "op": "get_object", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "acquiring", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(8)", "threadName": "rustfs-worker", "timeout_ms": 1000, "timestamp": "2026-10-22T13:37:20.076775592Z"} +{"acquire_ms": 0, "bucket": ".rustfs.sys", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "cab752a7-f3c2-4f03-a127-c474bb0828f0", "message": "Object namespace lock diagnostic", "mode": "read", "namespace": "pool-0", "object": "buckets/rv-heal-5dd2430e083c6140/.metadata.bin", "op": "get_object", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "acquired", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(8)", "threadName": "rustfs-worker", "timeout_ms": 1000, "timestamp": "2026-10-22T13:37:20.076806618Z"} +{"acquire_ms": 0, "bucket": ".rustfs.sys", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "cab752a7-f3c2-4f03-a127-c474bb0828f0", "message": "Object namespace lock diagnostic", "mode": "read", "namespace": "pool-0", "object": "buckets/rv-heal-5dd2430e083c6140/.metadata.bin", "op": "get_object", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "guard_dropped", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(7)", "threadName": "rustfs-worker", "timeout_ms": 1000, "timestamp": "2026-10-22T13:37:20.077111504Z"} +{"acquire_ms": 0, "bucket": ".rustfs.sys", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "681c651c-1aed-4441-8e63-0eb601b67c3e", "message": "Object namespace lock diagnostic", "mode": "read", "namespace": "pool-0", "object": "buckets/rv-heal-5dd2430e083c6140/.bucket-incarnation", "op": "get_object", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "acquiring", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(7)", "threadName": "rustfs-worker", "timeout_ms": 1000, "timestamp": "2026-10-22T13:37:20.077174268Z"} +{"acquire_ms": 0, "bucket": ".rustfs.sys", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "681c651c-1aed-4441-8e63-0eb601b67c3e", "message": "Object namespace lock diagnostic", "mode": "read", "namespace": "pool-0", "object": "buckets/rv-heal-5dd2430e083c6140/.bucket-incarnation", "op": "get_object", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "acquired", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(7)", "threadName": "rustfs-worker", "timeout_ms": 1000, "timestamp": "2026-10-22T13:37:20.077205675Z"} +{"acquire_ms": 0, "bucket": ".rustfs.sys", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "681c651c-1aed-4441-8e63-0eb601b67c3e", "message": "Object namespace lock diagnostic", "mode": "read", "namespace": "pool-0", "object": "buckets/rv-heal-5dd2430e083c6140/.bucket-incarnation", "op": "get_object", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "guard_dropped", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(9)", "threadName": "rustfs-worker", "timeout_ms": 1000, "timestamp": "2026-10-22T13:37:20.0773692Z"} +{"acquire_ms": 260, "bucket": "rv-heal-5dd2430e083c6140", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 8, "level": "TRACE", "line_number": 159, "lock_attempt_id": "b31b1a30-9431-425e-9a92-909b47ac6bfc", "message": "Object namespace lock diagnostic", "mode": "write", "namespace": "pool-0", "object": "objects/0000.bin", "op": "put_object_commit", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "guard_dropped", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(10)", "threadName": "rustfs-worker", "timeout_ms": 60000, "timestamp": "2026-10-22T13:37:20.085553232Z"} +{"acquire_ms": 0, "bucket": ".rustfs.sys", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "6c6613ad-fe92-47c6-949f-b4c934cf8ef0", "message": "Object namespace lock diagnostic", "mode": "read", "namespace": "pool-0", "object": "buckets/rv-heal-5dd2430e083c6140/.bucket-incarnation", "op": "get_object", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "acquiring", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(10)", "threadName": "rustfs-worker", "timeout_ms": 1000, "timestamp": "2026-10-22T13:37:20.085660141Z"} +{"acquire_ms": 0, "bucket": ".rustfs.sys", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "6c6613ad-fe92-47c6-949f-b4c934cf8ef0", "message": "Object namespace lock diagnostic", "mode": "read", "namespace": "pool-0", "object": "buckets/rv-heal-5dd2430e083c6140/.bucket-incarnation", "op": "get_object", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "acquired", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(10)", "threadName": "rustfs-worker", "timeout_ms": 1000, "timestamp": "2026-10-22T13:37:20.085684588Z"} +{"acquire_ms": 0, "bucket": ".rustfs.sys", "component": "ecstore", "event": "object_namespace_lock", "filename": "crates/ecstore/src/diagnostics/object_lock.rs", "hold_ms": 0, "level": "TRACE", "line_number": 159, "lock_attempt_id": "6c6613ad-fe92-47c6-949f-b4c934cf8ef0", "message": "Object namespace lock diagnostic", "mode": "read", "namespace": "pool-0", "object": "buckets/rv-heal-5dd2430e083c6140/.bucket-incarnation", "op": "get_object", "owner": ":::9000", "request-id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span": {"content_length": "1048773", "content_type": "application/octet-stream", "method": "PUT", "name": "http-request", "peer_addr": "::ffff:10.0.0.4", "request_id": "6afbeca5-eefd-4ad0-b2c3-39b6b92ae0a5", "span_id": "unknown", "trace_id": "unknown", "uri": "/rv-heal-5dd2430e083c6140/objects/0000.bin", "user_agent": "Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/N,Z,D,b,e cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/73a9d02c-6579-44ce-8152-011f367dc891", "version": "HTTP/1.1"}, "state": "guard_dropped", "subsystem": "object_lock", "target": "rustfs_ecstore::object_lock_diag", "threadId": "ThreadId(10)", "threadName": "rustfs-worker", "timeout_ms": 1000, "timestamp": "2026-10-22T13:37:20.085869895Z"} From 6fbc3c466a9eef9d0543c0cf7af6cd587485e0db Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 22 Oct 2026 23:13:56 +0800 Subject: [PATCH 300/343] fix(validation): verify successor shards after service stop --- docs/validation-heal-operations.md | 19 ++++- docs/validation-migration-map.md | 14 +++- scripts/validation_heal/lock_successor.py | 5 +- .../validation_heal/lock_timeout_runner.py | 19 +++++ test_validation_heal_lock_successor.py | 3 +- test_validation_heal_lock_timeout_runner.py | 83 ++++++++++++++++--- 6 files changed, 125 insertions(+), 18 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 33a30e54..358e2388 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -1025,7 +1025,24 @@ lifecycle, and excludes them from the unique target write attempt. The original fifteen-event trace is a regression fixture; foreign paths, write-mode substitutions and reused or malformed lifecycles still refuse. A new helper requires a newly bound native/tools build before another live -attempt. Independent live acceptance of this branch is still pending. +attempt. Fresh exact-helper native/tools +[39791663588/1](https://github.com/rustfs/rustfs/actions/runs/39791663588) +passed independent verification of all three official archives, 22 original +members, ten exact tests and 104 ordered milestones. The next live run +[39798325977/1](https://github.com/rustfs/rustfs/actions/runs/39798325977) +passed the metadata-read classifier and acknowledged one new PUT version, +then retained FAIL/cleanup FAIL because the controller invoked stopped-only +streaming shard verification while the service was running. + +[39799657074/1](https://github.com/rustfs/rustfs/actions/runs/39799657074) +isolated that stopped H1 scene. Independent official archive verification +byte-matched all 71 original files and all five pre/move/post tree, device +and inode baselines. The new shard's streaming checksums now run only after +all online peer reads and FD observations, under the unchanged stopped-only +verifier. Stopped physical snapshots must match the ACK-bound pre-read scene +both before and after the checksum pass. Drift or checksum failure retains +the original scene and prevents cleanup. Fresh helper-bound tools and +independent live acceptance are still required. Every successful diagnostic remains `BLOCKED/cleanup PASS` with `canonicalCasePassClaimed=false`. Exact disconnect-edge ownership, complete native timeout/stale fences, the diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index f5fe7db0..371ce8f5 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -60,8 +60,18 @@ failed at internal bucket metadata-read classification despite one unretried 200 PUT ACK. Its stopped H1 scene was preserved by [39789732916/1](https://github.com/rustfs/rustfs/actions/runs/39789732916). The parser fix retains exact request/owner/budget/lifecycle checks and has -a regression over the original fifteen events; a fresh helper-bound tools -build and live run are still required. Acceptance stays **108/158**. +a regression over the original fifteen events. Fresh helper-bound tools +[39791663588/1](https://github.com/rustfs/rustfs/actions/runs/39791663588) +passed independent archive/source/native audits. Live +[39798325977/1](https://github.com/rustfs/rustfs/actions/runs/39798325977) +then reached the new acknowledged generation but invoked the stopped-only +shard verifier on a running service. Its original FAIL/cleanup FAIL and +71 files were preserved unchanged by isolation +[39799657074/1](https://github.com/rustfs/rustfs/actions/runs/39799657074). +The controller now finishes all online peer/FD observations before stopping +the service and checking streaming shard integrity, with matching physical +snapshots before and after verification. Fresh tools and live acceptance +of that sequence remain pending. Acceptance stays **108/158**. [J02 diagnostic 38874900846/1](https://github.com/rustfs/rustfs/actions/runs/38874900846) completed all 20 H1/H2 × DB/DV branch instances on that exact candidate. Sixteen branches report PASS; the four original all-metadata-corruption probes report permanent Failed rather than Unknown and retain NOT_OBSERVED. Independent archive and provenance audits passed. Independent mixed-branch audits verified all four repaired shards against original complete bytes, exact receipts and counters, surviving below-quorum failures, original terminal preservation and post-rescue S3 inventories. All 50 node namespace finalization receipts passed audit. Real legacy writes separately produce Unknown with zero repairs and unchanged physical files, but do not replace the original source procedure. Full J02 canonical acceptance remains blocked; these partial results add no canonical acceptance. diff --git a/scripts/validation_heal/lock_successor.py b/scripts/validation_heal/lock_successor.py index fde9e98b..b105eaa1 100644 --- a/scripts/validation_heal/lock_successor.py +++ b/scripts/validation_heal/lock_successor.py @@ -16,14 +16,14 @@ from .heal_workload import Recorder, verify_seed from .healthy import physical_content from .lock_timeout import CALLER_RECEIVE_BUFFER_BYTES, MIN_BACKPRESSURE_BYTES, require_exact_inventory_entry - from .shards import files, map_target, verify_cohort + from .shards import files, map_target except ImportError: import lock_observer from heal_probe import CLOCK_DOMAIN, Probe, canonical, payload, require, save, validate_binding from heal_workload import Recorder, verify_seed from healthy import physical_content from lock_timeout import CALLER_RECEIVE_BUFFER_BYTES, MIN_BACKPRESSURE_BYTES, require_exact_inventory_entry - from shards import files, map_target, verify_cohort + from shards import files, map_target SUCCESSOR_BYTES = 1024 * 1024 + 197 @@ -290,7 +290,6 @@ def complete(sent, admitted, window): after = fleet.snapshot('I05-successor-after-ACK-before-public-read') after_ack = physical_content(after) target = physical_append(physical, after, bucket, baseline, added) - fleet.record('I05-successor-new-shard-integrity', verify_cohort(fleet, target, 'I05-successor-shard')) save(output / 'successor-complete.json', {**attempt, 'added': added, 'queuedBeforeDisconnectObserved': True, 'originalDiagnosticDropBeforeAcquire': True, 'heldAtDisconnectProven': False, 'canonicalCasePassClaimed': False}) diff --git a/scripts/validation_heal/lock_timeout_runner.py b/scripts/validation_heal/lock_timeout_runner.py index 3df828d5..2224d24c 100644 --- a/scripts/validation_heal/lock_timeout_runner.py +++ b/scripts/validation_heal/lock_timeout_runner.py @@ -23,6 +23,7 @@ from .lock_timeout import MIN_BACKPRESSURE_BYTES, observe_timeout from .lock_successor import observe_successor from .race_native_evidence import verify as verify_native + from .shards import map_target, verify_cohort from .timeout_retention import seal_successor_scene except ImportError: from heal_probe import Probe, compare_inventories, require, save, validate_binding @@ -33,6 +34,7 @@ from lock_timeout import MIN_BACKPRESSURE_BYTES, observe_timeout from lock_successor import observe_successor from race_native_evidence import verify as verify_native + from shards import map_target, verify_cohort from timeout_retention import seal_successor_scene @@ -175,6 +177,23 @@ def error_info(error): if all(resources[node]['fileDescriptors'] <= baseline_resources[node]['fileDescriptors'] for node in nodes): break time.sleep(.2) + if branch == 'queued-successor': + # Stream checksums only after all live ownership/FD observations. + # The stopped oracle must still be the ACK-bound pre-read scene. + fleet.stop() + stopped = fleet.snapshot('I05-successor-stopped-before-integrity', quiescent=True) + require(physical_content(physical) == physical_content(stopped), + 'I05 successor stopping the service changed the verified physical scene') + original_versions = {item['versionId'] for item in baseline['entries']} + added = [item for item in diagnostic['expectedEntries'] if item['versionId'] not in original_versions] + require(len(added) == 1 and added[0]['key'] == target['key'] + and added[0]['isLatest'] is True and added[0]['deleteMarker'] is False, + 'I05 successor stopped verification lacks one exact new data version') + new_target = map_target(stopped, bucket, target['key'], 'data', 1, added[0]['versionId']) + fleet.record('I05-successor-new-shard-integrity', verify_cohort(fleet, new_target, 'I05-successor-shard')) + require(physical_content(stopped) == physical_content( + fleet.snapshot('I05-successor-stopped-after-integrity', quiescent=True)), + 'I05 successor streaming verification changed physical content') result['diagnosticComplete'] = True except (Exception, KeyboardInterrupt) as error: result['failure'] = error_info(error) diff --git a/test_validation_heal_lock_successor.py b/test_validation_heal_lock_successor.py index 6f27b834..dc1b46f0 100644 --- a/test_validation_heal_lock_successor.py +++ b/test_validation_heal_lock_successor.py @@ -229,7 +229,7 @@ def s3(self, operation, params, body): return response peer = type('Peer', (), {'inventory': lambda _, bucket: owner.inventory(bucket)}) for name, value in [('SuccessorProbe', Writer), ('Probe', lambda *a, **k: peer()), - ('physical_append', self.physical), ('verify_cohort', lambda *a: {'allShardBitrotVerified': True})]: + ('physical_append', self.physical)]: p = patch.object(successor, name, value); p.start(); self.addCleanup(p.stop) self.probe.inventory = self.inventory def snapshot(label): @@ -239,6 +239,7 @@ def snapshot(label): self.fleet.snapshot = snapshot original_remote = self.fleet.remote def remote(node, action, **params): + self.assertNotEqual(action, 'verify-part', 'streaming checksums require a stopped service') value = original_remote(node, action, **params) if action in {'process-metrics', 'caller-socket'}: if action == 'caller-socket': diff --git a/test_validation_heal_lock_timeout_runner.py b/test_validation_heal_lock_timeout_runner.py index e179329b..24092016 100644 --- a/test_validation_heal_lock_timeout_runner.py +++ b/test_validation_heal_lock_timeout_runner.py @@ -65,6 +65,7 @@ def __init__(self, nodes, evidence, run_id, *args, **kwargs): self.nodes = nodes; self.evidence = evidence; self.run_id = run_id self.runtime_profile = kwargs.get('runtime_profile') self.secret_values = set(); self.reserved = []; self.sequence = 0 + self.stopped = False owner.fleets.append(self) def record(self, label, value): self.sequence += 1 @@ -75,13 +76,19 @@ def deploy(self, *args): if owner.change == 'reserve-error': raise ValueError('partial reservation') def snapshot(self, label, **kwargs): owner.events.append(label) - return {node: {'files': [], 'formats': []} for node in self.nodes} + if kwargs.get('quiescent'): + owner.assertTrue(self.stopped) + changed = ((owner.change == 'stopped-drift' and label == 'I05-successor-stopped-before-integrity') + or (owner.change == 'integrity-drift' and label == 'I05-successor-stopped-after-integrity')) + return {node: {'files': ['changed'] if changed else [], 'formats': []} for node in self.nodes} def remote(self, node, action): owner.events.append(action + '-' + node) if action == 'process-metrics': + owner.assertFalse(self.stopped) return {'identity': identity(), 'fileDescriptors': 11 if owner.change == 'fd-leak' else 10} return {'journal': 'original log'} - def stop(self): owner.events.append('stop') + def stop(self): + owner.events.append('stop'); self.stopped = True def cleanup(self): owner.events.append('cleanup'); return owner.change != 'cleanup-fail' def diagnostic(probe, fleet, bucket, entry, output): owner.events.append('diagnostic') @@ -204,17 +211,11 @@ def test_queued_successor_without_native_artifact_refuses_before_fleet_or_eviden def test_queued_successor_driver_binds_runtime_new_ack_and_every_peer(self): self.args.diagnostic_branch = 'queued-successor' - def successor(probe, fleet, bucket, entry, output): - self.events.append('successor') - probe.s3('put_object', {'Bucket': bucket, 'Key': entry['key']}, b'new-generation') - return {'diagnosticComplete': True, 'canonicalCasePassClaimed': False, - 'wholeCasePassClaimed': False, 'expectedEntries': copy.deepcopy(self.entries), - 'verifiedEndpoints': list(probe.binding['endpoints']), - 'callerObservation': {'baselineResources': {node: { - 'identity': identity(), 'fileDescriptors': 10} for node in fleet.nodes}}} with patch.object(runner, 'preflight', return_value=(self.candidate, self.inspector)), \ patch.object(runner, 'verify_native_prerequisite', return_value={'native': True}) as native, \ - patch.object(runner, 'observe_successor', side_effect=successor): + patch.object(runner, 'observe_successor', side_effect=self.successor), \ + patch.object(runner, 'map_target', side_effect=self.new_target), \ + patch.object(runner, 'verify_cohort', side_effect=self.integrity): summary = runner.run(self.args) self.assertTrue(summary['diagnosticComplete']) self.assertFalse(summary['canonicalCasePassClaimed'] or summary['wholeCasePassClaimed']) @@ -228,6 +229,66 @@ def successor(probe, fleet, bucket, entry, output): peers = list(fleet.evidence.glob('*-all-peer-ACK-proof.json')) self.assertEqual(len(peers), len(fleet.nodes)) self.assertTrue(all(json.loads(p.read_text())['seedAckAndFullGetAgree'] for p in peers)) + integrity = list(fleet.evidence.glob('*-I05-successor-new-shard-integrity.json')) + self.assertEqual(len(integrity), 1) + self.assertTrue(json.loads(integrity[0].read_text())['allShardBitrotVerified']) + for start, end in ((0, self.events.index('cleanup') + 1), + (self.events.index('cleanup') + 1, len(self.events))): + events = self.events[start:end] + self.assertLess(events.index('physical-after-all-peer-reads'), events.index('stop')) + self.assertLess(events.index('process-metrics-vm000'), events.index('stop')) + self.assertLess(events.index('stop'), events.index('I05-successor-stopped-before-integrity')) + self.assertLess(events.index('I05-successor-stopped-before-integrity'), events.index('verify-integrity')) + self.assertLess(events.index('verify-integrity'), events.index('I05-successor-stopped-after-integrity')) + self.assertLess(events.index('I05-successor-stopped-after-integrity'), events.index('cleanup')) + + def successor(self, probe, fleet, bucket, entry, output): + self.events.append('successor') + probe.s3('put_object', {'Bucket': bucket, 'Key': entry['key']}, b'new-generation') + return {'diagnosticComplete': True, 'canonicalCasePassClaimed': False, + 'wholeCasePassClaimed': False, 'expectedEntries': copy.deepcopy(self.entries), + 'verifiedEndpoints': list(probe.binding['endpoints']), + 'callerObservation': {'baselineResources': {node: { + 'identity': identity(), 'fileDescriptors': 10} for node in fleet.nodes}}} + + def new_target(self, snapshot, bucket, key, role, part, version): + self.assertEqual(set(snapshot), set(self.fleets[-1].nodes)) + self.assertTrue(self.fleets[-1].stopped) + added = next(item for item in self.entries if item['versionId'] == version) + self.assertEqual((added['key'], role, part), (key, 'data', 1)) + self.assertTrue(added['isLatest']) + self.assertFalse(added['deleteMarker']) + return {'versionId': version} + + def integrity(self, fleet, target, label): + self.assertTrue(fleet.stopped) + self.assertEqual(label, 'I05-successor-shard') + self.events.append('verify-integrity') + if self.change == 'integrity-error': + raise ValueError('corrupt acknowledged successor shard') + return {'allShardBitrotVerified': True} + + def test_successor_stopped_drift_or_checksum_failure_retains_scene_before_cleanup(self): + self.args.diagnostic_branch = 'queued-successor' + for change, message in (('stopped-drift', 'stopping the service changed'), + ('integrity-error', 'corrupt acknowledged successor shard'), + ('integrity-drift', 'streaming verification changed')): + with self.subTest(change=change): + self.change = change; self.events.clear() + self.args.evidence = self.root / change + with patch.object(runner, 'verify_native_prerequisite'), \ + patch.object(runner, 'observe_successor', side_effect=self.successor), \ + patch.object(runner, 'map_target', side_effect=self.new_target), \ + patch.object(runner, 'verify_cohort', side_effect=self.integrity), \ + patch.object(runner, 'seal_successor_scene', return_value={'sealed': True}) as seal: + result = self.invoke() + self.assertFalse(result['diagnosticComplete']) + self.assertEqual(result['cleanup'], 'FAIL') + self.assertIn(message, result['failure']['message']) + self.assertTrue(result['namespacesRetained']) + self.assertNotIn('cleanup', self.events) + self.assertIn('original-failure-scene', self.events) + seal.assert_called_once() def test_successor_unknown_outcome_seals_only_after_stop_scene_logs_and_result(self): self.args.diagnostic_branch = 'queued-successor' From 3012a652efdab5b945999077ab0c31dea476789b Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 00:17:57 +0800 Subject: [PATCH 301/343] fix(validation): distinguish successor slow-lock warnings --- docs/validation-heal-operations.md | 24 ++++++++++- docs/validation-migration-map.md | 30 +++++++++++++- scripts/validation_heal/lock_successor.py | 17 ++++++++ test_validation_heal_lock_successor.py | 41 +++++++++++++++++++ ...-successor-slow-acquire-39810035277.ndjson | 16 ++++++++ 5 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 testdata/i05-successor-slow-acquire-39810035277.ndjson diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 358e2388..26ed9173 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -1041,8 +1041,28 @@ and inode baselines. The new shard's streaming checksums now run only after all online peer reads and FD observations, under the unchanged stopped-only verifier. Stopped physical snapshots must match the ACK-bound pre-read scene both before and after the checksum pass. Drift or checksum failure retains -the original scene and prevents cleanup. Fresh helper-bound tools and -independent live acceptance are still required. +the original scene and prevents cleanup. Exact helper `6fbc3c4` tools +[39803149039/1](https://github.com/rustfs/rustfs/actions/runs/39803149039) +passed independent audits of all three official ZIPs, 22 original members, +ten exact native tests, 104 ordered milestones and inspector contracts. +Live [39810035277/1](https://github.com/rustfs/rustfs/actions/runs/39810035277) +completed the new H1 stopped-integrity sequence and cleanup, then retained +H2 because a slow-acquire WARN sharing the diagnostic target was parsed as +a lifecycle event. The WARN has no event/state/attempt/namespace fields; +it reports a 1214 ms acquisition against a 500 ms warning threshold. It is +not a timeout or a separate write attempt. The selector recognizes this +exact auxiliary shape only after the same request's acquired target write, +retaining strict HTTP identity, owner, numeric timing and lifecycle checks. +Unknown warnings and lifecycle substitutions still refuse; the auxiliary +warning cannot establish acquisition or guard drop. + +Isolation [39811143272/1](https://github.com/rustfs/rustfs/actions/runs/39811143272) +preserved all 180 original failed files without modification. Independent +archive audit verified all four nodes' pre/move/post tree, device and inode +baselines and twenty root-owned 0700 archive parents on their original +devices. No S3 replay, namespace deletion, old-generation restoration or +original verdict rewrite occurred. Rebuild exact-helper tools before the +next H1/H2 live attempt. Every successful diagnostic remains `BLOCKED/cleanup PASS` with `canonicalCasePassClaimed=false`. Exact disconnect-edge ownership, complete native timeout/stale fences, the diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 371ce8f5..0fd16628 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -70,8 +70,34 @@ shard verifier on a running service. Its original FAIL/cleanup FAIL and [39799657074/1](https://github.com/rustfs/rustfs/actions/runs/39799657074). The controller now finishes all online peer/FD observations before stopping the service and checking streaming shard integrity, with matching physical -snapshots before and after verification. Fresh tools and live acceptance -of that sequence remain pending. Acceptance stays **108/158**. +snapshots before and after verification. Tools +[39803149039/1](https://github.com/rustfs/rustfs/actions/runs/39803149039), +bound to helper `6fbc3c4`, passed independent official archive/source/native +and inspector audits. Live +[39810035277/1](https://github.com/rustfs/rustfs/actions/runs/39810035277) +completed H1 including stopped integrity and cleanup. H2 started and queued +its PUT, then the parser treated a same-target slow-acquire WARN as a lock +lifecycle row. The original result stays FAIL/cleanup FAIL. Isolation +[39811143272/1](https://github.com/rustfs/rustfs/actions/runs/39811143272) +preserves all 180 original files and twenty same-device private archive +parents; every four-node pre/move/post tree, device and inode baseline +matches. The parser now recognizes only the exact auxiliary warning bound +to an already acquired target write; it cannot replace lifecycle evidence. +The original sixteen-event H2 trace and incremental prefixes are covered by +positive and negative regression tests. Fresh helper-bound tools and H1/H2 +live acceptance remain required. Acceptance stays **108/158**. + +Fresh D03 [39801936817/1](https://github.com/rustfs/rustfs/actions/runs/39801936817) +retains FAIL/cleanup PASS. Independent H1 re-evaluation verified all four +branch physical and strict version-responsibility oracles, sixteen seed +request/ACK pairs, fourteen versions/markers, 48 HEAD observations and 144 +range payloads. H2 failed before seeding when a `pool.bin` write returned +`SystemMetadataRepairRequired` and the pool transaction remained blocked +at `transaction_unknown/prepare_cas`. All 392 original files and five node +cleanup receipts were independently checked. The exact candidate's bounded +owner-repair retries only cover identity metadata, so ordinary init retries +do not discharge this `pool.bin` responsibility. The initial replica/network +cause remains unproven; no D03 historical failure is disproved or promoted. [J02 diagnostic 38874900846/1](https://github.com/rustfs/rustfs/actions/runs/38874900846) completed all 20 H1/H2 × DB/DV branch instances on that exact candidate. Sixteen branches report PASS; the four original all-metadata-corruption probes report permanent Failed rather than Unknown and retain NOT_OBSERVED. Independent archive and provenance audits passed. Independent mixed-branch audits verified all four repaired shards against original complete bytes, exact receipts and counters, surviving below-quorum failures, original terminal preservation and post-rescue S3 inventories. All 50 node namespace finalization receipts passed audit. Real legacy writes separately produce Unknown with zero repairs and unchanged physical files, but do not replace the original source procedure. Full J02 canonical acceptance remains blocked; these partial results add no canonical acceptance. diff --git a/scripts/validation_heal/lock_successor.py b/scripts/validation_heal/lock_successor.py index b105eaa1..4d92eaff 100644 --- a/scripts/validation_heal/lock_successor.py +++ b/scripts/validation_heal/lock_successor.py @@ -73,6 +73,23 @@ def successor_attempt(rows, marker, holder, bucket, key): and span.get('uri') == '/' + quote(bucket, safe='') + '/' + quote(key, safe='/~'), 'I05 successor server request, method or exact URI differs') request_ids.add(request_id) + # Slow-acquire warnings share the target but carry no lock lifecycle. + # Bind this auxiliary row to the preceding acquired write observation. + if row.get('message') == 'object namespace lock acquisition exceeded threshold': + require(row.get('level') == 'WARN' and row.get('op') == 'put_object_commit' + and row.get('bucket') == bucket and row.get('object') == key + and row.get('mode') == 'write' and row.get('owner') == holder['owner'] + and not any(name in row for name in ('event', 'component', 'subsystem', 'state', + 'namespace', 'lock_attempt_id', 'timeout_ms', 'hold_ms', 'requested_version_id')) + and all(isinstance(row.get(name), str) and row[name].isascii() + and row[name].isdecimal() and str(int(row[name])) == row[name] + for name in ('acquire_ms', 'threshold_ms')) + and int(row['acquire_ms']) >= int(row['threshold_ms']) > 0 + and selected and selected[-1]['state'] == 'acquired' + and selected[-1]['request_id'] == request_id + and int(row['acquire_ms']) >= selected[-1]['acquire_ms'], + 'I05 successor slow-acquire warning lacks its exact acquired write') + continue # Bucket fencing reads both persisted metadata objects within this PUT. # They share its HTTP identity, but cannot establish write admission. metadata_read = (row.get('mode') == 'read' and row.get('op') == 'get_object' diff --git a/test_validation_heal_lock_successor.py b/test_validation_heal_lock_successor.py index dc1b46f0..3c738a34 100644 --- a/test_validation_heal_lock_successor.py +++ b/test_validation_heal_lock_successor.py @@ -114,6 +114,47 @@ def test_original_live_metadata_reads_keep_one_exact_target_write_attempt(self): with self.subTest(rows=changed), self.assertRaises(ValueError): successor.successor_attempt(changed, marker, holder, bucket, 'objects/0000.bin') + def test_real_slow_acquire_warning_never_replaces_target_lifecycle(self): + path = Path(__file__).with_name('testdata') / 'i05-successor-slow-acquire-39810035277.ndjson' + rows = [json.loads(line) for line in path.read_text().splitlines()] + marker = 'auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818' + holder = {'request_id': fixture.REQUEST, 'lock_attempt_id': fixture.ATTEMPT, + 'owner': 'rustfs-node1:9000', 'namespace': 'pool-0'} + bucket = 'rv-heal-e9da35d01389c2ae' + warning = next(i for i, row in enumerate(rows) if row['level'] == 'WARN') + for length in range(1, len(rows) + 1): + proof = successor.successor_attempt(rows[:length], marker, holder, bucket, 'objects/0000.bin') + writes = [row for row in rows[:length] if row.get('event') == 'object_namespace_lock' + and row['mode'] == 'write'] + if writes: + self.assertEqual(proof['events'], writes) + self.assertEqual(proof['lockAttemptId'], 'de759fe0-8dfe-4b32-841a-047a23b8ed7c') + else: + self.assertIsNone(proof) + mutations = [lambda r: r.update(event='object_namespace_lock'), + lambda r: r.update(state='acquired'), lambda r: r.update(level='ERROR'), + lambda r: r.update(namespace='pool-0'), lambda r: r.update(owner='foreign'), + lambda r: r.update(object='foreign'), lambda r: r.update(bucket='foreign'), + lambda r: r.update(mode='read'), lambda r: r.update(op='get_object'), + lambda r: r.update(acquire_ms=True), lambda r: r.update(acquire_ms='499'), + lambda r: r.update(threshold_ms='0'), lambda r: r.update(threshold_ms='0500'), + lambda r: r.update(message='unreviewed lock warning')] + for mutate in mutations: + changed = copy.deepcopy(rows); mutate(changed[warning]) + with self.subTest(mutate=mutate), self.assertRaises(ValueError): + successor.successor_attempt(changed, marker, holder, bucket, 'objects/0000.bin') + for changed in [rows[:warning - 1] + rows[warning:], + [rows[warning]], rows[:warning - 1] + [rows[warning]] + rows[warning - 1:]]: + with self.subTest(rows=changed), self.assertRaises(ValueError): + successor.successor_attempt(changed, marker, holder, bucket, 'objects/0000.bin') + # Removing the auxiliary warning leaves the same three target events. + plain = successor.successor_attempt(rows[:warning] + rows[warning + 1:], marker, holder, bucket, 'objects/0000.bin') + self.assertEqual(plain['states'], ['acquiring', 'acquired', 'guard_dropped']) + undrained = [row for row in rows if not (row.get('mode') == 'write' and row.get('state') == 'guard_dropped')] + proof = successor.successor_attempt(undrained, marker, holder, bucket, 'objects/0000.bin') + self.assertEqual(proof['states'], ['acquiring', 'acquired']) + self.assertNotIn('guard_dropped', proof['states']) + def test_real_recorder_ack_binding_rejects_unknown_retry_wrong_content_and_time(self): with tempfile.TemporaryDirectory() as folder: probe = type('Probe', (), {'binding': { diff --git a/testdata/i05-successor-slow-acquire-39810035277.ndjson b/testdata/i05-successor-slow-acquire-39810035277.ndjson new file mode 100644 index 00000000..92f1aa44 --- /dev/null +++ b/testdata/i05-successor-slow-acquire-39810035277.ndjson @@ -0,0 +1,16 @@ +{"acquire_ms":0,"bucket":".rustfs.sys","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"b44942f5-f89b-4b21-a00b-64716e353981","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"buckets/rv-heal-e9da35d01389c2ae/.bucket-incarnation","op":"get_object","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"acquiring","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(7)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T16:01:37.184272458Z"} +{"acquire_ms":0,"bucket":".rustfs.sys","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"b44942f5-f89b-4b21-a00b-64716e353981","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"buckets/rv-heal-e9da35d01389c2ae/.bucket-incarnation","op":"get_object","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"acquired","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(10)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T16:01:37.18460302Z"} +{"acquire_ms":0,"bucket":".rustfs.sys","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"b44942f5-f89b-4b21-a00b-64716e353981","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"buckets/rv-heal-e9da35d01389c2ae/.bucket-incarnation","op":"get_object","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"guard_dropped","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(8)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T16:01:37.185387906Z"} +{"acquire_ms":0,"bucket":"rv-heal-e9da35d01389c2ae","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"de759fe0-8dfe-4b32-841a-047a23b8ed7c","message":"Object namespace lock diagnostic","mode":"write","namespace":"pool-0","object":"objects/0000.bin","op":"put_object_commit","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"acquiring","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(8)","threadName":"rustfs-worker","timeout_ms":60000,"timestamp":"2026-10-22T16:01:37.185440815Z"} +{"acquire_ms":1213,"bucket":"rv-heal-e9da35d01389c2ae","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"de759fe0-8dfe-4b32-841a-047a23b8ed7c","message":"Object namespace lock diagnostic","mode":"write","namespace":"pool-0","object":"objects/0000.bin","op":"put_object_commit","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"acquired","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(9)","threadName":"rustfs-worker","timeout_ms":60000,"timestamp":"2026-10-22T16:01:38.399392933Z"} +{"acquire_ms":"1214","bucket":"rv-heal-e9da35d01389c2ae","filename":"crates/ecstore/src/set_disk/mod.rs","level":"WARN","line_number":5218,"message":"object namespace lock acquisition exceeded threshold","mode":"write","object":"objects/0000.bin","op":"put_object_commit","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(9)","threadName":"rustfs-worker","threshold_ms":"500","timestamp":"2026-10-22T16:01:38.399447324Z"} +{"acquire_ms":0,"bucket":".rustfs.sys","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"5986ab7e-ec8a-4721-a43d-8e43b08dd2c6","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"buckets/rv-heal-e9da35d01389c2ae/.metadata.bin","op":"get_object","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"acquiring","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(9)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T16:01:38.400853841Z"} +{"acquire_ms":0,"bucket":".rustfs.sys","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"5986ab7e-ec8a-4721-a43d-8e43b08dd2c6","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"buckets/rv-heal-e9da35d01389c2ae/.metadata.bin","op":"get_object","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"acquired","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(9)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T16:01:38.401178747Z"} +{"acquire_ms":0,"bucket":".rustfs.sys","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"5986ab7e-ec8a-4721-a43d-8e43b08dd2c6","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"buckets/rv-heal-e9da35d01389c2ae/.metadata.bin","op":"get_object","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"guard_dropped","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(10)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T16:01:38.402115996Z"} +{"acquire_ms":0,"bucket":".rustfs.sys","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"bd1959e7-754d-4142-a590-a49bde62f2c2","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"buckets/rv-heal-e9da35d01389c2ae/.bucket-incarnation","op":"get_object","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"acquiring","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(10)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T16:01:38.402176987Z"} +{"acquire_ms":0,"bucket":".rustfs.sys","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"bd1959e7-754d-4142-a590-a49bde62f2c2","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"buckets/rv-heal-e9da35d01389c2ae/.bucket-incarnation","op":"get_object","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"acquired","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(9)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T16:01:38.402496128Z"} +{"acquire_ms":0,"bucket":".rustfs.sys","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"bd1959e7-754d-4142-a590-a49bde62f2c2","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"buckets/rv-heal-e9da35d01389c2ae/.bucket-incarnation","op":"get_object","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"guard_dropped","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(10)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T16:01:38.403118789Z"} +{"acquire_ms":1213,"bucket":"rv-heal-e9da35d01389c2ae","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":10,"level":"TRACE","line_number":159,"lock_attempt_id":"de759fe0-8dfe-4b32-841a-047a23b8ed7c","message":"Object namespace lock diagnostic","mode":"write","namespace":"pool-0","object":"objects/0000.bin","op":"put_object_commit","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"guard_dropped","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(9)","threadName":"rustfs-worker","timeout_ms":60000,"timestamp":"2026-10-22T16:01:38.40996034Z"} +{"acquire_ms":0,"bucket":".rustfs.sys","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"9811a447-de36-4725-a3bc-62921e50d2cf","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"buckets/rv-heal-e9da35d01389c2ae/.bucket-incarnation","op":"get_object","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"acquiring","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(7)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T16:01:38.410458527Z"} +{"acquire_ms":0,"bucket":".rustfs.sys","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"9811a447-de36-4725-a3bc-62921e50d2cf","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"buckets/rv-heal-e9da35d01389c2ae/.bucket-incarnation","op":"get_object","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"acquired","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(9)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T16:01:38.410760226Z"} +{"acquire_ms":0,"bucket":".rustfs.sys","component":"ecstore","event":"object_namespace_lock","filename":"crates/ecstore/src/diagnostics/object_lock.rs","hold_ms":0,"level":"TRACE","line_number":159,"lock_attempt_id":"9811a447-de36-4725-a3bc-62921e50d2cf","message":"Object namespace lock diagnostic","mode":"read","namespace":"pool-0","object":"buckets/rv-heal-e9da35d01389c2ae/.bucket-incarnation","op":"get_object","owner":"rustfs-node1:9000","request-id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","spans":[{"content_length":"1048773","content_type":"application/octet-stream","method":"PUT","name":"http-request","peer_addr":"::ffff:10.0.0.4","request_id":"f159b0a8-ab69-48a9-ae2c-f96751aa8844","span_id":"unknown","trace_id":"unknown","uri":"/rv-heal-e9da35d01389c2ae/objects/0000.bin","user_agent":"Boto3/1.43.77 md/Botocore#1.43.77 ua/2.1 os/linux#6.17.0-1022-azure md/arch#x86_64 lang/python#3.12.3 md/pyimpl#CPython m/e,N,Z,b,D cfg/retry-mode#legacy Botocore/1.43.77 auto-testing-i05-successor/5a37a691-d1d8-4591-b5a4-f174b7fe4818","version":"HTTP/1.1"}],"state":"guard_dropped","subsystem":"object_lock","target":"rustfs_ecstore::object_lock_diag","threadId":"ThreadId(9)","threadName":"rustfs-worker","timeout_ms":1000,"timestamp":"2026-10-22T16:01:38.411429057Z"} From 7741ef3c0c2445b6ed2fb2340e4049beb0e3e37a Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 01:22:15 +0800 Subject: [PATCH 302/343] docs(validation): record audited I05 successor diagnostics --- docs/validation-heal-cases.md | 2 +- docs/validation-heal-environment-gates.md | 2 +- docs/validation-heal-operations.md | 28 ++++++++++++++++++--- docs/validation-migration-map.md | 20 +++++++++++---- scripts/validation_heal/execution_policy.py | 2 +- scripts/validation_heal/recipes.json | 2 +- 6 files changed, 43 insertions(+), 13 deletions(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index a8b6a05b..e45f2795 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -1770,7 +1770,7 @@ Topologies: H1,H2. Branches: object-lock-timeout, caller-disconnect, stale-read- Execution policy: `evidence-blocked`; disabled by default: `false`. -Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 单连接历史 GET、真实争用 PUT 超时、服务 admission/owner/socket 和有界 FD 收尾诊断已通过独立 live 审计。已实现独立 queued-successor controller、候选绑定的 native 前置及失败现场隔离，尚缺该分支的独立 live 验收；仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。 +Blocked reason (`NATIVE_LOCK_OWNER_ABORT_BARRIER`): 已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 timeout 与 queued-successor 诊断均已通过独立 live 审计（39767336741/1、39817606025/1），包括原始 ACK、历史代次、真实锁事件、各 peer inventory、停机后新分片完整性和有界 FD 收尾。仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。 1. LOCK_WINDOW用本例慢IO/受控原生锁屏障持有目标对象锁，先证明真实争用超过预注册预算。 2. 同时heal/覆盖或删除；调用方断开实例仅断本次TCP连接，不杀服务器或关闭跳板机；记录请求开始/断开/服务在途结束。 diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index 982f239f..dce3f3b2 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -40,7 +40,7 @@ These observations do not certify a clean environment. Independently audit retai | --- | --- | --- | --- | | C11 | `heal.bucket-incarnation-recreation` | H1/H2 旧元数据盘重返、原始 checkpoint/resume 与 committed MRF 重放编排已实现；精确候选的 9 项存储原生回归和 3 项 manager incarnation 前置证据已独立核验。完整现场分支尚无独立审计结果 | 旧/新 incarnation、正常 S3 删除与新 ACK、物理范围和原生 artifact 精确绑定；单分支不提升整例 | | F07 | `heal.checkpoint-and-mrf-corruption` | 已实现 H1/H2 checkpoint 截断、未来 schema、committed MRF CRC 故障及独立安全重扫；10 项原生回归作为前置证据。真实较早代次回放与私有限额卷的 journal ENOSPC 已实现；仍缺完整 native cut 审核，完整 live 尚未审计 | 真实 durable 记录故障与 publication 证据绑定；现有 interruption hooks 不等于每个 fsync 或断电 | -| I05 | `heal.lock-timeout-and-caller-abort` | 同源码 native 子集已保留；H1/H2 单连接历史 GET、真实争用 PUT timeout、服务 admission/owner/socket、有界 FD 收尾诊断通过独立 live 审计（39767336741/1）；queued-successor controller、native 前置与现场隔离已实现，完整用例仍 BLOCKED | queued-successor 独立 live 验收、实际断连边界所有权、完整 native timeout/stale 屏障、stale-generation controller 和 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环 | +| I05 | `heal.lock-timeout-and-caller-abort` | 同源码 native 子集已保留；H1/H2 timeout 与 queued-successor 诊断已通过独立 live 审计（39767336741/1、39817606025/1），完整用例仍 BLOCKED | 实际断连边界所有权、完整 native timeout/stale 屏障、stale-generation controller 和 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环 | | J02 | `heal.outcome-counters-and-error-truth` | 全部夹具与 manifest 对账均已实现。候选 `7951bb629821246868ee3761b321dd07c06db45c` 修复整桶重试覆盖 repaired 记录，210 项 manager 与 5 项独立进程集成测试通过。新实机 `38874900846/1` 完成 20 个分支：16 个报告通过，4 个原始全元数据损坏探针返回永久 Failed，保留 NOT_OBSERVED；四组混合任务与全部 50 条节点清理记录已独立审计。真实 legacy Unknown 已观察，不能代替原始探针 | 原始元数据损坏分支的 Unknown 仍未观察；完整 source-procedure 验收尚未闭合。原始 Failed、NOT_OBSERVED 和完整用例门控保持不变 | I05 与 A09、G01–G06 的精确屏障证据允许使用同一不可变候选源码单独构建的 `cfg(test)` 原生测试执行文件。不得给优化服务候选加入隐藏 hook 冒充原包，也不能把缺少 release hook 本身归为外部环境阻塞。原生子集、完整服务调用和现场最终 ACK/物理检查分别验收；已验收 native 证据不能代替尚缺的 controller 或完整 live 结果。第一批 G07/G08 的 native 分支与 H04 容量锁窗口保留各自未完成条件；外部请求重试或最终对象正确均不能证明精确锁或提交窗口。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 26ed9173..08b45419 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -838,8 +838,9 @@ absence-of-leaks proof. All five node cleanup receipts show stopped services and removed owned roots. The original result is `BLOCKED/cleanup PASS`: the workflow fails its deliberately retained canonical gate despite successful diagnostic execution. Exact disconnect-edge ownership, distributed release, -complete native timeout/stale fences, stale-generation/queued-successor -controllers and full source H1/H2 DL/DV coverage remain unaccepted. No new +complete native timeout/stale fences, the stale-generation controller +and full source H1/H2 DL/DV coverage remain unaccepted. The separately audited +queued-successor diagnostic is recorded below. No new canonical case is counted. `validation_cleanup_run_id=` selects the narrow I05 preservation @@ -1061,8 +1062,27 @@ preserved all 180 original failed files without modification. Independent archive audit verified all four nodes' pre/move/post tree, device and inode baselines and twenty root-owned 0700 archive parents on their original devices. No S3 replay, namespace deletion, old-generation restoration or -original verdict rewrite occurred. Rebuild exact-helper tools before the -next H1/H2 live attempt. +original verdict rewrite occurred. + +Exact helper `3012a65` tools +[39812960916/1](https://github.com/rustfs/rustfs/actions/runs/39812960916) +passed independent verification of three official ZIPs, 22 original members, +ten exact native tests, 104 ordered milestones, actual hosted checkouts, +source/tree/Cargo.lock, optimized ELF and inspector quorum/size contracts. +Fresh live +[39817606025/1](https://github.com/rustfs/rustfs/actions/runs/39817606025) +completed H1 and H2 with five passing node cleanup receipts. Independent +audit byte-matched all 246 official files and verified nine seed request/ACK +pairs, eight final versions/markers per topology, one unretried successor +PUT ACK, retained historical physical generations, actual lock lifecycle +events and every peer's complete inventory. Stopped physical snapshots match +the ACK-bound scene before and after all four H1 and sixteen H2 new-shard +streaming receipts. Final FD counts were H1 vm000 `22 -> 22`; H2 vm000 +`108 -> 96`, vm001 `117 -> 113`, vm002 `105 -> 101`, vm003 `108 -> 104`. +Both diagnostic branches are independently accepted, with the original +`BLOCKED/cleanup PASS` unchanged. Precise disconnect-edge ownership, +distributed release and global absence of leaks are explicitly unproven; +the complete source DL/DV fixtures were not covered. Every successful diagnostic remains `BLOCKED/cleanup PASS` with `canonicalCasePassClaimed=false`. Exact disconnect-edge ownership, complete native timeout/stale fences, the diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 0fd16628..0a0603eb 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -51,8 +51,8 @@ Independent audit of 211 artifact files verified exact 1001 ms write-lock timeouts, closed caller sockets, unchanged physical content, full diagnostic seed ACK/peer inventories and final FD counts at or below baseline. The original canonical result remains `BLOCKED/cleanup PASS`; source DL/DV, -exact disconnect/native fences, stale-generation coverage and independent -queued-successor live acceptance remain outstanding. Exact-helper native/tools +exact disconnect/native fences and stale-generation coverage remain outstanding. +Independent queued-successor diagnostic acceptance is recorded below. Exact-helper native/tools [39783997956/1](https://github.com/rustfs/rustfs/actions/runs/39783997956) passed independent audit. The initial successor live [39788321908/1](https://github.com/rustfs/rustfs/actions/runs/39788321908) @@ -84,8 +84,18 @@ parents; every four-node pre/move/post tree, device and inode baseline matches. The parser now recognizes only the exact auxiliary warning bound to an already acquired target write; it cannot replace lifecycle evidence. The original sixteen-event H2 trace and incremental prefixes are covered by -positive and negative regression tests. Fresh helper-bound tools and H1/H2 -live acceptance remain required. Acceptance stays **108/158**. +positive and negative regression tests. Exact helper `3012a65` tools +[39812960916/1](https://github.com/rustfs/rustfs/actions/runs/39812960916) +passed independent archive/source/native and inspector audits. Fresh successor +[39817606025/1](https://github.com/rustfs/rustfs/actions/runs/39817606025) +completed H1/H2 and all five node cleanups. Independent audit byte-matched all +246 official files, nine seed request/ACK pairs and eight final versions/markers +per topology, one successor ACK, historical generations, lock events, every +peer inventory, ACK-bound stopped physical invariance and four/sixteen streaming +shard receipts. FD counts returned to or below each peer's baseline. The original +result remains `BLOCKED/cleanup PASS`; exact disconnect-edge ownership, +distributed release, global no-leak and full source DL/DV coverage are unproven. +Acceptance stays **108/158**. Fresh D03 [39801936817/1](https://github.com/rustfs/rustfs/actions/runs/39801936817) retains FAIL/cleanup PASS. Independent H1 re-evaluation verified all four @@ -297,7 +307,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -Four Heal cases remain outside canonical dispatch: C11, F07, I05 and J02. All four have explicit diagnostic routes that preserve their blocking canonical verdicts; I05 has a live-audited timeout diagnostic and an implemented queued-successor diagnostic with exact-source native prerequisites and stopped-scene isolation. The successor branch still needs independent live acceptance; exact server barriers and other source branches remain required. The 13 environment-gated cases have disabled-by-default control flows and oracles in `environment_cases.py`; actual execution requires an assigned and reviewed auto-testing resource adapter, and canonical results remain review-gated. See [the environment gate reference](validation-heal-environment-gates.md) for each missing resource, capability and native barrier. +Four Heal cases remain outside canonical dispatch: C11, F07, I05 and J02. All four have explicit diagnostic routes that preserve their blocking canonical verdicts; I05's H1/H2 timeout and queued-successor diagnostics have independently audited live results, exact-source native prerequisites and stopped-scene isolation. Exact server barriers and the complete source DL/DV branches remain required. The 13 environment-gated cases have disabled-by-default control flows and oracles in `environment_cases.py`; actual execution requires an assigned and reviewed auto-testing resource adapter, and canonical results remain review-gated. See [the environment gate reference](validation-heal-environment-gates.md) for each missing resource, capability and native barrier. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. diff --git a/scripts/validation_heal/execution_policy.py b/scripts/validation_heal/execution_policy.py index 88c7dd19..f007b366 100644 --- a/scripts/validation_heal/execution_policy.py +++ b/scripts/validation_heal/execution_policy.py @@ -139,7 +139,7 @@ def _native(code: str, reason_code: str, reason: str, capabilities: tuple[str, . ), "heal.lock-timeout-and-caller-abort": _native( "I05", "NATIVE_LOCK_OWNER_ABORT_BARRIER", - "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 单连接历史 GET、真实争用 PUT 超时、服务 admission/owner/socket 和有界 FD 收尾诊断已通过独立 live 审计。已实现独立 queued-successor controller、候选绑定的 native 前置及失败现场隔离，尚缺该分支的独立 live 验收；仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。", + "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 timeout 与 queued-successor 诊断均已通过独立 live 审计（39767336741/1、39817606025/1），包括原始 ACK、历史代次、真实锁事件、各 peer inventory、停机后新分片完整性和有界 FD 收尾。仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。", ("server_lock_owner", "lock_timeout_fence", "caller_abort_sequence", "stale_read_repair_fence", "exact_source_native_evidence", "complete_live_orchestrator", "final_object_lock_fd_closure"), ), "heal.outcome-counters-and-error-truth": _native( diff --git a/scripts/validation_heal/recipes.json b/scripts/validation_heal/recipes.json index f278f4f2..a6935225 100644 --- a/scripts/validation_heal/recipes.json +++ b/scripts/validation_heal/recipes.json @@ -3253,7 +3253,7 @@ "dispatchable": false, "code": "I05", "reasonCode": "NATIVE_LOCK_OWNER_ABORT_BARRIER", - "reason": "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 单连接历史 GET、真实争用 PUT 超时、服务 admission/owner/socket 和有界 FD 收尾诊断已通过独立 live 审计。已实现独立 queued-successor controller、候选绑定的 native 前置及失败现场隔离，尚缺该分支的独立 live 验收；仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。", + "reason": "已有同候选源码的 lock owner、取消物理 unlink 与 stale read-repair 原生子集；H1/H2 timeout 与 queued-successor 诊断均已通过独立 live 审计（39767336741/1、39817606025/1），包括原始 ACK、历史代次、真实锁事件、各 peer inventory、停机后新分片完整性和有界 FD 收尾。仍缺实际断连边界的精确所有权、完整 native timeout/stale 屏障、stale-generation controller，以及 H1/H2 DL/DV 完整对象/ACK/物理/锁/FD 闭环。精确源码 cfg(test) 原生证据独立保留；诊断子集和原生子集不能充作完整 canonical PASS。", "requiredCapabilities": [ "server_lock_owner", "lock_timeout_fence", From 632e3c4c7b8ee94c2b7a579eb991bc8a787340f6 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 03:22:32 +0800 Subject: [PATCH 303/343] fix(validation): require complete B05 member readiness --- docs/validation-heal-cases.md | 9 ++ scripts/validation_heal/versions.py | 55 ++++++++++- test_validation_heal.py | 148 ++++++++++++++++++++++++++++ 3 files changed, 210 insertions(+), 2 deletions(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index e45f2795..1f087ca4 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -334,6 +334,15 @@ Source assertions: Topologies: H1,H2. Branches: loss-1-data, loss-1-parity, loss-2-through-m-data-parity. +Before seeding, before each fault and after each faulted restart, the executor +observes signed read-only Admin info from every H1/H2 endpoint for at most +90 seconds. All 4/16 slots must match the frozen format UUIDs and report online; +read readiness alone does not prove this prerequisite. Every observation is +retained. Incomplete or foreign identities fail immediately; persistent offline +members expire as `NOT_OBSERVED` without retrying the Heal task. The original +`38200772548` unknown-version receipts remain a retained failure. This startup +check alone does not establish its underlying cause or a new live PASS. + 1. 从候选/实际元数据确定 M=N-R，枚举 1..M 的合法缺片数及 data/parity 组合，写入本批展开参数表；不得只运行 loss=1。 2. 每实例用相同版本的新对象，将指定成员上的 part 分别 FAULT missing；多份元数据也要证明指向同 version/data_dir。 3. 在满足修复提交条件时 START deep/WAIT；先检查全部目标片与角色基线相同，再 VERIFY。 diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 06442d45..d8f55c3c 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -15,6 +15,7 @@ import secrets import signal import sys +import time from urllib.parse import quote try: @@ -33,7 +34,7 @@ from .markers import (BRANCHES as MARKER_BRANCHES, INSTANCES as MARKER_INSTANCES, seed_markers, marker_plan, prepare_stale, verify_markers, reveal_current) from .stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch - from .disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch + from .disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch, info_disks from .replacement import BRANCHES as REPLACEMENT_BRANCHES, AUTOMATIC_BRANCHES as AUTO_REPLACEMENT_BRANCHES, GRACEFUL_BRANCHES, CRASH_BRANCHES, run_branch as replacement_branch from .shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) @@ -53,7 +54,7 @@ from markers import (BRANCHES as MARKER_BRANCHES, INSTANCES as MARKER_INSTANCES, seed_markers, marker_plan, prepare_stale, verify_markers, reveal_current) from stale_rejoin import BRANCHES as STALE_BRANCHES, run_branch as stale_branch - from disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch + from disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch, info_disks from replacement import BRANCHES as REPLACEMENT_BRANCHES, AUTOMATIC_BRANCHES as AUTO_REPLACEMENT_BRANCHES, GRACEFUL_BRANCHES, CRASH_BRANCHES, run_branch as replacement_branch from shards import (SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort, verify_ranges) @@ -340,6 +341,50 @@ def metadata_target(snapshot, part_target): return {**part_target, "relativePath": relative, "sha256": entry["sha256"], "size": entry["size"]} +def validate_all_members_online(observed, snapshot): + topology = validate_topology(snapshot, "H1" if len(snapshot) == 1 else "H2") + indexed = info_disks(observed, topology["independentDrives"]) + require(len({disk["endpoint"] for disk in indexed.values()}) == len(indexed), + "multi-loss Admin members reuse a physical endpoint") + members = [] + for slot, expected_uuid in enumerate(topology["diskUUIDs"]): + disk = indexed[0, 0, slot] + require(disk.get("uuid") in {None, "", expected_uuid}, + "multi-loss Admin slot differs from the frozen format UUID") + members.append({"slot": [0, 0, slot], "uuid": disk.get("uuid"), "endpoint": disk["endpoint"], + "state": disk.get("state"), "runtimeState": disk.get("runtimeState")}) + online = all(member["uuid"] == topology["diskUUIDs"][member["slot"][2]] + and member["state"] == "ok" and member["runtimeState"] in {None, "online"} + for member in members) + return {"allFormattedMembersOnline": online, "members": members} + + +def wait_all_members_online(probe, fleet, snapshot, label): + # Read readiness only proves quorum. Poll complete member views before the + # single admitted repair; no task or fixture operation is retried here. + deadline = time.monotonic() + 90 + while True: + views = {} + for endpoint in probe.binding["endpoints"]: + require(time.monotonic() < deadline, + "NOT_OBSERVED: multi-loss full-member startup exceeded 90 seconds") + reader = Probe(probe.binding, endpoint) + observed = fleet.record("multi-loss-members-" + label + "-" + endpoint, + reader.admin("/rustfs/admin/v3/info", [], b"", method="GET")) + views[endpoint] = validate_all_members_online(observed, snapshot) + proof = fleet.record("multi-loss-members-proof-" + label, + {"views": views, "allEndpointViewsOnline": all( + view["allFormattedMembersOnline"] for view in views.values()), + "observationOnly": True, "healTaskRetried": False}) + if proof["allEndpointViewsOnline"]: + require(time.monotonic() < deadline, + "NOT_OBSERVED: multi-loss full-member startup exceeded 90 seconds") + return proof + remaining = deadline - time.monotonic() + require(remaining > 0, "NOT_OBSERVED: multi-loss full-member startup exceeded 90 seconds") + time.sleep(min(2, remaining)) + + def multi_loss_instances(snapshot, bucket, expected, fixture): sample = next(entry for entry in expected if not entry["deleteMarker"]) target = choose_target(snapshot, bucket, sample, "data") @@ -762,6 +807,8 @@ def run_topology(args, topology, nodes, fixture): fleet.deploy(args.binary_sha, bucket, access, secret) processes = fleet.snapshot("deployed-topology") fleet.record("topology-proof", validate_topology(processes, topology)) + if args.case_id == "heal.multiple-shard-loss": + wait_all_members_online(probe, fleet, processes, "before-seed") scanner = args.case_id == "heal.scanner-cold-object-repair" if scanner: fleet.record("scanner-registered-budget", {"perBranchSeconds": 240, "sourceBranches": list(SCANNER_BRANCHES), @@ -840,6 +887,8 @@ def run_topology(args, topology, nodes, fixture): if lock_context is not None: fleet.record("lock-before-fault", prove_locked(probe, fleet, bucket, lock_context, key=LOCK_KEYS[LOCK_BRANCHES.index(branch)], label="lock-before-" + branch)) + if args.case_id == "heal.multiple-shard-loss": + wait_all_members_online(probe, fleet, processes, "before-fault-" + branch) fleet.stop() if args.case_id in {"heal.disk-replacement-formatted", "heal.replacement-auto-rebuild", "heal.target-graceful-restart", "heal.target-process-crash"}: stale_marker_started = True @@ -904,6 +953,8 @@ def run_topology(args, topology, nodes, fixture): fleet.record("drive-gate-armed", fleet.remote(gate_node, "drive-gate", disk=plan["proofs"][0]["disk"], mode=plan["driveGate"])) fleet.start() + if args.case_id == "heal.multiple-shard-loss": + wait_all_members_online(probe, fleet, before, "before-task-" + branch) if gate_node is not None: fleet.record("service-write-denial", fleet.remote(gate_node, "drive-gate-probe")) witness = fleet.snapshot("physical-before-trigger-" + branch, metadata_faults=metadata_faults) diff --git a/test_validation_heal.py b/test_validation_heal.py index c66b2b90..2bc05bcf 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -24,6 +24,7 @@ map_inline_target, verify_proof) from scripts.validation_heal.versions import seed_semantics, verify_semantics from scripts.validation_heal.versions import multi_loss_instances, multi_loss_plan, SPECIAL_KEYS, verify_pagination, boundary_plan +from scripts.validation_heal.versions import validate_all_members_online, wait_all_members_online from scripts.validation_heal.heal_probe import Probe, validate_s3_request from scripts.validation_heal.prepare_heal_meta_inspector import quorum_contract, size_contract from scripts.validation_heal.boundaries import verify_size_contract, boundary_sizes, verify_boundary_ranges @@ -3037,6 +3038,153 @@ def test_nonce_bound_ack_reaches_inventory_checks_without_weakening_public_manif verify_seed([{**entry, "payloadNonce": nonce}], inventory) +class HealMultiLossMemberReadinessTest(unittest.TestCase): + def scene(self, total): + snapshot, expected = HealReadQuorumBoundaryTest().scene(total=total, data=2 if total == 4 else 12) + target = map_target(snapshot, HealReadQuorumBoundaryTest.bucket, expected[0]["key"], "data", 1) + observed = HealDiskLossTest().admin_info(target) + drives = observed["response"]["info"]["servers"][0]["drives"] + observed["response"]["info"]["servers"] = [{"drives": drives[index:index + 4]} + for index in range(0, total, 4)] + return snapshot, observed + + def test_complete_formatted_members_are_required_even_with_read_quorum(self): + for total in (4, 16): + snapshot, online = self.scene(total) + self.assertTrue(validate_all_members_online(online, snapshot)["allFormattedMembersOnline"]) + changed = copy.deepcopy(online) + drives = changed["response"]["info"]["servers"][0]["drives"] + for disk in drives[:1 if total == 4 else 4]: + disk.update(state="disk not found", runtimeState="offline", uuid="") + self.assertFalse(validate_all_members_online(changed, snapshot)["allFormattedMembersOnline"]) + for field, value in (("runtimeState", "offline"), ("runtimeState", "unknown"), + ("state", "initializing"), ("uuid", None)): + snapshot, changed = self.scene(16) + changed["response"]["info"]["servers"][0]["drives"][0][field] = value + with self.subTest(field=field, value=value): + self.assertFalse(validate_all_members_online(changed, snapshot)["allFormattedMembersOnline"]) + + def test_malformed_or_foreign_member_evidence_fails_without_polling(self): + for kind in ("http", "omitted", "duplicate-slot", "foreign-uuid", "duplicate-uuid", + "duplicate-endpoint", "wrong-pool", "bool-slot"): + snapshot, changed = self.scene(16) + servers = changed["response"]["info"]["servers"] + drives = servers[0]["drives"] + if kind == "http": changed["httpStatus"] = 503 + elif kind == "omitted": servers.pop() + elif kind == "duplicate-slot": drives[1]["disk_index"] = drives[0]["disk_index"] + elif kind == "foreign-uuid": drives[0]["uuid"] = "foreign" + elif kind == "duplicate-uuid": drives[1]["uuid"] = drives[0]["uuid"] + elif kind == "duplicate-endpoint": drives[1]["endpoint"] = drives[0]["endpoint"] + elif kind == "wrong-pool": drives[0]["pool_index"] = 1 + else: drives[0]["disk_index"] = False + probe, fleet, reader = Mock(), Mock(), Mock() + probe.binding = {"endpoints": {"node1": "http://rustfs-node1:9000"}} + fleet.record.side_effect = lambda label, value: value + reader.admin.return_value = changed + with self.subTest(kind=kind), patch("scripts.validation_heal.versions.Probe", return_value=reader), \ + patch("scripts.validation_heal.versions.time.sleep") as sleep, self.assertRaises(ValueError): + wait_all_members_online(probe, fleet, snapshot, "before-task") + reader.admin.assert_called_once_with("/rustfs/admin/v3/info", [], b"", method="GET") + sleep.assert_not_called() + + def test_polling_records_each_endpoint_and_never_admits_a_heal(self): + snapshot, online = self.scene(16) + offline = copy.deepcopy(online) + for disk in offline["response"]["info"]["servers"][1]["drives"]: + disk.update(state="offline", runtimeState="offline") + probe, fleet = Mock(), Mock() + probe.binding = {"endpoints": {f"node{index}": f"http://rustfs-node{index}:9000" for index in range(1, 5)}} + fleet.record.side_effect = lambda label, value: value + readers = {endpoint: Mock() for endpoint in probe.binding["endpoints"]} + for endpoint, reader in readers.items(): + reader.admin.side_effect = [offline if endpoint == "node2" else online, online] + with patch("scripts.validation_heal.versions.Probe", side_effect=lambda binding, endpoint: readers[endpoint]), \ + patch("scripts.validation_heal.versions.time.sleep") as sleep, \ + patch("scripts.validation_heal.versions.run_task") as task: + result = wait_all_members_online(probe, fleet, snapshot, "before-task") + self.assertTrue(result["allEndpointViewsOnline"]) + self.assertEqual(set(result["views"]), set(readers)) + for reader in readers.values(): + self.assertEqual(reader.admin.call_count, 2) + self.assertTrue(all(call.args == ("/rustfs/admin/v3/info", [], b"") + and call.kwargs == {"method": "GET"} for call in reader.admin.call_args_list)) + self.assertEqual(sum("proof" in call.args[0] for call in fleet.record.call_args_list), 2) + sleep.assert_called_once() + task.assert_not_called() + fleet.remote.assert_not_called() + + def test_persistent_offline_and_late_online_observations_expire(self): + for late_online in (False, True): + snapshot, observed = self.scene(4) + if not late_online: + observed["response"]["info"]["servers"][0]["drives"][0].update(state="offline") + probe, fleet, reader = Mock(), Mock(), Mock() + probe.binding = {"endpoints": {"node1": "http://rustfs-node1:9000"}} + fleet.record.side_effect = lambda label, value: value + reader.admin.return_value = observed + with self.subTest(late_online=late_online), \ + patch("scripts.validation_heal.versions.Probe", return_value=reader), \ + patch("scripts.validation_heal.versions.time.monotonic", side_effect=[0, 0, 91]), \ + patch("scripts.validation_heal.versions.time.sleep") as sleep, \ + self.assertRaisesRegex(ValueError, "NOT_OBSERVED.*90 seconds"): + wait_all_members_online(probe, fleet, snapshot, "before-task") + reader.admin.assert_called_once() + sleep.assert_not_called() + + def test_runner_checks_members_before_seed_fault_and_the_single_task(self): + from scripts.validation_heal import versions + snapshot, _ = self.scene(4) + for rejected in ("before-seed", "before-fault-instance", "before-task-instance", None): + with self.subTest(rejected=rejected), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + candidate = root / "candidate.json" + candidate.write_text(json.dumps({"source_sha": "a" * 40, "package_sha256": "b" * 64})) + rc = root / "rc" + rc.write_bytes(b"rc") + args = types.SimpleNamespace(evidence=root / "evidence", candidate=candidate, + case_id="heal.multiple-shard-loss", tool_directory=root, testing_commit="c" * 40, + binary_sha="d" * 64, rc_bin=rc) + fleet, probe = Mock(), Mock() + events = [] + fleet.record.side_effect = lambda label, value: value + fleet.snapshot.return_value = snapshot + fleet.cleanup.return_value = True + plan = {"scope": {}, "recursive": True, "expected": [], "proofs": [], "faults": [], + "requiredRepairs": set(), "metadataRepairs": set()} + def ready(probe, fleet, actual, label): + events.append(label) + if label == rejected: + raise ValueError("NOT_OBSERVED: startup members remain offline") + def seeded(*args): + events.append("seed") + return [] + def fault_plan(*args): + events.append("fault-plan") + return plan + def task(*args, **kwargs): + events.append("task") + raise ValueError("stop after the single admitted task") + with patch.dict("os.environ", {"GITHUB_RUN_ID": "123", "GITHUB_RUN_ATTEMPT": "1"}), \ + patch.object(versions, "Fleet", return_value=fleet), \ + patch.object(versions, "Probe", return_value=probe), \ + patch.object(versions, "verify_inspector", return_value={}), \ + patch.object(versions, "Recorder"), \ + patch.object(versions, "seed", side_effect=seeded), \ + patch.object(versions, "verify_seed", return_value={}), \ + patch.object(versions, "multi_loss_instances", return_value=[{"name": "instance"}]), \ + patch.object(versions, "multi_loss_plan", side_effect=fault_plan), \ + patch.object(versions, "wait_all_members_online", side_effect=ready), \ + patch.object(versions, "physical_content", return_value={}), \ + patch.object(versions, "run_task", side_effect=task) as repair: + result = versions.run_topology(args, "H1", ["vm000"], "DV") + wanted = ["before-seed", "seed", "before-fault-instance", "fault-plan", "before-task-instance", "task"] + self.assertEqual(events, wanted if rejected is None else wanted[:wanted.index(rejected) + 1]) + self.assertEqual(repair.call_count, int(rejected is None)) + self.assertEqual(result["verdict"], "FAIL") + self.assertEqual(result["cleanup"], "PASS") + + class HealDiskLossTest(unittest.TestCase): def scene(self, total=4, data=2): original, old = HealReadQuorumBoundaryTest().scene(total=total, data=data) From 1f1ed10987e2609245d61617d7e702dd6591a251 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 03:31:32 +0800 Subject: [PATCH 304/343] fix(validation): restart B05 service after retained branch failure --- scripts/validation_heal/versions.py | 5 +++++ test_validation_heal.py | 24 ++++++++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index d8f55c3c..9f76f812 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -878,6 +878,7 @@ def run_topology(args, topology, nodes, fixture): if args.case_id == "heal.delete-marker-semantics": branches = [entry["name"] for entry in instances] fleet.record("marker-parameter-matrix", {"instances": instances, "sourceBranches": list(MARKER_BRANCHES)}) + restart_after_failure = False for branch in branches: registrations = [] metadata_faults = {} @@ -888,6 +889,9 @@ def run_topology(args, topology, nodes, fixture): fleet.record("lock-before-fault", prove_locked(probe, fleet, bucket, lock_context, key=LOCK_KEYS[LOCK_BRANCHES.index(branch)], label="lock-before-" + branch)) if args.case_id == "heal.multiple-shard-loss": + if restart_after_failure: + fleet.start() + restart_after_failure = False wait_all_members_online(probe, fleet, processes, "before-fault-" + branch) fleet.stop() if args.case_id in {"heal.disk-replacement-formatted", "heal.replacement-auto-rebuild", "heal.target-graceful-restart", "heal.target-process-crash"}: @@ -1107,6 +1111,7 @@ def run_topology(args, topology, nodes, fixture): raise if isinstance(exc, KeyboardInterrupt) or active or stale_marker_started: raise + restart_after_failure = args.case_id == "heal.multiple-shard-loss" result["verdict"] = "PASS" if len(result["branches"]) == len(branches) and all( item["verdict"] == "PASS" for item in result["branches"].values()) else "FAIL" if args.case_id == "heal.versioning-suspended-null" and result["verdict"] == "PASS": diff --git a/test_validation_heal.py b/test_validation_heal.py index 2bc05bcf..44d5b4c0 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -3135,7 +3135,10 @@ def test_persistent_offline_and_late_online_observations_expire(self): def test_runner_checks_members_before_seed_fault_and_the_single_task(self): from scripts.validation_heal import versions snapshot, _ = self.scene(4) - for rejected in ("before-seed", "before-fault-instance", "before-task-instance", None): + scenarios = [(rejected, ["instance"]) for rejected in + ("before-seed", "before-fault-instance", "before-task-instance", None)] + scenarios.append(("before-task-instance", ["instance", "successor"])) + for rejected, names in scenarios: with self.subTest(rejected=rejected), tempfile.TemporaryDirectory() as directory: root = Path(directory) candidate = root / "candidate.json" @@ -3147,12 +3150,19 @@ def test_runner_checks_members_before_seed_fault_and_the_single_task(self): binary_sha="d" * 64, rc_bin=rc) fleet, probe = Mock(), Mock() events = [] + fleet.drive_gates = set() + fleet.faults = [] + service = {"running": False} + fleet.deploy.side_effect = lambda *args: service.update(running=True) + fleet.stop.side_effect = lambda: service.update(running=False) + fleet.start.side_effect = lambda: service.update(running=True) fleet.record.side_effect = lambda label, value: value fleet.snapshot.return_value = snapshot fleet.cleanup.return_value = True plan = {"scope": {}, "recursive": True, "expected": [], "proofs": [], "faults": [], "requiredRepairs": set(), "metadataRepairs": set()} def ready(probe, fleet, actual, label): + self.assertTrue(service["running"], "member observation requires the recovered service") events.append(label) if label == rejected: raise ValueError("NOT_OBSERVED: startup members remain offline") @@ -3172,15 +3182,21 @@ def task(*args, **kwargs): patch.object(versions, "Recorder"), \ patch.object(versions, "seed", side_effect=seeded), \ patch.object(versions, "verify_seed", return_value={}), \ - patch.object(versions, "multi_loss_instances", return_value=[{"name": "instance"}]), \ + patch.object(versions, "multi_loss_instances", return_value=[{"name": name} for name in names]), \ patch.object(versions, "multi_loss_plan", side_effect=fault_plan), \ patch.object(versions, "wait_all_members_online", side_effect=ready), \ patch.object(versions, "physical_content", return_value={}), \ patch.object(versions, "run_task", side_effect=task) as repair: result = versions.run_topology(args, "H1", ["vm000"], "DV") wanted = ["before-seed", "seed", "before-fault-instance", "fault-plan", "before-task-instance", "task"] - self.assertEqual(events, wanted if rejected is None else wanted[:wanted.index(rejected) + 1]) - self.assertEqual(repair.call_count, int(rejected is None)) + if len(names) == 2: + wanted = wanted[:-1] + ["before-fault-successor", "fault-plan", "before-task-successor", "task"] + elif rejected is not None: + wanted = wanted[:wanted.index(rejected) + 1] + self.assertEqual(events, wanted) + self.assertEqual(repair.call_count, int(rejected is None or len(names) == 2)) + if len(names) == 2: + self.assertEqual(result["branches"]["instance"]["verdict"], "FAIL") self.assertEqual(result["verdict"], "FAIL") self.assertEqual(result["cleanup"], "PASS") From 76dbf7029e82c1e3878c9160090804d4afc38078 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 03:33:48 +0800 Subject: [PATCH 305/343] fix(validation): bound Scanner logs to source attribution targets --- scripts/validation_heal/fleet_node.py | 4 +++- test_validation_heal.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 68271446..1e74416e 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -112,7 +112,9 @@ def runtime_config(profile): logging_filter = "info,rustfs_ecstore=debug,rustfs::heal=debug" return {**BACKGROUND_CONFIG, "RUSTFS_OBS_LOGGER_LEVEL": logging_filter, "RUST_LOG": logging_filter} if profile == "scanner": - return {**BACKGROUND_CONFIG, "RUSTFS_SCANNER_ENABLED": "true", "RUSTFS_OBS_LOGGER_LEVEL": "debug", + logging_filter = "info,rustfs::scanner=debug,rustfs::heal=debug" + return {**BACKGROUND_CONFIG, "RUSTFS_SCANNER_ENABLED": "true", + "RUSTFS_OBS_LOGGER_LEVEL": logging_filter, "RUST_LOG": logging_filter, "RUSTFS_SCANNER_START_DELAY_SECS": "5", "RUSTFS_SCANNER_CYCLE": "5", "RUSTFS_SCANNER_SPEED": "fastest", "RUSTFS_SCANNER_DELAY": "0", "RUSTFS_SCANNER_BITROT_CYCLE_SECS": "0", "RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS": "0", diff --git a/test_validation_heal.py b/test_validation_heal.py index 44d5b4c0..d26e7d8f 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -1492,6 +1492,9 @@ def remote(node, action, **params): def test_scanner_profile_enables_only_scanner_source_and_fixed_deep_timing(self): config = runtime_config("scanner") + logging_filter = "info,rustfs::scanner=debug,rustfs::heal=debug" + self.assertEqual(config["RUSTFS_OBS_LOGGER_LEVEL"], logging_filter) + self.assertEqual(config["RUST_LOG"], logging_filter) self.assertEqual(config["RUSTFS_SCANNER_ENABLED"], "true") for name in ("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "RUSTFS_HEAL_MRF_ENABLE"): self.assertEqual(config[name], "false") From 70f6146ee236b38e219a1f36cf8a217ed1d3c9ad Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 04:54:39 +0800 Subject: [PATCH 306/343] docs(validation): record complete B05 acceptance --- docs/validation-migration-map.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 0a0603eb..91ccf7b1 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -9,17 +9,17 @@ live run proves each source assertion on the required topology. ## Current acceptance and remaining work -Count reconciliation (2026-10-22): the four additions after the 104-case -baseline are C02, C03, C05 and I04. Their original raw-evidence auditors were +Count reconciliation (2026-10-23): the five additions after the 104-case +baseline are C02, C03, C05, I04 and B05. Their original raw-evidence auditors were rerun against the retained official archives; all results match the recorded audits. Candidate/helper identities, complete H1/H2 branch coverage, cleanup, acceptance review chains and unchanged earlier failures were reconciled to -`canonical-migration-ledger-20261020-i04.json`. Each acceptance retains its own +`canonical-migration-ledger-20261023-b05.json`. Each acceptance retains its own immutable candidate; this is migration coverage across those candidates, not a fresh replay of all cases on the current I05 package. Partial I05, native and recovery diagnostics add no canonical acceptance. -All 158 canonical IDs are retained. Independent complete live acceptance is **108/158**: the prior 104-case baseline plus the four reconciled cases below. The baseline includes B07 `heal.below-read-quorum` ([run 38727279347/1](https://github.com/rustfs/rustfs/actions/runs/38727279347)), A10 `heal.admin-option-contract` ([run 39197088183/1](https://github.com/rustfs/rustfs/actions/runs/39197088183)) and B06 `heal.read-quorum-boundary` ([run 39223475622/1](https://github.com/rustfs/rustfs/actions/runs/39223475622)). Complete source matrices, immutable candidate/package/tools and cleanup were independently audited; earlier failures remain retained. +All 158 canonical IDs are retained. Independent complete live acceptance is **109/158**: the prior 104-case baseline plus the five reconciled cases below. The baseline includes B07 `heal.below-read-quorum` ([run 38727279347/1](https://github.com/rustfs/rustfs/actions/runs/38727279347)), A10 `heal.admin-option-contract` ([run 39197088183/1](https://github.com/rustfs/rustfs/actions/runs/39197088183)) and B06 `heal.read-quorum-boundary` ([run 39223475622/1](https://github.com/rustfs/rustfs/actions/runs/39223475622)). Complete source matrices, immutable candidate/package/tools and cleanup were independently audited; earlier failures remain retained. | Reconciled case | Complete live run | Required coverage | Original result SHA-256 | | --- | --- | --- | --- | @@ -27,16 +27,21 @@ All 158 canonical IDs are retained. Independent complete live acceptance is **10 | C03 `heal.parity-shard-bitrot` | [39286997859/1](https://github.com/rustfs/rustfs/actions/runs/39286997859) | Six H1/H2 branch instances, 2,904 archive files | `da2df2e15aa973b6a04a0cb40491413db17426b63d6367f11682ef579b84235a` | | C05 `heal.missing-or-corrupt-xlmeta` | [39322217398/1](https://github.com/rustfs/rustfs/actions/runs/39322217398) | Ten H1/H2 DB/DV instances, 3,132 archive files | `489b8179ee90a7412b223e669db8fd4f6520b9e69051cbf927b6f33299de423b` | | I04 `heal.queue-pressure-and-admission` | [39354651483/1](https://github.com/rustfs/rustfs/actions/runs/39354651483) | Four branches per topology, six exact durable tasks, 5,565 archive files | `4dcde2f34bb7df52f03f5a3ec1e87663145a7134d3ef1a513fda2da52d1709a2` | +| B05 `heal.multiple-shard-loss` | [39843970314/1](https://github.com/rustfs/rustfs/actions/runs/39843970314) | All 38 DB/DV and H1/H2 instances, 9,059 archive files, ten node cleanups and four package purges | `2624f43fc8581eb1b533a477b1a869f9c72208861d7a268ea35ae8f36153fd7e` | | Acceptance state | Cases | Remaining requirement | | --- | ---: | --- | -| Independently accepted complete live PASS | 108 | Preserve original provenance and complete results. | -| Retained live failures with existing executors | 26 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | +| Independently accepted complete live PASS | 109 | Preserve original provenance and complete results. | +| Retained live failures with existing executors | 25 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | | Required native barriers | 7 | A09 and G01–G06 need exact candidate-bound server lock/version/commit barriers. | | Orchestration or evidence blocked | 4 | C11, F07, I05 and J02 need their remaining native or full live evidence closure. | | Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | -The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 37 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. +The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 36 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. + +B05 acceptance uses candidate `7ed6c9320db491c84b54e8232152d7b1282c1b3f`, helper `76dbf7029e82c1e3878c9160090804d4afc38078`, package `39827676705/1` and tools `39841369555/1`. The complete official ZIP digest is `4eb3405af1f47583cd7c2d8e5b9ce9fa6910b76665df529d7dfbe4f1399a20d8`. Independent recomputation covers seed ACKs, full versions/markers, the actual 2+2 and 12+4 role/loss matrices, every endpoint's full online membership before each required boundary, sequential faults, one unique START/token per instance, two identical terminal outcome queries, strict integer scope, exact repaired/healthy responsibility, all physical shard and bitrot proofs, complete S3/HEAD/Range evidence before finalization, ten private-node cleanups and four package purges. No backup restoration was used. Original failure `38200772548` remains unchanged; its vm001 offline root cause is not established by the later pass. + +E04 live [39842352595/1](https://github.com/rustfs/rustfs/actions/runs/39842352595) retains FAIL/cleanup PASS. All 1,402 official files, three H1 branches and five node cleanups were independently verified. H2's eight complete journal windows no longer exhaust pagination, but its healthy Scanner task completed on vm002 while the exact node3 token query was forwarded to vm000 and returned notFound. No H2 fault branch ran. RustFS commit `553be4b4c62ae0c590e82f91817bc018bdfdc660` lets authenticated original local automatic-task tokens return the source/path-bound in-memory snapshot; Admin/Internal/unknown/alias identities retain coordinator routing. The fixed candidate `9d90884c304dcfbf97057c263c6f2ee110230ce7` adds only that reviewed patch to `7ed6c932`, with unchanged Cargo.lock. A new immutable package, matching tools and complete six-branch live acceptance remain required; unit tests do not promote E04. I05 progress (2026-10-22): four-node startup scene `39764496504/1` was independently verified and isolated by @@ -95,7 +100,7 @@ peer inventory, ACK-bound stopped physical invariance and four/sixteen streaming shard receipts. FD counts returned to or below each peer's baseline. The original result remains `BLOCKED/cleanup PASS`; exact disconnect-edge ownership, distributed release, global no-leak and full source DL/DV coverage are unproven. -Acceptance stays **108/158**. +That I05 diagnostic added no acceptance; the current total is **109/158** after the separate complete B05 audit. Fresh D03 [39801936817/1](https://github.com/rustfs/rustfs/actions/runs/39801936817) retains FAIL/cleanup PASS. Independent H1 re-evaluation verified all four From ad07638bff44e8ee2c9310e275ca1f16f0c479e1 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 05:39:56 +0800 Subject: [PATCH 307/343] docs(validation): record independent J02 branch evidence --- docs/validation-migration-map.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 91ccf7b1..e7ceae30 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -120,6 +120,14 @@ A10 run `39197088183/1` uses immutable helper `b16e6072679d6a60a044b9d4520b30b4a The sections below retain earlier implementation and run history. Historical failed or partial results are not promoted by this summary. +## Supplemental J02 evidence audit + +The 2026-10-23 supplemental independent audit closes twelve positive branch instances from original J02 run 38874900846: grace Deferred/new-token cleanup, read-only terminal failure/new-token repair and dry-run across H1/H2 and DB/DV. Every original archive member was byte-checked again, all fixture requests/ACK digests and version/marker manifests were rebuilt, and the exact executed helper was pinned. + +Grace uses both original ACK versions, natural 120-second expiry under unchanged candidate processes, the exact typed future retry time, two unchanged original-token terminal queries and every H2 peer, a distinct compensating token, physical post-DELETE authority before GET/rescue and complete peer inventories with old-version refusal. Read-only verifies actual private-namespace EROFS, the protected-control incarnation, unchanged failed and released scenes, a distinct repair token, original full shard/bitrot evidence before GET/rescue and all peer inventories. Dry-run retains every original typed disposition, the actual R-1 fault geometry and unchanged fault bytes. All twelve node cleanup groups bind their original reservations. + +Together with the earlier four mixed-branch audits, sixteen branch instances now have independent acceptance; the four original opaque-metadata Unknown probes still prevent whole-case acceptance. The supplemental report SHA-256 is `4424b3e681931527d244a4ca0211dde534a8145cd95ea0b460f388262ccb8c10`. Historical results and the 109/158 canonical count remain unchanged. + ## Original reuse assessment | Source module | Cases | Existing auto-testing code to inspect | Current gap | From cd36287f1abe42662333fab31cf45ac23de91c8e Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 06:08:19 +0800 Subject: [PATCH 308/343] fix(validation): bind remote Scanner tasks to leader cycles --- docs/validation-migration-map.md | 4 +- scripts/validation_heal/scanner_repair.py | 51 ++++++++++++++++--- test_validation_heal.py | 62 +++++++++++++++++++++++ 3 files changed, 109 insertions(+), 8 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index e7ceae30..121cacdf 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -41,7 +41,9 @@ The current delivery keeps the 13 environment cases disabled with explicit reaso B05 acceptance uses candidate `7ed6c9320db491c84b54e8232152d7b1282c1b3f`, helper `76dbf7029e82c1e3878c9160090804d4afc38078`, package `39827676705/1` and tools `39841369555/1`. The complete official ZIP digest is `4eb3405af1f47583cd7c2d8e5b9ce9fa6910b76665df529d7dfbe4f1399a20d8`. Independent recomputation covers seed ACKs, full versions/markers, the actual 2+2 and 12+4 role/loss matrices, every endpoint's full online membership before each required boundary, sequential faults, one unique START/token per instance, two identical terminal outcome queries, strict integer scope, exact repaired/healthy responsibility, all physical shard and bitrot proofs, complete S3/HEAD/Range evidence before finalization, ten private-node cleanups and four package purges. No backup restoration was used. Original failure `38200772548` remains unchanged; its vm001 offline root cause is not established by the later pass. -E04 live [39842352595/1](https://github.com/rustfs/rustfs/actions/runs/39842352595) retains FAIL/cleanup PASS. All 1,402 official files, three H1 branches and five node cleanups were independently verified. H2's eight complete journal windows no longer exhaust pagination, but its healthy Scanner task completed on vm002 while the exact node3 token query was forwarded to vm000 and returned notFound. No H2 fault branch ran. RustFS commit `553be4b4c62ae0c590e82f91817bc018bdfdc660` lets authenticated original local automatic-task tokens return the source/path-bound in-memory snapshot; Admin/Internal/unknown/alias identities retain coordinator routing. The fixed candidate `9d90884c304dcfbf97057c263c6f2ee110230ce7` adds only that reviewed patch to `7ed6c932`, with unchanged Cargo.lock. A new immutable package, matching tools and complete six-branch live acceptance remain required; unit tests do not promote E04. +E04 live [39842352595/1](https://github.com/rustfs/rustfs/actions/runs/39842352595) retains FAIL/cleanup PASS. All 1,402 official files, three H1 branches and five node cleanups were independently verified. H2's healthy Scanner task completed on vm002 while the exact node3 token query was forwarded to vm000 and returned notFound. RustFS commit `553be4b4c62ae0c590e82f91817bc018bdfdc660` lets authenticated original local automatic-task tokens return the source/path-bound in-memory snapshot; Admin/Internal/unknown/alias identities retain coordinator routing. Fixed candidate `9d90884c304dcfbf97057c263c6f2ee110230ce7` adds only that reviewed patch to `7ed6c932`, with unchanged Cargo.lock. Package [39851206784/1](https://github.com/rustfs/rustfs/actions/runs/39851206784) and matching tools [39860218641/1](https://github.com/rustfs/rustfs/actions/runs/39860218641) passed independent complete provenance and byte audits. + +The subsequent E04 live [39860501067/1](https://github.com/rustfs/rustfs/actions/runs/39860501067) retains FAIL/cleanup PASS. All 1,795 official files, H1's three branches, H2's cycle-boundary branch, five node cleanup receipts and four package purges were independently verified. H2's missing-shard and bitrot branches failed attribution: real repair tasks ran on remote disk workers while complete Scanner cycles were logged on vm003. The old same-node cycle requirement omitted both exact tasks and queried later healthy tasks. Independent replay binds each missed task to its original remote Scanner RPC request and cycle, and verifies restored physical bytes before rescue; neither missing structured receipt is retroactively accepted. The corrected executor retains the task owner for exact token queries and requires the remote RPC identity, Deep disk scan, matching positive cycle number and one unambiguous complete leader cycle. Invalid or duplicate identities, wrong/failed/ambiguous cycles and healthy substitution remain rejected. Fresh complete six-branch live acceptance is still required; canonical acceptance remains 109/158. I05 progress (2026-10-22): four-node startup scene `39764496504/1` was independently verified and isolated by diff --git a/scripts/validation_heal/scanner_repair.py b/scripts/validation_heal/scanner_repair.py index e0f3bfae..5d6a10fe 100644 --- a/scripts/validation_heal/scanner_repair.py +++ b/scripts/validation_heal/scanner_repair.py @@ -3,7 +3,8 @@ from datetime import datetime, timezone import json import time -from urllib.parse import quote +from urllib.parse import parse_qs, quote, urlsplit +from uuid import UUID try: from .heal_probe import Probe, require @@ -90,12 +91,47 @@ def cycle_pairs(events): return pairs +def discovery_cycle(events, node, discovery): + """Bind remote disk discovery to its RPC cycle, retaining the task owner.""" + stamp = event_time(discovery) + spans = discovery.get("spans", []) + if stamp is None or not isinstance(spans, list) or any(not isinstance(span, dict) for span in spans): return None + remote = [span for span in spans if span.get("name") == "http-request"] + if not remote: + matches = [(node, pair) for pair in cycle_pairs(events[node]) + if pair["started"] <= stamp <= pair["completed"]] + request = None + else: + if len(remote) != 1: return None + span = remote[0] + request_id = discovery.get("request_id") + if not isinstance(request_id, str) or not request_id or span.get("request_id") != request_id or span.get("method") != "POST": return None + if not isinstance(span.get("uri"), str): return None + if not any(s.get("name") == "nsscanner_disk" and s.get("scan_mode") == "Deep" for s in spans): return None + try: + url = urlsplit(span["uri"]) + query = parse_qs(url.query, keep_blank_values=True, strict_parsing=True) + if url.scheme or url.netloc or url.fragment or url.path != "/rustfs/rpc/ns_scanner": return None + cycle, = query["ns_scanner_cycle"] + if not cycle.isascii() or not cycle.isdecimal() or int(cycle) <= 0: return None + rpc_id, = query["ns_scanner_request_id"] + if UUID(rpc_id).int == 0: return None + except (KeyError, TypeError, ValueError): + return None + matches = [(leader, pair) for leader, rows in events.items() for pair in cycle_pairs(rows) + if pair["cycle"] == int(cycle) and pair["started"] <= stamp <= pair["completed"]] + request = {"requestId": request_id, "scannerRequestId": rpc_id, "uri": span["uri"]} + if len(matches) != 1: return None + leader, pair = matches[0] + return {"node": leader, **{name: value.isoformat() if isinstance(value, datetime) else value + for name, value in pair.items()}, "remoteRequest": request} + + def source_tasks(events, bucket, entry): """Bind discovery -> channel -> accepted Scanner source -> exact object task.""" require(entry["versionId"] == "null" and not entry["deleteMarker"], "scanner cold fixture must be unversioned data") found = [] for node, rows in events.items(): - pairs = cycle_pairs(rows) discoveries = [event for event in rows if event.get("target") == "rustfs::scanner::folder" and event.get("event") == "scanner_heal_admission" and event.get("state") == "request_started" and event.get("bucket") == bucket and event.get("object") == entry["key"] @@ -123,12 +159,11 @@ def source_tasks(events, bucket, entry): admission = next((event for event in accepted if event_time(event) is not None and receipt_time <= event_time(event) <= stamp), None) done = next((event for event in completed if event_time(event) is not None and stamp <= event_time(event)), None) - cycle = next((pair for pair in pairs if pair["started"] <= event_time(discovery) - <= pair["completed"] and event_time(discovery) <= stamp), None) + cycle = discovery_cycle(events, node, discovery) if admission is not None and done is not None and cycle is not None: found.append({"node": node, "taskId": token, "discovery": discovery, "received": receipt, "admission": admission, "started": start, "completed": done, - "cycle": {name: value.isoformat() if isinstance(value, datetime) else value for name, value in cycle.items()}}) + "cycle": cycle}) break return found @@ -140,7 +175,7 @@ def observe_task(probe, fleet, window, bucket, entry, branch, *, healthy=False): events = window.capture() tasks = source_tasks(events, bucket, entry) if tasks: - if branch != "scanner-cycle-boundary" or healthy or any(len(cycle_pairs(events[task["node"]])) >= 2 for task in tasks): + if branch != "scanner-cycle-boundary" or healthy or any(len(cycle_pairs(events[task["cycle"]["node"]])) >= 2 for task in tasks): break require(time.monotonic() < deadline, "ATTRIBUTION_NOT_OBSERVED: scanner did not discover, admit and complete the exact target in its cycle") time.sleep(1) @@ -148,6 +183,8 @@ def observe_task(probe, fleet, window, bucket, entry, branch, *, healthy=False): # tasks in order and select the first exact repaired receipt for this run. observations = [] for task in tasks: + cycles = len(cycle_pairs(events[task["cycle"]["node"]])) + if branch == "scanner-cycle-boundary" and not healthy and cycles < 2: continue endpoint = "node" + str(fleet.nodes.index(task["node"]) + 1) observer = Probe(probe.binding, endpoint) path = "/rustfs/admin/v3/heal/" + quote(bucket, safe="") + "/" + quote(entry["key"], safe="/") @@ -161,6 +198,6 @@ def observe_task(probe, fleet, window, bucket, entry, branch, *, healthy=False): fleet.record("scanner-source-binding", {"source": "scanner", "branch": branch, "healthyPreflight": healthy, "budgetSeconds": BUDGET_SECONDS, "task": task, "targetReadsBeforePhysicalProof": 0, "adminMutations": 0, "observedTasks": observations, - "postFaultCycles": len(cycle_pairs(events[task["node"]]))}) + "postFaultCycles": cycles}) return status raise ValueError("scanner tasks did not provide the required exact healthy/repaired responsibility; inspect source evidence") diff --git a/test_validation_heal.py b/test_validation_heal.py index d26e7d8f..e6920f9d 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -1364,6 +1364,68 @@ def test_task_may_start_after_discovery_cycle_but_cannot_precede_admission(self) rows[4]["timestamp"] = "2026-10-10T12:00:03Z" self.assertFalse(scanner_sources({"host": rows}, self.bucket, self.entry())) + def remote_events(self): + rows = self.events() + uri = ("/rustfs/rpc/ns_scanner?ns_scanner_cycle=7&ns_scanner_request_id=" + "410f652d-2861-4780-913a-539ccab29c47") + rows[1].update(request_id="remote-request", spans=[ + {"name": "http-request", "method": "POST", "request_id": "remote-request", "uri": uri}, + {"name": "nsscanner_disk", "scan_mode": "Deep"}]) + return {"worker": rows[1:-1], "leader": [rows[0], rows[-1]]} + + def test_remote_discovery_keeps_owner_and_binds_exact_rpc_cycle(self): + events = self.remote_events() + tasks = scanner_sources(events, self.bucket, self.entry()) + self.assertEqual(len(tasks), 1) + self.assertEqual(tasks[0]["node"], "worker") + self.assertEqual(tasks[0]["cycle"]["node"], "leader") + self.assertEqual(tasks[0]["cycle"]["cycle"], 7) + self.assertEqual(tasks[0]["cycle"]["remoteRequest"]["requestId"], "remote-request") + for uri in ("/rustfs/rpc/other?ns_scanner_cycle=7", + "https://other/rustfs/rpc/ns_scanner?ns_scanner_cycle=7", + events["worker"][0]["spans"][0]["uri"].replace("cycle=7", "cycle=8"), + events["worker"][0]["spans"][0]["uri"] + "&ns_scanner_cycle=7", + events["worker"][0]["spans"][0]["uri"].replace("410f652d-2861-4780-913a-539ccab29c47", "invalid")): + changed = copy.deepcopy(events) + changed["worker"][0]["spans"][0]["uri"] = uri + with self.subTest(uri=uri): + self.assertFalse(scanner_sources(changed, self.bucket, self.entry())) + for field, value in (("request_id", "another"), ("method", "GET")): + changed = copy.deepcopy(events) + changed["worker"][0]["spans"][0][field] = value + self.assertFalse(scanner_sources(changed, self.bucket, self.entry())) + changed = copy.deepcopy(events) + changed["worker"][0]["spans"][1]["scan_mode"] = "Normal" + self.assertFalse(scanner_sources(changed, self.bucket, self.entry())) + changed = copy.deepcopy(events) + changed["other-leader"] = copy.deepcopy(events["leader"]) + self.assertFalse(scanner_sources(changed, self.bucket, self.entry())) + changed = copy.deepcopy(events) + changed["leader"][-1]["state"] = "failed" + self.assertFalse(scanner_sources(changed, self.bucket, self.entry())) + for spans in (None, "invalid", [None], [{"name": "http-request", "uri": None}]): + changed = copy.deepcopy(events) + changed["worker"][0]["spans"] = spans + with self.subTest(spans=spans): + self.assertFalse(scanner_sources(changed, self.bucket, self.entry())) + + def test_remote_boundary_queries_worker_and_counts_leader_cycles(self): + events = self.remote_events() + events["leader"] += [ + {**events["leader"][0], "cycle": 8, "timestamp": "2026-10-10T12:00:08Z"}, + {**events["leader"][1], "timestamp": "2026-10-10T12:00:09Z"}] + records = [] + fleet = types.SimpleNamespace(runtime_profile="scanner", nodes=["worker", "leader"], + record=lambda label, value: records.append((label, value)) or value) + window = types.SimpleNamespace(capture=lambda: events) + status = {"summary": "finished", "outcome": {"objects": [{"disposition": {"state": "repaired"}}]}} + observer = types.SimpleNamespace(admin=lambda *args: {"httpStatus": 200, "response": status}) + with patch("scripts.validation_heal.scanner_repair.Probe", return_value=observer) as probe: + observe_scanner(types.SimpleNamespace(binding={}), fleet, window, self.bucket, self.entry(), "scanner-cycle-boundary") + probe.assert_called_once_with({}, "node1") + self.assertEqual(records[-1][1]["postFaultCycles"], 2) + self.assertEqual(records[-1][1]["task"]["cycle"]["node"], "leader") + def test_cycle_pair_requires_ordered_positive_start_and_complete_boundary(self): rows = self.events() self.assertEqual(cycle_pairs(rows)[0]["cycle"], 7) From 53ef090906f264d089fd3007e6356d738c7e897c Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 06:31:00 +0800 Subject: [PATCH 309/343] docs(validation): record complete E04 acceptance --- docs/validation-migration-map.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 121cacdf..f08da46d 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -9,17 +9,17 @@ live run proves each source assertion on the required topology. ## Current acceptance and remaining work -Count reconciliation (2026-10-23): the five additions after the 104-case -baseline are C02, C03, C05, I04 and B05. Their original raw-evidence auditors were +Count reconciliation (2026-10-23): the six additions after the 104-case +baseline are C02, C03, C05, I04, B05 and E04. Their original raw-evidence auditors were rerun against the retained official archives; all results match the recorded audits. Candidate/helper identities, complete H1/H2 branch coverage, cleanup, acceptance review chains and unchanged earlier failures were reconciled to -`canonical-migration-ledger-20261023-b05.json`. Each acceptance retains its own +`canonical-migration-ledger-20261023-e04.json`. Each acceptance retains its own immutable candidate; this is migration coverage across those candidates, not a fresh replay of all cases on the current I05 package. Partial I05, native and recovery diagnostics add no canonical acceptance. -All 158 canonical IDs are retained. Independent complete live acceptance is **109/158**: the prior 104-case baseline plus the five reconciled cases below. The baseline includes B07 `heal.below-read-quorum` ([run 38727279347/1](https://github.com/rustfs/rustfs/actions/runs/38727279347)), A10 `heal.admin-option-contract` ([run 39197088183/1](https://github.com/rustfs/rustfs/actions/runs/39197088183)) and B06 `heal.read-quorum-boundary` ([run 39223475622/1](https://github.com/rustfs/rustfs/actions/runs/39223475622)). Complete source matrices, immutable candidate/package/tools and cleanup were independently audited; earlier failures remain retained. +All 158 canonical IDs are retained. Independent complete live acceptance is **110/158**: the prior 104-case baseline plus the six reconciled cases below. The baseline includes B07 `heal.below-read-quorum` ([run 38727279347/1](https://github.com/rustfs/rustfs/actions/runs/38727279347)), A10 `heal.admin-option-contract` ([run 39197088183/1](https://github.com/rustfs/rustfs/actions/runs/39197088183)) and B06 `heal.read-quorum-boundary` ([run 39223475622/1](https://github.com/rustfs/rustfs/actions/runs/39223475622)). Complete source matrices, immutable candidate/package/tools and cleanup were independently audited; earlier failures remain retained. | Reconciled case | Complete live run | Required coverage | Original result SHA-256 | | --- | --- | --- | --- | @@ -28,22 +28,23 @@ All 158 canonical IDs are retained. Independent complete live acceptance is **10 | C05 `heal.missing-or-corrupt-xlmeta` | [39322217398/1](https://github.com/rustfs/rustfs/actions/runs/39322217398) | Ten H1/H2 DB/DV instances, 3,132 archive files | `489b8179ee90a7412b223e669db8fd4f6520b9e69051cbf927b6f33299de423b` | | I04 `heal.queue-pressure-and-admission` | [39354651483/1](https://github.com/rustfs/rustfs/actions/runs/39354651483) | Four branches per topology, six exact durable tasks, 5,565 archive files | `4dcde2f34bb7df52f03f5a3ec1e87663145a7134d3ef1a513fda2da52d1709a2` | | B05 `heal.multiple-shard-loss` | [39843970314/1](https://github.com/rustfs/rustfs/actions/runs/39843970314) | All 38 DB/DV and H1/H2 instances, 9,059 archive files, ten node cleanups and four package purges | `2624f43fc8581eb1b533a477b1a869f9c72208861d7a268ea35ae8f36153fd7e` | +| E04 `heal.scanner-cold-object-repair` | [39863505276/1](https://github.com/rustfs/rustfs/actions/runs/39863505276) | All six H1/H2 instances, 2,307 archive files, five node cleanups and four package purges | `8bfcfb03bfa5cc88649e98c4f9a6fa7a27d64f9fd0cdbe8d851f57fc00bc0702` | | Acceptance state | Cases | Remaining requirement | | --- | ---: | --- | -| Independently accepted complete live PASS | 109 | Preserve original provenance and complete results. | -| Retained live failures with existing executors | 25 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | +| Independently accepted complete live PASS | 110 | Preserve original provenance and complete results. | +| Retained live failures with existing executors | 24 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | | Required native barriers | 7 | A09 and G01–G06 need exact candidate-bound server lock/version/commit barriers. | | Orchestration or evidence blocked | 4 | C11, F07, I05 and J02 need their remaining native or full live evidence closure. | | Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | -The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 36 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. +The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 35 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. B05 acceptance uses candidate `7ed6c9320db491c84b54e8232152d7b1282c1b3f`, helper `76dbf7029e82c1e3878c9160090804d4afc38078`, package `39827676705/1` and tools `39841369555/1`. The complete official ZIP digest is `4eb3405af1f47583cd7c2d8e5b9ce9fa6910b76665df529d7dfbe4f1399a20d8`. Independent recomputation covers seed ACKs, full versions/markers, the actual 2+2 and 12+4 role/loss matrices, every endpoint's full online membership before each required boundary, sequential faults, one unique START/token per instance, two identical terminal outcome queries, strict integer scope, exact repaired/healthy responsibility, all physical shard and bitrot proofs, complete S3/HEAD/Range evidence before finalization, ten private-node cleanups and four package purges. No backup restoration was used. Original failure `38200772548` remains unchanged; its vm001 offline root cause is not established by the later pass. E04 live [39842352595/1](https://github.com/rustfs/rustfs/actions/runs/39842352595) retains FAIL/cleanup PASS. All 1,402 official files, three H1 branches and five node cleanups were independently verified. H2's healthy Scanner task completed on vm002 while the exact node3 token query was forwarded to vm000 and returned notFound. RustFS commit `553be4b4c62ae0c590e82f91817bc018bdfdc660` lets authenticated original local automatic-task tokens return the source/path-bound in-memory snapshot; Admin/Internal/unknown/alias identities retain coordinator routing. Fixed candidate `9d90884c304dcfbf97057c263c6f2ee110230ce7` adds only that reviewed patch to `7ed6c932`, with unchanged Cargo.lock. Package [39851206784/1](https://github.com/rustfs/rustfs/actions/runs/39851206784) and matching tools [39860218641/1](https://github.com/rustfs/rustfs/actions/runs/39860218641) passed independent complete provenance and byte audits. -The subsequent E04 live [39860501067/1](https://github.com/rustfs/rustfs/actions/runs/39860501067) retains FAIL/cleanup PASS. All 1,795 official files, H1's three branches, H2's cycle-boundary branch, five node cleanup receipts and four package purges were independently verified. H2's missing-shard and bitrot branches failed attribution: real repair tasks ran on remote disk workers while complete Scanner cycles were logged on vm003. The old same-node cycle requirement omitted both exact tasks and queried later healthy tasks. Independent replay binds each missed task to its original remote Scanner RPC request and cycle, and verifies restored physical bytes before rescue; neither missing structured receipt is retroactively accepted. The corrected executor retains the task owner for exact token queries and requires the remote RPC identity, Deep disk scan, matching positive cycle number and one unambiguous complete leader cycle. Invalid or duplicate identities, wrong/failed/ambiguous cycles and healthy substitution remain rejected. Fresh complete six-branch live acceptance is still required; canonical acceptance remains 109/158. +The subsequent E04 live [39860501067/1](https://github.com/rustfs/rustfs/actions/runs/39860501067) retains FAIL/cleanup PASS. All 1,795 official files, H1's three branches, H2's cycle-boundary branch, five node cleanup receipts and four package purges were independently verified. H2's missing-shard and bitrot branches failed attribution: real repair tasks ran on remote disk workers while complete Scanner cycles were logged on vm003. The old same-node cycle requirement omitted both exact tasks and queried later healthy tasks. Independent replay binds each missed task to its original remote Scanner RPC request and cycle, and verifies restored physical bytes before rescue; neither missing structured receipt is retroactively accepted. The corrected executor retains the task owner for exact token queries and requires the remote RPC identity, Deep disk scan, matching positive cycle number and one unambiguous complete leader cycle. Invalid or duplicate identities, wrong/failed/ambiguous cycles and healthy substitution remain rejected. The later complete six-branch acceptance of run 39863505276, detailed below, raises canonical acceptance to 110/158 while retaining this original failure. I05 progress (2026-10-22): four-node startup scene `39764496504/1` was independently verified and isolated by @@ -102,7 +103,7 @@ peer inventory, ACK-bound stopped physical invariance and four/sixteen streaming shard receipts. FD counts returned to or below each peer's baseline. The original result remains `BLOCKED/cleanup PASS`; exact disconnect-edge ownership, distributed release, global no-leak and full source DL/DV coverage are unproven. -That I05 diagnostic added no acceptance; the current total is **109/158** after the separate complete B05 audit. +That I05 diagnostic added no acceptance. The separate complete B05 audit brought the historical total to 109/158; the subsequent E04 audit brings the current total to 110/158. Fresh D03 [39801936817/1](https://github.com/rustfs/rustfs/actions/runs/39801936817) retains FAIL/cleanup PASS. Independent H1 re-evaluation verified all four @@ -122,13 +123,19 @@ A10 run `39197088183/1` uses immutable helper `b16e6072679d6a60a044b9d4520b30b4a The sections below retain earlier implementation and run history. Historical failed or partial results are not promoted by this summary. +## E04 complete acceptance and A05 cancellation regression + +Fresh E04 live [39863505276/1](https://github.com/rustfs/rustfs/actions/runs/39863505276) passes complete independent acceptance on helper `cd36287f1abe42662333fab31cf45ac23de91c8e`, candidate `9d90884c304dcfbf97057c263c6f2ee110230ce7`, package `39851206784/1` and tools `39863244404/1`. All 2,307 official archive files were byte-checked against ZIP SHA-256 `ecbd537cb6422cc3af631d4a7d1145040ffd30c20da0008778e442752e0f193e`. All six source branches passed independently recomputed seed ACKs, healthy preflight, exact Scanner source/token/RPC/cycle binding, strict repaired responsibility, complete physical and bitrot proofs before S3 reads, 768 HEADs, 672 deterministic Range payloads, unchanged inventories and finalization without backup restoration. All five node cleanups and four package purges passed. Independent complete audit SHA-256 is `c96ab5fc91b13fecb475a0cee86c85a3f5c983ce125b912b4485f7cd080d3243`. This raises canonical acceptance to 110/158 and preserves every earlier failure. + +A05's cancellation after successful bucket storage I/O is deterministically reproduced at the receipt boundary: the unchanged code rejects the late positive receipt but falls back to Unknown. RustFS commit `78c803e1efd67cc04edc8fc4d08a8ea75af77ddf`, pushed to the existing PR, retains the positive-receipt rejection and routes observed cancellation through the existing typed Cancelled outcome, with partial coverage and zero positive or Unknown responsibility. The exact regression failed before the fix and passes after it; all 153 task tests and Rust formatting pass. This establishes the code defect and local regression coverage, not the exact historical live interleaving or a new A05 canonical result. Fresh immutable candidate and complete A05 live acceptance remain required. + ## Supplemental J02 evidence audit The 2026-10-23 supplemental independent audit closes twelve positive branch instances from original J02 run 38874900846: grace Deferred/new-token cleanup, read-only terminal failure/new-token repair and dry-run across H1/H2 and DB/DV. Every original archive member was byte-checked again, all fixture requests/ACK digests and version/marker manifests were rebuilt, and the exact executed helper was pinned. Grace uses both original ACK versions, natural 120-second expiry under unchanged candidate processes, the exact typed future retry time, two unchanged original-token terminal queries and every H2 peer, a distinct compensating token, physical post-DELETE authority before GET/rescue and complete peer inventories with old-version refusal. Read-only verifies actual private-namespace EROFS, the protected-control incarnation, unchanged failed and released scenes, a distinct repair token, original full shard/bitrot evidence before GET/rescue and all peer inventories. Dry-run retains every original typed disposition, the actual R-1 fault geometry and unchanged fault bytes. All twelve node cleanup groups bind their original reservations. -Together with the earlier four mixed-branch audits, sixteen branch instances now have independent acceptance; the four original opaque-metadata Unknown probes still prevent whole-case acceptance. The supplemental report SHA-256 is `4424b3e681931527d244a4ca0211dde534a8145cd95ea0b460f388262ccb8c10`. Historical results and the 109/158 canonical count remain unchanged. +Together with the earlier four mixed-branch audits, sixteen branch instances now have independent acceptance; the four original opaque-metadata Unknown probes still prevent whole-case acceptance. The supplemental report SHA-256 is `4424b3e681931527d244a4ca0211dde534a8145cd95ea0b460f388262ccb8c10`. This J02 audit preserves historical results and adds no canonical acceptance; the current total is 110/158 after the separate E04 audit. ## Original reuse assessment From 81ed5881a2aa0b1320e8f1101dfe03443ab71b38 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 07:33:26 +0800 Subject: [PATCH 310/343] fix(validation): pin released rc for J01 status checks --- docs/validation-migration-map.md | 10 ++++ scripts/validation_heal/rc_binary.py | 62 ++++++++++++++++++++ scripts/validation_heal/run_case.sh | 15 +++-- test_validation_heal.py | 86 ++++++++++++++++++++++++++++ 4 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 scripts/validation_heal/rc_binary.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index f08da46d..293e36f1 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -657,6 +657,16 @@ configuration outside artifacts and are removed during finalization. Local coverage/uncertainty/startup/config-cleanup negatives passed; live evidence remains pending. +J01 selects the official rc v0.1.36 Linux amd64 musl release in a new private +artifact directory. Both archive and executable SHA-256 values are fixed and +checked before execution; its manifest records the release source, target, +version output and exact bytes. This release includes the existing CLI fix +that preserves incomplete/unknown status in human-readable output. The musl +build avoids depending on the runner's glibc version. Other Heal cases retain +their existing client selection. The original failed J01 result is retained; +the complete raw/CLI startup, disabled, missing-peer and recovery observations +still require live acceptance with the fixed client. + J03 `heal.incremental-cursor-and-truncation` now has a current-fleet DX executor with 1025 pre-registered external objects of 1MiB+73 bytes. The diff --git a/scripts/validation_heal/rc_binary.py b/scripts/validation_heal/rc_binary.py new file mode 100644 index 00000000..76089e08 --- /dev/null +++ b/scripts/validation_heal/rc_binary.py @@ -0,0 +1,62 @@ +"""Prepare the released rc with truthful J01 runtime status rendering.""" + +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import tarfile + + +RELEASE = "v0.1.36" +SOURCE = "9131de5b0bcb3a28bb0d32b5dce2c86cbbf4724e" +# The official musl build avoids a dependency on the runner's glibc version. +URL = "https://github.com/rustfs/cli/releases/download/v0.1.36/rustfs-cli-linux-amd64-v0.1.36.tar.gz" +ARCHIVE_SHA256 = "4a8128911ccad4e7b481f26635a4cfd1ec064412210526e57ad2c748d356f3b7" +BINARY_SHA256 = "15490337489f5c1bb2d33f77e308be52ac29adaa27069d7c77f0d808b165f786" +MAX_BYTES = 64 * 1024**2 + + +def unpack(archive, directory): + if archive.stat().st_size > MAX_BYTES or hashlib.sha256(archive.read_bytes()).hexdigest() != ARCHIVE_SHA256: + raise ValueError("released rc archive digest differs") + with tarfile.open(archive) as incoming: + members = incoming.getmembers() + if (len(members) != 3 or {m.name for m in members} != {"rc", "LICENSE-MIT", "LICENSE-APACHE"} + or any(not m.isfile() or not 0 < m.size <= MAX_BYTES for m in members) + or sum(m.size for m in members) > MAX_BYTES): + raise ValueError("released rc archive has unexpected members") + binary = incoming.extractfile(next(m for m in members if m.name == "rc")).read() + if hashlib.sha256(binary).hexdigest() != BINARY_SHA256: + raise ValueError("released rc binary digest differs") + if binary[:6] != b"\x7fELF\x02\x01" or binary[18:20] != b"\x3e\x00": + raise ValueError("released rc is not Linux x86-64 ELF") + target = directory / "rc" + with target.open("xb") as output: + output.write(binary) + target.chmod(0o700) + return target.resolve() + + +def prepare(directory): + directory.mkdir(mode=0o700, parents=True, exist_ok=False) + archive = directory / "rc.tar.gz" + subprocess.run(["curl", "--fail", "--silent", "--show-error", "--location", "--proto", "=https", + "--proto-redir", "=https", "--connect-timeout", "15", "--max-time", "180", + "--max-filesize", str(MAX_BYTES), "--output", str(archive), URL], check=True, timeout=190) + target = unpack(archive, directory) + result = subprocess.run([str(target), "--version"], capture_output=True, text=True, check=True, timeout=10) + manifest = {"release": RELEASE, "sourceCommit": SOURCE, "target": "x86_64-unknown-linux-musl", "archiveUrl": URL, + "archiveSha256": ARCHIVE_SHA256, "binarySha256": BINARY_SHA256, + "binaryPath": str(target), "versionOutput": result.stdout.strip(), + "role": "validation client; not the RustFS service under test"} + with (directory / "manifest.json").open("x") as output: + json.dump(manifest, output, indent=2) + output.write("\n") + return target + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--directory", type=Path, required=True) + print(prepare(parser.parse_args().directory)) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 9a7f3cc0..c6974fd0 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -109,6 +109,10 @@ fi [ "${driver}" = healthy.py ] || [ "${#tool_args[@]}" -gt 0 ] || die "physical Heal cases require candidate-bound bitrot tools" evidence="${artifacts}/evidence/${case_id}" mkdir -p "${artifacts}/cases" "${evidence}" +rc_binary="$(command -v rc)" +if [ "${case_id}" = heal.status-partial-disabled-unknown ]; then + rc_binary="$(python3 -B "${ROOT}/scripts/validation_heal/rc_binary.py" --directory "${evidence}/rc-client")" +fi if [ "${driver}" = race.py ] || [ "${driver}" = nolock.py ]; then python3 -B "${ROOT}/scripts/validation_heal/${driver}" preflight \ --candidate "${candidate}" --case-id "${case_id}" --testing-commit "${testing_commit}" \ @@ -117,7 +121,7 @@ fi if [ "${driver}" = lock_timeout_runner.py ]; then python3 -B "${ROOT}/scripts/validation_heal/${driver}" preflight \ --candidate "${candidate}" --case-id "${case_id}" --testing-commit "${testing_commit}" \ - "${tool_args[@]}" "${i05_args[@]}" --rc-bin "$(command -v rc)" > "${evidence}/timeout-preflight.json" + "${tool_args[@]}" "${i05_args[@]}" --rc-bin "${rc_binary}" > "${evidence}/timeout-preflight.json" fi python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ --result-file "${artifacts}/cases/${case_id}.json" --verdict RUNNING --cleanup-status PENDING @@ -197,7 +201,7 @@ if [ "${bootstrap_rc}" -eq 0 ]; then python3 -B "${ROOT}/scripts/validation_heal/${driver}" --candidate "${candidate}" \ --case-id "${case_id}" \ "${tool_args[@]}" "${i05_args[@]}" \ - --testing-commit "${testing_commit}" --binary-sha "${binary_sha}" --rc-bin "$(command -v rc)" \ + --testing-commit "${testing_commit}" --binary-sha "${binary_sha}" --rc-bin "${rc_binary}" \ --evidence "${evidence}" > "${evidence}/test.log" 2>&1 || case_rc=$? cat "${evidence}/test.log" jq -e '.cleanup == "PASS"' "${evidence}/healthy-summary.json" >/dev/null 2>&1 || cleanup_status=FAIL @@ -210,7 +214,7 @@ cat "${evidence}/cleanup.log" [ -n "${binary_sha}" ] || { python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ --result-file "${artifacts}/cases/${case_id}.json" --verdict INFRA_ERROR --cleanup-status "${cleanup_status}" \ - --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --rc-bin "$(command -v rc)" \ + --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --rc-bin "${rc_binary}" \ --evidence "evidence/${case_id}/setup.log" --evidence "evidence/${case_id}/cleanup.log" \ --reason "Heal candidate bootstrap failed before a verified binary was installed" exit 1 @@ -251,9 +255,12 @@ fi args=("${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool --result-file "${artifacts}/cases/${case_id}.json" --verdict "${verdict}" --cleanup-status "${cleanup_status}" --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --binary-sha256 "${binary_sha}" - --rc-bin "$(command -v rc)" --evidence "evidence/${case_id}/setup.log" + --rc-bin "${rc_binary}" --evidence "evidence/${case_id}/setup.log" --evidence "evidence/${case_id}/test.log" --evidence "evidence/${case_id}/cleanup.log" --evidence "evidence/${case_id}/healthy-summary.json") +if [ "${case_id}" = heal.status-partial-disabled-unknown ]; then + args+=(--evidence "evidence/${case_id}/rc-client/manifest.json") +fi if [ -n "${reason}" ]; then args+=(--reason "${reason}"); fi python3 -B "${args[@]}" [ "${verdict}" = PASS ] diff --git a/test_validation_heal.py b/test_validation_heal.py index e6920f9d..a8a1c7ae 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -52,6 +52,7 @@ from scripts.validation_heal.cancel_restart import validate_journal, BRANCHES as CANCEL_RESTART_BRANCHES from scripts.validation_heal.fleet_node import task_records as node_task_records from scripts.validation_heal import overlap +from scripts.validation_heal import rc_binary from scripts.validation_heal import root_restart, fleet_node from scripts.validation_heal import quorum_boundary from scripts.validation_heal import stale_rejoin @@ -1679,6 +1680,91 @@ def test_permission_probe_checks_read_and_write_with_same_namespace_and_dac_limi self.assertEqual(invoke.call_args.args[0][-1], "/private/data/.rustfs.sys/format.json") +class HealReleasedClientTest(unittest.TestCase): + def archive(self, root, *, extra=None): + import io + import tarfile + binary = bytearray(64) + binary[:6] = b"\x7fELF\x02\x01" + binary[18:20] = b"\x3e\x00" + archive = root / "client.tar.gz" + with tarfile.open(archive, "w:gz") as output: + for name, data in [("rc", bytes(binary)), ("LICENSE-MIT", b"MIT"), ("LICENSE-APACHE", b"Apache")]: + entry = tarfile.TarInfo(name) + entry.size = len(data) + output.addfile(entry, io.BytesIO(data)) + if extra is not None: + output.addfile(extra) + return archive, bytes(binary) + + def test_exact_archive_and_binary_are_required_before_execution(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive, binary = self.archive(root) + with patch.object(rc_binary, "ARCHIVE_SHA256", hashlib.sha256(archive.read_bytes()).hexdigest()), \ + patch.object(rc_binary, "BINARY_SHA256", hashlib.sha256(binary).hexdigest()), \ + patch.object(rc_binary.subprocess, "run") as execute: + target = rc_binary.unpack(archive, root) + self.assertEqual(target.read_bytes(), binary) + self.assertEqual(target.stat().st_mode & 0o777, 0o700) + execute.assert_not_called() + with self.assertRaises(FileExistsError): + rc_binary.unpack(archive, root) + + def test_archive_or_binary_digest_mismatch_does_not_create_executable(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive, _ = self.archive(root) + with self.assertRaisesRegex(ValueError, "archive digest"): + rc_binary.unpack(archive, root) + with patch.object(rc_binary, "ARCHIVE_SHA256", hashlib.sha256(archive.read_bytes()).hexdigest()): + with self.assertRaisesRegex(ValueError, "binary digest"): + rc_binary.unpack(archive, root) + self.assertFalse((root / "rc").exists()) + + def test_unexpected_or_link_members_are_rejected(self): + import tarfile + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, kind in [("../outside", tarfile.REGTYPE), ("alias", tarfile.SYMTYPE)]: + entry = tarfile.TarInfo(name) + entry.type = kind + entry.linkname = "rc" if kind == tarfile.SYMTYPE else "" + archive, binary = self.archive(root, extra=entry) + with patch.object(rc_binary, "ARCHIVE_SHA256", hashlib.sha256(archive.read_bytes()).hexdigest()), \ + patch.object(rc_binary, "BINARY_SHA256", hashlib.sha256(binary).hexdigest()): + with self.assertRaisesRegex(ValueError, "unexpected members"): + rc_binary.unpack(archive, root) + self.assertFalse((root / "rc").exists()) + + def test_prepare_records_pinned_identity_and_uses_private_client(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive, binary = self.archive(root) + raw = archive.read_bytes() + destination = root / "selected-client" + def invoke(command, **kwargs): + if command[0] == "curl": + self.assertEqual(command[-1], rc_binary.URL) + self.assertIn("--max-filesize", command) + Path(command[command.index("--output") + 1]).write_bytes(raw) + return types.SimpleNamespace(returncode=0) + self.assertEqual(command, [str((destination / "rc").resolve()), "--version"]) + return types.SimpleNamespace(returncode=0, stdout="rc 0.1.36\n") + with patch.object(rc_binary, "ARCHIVE_SHA256", hashlib.sha256(raw).hexdigest()), \ + patch.object(rc_binary, "BINARY_SHA256", hashlib.sha256(binary).hexdigest()), \ + patch.object(rc_binary.subprocess, "run", side_effect=invoke) as execute: + target = rc_binary.prepare(destination) + self.assertEqual(execute.call_count, 2) + manifest = json.loads((destination / "manifest.json").read_text()) + self.assertEqual(manifest["binaryPath"], str(target)) + self.assertEqual(manifest["binarySha256"], hashlib.sha256(binary).hexdigest()) + self.assertEqual(manifest["sourceCommit"], rc_binary.SOURCE) + self.assertEqual(manifest["versionOutput"], "rc 0.1.36") + with self.assertRaises(FileExistsError): + rc_binary.prepare(destination) + + class HealRuntimeStatusTest(unittest.TestCase): def background(self, *, state="idle", missing=False): return {"state": state, "clusterStatusComplete": not missing, From 23b3812720d25667b90cd85e3cc0429debb558e9 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 08:32:10 +0800 Subject: [PATCH 311/343] fix(validation): recognize explicit drive access denial --- docs/validation-migration-map.md | 34 +++++++++++++++++++++++----- scripts/validation_heal/disk_loss.py | 2 +- test_validation_heal.py | 31 +++++++++++++++++++++++-- 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 293e36f1..413a68be 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -40,6 +40,22 @@ All 158 canonical IDs are retained. Independent complete live acceptance is **11 The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 35 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. +A05 scoped live [39875125045/1](https://github.com/rustfs/rustfs/actions/runs/39875125045) +was interrupted by the self-hosted runner's exhausted filesystem. The official +check annotation reports `No space left on device` while writing the runner's +worker log. The selected-case step remained in progress in the completed job; +cleanup and artifact upload stayed pending, and the official artifact listing +contains no files. Neither product failure nor clean finalization can be +concluded from that interruption. Its fixed candidate `c95bb396`, package +`39867606969/1` and matching tools `39874908888/1` retain their independent +byte/provenance audits. The J01 chain stopped before dispatch when A05 evidence +collection refused the empty artifact listing. Its released-client fix and +the B01 explicit disk-error correction have local coverage; both still need +complete fresh live acceptance. Preserve the interrupted runner's original +temporary evidence and all current fleet namespaces before another job or +recovery action. Resume fleet execution only after runner space, original +evidence and owned cleanup/isolation have been independently reconciled. + B05 acceptance uses candidate `7ed6c9320db491c84b54e8232152d7b1282c1b3f`, helper `76dbf7029e82c1e3878c9160090804d4afc38078`, package `39827676705/1` and tools `39841369555/1`. The complete official ZIP digest is `4eb3405af1f47583cd7c2d8e5b9ce9fa6910b76665df529d7dfbe4f1399a20d8`. Independent recomputation covers seed ACKs, full versions/markers, the actual 2+2 and 12+4 role/loss matrices, every endpoint's full online membership before each required boundary, sequential faults, one unique START/token per instance, two identical terminal outcome queries, strict integer scope, exact repaired/healthy responsibility, all physical shard and bitrot proofs, complete S3/HEAD/Range evidence before finalization, ten private-node cleanups and four package purges. No backup restoration was used. Original failure `38200772548` remains unchanged; its vm001 offline root cause is not established by the later pass. E04 live [39842352595/1](https://github.com/rustfs/rustfs/actions/runs/39842352595) retains FAIL/cleanup PASS. All 1,402 official files, three H1 branches and five node cleanups were independently verified. H2's healthy Scanner task completed on vm002 while the exact node3 token query was forwarded to vm000 and returned notFound. RustFS commit `553be4b4c62ae0c590e82f91817bc018bdfdc660` lets authenticated original local automatic-task tokens return the source/path-bound in-memory snapshot; Admin/Internal/unknown/alias identities retain coordinator routing. Fixed candidate `9d90884c304dcfbf97057c263c6f2ee110230ce7` adds only that reviewed patch to `7ed6c932`, with unchanged Cargo.lock. Package [39851206784/1](https://github.com/rustfs/rustfs/actions/runs/39851206784) and matching tools [39860218641/1](https://github.com/rustfs/rustfs/actions/runs/39860218641) passed independent complete provenance and byte audits. @@ -885,12 +901,18 @@ streaming checks and full all-peer S3 manifests. Missing/unchanged UUID or early automatic repair cannot satisfy this manual case. Partial rename/mkdir intent remains recoverable; archive drift retains the namespace. Live proof is pending. -B01 run `38081858808` retained FAIL and cleanup PASS. Both H1/H2 had real -same-service read/write EACCES, but the exact Admin disk never became explicitly -offline within the registered 90-second observation window. Degraded ACKs were -therefore not started. The missing double witness remains NOT_OBSERVED in the -failure evidence; ready, cached online state and an inaccessible directory do -not substitute for the required Admin offline proof. +B01 run `38081858808` retains its original FAIL and cleanup PASS. Both H1/H2 +had real same-service read/write EACCES. Independent replay of all 46 H1 and +45 H2 original Admin responses finds the exact original slot, endpoint and UUID with +`state="drive access denied"` and cached `runtimeState="online"`. The candidate's +`DiskAccessDenied` error renders that exact state, and its Admin disk-count +logic counts it as offline. The executor omitted this spelling from its +explicit error set, so it rejected the recorded witness and never started +degraded ACKs. The state mapping now recognizes that exact error while still +requiring the original identity and every other member's healthy UUID. Unknown, +initializing, unrecognized error and online-only observations remain refused. +Replaying this offline check does not supply the unexecuted ACK or repair +branches; complete fresh H1/H2 live evidence is still required. B06 corrected run `38082447868` retained FAIL and cleanup PASS. Both exact-R diff --git a/scripts/validation_heal/disk_loss.py b/scripts/validation_heal/disk_loss.py index 81b6820a..153ada72 100644 --- a/scripts/validation_heal/disk_loss.py +++ b/scripts/validation_heal/disk_loss.py @@ -77,7 +77,7 @@ def member_state(observed, target, total, baseline=None): "another Admin member is unavailable; single-drive admission is unproven") # The pinned candidate counts these explicit disk errors as offline. An # unknown/initializing server or aggregate count never witnesses this disk. - require(selected.get("state") in {"disk not found", "disk access denied", "file access denied", + require(selected.get("state") in {"disk not found", "drive access denied", "disk access denied", "file access denied", "volume access denied", "offline"} or selected.get("runtimeState") == "offline", "NOT_OBSERVED: exact gated member is not explicitly offline in Admin info") return {**baseline, "reportedUUID": selected.get("uuid"), "state": selected.get("state"), diff --git a/test_validation_heal.py b/test_validation_heal.py index a8a1c7ae..0bcca527 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -3431,21 +3431,48 @@ def test_offline_admin_requires_exact_slot_endpoint_uuid_and_complete_healthy_pe baseline = disk_loss.member_state(online, target, 4) offline = copy.deepcopy(online) selected = offline["response"]["info"]["servers"][0]["drives"][target["setSlot"]] - selected.update(state="disk access denied", runtimeState="offline") + selected.update(state="drive access denied", runtimeState="online") self.assertTrue(disk_loss.member_state(offline, target, 4, baseline)["exactOriginalMemberOffline"]) - for kind in ("wrong-slot", "unknown", "new-uuid", "new-endpoint", "peer-offline", "omitted", "duplicate"): + for kind in ("wrong-slot", "online", "unknown", "initializing", "unknown-error", "new-uuid", + "new-endpoint", "peer-offline", "peer-uuid", "omitted", "duplicate"): changed = copy.deepcopy(offline) drives = changed["response"]["info"]["servers"][0]["drives"] if kind == "wrong-slot": drives[target["setSlot"]]["disk_index"] = 0 + elif kind == "online": drives[target["setSlot"]].update(state="ok", runtimeState="online") elif kind == "unknown": drives[target["setSlot"]].update(state="unknown", runtimeState="unknown") + elif kind == "initializing": drives[target["setSlot"]].update(state="initializing", runtimeState=None) + elif kind == "unknown-error": drives[target["setSlot"]].update(state="unexpected access error", runtimeState="online") elif kind == "new-uuid": drives[target["setSlot"]]["uuid"] = "another-disk" elif kind == "new-endpoint": drives[target["setSlot"]]["endpoint"] = "http://another-node:9000/data" elif kind == "peer-offline": drives[0].update(state="offline", runtimeState="offline") + elif kind == "peer-uuid": drives[0]["uuid"] = "another-disk" elif kind == "omitted": drives.pop() else: drives.append(copy.deepcopy(drives[0])) with self.subTest(kind=kind), self.assertRaises(ValueError): disk_loss.member_state(changed, target, 4, baseline) + def test_explicit_drive_access_denial_is_offline_despite_cached_runtime_state(self): + _, _, _, _, target, _, _ = self.scene() + online = self.admin_info(target) + baseline = disk_loss.member_state(online, target, 4) + for runtime in ("online", None): + for uuid in (target["diskUUID"], ""): + observed = copy.deepcopy(online) + selected = observed["response"]["info"]["servers"][0]["drives"][target["setSlot"]] + selected.update(state="drive access denied", uuid=uuid) + if runtime is None: + selected.pop("runtimeState") + else: + selected["runtimeState"] = runtime + with self.subTest(runtime=runtime, uuid=uuid): + proof = disk_loss.member_state(observed, target, 4, baseline) + self.assertTrue(proof["exactOriginalMemberOffline"]) + self.assertEqual(proof["state"], "drive access denied") + self.assertEqual(proof["reportedUUID"], uuid) + self.assertEqual(proof["runtimeState"], runtime) + with self.assertRaisesRegex(ValueError, "not positively online"): + disk_loss.member_state(observed, target, 4) + def test_degraded_unique_puts_keep_each_ack_and_branches_match_source(self): self.assertEqual(list(disk_loss.BRANCHES), select("B01")[0]["variants"]) bucket = HealReadQuorumBoundaryTest.bucket From 25301fd968aec1f13d3c9ed9cb7ecf7c5b8f19ea Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 10:42:55 +0800 Subject: [PATCH 312/343] fix(validation): validate retired token stop during force start --- scripts/validation_heal/overlap.py | 6 ++-- test_validation_heal.py | 50 ++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/scripts/validation_heal/overlap.py b/scripts/validation_heal/overlap.py index a8732f02..98590d79 100644 --- a/scripts/validation_heal/overlap.py +++ b/scripts/validation_heal/overlap.py @@ -15,13 +15,13 @@ from .heal_probe import canonical, payload, require from .healthy import BASE_OPTIONS, physical_content from .cursor import SIZE, TASK_SECONDS, run_topology as dx_topology, main as dx_main, wanted_pairs - from .stop import start, status, idle_fleet, validate_unknown_token + from .stop import start, status, idle_fleet, validate_retired_stop from .versions import public_version_id except ImportError: from heal_probe import canonical, payload, require from healthy import BASE_OPTIONS, physical_content from cursor import SIZE, TASK_SECONDS, run_topology as dx_topology, main as dx_main, wanted_pairs - from stop import start, status, idle_fleet, validate_unknown_token + from stop import start, status, idle_fleet, validate_retired_stop from versions import public_version_id @@ -287,7 +287,7 @@ def sample_disjoint(): monotonic(before, first_old, cancelled=True) new_before = wait_window(probe, fleet, path, new) retired = fleet.record("force-old-stop", probe.admin(path, [("clientToken", old), ("forceStop", "true")], b"")) - proof["oldStopIsolation"] = validate_unknown_token(retired, old) + proof["oldStopIsolation"] = validate_retired_stop(retired, old) new_after = status(probe, fleet, path, new, "force-new-after-old-stop") require(new_after.get("summary") != "stopped", "old token stopped the replacement task") require(active_window(new_after), "NOT_OBSERVED: old stop was not bracketed by the same running replacement") diff --git a/test_validation_heal.py b/test_validation_heal.py index 0bcca527..def6800c 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -2483,6 +2483,12 @@ def test_restart_preserves_every_source_branch(self): class HealOverlapOracleTest(unittest.TestCase): bucket = "rv-heal-overlap-unit" + def retired_stop(self, path, query, body): + return {"httpStatus": 200, "query": query, "method": "POST", "auth": "signed", + "requestBody": body.decode(), "path": path, + "response": {"summary": "stopped", "detail": "", "startTime": "2026-10-23T02:00:00Z", + "settings": {**BASE_OPTIONS, "scanMode": 1, "pool": None, "set": None}}} + def expected(self): return [{"key": f"objects/a/child/{index:04d}.bin", "versionId": "null", "deleteMarker": False, "payload": {"size": CURSOR_SIZE}} for index in range(overlap.SCOPE_OBJECTS)] @@ -2683,14 +2689,14 @@ def observed(*args): return value fleet = types.SimpleNamespace(nodes=["node"], record=record, snapshot=lambda *args: {"node": {"pid": 1, "unit": "unit"}}) - probe = types.SimpleNamespace(admin=lambda *args: {}) + probe = types.SimpleNamespace(admin=self.retired_stop) samples = [self.state(processed=90), self.state()] with self.subTest(branch=branch), patch.object(overlap, "start", side_effect=admit), \ patch.object(overlap, "wait_window", side_effect=(before, new_before)), \ patch.object(overlap, "status", side_effect=observed), patch.object(overlap, "completed_scope", side_effect=complete), \ - patch.object(overlap, "idle_fleet", return_value={}), patch.object(overlap, "validate_unknown_token", return_value={}), \ + patch.object(overlap, "idle_fleet", return_value={}), \ patch.object(overlap, "physical_content", return_value={}), patch.object(overlap.time, "sleep"): - overlap.observe_branch(branch, probe, fleet, {}, self.bucket, primary + other, targets, active, original={}) + result = overlap.observe_branch(branch, probe, fleet, {}, self.bucket, primary + other, targets, active, original={}) self.assertEqual(active, []) self.assertEqual(calls[-1][1], "new") if branch == "disjoint": @@ -2699,9 +2705,47 @@ def observed(*args): self.assertEqual(calls[0][2], [before, after]) self.assertEqual(calls[-1][2], [new_before] + peer_samples) else: + self.assertEqual(result[branch]["oldStopIsolation"]["exactRetiredTaskId"], "old") self.assertEqual(calls[-1][2][:2], [new_before, new_after]) self.assertEqual(len(calls[-1][2]), 4) + def test_force_start_rejects_unknown_or_malformed_retired_stop_before_completion(self): + primary = self.expected() + targets = [primary[index] for index in overlap.MISSING] + for kind in ("unknown", "token", "duplicate-token", "method", "body", "transport", + "settings", "summary", "timestamp", "extra-field"): + active = [] + def admit(probe, fleet, path, identities, **options): + token = "new" if identities else "old" + identities.append((path, token)) + return token + def response(path, query, body): + observed = self.retired_stop(path, query, body) + if kind == "unknown": + observed.update(httpStatus=500, response={"Code": "InternalError", + "Message": "Heal task not found: old"}) + if kind == "token": observed["query"] = [("clientToken", "new"), ("forceStop", "true")] + if kind == "duplicate-token": observed["query"] += [("clientToken", "new")] + if kind == "method": observed["method"] = "GET" + if kind == "body": observed["requestBody"] = "{}" + if kind == "transport": observed["transportError"] = "timeout" + if kind == "settings": observed["response"]["settings"]["recursive"] = True + if kind == "summary": observed["response"]["summary"] = "finished" + if kind == "timestamp": observed["response"]["startTime"] = "2026-10-23T02:00:00" + if kind == "extra-field": observed["response"]["clientToken"] = "old" + return observed + fleet = types.SimpleNamespace(record=lambda label, value: value) + probe = types.SimpleNamespace(admin=response) + with self.subTest(kind=kind), patch.object(overlap, "start", side_effect=admit), \ + patch.object(overlap, "wait_window", side_effect=(self.state(processed=23), self.state(processed=70))), \ + patch.object(overlap, "status", return_value=self.state(cancelled=True)) as status_call, \ + patch.object(overlap, "completed_scope") as completion, \ + self.assertRaises(ValueError): + overlap.observe_branch("force-start", probe, fleet, {}, self.bucket, primary, targets, active, original={}) + completion.assert_not_called() + self.assertEqual(status_call.call_count, 1) + self.assertEqual([token for _, token in active], ["old", "new"]) + def test_finished_window_and_same_token_counter_rollback_cannot_pass(self): self.assertFalse(overlap.active_window(self.state())) before = self.state(cancelled=True) From ff610f358fd98396a04b5180c456128730c8c20b Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 12:05:25 +0800 Subject: [PATCH 313/343] fix(validation): return B01 disk without restarting its fleet --- scripts/validation_heal/disk_loss.py | 16 +++--- scripts/validation_heal/stale_rejoin.py | 24 +++++--- test_validation_heal.py | 74 +++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 16 deletions(-) diff --git a/scripts/validation_heal/disk_loss.py b/scripts/validation_heal/disk_loss.py index 153ada72..6502a726 100644 --- a/scripts/validation_heal/disk_loss.py +++ b/scripts/validation_heal/disk_loss.py @@ -9,13 +9,13 @@ from .heal_workload import verify_seed from .healthy import physical_content from .shards import files, map_target, run_task - from .stale_rejoin import logical_metadata, identities + from .stale_rejoin import logical_metadata, identities, return_original_member except ImportError: from heal_probe import Probe, canonical, compare_inventories, payload, require from heal_workload import verify_seed from healthy import physical_content from shards import files, map_target, run_task - from stale_rejoin import logical_metadata, identities + from stale_rejoin import logical_metadata, identities, return_original_member BRANCHES = ("original-drive-return", "offline-ack-repair") @@ -220,12 +220,14 @@ def witness_offline(label): baseline_s3 = fleet.record("disk-loss-degraded-full-inventory", recorder.probe.inventory(bucket, unversioned_current_tags=True)) fleet.record("disk-loss-degraded-full-proof", {"expected": manifest, **verify_seed(manifest, baseline_s3)}) last = witness_offline("after-acks") - fleet.stop() - authority = fleet.snapshot("disk-loss-online-authorities", quiescent=True) + # A restart with the original disk still inaccessible leaves its unclean + # marker intact. Returning it through startup can repair ACKs before the + # manual token; reuse the original-process rejoin used by stale versions. + authority = fleet.snapshot("disk-loss-online-authorities") + require(physical_content(authority) == physical_content(fleet.snapshot("disk-loss-online-authorities-stable")), + "NOT_OBSERVED: offline ACK authority is not physically stable before live rejoin") witness = fleet.record("disk-loss-ack-authority", authority_witness(before, authority, bucket, target, new)) - fleet.record("disk-loss-original-member-return", fleet.remote(node, "drive-gate", mode="clear")) - fleet.drive_gates.remove(node) - fleet.start() + return_original_member(fleet, node, "disk-loss") require(physical_content(authority) == physical_content(fleet.snapshot("disk-loss-before-manual")), "ATTRIBUTION_NOT_OBSERVED: original member gained ACK shards before the manual token") receipts = [] diff --git a/scripts/validation_heal/stale_rejoin.py b/scripts/validation_heal/stale_rejoin.py index 754e6d5b..ccc679ee 100644 --- a/scripts/validation_heal/stale_rejoin.py +++ b/scripts/validation_heal/stale_rejoin.py @@ -182,6 +182,20 @@ def responsibility(status, bucket, key, expected, *, deleted=None): "bucketIncarnationId": incarnation, "counters": counters} +def return_original_member(fleet, node, prefix): + """Release the owned permission gate without triggering startup recovery.""" + processes = {peer: fleet.remote(peer, "process") for peer in fleet.nodes} + release = fleet.record(prefix + "-original-member-rejoin", fleet.remote(node, "drive-gate-release-live")) + require(release.get("sameOriginalProcess") is True and release.get("serviceRestarted") is False, + "member rejoin restarted or replaced the original process") + fleet.drive_gates.remove(node) + rejoined = {peer: fleet.remote(peer, "process") for peer in fleet.nodes} + require(all(all(processes[peer].get(name) == rejoined[peer].get(name) for name in PROCESS_FIELDS) for peer in fleet.nodes), + "live rejoin changed an original peer process/unit/candidate/environment") + fleet.record(prefix + "-live-rejoin-process-proof", {"before": processes, "after": rejoined, + "allOriginalProcessesPreserved": True}) + + def run_branch(recorder, fleet, bucket, branch, expected, active, admissions, contract): # Lazy imports keep the shared version executor's dependency graph acyclic. try: @@ -219,15 +233,7 @@ def run_branch(recorder, fleet, bucket, branch, expected, active, admissions, co require(physical_content(authority) == physical_content(fleet.snapshot("stale-online-authorities-stable")), "NOT_OBSERVED: offline ACK authority is not physically stable before live rejoin") witness = fleet.record("stale-generation-authority-proof", authority_witness(before, authority, bucket, key, manifest, target)) - processes = {peer: fleet.remote(peer, "process") for peer in fleet.nodes} - release = fleet.record("stale-original-member-rejoin", fleet.remote(node, "drive-gate-release-live")) - require(release.get("sameOriginalProcess") is True and release.get("serviceRestarted") is False, - "stale member rejoin restarted or replaced the original process") - fleet.drive_gates.remove(node) - rejoined = {peer: fleet.remote(peer, "process") for peer in fleet.nodes} - require(all(all(processes[peer].get(name) == rejoined[peer].get(name) for name in PROCESS_FIELDS) for peer in fleet.nodes), - "stale live rejoin changed an original peer process/unit/candidate/environment") - fleet.record("stale-live-rejoin-process-proof", {"before": processes, "after": rejoined, "allOriginalProcessesPreserved": True}) + return_original_member(fleet, node, "stale") require(physical_content(authority) == physical_content(fleet.snapshot("stale-before-manual")), "ATTRIBUTION_NOT_OBSERVED: stale namespace changed before its manual token") status = run_task(recorder.probe, fleet, {"bucket": bucket, "key": key}, 2, active, diff --git a/test_validation_heal.py b/test_validation_heal.py index def6800c..1f880d8c 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -3397,6 +3397,80 @@ def task(*args, **kwargs): class HealDiskLossTest(unittest.TestCase): + def test_original_member_return_preserves_every_peer_process_identity(self): + identities = {node: {name: node + ":" + name for name in fleet_node.PROCESS_FIELDS} + for node in ("node1", "node2")} + for changed in (None, *fleet_node.PROCESS_FIELDS, "restart", "unbound"): + released = False + def remote(node, action): + nonlocal released + if action == "drive-gate-release-live": + released = True + return {"sameOriginalProcess": changed != "unbound", "serviceRestarted": changed == "restart"} + self.assertEqual(action, "process") + value = copy.deepcopy(identities[node]) + if released and node == "node1" and changed in fleet_node.PROCESS_FIELDS: + value[changed] = "changed" + return value + fleet = types.SimpleNamespace(nodes=list(identities), drive_gates={"node2"}, remote=remote, + record=Mock(side_effect=lambda label, value: value)) + with self.subTest(changed=changed): + if changed is None: + stale_rejoin.return_original_member(fleet, "node2", "disk-loss") + self.assertEqual(fleet.drive_gates, set()) + self.assertEqual(fleet.record.call_args.args[0], "disk-loss-live-rejoin-process-proof") + else: + with self.assertRaises(ValueError): + stale_rejoin.return_original_member(fleet, "node2", "disk-loss") + self.assertNotIn("disk-loss-live-rejoin-process-proof", [call.args[0] for call in fleet.record.call_args_list]) + + def test_offline_ack_rejoin_cannot_restart_or_admit_manual_work_after_early_repair(self): + original, authority, repaired, bucket, target, new, _ = self.scene() + online = self.admin_info(target) + offline = copy.deepcopy(online) + offline["response"]["info"]["servers"][0]["drives"][target["setSlot"]].update(state="drive access denied") + for failure in (None, "unstable", "early-repair"): + events = [] + def snapshot(label, **options): + if label.startswith("disk-loss-original-"): return original + self.assertFalse(options.get("quiescent", False), "offline ACKs must remain in their original running fleet") + if failure == "unstable" and label == "disk-loss-online-authorities-stable": return repaired + if failure == "early-repair" and label == "disk-loss-before-manual": return repaired + return authority + def remote(node, action, **options): + if action == "drive-gate": return {"dataPath": "/owned-root"} + if action == "drive-gate-probe": + return {"dataPath": "/owned-root", "readProbe": {"errno": 13}, "errno": 13, "dacBypassRemoved": True} + if action == "drive-gate-release-live": + events.append("live-return") + return {"sameOriginalProcess": True, "serviceRestarted": False} + self.assertEqual(action, "process") + return {name: node + ":" + name for name in fleet_node.PROCESS_FIELDS} + def ack(*args): + events.append("ack") + return new, new + fleet = types.SimpleNamespace(nodes=["node1"], drive_gates=set(), remote=remote, + record=lambda label, value: value, snapshot=snapshot, + start=lambda: events.append("start"), stop=lambda: events.append("stop")) + recorder = types.SimpleNamespace(probe=types.SimpleNamespace( + admin=Mock(side_effect=[online, offline, offline]), inventory=lambda *args, **kw: {})) + with self.subTest(failure=failure), patch.object(disk_loss, "map_target", return_value=target), \ + patch.object(disk_loss, "offline_ack", side_effect=ack), \ + patch.object(disk_loss, "verify_seed", return_value={}), \ + patch.object(disk_loss, "run_task", side_effect=RuntimeError("manual boundary reached")) as run: + if failure is None: + with self.assertRaisesRegex(RuntimeError, "manual boundary reached"): + disk_loss.run_branch(recorder, fleet, bucket, "offline-ack-repair", [], [], [], + HealReadQuorumBoundaryTest.contract) + self.assertEqual(run.call_count, 1) + else: + with self.assertRaisesRegex(ValueError, "NOT_OBSERVED"): + disk_loss.run_branch(recorder, fleet, bucket, "offline-ack-repair", [], [], [], + HealReadQuorumBoundaryTest.contract) + run.assert_not_called() + self.assertNotIn("stop", events[events.index("ack"):]) + self.assertNotIn("start", events[events.index("ack"):]) + def scene(self, total=4, data=2): original, old = HealReadQuorumBoundaryTest().scene(total=total, data=data) bucket, old_key, new_key = HealReadQuorumBoundaryTest.bucket, old[0]["key"], "disk-loss/offline-ack-repair/new-00.bin" From ad2f7c1bfd6aa235f3d0133208c0b256fa3bbb85 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 12:11:30 +0800 Subject: [PATCH 314/343] fix(validation): account for protected B01 repair sidecars --- scripts/validation_heal/disk_loss.py | 11 +++++++++++ test_validation_heal.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/scripts/validation_heal/disk_loss.py b/scripts/validation_heal/disk_loss.py index 6502a726..7430a7a7 100644 --- a/scripts/validation_heal/disk_loss.py +++ b/scripts/validation_heal/disk_loss.py @@ -3,6 +3,7 @@ import copy import hashlib import time +import uuid try: from .heal_probe import Probe, canonical, compare_inventories, payload, require @@ -175,6 +176,16 @@ def repaired_physical(authority, after, bucket, target, new, witness): require(shard["diskUUID"] == target["diskUUID"] and shard["setSlot"] == target["setSlot"], "repair used a replacement disk or different member slot") allowed.update((part, metadata)); cohorts.append(cohort) + protected = cohort["logical"]["parts"][0].get("integrityDescriptorPresent") is True + sidecars = [identity for identity in right if identity[:2] == member + and identity[2].startswith(part[2] + ".integrity.")] + require(len(sidecars) == int(protected), "rejoined protected part has missing or ambiguous integrity sidecars") + for identity in sidecars: + generation = identity[2].removeprefix(part[2] + ".integrity.") + require(str(uuid.UUID(generation)) == generation and uuid.UUID(generation).int != 0 + and 0 < right[identity]["size"] <= 65536, + "rejoined integrity sidecar has an invalid generation or size") + allowed.add(identity) require(set(right) - set(left) == allowed, "repair created unexpected or missing files outside the original member's ACKs") return cohorts, {"sameDiskUUIDAndSetSlot": True, "allDegradedACKTargetPartsCreated": True, "originalAndOnlineShardBytesUnchanged": True, "backupRestorationUsed": False, diff --git a/test_validation_heal.py b/test_validation_heal.py index 1f880d8c..54778cae 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -3513,6 +3513,34 @@ def test_actual_single_member_quorum_and_repair_preserve_all_original_bytes(self self.assertTrue(proof["allDegradedACKTargetPartsCreated"]) self.assertFalse(proof["backupRestorationUsed"]) + def test_repaired_protected_ack_requires_one_exact_target_integrity_sidecar(self): + original, authority, repaired, bucket, target, new, _ = self.scene() + for scene in (authority, repaired): + for value in scene.values(): + for item in value["files"]: + metadata = item.get("metadata", {}) + if metadata.get("key") == new[0]["key"]: + metadata["versions"][0]["parts"][0]["integrityDescriptorPresent"] = True + witness = disk_loss.authority_witness(original, authority, bucket, target, new) + cohort = map_target(repaired, bucket, new[0]["key"], "data", 1) + member = next(shard for shard in cohort["cohort"] if shard["node"] == target["node"] and shard["disk"] == target["disk"]) + sidecar = {"disk": target["disk"], "path": member["relativePath"] + ".integrity.12345678-1234-1234-1234-123456789abc", + "size": 1408, "sha256": "actual-sidecar-hash"} + repaired[target["node"]]["files"].append(sidecar) + disk_loss.repaired_physical(authority, repaired, bucket, target, new, witness) + for kind in ("missing", "duplicate", "foreign-generation", "nil", "empty", "oversize", "different-member"): + changed = copy.deepcopy(repaired) + row = changed[target["node"]]["files"][-1] + if kind == "missing": changed[target["node"]]["files"].pop() + elif kind == "duplicate": changed[target["node"]]["files"].append({**row, "path": row["path"][:-1] + "d"}) + elif kind == "foreign-generation": row["path"] = row["path"].replace(cohort["dataDir"], "other") + elif kind == "nil": row["path"] = member["relativePath"] + ".integrity.00000000-0000-0000-0000-000000000000" + elif kind == "empty": row["size"] = 0 + elif kind == "oversize": row["size"] = 65537 + else: row["disk"] = (target["disk"] + 1) % 4 + with self.subTest(kind=kind), self.assertRaises(ValueError): + disk_loss.repaired_physical(authority, changed, bucket, target, new, witness) + def test_offline_ack_authority_rejects_target_writes_loss_of_controls_and_donor_versions(self): original, authority, repaired, bucket, target, new, witness = self.scene() for kind in ("offline-write", "control", "donor-version", "format", "outside"): From 21afd51157800f2c1495bd2bac6fe74a37659860 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 12:26:28 +0800 Subject: [PATCH 315/343] fix(validation): bind quorum checks to candidate identity --- scripts/validation_heal/quorum_boundary.py | 11 ++++++++++ scripts/validation_heal/write_quorum.py | 23 ++++++++++++++++----- test_validation_heal_quorum_candidate.py | 24 ++++++++++++++++++++-- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/scripts/validation_heal/quorum_boundary.py b/scripts/validation_heal/quorum_boundary.py index b3e55d17..d191451f 100644 --- a/scripts/validation_heal/quorum_boundary.py +++ b/scripts/validation_heal/quorum_boundary.py @@ -78,6 +78,17 @@ **LOCK_REFERENCES['85c706cac09d53d0649487b8f1865f18beb53225'], 'candidateCommit': '62ff96774fa4139d45f14d70b70519aae8070ac0'} +LOCK_REFERENCES['1a3f038c90704f9ad9693eddcf7d861c9530f13c'] = { + **LOCK_REFERENCES['62ff96774fa4139d45f14d70b70519aae8070ac0'], + 'candidateCommit': '1a3f038c90704f9ad9693eddcf7d861c9530f13c', + 'manualHealLockPolicy': 'ordinary manual heal retains the exclusive namespace owner through storage completion; read-repair has a separate shared data phase and exclusive commit fence', + 'sourceFiles': { + **LOCK_REFERENCES['62ff96774fa4139d45f14d70b70519aae8070ac0']['sourceFiles'], + 'crates/ecstore/src/set_disk/ops/locking.rs': 'b75bb680ab648a1cac84a14318e7b0c5022897be43b624281f6fa277235a4e62', + 'crates/ecstore/src/set_disk/ops/heal.rs': '0b706ac476e02202d5ad217a166eb523abfd3cfd123902a9fb7edf27bd2bf376', + 'crates/ecstore/src/cluster/rpc/client.rs': '515b1ec94c2d88e8cf77646652dc9b84405f1c5e5eb4cc103b79be21746edc47'}, +} + def lock_reference(candidate_commit): require(isinstance(candidate_commit, str) and candidate_commit in LOCK_REFERENCES, diff --git a/scripts/validation_heal/write_quorum.py b/scripts/validation_heal/write_quorum.py index c6fdbb48..4904764b 100644 --- a/scripts/validation_heal/write_quorum.py +++ b/scripts/validation_heal/write_quorum.py @@ -2,6 +2,7 @@ """B08: independently witnessed write and distributed-lock quorum loss.""" import argparse +import copy from datetime import datetime, timedelta, timezone import hashlib import json @@ -23,7 +24,7 @@ from .shards import SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort from .versions import validate_version_receipts, verify_quorum_contract, verify_version_s3 from .drive_gate import validate_unwritable - from .quorum_boundary import LOCK_REFERENCE + from .quorum_boundary import LOCK_REFERENCE, lock_reference from .stop import idle_fleet except ImportError: from heal_probe import Probe, payload, require, save @@ -34,7 +35,7 @@ from shards import SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort from versions import validate_version_receipts, verify_quorum_contract, verify_version_s3 from drive_gate import validate_unwritable - from quorum_boundary import LOCK_REFERENCE + from quorum_boundary import LOCK_REFERENCE, lock_reference from stop import idle_fleet @@ -49,6 +50,18 @@ "H2LockFixture": "explicit EC:8 only for the fresh run-owned sixteen-drive lock-loss branch", "WriteFixture": "all N readable authorities; independent read-only gates remove W even when R equals W"} +STORAGE_REFERENCES = { + STORAGE_REFERENCE['candidateCommit']: STORAGE_REFERENCE, + '1a3f038c90704f9ad9693eddcf7d861c9530f13c': { + **STORAGE_REFERENCE, 'candidateCommit': '1a3f038c90704f9ad9693eddcf7d861c9530f13c'}, +} + + +def source_reference(candidate_commit): + locks = lock_reference(candidate_commit) + require(candidate_commit in STORAGE_REFERENCES, "B08 storage boundary is not reviewed for this candidate") + return {"lock": locks, "storage": copy.deepcopy(STORAGE_REFERENCES[candidate_commit])} + def plan_for(snapshot, target, branch, contract): require(branch in BRANCHES and contract.get("recognized") is True @@ -181,7 +194,7 @@ def run_branch(args, topology, nodes, branch): lock_fixture = topology == "H2" and branch == "lock-quorum-absent" fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="lock-quorum" if lock_fixture else "manual") candidate = json.loads(args.candidate.read_text()) - require(candidate["source_sha"] == LOCK_REFERENCE["candidateCommit"], "B08 lock client policy is unreviewed") + reference = source_reference(candidate["source_sha"]) bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] access, secret = secrets.token_hex(16), secrets.token_hex(24) os.environ.update(HEAL_ACCESS_KEY=access, HEAL_SECRET_KEY=secret) @@ -192,8 +205,8 @@ def run_branch(args, topology, nodes, branch): "candidatePackageSha256": candidate["package_sha256"], "candidateBinarySha256": args.binary_sha, "rcBinarySha256": hashlib.sha256(args.rc_bin.read_bytes()).hexdigest(), "maxInventoryObjects": 256, "maxInventoryBytes": 512 * 1024**2} - fleet.record("lock-policy-source-contract", LOCK_REFERENCE) - fleet.record("storage-boundary-source-contract", STORAGE_REFERENCE) + fleet.record("lock-policy-source-contract", reference["lock"]) + fleet.record("storage-boundary-source-contract", reference["storage"]) contract = verify_quorum_contract(fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)), candidate) fleet.record("binding", binding) probe, active = Probe(binding), [] diff --git a/test_validation_heal_quorum_candidate.py b/test_validation_heal_quorum_candidate.py index 84bb7e4e..963734fa 100644 --- a/test_validation_heal_quorum_candidate.py +++ b/test_validation_heal_quorum_candidate.py @@ -16,6 +16,7 @@ FIXED = '1c7fc818dd858247c45b39542abd8291b1e6e31b' DIAGNOSTIC = '85c706cac09d53d0649487b8f1865f18beb53225' RPC_FIXED = '62ff96774fa4139d45f14d70b70519aae8070ac0' +IDENTITY_FIXED = '1a3f038c90704f9ad9693eddcf7d861c9530f13c' class RecordedSourceContract(Exception): @@ -29,7 +30,7 @@ def execute_until_contract(self, module, source, *, allow_inspector=True): records = {} def record(label, value): records[label] = value - if label in {'quorum-source-contract', 'network-source-contract'}: + if label in {'quorum-source-contract', 'network-source-contract', 'storage-boundary-source-contract'}: raise RecordedSourceContract(records) return value fleet = types.SimpleNamespace(record=record) @@ -51,6 +52,8 @@ def record(label, value): # exists on this fleet and no Probe is constructed by these tests. if module is quorum_boundary: module.run_topology(args, 'H1', ['vm000']) + elif module is write_quorum: + module.run_branch(args, 'H1', ['vm000'], write_quorum.BRANCHES[0]) else: module.run_branch(args, ['vm000', 'vm001', 'vm002', 'vm003'], network.BRANCHES[0]) return records @@ -70,13 +73,24 @@ def test_actual_executors_record_the_selected_candidate_before_deploy(self): self.assertEqual(captured['binding']['candidateCommit'], source) def test_unknown_and_malformed_sources_never_reach_inspector_or_deploy(self): - for module in (quorum_boundary, network): + for module in (quorum_boundary, network, write_quorum): for source in ('0' * 40, FIXED[:7], FIXED.upper(), RPC_FIXED[:7], RPC_FIXED.upper(), RPC_FIXED[:-1] + '1', None, [], True): with self.subTest(executor=module.CASE_ID, source=source): with self.assertRaises(ValueError): self.execute_until_contract(module, source, allow_inspector=False) + def test_write_quorum_binds_both_source_reviews_before_inspector_or_deploy(self): + for source in (ORIGINAL, IDENTITY_FIXED): + with self.subTest(source=source), self.assertRaises(RecordedSourceContract) as completed: + self.execute_until_contract(write_quorum, source) + captured = completed.exception.records + self.assertEqual(captured['lock-policy-source-contract']['candidateCommit'], source) + self.assertEqual(captured['storage-boundary-source-contract']['candidateCommit'], source) + for source in (FIXED, DIAGNOSTIC, RPC_FIXED): + with self.subTest(source=source), self.assertRaisesRegex(ValueError, 'storage boundary'): + self.execute_until_contract(write_quorum, source, allow_inspector=False) + def test_network_storage_review_is_separate_from_lock_review(self): source = 'f' * 40 with patch.dict(quorum_boundary.LOCK_REFERENCES, {source: { @@ -97,6 +111,12 @@ def test_reference_exports_cannot_mutate_original_or_other_candidate(self): self.assertEqual(quorum_boundary.LOCK_REFERENCE, original) self.assertEqual(write_quorum.LOCK_REFERENCE['candidateCommit'], ORIGINAL) self.assertEqual(write_quorum.STORAGE_REFERENCE['candidateCommit'], ORIGINAL) + selected = write_quorum.source_reference(IDENTITY_FIXED) + selected['lock']['sourceFiles'].clear() + selected['storage']['sourceFiles'].clear() + self.assertTrue(write_quorum.source_reference(IDENTITY_FIXED)['lock']['sourceFiles']) + self.assertTrue(write_quorum.source_reference(IDENTITY_FIXED)['storage']['sourceFiles']) + self.assertEqual(write_quorum.STORAGE_REFERENCE['candidateCommit'], ORIGINAL) if __name__ == '__main__': From cc578a379048c650783769fbb07f13beab610d5b Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 13:55:49 +0800 Subject: [PATCH 316/343] fix(validation): retain quorum rejection recovery evidence --- scripts/validation_heal/shards.py | 33 +++++++++++++++- scripts/validation_heal/versions.py | 27 +++++++++++-- scripts/validation_heal/write_quorum.py | 32 ++++++++++++---- test_validation_heal.py | 50 ++++++++++++++++++++++++- 4 files changed, 128 insertions(+), 14 deletions(-) diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index 2bae48bb..5941f5a5 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -86,6 +86,34 @@ } SIZES = [0, 1, 4095, 4096, 4097, 1024**2 - 1, 1024**2, 1024**2 + 1, 5 * 1024**2 + 123] MULTIPART_SIZES = (5 * 1024**2 + 17, 5 * 1024**2 + 93, 1024**2 + 47) +COORDINATION_REJECTION_BODY = ( + 'InternalError' + 'cluster heal coordination unavailable') + + +def validate_coordination_rejection(observation, *, path=None): + require(observation.get("httpStatus") == 500 + and observation.get("response") == COORDINATION_REJECTION_BODY, + "Heal admission refusal differs from the reviewed coordinator failure") + require(path is None or observation.get("path") == path, + "Heal admission refusal belongs to another request path") + return {"admissionState": "coordination-rejected", "path": observation.get("path"), + "httpStatus": 500, "response": COORDINATION_REJECTION_BODY, + "errorCode": "InternalError", "errorMessage": "cluster heal coordination unavailable", + "clientToken": None, "taskAdmitted": False, "responsibilityReported": False} + + +def validate_coordination_rejection_logs(observations): + require(isinstance(observations, dict) and observations, "coordinator failure logs are missing") + matches = [] + for node, observation in observations.items(): + journal = observation.get("journal") if isinstance(observation, dict) else None + if (isinstance(journal, str) and "local_coordinator_failed" in journal + and "heal control start admission failed" in journal): + matches.append(node) + require(matches, "coordinator rejection is not attributed to the exact admission failure in service logs") + return {"coordinatorFailureLogged": True, "nodesWithFailure": sorted(matches), + "reason": "local_coordinator_failed", "admissionFailure": "heal control start admission failed"} def files(snapshot): @@ -365,12 +393,15 @@ def validate_absent_request(status, bucket, key, options): "completeResponsibility": True, "repairPassClaimed": False} -def run_task(probe, fleet, target, mode, active, *, status_endpoints=(), option_overrides=None, admissions=None): +def run_task(probe, fleet, target, mode, active, *, status_endpoints=(), option_overrides=None, admissions=None, + allow_coordination_rejection=False): path = "/rustfs/admin/v3/heal/" + quote(target["bucket"], safe="") if target["key"]: path += "/" + quote(target["key"], safe="/") started = fleet.record("trigger", probe.admin(path, [], json.dumps( {**BASE_OPTIONS, "scanMode": mode, **(option_overrides or {})}).encode())) + if allow_coordination_rejection and started.get("httpStatus") != 200: + return validate_coordination_rejection(started, path=path) require(started.get("httpStatus") == 200 and isinstance(started.get("response"), dict), "repair admission failed") token = started["response"].get("clientToken") require(isinstance(token, str) and token, "repair admission omitted token") diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 9f76f812..04cf78db 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -37,6 +37,7 @@ from .disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch, info_disks from .replacement import BRANCHES as REPLACEMENT_BRANCHES, AUTOMATIC_BRANCHES as AUTO_REPLACEMENT_BRANCHES, GRACEFUL_BRANCHES, CRASH_BRANCHES, run_branch as replacement_branch from .shards import (SIZES, files, map_target, run_task, validate_fault, + validate_coordination_rejection, validate_coordination_rejection_logs, validate_repaired_physical, verify_cohort, verify_ranges) except ImportError: from heal_probe import Probe, compare_inventories, payload, require, save @@ -57,6 +58,7 @@ from disk_loss import BRANCHES as DISK_LOSS_BRANCHES, run_branch as disk_loss_branch, info_disks from replacement import BRANCHES as REPLACEMENT_BRANCHES, AUTOMATIC_BRANCHES as AUTO_REPLACEMENT_BRANCHES, GRACEFUL_BRANCHES, CRASH_BRANCHES, run_branch as replacement_branch from shards import (SIZES, files, map_target, run_task, validate_fault, + validate_coordination_rejection, validate_coordination_rejection_logs, validate_repaired_physical, verify_cohort, verify_ranges) @@ -974,19 +976,30 @@ def run_topology(args, topology, nodes, fixture): validate_unrecoverable_get(denied, contract) require(physical_content(prior) == physical_content(fleet.snapshot("physical-after-negative-get")), "negative read widened damage or silently healed below-quorum fragments") + admissions_before_attempt = len(admissions) if args.case_id == "heal.read-repair-attribution": status = tracked_read_task(probe, fleet, bucket, branch, plan["expected"][0]) elif scanner: status = scanner_task(probe, fleet, scanner_window, bucket, plan["expected"][0], branch) else: status = run_task(probe, fleet, plan["scope"], 2, active, - option_overrides={"recursive": plan["recursive"]}, admissions=admissions) + option_overrides={"recursive": plan["recursive"]}, admissions=admissions, + allow_coordination_rejection=(args.case_id == "heal.disk-readonly-and-permission" + and branch == "permission-denied")) fleet.stop() repaired = fleet.snapshot("physical-after-" + branch, quiescent=True, metadata_faults=metadata_faults) if gate_node is not None: require(physical_content(prior) == physical_content(repaired), "unwritable window changed object files or wrote outside the target") - gated_proof = validate_unwritable(status, bucket, plan["expected"][0]) + if status.get("admissionState") == "coordination-rejected": + rejection = validate_coordination_rejection(status, path=status.get("path")) + logs = {node: fleet.record("coordination-rejection-logs", {"node": node, + **fleet.remote(node, "logs")}) for node in fleet.nodes} + rejection["logProof"] = validate_coordination_rejection_logs(logs) + rejection["physicalUnchanged"] = True + gated_proof = rejection + else: + gated_proof = validate_unwritable(status, bucket, plan["expected"][0]) fleet.record("unwritable-responsibility", gated_proof) fleet.record("drive-gate-released", fleet.remote(gate_node, "drive-gate", mode="clear")) fleet.drive_gates.remove(gate_node) @@ -994,8 +1007,14 @@ def run_topology(args, topology, nodes, fixture): fleet.start() restored_status = run_task(probe, fleet, plan["scope"], 2, active, option_overrides={"recursive": False}, admissions=admissions) - require(len(admissions) >= 2 and admissions[-1]["clientToken"] != admissions[-2]["clientToken"], - "restored writable repair reused the failed task identity") + if status.get("admissionState") == "coordination-rejected": + require(len(admissions) == admissions_before_attempt + 1 + and admissions[-1]["clientToken"] not in {item["clientToken"] + for item in admissions[:admissions_before_attempt]}, + "restored writable repair lacks a fresh unique task identity") + else: + require(len(admissions) >= 2 and admissions[-1]["clientToken"] != admissions[-2]["clientToken"], + "restored writable repair reused the failed task identity") status = restored_status fleet.stop() repaired = fleet.snapshot("physical-writable-repair", quiescent=True) diff --git a/scripts/validation_heal/write_quorum.py b/scripts/validation_heal/write_quorum.py index 4904764b..692e379c 100644 --- a/scripts/validation_heal/write_quorum.py +++ b/scripts/validation_heal/write_quorum.py @@ -21,7 +21,9 @@ from .healthy import Fleet, physical_content, validate_topology from .inspector import verify as verify_inspector from .matrix import select - from .shards import SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort + from .shards import (SIZES, files, map_target, run_task, validate_fault, + validate_coordination_rejection, validate_coordination_rejection_logs, + validate_repaired_physical, verify_cohort) from .versions import validate_version_receipts, verify_quorum_contract, verify_version_s3 from .drive_gate import validate_unwritable from .quorum_boundary import LOCK_REFERENCE, lock_reference @@ -32,7 +34,9 @@ from healthy import Fleet, physical_content, validate_topology from inspector import verify as verify_inspector from matrix import select - from shards import SIZES, files, map_target, run_task, validate_fault, validate_repaired_physical, verify_cohort + from shards import (SIZES, files, map_target, run_task, validate_fault, + validate_coordination_rejection, validate_coordination_rejection_logs, + validate_repaired_physical, verify_cohort) from versions import validate_version_receipts, verify_quorum_contract, verify_version_s3 from drive_gate import validate_unwritable from quorum_boundary import LOCK_REFERENCE, lock_reference @@ -209,7 +213,7 @@ def run_branch(args, topology, nodes, branch): fleet.record("storage-boundary-source-contract", reference["storage"]) contract = verify_quorum_contract(fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)), candidate) fleet.record("binding", binding) - probe, active = Probe(binding), [] + probe, active, admissions = Probe(binding), [], [] result = {"topology": topology, "branch": branch, "verdict": "RUNNING", "cleanup": "PENDING"} put_observed, put_key, put_payload, original = None, None, None, None try: @@ -272,8 +276,12 @@ def witness(label): put_observed = fleet.record("single-negative-PUT-no-retry", probe.s3("put_object", {"Bucket": bucket, "Key": put_key, "Metadata": {"heal-fixture": "DL"}, "Tagging": "suite=heal&fixture=DL"}, put_body)) result["put"] = negative_put(put_observed) - status = run_task(probe, fleet, {"bucket": bucket, "key": selected["key"]}, 2, active) - result["negativeResponsibility"] = validate_unwritable(status, bucket, selected) + status = run_task(probe, fleet, {"bucket": bucket, "key": selected["key"]}, 2, active, + admissions=admissions, allow_coordination_rejection=(topology == "H2")) + coordination_rejected = status.get("admissionState") == "coordination-rejected" + result["negativeResponsibility"] = validate_coordination_rejection(status) if coordination_rejected \ + else validate_unwritable(status, bucket, selected) + result["negativeResponsibility"]["physicalMutationClaimed"] = False fleet.record("quorums-after-operations", witness("service-quorum-after")) for node in plan["offlineNodes"]: fleet.record("lock-peer-still-refused", endpoint_refused(binding, nodes.index(node))) @@ -282,8 +290,10 @@ def witness(label): fleet.stop() scene = fleet.snapshot("physical-negative-before-recovery", quiescent=True) require(physical_content(faulted) == physical_content(scene), "negative write/Heal mutated target or intact controls") - for node in nodes: - fleet.record("negative-quorum-peer-logs", {"node": node, **fleet.remote(node, "logs")}) + logs = {node: fleet.record("negative-quorum-peer-logs", {"node": node, **fleet.remote(node, "logs")}) + for node in nodes} + if coordination_rejected: + result["coordinationRejectionLogs"] = validate_coordination_rejection_logs(logs) for node in tuple(fleet.quorum_gates): fleet.record("release-write-quorum-gate", fleet.remote(node, "quorum-gate", mode="clear")) fleet.quorum_gates.remove(node) @@ -291,7 +301,13 @@ def witness(label): require(physical_content(faulted) == physical_content(fleet.snapshot("physical-source-return-before-repair")), "source return repaired the target before the independent manual recovery task") recovered = run_task(probe, fleet, {"bucket": bucket, "key": selected["key"]}, 2, active, - status_endpoints=tuple(binding["endpoints"])[1:]) + status_endpoints=tuple(binding["endpoints"])[1:], admissions=admissions) + if coordination_rejected: + require(len(admissions) == 1 and admissions[0]["clientToken"], + "recovered quorum repair lacks its first admitted task identity") + else: + require(len(admissions) == 2 and admissions[-1]["clientToken"] != admissions[-2]["clientToken"], + "recovered quorum repair reused the negative-window task identity") result["recoveryResponsibility"] = validate_version_receipts(recovered, bucket, [selected], 2, required_repairs={(selected["key"], "null")}) idle_fleet(fleet, binding) diff --git a/test_validation_heal.py b/test_validation_heal.py index 54778cae..312304ae 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -15,7 +15,9 @@ from scripts.validation_heal.healthy import validate_healthy, validate_topology, validate_rejection, Fleet, BASE_OPTIONS from scripts.validation_heal.fleet_node import paths, stopped, inspect_metadata from scripts.validation_heal.inspector import verify as verify_inspector -from scripts.validation_heal.shards import BRANCHES, files, map_target, validate_fault, validate_repaired_physical, validate_repaired_status, peer_status_equal, validate_dry_run_status +from scripts.validation_heal.shards import (BRANCHES, files, map_target, validate_fault, validate_repaired_physical, + validate_repaired_status, peer_status_equal, validate_dry_run_status, validate_coordination_rejection, + validate_coordination_rejection_logs) from scripts.validation_heal.heal_fault import mutation from scripts.validation_heal.shards import validate_conditional_response from scripts.validation_heal.shards import run_task, verify_ranges @@ -287,6 +289,52 @@ def admin(path, query, body): self.assertEqual(active, []) self.assertEqual(admissions, [{"path": expected_path, "clientToken": "original-token"}]) + def test_exact_coordinator_rejection_is_branch_scoped_and_never_admits_a_task(self): + body = ('InternalError' + 'cluster heal coordination unavailable') + observed = {"httpStatus": 500, "response": body} + calls, records, active, admissions = [], [], [], [] + + def admin(path, query, request_body): + calls.append((path, query, request_body)) + return {**observed, "path": path} + + result = run_task(types.SimpleNamespace(admin=admin), + types.SimpleNamespace(record=lambda label, value: records.append((label, value)) or value), + {"bucket": "fixture", "key": "objects/0008.bin"}, 2, active, + admissions=admissions, allow_coordination_rejection=True) + self.assertEqual(result["admissionState"], "coordination-rejected") + self.assertEqual(result["httpStatus"], 500) + self.assertFalse(result["taskAdmitted"]) + self.assertIsNone(result["clientToken"]) + self.assertFalse(result["responsibilityReported"]) + self.assertEqual(len(calls), 1) + self.assertEqual(active, []) + self.assertEqual(admissions, []) + with self.assertRaisesRegex(ValueError, "repair admission failed"): + run_task(types.SimpleNamespace(admin=admin), + types.SimpleNamespace(record=lambda _label, value: value), + {"bucket": "fixture", "key": "objects/0008.bin"}, 2, [], + allow_coordination_rejection=False) + + def test_coordinator_rejection_requires_exact_status_and_xml(self): + exact = {"httpStatus": 500, "response": ('' + 'InternalErrorcluster heal coordination unavailable')} + self.assertEqual(validate_coordination_rejection(exact)["errorCode"], "InternalError") + for changed in ( + {**exact, "httpStatus": 503}, + {**exact, "response": exact["response"].replace("unavailable", "temporarily unavailable")}, + {**exact, "response": exact["response"] + " "}, + ): + with self.subTest(changed=changed): + with self.assertRaisesRegex(ValueError, "reviewed coordinator failure"): + validate_coordination_rejection(changed) + logs = {"node1": {"journal": ('{"reason":"local_coordinator_failed",' + '"error":"heal control start admission failed"}')}} + self.assertTrue(validate_coordination_rejection_logs(logs)["coordinatorFailureLogged"]) + with self.assertRaisesRegex(ValueError, "not attributed"): + validate_coordination_rejection_logs({"node1": {"journal": "unrelated failure"}}) + def test_condition_denial_is_not_a_transport_failure_or_success_payload(self): baseline = {"properties": {"ETag": '"etag"'}, "payload": {"size": 3, "sha256": "correct"}} good = {"response": {"ResponseMetadata": {"HTTPStatusCode": 412}, "Error": {"Code": "PreconditionFailed"}}, From a3d3c62a9f6d29667a1a2b57d13f5cb9e166de3f Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 14:51:59 +0800 Subject: [PATCH 317/343] fix(validation): bind coordinator rejection to service evidence --- scripts/validation_heal/shards.py | 89 ++++++++++++++++++++++--- scripts/validation_heal/versions.py | 3 +- scripts/validation_heal/write_quorum.py | 3 +- test_validation_heal.py | 58 ++++++++++++++-- 4 files changed, 135 insertions(+), 18 deletions(-) diff --git a/scripts/validation_heal/shards.py b/scripts/validation_heal/shards.py index 5941f5a5..54284a27 100644 --- a/scripts/validation_heal/shards.py +++ b/scripts/validation_heal/shards.py @@ -105,15 +105,76 @@ def validate_coordination_rejection(observation, *, path=None): def validate_coordination_rejection_logs(observations): require(isinstance(observations, dict) and observations, "coordinator failure logs are missing") + request = observations.get("request") + peers = observations.get("peers") + require(isinstance(request, dict) and isinstance(peers, dict), + "coordinator rejection lacks its request window and peer logs") + require(request.get("method") == "POST" and request.get("httpStatus") == 500 + and isinstance(request.get("path"), str) and request["path"].startswith("/rustfs/admin/v3/heal/") + and request.get("response") == COORDINATION_REJECTION_BODY, + "coordinator rejection request identity differs from the reviewed Heal request") + require(isinstance(request.get("requestId"), str) and request["requestId"] + and isinstance(request.get("unitFragment"), str) and request["unitFragment"], + "coordinator rejection request lacks its HTTP identity or run unit") + try: + started = datetime.fromisoformat(request["startedAt"]) + finished = datetime.fromisoformat(request["finishedAt"]) + except (KeyError, TypeError, ValueError): + require(False, "coordinator rejection request has no valid client time window") + require(started.tzinfo is not None and finished.tzinfo is not None and started <= finished, + "coordinator rejection request has an invalid client time window") + expected_node = request.get("serviceNode") + require(isinstance(expected_node, str) and expected_node in peers, + "coordinator rejection logs omit the request-serving node") + + service = peers[expected_node] + require(isinstance(service, dict) and service.get("node") == expected_node + and isinstance(service.get("unit"), str) + and request["unitFragment"] in service["unit"], + "coordinator rejection log unit does not belong to the active test run") + journal = service.get("journal") + require(isinstance(journal, str), "request-serving node has no service journal") matches = [] - for node, observation in observations.items(): - journal = observation.get("journal") if isinstance(observation, dict) else None - if (isinstance(journal, str) and "local_coordinator_failed" in journal - and "heal control start admission failed" in journal): - matches.append(node) - require(matches, "coordinator rejection is not attributed to the exact admission failure in service logs") - return {"coordinatorFailureLogged": True, "nodesWithFailure": sorted(matches), - "reason": "local_coordinator_failed", "admissionFailure": "heal control start admission failed"} + for line in journal.splitlines(): + offset = line.find("{") + if offset < 0: + continue + try: + event = json.loads(line[offset:]) + raw_timestamp = event.get("timestamp") + if not isinstance(raw_timestamp, str): + continue + timestamp = datetime.fromisoformat(raw_timestamp.replace("Z", "+00:00")) + except (json.JSONDecodeError, KeyError, TypeError, ValueError): + continue + if timestamp.tzinfo is None: + continue + timestamp = timestamp.astimezone(timezone.utc) + # Client and service clocks can differ slightly; the bounded margin is + # still narrow enough to distinguish this short POST from nearby Heal work. + if not started - timedelta(seconds=1) <= timestamp <= finished + timedelta(seconds=1): + continue + if (event.get("level") != "WARN" or event.get("message") != "cluster heal coordination failed" + or event.get("event") != "admin_request_failed" or event.get("component") != "admin_api" + or event.get("subsystem") != "heal_admin" or event.get("operation") != "heal_control" + or event.get("result") != "failed" or event.get("target") != "rustfs::admin::handlers::heal"): + continue + reason, error = event.get("reason"), event.get("error") + if reason == "local_coordinator_failed" and isinstance(error, str) \ + and 'message: "heal control start admission failed"' in error: + matches.append((timestamp, reason, "heal control start admission failed")) + elif reason == "cluster_capability_unavailable" and isinstance(error, str) \ + and "remote rpc client unavailable: can not get heal control client" in error \ + and "transport error" in error: + matches.append((timestamp, reason, "heal control capability preflight failed")) + require(matches, "coordinator rejection is not attributed to this request's exact pre-admission failure in service logs") + require(len(matches) == 1, "coordinator rejection has ambiguous matching service failures in its request window") + timestamp, reason, failure = matches[0] + return {"coordinatorFailureLogged": True, "serviceNode": expected_node, + "serviceUnit": service["unit"], "requestPath": request["path"], + "requestId": request["requestId"], "eventTimestamp": timestamp.isoformat(), + "reason": reason, "preAdmissionFailure": failure, + "correlation": "request-serving node + active run unit + bounded client request window + exact structured RustFS event"} def files(snapshot): @@ -401,7 +462,17 @@ def run_task(probe, fleet, target, mode, active, *, status_endpoints=(), option_ started = fleet.record("trigger", probe.admin(path, [], json.dumps( {**BASE_OPTIONS, "scanMode": mode, **(option_overrides or {})}).encode())) if allow_coordination_rejection and started.get("httpStatus") != 200: - return validate_coordination_rejection(started, path=path) + rejection = validate_coordination_rejection(started, path=path) + endpoint = started.get("endpoint") + match = re.fullmatch(r"node([1-9][0-9]*)", endpoint) if isinstance(endpoint, str) else None + require(match is not None and int(match.group(1)) <= len(fleet.nodes), + "rejected Heal request endpoint is not bound to a test service node") + request_id = (started.get("responseHeaders") or {}).get("x-request-id") + request_window = {name: started.get(name) for name in + ("method", "httpStatus", "path", "startedAt", "finishedAt", "response")} + request_window.update(requestId=request_id, endpoint=endpoint, + serviceNode=fleet.nodes[int(match.group(1)) - 1], unitFragment=fleet.run_id) + return {**rejection, "requestWindow": request_window} require(started.get("httpStatus") == 200 and isinstance(started.get("response"), dict), "repair admission failed") token = started["response"].get("clientToken") require(isinstance(token, str) and token, "repair admission omitted token") diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 04cf78db..784a376b 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -995,7 +995,8 @@ def run_topology(args, topology, nodes, fixture): rejection = validate_coordination_rejection(status, path=status.get("path")) logs = {node: fleet.record("coordination-rejection-logs", {"node": node, **fleet.remote(node, "logs")}) for node in fleet.nodes} - rejection["logProof"] = validate_coordination_rejection_logs(logs) + rejection["logProof"] = validate_coordination_rejection_logs( + {"request": status["requestWindow"], "peers": logs}) rejection["physicalUnchanged"] = True gated_proof = rejection else: diff --git a/scripts/validation_heal/write_quorum.py b/scripts/validation_heal/write_quorum.py index 692e379c..331c1506 100644 --- a/scripts/validation_heal/write_quorum.py +++ b/scripts/validation_heal/write_quorum.py @@ -293,7 +293,8 @@ def witness(label): logs = {node: fleet.record("negative-quorum-peer-logs", {"node": node, **fleet.remote(node, "logs")}) for node in nodes} if coordination_rejected: - result["coordinationRejectionLogs"] = validate_coordination_rejection_logs(logs) + result["coordinationRejectionLogs"] = validate_coordination_rejection_logs( + {"request": status["requestWindow"], "peers": logs}) for node in tuple(fleet.quorum_gates): fleet.record("release-write-quorum-gate", fleet.remote(node, "quorum-gate", mode="clear")) fleet.quorum_gates.remove(node) diff --git a/test_validation_heal.py b/test_validation_heal.py index 312304ae..1ec4ed3a 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -292,7 +292,10 @@ def admin(path, query, body): def test_exact_coordinator_rejection_is_branch_scoped_and_never_admits_a_task(self): body = ('InternalError' 'cluster heal coordination unavailable') - observed = {"httpStatus": 500, "response": body} + observed = {"method": "POST", "httpStatus": 500, "response": body, + "startedAt": "2026-10-23T06:03:32.490+00:00", + "finishedAt": "2026-10-23T06:03:32.520+00:00", "endpoint": "node1", + "responseHeaders": {"x-request-id": "request-123"}} calls, records, active, admissions = [], [], [], [] def admin(path, query, request_body): @@ -300,7 +303,8 @@ def admin(path, query, request_body): return {**observed, "path": path} result = run_task(types.SimpleNamespace(admin=admin), - types.SimpleNamespace(record=lambda label, value: records.append((label, value)) or value), + types.SimpleNamespace(record=lambda label, value: records.append((label, value)) or value, + nodes=["vm000"], run_id="gh-39909050467-1-h1"), {"bucket": "fixture", "key": "objects/0008.bin"}, 2, active, admissions=admissions, allow_coordination_rejection=True) self.assertEqual(result["admissionState"], "coordination-rejected") @@ -309,6 +313,8 @@ def admin(path, query, request_body): self.assertIsNone(result["clientToken"]) self.assertFalse(result["responsibilityReported"]) self.assertEqual(len(calls), 1) + self.assertEqual(result["requestWindow"]["requestId"], "request-123") + self.assertEqual(result["requestWindow"]["serviceNode"], "vm000") self.assertEqual(active, []) self.assertEqual(admissions, []) with self.assertRaisesRegex(ValueError, "repair admission failed"): @@ -329,11 +335,49 @@ def test_coordinator_rejection_requires_exact_status_and_xml(self): with self.subTest(changed=changed): with self.assertRaisesRegex(ValueError, "reviewed coordinator failure"): validate_coordination_rejection(changed) - logs = {"node1": {"journal": ('{"reason":"local_coordinator_failed",' - '"error":"heal control start admission failed"}')}} - self.assertTrue(validate_coordination_rejection_logs(logs)["coordinatorFailureLogged"]) - with self.assertRaisesRegex(ValueError, "not attributed"): - validate_coordination_rejection_logs({"node1": {"journal": "unrelated failure"}}) + request = {"method": "POST", "httpStatus": 500, + "path": "/rustfs/admin/v3/heal/fixture/objects/0008.bin", + "startedAt": "2026-10-23T06:03:32.490+00:00", + "finishedAt": "2026-10-23T06:03:32.520+00:00", "response": exact["response"], + "requestId": "request-123", "endpoint": "node1", "serviceNode": "vm000", + "unitFragment": "gh-39909050467-1-h1"} + base_event = {"timestamp": "2026-10-23T06:03:32.511021252Z", "level": "WARN", + "message": "cluster heal coordination failed", "event": "admin_request_failed", + "component": "admin_api", "subsystem": "heal_admin", "operation": "heal_control", + "result": "failed", "reason": "local_coordinator_failed", + "error": 'code: \'Internal error\', message: "heal control start admission failed"', + "target": "rustfs::admin::handlers::heal"} + def logs_for(event, *, node="vm000", unit="rustfs-validation-heal-gh-39909050467-1-h1.service"): + return {"request": request, "peers": {node: {"node": node, "unit": unit, + "journal": "Oct 23 06:03:32 rustfs000000 rustfs[874327]: " + json.dumps(event)}}} + proof = validate_coordination_rejection_logs(logs_for(base_event)) + self.assertTrue(proof["coordinatorFailureLogged"]) + self.assertEqual(proof["preAdmissionFailure"], "heal control start admission failed") + + capability = {**base_event, "reason": "cluster_capability_unavailable", + "error": "remote rpc client unavailable: can not get heal control client, err: transport error"} + request["startedAt"] = "2026-10-23T06:30:38.471+00:00" + request["finishedAt"] = "2026-10-23T06:30:38.493+00:00" + request["unitFragment"] = "gh-39911462644-1-h2" + capability["timestamp"] = "2026-10-23T06:30:38.4969218Z" + self.assertEqual(validate_coordination_rejection_logs(logs_for(capability, + unit="rustfs-validation-heal-gh-39911462644-1-h2.service"))["reason"], + "cluster_capability_unavailable") + + unrelated = {**capability, "reason": "coordinator_request_failed"} + with self.assertRaisesRegex(ValueError, "exact pre-admission failure"): + validate_coordination_rejection_logs(logs_for(unrelated, + unit="rustfs-validation-heal-gh-39911462644-1-h2.service")) + with self.assertRaisesRegex(ValueError, "request-serving node"): + validate_coordination_rejection_logs(logs_for(capability, node="vm001", + unit="rustfs-validation-heal-gh-39911462644-1-h2.service")) + with self.assertRaisesRegex(ValueError, "active test run"): + validate_coordination_rejection_logs(logs_for(capability, + unit="rustfs-validation-heal-gh-39909050467-1-h1.service")) + late = {**capability, "timestamp": "2026-10-23T06:30:41.000Z"} + with self.assertRaisesRegex(ValueError, "exact pre-admission failure"): + validate_coordination_rejection_logs(logs_for(late, + unit="rustfs-validation-heal-gh-39911462644-1-h2.service")) def test_condition_denial_is_not_a_transport_failure_or_success_payload(self): baseline = {"properties": {"ETag": '"etag"'}, "payload": {"size": 3, "sha256": "correct"}} From adad664ce43caa07c33eabbc2559054d87850e0b Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 16:11:07 +0800 Subject: [PATCH 318/343] docs(validation): reconcile B02 and B08 acceptance --- docs/validation-migration-map.md | 54 +++++++++++++++++++------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 413a68be..f93f82fa 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -9,17 +9,18 @@ live run proves each source assertion on the required topology. ## Current acceptance and remaining work -Count reconciliation (2026-10-23): the six additions after the 104-case -baseline are C02, C03, C05, I04, B05 and E04. Their original raw-evidence auditors were -rerun against the retained official archives; all results match the recorded -audits. Candidate/helper identities, complete H1/H2 branch coverage, cleanup, -acceptance review chains and unchanged earlier failures were reconciled to -`canonical-migration-ledger-20261023-e04.json`. Each acceptance retains its own -immutable candidate; this is migration coverage across those candidates, not -a fresh replay of all cases on the current I05 package. Partial I05, native -and recovery diagnostics add no canonical acceptance. - -All 158 canonical IDs are retained. Independent complete live acceptance is **110/158**: the prior 104-case baseline plus the six reconciled cases below. The baseline includes B07 `heal.below-read-quorum` ([run 38727279347/1](https://github.com/rustfs/rustfs/actions/runs/38727279347)), A10 `heal.admin-option-contract` ([run 39197088183/1](https://github.com/rustfs/rustfs/actions/runs/39197088183)) and B06 `heal.read-quorum-boundary` ([run 39223475622/1](https://github.com/rustfs/rustfs/actions/runs/39223475622)). Complete source matrices, immutable candidate/package/tools and cleanup were independently audited; earlier failures remain retained. +Count reconciliation (2026-10-23): the append-only ledger +`canonical-migration-ledger-20261023-b08-39915022106.json` follows the +B01 ledger and attaches the complete B08 live acceptance plus the latest B02 +failure evidence. It preserves every prior result and does not replay all 158 +cases on one candidate. + +All 158 canonical IDs are retained. Independent complete live acceptance is +**113/158**. The current ledger has 21 retained live failures, seven required +native barriers, four orchestration/evidence blockers and 13 environment cases +disabled by default. The active acceptance backlog is 32 cases. Counts are +case-level canonical results across immutable candidates; diagnostic, native, +isolation and partial results do not increment PASS. | Reconciled case | Complete live run | Required coverage | Original result SHA-256 | | --- | --- | --- | --- | @@ -29,16 +30,27 @@ All 158 canonical IDs are retained. Independent complete live acceptance is **11 | I04 `heal.queue-pressure-and-admission` | [39354651483/1](https://github.com/rustfs/rustfs/actions/runs/39354651483) | Four branches per topology, six exact durable tasks, 5,565 archive files | `4dcde2f34bb7df52f03f5a3ec1e87663145a7134d3ef1a513fda2da52d1709a2` | | B05 `heal.multiple-shard-loss` | [39843970314/1](https://github.com/rustfs/rustfs/actions/runs/39843970314) | All 38 DB/DV and H1/H2 instances, 9,059 archive files, ten node cleanups and four package purges | `2624f43fc8581eb1b533a477b1a869f9c72208861d7a268ea35ae8f36153fd7e` | | E04 `heal.scanner-cold-object-repair` | [39863505276/1](https://github.com/rustfs/rustfs/actions/runs/39863505276) | All six H1/H2 instances, 2,307 archive files, five node cleanups and four package purges | `8bfcfb03bfa5cc88649e98c4f9a6fa7a27d64f9fd0cdbe8d851f57fc00bc0702` | +| A05 `heal.admin-overlap-and-force-start` | [39892106641/1](https://github.com/rustfs/rustfs/actions/runs/39892106641) | All eight H1/H2 overlap and force-start branches, 20 cleanup receipts | `4a8073b6fe129a5bf696ffbc74b0024f1b23e66232606df13c7f606e0511b036` | +| B01 `heal.disk-loss-and-repair` | [39899825839/1](https://github.com/rustfs/rustfs/actions/runs/39899825839) | H1/H2 original-drive-return and offline-ACK repair, 272 fixture ACK/request pairs, five cleanups | `fe89fbb0e4e2d93aa5c4408b9218c66d6caa9120f067f3aca3560f4ca360c4e5` | +| B08 `heal.below-write-quorum` | [39915022106/1](https://github.com/rustfs/rustfs/actions/runs/39915022106) | H1/H2 write/lock quorum loss, H1 lock-loss not applicable, 3,463 archive members, ten cleanups | `b5fbc87b2a9e7dab2b07301c8703f0d1733ee44f11ef984b3bce55b65b5df87e` | | Acceptance state | Cases | Remaining requirement | | --- | ---: | --- | -| Independently accepted complete live PASS | 110 | Preserve original provenance and complete results. | -| Retained live failures with existing executors | 24 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | +| Independently accepted complete live PASS | 113 | Preserve original provenance and complete results. | +| Retained live failures with existing executors | 21 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | | Required native barriers | 7 | A09 and G01–G06 need exact candidate-bound server lock/version/commit barriers. | | Orchestration or evidence blocked | 4 | C11, F07, I05 and J02 need their remaining native or full live evidence closure. | | Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | -The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 35 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. +The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 32 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. + +### B08 complete live acceptance + +[B08 run 39915022106/1](https://github.com/rustfs/rustfs/actions/runs/39915022106) is independently accepted on candidate `1a3f038c90704f9ad9693eddcf7d861c9530f13c`, package `39898678364/1`, tools `39914732144/1` and helper `a3d3c62a9f6d29667a1a2b57d13f5cb9e166de3f`. The official 3,463-member archive has SHA-256 `59ddf692cdca495abcd37a1affacf25e4cd3da24c931509bb9b4facc386c2f11`; the independent audit has SHA-256 `63f143d278c5e6a43f0068a31c692cae1064fd2612b54060198c448f52b7e640`. H1 read-present/write-absent and H2 read-present/write-absent plus lock-quorum-absent passed; H1 lock-quorum-absent is explicitly NOT_APPLICABLE under the candidate's in-process local manager. The audit verified all peer inventories, unique PUT absence, physical target restoration, streaming shard proofs and ten cleanup receipts. The prior B08 failure remains retained in the append-only ledger. + +### B02 latest live result remains retained failure + +[B02 run 39916358715/1](https://github.com/rustfs/rustfs/actions/runs/39916358715) used the same immutable candidate package `39898678364/1` and helper `a3d3c62a9f6d29667a1a2b57d13f5cb9e166de3f`. The complete official artifact `12485117739` has SHA-256 `435fe989937adbad376c93506dfe7db7eec914f3fafb68ea98068d8c79ea5d7a`; its case result SHA-256 is `a92ab5c1e312f0e5f25b83f7f8802b1de3e43c5fd93bff05dd3651c9d1e1f978`. Read-only passed on H1 and H2, cleanup passed, and both permission-denied branches failed with `faulted version lacks a repaired responsibility`. The independent artifact audit `B02-live-39916358715-independent-failure-artifact-audit.json` (SHA-256 `8c9b2e03e437542f1d3bd6894cb8dab225e46e0e9f65aecf4b16f4a3db5d27eb`) confirms the follow-up queried the expected `objects/0017.bin` and returned `progress.objectsHealed=1`, while terminal responsibility reports `verified_healthy`, `healed=0` and `unchanged=1`. Physical snapshots show the faulted shard absent after the denied attempt and present with its baseline SHA after the writable follow-up sequence. The runner stops and restarts RustFS after releasing the gate and before that follow-up; there is no physical snapshot between service startup and the new Admin token, so the artifact cannot attribute the restoration to that token or establish a terminal receipt classification bug. The strict source assertion remains unsatisfied; B02 stays `RETAINED_LIVE_FAILURE` pending an attribution-safe replay. A05 scoped live [39875125045/1](https://github.com/rustfs/rustfs/actions/runs/39875125045) was interrupted by the self-hosted runner's exhausted filesystem. The official @@ -49,12 +61,12 @@ contains no files. Neither product failure nor clean finalization can be concluded from that interruption. Its fixed candidate `c95bb396`, package `39867606969/1` and matching tools `39874908888/1` retain their independent byte/provenance audits. The J01 chain stopped before dispatch when A05 evidence -collection refused the empty artifact listing. Its released-client fix and -the B01 explicit disk-error correction have local coverage; both still need -complete fresh live acceptance. Preserve the interrupted runner's original -temporary evidence and all current fleet namespaces before another job or -recovery action. Resume fleet execution only after runner space, original -evidence and owned cleanup/isolation have been independently reconciled. +collection refused the empty artifact listing. Its released-client fix remains pending a fresh live acceptance; the B01 +explicit disk-error correction is covered by the accepted B01 run above. The administrator has since deleted the interrupted runner temporary evidence; +the user explicitly accepts retaining this attempt as unaccepted. A05 later +passed on a separate complete run 39892106641; that pass does not reconstruct +or change the status of run 39875125045. No product verdict or cleanup PASS is +inferred for the deleted scene. B05 acceptance uses candidate `7ed6c9320db491c84b54e8232152d7b1282c1b3f`, helper `76dbf7029e82c1e3878c9160090804d4afc38078`, package `39827676705/1` and tools `39841369555/1`. The complete official ZIP digest is `4eb3405af1f47583cd7c2d8e5b9ce9fa6910b76665df529d7dfbe4f1399a20d8`. Independent recomputation covers seed ACKs, full versions/markers, the actual 2+2 and 12+4 role/loss matrices, every endpoint's full online membership before each required boundary, sequential faults, one unique START/token per instance, two identical terminal outcome queries, strict integer scope, exact repaired/healthy responsibility, all physical shard and bitrot proofs, complete S3/HEAD/Range evidence before finalization, ten private-node cleanups and four package purges. No backup restoration was used. Original failure `38200772548` remains unchanged; its vm001 offline root cause is not established by the later pass. @@ -1253,7 +1265,7 @@ After gate removal/source return, an independent task must reconstruct the original missing target before any target GET or backup rescue. Streaming verification, all-peer complete S3 manifests and clean finalization remain required. Ten boundary, identity, mount, cleanup and unknown-ACK negatives join -the focused checks. B08 has no live PASS claim yet. +the focused checks. This implementation section predates the complete B08 acceptance above; its earlier no-PASS statement remains historical. A06 run `38090151509` retained FAIL/cleanup PASS on H1/H2 after its repeat-stop assertion. It exposed a harness contract defect: the reviewed manager's From fb3ab3d9764ca53b511d77d95e2b0794c0b7882d Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 16:27:35 +0800 Subject: [PATCH 319/343] test(heal): preserve B02 permission process attribution --- docs/validation-heal-cases.md | 2 ++ scripts/validation_heal/versions.py | 54 ++++++++++++++++++++++++++--- test_validation_heal.py | 33 +++++++++++++++++- 3 files changed, 83 insertions(+), 6 deletions(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index 1f087ca4..bd479d73 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -273,6 +273,8 @@ Topologies: H1,H2. Branches: read-only, permission-denied. 3. 目标不可写期间必须保留失败/待修责任，不能 reported repaired；PHYSICAL 确認没有写到父目录/系统盘。 4. 解除只读或恢复原 ACL，重新修复并 VERIFY；保留故障期间错误和恢复后物理补片证据。 +permission-denied 分支释放 ACL gate 后，保持同一候选进程运行，并用进程 birth、候选摘要及释放后的物理快照证明状态。释放后、提交新的唯一 Admin token 前，目标分片必须仍保持故障态；若自动恢复先改变分片，该分支记为 `NOT_OBSERVED`，不能将恢复归因给新的手工任务。新的修复 token 终态后再停止服务并验证原始分片摘要。read-only 分支按其 unit mount gate 的生命周期清理 gate，再执行独立修复任务。 + Operations: [SEED_DB](validation-heal-operations.md#SEED_DB) → [FAULT](validation-heal-operations.md#FAULT) → [DRIVE_GATE](validation-heal-operations.md#DRIVE_GATE) → [START](validation-heal-operations.md#START) → [WAIT](validation-heal-operations.md#WAIT) → [PHYSICAL](validation-heal-operations.md#PHYSICAL) → [RECOVER](validation-heal-operations.md#RECOVER) → [VERIFY](validation-heal-operations.md#VERIFY) Source assertions: diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 784a376b..3019f0bd 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -493,6 +493,36 @@ def verify_proof(fleet, target, label): return {"allInlineShardsBitrotVerified": True, "originalInlineSha256Restored": True, "shards": proofs} +def release_permission_gate_live(fleet, node, before_release, metadata_faults): + """Restore a B02 permission gate without restarting its candidate process.""" + released = fleet.record("drive-gate-released-live", + fleet.remote(node, "drive-gate-release-live")) + require(released.get("sameOriginalProcess") is True and released.get("serviceRestarted") is False + and isinstance(released.get("processBefore"), dict) + and released.get("processBefore") == released.get("processAfter") + and released["processAfter"].get("pid", 0) > 1, + "B02 permission gate release did not preserve the original candidate process") + fleet.drive_gates.remove(node) + + expected_process = released["processAfter"] + process_before_snapshot = fleet.record("process-after-live-gate-release-before-snapshot", + fleet.remote(node, "process")) + require(process_before_snapshot == expected_process, + "B02 candidate process changed after live permission gate release") + after_release = fleet.snapshot("physical-after-live-gate-release", metadata_faults=metadata_faults) + require(after_release[node].get("pid") == expected_process["pid"] + and after_release[node].get("binarySha256") == expected_process["binarySha256"], + "B02 live physical snapshot is not bound to the released candidate process") + process_after_snapshot = fleet.record("process-after-live-gate-release-after-snapshot", + fleet.remote(node, "process")) + require(process_after_snapshot == expected_process, + "B02 candidate process changed while taking the post-release physical snapshot") + require(physical_content(before_release) == physical_content(after_release), + "ATTRIBUTION_NOT_OBSERVED: target changed after same-process gate release before the fresh manual token") + return {"release": released, "physicalUnchangedUntilFreshToken": True, + "processBoundSnapshot": True, "processIdentity": expected_process} + + def boundary_plan(instance, snapshot, bucket, expected): selected = [entry for entry in expected if entry["key"] == instance["key"]] require(len(selected) == 1 and selected[0]["payload"]["size"] == instance["size"], "boundary lost its ACK size") @@ -986,8 +1016,17 @@ def run_topology(args, topology, nodes, fixture): option_overrides={"recursive": plan["recursive"]}, admissions=admissions, allow_coordination_rejection=(args.case_id == "heal.disk-readonly-and-permission" and branch == "permission-denied")) - fleet.stop() - repaired = fleet.snapshot("physical-after-" + branch, quiescent=True, metadata_faults=metadata_faults) + permission_live_rejoin = (gate_node is not None + and args.case_id == "heal.disk-readonly-and-permission" and branch == "permission-denied") + if permission_live_rejoin: + # Keep the candidate alive: the old stop/clear/start sequence + # allowed a restarted process to restore the part before the + # fresh writable Admin token could be attributed. + repaired = fleet.snapshot("physical-after-" + branch, metadata_faults=metadata_faults) + else: + fleet.stop() + repaired = fleet.snapshot("physical-after-" + branch, quiescent=True, + metadata_faults=metadata_faults) if gate_node is not None: require(physical_content(prior) == physical_content(repaired), "unwritable window changed object files or wrote outside the target") @@ -1002,10 +1041,15 @@ def run_topology(args, topology, nodes, fixture): else: gated_proof = validate_unwritable(status, bucket, plan["expected"][0]) fleet.record("unwritable-responsibility", gated_proof) - fleet.record("drive-gate-released", fleet.remote(gate_node, "drive-gate", mode="clear")) - fleet.drive_gates.remove(gate_node) + if permission_live_rejoin: + release_proof = release_permission_gate_live(fleet, gate_node, repaired, metadata_faults) + fleet.record("same-process-writable-repair-boundary", release_proof) + else: + fleet.record("drive-gate-released", fleet.remote(gate_node, "drive-gate", mode="clear")) + fleet.drive_gates.remove(gate_node) gate_node = None - fleet.start() + if not permission_live_rejoin: + fleet.start() restored_status = run_task(probe, fleet, plan["scope"], 2, active, option_overrides={"recursive": False}, admissions=admissions) if status.get("admissionState") == "coordination-rejected": diff --git a/test_validation_heal.py b/test_validation_heal.py index 1ec4ed3a..4a54d40a 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -26,7 +26,7 @@ map_inline_target, verify_proof) from scripts.validation_heal.versions import seed_semantics, verify_semantics from scripts.validation_heal.versions import multi_loss_instances, multi_loss_plan, SPECIAL_KEYS, verify_pagination, boundary_plan -from scripts.validation_heal.versions import validate_all_members_online, wait_all_members_online +from scripts.validation_heal.versions import validate_all_members_online, wait_all_members_online, release_permission_gate_live from scripts.validation_heal.heal_probe import Probe, validate_s3_request from scripts.validation_heal.prepare_heal_meta_inspector import quorum_contract, size_contract from scripts.validation_heal.boundaries import verify_size_contract, boundary_sizes, verify_boundary_ranges @@ -1168,6 +1168,37 @@ def test_deleted_identity_requires_exact_s3_absence_without_payload_or_transport class HealDriveGateTest(unittest.TestCase): + def test_permission_gate_release_keeps_same_process_before_fresh_repair_token(self): + process = {name: "candidate-" + name for name in ("pid", "startTicks", "bootId", "unit", + "executable", "binarySha256", "mountNamespace", "commandSha256", "environmentFileSha256", "settings")} + process["pid"] = 4242 + before = {"node1": {"files": [{"path": "bucket/objects/0000.bin/part.1", "sha256": "before"}], + "formats": [{"value": {"id": "format"}}]}} + after = copy.deepcopy(before) + after["node1"].update(pid=4242, binarySha256=process["binarySha256"]) + events = [] + + def remote(node, action): + events.append(action) + if action == "drive-gate-release-live": + return {"sameOriginalProcess": True, "serviceRestarted": False, + "processBefore": process, "processAfter": process, + "originalPermissionsRestored": True} + if action == "process": + return process + raise AssertionError(action) + + fleet = types.SimpleNamespace(drive_gates={"node1"}, remote=remote, + record=lambda label, value: events.append(label) or value, + snapshot=lambda label, **kwargs: after) + proof = release_permission_gate_live(fleet, "node1", before, {}) + self.assertEqual(events, ["drive-gate-release-live", "drive-gate-released-live", "process", + "process-after-live-gate-release-before-snapshot", "process", + "process-after-live-gate-release-after-snapshot"]) + self.assertEqual(fleet.drive_gates, set()) + self.assertTrue(proof["physicalUnchangedUntilFreshToken"]) + self.assertTrue(proof["processBoundSnapshot"]) + def test_live_permission_activation_follows_actual_start_and_initial_readiness(self): with tempfile.TemporaryDirectory() as directory: fleet = Fleet(["node1"], Path(directory), "gh-12345-1-h1") From c98028b697f8f9349a2faca277c14810d9b3e466 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 16:29:51 +0800 Subject: [PATCH 320/343] fix(heal): compare B02 process birth fields --- scripts/validation_heal/versions.py | 13 ++++++++++--- test_validation_heal.py | 6 ++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 3019f0bd..5f47c6b0 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -20,6 +20,7 @@ try: from .heal_probe import Probe, compare_inventories, payload, require, save + from .fleet_node import PROCESS_FIELDS from .heal_workload import Recorder, seed, verify_seed from .healthy import Fleet, physical_content, validate_topology from .inspector import verify as verify_inspector @@ -41,6 +42,7 @@ validate_repaired_physical, verify_cohort, verify_ranges) except ImportError: from heal_probe import Probe, compare_inventories, payload, require, save + from fleet_node import PROCESS_FIELDS from heal_workload import Recorder, seed, verify_seed from healthy import Fleet, physical_content, validate_topology from inspector import verify as verify_inspector @@ -499,7 +501,10 @@ def release_permission_gate_live(fleet, node, before_release, metadata_faults): fleet.remote(node, "drive-gate-release-live")) require(released.get("sameOriginalProcess") is True and released.get("serviceRestarted") is False and isinstance(released.get("processBefore"), dict) - and released.get("processBefore") == released.get("processAfter") + and isinstance(released.get("processAfter"), dict) + and all(name in released["processBefore"] and name in released["processAfter"] + and released["processBefore"][name] == released["processAfter"][name] + for name in PROCESS_FIELDS) and released["processAfter"].get("pid", 0) > 1, "B02 permission gate release did not preserve the original candidate process") fleet.drive_gates.remove(node) @@ -507,7 +512,8 @@ def release_permission_gate_live(fleet, node, before_release, metadata_faults): expected_process = released["processAfter"] process_before_snapshot = fleet.record("process-after-live-gate-release-before-snapshot", fleet.remote(node, "process")) - require(process_before_snapshot == expected_process, + require(all(name in process_before_snapshot and process_before_snapshot[name] == expected_process[name] + for name in PROCESS_FIELDS), "B02 candidate process changed after live permission gate release") after_release = fleet.snapshot("physical-after-live-gate-release", metadata_faults=metadata_faults) require(after_release[node].get("pid") == expected_process["pid"] @@ -515,7 +521,8 @@ def release_permission_gate_live(fleet, node, before_release, metadata_faults): "B02 live physical snapshot is not bound to the released candidate process") process_after_snapshot = fleet.record("process-after-live-gate-release-after-snapshot", fleet.remote(node, "process")) - require(process_after_snapshot == expected_process, + require(all(name in process_after_snapshot and process_after_snapshot[name] == expected_process[name] + for name in PROCESS_FIELDS), "B02 candidate process changed while taking the post-release physical snapshot") require(physical_content(before_release) == physical_content(after_release), "ATTRIBUTION_NOT_OBSERVED: target changed after same-process gate release before the fresh manual token") diff --git a/test_validation_heal.py b/test_validation_heal.py index 4a54d40a..7a177e51 100644 --- a/test_validation_heal.py +++ b/test_validation_heal.py @@ -1172,6 +1172,8 @@ def test_permission_gate_release_keeps_same_process_before_fresh_repair_token(se process = {name: "candidate-" + name for name in ("pid", "startTicks", "bootId", "unit", "executable", "binarySha256", "mountNamespace", "commandSha256", "environmentFileSha256", "settings")} process["pid"] = 4242 + process_before = {**process, "state": "S"} + process_after = {**process, "state": "R"} before = {"node1": {"files": [{"path": "bucket/objects/0000.bin/part.1", "sha256": "before"}], "formats": [{"value": {"id": "format"}}]}} after = copy.deepcopy(before) @@ -1182,10 +1184,10 @@ def remote(node, action): events.append(action) if action == "drive-gate-release-live": return {"sameOriginalProcess": True, "serviceRestarted": False, - "processBefore": process, "processAfter": process, + "processBefore": process_before, "processAfter": process_after, "originalPermissionsRestored": True} if action == "process": - return process + return {**process, "state": "D"} raise AssertionError(action) fleet = types.SimpleNamespace(drive_gates={"node1"}, remote=remote, From dd53393e6ee713f5831254be92a7a90ea40bdf62 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 17:04:07 +0800 Subject: [PATCH 321/343] docs(validation): record complete B02 acceptance --- docs/validation-migration-map.md | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index f93f82fa..7d6352b7 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -12,13 +12,14 @@ live run proves each source assertion on the required topology. Count reconciliation (2026-10-23): the append-only ledger `canonical-migration-ledger-20261023-b08-39915022106.json` follows the B01 ledger and attaches the complete B08 live acceptance plus the latest B02 -failure evidence. It preserves every prior result and does not replay all 158 -cases on one candidate. +failure evidence. The B02 complete acceptance below is a later independent +record; it does not replace that retained failure or replay all 158 cases on +one candidate. All 158 canonical IDs are retained. Independent complete live acceptance is -**113/158**. The current ledger has 21 retained live failures, seven required +**114/158**. The current ledger has 20 retained live failures, seven required native barriers, four orchestration/evidence blockers and 13 environment cases -disabled by default. The active acceptance backlog is 32 cases. Counts are +disabled by default. The active acceptance backlog is 31 cases. Counts are case-level canonical results across immutable candidates; diagnostic, native, isolation and partial results do not increment PASS. @@ -33,24 +34,31 @@ isolation and partial results do not increment PASS. | A05 `heal.admin-overlap-and-force-start` | [39892106641/1](https://github.com/rustfs/rustfs/actions/runs/39892106641) | All eight H1/H2 overlap and force-start branches, 20 cleanup receipts | `4a8073b6fe129a5bf696ffbc74b0024f1b23e66232606df13c7f606e0511b036` | | B01 `heal.disk-loss-and-repair` | [39899825839/1](https://github.com/rustfs/rustfs/actions/runs/39899825839) | H1/H2 original-drive-return and offline-ACK repair, 272 fixture ACK/request pairs, five cleanups | `fe89fbb0e4e2d93aa5c4408b9218c66d6caa9120f067f3aca3560f4ca360c4e5` | | B08 `heal.below-write-quorum` | [39915022106/1](https://github.com/rustfs/rustfs/actions/runs/39915022106) | H1/H2 write/lock quorum loss, H1 lock-loss not applicable, 3,463 archive members, ten cleanups | `b5fbc87b2a9e7dab2b07301c8703f0d1733ee44f11ef984b3bce55b65b5df87e` | +| B02 `heal.disk-readonly-and-permission` | [39927051614/1](https://github.com/rustfs/auto-testing/actions/runs/39927051614) | H1/H2 read-only and permission-denied, same-process gate release, exact shard restoration and five cleanups | `ba2f2569f3799a2aed014c9b89d37a80075774e76dfbfd1b3a06c04c38cb533e` | | Acceptance state | Cases | Remaining requirement | | --- | ---: | --- | -| Independently accepted complete live PASS | 113 | Preserve original provenance and complete results. | -| Retained live failures with existing executors | 21 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | +| Independently accepted complete live PASS | 114 | Preserve original provenance and complete results. | +| Retained live failures with existing executors | 20 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | | Required native barriers | 7 | A09 and G01–G06 need exact candidate-bound server lock/version/commit barriers. | | Orchestration or evidence blocked | 4 | C11, F07, I05 and J02 need their remaining native or full live evidence closure. | | Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | -The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 32 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. +The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 31 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. ### B08 complete live acceptance [B08 run 39915022106/1](https://github.com/rustfs/rustfs/actions/runs/39915022106) is independently accepted on candidate `1a3f038c90704f9ad9693eddcf7d861c9530f13c`, package `39898678364/1`, tools `39914732144/1` and helper `a3d3c62a9f6d29667a1a2b57d13f5cb9e166de3f`. The official 3,463-member archive has SHA-256 `59ddf692cdca495abcd37a1affacf25e4cd3da24c931509bb9b4facc386c2f11`; the independent audit has SHA-256 `63f143d278c5e6a43f0068a31c692cae1064fd2612b54060198c448f52b7e640`. H1 read-present/write-absent and H2 read-present/write-absent plus lock-quorum-absent passed; H1 lock-quorum-absent is explicitly NOT_APPLICABLE under the candidate's in-process local manager. The audit verified all peer inventories, unique PUT absence, physical target restoration, streaming shard proofs and ten cleanup receipts. The prior B08 failure remains retained in the append-only ledger. -### B02 latest live result remains retained failure +### B02 earlier live failure remains retained -[B02 run 39916358715/1](https://github.com/rustfs/rustfs/actions/runs/39916358715) used the same immutable candidate package `39898678364/1` and helper `a3d3c62a9f6d29667a1a2b57d13f5cb9e166de3f`. The complete official artifact `12485117739` has SHA-256 `435fe989937adbad376c93506dfe7db7eec914f3fafb68ea98068d8c79ea5d7a`; its case result SHA-256 is `a92ab5c1e312f0e5f25b83f7f8802b1de3e43c5fd93bff05dd3651c9d1e1f978`. Read-only passed on H1 and H2, cleanup passed, and both permission-denied branches failed with `faulted version lacks a repaired responsibility`. The independent artifact audit `B02-live-39916358715-independent-failure-artifact-audit.json` (SHA-256 `8c9b2e03e437542f1d3bd6894cb8dab225e46e0e9f65aecf4b16f4a3db5d27eb`) confirms the follow-up queried the expected `objects/0017.bin` and returned `progress.objectsHealed=1`, while terminal responsibility reports `verified_healthy`, `healed=0` and `unchanged=1`. Physical snapshots show the faulted shard absent after the denied attempt and present with its baseline SHA after the writable follow-up sequence. The runner stops and restarts RustFS after releasing the gate and before that follow-up; there is no physical snapshot between service startup and the new Admin token, so the artifact cannot attribute the restoration to that token or establish a terminal receipt classification bug. The strict source assertion remains unsatisfied; B02 stays `RETAINED_LIVE_FAILURE` pending an attribution-safe replay. +[B02 run 39916358715/1](https://github.com/rustfs/rustfs/actions/runs/39916358715) used the same immutable candidate package `39898678364/1` and helper `a3d3c62a9f6d29667a1a2b57d13f5cb9e166de3f`. The complete official artifact `12485117739` has SHA-256 `435fe989937adbad376c93506dfe7db7eec914f3fafb68ea98068d8c79ea5d7a`; its case result SHA-256 is `a92ab5c1e312f0e5f25b83f7f8802b1de3e43c5fd93bff05dd3651c9d1e1f978`. Read-only passed on H1 and H2, cleanup passed, and both permission-denied branches failed with `faulted version lacks a repaired responsibility`. The independent artifact audit `B02-live-39916358715-independent-failure-artifact-audit.json` (SHA-256 `8c9b2e03e437542f1d3bd6894cb8dab225e46e0e9f65aecf4b16f4a3db5d27eb`) confirms the follow-up queried the expected `objects/0017.bin` and returned `progress.objectsHealed=1`, while terminal responsibility reports `verified_healthy`, `healed=0` and `unchanged=1`. Physical snapshots show the faulted shard absent after the denied attempt and present with its baseline SHA after the writable follow-up sequence. The runner stops and restarts RustFS after releasing the gate and before that follow-up; there is no physical snapshot between service startup and the new Admin token, so the artifact cannot attribute the restoration to that token or establish a terminal receipt classification bug. The strict source assertion remained unsatisfied in that run; this original failure record remains unchanged. + +### B02 complete live acceptance + +Run [39927051614/1](https://github.com/rustfs/auto-testing/actions/runs/39927051614) independently passed both source assertions across H1/H2 read-only and permission-denied branches. It used candidate commit `1a3f038c90704f9ad9693eddcf7d861c9530f13c`, package `39898678364/1` (SHA-256 `33f97e40add31ebe9bc4e923f68bc11d2e8855b4eedf5aad7bc8fce1f07da9d6`), tools `39925186107/1` and helper commit `c98028b697f8f9349a2faca277c14810d9b3e466`. Official artifact `12490700181` has SHA-256 `ba2f2569f3799a2aed014c9b89d37a80075774e76dfbfd1b3a06c04c38cb533e`; the independent complete audit has SHA-256 `6e7a2c9e21ed3bd031d7189da0c2d8cfd607c56626745cb4a797e0099070771b`. + +For each permission-denied branch, the initial request was rejected before task admission and had no client token. After the gate cleared, candidate process birth identity remained unchanged and the pre-token physical snapshot matched the denied-attempt snapshot. A fresh Admin token then repaired the exact planned shard to its baseline size and SHA-256. The read-only branches also restored their exact planned shards. S3 inventories matched, all five H1/H2 cleanup receipts passed, and both source assertions were present as PASS. This adds one canonical acceptance while retaining the prior failed run. A05 scoped live [39875125045/1](https://github.com/rustfs/rustfs/actions/runs/39875125045) was interrupted by the self-hosted runner's exhausted filesystem. The official From af9da087393ec0db70a86bff298f15151b6c3ae8 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 18:03:06 +0800 Subject: [PATCH 322/343] docs(validation): record J01 and B03 live diagnostics --- docs/validation-migration-map.md | 64 ++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 7d6352b7..d8611326 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -76,6 +76,70 @@ passed on a separate complete run 39892106641; that pass does not reconstruct or change the status of run 39875125045. No product verdict or cleanup PASS is inferred for the deleted scene. +### J01 startup window diagnosis (2026-10-23) + +J01 run [39929436131/1](https://github.com/rustfs/rustfs/actions/runs/39929436131) +used the current auto-testing fleet, candidate `1a3f038c90704f9ad9693eddcf7d861c9530f13c`, +and helper `c98028b697f8f9349a2faca277c14810d9b3e466`. Its official artifact +`12491201856` has SHA-256 +`bd703e00ed88297794a6adb72760ede313f7e9177a30978cac45eb60eddbae5d`. +Enabled-idle, disabled, missing-peer, recovery and all four cleanups passed; +the complete case remains FAIL because H1 and H2 did not expose the required +raw/rc `uninitialized` window. The artifact and original verdict remain intact. + +The H1 startup journal narrows the cause. RustFS began listening at +`09:07:12.377958Z`. The background-heal endpoint returned HTTP 503 with +`server context is not ready` at `.400036Z` and `.411255Z`. Heal runtime +initialization followed at `.438696Z`, and `server_ready` at `.438747Z`. The +first successful signed status response in the collector evidence arrived at +`09:07:12.514298Z` and already reported `idle`. Thus the startup listener +rejects status requests until the heal runtime is initialized; the source +required HTTP 200 `uninitialized` state is not externally observable on this +candidate during that transition. Connection refusal or pre-ready 503 remains +non-evidence. Keep this branch `NOT_OBSERVED` and the canonical J01 result +unaccepted unless a product/source change supplies the required truthful +observable window; do not weaken the source oracle or promote this run. + +### B03 current auto-testing result and mount-admission diagnosis (2026-10-23) + +B03 run [39931810677/1](https://github.com/rustfs/rustfs/actions/runs/39931810677) +used candidate `1a3f038c90704f9ad9693eddcf7d861c9530f13c`, package +`39898678364/1` (SHA-256 +`33f97e40add31ebe9bc4e923f68bc11d2e8855b4eedf5aad7bc8fce1f07da9d6`), +and auto-testing commit `dd53393e6ee713f5831254be92a7a90ea40bdf62`. +Official artifact `12491823352` has SHA-256 +`df3770cf8f2b9a8ba22e6cc2e5064f96c030121e5bfe105f5de2a894adacad01`. +All four H1/H2 × DB/DV instances retained FAIL and cleanup PASS. Their +registered 90-second format observations never found the replacement +`format.json`; candidate journals repeatedly report +`Replacement target is not ready: target mount admission failed`. + +The runner gives RustFS endpoints under +`/data/rustfsN/heal-validation//data`. The parent `/data/rustfsN` +mounts are independent devices, but the candidate's replacement admission +requires a valid mount lease for the exact target endpoint and each local +sibling endpoint, plus target physical-device IDs disjoint from `/` and those +siblings. The run-private endpoint directories do not meet that exact-mount +contract. The service logs show the actual sequence: `renew_disk` attached the +empty endpoint and submitted an automatic replacement request; the task then +started and repeatedly failed with `Replacement target is not ready: target +mount admission failed`. `RUSTFS_HEAL_AUTO_HEAL_ENABLE=false` only disables the +periodic scanner; it does not suppress this replacement request generated by +`renew_disk`. The executor's `wait_format()` therefore expired after 90 seconds +and the explicit manual `run_task()` call was never reached. The source +procedure and strict UUID/set-slot and physical-byte oracles remain unchanged; +do not lengthen the wait, copy from quarantine, or infer a product verdict from +this environment mismatch. + +Before another canonical attempt, the auto-testing adapter must demonstrate a +candidate-supported way to initialize the blank replacement before the +manual token, and expose every local endpoint through mount leases that pass +the candidate's exact mount and sibling-device checks. Record mount IDs and +device identities before and after replacement, keep the old archive unreadable +to the service, then rerun the full H1/H2 DB/DV matrix. If the current fleet +cannot supply those mounts, retain this failure and record the precise resource +blocker instead of emulating mount admission. + B05 acceptance uses candidate `7ed6c9320db491c84b54e8232152d7b1282c1b3f`, helper `76dbf7029e82c1e3878c9160090804d4afc38078`, package `39827676705/1` and tools `39841369555/1`. The complete official ZIP digest is `4eb3405af1f47583cd7c2d8e5b9ce9fa6910b76665df529d7dfbe4f1399a20d8`. Independent recomputation covers seed ACKs, full versions/markers, the actual 2+2 and 12+4 role/loss matrices, every endpoint's full online membership before each required boundary, sequential faults, one unique START/token per instance, two identical terminal outcome queries, strict integer scope, exact repaired/healthy responsibility, all physical shard and bitrot proofs, complete S3/HEAD/Range evidence before finalization, ten private-node cleanups and four package purges. No backup restoration was used. Original failure `38200772548` remains unchanged; its vm001 offline root cause is not established by the later pass. E04 live [39842352595/1](https://github.com/rustfs/rustfs/actions/runs/39842352595) retains FAIL/cleanup PASS. All 1,402 official files, three H1 branches and five node cleanups were independently verified. H2's healthy Scanner task completed on vm002 while the exact node3 token query was forwarded to vm000 and returned notFound. RustFS commit `553be4b4c62ae0c590e82f91817bc018bdfdc660` lets authenticated original local automatic-task tokens return the source/path-bound in-memory snapshot; Admin/Internal/unknown/alias identities retain coordinator routing. Fixed candidate `9d90884c304dcfbf97057c263c6f2ee110230ce7` adds only that reviewed patch to `7ed6c932`, with unchanged Cargo.lock. Package [39851206784/1](https://github.com/rustfs/rustfs/actions/runs/39851206784) and matching tools [39860218641/1](https://github.com/rustfs/rustfs/actions/runs/39860218641) passed independent complete provenance and byte audits. From 3c867aa22fedc80ab475e5d1517c01dfaca25d62 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 18:17:52 +0800 Subject: [PATCH 323/343] docs(validation): clarify B03 mount adapter gate --- docs/validation-heal-operations.md | 4 ++-- docs/validation-migration-map.md | 33 ++++++++++++++++++++++-------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 08b45419..f1d68049 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -2,7 +2,7 @@ 此手册和 [84 项逐例操作卡](validation-heal-cases.md) 一起构成可执行的**人工操作方案 + 脚本辅助**。不是 84 个无人值守自动测试，也不是通过报告。操作卡列出 302 个分支模板；真实盘角色、阈值、损坏数量和拓扑仍需在每批 `parameterBindings` 展开，不能把模板数量当实测次数。 -公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 已接入 60 个 Heal 用例，完整清单通过 `matrix.py list` 查看，以 `automaticExecutor` 为准。入口使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 +公共验收契约见 [Heal 规范](validation-heal-contract.md)。源仓库历史 pilot 结果不计入当前 canonical 分母。自动入口 `scripts/run_validation_case.sh CASE_ID CANDIDATE_MANIFEST TESTING_COMMIT ARTIFACTS_DIR` 已接入 67 个 Heal 用例，完整清单通过 `matrix.py list` 查看，以 `automaticExecutor` 为准。入口使用当前 Heal 节点、候选 deb 和独立 H1/H2 服务；其余操作卡仍需按本手册操作留证。 ## 1. 使用入口与交付物 @@ -54,7 +54,7 @@ python scripts/validation_heal/heal_probe.py --binding lab.json --out reports/bi ### 独立实验服务的部署 -H1 使用当前 Heal 节点经核验的四个独立 mount，**另建** `/data/rustfsN/heal-validation//data` 和同级 `quarantine`，root-owned 0700；每个 `` 根写入包含 `runId` 的 root-owned `owner.json`。先检查每个根不存在；已存在的根不能覆盖。复用 `rustfs_heal_test.sh` 的 candidate deb 下载、SHA 校验和安装流程，把已安装且摘要已核验的 `/usr/bin/rustfs` 放在本批 state 下；执行用例不需要 RustFS 源码 checkout。凭据文件为 0600。以下专用 unit 仍运行在当前 Heal 节点，使用已分配的 9000 测试端口： +H1 使用当前 Heal 节点经核验的四个独立 mount，**另建** `/data/rustfsN/heal-validation//data` 和同级 `quarantine`，root-owned 0700；每个 `` 根写入包含 `runId` 的 root-owned `owner.json`。普通对象故障可以使用这些独立父盘下的 run 私有目录。B03 还要求替换 target 和同组兄弟 endpoint 都能取得 candidate 认可的精确 mount lease；替换过程中必须证明旧 mount 与 quarantine 对服务不可读、新空设备挂到原 endpoint 后 candidate 取得新的 mount lease，并前后记录 mount ID、物理设备 ID 和 filesystem identity。独立文件系统或隔离 mount namespace 中的 bind mount 都可能作为实现方式，但仅创建 bind mount 不足以满足要求：adapter 还必须安全切换被替换 endpoint，且通过 candidate 对 root 和 sibling 物理设备身份的检查。当前 runner 只把 run 私有子目录作为 endpoint，尚无此 mount-switch adapter，因此它现有的四个独立父盘不能证明 B03 满足资源条件。缺少已验证 adapter 时应保留 FAIL/BLOCKED 证据，不延长等待时间或复制 quarantine 数据。先检查每个根不存在；已存在的根不能覆盖。复用 `rustfs_heal_test.sh` 的 candidate deb 下载、SHA 校验和安装流程，把已安装且摘要已核验的 `/usr/bin/rustfs` 放在本批 state 下；执行用例不需要 RustFS 源码 checkout。凭据文件为 0600。以下专用 unit 仍运行在当前 Heal 节点，使用已分配的 9000 测试端口： ```bash # 在目标节点；所有变量来自已审核的 lab binding，不接受任意在线输入。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index d8611326..fa022cec 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -114,6 +114,18 @@ registered 90-second format observations never found the replacement `format.json`; candidate journals repeatedly report `Replacement target is not ready: target mount admission failed`. +I re-downloaded the official artifact and checked its runner storage preflight, +H1 deployment preflight, replacement receipt, post-start format observations, +and candidate journal. The runner had about 258 GB free at preflight, and the +four parent mounts were four distinct NVMe devices; this rules out the old +disk-space concern for this run. The deployment record lists the actual +endpoint roots under each `/data/rustfsN` mount, but does not establish a mount +lease for those run-private endpoint paths. The service journal confirms that +`renew_disk` attached the empty endpoint and queued the replacement request, +which then failed admission. Thus the missing capability is endpoint mount +binding and safe mount switching, rather than capacity or missing replacement +bytes. + The runner gives RustFS endpoints under `/data/rustfsN/heal-validation//data`. The parent `/data/rustfsN` mounts are independent devices, but the candidate's replacement admission @@ -131,14 +143,19 @@ procedure and strict UUID/set-slot and physical-byte oracles remain unchanged; do not lengthen the wait, copy from quarantine, or infer a product verdict from this environment mismatch. -Before another canonical attempt, the auto-testing adapter must demonstrate a -candidate-supported way to initialize the blank replacement before the -manual token, and expose every local endpoint through mount leases that pass -the candidate's exact mount and sibling-device checks. Record mount IDs and -device identities before and after replacement, keep the old archive unreadable -to the service, then rerun the full H1/H2 DB/DV matrix. If the current fleet -cannot supply those mounts, retain this failure and record the precise resource -blocker instead of emulating mount admission. +Before another canonical attempt, add a read-only preflight that checks every +local endpoint's exact mount lease, mount ID, filesystem identity, and physical +device separation from `/` and its local siblings. The replacement adapter +must then demonstrate a candidate-supported way to switch the target endpoint +to a blank mount, let the candidate obtain a new lease and initialize it before +the manual token, while keeping the old archive unreadable to the service. A +bind mount may be used only if the adapter proves this full lease switch and +all candidate device-identity checks; a bind mount by itself is not evidence. +Run the admission preflight first and stop before seeding if it fails. Once it +passes, record mount identities before and after replacement and rerun the +full H1/H2 DB/DV matrix. If the current fleet cannot supply the mount-switch +adapter, retain this failure and report B03 as environment-blocked instead of +emulating mount admission. B05 acceptance uses candidate `7ed6c9320db491c84b54e8232152d7b1282c1b3f`, helper `76dbf7029e82c1e3878c9160090804d4afc38078`, package `39827676705/1` and tools `39841369555/1`. The complete official ZIP digest is `4eb3405af1f47583cd7c2d8e5b9ce9fa6910b76665df529d7dfbe4f1399a20d8`. Independent recomputation covers seed ACKs, full versions/markers, the actual 2+2 and 12+4 role/loss matrices, every endpoint's full online membership before each required boundary, sequential faults, one unique START/token per instance, two identical terminal outcome queries, strict integer scope, exact repaired/healthy responsibility, all physical shard and bitrot proofs, complete S3/HEAD/Range evidence before finalization, ten private-node cleanups and four package purges. No backup restoration was used. Original failure `38200772548` remains unchanged; its vm001 offline root cause is not established by the later pass. From 60f660627baf589705863f301d44bbd246455dd7 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 18:35:24 +0800 Subject: [PATCH 324/343] test(heal): fail fast on B03 mount admission --- docs/validation-heal-operations.md | 2 +- docs/validation-migration-map.md | 16 +++- scripts/validation_heal/fleet_node.py | 131 ++++++++++++++++++++++++++ scripts/validation_heal/healthy.py | 19 +++- scripts/validation_heal/run_case.sh | 5 + scripts/validation_heal/versions.py | 25 ++++- test_validation_heal_environment.py | 38 +++++++- 7 files changed, 224 insertions(+), 12 deletions(-) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index f1d68049..11875e9d 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -54,7 +54,7 @@ python scripts/validation_heal/heal_probe.py --binding lab.json --out reports/bi ### 独立实验服务的部署 -H1 使用当前 Heal 节点经核验的四个独立 mount，**另建** `/data/rustfsN/heal-validation//data` 和同级 `quarantine`，root-owned 0700；每个 `` 根写入包含 `runId` 的 root-owned `owner.json`。普通对象故障可以使用这些独立父盘下的 run 私有目录。B03 还要求替换 target 和同组兄弟 endpoint 都能取得 candidate 认可的精确 mount lease；替换过程中必须证明旧 mount 与 quarantine 对服务不可读、新空设备挂到原 endpoint 后 candidate 取得新的 mount lease，并前后记录 mount ID、物理设备 ID 和 filesystem identity。独立文件系统或隔离 mount namespace 中的 bind mount 都可能作为实现方式，但仅创建 bind mount 不足以满足要求：adapter 还必须安全切换被替换 endpoint，且通过 candidate 对 root 和 sibling 物理设备身份的检查。当前 runner 只把 run 私有子目录作为 endpoint，尚无此 mount-switch adapter，因此它现有的四个独立父盘不能证明 B03 满足资源条件。缺少已验证 adapter 时应保留 FAIL/BLOCKED 证据，不延长等待时间或复制 quarantine 数据。先检查每个根不存在；已存在的根不能覆盖。复用 `rustfs_heal_test.sh` 的 candidate deb 下载、SHA 校验和安装流程，把已安装且摘要已核验的 `/usr/bin/rustfs` 放在本批 state 下；执行用例不需要 RustFS 源码 checkout。凭据文件为 0600。以下专用 unit 仍运行在当前 Heal 节点，使用已分配的 9000 测试端口： +H1 使用当前 Heal 节点经核验的四个独立 mount，**另建** `/data/rustfsN/heal-validation//data` 和同级 `quarantine`，root-owned 0700；每个 `` 根写入包含 `runId` 的 root-owned `owner.json`。普通对象故障可以使用这些独立父盘下的 run 私有目录。B03 还要求替换 target 和同组兄弟 endpoint 都能取得 candidate 认可的精确 mount lease；替换过程中必须证明旧 mount 与 quarantine 对服务不可读、新空设备挂到原 endpoint 后 candidate 取得新的 mount lease，并前后记录 mount ID、物理设备 ID 和 filesystem identity。独立文件系统或隔离 mount namespace 中的 bind mount 都可能作为实现方式，但仅创建 bind mount 不足以满足要求：adapter 还必须安全切换被替换 endpoint，且通过 candidate 对 root 和 sibling 物理设备身份的检查。B03 的 `replacement-admission-preflight` 在普通盘预检和 run 目录创建前，以只读方式检查四个预期 endpoint、mount ID、filesystem identity、物理设备以及 root/sibling 隔离；缺少真实 endpoint mount 或已审核 mount-switch adapter 时应 BLOCKED 并在服务启动/seed 前结束。当前 runner 只把 run 私有子目录作为 endpoint，尚无此 mount-switch adapter，因此它现有的四个独立父盘不能证明 B03 满足资源条件。不得延长等待时间或复制 quarantine 数据。先检查每个根不存在；已存在的根不能覆盖。复用 `rustfs_heal_test.sh` 的 candidate deb 下载、SHA 校验和安装流程，把已安装且摘要已核验的 `/usr/bin/rustfs` 放在本批 state 下；执行用例不需要 RustFS 源码 checkout。凭据文件为 0600。以下专用 unit 仍运行在当前 Heal 节点，使用已分配的 9000 测试端口： ```bash # 在目标节点；所有变量来自已审核的 lab binding，不接受任意在线输入。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index fa022cec..f9767cad 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -151,11 +151,17 @@ to a blank mount, let the candidate obtain a new lease and initialize it before the manual token, while keeping the old archive unreadable to the service. A bind mount may be used only if the adapter proves this full lease switch and all candidate device-identity checks; a bind mount by itself is not evidence. -Run the admission preflight first and stop before seeding if it fails. Once it -passes, record mount identities before and after replacement and rerun the -full H1/H2 DB/DV matrix. If the current fleet cannot supply the mount-switch -adapter, retain this failure and report B03 as environment-blocked instead of -emulating mount admission. +The executor now runs a read-only `replacement-admission-preflight` before +the normal parent-mount preflight, package-backed reservation, service start, +or fixture seed. It records each endpoint's exact mountpoint, mount ID, +filesystem identity, nearest existing ancestor, physical-device IDs and +root/sibling overlap; it also records whether a reviewed mount-switch adapter +is registered. The current runner reports ordinary run directories and no +adapter, so B03 is stopped as `BLOCKED` with cleanup evidence before any +90-second format wait. Once an adapter is installed and independently +reviewed, the gate must pass before recording mount identities before and +after replacement and rerunning the full H1/H2 DB/DV matrix. The source oracle +and prior FAIL evidence remain unchanged. B05 acceptance uses candidate `7ed6c9320db491c84b54e8232152d7b1282c1b3f`, helper `76dbf7029e82c1e3878c9160090804d4afc38078`, package `39827676705/1` and tools `39841369555/1`. The complete official ZIP digest is `4eb3405af1f47583cd7c2d8e5b9ce9fa6910b76665df529d7dfbe4f1399a20d8`. Independent recomputation covers seed ACKs, full versions/markers, the actual 2+2 and 12+4 role/loss matrices, every endpoint's full online membership before each required boundary, sequential faults, one unique START/token per instance, two identical terminal outcome queries, strict integer scope, exact repaired/healthy responsibility, all physical shard and bitrot proofs, complete S3/HEAD/Range evidence before finalization, ten private-node cleanups and four package purges. No backup restoration was used. Original failure `38200772548` remains unchanged; its vm001 offline root cause is not established by the later pass. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index 1e74416e..c39bbcb7 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -334,6 +334,136 @@ def mounts(): return observed +def _unescape_mountinfo(value): + return (value.replace("\\040", " ").replace("\\011", "\t") + .replace("\\012", "\n").replace("\\134", "\\")) + + +def _mountinfo_entries(): + entries = [] + for line in Path("/proc/self/mountinfo").read_text().splitlines(): + fields = line.split() + if len(fields) < 10 or "-" not in fields: + continue + separator = fields.index("-") + if separator + 2 >= len(fields): + continue + entries.append({ + "mountId": fields[0], "device": fields[2], "root": _unescape_mountinfo(fields[3]), + "target": _unescape_mountinfo(fields[4]), "options": fields[5], + "fstype": fields[separator + 1], "source": _unescape_mountinfo(fields[separator + 2]), + }) + return entries + + +def _physical_device_ids(path): + """Resolve leaf block identities in the same failure-domain shape as RustFS.""" + info = path.stat() + device = f"{os.major(info.st_dev)}:{os.minor(info.st_dev)}" + sysfs = Path("/sys/dev/block") / device + if not sysfs.exists(): + return [device] + + def leaves(node): + slaves = node / "slaves" + entries = sorted(slaves.iterdir()) if slaves.is_dir() else [] + if entries: + result = [] + for entry in entries: + result.extend(leaves(entry.resolve(strict=True))) + return result + if (node / "partition").exists() and node.parent.name: + return [node.parent.name] + return [node.name] + + return sorted(set(leaves(sysfs.resolve(strict=True)))) + + +def _replacement_mount_identity(path): + """Inspect one endpoint without creating, remounting or switching anything.""" + require(path.is_absolute() and not any(part == ".." for part in path.parts), + "replacement endpoint path is invalid") + mount_root = next((mount for mount in MOUNTS if path.is_relative_to(mount)), None) + if mount_root is not None: + for entry in (path, *path.parents): + require(not entry.is_symlink(), "symlink in replacement endpoint path") + if entry == mount_root: + break + probe = path + while not probe.exists() and probe != probe.parent: + probe = probe.parent + require(probe.exists() and not probe.is_symlink(), "replacement endpoint has no safe existing ancestor") + canonical_probe = probe.resolve(strict=True) + canonical = path.resolve(strict=False) + info = canonical_probe.lstat() + require(stat.S_ISDIR(info.st_mode) and not stat.S_ISLNK(info.st_mode), + "replacement endpoint or nearest existing ancestor must be a canonical directory") + entries = _mountinfo_entries() + exact = [item for item in entries if item["target"] == str(canonical)] if path.exists() else [] + mount = exact[-1] if len(exact) == 1 else None + mount_id = mount["mountId"] if mount else None + ancestors = [item for item in entries if canonical_probe == Path(item["target"]) + or Path(item["target"]) in canonical_probe.parents] + nearest_mount = max(ancestors, key=lambda item: len(Path(item["target"]).parts), default=None) + physical = _physical_device_ids(canonical_probe) + return { + "endpoint": str(path), + "canonicalPath": str(canonical), + "endpointExists": path.exists(), + "mounted": mount is not None and mount_id is not None, + "mount": mount, + "mountId": mount_id, + "filesystemIdentity": None if mount_id is None else f"{mount_id}:{info.st_dev}:{info.st_ino}", + "device": f"{os.major(info.st_dev)}:{os.minor(info.st_dev)}", + "physicalDeviceIds": physical, + "nearestExistingAncestor": str(canonical_probe), + "ancestorMount": nearest_mount, + "writable": os.access(canonical_probe, os.W_OK), + } + + +def replacement_admission_preflight(request): + """Read-only B03 admission proof; fail closed before service start or seed.""" + require(set(request) == {"runId", "action"}, "replacement admission accepts no paths or mutation options") + _, roots = paths(request["runId"]) + endpoints = [_replacement_mount_identity(root / "data") for root in roots] + root_identity = _replacement_mount_identity(Path("/")) + root_devices = set(root_identity["physicalDeviceIds"]) + endpoint_devices = [set(item["physicalDeviceIds"]) for item in endpoints] + sibling_overlap = [sorted(left & right) for index, left in enumerate(endpoint_devices) + for right in endpoint_devices[index + 1:] if left & right] + root_overlap = [sorted(devices & root_devices) for devices in endpoint_devices if devices & root_devices] + exact_mounts = all(item["mounted"] for item in endpoints) + distinct_filesystems = len({item["filesystemIdentity"] for item in endpoints}) == len(endpoints) + adapter = { + "registered": False, + "reason": "auto-testing reserves ordinary run directories; no reviewed endpoint mount-switch adapter is installed", + } + result = { + "runId": request["runId"], + "readOnly": True, + "mutated": False, + "canonicalPassClaimed": False, + "endpoints": endpoints, + "root": root_identity, + "mountSwitchAdapter": adapter, + "checks": { + "exactEndpointMounts": exact_mounts, + "mountIdsPresent": all(item["mountId"] is not None for item in endpoints), + "filesystemIdentitiesDistinct": distinct_filesystems, + "endpointPhysicalDevicesDistinct": not sibling_overlap, + "endpointDevicesSeparateFromRoot": not root_overlap, + "mountSwitchAdapterReviewed": False, + }, + "admitted": bool(exact_mounts and distinct_filesystems and not sibling_overlap + and not root_overlap and adapter["registered"] is True), + "reason": None, + } + if not result["admitted"]: + result["reason"] = "B03 requires exact endpoint mount leases and a reviewed mount-switch adapter; run endpoints are ordinary directories" + return result + + def environment_service_state(): """Observe fixed service/port names; never stop or certify a clean scene.""" fields = {'LoadState', 'ActiveState', 'SubState', 'MainPID'} @@ -1984,6 +2114,7 @@ def main(request): actions["version-race-backups"] = lambda value: globals()["mpu_race_backups_backend"].run(value, globals()) actions["process-clock"] = process_clock actions["environment-capabilities"] = environment_capabilities + actions["replacement-admission-preflight"] = replacement_admission_preflight for action in ("retained-scene-audit", "retained-scene-isolate"): actions[action] = retained_scene_operation for action in ("owner-markers-audit", "owner-markers-recover"): diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 1050b921..43d6fd80 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -36,6 +36,14 @@ "dryRun": False, "updateParity": False, "nolock": False, "readRepair": False, "pool": 0, "set": 0} +class ReplacementAdmissionBlocked(ValueError): + """B03 cannot start until the actual endpoint mounts pass candidate admission.""" + + def __init__(self, evidence): + self.evidence = evidence + super().__init__("B03 replacement mount admission preflight failed; deployment and fixture seed were skipped") + + def validate_rejection(observed, expected_code): require(observed.get("httpStatus") == 400, "invalid Heal request was not rejected with HTTP 400") body = observed.get("response") @@ -136,12 +144,14 @@ def validate_healthy(status, bucket, key, disposition): class Fleet: - def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard"): + def __init__(self, nodes, evidence, run_id, tool_directory=None, *, runtime_profile="manual", resource_profile="standard", + require_replacement_admission=False): require(runtime_profile in {"lock-timeout", "lock-successor", "frozen-race-legacy", "outcome-legacy", "outcome-grace", "queue-pressure", "orphan-preserve", "orphan-before", "orphan-after", "manual", "authority-split", "multi-pool", "bounded-capacity", "journal-bounded", "lock-quorum", "offline-replay", "durable-mrf", "mrf-crash", "flapping", "read-repair", "scanner", "disabled", "overlap", "root-restart", "stale-rejoin", "replacement", "schema-refusal", "scope-isolation", "incarnation-checkpoint", "incarnation-mrf"}, "unsupported Heal runtime profile") require(resource_profile in {"standard", "cursor", "full-restart"}, "unsupported Heal resource profile") self.nodes, self.evidence, self.run_id = nodes, evidence, run_id self.tool_directory = tool_directory self.runtime_profile = runtime_profile + self.require_replacement_admission = bool(require_replacement_admission) self.node_runtime_profiles = {} self.resource_profile = resource_profile self.sequence = 0 @@ -294,6 +304,13 @@ def start(self): def deploy(self, binary_sha, bucket, access, secret, *, before_start=None): self.secret_values.update({access, secret}) + if self.require_replacement_admission: + admission = {node: self.remote(node, "replacement-admission-preflight") for node in self.nodes} + self.record("replacement-admission-preflight", admission) + if not all(value.get("readOnly") is True and value.get("mutated") is False + and value.get("canonicalPassClaimed") is False and value.get("admitted") is True + for value in admission.values()): + raise ReplacementAdmissionBlocked(admission) checked = {node: self.remote(node, "preflight", resourceProfile=self.resource_profile, **({'runtimeProfile': self.runtime_profile} if self.runtime_profile in {'lock-timeout', 'lock-successor'} else {})) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index c6974fd0..f53ff8f4 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -225,6 +225,11 @@ if [ "${case_rc}" -eq 0 ] && [ "${cleanup_status}" = PASS ]; then elif [ "${case_rc}" -eq 0 ]; then verdict=INFRA_ERROR; reason="Heal environment finalization failed" fi +if [ "${case_id}" = heal.disk-replacement-formatted ] && [ "${cleanup_status}" = PASS ] && \ + jq -e '.replacementAdmissionBlocked == true and .complete == false and .canonicalPassClaimed != true' \ + "${evidence}/healthy-summary.json" >/dev/null 2>&1; then + verdict=BLOCKED; reason="B03 exact endpoint mount admission and a reviewed mount-switch adapter are unavailable; preflight stopped before deployment and fixture seed" +fi if [ "${case_id}" = heal.bucket-incarnation-recreation ] && [ "${cleanup_status}" = PASS ] && \ jq -e '.diagnosticBranchesPassed == true and .complete == false and .canonicalCasePassClaimed == false' \ "${evidence}/healthy-summary.json" >/dev/null 2>&1; then diff --git a/scripts/validation_heal/versions.py b/scripts/validation_heal/versions.py index 5f47c6b0..d7d37ec5 100644 --- a/scripts/validation_heal/versions.py +++ b/scripts/validation_heal/versions.py @@ -22,7 +22,7 @@ from .heal_probe import Probe, compare_inventories, payload, require, save from .fleet_node import PROCESS_FIELDS from .heal_workload import Recorder, seed, verify_seed - from .healthy import Fleet, physical_content, validate_topology + from .healthy import Fleet, ReplacementAdmissionBlocked, physical_content, validate_topology from .inspector import verify as verify_inspector from .matrix import select from .heal_fault import bounded_fault_id @@ -44,7 +44,7 @@ from heal_probe import Probe, compare_inventories, payload, require, save from fleet_node import PROCESS_FIELDS from heal_workload import Recorder, seed, verify_seed - from healthy import Fleet, physical_content, validate_topology + from healthy import Fleet, ReplacementAdmissionBlocked, physical_content, validate_topology from inspector import verify as verify_inspector from matrix import select from heal_fault import bounded_fault_id @@ -803,7 +803,8 @@ def run_topology(args, topology, nodes, fixture): runtime_profile="scanner" if args.case_id == "heal.scanner-cold-object-repair" else "read-repair" if args.case_id == "heal.read-repair-attribution" else "replacement" if args.case_id == "heal.replacement-auto-rebuild" or target_restart else - "stale-rejoin" if args.case_id == "heal.stale-version-rejoin" else "manual") + "stale-rejoin" if args.case_id == "heal.stale-version-rejoin" else "manual", + require_replacement_admission=args.case_id == "heal.disk-replacement-formatted") if target_restart: fleet.node_runtime_profiles = {node: "replacement-target" for node in nodes[1:]} candidate = json.loads(args.candidate.read_text()) @@ -1187,6 +1188,10 @@ def run_topology(args, topology, nodes, fixture): item["verdict"] == "PASS" for item in result["branches"].values()) else "FAIL" if args.case_id == "heal.versioning-suspended-null" and result["verdict"] == "PASS": result["reenable"] = reenable_and_verify(recorder, fleet, bucket, baseline) + except ReplacementAdmissionBlocked as exc: + result.update(verdict="BLOCKED", failure={"type": type(exc).__name__, "message": str(exc)}, + replacementAdmissionBlocked=True, admissionEvidence=exc.evidence, + deploymentSkipped=True, fixtureSeedSkipped=True) except (Exception, KeyboardInterrupt) as exc: result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) finally: @@ -1238,15 +1243,27 @@ def interrupted(signum, frame): for topology, selected in topologies: for fixture in fixtures: results.append(run_topology(args, topology, selected, fixture)) + if results[-1].get("replacementAdmissionBlocked") is True: + break if results[-1]["cleanup"] != "PASS": break + if results and results[-1].get("replacementAdmissionBlocked") is True: + break if results[-1]["cleanup"] != "PASS": break finally: complete = len(results) == len(topologies) * len(fixtures) and all( item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) + replacement_admission_blocked = (args.case_id == "heal.disk-replacement-formatted" and bool(results) + and all(item.get("replacementAdmissionBlocked") is True and item.get("cleanup") == "PASS" + for item in results)) save(args.evidence / "healthy-summary.json", {"caseId": args.case_id, "results": results, - "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", "complete": complete}) + "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", + "complete": complete, "replacementAdmissionBlocked": replacement_admission_blocked, + "canonicalPassClaimed": False}) + if replacement_admission_blocked: + print("heal.disk-replacement-formatted: BLOCKED before deployment or fixture seed by mount admission preflight") + return require(complete, "version case requires every H1/H2 source branch; inspect topology-result evidence") diff --git a/test_validation_heal_environment.py b/test_validation_heal_environment.py index adb3f035..f05c04fc 100644 --- a/test_validation_heal_environment.py +++ b/test_validation_heal_environment.py @@ -62,15 +62,51 @@ def test_directories_stacked_devices_readonly_mounts_or_low_space_do_not_count(s self.assertFalse(result['resourceShapes']['H1']['mountsAvailable']) def test_current_remote_adapter_includes_a_compilable_readonly_inventory_action(self): + actions = [] def invoke(argv, **params): source = params['input'] compile(source, 'remote-inventory', 'exec') + actions.append(source) self.assertIn('environment_capabilities', source) - self.assertIn('environment-capabilities', source) return types.SimpleNamespace(returncode=0, stdout='{"inspectionOnly":true,"mutated":false}', stderr='') with tempfile.TemporaryDirectory() as name, patch('scripts.validation_heal.healthy.subprocess.run', side_effect=invoke): fleet = Fleet(['n0'], Path(name), 'gh-1234-1-h2') self.assertFalse(fleet.remote('n0', 'environment-capabilities')['mutated']) + self.assertFalse(fleet.remote('n0', 'replacement-admission-preflight')['mutated']) + self.assertIn('replacement_admission_preflight', actions[-1]) + + def test_b03_admission_requires_exact_endpoint_mounts_and_reviewed_switch_adapter(self): + roots = [Path(f'/data/rustfs{index}/heal-validation/gh-1234-1-h1') for index in range(1, 5)] + ordinary = lambda path: { + 'endpoint': str(path), 'canonicalPath': str(path), 'endpointExists': False, + 'mounted': False, 'mount': None, 'mountId': None, 'filesystemIdentity': None, + 'device': '8:1', 'physicalDeviceIds': ['nvme0n1'], + 'nearestExistingAncestor': str(path.parent.parent), 'ancestorMount': None, 'writable': True, + } + root = {**ordinary(Path('/')), 'endpointExists': True, 'mounted': True, 'mountId': '1', + 'filesystemIdentity': '1:0:1', 'physicalDeviceIds': ['nvme-root']} + with patch.object(fleet_node, 'paths', return_value=(Path('/state'), roots)), \ + patch.object(fleet_node, '_replacement_mount_identity', side_effect=lambda path: root if path == Path('/') else ordinary(path)): + result = fleet_node.replacement_admission_preflight({'runId': 'gh-1234-1-h1', 'action': 'replacement-admission-preflight'}) + self.assertFalse(result['admitted']) + self.assertFalse(result['mutated']) + self.assertFalse(result['checks']['exactEndpointMounts']) + self.assertFalse(result['checks']['mountSwitchAdapterReviewed']) + self.assertIn('ordinary directories', result['reason']) + + def test_b03_admission_stops_before_parent_mount_preflight_or_start(self): + with tempfile.TemporaryDirectory() as name: + fleet = Fleet(['n0'], Path(name), 'gh-1234-1-h1', runtime_profile='replacement', + require_replacement_admission=True) + actions = [] + fleet.record = lambda label, value: value + fleet.remote = lambda node, action, **params: actions.append(action) or { + 'readOnly': True, 'mutated': False, 'canonicalPassClaimed': False, 'admitted': False, + } + with self.assertRaisesRegex(ValueError, 'admission preflight failed'): + fleet.deploy('a' * 64, 'rv-heal-' + 'a' * 16, 'access', 'secret') + self.assertEqual(actions, ['replacement-admission-preflight']) + self.assertEqual(fleet.reserved, []) def runtime_command(self, *, units='', listeners='', state=None): state = state or 'LoadState=loaded\nActiveState=active\nSubState=running\nMainPID=42\n' From 49443e7559578cba8be285ed723bb68dfbaf8656 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 18:53:04 +0800 Subject: [PATCH 325/343] test(heal): gate B03 before package bootstrap --- docs/validation-heal-contract.md | 2 + docs/validation-heal-operations.md | 2 +- docs/validation-migration-map.md | 19 ++- .../validation_heal/replacement_preflight.py | 130 ++++++++++++++++++ scripts/validation_heal/run_case.sh | 37 ++++- scripts/validation_result.py | 17 ++- test_validation_heal_environment.py | 68 +++++++++ test_validation_migration.py | 29 ++++ 8 files changed, 291 insertions(+), 13 deletions(-) create mode 100644 scripts/validation_heal/replacement_preflight.py diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index 5ebc447e..4db8e232 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -8,6 +8,8 @@ 目前有 67 个自动执行器和 13 个默认关闭的环境场景入口；C11/F07/I05/J02 尚未接入调度。注册执行器不代表完整 canonical PASS，A09/G01–G06 的必需 native 分支仍被阻断。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。13 个环境场景保留源操作分支、未满足条件和显式启用入口；其完整 live 验证仍需资源适配器与源断言审核。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 +B03 的 runner 先执行只读 mount admission preflight，再允许停止服务、下载/安装候选包或创建远端 run namespace；阻断结果明确记录 `execution_stage=pre-deployment`、候选/rc 来源和 `binary_sha256=null`，表示尚未观察到安装候选二进制。通过预检后，部署阶段仍会在 reservation 前再次读取同一组 mount/设备身份，避免预检与实际变更之间的环境漂移。 + H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 目标不是只证明“坏盘后仍能 GET”，而是分别证明： diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 11875e9d..01d31f0a 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -54,7 +54,7 @@ python scripts/validation_heal/heal_probe.py --binding lab.json --out reports/bi ### 独立实验服务的部署 -H1 使用当前 Heal 节点经核验的四个独立 mount，**另建** `/data/rustfsN/heal-validation//data` 和同级 `quarantine`，root-owned 0700；每个 `` 根写入包含 `runId` 的 root-owned `owner.json`。普通对象故障可以使用这些独立父盘下的 run 私有目录。B03 还要求替换 target 和同组兄弟 endpoint 都能取得 candidate 认可的精确 mount lease；替换过程中必须证明旧 mount 与 quarantine 对服务不可读、新空设备挂到原 endpoint 后 candidate 取得新的 mount lease，并前后记录 mount ID、物理设备 ID 和 filesystem identity。独立文件系统或隔离 mount namespace 中的 bind mount 都可能作为实现方式，但仅创建 bind mount 不足以满足要求：adapter 还必须安全切换被替换 endpoint，且通过 candidate 对 root 和 sibling 物理设备身份的检查。B03 的 `replacement-admission-preflight` 在普通盘预检和 run 目录创建前，以只读方式检查四个预期 endpoint、mount ID、filesystem identity、物理设备以及 root/sibling 隔离；缺少真实 endpoint mount 或已审核 mount-switch adapter 时应 BLOCKED 并在服务启动/seed 前结束。当前 runner 只把 run 私有子目录作为 endpoint，尚无此 mount-switch adapter，因此它现有的四个独立父盘不能证明 B03 满足资源条件。不得延长等待时间或复制 quarantine 数据。先检查每个根不存在；已存在的根不能覆盖。复用 `rustfs_heal_test.sh` 的 candidate deb 下载、SHA 校验和安装流程，把已安装且摘要已核验的 `/usr/bin/rustfs` 放在本批 state 下；执行用例不需要 RustFS 源码 checkout。凭据文件为 0600。以下专用 unit 仍运行在当前 Heal 节点，使用已分配的 9000 测试端口： +H1 使用当前 Heal 节点经核验的四个独立 mount，**另建** `/data/rustfsN/heal-validation//data` 和同级 `quarantine`，root-owned 0700；每个 `` 根写入包含 `runId` 的 root-owned `owner.json`。普通对象故障可以使用这些独立父盘下的 run 私有目录。B03 还要求替换 target 和同组兄弟 endpoint 都能取得 candidate 认可的精确 mount lease；替换过程中必须证明旧 mount 与 quarantine 对服务不可读、新空设备挂到原 endpoint 后 candidate 取得新的 mount lease，并前后记录 mount ID、物理设备 ID 和 filesystem identity。独立文件系统或隔离 mount namespace 中的 bind mount 都可能作为实现方式，但仅创建 bind mount 不足以满足要求：adapter 还必须安全切换被替换 endpoint，且通过 candidate 对 root 和 sibling 物理设备身份的检查。Functional Chain 会先在 `stop_existing` 和候选 deb 下载/安装之前调用只读 `replacement-admission-preflight`，覆盖 H1/H2 endpoints；未通过时以 `BLOCKED` 保留 package/client provenance 和完整现场读数，不停服务、不碰已安装 package，也不创建远端 run namespace。`Fleet.deploy()` 在真正 reservation 前再重复该检查。缺少真实 endpoint mount 或已审核 mount-switch adapter 时应 BLOCKED；当前 runner 只把 run 私有子目录作为 endpoint，尚无此 mount-switch adapter，因此它现有的四个独立父盘不能证明 B03 满足资源条件。不得延长等待时间或复制 quarantine 数据。先检查每个根不存在；已存在的根不能覆盖。复用 `rustfs_heal_test.sh` 的 candidate deb 下载、SHA 校验和安装流程，把已安装且摘要已核验的 `/usr/bin/rustfs` 放在本批 state 下；执行用例不需要 RustFS 源码 checkout。凭据文件为 0600。以下专用 unit 仍运行在当前 Heal 节点，使用已分配的 9000 测试端口： ```bash # 在目标节点；所有变量来自已审核的 lab binding，不接受任意在线输入。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index f9767cad..1794e5be 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -151,14 +151,19 @@ to a blank mount, let the candidate obtain a new lease and initialize it before the manual token, while keeping the old archive unreadable to the service. A bind mount may be used only if the adapter proves this full lease switch and all candidate device-identity checks; a bind mount by itself is not evidence. -The executor now runs a read-only `replacement-admission-preflight` before -the normal parent-mount preflight, package-backed reservation, service start, -or fixture seed. It records each endpoint's exact mountpoint, mount ID, +The Functional Chain now runs a read-only `replacement-admission-preflight` +before `stop_existing`, candidate package download/install, the normal +parent-mount preflight, reservation, service start, or fixture seed. It +inspects all H1/H2 endpoints and records each exact mountpoint, mount ID, filesystem identity, nearest existing ancestor, physical-device IDs and -root/sibling overlap; it also records whether a reviewed mount-switch adapter -is registered. The current runner reports ordinary run directories and no -adapter, so B03 is stopped as `BLOCKED` with cleanup evidence before any -90-second format wait. Once an adapter is installed and independently +root/sibling overlap, along with whether a reviewed mount-switch adapter is +registered. If admission is unavailable, the result is `BLOCKED` with +candidate package and client provenance, `binary_sha256: null`, and an +explicit `NOT_REQUIRED_NO_MUTATION` cleanup disposition; the existing RustFS +package and service remain untouched. If admission passes, `Fleet.deploy()` +repeats the read-only check immediately before reservation. The current runner +reports ordinary run directories and no adapter, so B03 stops before bootstrap +and any 90-second format wait. Once an adapter is installed and independently reviewed, the gate must pass before recording mount identities before and after replacement and rerunning the full H1/H2 DB/DV matrix. The source oracle and prior FAIL evidence remain unchanged. diff --git a/scripts/validation_heal/replacement_preflight.py b/scripts/validation_heal/replacement_preflight.py new file mode 100644 index 00000000..115db90c --- /dev/null +++ b/scripts/validation_heal/replacement_preflight.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Run B03's read-only endpoint admission check before package/service changes.""" + +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +import hashlib +import json +import os +from pathlib import Path +import re +import sys + +try: + from .healthy import Fleet + from .heal_probe import save +except ImportError: + from healthy import Fleet + from heal_probe import save + + +def checked_identity(value: object) -> bool: + return (isinstance(value, dict) and value.get("readOnly") is True + and value.get("mutated") is False and value.get("canonicalPassClaimed") is False + and isinstance(value.get("checks"), dict) + and set(value["checks"]) == { + "exactEndpointMounts", "mountIdsPresent", "filesystemIdentitiesDistinct", + "endpointPhysicalDevicesDistinct", "endpointDevicesSeparateFromRoot", + "mountSwitchAdapterReviewed", + } + and type(value.get("admitted")) is bool) + + +def run_preflight(nodes: list[str], evidence: Path, workflow_run_id: str, attempt: str, + testing_commit: str, candidate: dict) -> dict: + if not re.fullmatch(r"[1-9][0-9]*", workflow_run_id) or not re.fullmatch(r"[1-9][0-9]*", attempt): + raise ValueError("B03 preflight requires the immutable workflow run and attempt") + if not re.fullmatch(r"[0-9a-f]{40}", testing_commit): + raise ValueError("B03 preflight requires the exact auto-testing commit") + if (not re.fullmatch(r"[0-9a-f]{40}", str(candidate.get("source_sha", ""))) + or not re.fullmatch(r"[0-9a-f]{64}", str(candidate.get("package_sha256", ""))) + or not isinstance(candidate.get("package_url"), str) or not candidate["package_url"]): + raise ValueError("B03 preflight candidate package provenance is incomplete") + if len(nodes) != 4 or len(set(nodes)) != 4 or any( + not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes): + raise ValueError("B03 preflight requires the four current auto-testing Heal nodes") + + results = [] + infrastructure_errors = [] + blocked = False + for topology, selected in (("H1", nodes[:1]), ("H2", nodes)): + run_id = f"gh-{workflow_run_id}-{attempt}-{topology.lower()}" + directory = evidence / "replacement-admission" / topology + directory.mkdir(parents=True, exist_ok=True) + fleet = Fleet(selected, directory, run_id) + observed = {} + with ThreadPoolExecutor(max_workers=len(selected)) as executor: + pending = {executor.submit(fleet.remote, node, "replacement-admission-preflight"): node + for node in selected} + for future in as_completed(pending): + node = pending[future] + try: + value = future.result() + if not checked_identity(value): + infrastructure_errors.append({"topology": topology, "node": node, + "reason": "remote preflight response is incomplete or malformed"}) + observed[node] = {"invalidResponse": True} + else: + observed[node] = value + blocked = blocked or value["admitted"] is not True + except Exception as exc: + infrastructure_errors.append({"topology": topology, "node": node, + "reason": f"{type(exc).__name__}: {exc}"}) + observed[node] = {"preflightError": f"{type(exc).__name__}: {exc}"} + ordered = {node: observed[node] for node in selected} + fleet.record("replacement-admission-preflight", {"topology": topology, "runId": run_id, + "readOnly": True, "mutated": False, "nodes": ordered}) + results.append({"topology": topology, "runId": run_id, "nodes": ordered, + "deploymentSkipped": True, "fixtureSeedSkipped": True}) + + infrastructure_errors.sort(key=lambda item: (item["topology"], item["node"], item["reason"])) + status = "INFRA_ERROR" if infrastructure_errors else "BLOCKED" if blocked else "ADMITTED" + summary = { + "schemaVersion": 1, + "caseId": "heal.disk-replacement-formatted", + "status": status, + "executionStage": "pre-deployment", + "workflowRunId": int(workflow_run_id), + "workflowRunAttempt": int(attempt), + "testingCommit": testing_commit, + "candidate": {"sourceSha": candidate["source_sha"], "packageUrl": candidate["package_url"], + "packageSha256": candidate["package_sha256"]}, + "preflightHelperSha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "fleetNodeSha256": hashlib.sha256(Path(__file__).with_name("fleet_node.py").read_bytes()).hexdigest(), + "mutationStarted": False, + "replacementAdmissionBlocked": status == "BLOCKED", + "deploymentSkipped": status != "ADMITTED", + "fixtureSeedSkipped": status != "ADMITTED", + "canonicalPassClaimed": False, + "cleanup": "PASS", + "cleanupDisposition": "NOT_REQUIRED_NO_MUTATION", + "results": results, + "infrastructureErrors": infrastructure_errors, + } + save(evidence / "healthy-summary.json", summary) + return summary + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--testing-commit", required=True) + parser.add_argument("--evidence", type=Path, required=True) + args = parser.parse_args() + try: + nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() + candidate = json.loads(args.candidate.read_text(encoding="utf-8")) + summary = run_preflight(nodes, args.evidence, os.environ.get("GITHUB_RUN_ID", ""), + os.environ.get("GITHUB_RUN_ATTEMPT", "1"), args.testing_commit, candidate) + print(json.dumps({"status": summary["status"], "mutationStarted": False, + "replacementAdmissionBlocked": summary["replacementAdmissionBlocked"]}, sort_keys=True)) + return 0 + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(f"B03 pre-deployment preflight failed: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index f53ff8f4..5dd7bf2a 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -177,7 +177,7 @@ test "${status}:${state}" = "1:"' || failed=1 } bootstrap_rc=0 case_rc=0 cleanup_status=PASS binary_sha="" -package_active=1 +package_active=0 on_exit() { local exit_code=$? trap - EXIT @@ -189,6 +189,41 @@ on_exit() { trap on_exit EXIT trap 'exit 130' INT trap 'exit 143' TERM +if [ "${case_id}" = heal.disk-replacement-formatted ]; then + preflight_rc=0 + python3 -B "${ROOT}/scripts/validation_heal/replacement_preflight.py" \ + --candidate "${candidate}" --testing-commit "${testing_commit}" --evidence "${evidence}" \ + > "${evidence}/predeployment-preflight.log" 2>&1 || preflight_rc=$? + cat "${evidence}/predeployment-preflight.log" + if [ "${preflight_rc}" -ne 0 ]; then + python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ + --result-file "${artifacts}/cases/${case_id}.json" --verdict INFRA_ERROR --cleanup-status PASS \ + --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --rc-bin "${rc_binary}" \ + --evidence "evidence/${case_id}/predeployment-preflight.log" \ + --reason "B03 read-only predeployment admission could not complete; package and service state were left untouched" + exit 1 + fi + preflight_status="$(jq -er '.status' "${evidence}/healthy-summary.json")" + if [ "${preflight_status}" = BLOCKED ]; then + python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ + --result-file "${artifacts}/cases/${case_id}.json" --verdict BLOCKED --cleanup-status PASS \ + --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --rc-bin "${rc_binary}" \ + --predeployment-blocked --evidence "evidence/${case_id}/predeployment-preflight.log" \ + --evidence "evidence/${case_id}/healthy-summary.json" \ + --reason "B03 exact endpoint mount admission and a reviewed mount-switch adapter are unavailable; the read-only preflight stopped before service or package changes" + exit 1 + elif [ "${preflight_status}" = INFRA_ERROR ]; then + python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ + --result-file "${artifacts}/cases/${case_id}.json" --verdict INFRA_ERROR --cleanup-status PASS \ + --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --rc-bin "${rc_binary}" \ + --evidence "evidence/${case_id}/healthy-summary.json" \ + --reason "B03 read-only admission returned incomplete node evidence; package and service state were left untouched" + exit 1 + elif [ "${preflight_status}" != ADMITTED ]; then + die "B03 predeployment admission returned an unknown status: ${preflight_status}" + fi +fi +package_active=1 (bootstrap) > "${evidence}/setup.log" 2>&1 || bootstrap_rc=$? cat "${evidence}/setup.log" if [ "${bootstrap_rc}" -eq 0 ]; then diff --git a/scripts/validation_result.py b/scripts/validation_result.py index 32ea51cf..4737e8d4 100644 --- a/scripts/validation_result.py +++ b/scripts/validation_result.py @@ -46,14 +46,19 @@ def main() -> int: parser.add_argument("--rc-bin", type=Path) parser.add_argument("--testing-commit") parser.add_argument("--binary-sha256") + parser.add_argument("--predeployment-blocked", action="store_true", + help="record a BLOCKED result before any candidate package was installed") args = parser.parse_args() + if args.predeployment_blocked and (args.verdict != "BLOCKED" + or args.case_id != "heal.disk-replacement-formatted"): + parser.error("--predeployment-blocked is reserved for the B03 mount admission result") catalog = json.loads(args.catalog.read_text(encoding="utf-8")) case = case_from_catalog(catalog, args.case_id) result = { "schema_version": 1, "case_id": args.case_id, "source_status": case["source_status"], - "migration_status": "unverified" if args.verdict in {"RUNNING", "BLOCKED"} else "verified", + "migration_status": "unverified" if args.verdict in {"RUNNING", "BLOCKED", "INFRA_ERROR"} else "verified", "profile": args.profile, "verdict": args.verdict, "cleanup_status": args.cleanup_status, @@ -62,13 +67,14 @@ def main() -> int: if not args.candidate_manifest or not args.rc_bin or not args.testing_commit: parser.error("terminal results require candidate manifest, rc binary, and testing commit") candidate = json.loads(args.candidate_manifest.read_text(encoding="utf-8")) - binary_sha256 = args.binary_sha256 or candidate.get("binary_sha256") + binary_sha256 = (None if args.predeployment_blocked else + args.binary_sha256 or candidate.get("binary_sha256")) bootstrap_error = args.verdict == "INFRA_ERROR" and binary_sha256 is None - if not bootstrap_error and (not isinstance(binary_sha256, str) or len(binary_sha256) != 64 or any( + if not bootstrap_error and not args.predeployment_blocked and (not isinstance(binary_sha256, str) or len(binary_sha256) != 64 or any( character not in "0123456789abcdef" for character in binary_sha256 )): parser.error("terminal results require a valid installed binary SHA-256") - if candidate.get("binary_sha256") and candidate["binary_sha256"] != binary_sha256: + if not args.predeployment_blocked and candidate.get("binary_sha256") and candidate["binary_sha256"] != binary_sha256: parser.error("installed binary SHA-256 differs from candidate manifest") rc_path = args.rc_bin.resolve(strict=True) result["provenance"] = { @@ -80,6 +86,9 @@ def main() -> int: "rc_path": str(rc_path), "rc_sha256": hashlib.sha256(rc_path.read_bytes()).hexdigest(), } + if args.predeployment_blocked: + result["execution_stage"] = "pre-deployment" + result["provenance"]["binary_sha256"] = None result["evidence"] = args.evidence if args.verdict == "PASS": if not args.evidence or args.cleanup_status != "PASS": diff --git a/test_validation_heal_environment.py b/test_validation_heal_environment.py index f05c04fc..42e64747 100644 --- a/test_validation_heal_environment.py +++ b/test_validation_heal_environment.py @@ -8,6 +8,7 @@ from scripts.validation_heal import environment from scripts.validation_heal import fleet_node +from scripts.validation_heal import replacement_preflight from scripts.validation_heal.healthy import Fleet @@ -108,6 +109,73 @@ def test_b03_admission_stops_before_parent_mount_preflight_or_start(self): self.assertEqual(actions, ['replacement-admission-preflight']) self.assertEqual(fleet.reserved, []) + def test_b03_predeployment_gate_is_read_only_and_checks_h1_h2_before_bootstrap(self): + check_names = { + 'exactEndpointMounts', 'mountIdsPresent', 'filesystemIdentitiesDistinct', + 'endpointPhysicalDevicesDistinct', 'endpointDevicesSeparateFromRoot', + 'mountSwitchAdapterReviewed', + } + calls = [] + class ReadOnlyFleet: + def __init__(self, nodes, evidence, run_id): + self.nodes, self.evidence, self.run_id = nodes, evidence, run_id + self.evidence.mkdir(parents=True, exist_ok=True) + def remote(self, node, action): + calls.append((self.run_id, node, action)) + admitted = node != 'vm002' + return {'runId': self.run_id, 'readOnly': True, 'mutated': False, + 'canonicalPassClaimed': False, 'admitted': admitted, + 'checks': {name: admitted for name in check_names}} + def record(self, label, value): + (self.evidence / (label + '.json')).write_text(json.dumps(value)) + return value + candidate = {'source_sha': 'a' * 40, 'package_sha256': 'b' * 64, + 'package_url': 'https://example.invalid/rustfs.deb'} + with tempfile.TemporaryDirectory() as folder, patch.object(replacement_preflight, 'Fleet', ReadOnlyFleet): + summary = replacement_preflight.run_preflight(['vm000', 'vm001', 'vm002', 'vm003'], + Path(folder), '12345', '2', 'c' * 40, candidate) + self.assertEqual(summary['status'], 'BLOCKED') + self.assertTrue(summary['replacementAdmissionBlocked']) + self.assertFalse(summary['mutationStarted']) + self.assertEqual(summary['cleanupDisposition'], 'NOT_REQUIRED_NO_MUTATION') + self.assertEqual([row['topology'] for row in summary['results']], ['H1', 'H2']) + self.assertEqual(len(calls), 5) + self.assertTrue(all(action == 'replacement-admission-preflight' for _, _, action in calls)) + self.assertTrue((Path(folder) / 'healthy-summary.json').is_file()) + + def test_b03_predeployment_transport_failure_is_infrastructure_error(self): + class ReadOnlyFleet: + def __init__(self, nodes, evidence, run_id): + self.nodes, self.evidence = nodes, evidence + self.evidence.mkdir(parents=True, exist_ok=True) + def remote(self, node, action): + if node == 'vm002': raise OSError('SSH unavailable') + return {'readOnly': True, 'mutated': False, 'canonicalPassClaimed': False, + 'admitted': False, 'checks': {'exactEndpointMounts': False, + 'mountIdsPresent': False, 'filesystemIdentitiesDistinct': False, + 'endpointPhysicalDevicesDistinct': False, 'endpointDevicesSeparateFromRoot': False, + 'mountSwitchAdapterReviewed': False}} + def record(self, label, value): return value + candidate = {'source_sha': 'a' * 40, 'package_sha256': 'b' * 64, + 'package_url': 'https://example.invalid/rustfs.deb'} + with tempfile.TemporaryDirectory() as folder, patch.object(replacement_preflight, 'Fleet', ReadOnlyFleet): + summary = replacement_preflight.run_preflight(['vm000', 'vm001', 'vm002', 'vm003'], + Path(folder), '12345', '2', 'c' * 40, candidate) + self.assertEqual(summary['status'], 'INFRA_ERROR') + self.assertFalse(summary['replacementAdmissionBlocked']) + self.assertEqual(summary['infrastructureErrors'][0]['node'], 'vm002') + + def test_b03_runner_preflight_precedes_bootstrap_and_package_activation(self): + source = (Path(__file__).parent / 'scripts/validation_heal/run_case.sh').read_text() + gate = source.index('replacement_preflight.py') + bootstrap = source.index('(bootstrap)') + package_activation = source.index('package_active=1', gate) + self.assertLess(gate, package_activation) + self.assertLess(package_activation, bootstrap) + for status in ('BLOCKED', 'INFRA_ERROR'): + self.assertLess(source.index('preflight_status}" = ' + status, gate), package_activation) + self.assertIn('--predeployment-blocked', source[gate:bootstrap]) + def runtime_command(self, *, units='', listeners='', state=None): state = state or 'LoadState=loaded\nActiveState=active\nSubState=running\nMainPID=42\n' def invoke(*args, **params): diff --git a/test_validation_migration.py b/test_validation_migration.py index bda5d4b3..eb6b129d 100644 --- a/test_validation_migration.py +++ b/test_validation_migration.py @@ -186,6 +186,7 @@ def test_writer_and_report_contract(self) -> None: "--reason", "candidate install failed"], check=True, capture_output=True) bootstrap = json.loads((results / "s3.basic-object-round-trip.json").read_text()) self.assertEqual(bootstrap["verdict"], "INFRA_ERROR") + self.assertEqual(bootstrap["migration_status"], "unverified") self.assertIsNone(bootstrap["provenance"]["binary_sha256"]) subprocess.run(command + identity + ["--verdict", "BLOCKED", "--cleanup-status", "PASS", "--binary-sha256", "b" * 64, "--reason", "diagnostic branches require independent canonical audit", @@ -214,6 +215,34 @@ def test_writer_and_report_contract(self) -> None: _, errors = validate_results(catalog, results, gate_statuses=set()) self.assertEqual([], errors) + def test_predeployment_blocked_result_does_not_claim_an_installed_binary(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + rc = root / "rc" + rc.write_bytes(b"test-rc") + candidate = root / "candidate.json" + candidate.write_text(json.dumps({ + "package_url": "https://example.invalid/rustfs.deb", + "package_sha256": "a" * 64, + "source_sha": "c" * 40, + }), encoding="utf-8") + result = root / "b03.json" + command = [sys.executable, str(ROOT / "scripts/validation_result.py"), + "--case-id", "heal.disk-replacement-formatted", "--profile", "cluster-1pool", + "--result-file", str(result), "--verdict", "BLOCKED", "--cleanup-status", "PASS", + "--candidate-manifest", str(candidate), "--testing-commit", "d" * 40, + "--rc-bin", str(rc), "--predeployment-blocked", "--reason", "mount gate failed"] + subprocess.run(command, check=True, capture_output=True) + value = json.loads(result.read_text(encoding="utf-8")) + self.assertEqual(value["execution_stage"], "pre-deployment") + self.assertEqual(value["migration_status"], "unverified") + self.assertIsNone(value["provenance"]["binary_sha256"]) + self.assertNotIn("assertions", value) + invalid_command = command.copy() + invalid_command[invalid_command.index("--verdict") + 1] = "PASS" + invalid = subprocess.run(invalid_command, capture_output=True) + self.assertNotEqual(invalid.returncode, 0) + if __name__ == "__main__": unittest.main() From 06466a084f938ecb4ed215f8225f07e0751f2dc6 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 19:00:19 +0800 Subject: [PATCH 326/343] docs(validation): record B03 preflight block --- docs/validation-migration-map.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 1794e5be..78a5e9f1 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -168,6 +168,26 @@ reviewed, the gate must pass before recording mount identities before and after replacement and rerunning the full H1/H2 DB/DV matrix. The source oracle and prior FAIL evidence remain unchanged. +The new pre-deployment gate was exercised on the current auto-testing +environment in B03 run +[39942624978/1](https://github.com/rustfs/rustfs/actions/runs/39942624978), +using the same candidate package and helper commit +`49443e7559578cba8be285ed723bb68dfbaf8656`. H1 and H2 both returned +`BLOCKED` before deployment: all endpoint paths were absent ordinary +directories beneath the four distinct NVMe parent mounts, with no exact +endpoint mount IDs or filesystem identities and no reviewed mount-switch +adapter. Root and sibling physical-device separation passed, but that does +not satisfy the endpoint lease contract. The case result records +`execution_stage=pre-deployment`, `binary_sha256=null`, +`mutationStarted=false`, deployment and fixture seeding skipped, and +`cleanup=PASS` with `NOT_REQUIRED_NO_MUTATION`. The case result SHA-256 is +`68ed1713fa3eab0e0d2d972369fca4af00ae58bdb4e6a1e64b072cfae6194621`; the +preflight summary SHA-256 is +`fb47d91ee77d21104be7a400bbfda1aa0d73b30e090476c853a9a53b5036ea12`. +This is verified BLOCKED evidence only: it adds no canonical PASS and does +not replace the retained B03 live failures. The next valid B03 attempt still +depends on installing and reviewing the mount-switch adapter. + B05 acceptance uses candidate `7ed6c9320db491c84b54e8232152d7b1282c1b3f`, helper `76dbf7029e82c1e3878c9160090804d4afc38078`, package `39827676705/1` and tools `39841369555/1`. The complete official ZIP digest is `4eb3405af1f47583cd7c2d8e5b9ce9fa6910b76665df529d7dfbe4f1399a20d8`. Independent recomputation covers seed ACKs, full versions/markers, the actual 2+2 and 12+4 role/loss matrices, every endpoint's full online membership before each required boundary, sequential faults, one unique START/token per instance, two identical terminal outcome queries, strict integer scope, exact repaired/healthy responsibility, all physical shard and bitrot proofs, complete S3/HEAD/Range evidence before finalization, ten private-node cleanups and four package purges. No backup restoration was used. Original failure `38200772548` remains unchanged; its vm001 offline root cause is not established by the later pass. E04 live [39842352595/1](https://github.com/rustfs/rustfs/actions/runs/39842352595) retains FAIL/cleanup PASS. All 1,402 official files, three H1 branches and five node cleanups were independently verified. H2's healthy Scanner task completed on vm002 while the exact node3 token query was forwarded to vm000 and returned notFound. RustFS commit `553be4b4c62ae0c590e82f91817bc018bdfdc660` lets authenticated original local automatic-task tokens return the source/path-bound in-memory snapshot; Admin/Internal/unknown/alias identities retain coordinator routing. Fixed candidate `9d90884c304dcfbf97057c263c6f2ee110230ce7` adds only that reviewed patch to `7ed6c932`, with unchanged Cargo.lock. Package [39851206784/1](https://github.com/rustfs/rustfs/actions/runs/39851206784) and matching tools [39860218641/1](https://github.com/rustfs/rustfs/actions/runs/39860218641) passed independent complete provenance and byte audits. From 6f2c0d1488e5e759140379e4a431234da3fb2b46 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 23 Oct 2026 19:04:53 +0800 Subject: [PATCH 327/343] test(heal): retain B03 topology preflight evidence --- scripts/validation_heal/run_case.sh | 14 +++++++++++--- test_validation_heal_environment.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 5dd7bf2a..f1ada70d 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -203,20 +203,28 @@ if [ "${case_id}" = heal.disk-replacement-formatted ]; then --reason "B03 read-only predeployment admission could not complete; package and service state were left untouched" exit 1 fi + preflight_evidence_args=( + --evidence "evidence/${case_id}/predeployment-preflight.log" + --evidence "evidence/${case_id}/healthy-summary.json" + ) + while IFS= read -r preflight_file; do + preflight_relative="${preflight_file#${artifacts}/}" + [ "${preflight_relative}" != "${preflight_file}" ] || die "B03 preflight evidence escaped the artifact root" + preflight_evidence_args+=(--evidence "${preflight_relative}") + done < <(find "${evidence}/replacement-admission" -type f -name '*.json' -print | LC_ALL=C sort) preflight_status="$(jq -er '.status' "${evidence}/healthy-summary.json")" if [ "${preflight_status}" = BLOCKED ]; then python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ --result-file "${artifacts}/cases/${case_id}.json" --verdict BLOCKED --cleanup-status PASS \ --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --rc-bin "${rc_binary}" \ - --predeployment-blocked --evidence "evidence/${case_id}/predeployment-preflight.log" \ - --evidence "evidence/${case_id}/healthy-summary.json" \ + --predeployment-blocked "${preflight_evidence_args[@]}" \ --reason "B03 exact endpoint mount admission and a reviewed mount-switch adapter are unavailable; the read-only preflight stopped before service or package changes" exit 1 elif [ "${preflight_status}" = INFRA_ERROR ]; then python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ --result-file "${artifacts}/cases/${case_id}.json" --verdict INFRA_ERROR --cleanup-status PASS \ --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --rc-bin "${rc_binary}" \ - --evidence "evidence/${case_id}/healthy-summary.json" \ + "${preflight_evidence_args[@]}" \ --reason "B03 read-only admission returned incomplete node evidence; package and service state were left untouched" exit 1 elif [ "${preflight_status}" != ADMITTED ]; then diff --git a/test_validation_heal_environment.py b/test_validation_heal_environment.py index 42e64747..4ecc5d65 100644 --- a/test_validation_heal_environment.py +++ b/test_validation_heal_environment.py @@ -176,6 +176,16 @@ def test_b03_runner_preflight_precedes_bootstrap_and_package_activation(self): self.assertLess(source.index('preflight_status}" = ' + status, gate), package_activation) self.assertIn('--predeployment-blocked', source[gate:bootstrap]) + def test_b03_case_result_references_each_topology_preflight_record(self): + source = (Path(__file__).parent / 'scripts/validation_heal/run_case.sh').read_text() + gate = source.index('replacement_preflight.py') + bootstrap = source.index('(bootstrap)') + preflight = source[gate:bootstrap] + self.assertIn('find "${evidence}/replacement-admission" -type f -name \'*.json\'', preflight) + self.assertIn('preflight_relative="${preflight_file#${artifacts}/}"', preflight) + self.assertIn('--predeployment-blocked "${preflight_evidence_args[@]}"', preflight) + self.assertIn('"${preflight_evidence_args[@]}"', preflight) + def runtime_command(self, *, units='', listeners='', state=None): state = state or 'LoadState=loaded\nActiveState=active\nSubState=running\nMainPID=42\n' def invoke(*args, **params): From 67b6c585c638066855a037b787021c9f4094ed93 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 24 Oct 2026 09:55:05 +0800 Subject: [PATCH 328/343] test(heal): gate unformatted replacement admission --- docs/validation-heal-contract.md | 2 +- docs/validation-migration-map.md | 8 +++++ scripts/validation_heal/fleet_node.py | 4 +-- scripts/validation_heal/healthy.py | 4 +-- .../validation_heal/replacement_preflight.py | 28 ++++++++++----- scripts/validation_heal/run_case.sh | 22 ++++++------ scripts/validation_heal/unformatted.py | 3 +- scripts/validation_result.py | 7 ++-- test_validation_heal_environment.py | 36 +++++++++++++++++++ test_validation_migration.py | 9 +++++ 10 files changed, 97 insertions(+), 26 deletions(-) diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index 4db8e232..b7376bc7 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -8,7 +8,7 @@ 目前有 67 个自动执行器和 13 个默认关闭的环境场景入口；C11/F07/I05/J02 尚未接入调度。注册执行器不代表完整 canonical PASS，A09/G01–G06 的必需 native 分支仍被阻断。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。13 个环境场景保留源操作分支、未满足条件和显式启用入口；其完整 live 验证仍需资源适配器与源断言审核。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 -B03 的 runner 先执行只读 mount admission preflight，再允许停止服务、下载/安装候选包或创建远端 run namespace；阻断结果明确记录 `execution_stage=pre-deployment`、候选/rc 来源和 `binary_sha256=null`，表示尚未观察到安装候选二进制。通过预检后，部署阶段仍会在 reservation 前再次读取同一组 mount/设备身份，避免预检与实际变更之间的环境漂移。 +B03/B04 replacement runner 先执行只读 mount admission preflight，再允许停止服务、下载/安装候选包或创建远端 run namespace；阻断结果明确记录 `execution_stage=pre-deployment`、候选/rc 来源和 `binary_sha256=null`，表示尚未观察到安装候选二进制。通过预检后，部署阶段仍会在 reservation 前再次读取同一组 mount/设备身份，避免预检与实际变更之间的环境漂移。 H07 的单 set 两个分支只属于 H3，双 set 两个分支只属于 H4；四个不匹配组合记为 `nonApplicableCombinations`。缺少机器属于资源阻塞。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 78a5e9f1..4bf8a5d0 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -188,6 +188,14 @@ This is verified BLOCKED evidence only: it adds no canonical PASS and does not replace the retained B03 live failures. The next valid B03 attempt still depends on installing and reviewing the mount-switch adapter. +B04 `heal.disk-replacement-unformatted` now uses the same replacement +admission contract. Its runner checks H1/H2 exact endpoint mounts before +stopping any service or installing the candidate, and its executor repeats +the read-only admission immediately before reservation. If the current +ordinary-directory layout remains in place, B04 is recorded as `BLOCKED` +with `binary_sha256=null`, `cleanup=PASS`, and +`NOT_REQUIRED_NO_MUTATION`; no blank or foreign-format fixture is created. + B05 acceptance uses candidate `7ed6c9320db491c84b54e8232152d7b1282c1b3f`, helper `76dbf7029e82c1e3878c9160090804d4afc38078`, package `39827676705/1` and tools `39841369555/1`. The complete official ZIP digest is `4eb3405af1f47583cd7c2d8e5b9ce9fa6910b76665df529d7dfbe4f1399a20d8`. Independent recomputation covers seed ACKs, full versions/markers, the actual 2+2 and 12+4 role/loss matrices, every endpoint's full online membership before each required boundary, sequential faults, one unique START/token per instance, two identical terminal outcome queries, strict integer scope, exact repaired/healthy responsibility, all physical shard and bitrot proofs, complete S3/HEAD/Range evidence before finalization, ten private-node cleanups and four package purges. No backup restoration was used. Original failure `38200772548` remains unchanged; its vm001 offline root cause is not established by the later pass. E04 live [39842352595/1](https://github.com/rustfs/rustfs/actions/runs/39842352595) retains FAIL/cleanup PASS. All 1,402 official files, three H1 branches and five node cleanups were independently verified. H2's healthy Scanner task completed on vm002 while the exact node3 token query was forwarded to vm000 and returned notFound. RustFS commit `553be4b4c62ae0c590e82f91817bc018bdfdc660` lets authenticated original local automatic-task tokens return the source/path-bound in-memory snapshot; Admin/Internal/unknown/alias identities retain coordinator routing. Fixed candidate `9d90884c304dcfbf97057c263c6f2ee110230ce7` adds only that reviewed patch to `7ed6c932`, with unchanged Cargo.lock. Package [39851206784/1](https://github.com/rustfs/rustfs/actions/runs/39851206784) and matching tools [39860218641/1](https://github.com/rustfs/rustfs/actions/runs/39860218641) passed independent complete provenance and byte audits. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index c39bbcb7..e751d36b 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -423,7 +423,7 @@ def _replacement_mount_identity(path): def replacement_admission_preflight(request): - """Read-only B03 admission proof; fail closed before service start or seed.""" + """Read-only replacement admission proof; fail closed before service start or seed.""" require(set(request) == {"runId", "action"}, "replacement admission accepts no paths or mutation options") _, roots = paths(request["runId"]) endpoints = [_replacement_mount_identity(root / "data") for root in roots] @@ -460,7 +460,7 @@ def replacement_admission_preflight(request): "reason": None, } if not result["admitted"]: - result["reason"] = "B03 requires exact endpoint mount leases and a reviewed mount-switch adapter; run endpoints are ordinary directories" + result["reason"] = "replacement cases require exact endpoint mount leases and a reviewed mount-switch adapter; run endpoints are ordinary directories" return result diff --git a/scripts/validation_heal/healthy.py b/scripts/validation_heal/healthy.py index 43d6fd80..9a3a1ed4 100644 --- a/scripts/validation_heal/healthy.py +++ b/scripts/validation_heal/healthy.py @@ -37,11 +37,11 @@ class ReplacementAdmissionBlocked(ValueError): - """B03 cannot start until the actual endpoint mounts pass candidate admission.""" + """Replacement cases cannot start until endpoint mounts pass admission.""" def __init__(self, evidence): self.evidence = evidence - super().__init__("B03 replacement mount admission preflight failed; deployment and fixture seed were skipped") + super().__init__("replacement mount admission preflight failed; deployment and fixture seed were skipped") def validate_rejection(observed, expected_code): diff --git a/scripts/validation_heal/replacement_preflight.py b/scripts/validation_heal/replacement_preflight.py index 115db90c..9fda253f 100644 --- a/scripts/validation_heal/replacement_preflight.py +++ b/scripts/validation_heal/replacement_preflight.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Run B03's read-only endpoint admission check before package/service changes.""" +"""Run the replacement cases' read-only endpoint admission check before changes.""" from __future__ import annotations @@ -32,19 +32,28 @@ def checked_identity(value: object) -> bool: and type(value.get("admitted")) is bool) +REPLACEMENT_CASES = { + "heal.disk-replacement-formatted", + "heal.disk-replacement-unformatted", +} + + def run_preflight(nodes: list[str], evidence: Path, workflow_run_id: str, attempt: str, - testing_commit: str, candidate: dict) -> dict: + testing_commit: str, candidate: dict, + case_id: str = "heal.disk-replacement-formatted") -> dict: + if case_id not in REPLACEMENT_CASES: + raise ValueError(f"unsupported replacement admission case: {case_id}") if not re.fullmatch(r"[1-9][0-9]*", workflow_run_id) or not re.fullmatch(r"[1-9][0-9]*", attempt): - raise ValueError("B03 preflight requires the immutable workflow run and attempt") + raise ValueError("replacement preflight requires the immutable workflow run and attempt") if not re.fullmatch(r"[0-9a-f]{40}", testing_commit): - raise ValueError("B03 preflight requires the exact auto-testing commit") + raise ValueError("replacement preflight requires the exact auto-testing commit") if (not re.fullmatch(r"[0-9a-f]{40}", str(candidate.get("source_sha", ""))) or not re.fullmatch(r"[0-9a-f]{64}", str(candidate.get("package_sha256", ""))) or not isinstance(candidate.get("package_url"), str) or not candidate["package_url"]): - raise ValueError("B03 preflight candidate package provenance is incomplete") + raise ValueError("replacement preflight candidate package provenance is incomplete") if len(nodes) != 4 or len(set(nodes)) != 4 or any( not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", node) for node in nodes): - raise ValueError("B03 preflight requires the four current auto-testing Heal nodes") + raise ValueError("replacement preflight requires the four current auto-testing Heal nodes") results = [] infrastructure_errors = [] @@ -83,7 +92,7 @@ def run_preflight(nodes: list[str], evidence: Path, workflow_run_id: str, attemp status = "INFRA_ERROR" if infrastructure_errors else "BLOCKED" if blocked else "ADMITTED" summary = { "schemaVersion": 1, - "caseId": "heal.disk-replacement-formatted", + "caseId": case_id, "status": status, "executionStage": "pre-deployment", "workflowRunId": int(workflow_run_id), @@ -109,6 +118,8 @@ def run_preflight(nodes: list[str], evidence: Path, workflow_run_id: str, attemp def main() -> int: parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--case-id", choices=sorted(REPLACEMENT_CASES), + default="heal.disk-replacement-formatted") parser.add_argument("--candidate", type=Path, required=True) parser.add_argument("--testing-commit", required=True) parser.add_argument("--evidence", type=Path, required=True) @@ -117,7 +128,8 @@ def main() -> int: nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() candidate = json.loads(args.candidate.read_text(encoding="utf-8")) summary = run_preflight(nodes, args.evidence, os.environ.get("GITHUB_RUN_ID", ""), - os.environ.get("GITHUB_RUN_ATTEMPT", "1"), args.testing_commit, candidate) + os.environ.get("GITHUB_RUN_ATTEMPT", "1"), args.testing_commit, candidate, + args.case_id) print(json.dumps({"status": summary["status"], "mutationStarted": False, "replacementAdmissionBlocked": summary["replacementAdmissionBlocked"]}, sort_keys=True)) return 0 diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index f1ada70d..90aefd56 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -7,6 +7,7 @@ ASSUME_YES=1 candidate="" testing_commit="" artifacts="" case_id="" driver=healthy.py environment_gate=0 +replacement_admission_gate=0 i05_args=() while [ "$#" -gt 0 ]; do case "$1" in @@ -39,7 +40,8 @@ case "${case_id}" in heal.admin-option-contract) driver=options.py ;; heal.queue-pressure-and-admission) driver=queue_pressure.py ;; heal.bucket-metadata-integrity) driver=bucket_metadata.py ;; - heal.disk-replacement-unformatted) driver=unformatted.py ;; + heal.disk-replacement-unformatted) driver=unformatted.py; replacement_admission_gate=1 ;; + heal.disk-replacement-formatted) driver=versions.py; replacement_admission_gate=1 ;; heal.below-write-quorum) driver=write_quorum.py ;; heal.foreign-format-and-unknown-schema) driver=schema.py ;; heal.encrypted-objects) driver=encrypted.py ;; @@ -55,7 +57,7 @@ case "${case_id}" in heal.mixed-version-upgrade-rollback|heal.dedicated-host-forced-restart|heal.race-ilm-expiry-and-transition|heal.race-replication-resync|heal.with-expansion-rebalance|heal.with-rebalance-stop-resume|heal.with-decommission|heal.with-decommission-cancel-clear|heal.after-retired-pool-removal|heal.ec84-multi-set-restart|heal.slow-io-and-eio|heal.kms-and-tier-outage|heal.long-soak) driver=environment_gate.py; environment_gate=1 ;; heal.read-quorum-boundary) driver=quorum_boundary.py ;; - heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.disk-replacement-formatted|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; + heal.historical-versions|heal.repeat-idempotency|heal.versioning-suspended-null|heal.below-read-quorum|heal.missing-or-corrupt-xlmeta|heal.object-metadata-and-tags|heal.multiple-shard-loss|heal.special-keys-and-pagination|heal.object-size-boundaries|heal.object-lock-retention|heal.metadata-payload-mismatch|heal.read-repair-attribution|heal.disk-readonly-and-permission|heal.scanner-cold-object-repair|heal.delete-marker-semantics|heal.stale-version-rejoin|heal.disk-loss-and-repair|heal.replacement-auto-rebuild|heal.target-graceful-restart|heal.target-process-crash) driver=versions.py ;; heal.admin-token-cross-node-status|heal.admin-dry-run|heal.missing-part-file|heal.data-shard-bitrot|heal.parity-shard-bitrot|heal.truncated-and-zeroed-shard|heal.completed-multipart-object|heal.range-head-and-conditions|heal.manual-normal-attribution|heal.manual-deep-attribution) driver=shards.py ;; *) die "automatic Heal executor is not migrated: ${case_id}" ;; @@ -189,10 +191,10 @@ on_exit() { trap on_exit EXIT trap 'exit 130' INT trap 'exit 143' TERM -if [ "${case_id}" = heal.disk-replacement-formatted ]; then +if [ "${replacement_admission_gate}" -eq 1 ]; then preflight_rc=0 python3 -B "${ROOT}/scripts/validation_heal/replacement_preflight.py" \ - --candidate "${candidate}" --testing-commit "${testing_commit}" --evidence "${evidence}" \ + --case-id "${case_id}" --candidate "${candidate}" --testing-commit "${testing_commit}" --evidence "${evidence}" \ > "${evidence}/predeployment-preflight.log" 2>&1 || preflight_rc=$? cat "${evidence}/predeployment-preflight.log" if [ "${preflight_rc}" -ne 0 ]; then @@ -200,7 +202,7 @@ if [ "${case_id}" = heal.disk-replacement-formatted ]; then --result-file "${artifacts}/cases/${case_id}.json" --verdict INFRA_ERROR --cleanup-status PASS \ --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --rc-bin "${rc_binary}" \ --evidence "evidence/${case_id}/predeployment-preflight.log" \ - --reason "B03 read-only predeployment admission could not complete; package and service state were left untouched" + --reason "${case_id} read-only predeployment admission could not complete; package and service state were left untouched" exit 1 fi preflight_evidence_args=( @@ -218,17 +220,17 @@ if [ "${case_id}" = heal.disk-replacement-formatted ]; then --result-file "${artifacts}/cases/${case_id}.json" --verdict BLOCKED --cleanup-status PASS \ --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --rc-bin "${rc_binary}" \ --predeployment-blocked "${preflight_evidence_args[@]}" \ - --reason "B03 exact endpoint mount admission and a reviewed mount-switch adapter are unavailable; the read-only preflight stopped before service or package changes" + --reason "${case_id} exact endpoint mount admission and a reviewed mount-switch adapter are unavailable; the read-only preflight stopped before service or package changes" exit 1 elif [ "${preflight_status}" = INFRA_ERROR ]; then python3 -B "${ROOT}/scripts/validation_result.py" --case-id "${case_id}" --profile cluster-1pool \ --result-file "${artifacts}/cases/${case_id}.json" --verdict INFRA_ERROR --cleanup-status PASS \ --candidate-manifest "${candidate}" --testing-commit "${testing_commit}" --rc-bin "${rc_binary}" \ "${preflight_evidence_args[@]}" \ - --reason "B03 read-only admission returned incomplete node evidence; package and service state were left untouched" + --reason "${case_id} read-only admission returned incomplete node evidence; package and service state were left untouched" exit 1 elif [ "${preflight_status}" != ADMITTED ]; then - die "B03 predeployment admission returned an unknown status: ${preflight_status}" + die "${case_id} predeployment admission returned an unknown status: ${preflight_status}" fi fi package_active=1 @@ -268,10 +270,10 @@ if [ "${case_rc}" -eq 0 ] && [ "${cleanup_status}" = PASS ]; then elif [ "${case_rc}" -eq 0 ]; then verdict=INFRA_ERROR; reason="Heal environment finalization failed" fi -if [ "${case_id}" = heal.disk-replacement-formatted ] && [ "${cleanup_status}" = PASS ] && \ +if [ "${replacement_admission_gate}" -eq 1 ] && [ "${cleanup_status}" = PASS ] && \ jq -e '.replacementAdmissionBlocked == true and .complete == false and .canonicalPassClaimed != true' \ "${evidence}/healthy-summary.json" >/dev/null 2>&1; then - verdict=BLOCKED; reason="B03 exact endpoint mount admission and a reviewed mount-switch adapter are unavailable; preflight stopped before deployment and fixture seed" + verdict=BLOCKED; reason="${case_id} exact endpoint mount admission and a reviewed mount-switch adapter are unavailable; preflight stopped before deployment and fixture seed" fi if [ "${case_id}" = heal.bucket-incarnation-recreation ] && [ "${cleanup_status}" = PASS ] && \ jq -e '.diagnosticBranchesPassed == true and .complete == false and .canonicalCasePassClaimed == false' \ diff --git a/scripts/validation_heal/unformatted.py b/scripts/validation_heal/unformatted.py index 8627f4d9..b0746933 100644 --- a/scripts/validation_heal/unformatted.py +++ b/scripts/validation_heal/unformatted.py @@ -126,7 +126,8 @@ def run_branch(args, topology, nodes, branch, foreign): directory = args.evidence / f"{topology}-{branch}" directory.mkdir() run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" - fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="replacement") + fleet = Fleet(nodes, directory, run_id, args.tool_directory, runtime_profile="replacement", + require_replacement_admission=True) candidate = json.loads(args.candidate.read_text()) fleet.record("inspector-build", verify_inspector(args.tool_directory, candidate)) bucket = "rv-heal-" + hashlib.sha256(run_id.encode()).hexdigest()[:16] diff --git a/scripts/validation_result.py b/scripts/validation_result.py index 4737e8d4..0cdfe291 100644 --- a/scripts/validation_result.py +++ b/scripts/validation_result.py @@ -50,8 +50,11 @@ def main() -> int: help="record a BLOCKED result before any candidate package was installed") args = parser.parse_args() if args.predeployment_blocked and (args.verdict != "BLOCKED" - or args.case_id != "heal.disk-replacement-formatted"): - parser.error("--predeployment-blocked is reserved for the B03 mount admission result") + or args.case_id not in { + "heal.disk-replacement-formatted", + "heal.disk-replacement-unformatted", + }): + parser.error("--predeployment-blocked is reserved for replacement admission results") catalog = json.loads(args.catalog.read_text(encoding="utf-8")) case = case_from_catalog(catalog, args.case_id) result = { diff --git a/test_validation_heal_environment.py b/test_validation_heal_environment.py index 4ecc5d65..bace1a76 100644 --- a/test_validation_heal_environment.py +++ b/test_validation_heal_environment.py @@ -165,6 +165,32 @@ def record(self, label, value): return value self.assertFalse(summary['replacementAdmissionBlocked']) self.assertEqual(summary['infrastructureErrors'][0]['node'], 'vm002') + def test_b04_reuses_replacement_admission_summary_without_claiming_mutation(self): + class ReadOnlyFleet: + def __init__(self, nodes, evidence, run_id): + self.nodes, self.evidence = nodes, evidence + self.evidence.mkdir(parents=True, exist_ok=True) + def remote(self, node, action): + return {'readOnly': True, 'mutated': False, 'canonicalPassClaimed': False, + 'admitted': False, 'checks': { + 'exactEndpointMounts': False, 'mountIdsPresent': False, + 'filesystemIdentitiesDistinct': False, + 'endpointPhysicalDevicesDistinct': True, + 'endpointDevicesSeparateFromRoot': True, + 'mountSwitchAdapterReviewed': False}} + def record(self, label, value): return value + candidate = {'source_sha': 'a' * 40, 'package_sha256': 'b' * 64, + 'package_url': 'https://example.invalid/rustfs.deb'} + with tempfile.TemporaryDirectory() as folder, patch.object(replacement_preflight, 'Fleet', ReadOnlyFleet): + summary = replacement_preflight.run_preflight(['vm000', 'vm001', 'vm002', 'vm003'], + Path(folder), '12345', '2', 'c' * 40, candidate, + 'heal.disk-replacement-unformatted') + self.assertEqual(summary['caseId'], 'heal.disk-replacement-unformatted') + self.assertEqual(summary['status'], 'BLOCKED') + self.assertTrue(summary['replacementAdmissionBlocked']) + self.assertFalse(summary['mutationStarted']) + self.assertEqual(summary['cleanupDisposition'], 'NOT_REQUIRED_NO_MUTATION') + def test_b03_runner_preflight_precedes_bootstrap_and_package_activation(self): source = (Path(__file__).parent / 'scripts/validation_heal/run_case.sh').read_text() gate = source.index('replacement_preflight.py') @@ -186,6 +212,16 @@ def test_b03_case_result_references_each_topology_preflight_record(self): self.assertIn('--predeployment-blocked "${preflight_evidence_args[@]}"', preflight) self.assertIn('"${preflight_evidence_args[@]}"', preflight) + def test_b04_runner_and_executor_use_the_same_replacement_admission_contract(self): + source = (Path(__file__).parent / 'scripts/validation_heal/run_case.sh').read_text() + gate = source.index('replacement_preflight.py') + bootstrap = source.index('(bootstrap)') + self.assertIn('heal.disk-replacement-unformatted) driver=unformatted.py; replacement_admission_gate=1', source) + self.assertIn('--case-id "${case_id}"', source[gate:bootstrap]) + self.assertLess(gate, bootstrap) + executor = (Path(__file__).parent / 'scripts/validation_heal/unformatted.py').read_text() + self.assertIn('require_replacement_admission=True', executor) + def runtime_command(self, *, units='', listeners='', state=None): state = state or 'LoadState=loaded\nActiveState=active\nSubState=running\nMainPID=42\n' def invoke(*args, **params): diff --git a/test_validation_migration.py b/test_validation_migration.py index eb6b129d..831109a8 100644 --- a/test_validation_migration.py +++ b/test_validation_migration.py @@ -243,6 +243,15 @@ def test_predeployment_blocked_result_does_not_claim_an_installed_binary(self): invalid = subprocess.run(invalid_command, capture_output=True) self.assertNotEqual(invalid.returncode, 0) + b04 = root / "b04.json" + b04_command = command.copy() + b04_command[b04_command.index("--case-id") + 1] = "heal.disk-replacement-unformatted" + b04_command[b04_command.index("--result-file") + 1] = str(b04) + subprocess.run(b04_command, check=True, capture_output=True) + b04_value = json.loads(b04.read_text(encoding="utf-8")) + self.assertEqual(b04_value["case_id"], "heal.disk-replacement-unformatted") + self.assertIsNone(b04_value["provenance"]["binary_sha256"]) + if __name__ == "__main__": unittest.main() From 0e3e84298770f26c2211ab12acf3cf18eaf474e0 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 24 Oct 2026 09:57:55 +0800 Subject: [PATCH 329/343] test(heal): keep verdict snippet compatible --- scripts/validation_heal/run_case.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 90aefd56..7f3c95e4 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -270,7 +270,7 @@ if [ "${case_rc}" -eq 0 ] && [ "${cleanup_status}" = PASS ]; then elif [ "${case_rc}" -eq 0 ]; then verdict=INFRA_ERROR; reason="Heal environment finalization failed" fi -if [ "${replacement_admission_gate}" -eq 1 ] && [ "${cleanup_status}" = PASS ] && \ +if [ "${replacement_admission_gate:-0}" -eq 1 ] && [ "${cleanup_status}" = PASS ] && \ jq -e '.replacementAdmissionBlocked == true and .complete == false and .canonicalPassClaimed != true' \ "${evidence}/healthy-summary.json" >/dev/null 2>&1; then verdict=BLOCKED; reason="${case_id} exact endpoint mount admission and a reviewed mount-switch adapter are unavailable; preflight stopped before deployment and fixture seed" From e26b5966614bd6220059928673d1e7acb57c0181 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 24 Oct 2026 10:00:38 +0800 Subject: [PATCH 330/343] docs(validation): document shared replacement gate --- docs/validation-heal-cases.md | 6 ++++++ docs/validation-heal-operations.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/validation-heal-cases.md b/docs/validation-heal-cases.md index bd479d73..72b29d74 100644 --- a/docs/validation-heal-cases.md +++ b/docs/validation-heal-cases.md @@ -314,6 +314,12 @@ Source assertions: Topologies: H1,H2. Branches: blank-unformatted, foreign-format. +The Functional Chain runs the shared read-only replacement admission gate +before package/service changes and the executor repeats it before reservation. +If exact endpoint mounts or the reviewed mount-switch adapter are unavailable, +the case is recorded as pre-deployment `BLOCKED` without creating either +the blank or foreign-format fixture. + 1. REPLACE 建没有 format 的空逻辑盘；记录空目录证据，启动并观察候选支持的安全初始化或明确拒绝。 2. 第二实例使用同 run 的第二个隔离小集群生成外来 format，将其按 FORMAT_BAD 手册装入替换根，保留原始副本和所有 UUID。 3. 外来身份不能被静默覆盖或挂到错误 set；若拒绝，等待有界 deadline 并确认其他正常盘数据不变。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 01d31f0a..dc88f3de 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -54,7 +54,7 @@ python scripts/validation_heal/heal_probe.py --binding lab.json --out reports/bi ### 独立实验服务的部署 -H1 使用当前 Heal 节点经核验的四个独立 mount，**另建** `/data/rustfsN/heal-validation//data` 和同级 `quarantine`，root-owned 0700；每个 `` 根写入包含 `runId` 的 root-owned `owner.json`。普通对象故障可以使用这些独立父盘下的 run 私有目录。B03 还要求替换 target 和同组兄弟 endpoint 都能取得 candidate 认可的精确 mount lease；替换过程中必须证明旧 mount 与 quarantine 对服务不可读、新空设备挂到原 endpoint 后 candidate 取得新的 mount lease，并前后记录 mount ID、物理设备 ID 和 filesystem identity。独立文件系统或隔离 mount namespace 中的 bind mount 都可能作为实现方式，但仅创建 bind mount 不足以满足要求：adapter 还必须安全切换被替换 endpoint，且通过 candidate 对 root 和 sibling 物理设备身份的检查。Functional Chain 会先在 `stop_existing` 和候选 deb 下载/安装之前调用只读 `replacement-admission-preflight`，覆盖 H1/H2 endpoints；未通过时以 `BLOCKED` 保留 package/client provenance 和完整现场读数，不停服务、不碰已安装 package，也不创建远端 run namespace。`Fleet.deploy()` 在真正 reservation 前再重复该检查。缺少真实 endpoint mount 或已审核 mount-switch adapter 时应 BLOCKED；当前 runner 只把 run 私有子目录作为 endpoint，尚无此 mount-switch adapter，因此它现有的四个独立父盘不能证明 B03 满足资源条件。不得延长等待时间或复制 quarantine 数据。先检查每个根不存在；已存在的根不能覆盖。复用 `rustfs_heal_test.sh` 的 candidate deb 下载、SHA 校验和安装流程，把已安装且摘要已核验的 `/usr/bin/rustfs` 放在本批 state 下；执行用例不需要 RustFS 源码 checkout。凭据文件为 0600。以下专用 unit 仍运行在当前 Heal 节点，使用已分配的 9000 测试端口： +H1 使用当前 Heal 节点经核验的四个独立 mount，**另建** `/data/rustfsN/heal-validation//data` 和同级 `quarantine`，root-owned 0700；每个 `` 根写入包含 `runId` 的 root-owned `owner.json`。普通对象故障可以使用这些独立父盘下的 run 私有目录。B03 和 B04 还要求替换 target 和同组兄弟 endpoint 都能取得 candidate 认可的精确 mount lease；替换过程中必须证明旧 mount 与 quarantine 对服务不可读、新空设备挂到原 endpoint 后 candidate 取得新的 mount lease，并前后记录 mount ID、物理设备 ID 和 filesystem identity。独立文件系统或隔离 mount namespace 中的 bind mount 都可能作为实现方式，但仅创建 bind mount 不足以满足要求：adapter 还必须安全切换被替换 endpoint，且通过 candidate 对 root 和 sibling 物理设备身份的检查。Functional Chain 会先在 `stop_existing` 和候选 deb 下载/安装之前调用只读 `replacement-admission-preflight`，覆盖 H1/H2 endpoints；未通过时以 `BLOCKED` 保留 package/client provenance 和完整现场读数，不停服务、不碰已安装 package，也不创建远端 run namespace。`Fleet.deploy()` 在真正 reservation 前再重复该检查。缺少真实 endpoint mount 或已审核 mount-switch adapter 时应 BLOCKED；当前 runner 只把 run 私有子目录作为 endpoint，尚无此 mount-switch adapter，因此它现有的四个独立父盘不能证明 B03/B04 满足资源条件。不得延长等待时间或复制 quarantine 数据。先检查每个根不存在；已存在的根不能覆盖。复用 `rustfs_heal_test.sh` 的 candidate deb 下载、SHA 校验和安装流程，把已安装且摘要已核验的 `/usr/bin/rustfs` 放在本批 state 下；执行用例不需要 RustFS 源码 checkout。凭据文件为 0600。以下专用 unit 仍运行在当前 Heal 节点，使用已分配的 9000 测试端口： ```bash # 在目标节点；所有变量来自已审核的 lab binding，不接受任意在线输入。 From 372f2cb8ca28bb6c33a25d8c719eff441e73cff6 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 24 Oct 2026 10:19:38 +0800 Subject: [PATCH 331/343] test(heal): generalize replacement preflight error --- scripts/validation_heal/replacement_preflight.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validation_heal/replacement_preflight.py b/scripts/validation_heal/replacement_preflight.py index 9fda253f..d68722fe 100644 --- a/scripts/validation_heal/replacement_preflight.py +++ b/scripts/validation_heal/replacement_preflight.py @@ -134,7 +134,7 @@ def main() -> int: "replacementAdmissionBlocked": summary["replacementAdmissionBlocked"]}, sort_keys=True)) return 0 except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: - print(f"B03 pre-deployment preflight failed: {exc}", file=sys.stderr) + print(f"replacement pre-deployment preflight failed: {exc}", file=sys.stderr) return 1 From de310e317338c086d27cf44eb1e55779936fbc3b Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 24 Oct 2026 10:39:12 +0800 Subject: [PATCH 332/343] fix(heal): classify replacement admission drift as blocked --- scripts/validation_heal/unformatted.py | 25 +++++++++++++--- test_validation_heal_unformatted.py | 40 ++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/scripts/validation_heal/unformatted.py b/scripts/validation_heal/unformatted.py index b0746933..6c42da40 100644 --- a/scripts/validation_heal/unformatted.py +++ b/scripts/validation_heal/unformatted.py @@ -23,7 +23,7 @@ try: from .heal_probe import Probe, compare_inventories, require, save from .heal_workload import Recorder, seed, verify_seed - from .healthy import Fleet, physical_content, validate_topology + from .healthy import Fleet, ReplacementAdmissionBlocked, physical_content, validate_topology from .inspector import verify as verify_inspector from .matrix import select from .disk_loss import info_disks, member_state @@ -32,7 +32,7 @@ except ImportError: from heal_probe import Probe, compare_inventories, require, save from heal_workload import Recorder, seed, verify_seed - from healthy import Fleet, physical_content, validate_topology + from healthy import Fleet, ReplacementAdmissionBlocked, physical_content, validate_topology from inspector import verify as verify_inspector from matrix import select from disk_loss import info_disks, member_state @@ -230,6 +230,11 @@ def run_branch(args, topology, nodes, branch, foreign): fleet.stop() require(physical_content(rebuilt) == physical_content(fleet.snapshot("after-S3-physical", quiescent=True)), "S3 verification changed the proven physical result") result.update(verdict="PASS", all128ACKsPreserved=True, allPeerS3ManifestEqual=True, syntheticFormatOrUUIDUsed=False) + except ReplacementAdmissionBlocked as exc: + result.update(verdict="BLOCKED", replacementAdmissionBlocked=True, + admissionEvidence=exc.evidence, deploymentSkipped=True, + fixtureSeedSkipped=True, + failure={"type": type(exc).__name__, "message": str(exc)}) except (Exception, KeyboardInterrupt) as exc: result.update(verdict="FAIL", failure={"type": type(exc).__name__, "message": str(exc)}) if fleet.reserved: @@ -267,14 +272,26 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"Heal workflow interrup for topology, selected in (("H1", nodes[:1]), ("H2", nodes[:4])): for branch in BRANCHES: results.append(run_branch(args, topology, selected, branch, donor)) + if results[-1].get("replacementAdmissionBlocked") is True: + break if results[-1]["cleanup"] != "PASS": break - if results[-1]["cleanup"] != "PASS": break + if (results[-1].get("replacementAdmissionBlocked") is True + or results[-1]["cleanup"] != "PASS"): break except (Exception, KeyboardInterrupt) as exc: donor_error = {"type": type(exc).__name__, "message": str(exc)} finally: complete = len(results) == 4 and all(item["verdict"] == "PASS" and item["cleanup"] == "PASS" for item in results) + replacement_admission_blocked = (bool(results) + and any(item.get("replacementAdmissionBlocked") is True for item in results) + and all(item.get("cleanup") == "PASS" for item in results) + and donor_error is None) save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "donorFailure": donor_error, - "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) else "FAIL", "complete": complete}) + "cleanup": "PASS" if results and all(item["cleanup"] == "PASS" for item in results) and donor_error is None else "FAIL", + "complete": complete, "replacementAdmissionBlocked": replacement_admission_blocked, + "canonicalPassClaimed": False}) + if replacement_admission_blocked: + print("heal.disk-replacement-unformatted: BLOCKED before reservation or fixture seed by mount admission preflight") + return require(complete, "B04 requires both source branches on H1/H2, released foreign donor and complete cleanup") diff --git a/test_validation_heal_unformatted.py b/test_validation_heal_unformatted.py index 2fad69e4..a13aeb9e 100644 --- a/test_validation_heal_unformatted.py +++ b/test_validation_heal_unformatted.py @@ -4,6 +4,7 @@ import json from pathlib import Path import tempfile +from types import SimpleNamespace import unittest from unittest.mock import patch @@ -110,5 +111,44 @@ def test_both_source_branches_have_an_automatic_entry(self): self.assertEqual(select("B04")[0]["variants"], list(unformatted.BRANCHES)) self.assertIn(unformatted.CASE_ID, AUTOMATIC_CASES) + def test_second_admission_block_is_blocked_and_does_not_seed_fixture(self): + class BlockedFleet: + instances = [] + def __init__(self, *args, **kwargs): + self.reserved = [] + self.records = [] + self.instances.append(self) + def record(self, label, value): + self.records.append((label, value)) + return value + def deploy(self, *args, **kwargs): + raise unformatted.ReplacementAdmissionBlocked({ + "vm000": {"admitted": False, "mutated": False, "readOnly": True}}) + def cleanup(self): return True + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + candidate = root / "candidate.json" + candidate.write_text(json.dumps({"source_sha": "072b57b37559fbe626aeda8abf61617fdcbdd926", + "package_sha256": "b" * 64, "package_url": "https://example.invalid/rustfs.deb"})) + evidence = root / "evidence" + evidence.mkdir() + rc = root / "rc" + rc.write_bytes(b"rc") + args = SimpleNamespace(evidence=evidence, candidate=candidate, binary_sha="c" * 64, + testing_commit="d" * 40, tool_directory=root, rc_bin=rc) + with patch.dict("os.environ", {"GITHUB_RUN_ID": "12345", "GITHUB_RUN_ATTEMPT": "1"}), \ + patch.object(unformatted, "Fleet", BlockedFleet), \ + patch.object(unformatted, "verify_inspector", return_value={"verified": True}): + result = unformatted.run_branch(args, "H1", ["vm000"], "blank-unformatted", None) + self.assertEqual(result["verdict"], "BLOCKED") + self.assertTrue(result["replacementAdmissionBlocked"]) + self.assertTrue(result["deploymentSkipped"]) + self.assertTrue(result["fixtureSeedSkipped"]) + self.assertEqual(result["cleanup"], "PASS") + self.assertEqual(result["admissionEvidence"]["vm000"]["admitted"], False) + self.assertEqual(len(BlockedFleet.instances), 1) + self.assertEqual(BlockedFleet.instances[0].reserved, []) + self.assertFalse((evidence / "H1-blank-unformatted" / "fixture").exists()) + if __name__ == "__main__": unittest.main() From 44312c5590f1c060091ece78e379c76aedcb6234 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 24 Oct 2026 12:06:48 +0800 Subject: [PATCH 333/343] fix(heal): clarify replacement admission block reason --- scripts/validation_heal/run_case.sh | 2 +- test_validation_heal_environment.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 7f3c95e4..2a4b1bf1 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -273,7 +273,7 @@ fi if [ "${replacement_admission_gate:-0}" -eq 1 ] && [ "${cleanup_status}" = PASS ] && \ jq -e '.replacementAdmissionBlocked == true and .complete == false and .canonicalPassClaimed != true' \ "${evidence}/healthy-summary.json" >/dev/null 2>&1; then - verdict=BLOCKED; reason="${case_id} exact endpoint mount admission and a reviewed mount-switch adapter are unavailable; preflight stopped before deployment and fixture seed" + verdict=BLOCKED; reason="${case_id} replacement admission did not permit deployment or fixture seed; the read-only gate or the reservation-time recheck blocked the case" fi if [ "${case_id}" = heal.bucket-incarnation-recreation ] && [ "${cleanup_status}" = PASS ] && \ jq -e '.diagnosticBranchesPassed == true and .complete == false and .canonicalCasePassClaimed == false' \ diff --git a/test_validation_heal_environment.py b/test_validation_heal_environment.py index bace1a76..5c493fad 100644 --- a/test_validation_heal_environment.py +++ b/test_validation_heal_environment.py @@ -219,6 +219,8 @@ def test_b04_runner_and_executor_use_the_same_replacement_admission_contract(sel self.assertIn('heal.disk-replacement-unformatted) driver=unformatted.py; replacement_admission_gate=1', source) self.assertIn('--case-id "${case_id}"', source[gate:bootstrap]) self.assertLess(gate, bootstrap) + self.assertIn('replacement admission did not permit deployment or fixture seed', source) + self.assertIn('reservation-time recheck blocked the case', source) executor = (Path(__file__).parent / 'scripts/validation_heal/unformatted.py').read_text() self.assertIn('require_replacement_admission=True', executor) From 002473b51f5ceeccadedf7325a1b1691e4c71dcd Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 24 Oct 2026 17:13:59 +0800 Subject: [PATCH 334/343] docs(validation): clarify two-stage replacement admission --- docs/validation-heal-operations.md | 2 ++ docs/validation-migration-map.md | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index dc88f3de..bf89597a 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -56,6 +56,8 @@ python scripts/validation_heal/heal_probe.py --binding lab.json --out reports/bi H1 使用当前 Heal 节点经核验的四个独立 mount，**另建** `/data/rustfsN/heal-validation//data` 和同级 `quarantine`，root-owned 0700；每个 `` 根写入包含 `runId` 的 root-owned `owner.json`。普通对象故障可以使用这些独立父盘下的 run 私有目录。B03 和 B04 还要求替换 target 和同组兄弟 endpoint 都能取得 candidate 认可的精确 mount lease；替换过程中必须证明旧 mount 与 quarantine 对服务不可读、新空设备挂到原 endpoint 后 candidate 取得新的 mount lease，并前后记录 mount ID、物理设备 ID 和 filesystem identity。独立文件系统或隔离 mount namespace 中的 bind mount 都可能作为实现方式，但仅创建 bind mount 不足以满足要求：adapter 还必须安全切换被替换 endpoint，且通过 candidate 对 root 和 sibling 物理设备身份的检查。Functional Chain 会先在 `stop_existing` 和候选 deb 下载/安装之前调用只读 `replacement-admission-preflight`，覆盖 H1/H2 endpoints；未通过时以 `BLOCKED` 保留 package/client provenance 和完整现场读数，不停服务、不碰已安装 package，也不创建远端 run namespace。`Fleet.deploy()` 在真正 reservation 前再重复该检查。缺少真实 endpoint mount 或已审核 mount-switch adapter 时应 BLOCKED；当前 runner 只把 run 私有子目录作为 endpoint，尚无此 mount-switch adapter，因此它现有的四个独立父盘不能证明 B03/B04 满足资源条件。不得延长等待时间或复制 quarantine 数据。先检查每个根不存在；已存在的根不能覆盖。复用 `rustfs_heal_test.sh` 的 candidate deb 下载、SHA 校验和安装流程，把已安装且摘要已核验的 `/usr/bin/rustfs` 放在本批 state 下；执行用例不需要 RustFS 源码 checkout。凭据文件为 0600。以下专用 unit 仍运行在当前 Heal 节点，使用已分配的 9000 测试端口： +如果首次 gate 通过后、`Fleet.deploy()` 的 reservation-time 复检发现 mount 漂移，仍记录 `BLOCKED` 并保留已安装候选的 binary provenance；不得把该结果改写成产品 `FAIL`，也不得把它伪装成 pre-deployment 的 `NOT_REQUIRED_NO_MUTATION`。两阶段检查都必须跳过 reservation 和 fixture seed，并完成 cleanup。 + ```bash # 在目标节点；所有变量来自已审核的 lab binding，不接受任意在线输入。 sudo systemd-run --unit="$HEAL_UNIT" --property=Type=exec \ diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 4bf8a5d0..f823962e 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -195,6 +195,11 @@ the read-only admission immediately before reservation. If the current ordinary-directory layout remains in place, B04 is recorded as `BLOCKED` with `binary_sha256=null`, `cleanup=PASS`, and `NOT_REQUIRED_NO_MUTATION`; no blank or foreign-format fixture is created. +If the first gate passes but the reservation-time recheck observes environment +drift, the executor records the same `BLOCKED` verdict while retaining the +installed candidate provenance, skips reservation and fixture seeding, and +finishes cleanup. That post-bootstrap block is distinct from the +pre-deployment `NOT_REQUIRED_NO_MUTATION` result and cannot claim canonical PASS. B05 acceptance uses candidate `7ed6c9320db491c84b54e8232152d7b1282c1b3f`, helper `76dbf7029e82c1e3878c9160090804d4afc38078`, package `39827676705/1` and tools `39841369555/1`. The complete official ZIP digest is `4eb3405af1f47583cd7c2d8e5b9ce9fa6910b76665df529d7dfbe4f1399a20d8`. Independent recomputation covers seed ACKs, full versions/markers, the actual 2+2 and 12+4 role/loss matrices, every endpoint's full online membership before each required boundary, sequential faults, one unique START/token per instance, two identical terminal outcome queries, strict integer scope, exact repaired/healthy responsibility, all physical shard and bitrot proofs, complete S3/HEAD/Range evidence before finalization, ten private-node cleanups and four package purges. No backup restoration was used. Original failure `38200772548` remains unchanged; its vm001 offline root cause is not established by the later pass. From 6b628bf091be67fdc513e788bd76ea3f8de58dfe Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 26 Oct 2026 10:03:29 +0800 Subject: [PATCH 335/343] docs(validation): record native audit and offline runner evidence --- docs/validation-heal-environment-gates.md | 4 ++++ docs/validation-heal-operations.md | 11 +++++++++++ docs/validation-migration-map.md | 20 +++++++++++++++++--- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/validation-heal-environment-gates.md b/docs/validation-heal-environment-gates.md index dce3f3b2..6486284e 100644 --- a/docs/validation-heal-environment-gates.md +++ b/docs/validation-heal-environment-gates.md @@ -6,6 +6,8 @@ The Functional Chain `environment.inspect` entry records the current four Heal h These observations do not certify a clean environment. Independently audit retained validation scenes and the complete shared-workflow census before dispatching another case. A completed workflow's success or failure alone does not prove recovery. The regular case installer remains responsible for stopping the standard service, and its existing namespace/port reservation guards remain authoritative. +When the shared runner cannot start `environment.inspect`, the separate `validation_runner_inspect_only=true` entry inspects GitHub registration and queue metadata on a hosted runner without fleet access. Inspection [40256930895/1](https://github.com/rustfs/rustfs/actions/runs/40256930895) observed `rustfs-smoke-testing` (ID 139653) offline at `2026-10-26T02:00:24Z`. This is not a diagnosis of host or disk health. API failures remain incomplete; restoring registration does not certify retained-scene cleanup or release the existing live admission gates. + 第一批 13 个用例保留全部 recipe、分支和 source assertions，并新增 `environment_cases.py` 的控制流程与断言，接入 validation 调度。它们默认关闭，只产生 `BLOCKED` 结果与未满足原因；默认路径不会连接或修改 fleet。真实执行沿用 auto-testing 的 S3/Admin collector、ACK ledger、候选包绑定、物理故障 oracle 和恢复模式，专用资源操作由部署环境的适配器提供。当前没有把适配器协议或未运行代码计为 canonical PASS。 设置 `AUTO_TESTING_ENABLE_ENV_GATED_HEAL=1` 并通过 `VALIDATION_HEAL_ENVIRONMENT_CAPABILITIES` 指向 JSON 能力清单后，门控会检查全部能力。缺项仍返回 `BLOCKED`。能力齐全后还要用 `VALIDATION_HEAL_ENVIRONMENT_RUNTIME` 指向 runtime JSON；只有候选/checkout/client 摘要、全部拓扑、当前 Heal SSH 节点和适配器摘要校验通过后才会运行分支。未绑定适配器仍为 `BLOCKED`；执行结果需要 source assertion 审核，当前实现不会自动生成 canonical PASS。 @@ -45,6 +47,8 @@ These observations do not certify a clean environment. Independently audit retai I05 与 A09、G01–G06 的精确屏障证据允许使用同一不可变候选源码单独构建的 `cfg(test)` 原生测试执行文件。不得给优化服务候选加入隐藏 hook 冒充原包，也不能把缺少 release hook 本身归为外部环境阻塞。原生子集、完整服务调用和现场最终 ACK/物理检查分别验收；已验收 native 证据不能代替尚缺的 controller 或完整 live 结果。第一批 G07/G08 的 native 分支与 H04 容量锁窗口保留各自未完成条件；外部请求重试或最终对象正确均不能证明精确锁或提交窗口。 +2026-10-26 已独立复核 hosted-only tools [40086026324/1](https://github.com/rustfs/rustfs/actions/runs/40086026324)：候选 `757b751482bb970bc321e92059690a7698554358` 上 10 项精确原生测试和 104 条有序 milestone 全部通过，覆盖 A09、G01–G06 与 I05 的原生子集。固定 helper `002473b` 的 runtime admission 在该候选上仅开放 A09/G05/G06；G01–G04 仍须选择各自审核过的候选及匹配工具，或先完成新的 runtime 源码审核。完整 live 用例和 I05 剩余控制流程均未因此通过，canonical 计数保持 114/158。 + 资源适配器协议使用 stdin/stdout JSON，不调用 shell。runtime 中的 `adapter` 包含绝对可执行路径、SHA-256 和操作 allowlist；`environment_cases.required_operations(case_id)` 给出所需操作。每个请求携带 run ID、原候选 commit、sequence、拓扑/分支 scope、bucket prefix、租约和参数。响应必须回显上述身份、按 `heal_probe.canonical(request)` 的 UTF-8 SHA-256 绑定原请求，并把真实观察写在 `observation`，禁止返回 verdict。资源适配器需独立实现平台 power control、故障层、pool lifecycle 或 KMS/tier 隔离；本 PR 未向当前共享环境部署适配器。 runtime 的 `binding` 使用现有 `heal_probe.validate_binding` 契约，另含 `schemaVersion: 1`、`provider: auto-testing`、`sshNodes`、`rcBinary`、`topologies`、`capabilities`、`adapter` 和对应资源字段。F08 需要 `releasePair`，F09 需要 `hostResourceId`，I01 需要 `faultDeviceId/ioLatencyMs`，G07/I06 需要 `kmsKeyId/tierEndpoint/tenantId/dependencyNamespaceId`，G08 需要 `replicaEndpoint`，J05 需要 `soakLimits`。H6 专用主机应由适配器按 `hostResourceId` 独占绑定；SSH fleet 仍记录当前 auto-testing Heal 节点。secret 通过环境传入，适配器诊断会脱敏，`redactionEnvironmentVariables` 可指定额外 secret 环境变量名。 diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index bf89597a..071d6691 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -34,6 +34,17 @@ python scripts/validation_heal/matrix.py plan --selection A01,A04,C01,C02,C04,J0 live 命令使用当前 auto-testing 的 `smoke-testing` runner、`RUSTFS_NODES` 和 `RUSTFS_SSH_USER`，复用 `rustfs_heal_test.sh` 的 Heal 节点。运行前取得与 Functional Chain 一致的共享节点并发锁。H1–H6 是测试拓扑要求，不是另一组 SSH 主机；不得按源仓库历史主机名或 IP 选择机器。 +### Runner 无法接单时的只读诊断 + +RustFS Functional Chain 提供 `validation_runner_inspect_only=true` 入口。它在 `ubuntu-latest` 上读取 GitHub 的 runner 注册信息与活动任务，使用独立并发组，不占用或替换测试 fleet 队列。一次请求只能选择该模式，不同时传入 candidate、tools、case 或 recovery 参数；使用包含此入口的已审核 workflow ref： + +```bash +gh workflow run rustfs-functional-chain.yml --repo rustfs/rustfs \ + --ref "$VALIDATION_DRIVER_REF" -f validation_runner_inspect_only=true +``` + +下载该次 `validation-runner-inspection--` artifact，核对 `workflowCommit`、`runId`、`runAttempt`、`observedAt`、`complete` 和 `runnerAvailability`。`offline`、`online-busy`、`online-idle`、`unregistered` 是 GitHub 注册状态的不同观察；API 拒绝、分页超过上限或其他错误保留 `complete=false` 与失败阶段，不能解释为空闲。报告不连接节点、不重启服务、不清理文件、不取消 workflow，也不证明环境已恢复或共享 workflow 全局空闲。runner 服务恢复后，仍需重新执行资源检查及原有现场/并发门控。 + 用 `matrix.py plan` 冻结当前 SSH 节点与所有适用操作分支，再根据本次只读 preflight 填写私有 `lab.json`。计划中的 `verified:false` 不可直接改成资源验证通过。H4 需要至少 24 个已分配独立盘端点；H6 需要独占主机及平台事件/恢复绑定。缺少资源的分支保留为 BLOCKED。 填写内容须包含： diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index f823962e..c4be52a2 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -17,8 +17,8 @@ record; it does not replace that retained failure or replay all 158 cases on one candidate. All 158 canonical IDs are retained. Independent complete live acceptance is -**114/158**. The current ledger has 20 retained live failures, seven required -native barriers, four orchestration/evidence blockers and 13 environment cases +**114/158**. The current ledger has 20 retained live failures, seven pending +native/live race acceptances, four orchestration/evidence blockers and 13 environment cases disabled by default. The active acceptance backlog is 31 cases. Counts are case-level canonical results across immutable candidates; diagnostic, native, isolation and partial results do not increment PASS. @@ -40,12 +40,26 @@ isolation and partial results do not increment PASS. | --- | ---: | --- | | Independently accepted complete live PASS | 114 | Preserve original provenance and complete results. | | Retained live failures with existing executors | 20 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | -| Required native barriers | 7 | A09 and G01–G06 need exact candidate-bound server lock/version/commit barriers. | +| Native and live race acceptance pending | 7 | A09 and G01–G06 require independently verified native groups, reviewed candidate-specific runtime admission and complete H1/H2 live evidence. | | Orchestration or evidence blocked | 4 | C11, F07, I05 and J02 need their remaining native or full live evidence closure. | | Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 31 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. +### Native prerequisites and runtime admission (2026-10-26) + +Hosted-only tools [40086026324/1](https://github.com/rustfs/rustfs/actions/runs/40086026324) completed successfully on candidate `757b751482bb970bc321e92059690a7698554358`, auto-testing `002473b51f5ceeccadedf7325a1b1691e4c71dcd` and driver `b1f7fb276941d7128b5ddbbda9c3e6536931c89d`. Its immutable package is [40081956660/1](https://github.com/rustfs/rustfs/actions/runs/40081956660), SHA-256 `1a52387a8cbdef586be2616b02298f351aec4ac2d5c0f7a76d094409c89c0b3c`. Independent read-only verification checked the candidate source tree, registered source-file digests, Cargo.lock, executable digest, ten exact nonignored passing test invocations, all 104 ordered milestones and the inspector identity. The original artifacts and audit are retained in the migration evidence store. No downloaded native executable was run during this audit. + +The registry exposes eight native prerequisite groups: A09, G01–G06 and part of I05. This is separate from runtime admission. The pinned helper admits A09, G05 and G06 on this candidate; it intentionally rejects G01–G04 on the same source before bootstrap because their runtime candidate contracts remain pinned to other reviewed sources. G01–G03 may use their reviewed version candidate `bd0671ba781d8ebf509f4f03533228dfb44496f1`, and G04 uses marker candidate `8380466f3a27c3c3ef4115084178a0ab99a78f32`, each with its own exact package and matching helper/tools. Alternatively, a separate source review and matching tools build must precede extending runtime admission. Do not infer admission from registry membership or mix artifacts from different candidates or helpers. + +All ten native tests pass, but complete live acceptance remains **114/158**. A09/G01–G06 still need their complete source branches, SDK ACK ledgers, physical checks and clean finalization on H1/H2. I05's native subset does not supply its missing full timeout/stale-generation controller and live closure. This tools run neither accessed the fleet nor adds canonical PASS. + +### Hosted runner inspection (2026-10-26) + +Read-only [40256930895/1](https://github.com/rustfs/rustfs/actions/runs/40256930895) completed on a GitHub-hosted runner with driver `31c9d85645c1a8051562983f18631098b9bc34e7`. At `2026-10-26T02:00:24Z`, GitHub reported the only matching `smoke-testing` runner, `rustfs-smoke-testing` (ID 139653), as **offline**, with `busy=false`. The report retained two queued fleet jobs, including the Functional Chain summary and the health collector, plus pending workflow runs without jobs. This establishes the runner registration state at observation time; it does not diagnose the host's service, disk, network or power state. + +The new `validation_runner_inspect_only=true` dispatch path uses a separate workflow concurrency group, reads only GitHub API metadata and uploads an incomplete report if API access fails. It rejects simultaneous candidate, build, fleet, tools or recovery inputs; all fleet jobs were skipped in the real inspection. No queued workflow was cancelled or replaced, no test-node connection was made, and `mutated`, `cleanStateCertified`, `sharedWorkflowIdleCertified` and `canonicalCasePassClaimed` remain false. Live resumption requires a working maintenance entry to restore this runner, followed by a new resource inspection and the existing retained-scene and shared-workflow admission checks. + ### B08 complete live acceptance [B08 run 39915022106/1](https://github.com/rustfs/rustfs/actions/runs/39915022106) is independently accepted on candidate `1a3f038c90704f9ad9693eddcf7d861c9530f13c`, package `39898678364/1`, tools `39914732144/1` and helper `a3d3c62a9f6d29667a1a2b57d13f5cb9e166de3f`. The official 3,463-member archive has SHA-256 `59ddf692cdca495abcd37a1affacf25e4cd3da24c931509bb9b4facc386c2f11`; the independent audit has SHA-256 `63f143d278c5e6a43f0068a31c692cae1064fd2612b54060198c448f52b7e640`. H1 read-present/write-absent and H2 read-present/write-absent plus lock-quorum-absent passed; H1 lock-quorum-absent is explicitly NOT_APPLICABLE under the candidate's in-process local manager. The audit verified all peer inventories, unique PUT absence, physical target restoration, streaming shard proofs and ten cleanup receipts. The prior B08 failure remains retained in the append-only ledger. From ee657fb81e280d910e21ff021ab742359cc4c14a Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 26 Oct 2026 14:20:08 +0800 Subject: [PATCH 336/343] ci(validation): run the frozen G06 source contract regressions --- .github/workflows/validation-catalog-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index cd95e18c..5f9c43e0 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -48,6 +48,7 @@ on: - test_validation_heal_queue_pressure.py - test_validation_heal_nolock.py - test_validation_heal_race.py + - test_validation_heal_g06_source.py - test_validation_heal_environment_gate.py - test_validation_heal_outcome.py - test_validation_heal_outcome_grace.py @@ -118,6 +119,7 @@ on: - test_validation_heal_queue_pressure.py - test_validation_heal_nolock.py - test_validation_heal_race.py + - test_validation_heal_g06_source.py - test_validation_heal_environment_gate.py - test_validation_heal_outcome.py - test_validation_heal_outcome_grace.py @@ -171,7 +173,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py - run: python3 -B -m unittest -v test_validation_heal_incarnation.py test_validation_heal_incarnation_replay.py - run: python3 -B -m unittest -v test_validation_heal_incarnation_native.py - - run: python3 -B -m unittest -v test_validation_heal_race_native.py + - run: python3 -B -m unittest -v test_validation_heal_race_native.py test_validation_heal_g06_source.py - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh - run: bash test_validation_scanner_usage.sh From 8efc57d7ed89531a8f63cc4f98d06dafa2361499 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 26 Oct 2026 15:31:40 +0800 Subject: [PATCH 337/343] fix(validation): isolate interrupted COPY after ACK authority --- docs/validation-migration-map.md | 12 + scripts/validation_heal/interrupted_race.py | 212 ++++++++++++++++++ .../validation_heal/recover_environment.py | 8 + test_validation_heal_race_retention.py | 75 +++++++ 4 files changed, 307 insertions(+) create mode 100644 scripts/validation_heal/interrupted_race.py diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index c4be52a2..13d127a9 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -60,6 +60,18 @@ Read-only [40256930895/1](https://github.com/rustfs/rustfs/actions/runs/40256930 The new `validation_runner_inspect_only=true` dispatch path uses a separate workflow concurrency group, reads only GitHub API metadata and uploads an incomplete report if API access fails. It rejects simultaneous candidate, build, fleet, tools or recovery inputs; all fleet jobs were skipped in the real inspection. No queued workflow was cancelled or replaced, no test-node connection was made, and `mutated`, `cleanStateCertified`, `sharedWorkflowIdleCertified` and `canonicalCasePassClaimed` remain false. Live resumption requires a working maintenance entry to restore this runner, followed by a new resource inspection and the existing retained-scene and shared-workflow admission checks. +### Interrupted G06 evidence and recovery boundary (2026-10-26) + +The re-registered `rustfs-smoke-testing` runner (ID 139936) was online in hosted inspections [40277791903](https://github.com/rustfs/rustfs/actions/runs/40277791903) and [40282750869](https://github.com/rustfs/rustfs/actions/runs/40282750869). Resource inspection [40278133051/1](https://github.com/rustfs/rustfs/actions/runs/40278133051) observed the current auto-testing fleet's 16 independent endpoints. Neither inspection certifies global workflow idleness or restores an interrupted scene. + +G06 [40281440070/1](https://github.com/rustfs/rustfs/actions/runs/40281440070) was cancelled during its selected-case step. The only annotation is `The operation was canceled.`; its cause is unconfirmed. The workflow uploaded no original artifact. Read-only collection [40284040202/1](https://github.com/rustfs/rustfs/actions/runs/40284040202) subsequently retained 1,777 original files (38,765,135 bytes), independently checked against every collection-receipt size and SHA-256. Collection provenance remains distinct from the original run; its unfinished result stays `RUNNING/PENDING`, unaccepted. + +H1 `copy-source` finalized as `NOT_OBSERVED` with cleanup PASS. H1 `copy-destination` retained all 258 seed request/ACK pairs and its one COPY pair, complete last-ACK physical authority and 1,028 streamed part proofs. It restarted before final peer S3 inspection, then was interrupted. Its recorded public-call overlap is false. No H2 or later source branch was completed, so this evidence adds no canonical PASS. + +`interrupted_race.py` admits only this bounded H1 COPY boundary: one unchanged unfinished case, one finalized predecessor, complete deterministic SDK journals, recomputed ACK-derived final state and physical/streaming proofs, and no later operation. Recovery checks all five original owner markers before stopping the exact owned unit, then requires the current stopped business scene to match the archived last-ACK scene. Existing same-filesystem private isolation preserves the original bytes, backup files, inodes and credentials without reading credential contents. Drift refuses isolation; no old-generation restoration or SDK replay is permitted. A new recovery receipt never rewrites the original verdict or supplies missing canonical assertions. + +The driver may explicitly select `validation_cleanup_retained_evidence=true` together with an exact failed cleanup run. It verifies that run/attempt and same-runner storage ownership, copies through no-follow file descriptors, checks every original and copied digest and records a new collection identity. This is not an original-run artifact fallback. The ordinary artifact-based recovery path remains the default. + ### B08 complete live acceptance [B08 run 39915022106/1](https://github.com/rustfs/rustfs/actions/runs/39915022106) is independently accepted on candidate `1a3f038c90704f9ad9693eddcf7d861c9530f13c`, package `39898678364/1`, tools `39914732144/1` and helper `a3d3c62a9f6d29667a1a2b57d13f5cb9e166de3f`. The official 3,463-member archive has SHA-256 `59ddf692cdca495abcd37a1affacf25e4cd3da24c931509bb9b4facc386c2f11`; the independent audit has SHA-256 `63f143d278c5e6a43f0068a31c692cae1064fd2612b54060198c448f52b7e640`. H1 read-present/write-absent and H2 read-present/write-absent plus lock-quorum-absent passed; H1 lock-quorum-absent is explicitly NOT_APPLICABLE under the candidate's in-process local manager. The audit verified all peer inventories, unique PUT absence, physical target restoration, streaming shard proofs and ten cleanup receipts. The prior B08 failure remains retained in the append-only ledger. diff --git a/scripts/validation_heal/interrupted_race.py b/scripts/validation_heal/interrupted_race.py new file mode 100644 index 00000000..d92c1ce6 --- /dev/null +++ b/scripts/validation_heal/interrupted_race.py @@ -0,0 +1,212 @@ +"""Isolate only G06's interrupted H1 COPY after complete last-ACK authority. + +This is an environment recovery boundary, never a completed canonical result. +Original request/ACK bytes and the stopped scene must still agree. No backup +restoration, new SDK request or inferred acknowledgement is permitted. +""" + +from collections import Counter +import hashlib +import json +import re +from types import SimpleNamespace + +try: + from . import race, race_retention + from .heal_probe import require + from .healthy import physical_content +except ImportError: + import race + import race_retention + from heal_probe import require + from healthy import physical_content + + +CASE = "heal.race-copy-and-metadata-update" + + +class Replay: + """Consume every original SDK operation once, without invoking a client.""" + + def __init__(self, folder, run_id, bucket): + self.folder, self.run_id, self.sequence = folder, run_id, 0 + self.probe = SimpleNamespace(binding={"bucketPrefix": bucket}) + self.files = {} + require(folder.is_dir() and all(re.fullmatch(r"[0-9]{6}-(request|ack)\.json", p.name) + and p.is_file() and not p.is_symlink() for p in folder.iterdir()), + "interrupted COPY journal has an unknown member") + + def call(self, operation, params, body=None): + self.sequence += 1 + request_path = self.folder / f"{self.sequence:06d}-request.json" + ack_path = self.folder / f"{self.sequence:06d}-ack.json" + request_raw, ack_raw = request_path.read_bytes(), ack_path.read_bytes() + require(max(len(request_raw), len(ack_raw)) <= 1024**2, "interrupted COPY journal exceeds its bound") + request, ack = json.loads(request_raw), json.loads(ack_raw) + observation = ack.get("observation") or {} + require(request.get("schemaVersion") == ack.get("schemaVersion") == 2 + and request.get("runId") == ack.get("runId") == self.run_id + and request.get("sequence") == ack.get("sequence") == self.sequence + and request.get("operation") == observation.get("operation") == operation + and request.get("params") == observation.get("params") == params + and ack.get("requestFile") == request_path.name + and ack.get("requestSha256") == hashlib.sha256(request_raw).hexdigest() + and ack.get("operationCheck") == "PASS" + and observation.get("acknowledged") is True and observation.get("resultUnknown") is False + and observation.get("clientCallInvoked") is True, + "interrupted COPY has an unresolved, changed or foreign SDK operation") + require(request.get("payload") == (None if body is None else { + "size": len(body), "sha256": hashlib.sha256(body).hexdigest()}), + "interrupted COPY deterministic seed body differs") + for path, raw in ((request_path, request_raw), (ack_path, ack_raw)): + self.files[self.folder.name + "/" + path.name] = hashlib.sha256(raw).hexdigest() + return observation["response"] + + def finish(self, count): + require(self.sequence == count and len(list(self.folder.iterdir())) == 2 * count, + "interrupted COPY journal has an omitted or additional SDK operation") + return self.files + + +def archived_plan(artifact, failed_run, candidate): + results = list(artifact.glob("*/run/*/cases/*.json")) + require(len(results) == 1 and results[0].name == CASE + ".json", "interrupted COPY requires one unmixed case") + raw = results[0].read_bytes() + result = json.loads(raw) + require(result.get("case_id") == CASE and result.get("verdict") == "RUNNING" + and result.get("cleanup_status") == "PENDING" and not result.get("provenance"), + "interrupted COPY original result must remain unfinished") + require(json.loads((results[0].parents[3] / "candidate.json").read_text()) == candidate, + "interrupted COPY candidate differs") + evidence = results[0].parents[1] / "evidence" / CASE + require(not (evidence / "healthy-summary.json").exists() + and {p.relative_to(evidence).as_posix() for p in evidence.glob("*/*") if p.is_dir()} + == {"H1/copy-source", "H1/copy-destination"}, + "interrupted COPY cannot omit another unfinished branch") + previous = evidence / "H1/copy-source" + def one(folder, label): + paths = list(folder.glob("*-" + label + ".json")) + require(len(paths) == 1, "interrupted COPY lacks one original " + label) + return json.loads(paths[0].read_text()) + prior = one(previous, "branch-result") + cleanup = one(previous, "cleanup") + require(prior.get("topology") == "H1" and prior.get("branch") == "copy-source" + and prior.get("verdict") in {"PASS", "NOT_OBSERVED", "FAIL"} and prior.get("cleanup") == "PASS" + and cleanup.get("node") == "vm000" and cleanup.get("cleanup") == "PASS" + and cleanup.get("unitStopped") is True and cleanup.get("rootsRemoved") is True + and one(previous, "recovery") == {"cleanup": "PASS", "errors": []}, + "interrupted COPY prior shared namespace was not finalized") + folder = evidence / "H1/copy-destination" + require({p.name for p in folder.iterdir() if p.is_dir()} == {"seed-ACKs", "foreground-ACKs"} + and all(p.is_dir() or re.fullmatch(r"[0-9]{5}-.+\.json", p.name) for p in folder.iterdir()), + "interrupted COPY has an unknown operation") + files = sorted(folder.glob("*.json")) + labels = Counter(p.name[6:-5] for p in files) + expected = {"binding": 1, "inspector-build": 1, "source-contract": 1, "preflight": 1, + "reserved": 1, "inspector-upload": 1, "started": 3, "deployed-topology": 1, + "topology-proof": 1, "complete-baseline-S3": 1, "frozen-foreground-plan": 1, + "quiescent": 2, "complete-baseline-physical": 1, "baseline-streaming": 9, + "exact-missing-part": 257, "complete-fault-physical": 1, "fault-batch-proof": 1, + "race-io-cap-intent": 2, "race-io-process": 2, "race-io-cap-applied": 2, + "attribution-before-START": 1, "prepared-Admin-request": 1, "original-Admin-start": 1, + "target-missing-before-foreground": 1, "read-repair-overlap-proof": 1, + "public-call-overlap-proof": 1, "ACK-derived-final-manifest": 1, + "product-physical-before-GET-or-rescue": 1, "last-ACK-physical-proof": 1, + "final-streaming": 9, "obsolete-target-fault-retired": 1, "integrity-fleet-attestation": 1} + statuses = labels.pop("original-token-status", 0) + require(labels == Counter(expected) and 3 <= statuses <= 2000 + and all(p.name.startswith(f"{i:05d}-") for i, p in enumerate(files, 1)) + and files[-1].name.endswith("-started.json") + and files[-2].name.endswith("-obsolete-target-fault-retired.json"), + "interrupted COPY is outside its reviewed after-authority boundary") + binding = one(folder, "binding") + require(re.fullmatch(rf"gh-{failed_run}-[1-9][0-9]*-h1", binding.get("runId", "")) + and binding.get("candidateCommit") == candidate["source_sha"] + and binding.get("candidatePackageSha256") == candidate["package_sha256"], + "interrupted COPY run or candidate binding differs") + reservation = one(folder, "reserved") + require(reservation.get("node") == "vm000" and reservation.get("reserved") is True, + "interrupted COPY original owner is incomplete") + owner = reservation["owner"] + run_id, bucket = binding["runId"], binding["bucketPrefix"] + seed = Replay(folder / "seed-ACKs", run_id, bucket) + expected_seed = race.seed_fixture(seed, bucket, "copy-destination") + ack_files = seed.finish(258) + baseline = one(folder, "complete-baseline-S3") + race.verify_seed(expected_seed, baseline) + plan = race.operation_plan(bucket, "copy-destination", expected_seed) + for step in plan: + step["branch"] = "copy-destination" + require(one(folder, "frozen-foreground-plan") == plan, "interrupted COPY changed its frozen operation") + writer = Replay(folder / "foreground-ACKs", run_id, bucket) + responses = [writer.call(step["operation"], step["params"]) for step in plan] + ack_files.update(writer.finish(1)) + final = race.final_inventory(baseline, "copy-destination", plan, responses) + require(final == one(folder, "ACK-derived-final-manifest"), "interrupted COPY final ACK authority differs") + before, after = (one(folder, label) for label in ( + "complete-baseline-physical", "product-physical-before-GET-or-rescue")) + for scene in (before, after): + require(set(scene) == {"vm000"} and scene["vm000"].get("quiescent") is True + and type(scene["vm000"].get("pid")) is int and scene["vm000"]["pid"] == 0 + and scene["vm000"].get("binarySha256") == binding["candidateBinarySha256"] + and scene["vm000"].get("hostname") == owner["hostname"] + and scene["vm000"].get("unit") == owner["unit"], + "interrupted COPY scene is not the stopped exact owner") + mapped = race.map_fixture(before, bucket, expected_seed) + survivors, proof = race.physical_transition(before, after, bucket, mapped, final, "copy-destination") + require(proof == one(folder, "last-ACK-physical-proof"), "interrupted COPY physical authority differs") + # Reuse the complete streaming oracle against archived receipts, without SSH. + class ArchivedParts: + nodes = ["vm000"] + def __init__(self): + self.proofs = iter(json.loads(p.read_text()) for p in sorted(folder.glob("*-final-streaming.json"))) + def remote(self, node, action, *, parts): + observed = next(self.proofs) + require(node == observed.get("node") and action == "verify-parts" + and [{k: p[k] for k in ("disk", "relativePath", "versionId", "partNumber")} + for p in observed["parts"]] == parts, "interrupted COPY streaming batch differs") + return observed + def record(self, label, value): + return value + stream = ArchivedParts() + race.verify_part_manifest(stream, list(survivors.values()), "final-streaming", count=257) + require(next(stream.proofs, None) is None, "interrupted COPY omitted a streaming batch") + retired = one(folder, "obsolete-target-fault-retired") + require(retired.get("oldBytesRestored") is False and retired.get("backupRetainedUntilCleanup") is True + and json.loads(files[-1].read_text()).get("vm000", {}).get("unit") == owner["unit"] + and json.loads(files[-1].read_text()).get("vm000", {}).get("started") is True, + "interrupted COPY retirement or final start differs") + isolation = {"kind": "unresolved-g06-race", "runId": run_id, "nodes": ["vm000"], "bucket": bucket, + "owners": {"vm000": owner}, "candidateCommit": candidate["source_sha"], + "harnessCommit": binding["harnessCommit"], "binarySha256": binding["candidateBinarySha256"], + "businessSceneSha256": {"vm000": race_retention.digest(physical_content(after)["vm000"])}, + "ackFilesSha256": race_retention.digest(ack_files)} + race_retention.validate(isolation) + return [{"kind": "interrupted-g06-after-authority", "isolation": isolation, + "runId": run_id, "nodes": ["vm000"], "originalCaseSha256": hashlib.sha256(raw).hexdigest()}] + + +def isolate(fleet, plan): + isolation = plan["isolation"] + race_retention.validate(isolation) + require(plan["kind"] == "interrupted-g06-after-authority" and fleet.run_id == isolation["runId"] + and list(fleet.nodes) == isolation["nodes"], "interrupted COPY recovery fleet differs") + states = [fleet.record("interrupted-COPY-namespace", {"node": node, + **fleet.remote(node, "namespace-state")}) for node in fleet.nodes] + for state in states: + if state.get("retained", {}).get("state") or any(state.get("retained", {}).get("roots", [])): + continue # Existing durable isolation intents are revalidated by isolate(). + owner_sha = race_retention.digest(isolation["owners"][state["node"]]) + entries = [state["state"], *state["roots"]] + require(len(entries) == 5 and all(entry.get("exists") is True + and entry.get("ownerEvidence", {}).get("uid") == 0 + and entry.get("ownerEvidence", {}).get("canonicalSha256") == owner_sha for entry in entries), + "interrupted COPY original owner drifted; no service stopped") + for state in states: + if not (state.get("retained", {}).get("state") or any(state.get("retained", {}).get("roots", []))): + stopped = fleet.record("interrupted-COPY-stop", {"node": state["node"], + **fleet.remote(state["node"], "stop")}) + require(stopped.get("unit") == isolation["owners"][state["node"]]["unit"] + and stopped.get("quiescent") is True, "interrupted COPY exact service did not stop") + return {**race_retention.isolate(fleet, isolation), "originalCaseSha256": plan["originalCaseSha256"], + "interruptedAfterLastACKAuthority": True, "canonicalCaseComplete": False} diff --git a/scripts/validation_heal/recover_environment.py b/scripts/validation_heal/recover_environment.py index 08666547..a0857716 100644 --- a/scripts/validation_heal/recover_environment.py +++ b/scripts/validation_heal/recover_environment.py @@ -17,6 +17,7 @@ from .network_retention import archived_plan as network_retention_plan, isolate as isolate_network_scene from .race_retention import archived_plan as race_retention_plan, isolate as isolate_race_scene from .timeout_retention import archived_plan as timeout_retention_plan + from .interrupted_race import archived_plan as interrupted_race_plan, isolate as isolate_interrupted_race except ImportError: from healthy import Fleet, physical_content from heal_probe import require, save @@ -25,6 +26,7 @@ from network_retention import archived_plan as network_retention_plan, isolate as isolate_network_scene from race_retention import archived_plan as race_retention_plan, isolate as isolate_race_scene from timeout_retention import archived_plan as timeout_retention_plan + from interrupted_race import archived_plan as interrupted_race_plan, isolate as isolate_interrupted_race def quota_recovery_plan(artifact, failed_run, candidate): @@ -151,6 +153,9 @@ def network_recovery_plan(artifact, failed_run, candidate): def recovery_plan(artifact, failed_run, candidate): require(type(failed_run) is int and failed_run > 0, "exact failed workflow run required") + unfinished_copy = list(artifact.glob("*/run/heal.race-copy-and-metadata-update/cases/*.json")) + if any(json.loads(path.read_text()).get("verdict") == "RUNNING" for path in unfinished_copy): + return interrupted_race_plan(artifact, failed_run, candidate) if list(artifact.glob("*/run/heal.lock-timeout-and-caller-abort/cases/*.json")): return timeout_retention_plan(artifact, failed_run, candidate) if (list(artifact.glob("*/run/heal.race-unversioned-overwrite/cases/*.json")) @@ -656,6 +661,9 @@ def main(): if plan.get("buckets"): fleet.snapshot_buckets = plan["buckets"] fleet.record("archived-recovery-plan", {key: value for key, value in plan.items() if key != "before"}) + if plan.get("kind") == "interrupted-g06-after-authority": + complete.append({"failedRunId": args.failed_run_id, **isolate_interrupted_race(fleet, plan)}) + continue if plan.get("kind") in {"unresolved-g01-race", "unresolved-g02-race", "unresolved-a09-race", "unresolved-g04-race", "unresolved-g05-race", "unresolved-g06-race", "unresolved-i05-timeout", "unresolved-i05-successor"}: complete.append({"failedRunId": args.failed_run_id, **isolate_race_scene(fleet, plan)}) continue diff --git a/test_validation_heal_race_retention.py b/test_validation_heal_race_retention.py index d7a59b7f..63ce64af 100644 --- a/test_validation_heal_race_retention.py +++ b/test_validation_heal_race_retention.py @@ -12,9 +12,84 @@ from unittest.mock import patch from scripts.validation_heal import race_retention as controller, retained_scene as retained, recover_environment as recovery +from scripts.validation_heal import interrupted_race import test_validation_heal_retained_scene as retained_tests +class InterruptedCopyTests(unittest.TestCase): + def journal(self, directory, mutation=None): + folder = Path(directory) / 'foreground-ACKs'; folder.mkdir() + params = {'Bucket': 'rv-heal-0123456789abcdef', 'Key': 'destination', + 'CopySource': {'Bucket': 'rv-heal-0123456789abcdef', 'Key': 'source'}} + request = {'schemaVersion': 2, 'runId': 'gh-12345-1-h1', 'sequence': 1, + 'operation': 'copy_object', 'params': params} + raw = json.dumps(request).encode() + ack = {'schemaVersion': 2, 'runId': request['runId'], 'sequence': 1, + 'requestFile': '000001-request.json', 'requestSha256': hashlib.sha256(raw).hexdigest(), + 'operationCheck': 'PASS', 'observation': {'operation': 'copy_object', 'params': params, + 'acknowledged': True, 'resultUnknown': False, 'clientCallInvoked': True, + 'response': {'CopyObjectResult': {'ETag': '"committed"'}}}} + if mutation == 'unknown': ack['observation']['resultUnknown'] = True + elif mutation == 'refusal': ack['observation']['acknowledged'] = False + elif mutation == 'request-hash': ack['requestSha256'] = '0' * 64 + elif mutation == 'foreign-run': ack['runId'] = 'gh-99999-1-h1' + elif mutation == 'foreign-operation': ack['observation']['operation'] = 'delete_object' + (folder / '000001-request.json').write_bytes(raw) + if mutation != 'missing-ack': (folder / '000001-ack.json').write_text(json.dumps(ack)) + return folder, request + + def test_only_exact_ack_can_be_replayed_without_a_client(self): + with tempfile.TemporaryDirectory() as directory: + folder, request = self.journal(directory) + reader = interrupted_race.Replay(folder, request['runId'], request['params']['Bucket']) + response = reader.call(request['operation'], request['params']) + self.assertEqual(response, {'CopyObjectResult': {'ETag': '"committed"'}}) + self.assertEqual(len(reader.finish(1)), 2) + (folder / '000002-request.json').write_text('{}') + with self.assertRaises(ValueError): reader.finish(1) + + def test_unknown_refused_changed_or_missing_ack_never_becomes_authority(self): + for mutation in ('unknown', 'refusal', 'request-hash', 'foreign-run', 'foreign-operation', 'missing-ack'): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + folder, request = self.journal(directory, mutation) + reader = interrupted_race.Replay(folder, request['runId'], request['params']['Bucket']) + with self.assertRaises((ValueError, FileNotFoundError)): + reader.call(request['operation'], request['params']) + + def test_foreign_owner_refuses_before_stop_and_scene_drift_refuses_before_move(self): + for mutation in ('foreign-owner', 'scene-drift'): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + helper, state, roots, node, request, scene = RaceRetentionTests().scene(directory) + owner = request['expectedOwner'] + isolation = {'kind': 'unresolved-g06-race', 'runId': owner['runId'], + 'nodes': ['vm000', 'vm001', 'vm002', 'vm003'], 'bucket': owner['bucket'], + 'candidateCommit': '1' * 40, 'harnessCommit': '2' * 40, + 'binarySha256': owner['binarySha256'], 'ackFilesSha256': '3' * 64, + 'owners': {}, 'businessSceneSha256': {}} + for i, name in enumerate(isolation['nodes']): + isolation['owners'][name] = {**owner, 'hostname': 'host-' + str(i)} + isolation['businessSceneSha256'][name] = request['expectedOriginalSceneSha256'] + calls = [] + def remote(name, action, **params): + calls.append(action) + if action == 'namespace-state': + marker = {'exists': True, 'ownerEvidence': {'uid': 0, + 'canonicalSha256': controller.digest(isolation['owners'][name])}} + if mutation == 'foreign-owner' and name == 'vm003': marker['ownerEvidence']['canonicalSha256'] = '0' * 64 + return {'state': copy.deepcopy(marker), 'roots': [copy.deepcopy(marker) for _ in range(4)]} + if action == 'stop': return {'unit': owner['unit'], 'quiescent': True} + self.assertEqual(action, 'retained-scene-audit') + raise ValueError('current business scene differs') + fleet = SimpleNamespace(nodes=isolation['nodes'], run_id=owner['runId'], + remote=remote, record=lambda label, value: value) + plan = {'kind': 'interrupted-g06-after-authority', 'isolation': isolation, + 'originalCaseSha256': '4' * 64} + with self.assertRaises(ValueError): interrupted_race.isolate(fleet, plan) + self.assertNotIn('retained-scene-isolate', calls) + self.assertNotIn('recover-fault', calls) + if mutation == 'foreign-owner': self.assertNotIn('stop', calls) + + class RaceRetentionTests(unittest.TestCase): def scene(self, directory): helper = retained_tests.RetainedSceneTests() From eae7ac2b04807ad609cd4615dbeb2e9063067b9e Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 26 Oct 2026 16:12:53 +0800 Subject: [PATCH 338/343] fix(validation): observe G06 from actual SDK dispatch --- docs/validation-migration-map.md | 4 ++ scripts/validation_heal/race.py | 35 ++++++++++++- test_validation_heal_g06_source.py | 79 ++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 2 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 13d127a9..4b868e19 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -72,6 +72,10 @@ H1 `copy-source` finalized as `NOT_OBSERVED` with cleanup PASS. H1 `copy-destina The driver may explicitly select `validation_cleanup_retained_evidence=true` together with an exact failed cleanup run. It verifies that run/attempt and same-runner storage ownership, copies through no-follow file descriptors, checks every original and copied digest and records a new collection identity. This is not an original-run artifact fallback. The ordinary artifact-based recovery path remains the default. +Isolation [40286516720/1](https://github.com/rustfs/rustfs/actions/runs/40286516720) completed with helper `8efc57d7ed89531a8f63cc4f98d06dafa2361499` and driver `3e337bad97a0263fa663f7909b33f247d6f76f17`. Independent audit verified official archive SHA-256 `11f691947bcbf6d707f0ed04ff7bfd625eeb4994221ad0f690bcc8c63cf43645`, all 1,777 original file bytes, the unchanged unfinished-case digest, stopped owned service and identical pre/move/post baselines for all five trees. All archive parents are root/root 0700 on their original filesystems. The disposition is ISOLATED; no namespace was deleted, no backup restored and no canonical verdict rewritten. Follow-up inspection [40287577171/1](https://github.com/rustfs/rustfs/actions/runs/40287577171) observed 16 eligible endpoints, inactive standard services, no port 9000 listeners and no active validation service. The original failed unit remains recorded; inspection does not certify global idleness. + +The original COPY SDK calls lasted approximately 23 ms and 17 ms. Their nearest running samples began before those calls or finished after them, so the frozen strict interval oracle correctly refused overlap. G06 now schedules observation from botocore's actual `before-send.s3` notification inside the SDK call, after client preparation and durable request intent. The callback only queues a timestamp; it never waits, delays a response, repeats a mutation or alters the SDK call interval. The unchanged oracle still requires an entire running original-token status call strictly inside every acknowledged SDK call. Real local HTTP/SDK tests cover nested running acceptance, finished-task refusal and response completion without any observer. The previous polling scheduler fails the positive regression. This improves observation scheduling without certifying a live race or a server commit fence; a new exact-helper tools bundle and full H1/H2 replay remain required. + ### B08 complete live acceptance [B08 run 39915022106/1](https://github.com/rustfs/rustfs/actions/runs/39915022106) is independently accepted on candidate `1a3f038c90704f9ad9693eddcf7d861c9530f13c`, package `39898678364/1`, tools `39914732144/1` and helper `a3d3c62a9f6d29667a1a2b57d13f5cb9e166de3f`. The official 3,463-member archive has SHA-256 `59ddf692cdca495abcd37a1affacf25e4cd3da24c931509bb9b4facc386c2f11`; the independent audit has SHA-256 `63f143d278c5e6a43f0068a31c692cae1064fd2612b54060198c448f52b7e640`. H1 read-present/write-absent and H2 read-present/write-absent plus lock-quorum-absent passed; H1 lock-quorum-absent is explicitly NOT_APPLICABLE under the candidate's in-process local manager. The audit verified all peer inventories, unique PUT absence, physical target restoration, streaming shard proofs and ten cleanup receipts. The prior B08 failure remains retained in the append-only ledger. diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index 2a1a9f81..02da32b5 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -11,6 +11,7 @@ import json import os from pathlib import Path +from queue import Empty, SimpleQueue import re import secrets import signal @@ -156,6 +157,22 @@ def reference_key(branch): return G06_REFERENCE_KEY if branch in G06_BRANCHES else REFERENCE_KEY +class DispatchProbe(Probe): + """Wake G06 observation from inside the SDK, immediately before sending. + + No wait or delay is added to the SDK call or its response. The notification + schedules observation; only the unchanged recorded intervals prove overlap. + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.dispatched = SimpleQueue() + + def s3_client(self): + client = super().s3_client() + client.meta.events.register("before-send.s3", lambda **_: self.dispatched.put(time.monotonic_ns())) + return client + + class RaceRecorder(Recorder): """Keep unresolved mutations and exact SDK receipts after any assertion.""" def __init__(self, *args, versioned, **kwargs): @@ -647,6 +664,8 @@ def read_versions(recorder, ledger, lock, done): def race(recorder, plan, probe, fleet, path, token, reference=REFERENCE, *, baseline=None): samples = [] + dispatch_times = [] + dispatch = recorder.probe.dispatched if plan and plan[0].get('branch') in G06_BRANCHES else None # Only the worker mutates Recorder state. All Fleet.record writes run on # the controller, so evidence sequence allocation never races. readers = [] @@ -690,8 +709,14 @@ def read_repair(): deadline = time.monotonic() + (90 if version_race else 120) try: while (not worker.done() or version_reads and read_future is not None and not read_future.done()) and time.monotonic() < deadline: + if dispatch is not None: + try: + dispatch_times.append(dispatch.get(timeout=.05)) + except Empty: + continue samples.append(observe(probe, fleet, path, token, reference)) - time.sleep(.025 if version_race else .01) + if dispatch is None: + time.sleep(.025 if version_race else .01) except BaseException: recorder.mutations.poison("status observation failed; drain in-flight mutations only") raise @@ -707,6 +732,11 @@ def read_repair(): require(len(acks) == len(plan) and all(row.get("operationCheck") == "PASS" for row in acks), "G01/G03 lacks every durable foreground ACK") overlaps = [any(sample_overlap(ack, sample, path, token, reference) for sample in samples) for ack in acks] + if dispatch is not None: + fleet.record('G06-request-dispatch-observation', {'requestSendMonotonicNs': dispatch_times, + 'statusSamples': len(samples), 'notificationScope': 'botocore before-send.s3; inside SDK call', + 'SDKCallDelayed': False, 'publicOverlapProved': False, 'nativeFenceProved': False, + 'canonicalCasePassClaimed': False}) read_complete = True reader_overlap = True if version_reads: @@ -820,7 +850,8 @@ def run_branch(args, topology, nodes, branch): target = next(target for target in targets if target["key"] == current_target_key) if (target["node"], target["disk"], target["relativePath"]) in files(witness): raise WindowNotObserved("G01/G03 old target already repaired before foreground mutations") - writer = RaceRecorder(probe, directory / "foreground-ACKs", + writer_probe = DispatchProbe(binding) if branch in G06_BRANCHES else probe + writer = RaceRecorder(writer_probe, directory / "foreground-ACKs", versioned=branch in (*G02_BRANCHES, *G03_BRANCHES)) writer.seen.update((row["key"], row["versionId"]) for row in expected) if branch in G02_BRANCHES: diff --git a/test_validation_heal_g06_source.py b/test_validation_heal_g06_source.py index 15421fa0..4e7d5c23 100644 --- a/test_validation_heal_g06_source.py +++ b/test_validation_heal_g06_source.py @@ -1,5 +1,11 @@ """Frozen DS and COPY direction contracts; synthetic scenes are not live proof.""" import copy +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import tempfile +import threading from types import SimpleNamespace import unittest from unittest.mock import Mock, patch @@ -9,6 +15,79 @@ from scripts.validation_heal import race +class G06DispatchTests(unittest.TestCase): + def server(self, *, summary='running', hold_copy=True): + copy_seen, sampled = threading.Event(), threading.Event() + failures = [] + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + def reply(self, raw, kind): + self.send_response(200); self.send_header('Content-Type', kind) + self.send_header('Content-Length', str(len(raw))); self.end_headers(); self.wfile.write(raw) + def do_PUT(self): + copy_seen.set() + if hold_copy and not sampled.wait(3): failures.append('COPY response did not observe Admin completion') + self.reply(b'"copy-ack"2026-10-26T00:00:00Z', 'application/xml') + def do_POST(self): + if not copy_seen.wait(3): failures.append('Admin request preceded an actual COPY') + self.reply(json.dumps({'summary': summary, 'settings': race.COPY_REFERENCE['effective'], + 'outcome': {'counters': {'processed': 1}}}).encode(), 'application/json') + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + worker = threading.Thread(target=server.serve_forever, daemon=True); worker.start() + self.addCleanup(server.server_close); self.addCleanup(server.shutdown) + binding = {'schemaVersion': 1, 'runId': 'gh-12345-1-h1', 'jumpHostOnly': True, + 'exclusiveHealService': True, 'leaseExpiresAt': (datetime.now(timezone.utc) + timedelta(minutes=5)).isoformat(), + 'bucketPrefix': 'rv-heal-0123456789abcdef', 'endpoints': {'node1': f'http://127.0.0.1:{server.server_port}'}, + 'defaultEndpoint': 'node1', 'candidateCommit': race.COPY_CANDIDATE, 'harnessCommit': '2' * 40, + 'candidatePackageSha256': '3' * 64, 'candidateBinarySha256': '4' * 64, 'rcBinarySha256': '5' * 64, + 'maxInventoryObjects': 512, 'maxInventoryBytes': 512 * 1024**2} + return binding, sampled, failures + + def test_real_sdk_dispatch_schedules_original_status_and_keeps_strict_interval_oracle(self): + for summary in ('running', 'finished'): + with self.subTest(summary=summary), tempfile.TemporaryDirectory() as directory: + binding, sampled, failures = self.server(summary=summary) + writer_probe = race.DispatchProbe(binding, credential_pair=('test-access', 'test-secret')) + admin_probe = race.Probe(binding, credential_pair=('test-access', 'test-secret')) + original_admin = admin_probe.admin + def admin(*args): + result = original_admin(*args) + sampled.set() # Server-only fixture barrier; production notification never waits. + return result + admin_probe.admin = admin + writer = race.RaceRecorder(writer_probe, Path(directory) / 'foreground-ACKs', versioned=False) + plan = race.operation_plan(binding['bucketPrefix'], 'copy-destination', []) + plan[0]['branch'] = 'copy-destination' + records = {} + fleet = SimpleNamespace(record=lambda label, value: records.setdefault(label, value)) + responses, accepted = race.race(writer, plan, admin_probe, fleet, + '/rustfs/admin/v3/heal/' + binding['bucketPrefix'], 'original-token', race.COPY_REFERENCE) + self.assertEqual(responses[0]['CopyObjectResult']['ETag'], '"copy-ack"') + self.assertEqual(accepted, summary == 'running') + self.assertFalse(failures) + ack = json.loads(next(writer.output.glob('*-ack.json')).read_text()) + dispatch = records['G06-request-dispatch-observation'] + sent = dispatch['requestSendMonotonicNs'][0] + observation = ack['observation']; status = records['original-token-status'] + self.assertLess(observation['callStartedMonotonicNs'], sent) + self.assertLess(sent, status['callStartedMonotonicNs']) + self.assertLess(status['callFinishedMonotonicNs'], observation['callFinishedMonotonicNs']) + self.assertFalse(dispatch['SDKCallDelayed'] or dispatch['publicOverlapProved'] or dispatch['canonicalCasePassClaimed']) + + def test_notification_never_waits_for_an_observer_or_changes_an_ack(self): + binding, _, failures = self.server(hold_copy=False) + probe = race.DispatchProbe(binding, credential_pair=('test-access', 'test-secret')) + plan = race.operation_plan(binding['bucketPrefix'], 'copy-source', []) + result = probe.s3(plan[0]['operation'], plan[0]['params']) + self.assertTrue(result['acknowledged']) + sent = probe.dispatched.get_nowait() + self.assertLess(result['callStartedMonotonicNs'], sent) + self.assertLess(sent, result['callFinishedMonotonicNs']) + self.assertEqual(result['response']['CopyObjectResult']['ETag'], '"copy-ack"') + self.assertFalse(failures) + + class G06SourceTests(unittest.TestCase): bucket = 'rv-heal-0123456789abcdef' From 1f8bcd7387f5b4b3055e3193cafac5672674371e Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 26 Oct 2026 16:37:08 +0800 Subject: [PATCH 339/343] fix(validation): preserve the source G06 tag overlap contract --- docs/validation-migration-map.md | 4 ++- scripts/validation_heal/race.py | 8 +++++- test_validation_heal_g06_source.py | 43 ++++++++++++++++++++++++++---- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 4b868e19..910900c6 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -74,7 +74,9 @@ The driver may explicitly select `validation_cleanup_retained_evidence=true` tog Isolation [40286516720/1](https://github.com/rustfs/rustfs/actions/runs/40286516720) completed with helper `8efc57d7ed89531a8f63cc4f98d06dafa2361499` and driver `3e337bad97a0263fa663f7909b33f247d6f76f17`. Independent audit verified official archive SHA-256 `11f691947bcbf6d707f0ed04ff7bfd625eeb4994221ad0f690bcc8c63cf43645`, all 1,777 original file bytes, the unchanged unfinished-case digest, stopped owned service and identical pre/move/post baselines for all five trees. All archive parents are root/root 0700 on their original filesystems. The disposition is ISOLATED; no namespace was deleted, no backup restored and no canonical verdict rewritten. Follow-up inspection [40287577171/1](https://github.com/rustfs/rustfs/actions/runs/40287577171) observed 16 eligible endpoints, inactive standard services, no port 9000 listeners and no active validation service. The original failed unit remains recorded; inspection does not certify global idleness. -The original COPY SDK calls lasted approximately 23 ms and 17 ms. Their nearest running samples began before those calls or finished after them, so the frozen strict interval oracle correctly refused overlap. G06 now schedules observation from botocore's actual `before-send.s3` notification inside the SDK call, after client preparation and durable request intent. The callback only queues a timestamp; it never waits, delays a response, repeats a mutation or alters the SDK call interval. The unchanged oracle still requires an entire running original-token status call strictly inside every acknowledged SDK call. Real local HTTP/SDK tests cover nested running acceptance, finished-task refusal and response completion without any observer. The previous polling scheduler fails the positive regression. This improves observation scheduling without certifying a live race or a server commit fence; a new exact-helper tools bundle and full H1/H2 replay remain required. +The original COPY SDK calls lasted approximately 23 ms and 17 ms. Their nearest running samples began before those calls or finished after them, so the frozen strict interval oracle correctly refused overlap. G06 now schedules observation from botocore's actual `before-send.s3` notification inside the SDK call, after client preparation and durable request intent. The callback only queues a timestamp; it never waits, delays a response, repeats a mutation or alters the SDK call interval. A qualifying original-token running sample must still fit strictly inside one actual acknowledged SDK call. Real local HTTP/SDK tests cover nested running acceptance, finished-task refusal and response completion without any observer. The previous polling scheduler fails the positive regression. This improves observation scheduling without certifying a live race or a server commit fence; a new exact-helper tools bundle and full H1/H2 replay remain required. + +The source's `heal_h1_concurrency_batch.py::observe_race` at clean commit `524eb5b32c3236497fdd8a9271df0494fa3d9974` requires every planned request to have an ACK and returns `bool(matches)` for its nested running samples. The frozen G06 tag branch has exactly 16 distinct serial updates. The migrated controller incorrectly required all 16 calls to overlap. G06 tag aggregation now restores the source's requirement of at least one qualifying call, while retaining all 16 original requests, all 16 ACKs, exact intervals, final tags and full physical/S3 checks. The evidence retains every per-call overlap flag and explicitly records the required aggregation, so no match still yields NOT_OBSERVED. Other race aggregation is unchanged. Real local SDK tests verify one matching tag call plus fifteen nonmatching calls, all 16 durable ACKs and complete refusal when every sample is finished. The preceding migrated helper fails that positive test; no historical verdict is rewritten. ### B08 complete live acceptance diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index 02da32b5..003d1950 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -732,6 +732,10 @@ def read_repair(): require(len(acks) == len(plan) and all(row.get("operationCheck") == "PASS" for row in acks), "G01/G03 lacks every durable foreground ACK") overlaps = [any(sample_overlap(ack, sample, path, token, reference) for sample in samples) for ack in acks] + # The frozen G06 source fixes 16 tag writes and accepts one fully nested + # original-token sample among them; every write still needs its own ACK. + tag_updates = bool(plan and plan[0].get('branch') == 'update-tags') + overlap_complete = any(overlaps) if tag_updates else all(overlaps) if dispatch is not None: fleet.record('G06-request-dispatch-observation', {'requestSendMonotonicNs': dispatch_times, 'statusSamples': len(samples), 'notificationScope': 'botocore before-send.s3; inside SDK call', @@ -751,8 +755,10 @@ def read_repair(): "readCalls": len(readers), "allReadsObserved": bool(readers) if read_branch else True, "nativeFenceProved": False}) fleet.record("public-call-overlap-proof", {"everyMutationOverlapped": all(overlaps), "mutations": overlaps, + "requiredCallOverlapObserved": overlap_complete, + "overlapRequirement": "at-least-one-of-frozen-16-tag-updates" if tag_updates else "every-mutation", "nativeFenceProved": False, "serverCommitOverlapProved": False}) - return responses, all(overlaps) and read_complete and reader_overlap + return responses, overlap_complete and read_complete and reader_overlap def run_branch(args, topology, nodes, branch): diff --git a/test_validation_heal_g06_source.py b/test_validation_heal_g06_source.py index 4e7d5c23..95c7e840 100644 --- a/test_validation_heal_g06_source.py +++ b/test_validation_heal_g06_source.py @@ -16,9 +16,9 @@ class G06DispatchTests(unittest.TestCase): - def server(self, *, summary='running', hold_copy=True): + def server(self, *, summary='running', hold_copy=True, first_only=False): copy_seen, sampled = threading.Event(), threading.Event() - failures = [] + failures, calls = [], [0] class Handler(BaseHTTPRequestHandler): def log_message(self, *args): pass @@ -26,12 +26,17 @@ def reply(self, raw, kind): self.send_response(200); self.send_header('Content-Type', kind) self.send_header('Content-Length', str(len(raw))); self.end_headers(); self.wfile.write(raw) def do_PUT(self): + self.rfile.read(int(self.headers.get('Content-Length', '0'))) + calls[0] += 1 copy_seen.set() - if hold_copy and not sampled.wait(3): failures.append('COPY response did not observe Admin completion') - self.reply(b'"copy-ack"2026-10-26T00:00:00Z', 'application/xml') + if hold_copy and (not first_only or calls[0] == 1) and not sampled.wait(3): + failures.append('SDK response did not observe Admin completion') + self.reply(b'' if '?tagging' in self.path else + b'"copy-ack"2026-10-26T00:00:00Z', 'application/xml') def do_POST(self): if not copy_seen.wait(3): failures.append('Admin request preceded an actual COPY') - self.reply(json.dumps({'summary': summary, 'settings': race.COPY_REFERENCE['effective'], + state = summary if not first_only or calls[0] == 1 else 'finished' + self.reply(json.dumps({'summary': state, 'settings': race.COPY_REFERENCE['effective'], 'outcome': {'counters': {'processed': 1}}}).encode(), 'application/json') server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) worker = threading.Thread(target=server.serve_forever, daemon=True); worker.start() @@ -87,6 +92,34 @@ def test_notification_never_waits_for_an_observer_or_changes_an_ack(self): self.assertEqual(result['response']['CopyObjectResult']['ETag'], '"copy-ack"') self.assertFalse(failures) + def test_frozen_sixteen_tag_writes_require_all_acks_and_one_nested_running_sample(self): + for summary in ('running', 'finished'): + with self.subTest(summary=summary), tempfile.TemporaryDirectory() as directory: + binding, sampled, failures = self.server(summary=summary, first_only=True) + writer_probe = race.DispatchProbe(binding, credential_pair=('test-access', 'test-secret')) + admin_probe = race.Probe(binding, credential_pair=('test-access', 'test-secret')) + original_admin = admin_probe.admin + def admin(*args): + value = original_admin(*args); sampled.set(); return value + admin_probe.admin = admin + writer = race.RaceRecorder(writer_probe, Path(directory) / 'foreground-ACKs', versioned=False) + plan = race.operation_plan(binding['bucketPrefix'], 'update-tags', []) + for step in plan: step['branch'] = 'update-tags' + records = {}; fleet = SimpleNamespace(record=lambda label, value: records.setdefault(label, value)) + responses, accepted = race.race(writer, plan, admin_probe, fleet, + '/rustfs/admin/v3/heal/' + binding['bucketPrefix'], 'original-token', race.COPY_REFERENCE) + self.assertEqual(len(responses), 16) + self.assertEqual(len(list(writer.output.glob('*-request.json'))), 16) + acks = [json.loads(p.read_text()) for p in writer.output.glob('*-ack.json')] + self.assertEqual(len(acks), 16) + self.assertTrue(all(row['operationCheck'] == 'PASS' for row in acks)) + self.assertEqual(accepted, summary == 'running') + proof = records['public-call-overlap-proof'] + self.assertEqual(proof['overlapRequirement'], 'at-least-one-of-frozen-16-tag-updates') + self.assertEqual(proof['requiredCallOverlapObserved'], summary == 'running') + self.assertFalse(proof['everyMutationOverlapped']) + self.assertFalse(failures) + class G06SourceTests(unittest.TestCase): bucket = 'rv-heal-0123456789abcdef' From 7094c16a5a5cf3b0f818c7be463f3bc0f8fd7645 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 26 Oct 2026 18:29:50 +0800 Subject: [PATCH 340/343] fix(validation): preserve race role evidence and status observation --- .../workflows/validation-catalog-tests.yml | 1 + docs/validation-heal-operations.md | 27 +++ docs/validation-migration-map.md | 6 +- scripts/validation_heal/fleet_node.py | 22 +++ scripts/validation_heal/race.py | 155 ++++++++++++++---- scripts/validation_heal/race_retention.py | 5 +- test_validation_heal_endpoint_resolution.py | 96 +++++++++++ test_validation_heal_g02_source.py | 11 +- test_validation_heal_g06_source.py | 46 +++++- test_validation_heal_race.py | 145 ++++++++++++++++ test_validation_heal_race_io.py | 3 +- test_validation_heal_race_physical.py | 1 + test_validation_heal_race_retention.py | 22 +++ 13 files changed, 494 insertions(+), 46 deletions(-) create mode 100644 test_validation_heal_endpoint_resolution.py diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 5f9c43e0..1a5d2639 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -174,6 +174,7 @@ jobs: - run: python3 -B -m unittest -v test_validation_heal_incarnation.py test_validation_heal_incarnation_replay.py - run: python3 -B -m unittest -v test_validation_heal_incarnation_native.py - run: python3 -B -m unittest -v test_validation_heal_race_native.py test_validation_heal_g06_source.py + - run: python3 -B -m unittest -v test_validation_heal_g02_source.py test_validation_heal_endpoint_resolution.py - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh - run: bash test_validation_scanner_usage.sh diff --git a/docs/validation-heal-operations.md b/docs/validation-heal-operations.md index 071d6691..71247e0e 100644 --- a/docs/validation-heal-operations.md +++ b/docs/validation-heal-operations.md @@ -693,6 +693,33 @@ Source `757b751482bb970bc321e92059690a7698554358` adds two collector-only groups The G06 public adapter uses the existing DS workload: all 257 original keys, including Chinese, spaces, literal `%2F`, actual slashes and parent-like prefixes. The observed missing key is `中文/文件/0252.bin`; its companion is `literal%2Fslash/0254.bin`. `copy-source` copies from the observed key into the companion; `copy-destination` copies in the reverse direction. Default COPY expectations inherit the source payload, content properties, metadata and tags. Both directions verify the unchanged source, exact destination coding-index shard bytes and a distinct destination generation before GET, rejecting retained old destination directories. Registry, fixture and local transition tests do not establish public overlap or complete canonical PASS. +G06 reuses `KeepAliveStatusProbe` only while observing the original token and +foreground operation. Existing readiness polls warm the same direct connection; +the controller still starts each foreground observation from the real SDK send +notification. Every connection attempt, actual socket identity, response and +phase remains in `G06-status-transport`. A transport error poisons the session +without an implicit retry; leaving the session closes its connection. The strict +same-clock nested interval, complete ACK ledger and finished-task refusal remain +unchanged. No delay is injected into the SDK call. + +Before race allocation and each explicit service restart, the runner and every +participating node resolve the existing `rustfs-node1` alias (H1) or all four +aliases (H2). `endpoint-resolution` retains bounded, single-attempt `getent` +results and refuses a failed precondition. It never edits hosts files or DNS, +substitutes IP addresses, extends the readiness budget or certifies node identity. +Later topology and physical checks remain required; successful preflight cannot +guarantee that a resolver will remain available during startup. + +G01/G02 retain every serial PUT ACK but follow the frozen source's overlap rule: +at least one strictly nested running status sample per required workload role. +G01's read-repair branch additionally journals every actual GET, verifies its +complete body and joint ETag/length/type/metadata against one original or +ACK-known foreground generation, and requires a reader overlap. Zero reads, +unknown results and mixed generations cannot pass. G02 requires all six +ACK-known historical versions and a separate reader overlap in its read branch. +Sampling continues until an in-flight reader drains after the writer. These +conditions preserve the source workload; they do not supply live acceptance. + ### G03 serial deletion authority G03's frozen `VersionLedger.history` determines the new current version after an explicit DELETE. The public S3 inventory is UUID-sorted and cannot supply that order. Each original foreground plan now carries the three seed ACK version IDs in order; the final oracle rejects missing, duplicate, foreign or differently selected identities, removes exactly the acknowledged version and promotes the last retained seed ACK. The physical oracle permits only that `isLatest` change while preserving every other retained descriptor, coding-index byte and data directory. Every original drive must contain exactly the retained version/latest population, with no extra marker or duplicate version. Focused tests use nonmonotonic UUIDs, both delete branches and corrupted physical populations. This source adaptation adds no native or live acceptance. diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 910900c6..331146e4 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -74,7 +74,11 @@ The driver may explicitly select `validation_cleanup_retained_evidence=true` tog Isolation [40286516720/1](https://github.com/rustfs/rustfs/actions/runs/40286516720) completed with helper `8efc57d7ed89531a8f63cc4f98d06dafa2361499` and driver `3e337bad97a0263fa663f7909b33f247d6f76f17`. Independent audit verified official archive SHA-256 `11f691947bcbf6d707f0ed04ff7bfd625eeb4994221ad0f690bcc8c63cf43645`, all 1,777 original file bytes, the unchanged unfinished-case digest, stopped owned service and identical pre/move/post baselines for all five trees. All archive parents are root/root 0700 on their original filesystems. The disposition is ISOLATED; no namespace was deleted, no backup restored and no canonical verdict rewritten. Follow-up inspection [40287577171/1](https://github.com/rustfs/rustfs/actions/runs/40287577171) observed 16 eligible endpoints, inactive standard services, no port 9000 listeners and no active validation service. The original failed unit remains recorded; inspection does not certify global idleness. -The original COPY SDK calls lasted approximately 23 ms and 17 ms. Their nearest running samples began before those calls or finished after them, so the frozen strict interval oracle correctly refused overlap. G06 now schedules observation from botocore's actual `before-send.s3` notification inside the SDK call, after client preparation and durable request intent. The callback only queues a timestamp; it never waits, delays a response, repeats a mutation or alters the SDK call interval. A qualifying original-token running sample must still fit strictly inside one actual acknowledged SDK call. Real local HTTP/SDK tests cover nested running acceptance, finished-task refusal and response completion without any observer. The previous polling scheduler fails the positive regression. This improves observation scheduling without certifying a live race or a server commit fence; a new exact-helper tools bundle and full H1/H2 replay remain required. +The original COPY SDK calls lasted approximately 23 ms and 17 ms. Their nearest running samples began before those calls or finished after them, so the frozen strict interval oracle correctly refused overlap. G06 schedules observation from botocore's actual `before-send.s3` notification inside the SDK call, after client preparation and durable request intent. The callback only queues a timestamp; it never waits, delays a response, repeats a mutation or alters the SDK call interval. A qualifying original-token running sample must still fit strictly inside one actual acknowledged SDK call. Real local HTTP/SDK tests cover nested running acceptance, finished-task refusal and response completion without any observer. The previous polling scheduler fails the positive regression. + +Complete G06 replay [40299213418/1](https://github.com/rustfs/rustfs/actions/runs/40299213418), pinned to helper `1f8bcd7387f5b4b3055e3193cafac5672674371e`, retained all eight public branches and uploaded its official artifact. Five branches passed; H1 `copy-source` remained `NOT_OBSERVED`, and H2's two COPY branches failed deployment before fixture writes because all peers could not resolve `rustfs-node1`. The nearest H1 status completed 0.575 ms after the SDK call; the strict oracle correctly refused it. Independent replay with the executing helper recomputed all six seeded branches' deterministic journals, original seed properties/tags, final manifests, physical transitions, complete streaming receipts, full peer S3 inventories, terminal outcomes and public overlap. All 20 cleanup receipts match the original reserved hosts. The case remains FAIL, adds no canonical PASS and does not establish an external disk deletion. + +The next executor reuses the existing same-token status transport for G06, preserving its strict overlap oracle, and records read-only endpoint resolution before allocation and restarts. G01/G02 separately restore the frozen source's per-role overlap rule while requiring every PUT ACK and complete concurrent reader evidence. New exact-helper tools and complete H1/H2 live evidence remain required for acceptance. The source's `heal_h1_concurrency_batch.py::observe_race` at clean commit `524eb5b32c3236497fdd8a9271df0494fa3d9974` requires every planned request to have an ACK and returns `bool(matches)` for its nested running samples. The frozen G06 tag branch has exactly 16 distinct serial updates. The migrated controller incorrectly required all 16 calls to overlap. G06 tag aggregation now restores the source's requirement of at least one qualifying call, while retaining all 16 original requests, all 16 ACKs, exact intervals, final tags and full physical/S3 checks. The evidence retains every per-call overlap flag and explicitly records the required aggregation, so no match still yields NOT_OBSERVED. Other race aggregation is unchanged. Real local SDK tests verify one matching tag call plus fifteen nonmatching calls, all 16 durable ACKs and complete refusal when every sample is finished. The preceding migrated helper fails that positive test; no historical verdict is rewritten. diff --git a/scripts/validation_heal/fleet_node.py b/scripts/validation_heal/fleet_node.py index e751d36b..549e4fb8 100644 --- a/scripts/validation_heal/fleet_node.py +++ b/scripts/validation_heal/fleet_node.py @@ -8,6 +8,7 @@ from __future__ import annotations import hashlib +import ipaddress import base64 import json import os @@ -598,6 +599,26 @@ def resource_limits(profile): "runtimeMaxSeconds": 1800 if profile == "standard" else 2700} +def endpoint_resolution(request): + """Observe the existing resolver only; never install hosts entries or retry.""" + require(set(request) == {'runId', 'action'} and request['action'] == 'endpoint-resolution' + and isinstance(request['runId'], str) and RUN_PATTERN.fullmatch(request['runId']), + 'endpoint resolution accepts no host, path or mutation options') + hosts = ['rustfs-node1'] if request['runId'].endswith('-h1') else [f'rustfs-node{i}' for i in range(1, 5)] + entries = {} + for host in hosts: + try: + result = subprocess.run(['getent', 'ahosts', host], capture_output=True, text=True, timeout=10) + addresses = sorted({str(ipaddress.ip_address(line.split()[0])) for line in result.stdout.splitlines() if line.split()}) + entries[host] = {'addresses': addresses, 'resolved': result.returncode == 0 and bool(addresses), + 'returnCode': result.returncode} + except (OSError, ValueError, subprocess.TimeoutExpired) as exc: + entries[host] = {'addresses': [], 'resolved': False, 'failureType': type(exc).__name__} + return {'hostname': socket.gethostname(), 'entries': entries, 'readOnly': True, 'mutated': False, + 'resolved': all(row['resolved'] for row in entries.values()), + 'hostIdentityCertified': False, 'canonicalPassClaimed': False} + + def preflight(request): state, roots = paths(request["runId"]) require(not state.exists() and all(not root.exists() for root in roots), "run namespace already exists") @@ -2114,6 +2135,7 @@ def main(request): actions["version-race-backups"] = lambda value: globals()["mpu_race_backups_backend"].run(value, globals()) actions["process-clock"] = process_clock actions["environment-capabilities"] = environment_capabilities + actions['endpoint-resolution'] = endpoint_resolution actions["replacement-admission-preflight"] = replacement_admission_preflight for action in ("retained-scene-audit", "retained-scene-isolate"): actions[action] = retained_scene_operation diff --git a/scripts/validation_heal/race.py b/scripts/validation_heal/race.py index 003d1950..6ec68576 100644 --- a/scripts/validation_heal/race.py +++ b/scripts/validation_heal/race.py @@ -5,6 +5,7 @@ import argparse import copy +from contextlib import contextmanager from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta, timezone import hashlib @@ -34,6 +35,8 @@ from .race_native_evidence import verify as verify_native from .race_io import set_fleet as set_race_io_cap from .mpu_race_backups import verify_versions as verify_version_backups + from .status_keepalive import KeepAliveStatusProbe + from .fleet_node import endpoint_resolution except ImportError: from heal_probe import Probe, canonical, compare_inventories, payload, require, save from heal_workload import Recorder, seed as seed_workload, verify_seed @@ -47,6 +50,8 @@ from race_native_evidence import verify as verify_native from race_io import set_fleet as set_race_io_cap from mpu_race_backups import verify_versions as verify_version_backups + from status_keepalive import KeepAliveStatusProbe + from fleet_node import endpoint_resolution CASE_IDS = ("heal.race-unversioned-overwrite", "heal.race-versioned-put", "heal.race-version-delete", "heal.race-markers-and-batch-delete", "heal.race-multipart-complete-abort", "heal.race-copy-and-metadata-update") @@ -570,6 +575,31 @@ def observe(probe, fleet, path, token, reference=REFERENCE): return value +def verify_endpoint_resolution(fleet): + """Fail before allocation/start if an existing fleet alias cannot resolve.""" + request = {'runId': fleet.run_id, 'action': 'endpoint-resolution'} + runner = endpoint_resolution(request) + with ThreadPoolExecutor(max_workers=len(fleet.nodes)) as pool: + peers = dict(zip(fleet.nodes, pool.map(lambda node: fleet.remote(node, 'endpoint-resolution'), fleet.nodes), strict=True)) + observed = fleet.record('endpoint-resolution', {'runner': runner, 'peers': peers}) + for entry in [observed['runner'], *observed['peers'].values()]: + require(entry.get('readOnly') is True and entry.get('mutated') is False + and entry.get('resolved') is True, + 'Heal endpoint name resolution failed before allocation/start; inspect endpoint-resolution evidence') + + +@contextmanager +def foreground_status_session(probe, fleet, branch, path, token): + if branch not in G06_BRANCHES: + yield + return + with probe.status_session(token, path) as transport: + try: + yield + finally: + fleet.record('G06-status-transport', transport) + + def wait_running(probe, fleet, path, token, total, reference=REFERENCE): deadline = time.monotonic() + 30 while True: @@ -662,6 +692,51 @@ def read_versions(recorder, ledger, lock, done): return proofs +def read_unversioned(recorder, params, done): + """Retain every actual G01 GET request/ACK independently of the PUT worker.""" + proofs = [] + deadline = time.monotonic() + 90 + for cycle in range(512): + if done.is_set() or time.monotonic() >= deadline: + break + recorder.call('get_object', params, identity={'readCycle': cycle}) + proofs.append(json.loads((recorder.output / f'{recorder.sequence:06d}-ack.json').read_text())) + time.sleep(.01) + return proofs + + +def verify_unversioned_reads(readers, baseline, plan, responses): + """A concurrent unversioned GET may see only one complete ACK-known generation.""" + require(baseline is not None and len(plan) == len(responses) == 3 and len(readers) <= 512, + 'G01 reader lacks its complete bounded original/foreground ACK model') + params = {'Bucket': plan[0]['params']['Bucket'], 'Key': plan[0]['params']['Key']} + originals = [entry for entry in baseline['entries'] if entry['key'] == params['Key']] + require(len(originals) == 1 and originals[0]['versionId'] == 'null' + and originals[0]['deleteMarker'] is False and originals[0]['isLatest'] is True, + 'G01 reader lost its exact original unversioned generation') + known = [{'payload': originals[0]['payload'], 'properties': { + key: originals[0]['properties'][key] for key in ('ETag', 'ContentLength', 'ContentType', 'Metadata')}}] + for step, response in zip(plan, responses, strict=True): + body = payload(step['nonce'], SIZE) + require(step['operation'] == 'put_object' and step['params']['Bucket'] == params['Bucket'] + and step['params']['Key'] == params['Key'] and response.get('ETag') + and response.get('VersionId') in {None, 'null'}, 'G01 reader has a foreign or unresolved PUT generation') + known.append({'payload': {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}, + 'properties': {'ETag': response['ETag'], 'ContentLength': len(body), + 'ContentType': step['params']['ContentType'], 'Metadata': step['params']['Metadata']}}) + for ack in readers: + observed = ack.get('observation') or {}; response = observed.get('response') or {} + require(ack.get('operationCheck') == 'PASS' and observed.get('operation') == 'get_object' + and observed.get('params') == params and observed.get('acknowledged') is True + and observed.get('resultUnknown') is False and not observed.get('clientCleanupError') + and response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 200 + and response.get('VersionId') in {None, 'null'} + and any(observed.get('payload') == entry['payload'] + and all(response.get(key) == value for key, value in entry['properties'].items()) for entry in known), + 'G01 read-repair GET lost its full ACK-known body or mixed generation properties') + return bool(readers) + + def race(recorder, plan, probe, fleet, path, token, reference=REFERENCE, *, baseline=None): samples = [] dispatch_times = [] @@ -674,10 +749,15 @@ def race(recorder, plan, probe, fleet, path, token, reference=REFERENCE, *, base version_race = bool(plan and plan[0].get('branch') in G02_BRANCHES) done = threading.Event(); lock = threading.Lock() ledger = {'bucket': plan[0]['params']['Bucket'], 'entries': []} + read_recorder = None + if read_branch: + require(baseline is not None, 'concurrent reader needs its original complete S3 baseline') if version_reads: - require(baseline is not None, 'G02 historical reads need their complete original S3 baseline') ledger['entries'] = [copy.deepcopy(row) for row in baseline['entries'] if row['key'] == TARGET_KEY] require(len(ledger['entries']) == 3, 'G02 historical reader lacks three original ACK-known versions') + if read_branch: + read_recorder = Recorder(probe, recorder.output.parent / ( + 'historical-read-ACKs' if version_reads else 'read-repair-ACKs')) def committed(responses): added = version_put_entries(baseline, plan, responses) with lock: @@ -692,23 +772,15 @@ def writing(): with ThreadPoolExecutor(max_workers=2 if read_branch else 1) as pool: worker = pool.submit(writing) if version_reads: - read_recorder = Recorder(probe, recorder.output.parent / 'historical-read-ACKs') read_future = pool.submit(read_versions, read_recorder, ledger, lock, done) elif read_branch: - def read_repair(): - deadline = time.monotonic() + 90 - while not worker.done() and time.monotonic() < deadline: - params = {"Bucket": plan[0]["params"]["Bucket"], "Key": plan[0]["params"]["Key"]} - if plan[0].get("readVersionId"): params["VersionId"] = plan[0]["readVersionId"] - observed = probe.s3("get_object", params) - readers.append(observed) - time.sleep(.01) - read_future = pool.submit(read_repair) + params = {'Bucket': plan[0]['params']['Bucket'], 'Key': plan[0]['params']['Key']} + read_future = pool.submit(read_unversioned, read_recorder, params, done) else: read_future = None deadline = time.monotonic() + (90 if version_race else 120) try: - while (not worker.done() or version_reads and read_future is not None and not read_future.done()) and time.monotonic() < deadline: + while (not worker.done() or read_branch and read_future is not None and not read_future.done()) and time.monotonic() < deadline: if dispatch is not None: try: dispatch_times.append(dispatch.get(timeout=.05)) @@ -726,16 +798,15 @@ def read_repair(): plan[0]["runtime"] = runtime if read_future is not None: value = read_future.result(timeout=60) - if version_reads: - readers = value + readers = value acks = [json.loads(path.read_text()) for path in sorted(recorder.output.glob("*-ack.json"))] require(len(acks) == len(plan) and all(row.get("operationCheck") == "PASS" for row in acks), "G01/G03 lacks every durable foreground ACK") overlaps = [any(sample_overlap(ack, sample, path, token, reference) for sample in samples) for ack in acks] - # The frozen G06 source fixes 16 tag writes and accepts one fully nested - # original-token sample among them; every write still needs its own ACK. + # The frozen controllers require one nested sample per workload role, + # while every serial mutation still needs its own durable ACK. tag_updates = bool(plan and plan[0].get('branch') == 'update-tags') - overlap_complete = any(overlaps) if tag_updates else all(overlaps) + overlap_complete = any(overlaps) if dispatch is not None: fleet.record('G06-request-dispatch-observation', {'requestSendMonotonicNs': dispatch_times, 'statusSamples': len(samples), 'notificationScope': 'botocore before-send.s3; inside SDK call', @@ -751,12 +822,22 @@ def read_repair(): fleet.record('G02-historical-read-coverage', {'requiredVersionIds': sorted(required), 'observedVersionIds': sorted(actual), 'complete': read_complete, 'readerCallOverlapObserved': reader_overlap, 'allReadsBoundToDurableACKs': True, 'nativeFenceProved': False, 'canonicalCasePassClaimed': False}) + elif read_branch: + read_complete = verify_unversioned_reads(readers, baseline, plan, responses) + reader_overlap = any(sample_overlap(ack, sample, path, token, reference) for ack in readers for sample in samples) + fleet.record('G01-read-repair-coverage', {'readCalls': len(readers), 'maxReadCalls': 512, + 'complete': read_complete, 'readerCallOverlapObserved': reader_overlap, + 'allReadsBoundToDurableACKs': True, 'fullACKKnownBodiesAndPropertiesVerified': read_complete, + 'nativeFenceProved': False, 'canonicalCasePassClaimed': False}) fleet.record("read-repair-overlap-proof", {"branch": plan[0].get("branch") if plan else None, "readCalls": len(readers), "allReadsObserved": bool(readers) if read_branch else True, "nativeFenceProved": False}) fleet.record("public-call-overlap-proof", {"everyMutationOverlapped": all(overlaps), "mutations": overlaps, - "requiredCallOverlapObserved": overlap_complete, - "overlapRequirement": "at-least-one-of-frozen-16-tag-updates" if tag_updates else "every-mutation", + "writerCallOverlapObserved": overlap_complete, + "readerCallOverlapObserved": reader_overlap if read_branch else None, + "requiredCallOverlapObserved": overlap_complete and reader_overlap, + "allRequiredReadContentVerified": read_complete, + "overlapRequirement": "at-least-one-of-frozen-16-tag-updates" if tag_updates else "at-least-one-per-required-workload-role", "nativeFenceProved": False, "serverCommitOverlapProved": False}) return responses, overlap_complete and read_complete and reader_overlap @@ -799,12 +880,14 @@ def run_branch(args, topology, nodes, branch): current_target_key, current_reference_key = target_key(branch), reference_key(branch) fleet.record("source-contract", {**reference, "branch": branch, "objectCount": OBJECTS, "objectSize": SIZE, "requiredPUTs": 3 if branch in (*G01_BRANCHES, *G02_BRANCHES) else 0, "serverCommitOverlapProved": False}) - probe, active, recorder, writer = Probe(binding), [], None, None + probe = KeepAliveStatusProbe(binding) if branch in G06_BRANCHES else Probe(binding) + active, recorder, writer = [], None, None result = {"topology": topology, "branch": branch, "fixture": "DL" if branch in G01_BRANCHES else "DS" if branch in G06_BRANCHES else "DV", "verdict": "RUNNING", "cleanup": "PENDING", "canonicalCasePassClaimed": False} changed_target_registered, target_authority = None, False try: + verify_endpoint_resolution(fleet) fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record("topology-proof", validate_topology(fleet.snapshot("deployed-topology"), topology)) recorder = RaceRecorder(probe, directory / "seed-ACKs", @@ -838,6 +921,7 @@ def run_branch(args, topology, nodes, branch): expectedSha256=target["sha256"], kind="missing")})) faulted = fleet.snapshot("complete-fault-physical", quiescent=True) fleet.record("fault-batch-proof", validate_missing_batch(before, faulted, targets, receipts)) + verify_endpoint_resolution(fleet) fleet.start() set_race_io_cap(fleet, args.case_id, branch, True, nodes=sorted({target['node'] for target in targets})) require(physical_content(faulted) == physical_content(fleet.snapshot("attribution-before-START")), @@ -848,22 +932,20 @@ def run_branch(args, topology, nodes, branch): token = start.get("response", {}).get("clientToken") if isinstance(start.get("response"), dict) else None require(start.get("httpStatus") == 200 and isinstance(token, str) and token, "G01/G03 admission lacks an original token") active.append((path, token)) - if branch in G02_BRANCHES: - witness = wait_version_window(probe, fleet, path, token, reference, before, targets, bucket) - else: - wait_running(probe, fleet, path, token, len(expected), reference) - witness = fleet.snapshot("target-missing-before-foreground") - target = next(target for target in targets if target["key"] == current_target_key) - if (target["node"], target["disk"], target["relativePath"]) in files(witness): - raise WindowNotObserved("G01/G03 old target already repaired before foreground mutations") - writer_probe = DispatchProbe(binding) if branch in G06_BRANCHES else probe - writer = RaceRecorder(writer_probe, directory / "foreground-ACKs", - versioned=branch in (*G02_BRANCHES, *G03_BRANCHES)) - writer.seen.update((row["key"], row["versionId"]) for row in expected) - if branch in G02_BRANCHES: + with foreground_status_session(probe, fleet, branch, path, token): + if branch in G02_BRANCHES: + witness = wait_version_window(probe, fleet, path, token, reference, before, targets, bucket) + else: + wait_running(probe, fleet, path, token, len(expected), reference) + witness = fleet.snapshot("target-missing-before-foreground") + target = next(target for target in targets if target["key"] == current_target_key) + if (target["node"], target["disk"], target["relativePath"]) in files(witness): + raise WindowNotObserved("G01/G03 old target already repaired before foreground mutations") + writer_probe = DispatchProbe(binding) if branch in G06_BRANCHES else probe + writer = RaceRecorder(writer_probe, directory / "foreground-ACKs", + versioned=branch in (*G02_BRANCHES, *G03_BRANCHES)) + writer.seen.update((row["key"], row["versionId"]) for row in expected) responses, overlapped = race(writer, plan, probe, fleet, path, token, reference, baseline=baseline) - else: - responses, overlapped = race(writer, plan, probe, fleet, path, token, reference) set_race_io_cap(fleet, args.case_id, branch, False, nodes=sorted({target['node'] for target in targets})) final = fleet.record("ACK-derived-final-manifest", final_inventory(baseline, branch, plan, responses)) terminal = wait_terminal(probe, fleet, path, token, reference) @@ -895,6 +977,7 @@ def run_branch(args, topology, nodes, branch): fleet.record("obsolete-target-fault-retired", {"fault": changed_target_registered, "reason": "complete last-ACK physical authority proves the original generation obsolete", "oldBytesRestored": False, "backupRetainedUntilCleanup": True}) + verify_endpoint_resolution(fleet) fleet.start() for endpoint in binding["endpoints"]: current = fleet.record("all-peer-final-S3", {"endpoint": endpoint, **Probe(binding, endpoint).inventory( @@ -911,7 +994,7 @@ def run_branch(args, topology, nodes, branch): require(physical_content(after) == physical_content(fleet.snapshot("physical-after-S3", quiescent=True)), "G01/G03 final GET changed the already-proven physical result") result["terminal"] = terminal_proof(terminal, bucket, baseline, final) - if not overlapped: raise WindowNotObserved("G01/G03 did not observe every required SDK call overlap") + if not overlapped: raise WindowNotObserved("race did not observe a nested SDK call for every required workload role") result.update(verdict="PASS", physical=physical, publicCallOverlapObserved=True, nativeFenceProved=False, serverCommitOverlapProved=False) except (Exception, KeyboardInterrupt) as exc: diff --git a/scripts/validation_heal/race_retention.py b/scripts/validation_heal/race_retention.py index d14a27c4..010dc279 100644 --- a/scripts/validation_heal/race_retention.py +++ b/scripts/validation_heal/race_retention.py @@ -107,7 +107,8 @@ def one(label): multipart = kind == 'unresolved-g05-race' markers = kind == 'unresolved-g04-race' read_journals = (['mpu-read-ACKs', *['final-read-ACKs-node' + str(i) for i in range(1, len(owners) + 1)]] - if multipart else ['negative-read-ACKs-node'+str(i) for i in range(1,len(owners)+1)] if markers else []) + if multipart else ['negative-read-ACKs-node'+str(i) for i in range(1,len(owners)+1)] if markers else + ['read-repair-ACKs'] if case == CASE and row['branch'] == 'overwrite-vs-read-repair' else []) if multipart: require({path.name for path in folder.glob('final-read-ACKs-*')} <= set(read_journals), 'G05 isolation has an unbound final peer read journal') @@ -178,7 +179,7 @@ def one(label): 'G04 isolation SDK request changed its bounded flat key set') allowed = ({'list_multipart_uploads', 'list_parts', 'list_object_versions', 'get_object', 'head_object', 'get_object_tagging'} if multipart and name in read_journals else - {'get_object'} if name == 'historical-read-ACKs' or markers and name in read_journals else + {'get_object'} if name in {'historical-read-ACKs', 'read-repair-ACKs'} or markers and name in read_journals else {'create_bucket','put_bucket_versioning','put_object','delete_object','delete_objects'} if markers else {'create_bucket', 'put_bucket_versioning', 'put_object', 'create_multipart_upload', 'upload_part', 'complete_multipart_upload', 'abort_multipart_upload'} if multipart else diff --git a/test_validation_heal_endpoint_resolution.py b/test_validation_heal_endpoint_resolution.py new file mode 100644 index 00000000..31613085 --- /dev/null +++ b/test_validation_heal_endpoint_resolution.py @@ -0,0 +1,96 @@ +"""Read-only DNS admission for the current auto-testing race endpoints.""" +import subprocess +import json +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +from scripts.validation_heal import fleet_node, race + + +class EndpointResolutionTests(unittest.TestCase): + def request(self, topology='h2'): + return {'runId': 'gh-12345-1-' + topology, 'action': 'endpoint-resolution'} + + def test_observation_resolves_only_existing_fixed_aliases_without_retries_or_mutations(self): + def resolve(args, **kwargs): + self.assertEqual(args[:2], ['getent', 'ahosts']) + self.assertEqual(kwargs['timeout'], 10) + return SimpleNamespace(returncode=0, stdout='10.0.0.1 STREAM\n10.0.0.1 DGRAM\n') + for topology, count in [('h1', 1), ('h2', 4)]: + with self.subTest(topology=topology), patch.object(fleet_node.subprocess, 'run', side_effect=resolve) as run: + observed = fleet_node.endpoint_resolution(self.request(topology)) + self.assertTrue(observed['resolved'] and observed['readOnly']) + self.assertFalse(observed['mutated'] or observed['canonicalPassClaimed'] or observed['hostIdentityCertified']) + self.assertEqual(run.call_count, count) + self.assertEqual(list(observed['entries']), [f'rustfs-node{i}' for i in range(1, count + 1)]) + self.assertTrue(all(row['addresses'] == ['10.0.0.1'] for row in observed['entries'].values())) + + def test_unresolved_empty_timeout_or_malformed_resolver_output_is_retained_and_refused(self): + variants = [SimpleNamespace(returncode=2, stdout=''), SimpleNamespace(returncode=0, stdout=''), + SimpleNamespace(returncode=0, stdout='not-an-ip STREAM\n'), + subprocess.TimeoutExpired(['getent'], 10), OSError('getent unavailable')] + for value in variants: + with self.subTest(value=value), patch.object(fleet_node.subprocess, 'run') as run: + if isinstance(value, Exception): run.side_effect = value + else: run.return_value = value + observed = fleet_node.endpoint_resolution(self.request('h1')) + self.assertFalse(observed['resolved']) + self.assertEqual(run.call_count, 1) + self.assertFalse(observed['entries']['rustfs-node1']['resolved']) + + def test_operator_cannot_supply_a_different_hostname_or_command(self): + with patch.object(fleet_node.subprocess, 'run') as run: + with self.assertRaises(ValueError): + fleet_node.endpoint_resolution({**self.request(), 'host': 'other'}) + with self.assertRaises(ValueError): + fleet_node.endpoint_resolution({**self.request(), 'action': 'reserve'}) + run.assert_not_called() + + def test_controller_retains_all_read_only_results_and_stops_before_reservation(self): + good = {'readOnly': True, 'mutated': False, 'resolved': True} + bad = {**good, 'resolved': False} + for bad_host in ('runner', 'vm002', None): + with self.subTest(bad_host=bad_host), tempfile.TemporaryDirectory() as directory: + fleet = race.Fleet(['vm000','vm001','vm002','vm003'], Path(directory), 'gh-12345-1-h2') + calls = []; records = {} + def remote(node, action): + calls.append((node, action)) + return bad if node == bad_host else good + fleet.remote = remote + fleet.record = lambda label, value: records.setdefault(label, value) + with patch.object(race, 'endpoint_resolution', return_value=bad if bad_host == 'runner' else good): + if bad_host is None: race.verify_endpoint_resolution(fleet) + else: + with self.assertRaisesRegex(ValueError, 'name resolution failed'): + race.verify_endpoint_resolution(fleet) + self.assertEqual(set(calls), {(n,'endpoint-resolution') for n in fleet.nodes}) + self.assertEqual(set(records['endpoint-resolution']['peers']), set(fleet.nodes)) + self.assertEqual(fleet.reserved, []) + + def test_real_case_refuses_dns_precondition_before_deploy_seed_or_fault(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + candidate = root / 'candidate.json' + candidate.write_text(json.dumps({'source_sha': race.VERSION_CANDIDATE,'package_sha256': 'a' * 64})) + rc = root / 'rc'; rc.write_bytes(b'fixture-only') + args = SimpleNamespace(evidence=root / 'evidence',candidate=candidate,rc_bin=rc, + tool_directory=root,testing_commit='b' * 40,binary_sha='c' * 64,case_id='heal.race-unversioned-overwrite') + fleet = Mock(nodes=['vm000'],run_id='gh-12345-1-h1') + fleet.record.side_effect = lambda label,value:value + fleet.remote.return_value = {'readOnly':True,'mutated':False,'resolved':False} + fleet.cleanup.return_value = True + with patch.object(race,'Fleet',return_value=fleet), patch.object(race,'verify_inspector',return_value={}), \ + patch.object(race,'endpoint_resolution',return_value={'readOnly':True,'mutated':False,'resolved':False}), \ + patch.object(race,'seed_fixture') as seed, patch.dict(race.os.environ,{'GITHUB_RUN_ID':'12345'}): + result = race.run_branch(args,'H1',['vm000'],'overwrite-vs-deep') + self.assertEqual(result['verdict'],'FAIL') + self.assertIn('name resolution failed',result['failure']['message']) + fleet.deploy.assert_not_called();seed.assert_not_called() + self.assertTrue(all(call.args[1] == 'endpoint-resolution' for call in fleet.remote.call_args_list)) + + +if __name__ == '__main__': + unittest.main() diff --git a/test_validation_heal_g02_source.py b/test_validation_heal_g02_source.py index 3f4f0085..947ec433 100644 --- a/test_validation_heal_g02_source.py +++ b/test_validation_heal_g02_source.py @@ -313,6 +313,7 @@ def remote(node, action, **params): patch.object(race, 'Probe', return_value=probe), patch.object(race, 'validate_topology'), \ patch.object(race, 'seed_fixture', return_value=baseline['entries']), patch.object(race, 'verify_seed'), \ patch.object(race, 'map_fixture', return_value=mapped), patch.object(race, 'OBJECTS', 4), \ + patch.object(race, 'verify_endpoint_resolution'), \ patch.object(race, 'verify_part_manifest') as verify, \ patch.dict(race.os.environ, {'GITHUB_RUN_ID': '12345', 'GITHUB_RUN_ATTEMPT': '1'}): # The actual binding selects a fresh bucket; its read-only @@ -323,8 +324,9 @@ def remote(node, action, **params): self.assertEqual(verify.call_args.kwargs['count'], len(mapped)) def test_controller_requires_both_complete_six_version_reads_and_original_token_reader_overlap(self): - for complete, reader_overlap in ((True, True), (True, False), (False, True)): - with self.subTest(complete=complete, reader_overlap=reader_overlap), tempfile.TemporaryDirectory() as directory: + for complete, reader_overlap, writer_overlap in ((True, True, True), (True, False, True), + (False, True, True), (True, True, False)): + with self.subTest(complete=complete, reader_overlap=reader_overlap, writer_overlap=writer_overlap), tempfile.TemporaryDirectory() as directory: baseline = self.baseline(); branch = race.G02_BRANCHES[1] plan = race.operation_plan(self.bucket, branch, baseline['entries']) for step in plan: step['branch'] = branch @@ -351,12 +353,13 @@ def read(recorder, ledger, lock, done): probe = SimpleNamespace(binding={'runId': 'gh-12345-1-h1', 'bucketPrefix': self.bucket, 'maxInventoryBytes': 512 * 1024**2}) def overlap(ack, *args): - return ack['observation']['operation'] == 'put_object' or reader_overlap + return (writer_overlap and ack['observation']['params']['Metadata']['nonce'] == '1' + if ack['observation']['operation'] == 'put_object' else reader_overlap) with patch.object(race, 'observe', side_effect=observe), patch.object(race, 'read_versions', side_effect=read), \ patch.object(race, 'sample_overlap', side_effect=overlap): responses, accepted = race.race(writer, plan, probe, fleet, '/original/path', 'original-token', baseline=baseline) self.assertEqual(len(responses), 3) - self.assertEqual(accepted, complete and reader_overlap) + self.assertEqual(accepted, complete and reader_overlap and writer_overlap) self.assertEqual(records['G02-historical-read-coverage']['complete'], complete) self.assertEqual(records['G02-historical-read-coverage']['readerCallOverlapObserved'], reader_overlap) diff --git a/test_validation_heal_g06_source.py b/test_validation_heal_g06_source.py index 95c7e840..f424866f 100644 --- a/test_validation_heal_g06_source.py +++ b/test_validation_heal_g06_source.py @@ -16,10 +16,11 @@ class G06DispatchTests(unittest.TestCase): - def server(self, *, summary='running', hold_copy=True, first_only=False): + def server(self, *, summary='running', hold_copy=True, first_only=False, allow_warmup=False): copy_seen, sampled = threading.Event(), threading.Event() failures, calls = [], [0] class Handler(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' def log_message(self, *args): pass def reply(self, raw, kind): @@ -34,7 +35,8 @@ def do_PUT(self): self.reply(b'' if '?tagging' in self.path else b'"copy-ack"2026-10-26T00:00:00Z', 'application/xml') def do_POST(self): - if not copy_seen.wait(3): failures.append('Admin request preceded an actual COPY') + if not (allow_warmup and calls[0] == 0) and not copy_seen.wait(3): + failures.append('Admin request preceded an actual COPY') state = summary if not first_only or calls[0] == 1 else 'finished' self.reply(json.dumps({'summary': state, 'settings': race.COPY_REFERENCE['effective'], 'outcome': {'counters': {'processed': 1}}}).encode(), 'application/json') @@ -49,6 +51,46 @@ def do_POST(self): 'maxInventoryObjects': 512, 'maxInventoryBytes': 512 * 1024**2} return binding, sampled, failures + def test_g06_reuses_the_original_status_connection_without_changing_sdk_intervals(self): + with tempfile.TemporaryDirectory() as directory: + binding, sampled, failures = self.server(allow_warmup=True) + admin_probe = race.KeepAliveStatusProbe(binding, credential_pair=('test-access', 'test-secret')) + writer = race.RaceRecorder(race.DispatchProbe(binding, credential_pair=('test-access', 'test-secret')), + Path(directory) / 'foreground-ACKs', versioned=False) + plan = race.operation_plan(binding['bucketPrefix'], 'copy-source', []) + plan[0]['branch'] = 'copy-source' + path, token = '/rustfs/admin/v3/heal/' + binding['bucketPrefix'], str(uuid.uuid4()) + records = {}; fleet = SimpleNamespace(record=lambda label, value: records.setdefault(label, value)) + with race.foreground_status_session(admin_probe, fleet, 'copy-source', path, token): + warm = admin_probe.admin(path, [('clientToken', token)], b'') + original = admin_probe.admin + def admin(*args): + result = original(*args); sampled.set(); return result + admin_probe.admin = admin + _, accepted = race.race(writer, plan, admin_probe, fleet, path, token, race.COPY_REFERENCE) + self.assertTrue(accepted) + self.assertFalse(failures) + state = records['G06-status-transport'] + self.assertIsNone(admin_probe._status) + self.assertEqual((state['requests'], state['connectionsOpened']), (2, 1)) + first, second = state['observations'] + self.assertEqual(first['transport']['connectionId'], second['transport']['connectionId']) + self.assertFalse(second['transport']['newConnection']) + ack = json.loads(next(writer.output.glob('*-ack.json')).read_text()) + self.assertLess(warm['callFinishedMonotonicNs'], ack['observation']['callStartedMonotonicNs']) + self.assertTrue(race.sample_overlap(ack, records['original-token-status'], path, token, race.COPY_REFERENCE)) + + def test_status_session_failure_preserves_transport_diagnostics_and_closes_connection(self): + binding, _, _ = self.server(hold_copy=False) + probe = race.KeepAliveStatusProbe(binding, credential_pair=('test-access', 'test-secret')) + records = {}; fleet = SimpleNamespace(record=lambda label, value: records.setdefault(label, value)) + with self.assertRaisesRegex(ValueError, 'fixture failure'): + with race.foreground_status_session(probe, fleet, 'replace-metadata', + '/rustfs/admin/v3/heal/' + binding['bucketPrefix'], str(uuid.uuid4())): + raise ValueError('fixture failure') + self.assertIsNone(probe._status) + self.assertIn('G06-status-transport', records) + def test_real_sdk_dispatch_schedules_original_status_and_keeps_strict_interval_oracle(self): for summary in ('running', 'finished'): with self.subTest(summary=summary), tempfile.TemporaryDirectory() as directory: diff --git a/test_validation_heal_race.py b/test_validation_heal_race.py index f347f11d..d948de58 100644 --- a/test_validation_heal_race.py +++ b/test_validation_heal_race.py @@ -1,12 +1,157 @@ +import copy +import hashlib import json from pathlib import Path import unittest from types import SimpleNamespace from unittest.mock import Mock, patch import tempfile +import threading from scripts.validation_heal import race from scripts.validation_heal.matrix import AUTOMATIC_CASES, select +from scripts.validation_heal.heal_workload import RecordedCallError + + +class UnversionedReaderTests(unittest.TestCase): + bucket = 'rv-heal-0123456789abcdef' + + def model(self): + body = race.payload('original', 31) + baseline = {'entries': [{'key': race.TARGET_KEY, 'versionId': 'null', 'isLatest': True, + 'deleteMarker': False, 'payload': {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}, + 'properties': {'ETag': '"original"', 'ContentLength': len(body), + 'ContentType': 'application/octet-stream', 'Metadata': {'nonce': 'original'}}}]} + plan = race.operation_plan(self.bucket, 'overwrite-vs-read-repair', baseline['entries']) + for step in plan: step['branch'] = 'overwrite-vs-read-repair' + responses = [{'ETag': f'"new-{index}"'} for index in range(3)] + return baseline, plan, responses + + def ack(self, entry): + return {'operationCheck': 'PASS', 'observation': {'operation': 'get_object', + 'params': {'Bucket': self.bucket, 'Key': race.TARGET_KEY}, 'acknowledged': True, + 'resultUnknown': False, 'payload': copy.deepcopy(entry['payload']), + 'response': {**copy.deepcopy(entry['properties']), 'ResponseMetadata': {'HTTPStatusCode': 200}}}} + + def test_reads_accept_only_complete_original_or_ack_known_generations(self): + baseline, plan, responses = self.model() + reads = [self.ack(baseline['entries'][0])] + for step, response in zip(plan, responses, strict=True): + body = race.payload(step['nonce'], 31) + reads.append(self.ack({'payload': {'size': len(body), 'sha256': hashlib.sha256(body).hexdigest()}, + 'properties': {'ETag': response['ETag'], 'ContentLength': len(body), + 'ContentType': step['params']['ContentType'], 'Metadata': step['params']['Metadata']}})) + with patch.object(race, 'SIZE', 31): + self.assertTrue(race.verify_unversioned_reads(reads, baseline, plan, responses)) + self.assertFalse(race.verify_unversioned_reads([], baseline, plan, responses)) + for mutation in ('body', 'etag', 'metadata', 'length', 'version', 'unknown', 'foreign', 'refused'): + with self.subTest(mutation=mutation): + changed = copy.deepcopy(reads); observed = changed[0]['observation'] + if mutation == 'body': observed['payload']['sha256'] = '0' * 64 + elif mutation == 'etag': observed['response']['ETag'] = responses[1]['ETag'] + elif mutation == 'metadata': observed['response']['Metadata'] = plan[1]['params']['Metadata'] + elif mutation == 'length': observed['response']['ContentLength'] += 1 + elif mutation == 'version': observed['response']['VersionId'] = 'foreign-version' + elif mutation == 'unknown': observed['resultUnknown'] = True + elif mutation == 'foreign': observed['params']['Bucket'] += '-other' + else: changed[0]['operationCheck'] = 'FAIL' + with self.assertRaises(ValueError): race.verify_unversioned_reads(changed, baseline, plan, responses) + + def test_real_recorder_retains_read_requests_and_acks_without_retrying_unknowns(self): + for unknown in (False, True): + with self.subTest(unknown=unknown), tempfile.TemporaryDirectory() as directory: + baseline, _, _ = self.model(); done = threading.Event(); calls = [] + def s3(operation, params, body): + calls.append((operation, params)); done.set() + observed = self.ack(baseline['entries'][0])['observation'] + if unknown: observed.update(acknowledged=False, resultUnknown=True, response=None) + return observed + probe = SimpleNamespace(binding={'runId': 'gh-12345-1-h1', 'bucketPrefix': self.bucket, + 'maxInventoryBytes': 512 * 1024**2}, s3=s3) + recorder = race.Recorder(probe, Path(directory) / 'read-repair-ACKs') + params = {'Bucket': self.bucket, 'Key': race.TARGET_KEY} + if unknown: + with self.assertRaises(RecordedCallError): race.read_unversioned(recorder, params, done) + else: + self.assertEqual(len(race.read_unversioned(recorder, params, done)), 1) + self.assertEqual(calls, [('get_object', params)]) + request = recorder.output / '000001-request.json'; ack = json.loads((recorder.output / '000001-ack.json').read_text()) + self.assertEqual(ack['requestSha256'], hashlib.sha256(request.read_bytes()).hexdigest()) + self.assertEqual(ack['operationCheck'], 'FAIL' if unknown else 'PASS') + + def test_controller_requires_a_successful_reader_and_its_own_original_token_window(self): + for present, reader_window, writer_window in ((True, True, True), (False, True, True), + (True, False, True), (True, True, False)): + with self.subTest(present=present, reader_window=reader_window, writer_window=writer_window), tempfile.TemporaryDirectory() as directory: + baseline, plan, responses = self.model() + output = Path(directory) / 'foreground-ACKs'; output.mkdir() + writer = SimpleNamespace(output=output, mutations=SimpleNamespace(poison=Mock())) + ready, sampled = threading.Event(), threading.Event(); records = {} + def foreground(*args, **kwargs): + for index in range(3): + (output / f'{index+1:06d}-ack.json').write_text(json.dumps({'operationCheck': 'PASS', + 'sequence': index + 1, 'observation': {'operation': 'put_object'}})) + self.assertTrue(ready.wait(2)); self.assertTrue(sampled.wait(2)) + return responses, {} + def read(*args): + ready.set(); self.assertTrue(sampled.wait(2)) + return [self.ack(baseline['entries'][0])] if present else [] + def observe(*args): sampled.set(); return {'originalToken': True} + def overlap(ack, *args): + return writer_window and ack.get('sequence') == 2 if ack['observation']['operation'] == 'put_object' else reader_window + probe = SimpleNamespace(binding={'runId': 'gh-12345-1-h1', 'bucketPrefix': self.bucket}) + fleet = SimpleNamespace(record=lambda label, value: records.setdefault(label, value)) + with patch.object(race, 'SIZE', 31), patch.object(race, 'foreground', side_effect=foreground), \ + patch.object(race, 'read_unversioned', side_effect=read), patch.object(race, 'observe', side_effect=observe), \ + patch.object(race, 'sample_overlap', side_effect=overlap): + _, accepted = race.race(writer, plan, probe, fleet, '/original-path', 'original-token', baseline=baseline) + self.assertEqual(accepted, present and reader_window and writer_window) + self.assertEqual(records['G01-read-repair-coverage']['complete'], present) + self.assertEqual(records['public-call-overlap-proof']['requiredCallOverlapObserved'], + present and reader_window and writer_window) + self.assertFalse(records['public-call-overlap-proof']['everyMutationOverlapped']) + + def test_missing_baseline_or_existing_reader_journal_refuses_before_any_put(self): + for stale in (False, True): + with self.subTest(stale=stale), tempfile.TemporaryDirectory() as directory: + baseline, plan, _ = self.model(); output = Path(directory) / 'foreground-ACKs'; output.mkdir() + if stale: (output.parent / 'read-repair-ACKs').mkdir() + writer = SimpleNamespace(output=output, mutations=SimpleNamespace(poison=Mock())) + probe = SimpleNamespace(binding={'runId': 'gh-12345-1-h1', 'bucketPrefix': self.bucket}) + with patch.object(race, 'foreground') as foreground, self.assertRaises((ValueError, FileExistsError)): + race.race(writer, plan, probe, SimpleNamespace(), '/original-path', 'original-token', + baseline=baseline if stale else None) + foreground.assert_not_called() + + def test_status_observation_continues_until_the_inflight_reader_drains(self): + with tempfile.TemporaryDirectory() as directory: + baseline, plan, responses = self.model(); output = Path(directory) / 'foreground-ACKs'; output.mkdir() + writer = SimpleNamespace(output=output, mutations=SimpleNamespace(poison=Mock())) + ready, observed_after_drain = threading.Event(), threading.Event(); shared = {}; samples = [] + def foreground(*args, **kwargs): + for index in range(3): + (output / f'{index+1:06d}-ack.json').write_text(json.dumps({'operationCheck': 'PASS', + 'sequence': index + 1, 'observation': {'operation': 'put_object'}})) + self.assertTrue(ready.wait(2)); return responses, {} + def read(recorder, params, done): + shared['done'] = done; ready.set() + self.assertTrue(done.wait(2)); self.assertTrue(observed_after_drain.wait(2)) + return [self.ack(baseline['entries'][0])] + def observe(*args): + after = shared.get('done') is not None and shared['done'].is_set() + samples.append(after) + if samples.count(True) >= 2: observed_after_drain.set() + return {'afterWriterDrained': after} + def overlap(ack, sample, *args): + return ack.get('sequence') == 2 if ack['observation']['operation'] == 'put_object' else sample['afterWriterDrained'] + probe = SimpleNamespace(binding={'runId': 'gh-12345-1-h1', 'bucketPrefix': self.bucket}) + fleet = SimpleNamespace(record=lambda label, value: value) + with patch.object(race, 'SIZE', 31), patch.object(race, 'foreground', side_effect=foreground), \ + patch.object(race, 'read_unversioned', side_effect=read), patch.object(race, 'observe', side_effect=observe), \ + patch.object(race, 'sample_overlap', side_effect=overlap): + _, accepted = race.race(writer, plan, probe, fleet, '/original-path', 'original-token', baseline=baseline) + self.assertTrue(accepted) + self.assertGreaterEqual(samples.count(True), 2) class RaceTests(unittest.TestCase): diff --git a/test_validation_heal_race_io.py b/test_validation_heal_race_io.py index 9061f837..e18d3bfa 100644 --- a/test_validation_heal_race_io.py +++ b/test_validation_heal_race_io.py @@ -262,7 +262,7 @@ class ReachedTerminalBoundary(BaseException): pass probe.inventory.return_value = {'entries':expected} def cap(f,c,b,enabled,*,nodes): self.assertIs(f,fleet); self.assertEqual(nodes,['vm000']); trace.append('cap-on' if enabled else 'cap-off') - def foreground(*args): trace.append('foreground'); return [],False + def foreground(*args, **kwargs): trace.append('foreground'); return [],False def terminal(*args): trace.append('terminal'); raise ReachedTerminalBoundary() values = {'Fleet':Mock(return_value=fleet),'Probe':Mock(return_value=probe),'verify_inspector':Mock(return_value={}), 'validate_topology':Mock(),'seed_fixture':Mock(return_value=expected),'verify_seed':Mock(), @@ -270,6 +270,7 @@ def terminal(*args): trace.append('terminal'); raise ReachedTerminalBoundary() 'validate_missing_batch':Mock(),'wait_running':Mock(),'race':Mock(side_effect=foreground), 'final_inventory':Mock(return_value={'entries':expected}), 'wait_terminal':Mock(side_effect=terminal),'set_race_io_cap':Mock(side_effect=cap),'OBJECTS':3} + if driver is race: values['verify_endpoint_resolution'] = Mock() for name,value in values.items(): stack.enter_context(patch.object(driver,name,value)) stack.enter_context(patch.dict(race.os.environ,{'GITHUB_RUN_ID':'12345','GITHUB_RUN_ATTEMPT':'1'})) args = SimpleNamespace(evidence=root/'evidence',candidate=candidate,rc_bin=rc,tool_directory=root, diff --git a/test_validation_heal_race_physical.py b/test_validation_heal_race_physical.py index eaf6f38b..dd6e7179 100644 --- a/test_validation_heal_race_physical.py +++ b/test_validation_heal_race_physical.py @@ -112,6 +112,7 @@ def remote(node, action, **params): fleet.remote.side_effect = remote for name, value in {"Fleet": Mock(return_value=fleet), "verify_inspector": Mock(return_value={}), "validate_topology": Mock(return_value={}), "Probe": Mock(), + "KeepAliveStatusProbe": Mock(), "verify_endpoint_resolution": Mock(), "seed_fixture": Mock(return_value=expected), "verify_seed": Mock(), "operation_plan": Mock(return_value=[]), "map_fixture": Mock(return_value=mapped), "verify_part_manifest": Mock(), "OBJECTS": 3}.items(): diff --git a/test_validation_heal_race_retention.py b/test_validation_heal_race_retention.py index 63ce64af..e397ba3f 100644 --- a/test_validation_heal_race_retention.py +++ b/test_validation_heal_race_retention.py @@ -201,6 +201,28 @@ def write(path, value): path.write_text(json.dumps(value)) root = Path(directory) / 'artifact' return root, candidate, folder + def test_g01_read_repair_journal_is_digest_bound_and_cannot_hide_a_mutation(self): + with tempfile.TemporaryDirectory() as directory: + root, candidate, original = self.archive(directory) + folder = original.with_name('overwrite-vs-read-repair'); original.rename(folder) + summary_path = folder.parents[1] / 'healthy-summary.json' + summary = json.loads(summary_path.read_text()); summary['results'][0]['branch'] = folder.name + summary_path.write_text(json.dumps(summary)) + binding = json.loads((folder / '00001-binding.json').read_text()) + journal = folder / 'read-repair-ACKs'; journal.mkdir() + request = {'runId': binding['runId'], 'operation': 'get_object', + 'params': {'Bucket': binding['bucketPrefix'], 'Key': 'objects/0256.bin'}} + (journal / '000001-request.json').write_text(json.dumps(request)) + ack = {'runId': binding['runId'], 'observation': {'operation': 'get_object', 'resultUnknown': True}} + receipt = journal / '000001-ack.json'; receipt.write_text(json.dumps(ack)) + first = recovery.recovery_plan(root, 12345, candidate)[0] + ack['preservedUnknownReceipt'] = True; receipt.write_text(json.dumps(ack)) + second = recovery.recovery_plan(root, 12345, candidate)[0] + self.assertNotEqual(first['ackFilesSha256'], second['ackFilesSha256']) + self.assertEqual(first['businessSceneSha256'], second['businessSceneSha256']) + request['operation'] = 'put_object'; (journal / '000001-request.json').write_text(json.dumps(request)) + with self.assertRaises(ValueError): recovery.recovery_plan(root, 12345, candidate) + def test_archived_unknown_or_explicit_refusal_can_only_select_isolation_with_exact_identity(self): for kind in ('valid', 'unknown', 'refusal', 'candidate', 'binary', 'host', 'live', 'mpu', 'omitted-owner', 'mixed-case', 'extra-failure'): with self.subTest(kind=kind), tempfile.TemporaryDirectory() as directory: From b20b3d802d356b88ad1a7d4a654d248ac960b1ae Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 27 Oct 2026 09:27:06 +0800 Subject: [PATCH 341/343] docs(validation): record G06 canonical acceptance --- docs/validation-migration-map.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 331146e4..5ef0b275 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -17,9 +17,9 @@ record; it does not replace that retained failure or replay all 158 cases on one candidate. All 158 canonical IDs are retained. Independent complete live acceptance is -**114/158**. The current ledger has 20 retained live failures, seven pending +**115/158**. The current ledger has 20 retained live failures, six pending native/live race acceptances, four orchestration/evidence blockers and 13 environment cases -disabled by default. The active acceptance backlog is 31 cases. Counts are +disabled by default. The active acceptance backlog is 30 cases. Counts are case-level canonical results across immutable candidates; diagnostic, native, isolation and partial results do not increment PASS. @@ -38,13 +38,13 @@ isolation and partial results do not increment PASS. | Acceptance state | Cases | Remaining requirement | | --- | ---: | --- | -| Independently accepted complete live PASS | 114 | Preserve original provenance and complete results. | +| Independently accepted complete live PASS | 115 | Preserve original provenance and complete results. | | Retained live failures with existing executors | 20 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | -| Native and live race acceptance pending | 7 | A09 and G01–G06 require independently verified native groups, reviewed candidate-specific runtime admission and complete H1/H2 live evidence. | +| Native and live race acceptance pending | 6 | A09 and G01–G05 require independently verified native groups, reviewed candidate-specific runtime admission and complete H1/H2 live evidence. | | Orchestration or evidence blocked | 4 | C11, F07, I05 and J02 need their remaining native or full live evidence closure. | | Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | -The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 31 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. +The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 30 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. ### Native prerequisites and runtime admission (2026-10-26) @@ -52,7 +52,7 @@ Hosted-only tools [40086026324/1](https://github.com/rustfs/rustfs/actions/runs/ The registry exposes eight native prerequisite groups: A09, G01–G06 and part of I05. This is separate from runtime admission. The pinned helper admits A09, G05 and G06 on this candidate; it intentionally rejects G01–G04 on the same source before bootstrap because their runtime candidate contracts remain pinned to other reviewed sources. G01–G03 may use their reviewed version candidate `bd0671ba781d8ebf509f4f03533228dfb44496f1`, and G04 uses marker candidate `8380466f3a27c3c3ef4115084178a0ab99a78f32`, each with its own exact package and matching helper/tools. Alternatively, a separate source review and matching tools build must precede extending runtime admission. Do not infer admission from registry membership or mix artifacts from different candidates or helpers. -All ten native tests pass, but complete live acceptance remains **114/158**. A09/G01–G06 still need their complete source branches, SDK ACK ledgers, physical checks and clean finalization on H1/H2. I05's native subset does not supply its missing full timeout/stale-generation controller and live closure. This tools run neither accessed the fleet nor adds canonical PASS. +All ten native tests pass. G06 subsequently passed all eight H1/H2 public branches in [40427815826/1](https://github.com/rustfs/rustfs/actions/runs/40427815826); an independent audit verified its ACK, physical, peer S3, overlap and 20 cleanup receipts, adding one canonical PASS. A09 and G01–G05 still need their complete source branches and clean finalization; I05's native subset does not supply its missing full timeout/stale-generation controller and live closure. ### Hosted runner inspection (2026-10-26) @@ -76,7 +76,7 @@ Isolation [40286516720/1](https://github.com/rustfs/rustfs/actions/runs/40286516 The original COPY SDK calls lasted approximately 23 ms and 17 ms. Their nearest running samples began before those calls or finished after them, so the frozen strict interval oracle correctly refused overlap. G06 schedules observation from botocore's actual `before-send.s3` notification inside the SDK call, after client preparation and durable request intent. The callback only queues a timestamp; it never waits, delays a response, repeats a mutation or alters the SDK call interval. A qualifying original-token running sample must still fit strictly inside one actual acknowledged SDK call. Real local HTTP/SDK tests cover nested running acceptance, finished-task refusal and response completion without any observer. The previous polling scheduler fails the positive regression. -Complete G06 replay [40299213418/1](https://github.com/rustfs/rustfs/actions/runs/40299213418), pinned to helper `1f8bcd7387f5b4b3055e3193cafac5672674371e`, retained all eight public branches and uploaded its official artifact. Five branches passed; H1 `copy-source` remained `NOT_OBSERVED`, and H2's two COPY branches failed deployment before fixture writes because all peers could not resolve `rustfs-node1`. The nearest H1 status completed 0.575 ms after the SDK call; the strict oracle correctly refused it. Independent replay with the executing helper recomputed all six seeded branches' deterministic journals, original seed properties/tags, final manifests, physical transitions, complete streaming receipts, full peer S3 inventories, terminal outcomes and public overlap. All 20 cleanup receipts match the original reserved hosts. The case remains FAIL, adds no canonical PASS and does not establish an external disk deletion. +Complete G06 replay [40427815826/1](https://github.com/rustfs/rustfs/actions/runs/40427815826), pinned to helper `7094c16`, passed all eight H1/H2 public branches. Independent replay with the executing helper recomputed all six seeded branches' deterministic journals, original seed properties/tags, final manifests, physical transitions, complete streaming receipts, full peer S3 inventories, terminal outcomes and public overlap. All 20 cleanup receipts match the reserved hosts. G06 is now canonical PASS; the earlier failed run [40299213418/1](https://github.com/rustfs/rustfs/actions/runs/40299213418) remains retained unchanged. The next executor reuses the existing same-token status transport for G06, preserving its strict overlap oracle, and records read-only endpoint resolution before allocation and restarts. G01/G02 separately restore the frozen source's per-role overlap rule while requiring every PUT ACK and complete concurrent reader evidence. New exact-helper tools and complete H1/H2 live evidence remain required for acceptance. From 0f5a4cf2ec0a1696374cd3dd9be1ddf8cd93453e Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 28 Oct 2026 14:07:28 +0800 Subject: [PATCH 342/343] docs(validation): record G02 and candidate E2E status --- docs/validation-migration-map.md | 60 +++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 5ef0b275..9ce13635 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -9,7 +9,7 @@ live run proves each source assertion on the required topology. ## Current acceptance and remaining work -Count reconciliation (2026-10-23): the append-only ledger +Count reconciliation (2026-10-23; updated 2026-10-28): the append-only ledger `canonical-migration-ledger-20261023-b08-39915022106.json` follows the B01 ledger and attaches the complete B08 live acceptance plus the latest B02 failure evidence. The B02 complete acceptance below is a later independent @@ -17,11 +17,13 @@ record; it does not replace that retained failure or replay all 158 cases on one candidate. All 158 canonical IDs are retained. Independent complete live acceptance is -**115/158**. The current ledger has 20 retained live failures, six pending -native/live race acceptances, four orchestration/evidence blockers and 13 environment cases -disabled by default. The active acceptance backlog is 30 cases. Counts are -case-level canonical results across immutable candidates; diagnostic, native, -isolation and partial results do not increment PASS. +**115/158**. The current ledger has 21 retained live failures, five pending +native/live race acceptances, four orchestration/evidence blockers and 13 +environment cases disabled by default. The active acceptance backlog remains +30 cases: G02 moved from pending race acceptance to retained live failure +after its first live run failed before completing the required matrix. +Counts are case-level canonical results across immutable candidates; +diagnostic, native, isolation and partial results do not increment PASS. | Reconciled case | Complete live run | Required coverage | Original result SHA-256 | | --- | --- | --- | --- | @@ -39,8 +41,8 @@ isolation and partial results do not increment PASS. | Acceptance state | Cases | Remaining requirement | | --- | ---: | --- | | Independently accepted complete live PASS | 115 | Preserve original provenance and complete results. | -| Retained live failures with existing executors | 20 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | -| Native and live race acceptance pending | 6 | A09 and G01–G05 require independently verified native groups, reviewed candidate-specific runtime admission and complete H1/H2 live evidence. | +| Retained live failures with existing executors | 21 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | +| Native and live race acceptance pending | 5 | A09 and G01/G03–G05 require independently verified native groups, reviewed candidate-specific runtime admission and complete H1/H2 live evidence. | | Orchestration or evidence blocked | 4 | C11, F07, I05 and J02 need their remaining native or full live evidence closure. | | Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | @@ -52,7 +54,47 @@ Hosted-only tools [40086026324/1](https://github.com/rustfs/rustfs/actions/runs/ The registry exposes eight native prerequisite groups: A09, G01–G06 and part of I05. This is separate from runtime admission. The pinned helper admits A09, G05 and G06 on this candidate; it intentionally rejects G01–G04 on the same source before bootstrap because their runtime candidate contracts remain pinned to other reviewed sources. G01–G03 may use their reviewed version candidate `bd0671ba781d8ebf509f4f03533228dfb44496f1`, and G04 uses marker candidate `8380466f3a27c3c3ef4115084178a0ab99a78f32`, each with its own exact package and matching helper/tools. Alternatively, a separate source review and matching tools build must precede extending runtime admission. Do not infer admission from registry membership or mix artifacts from different candidates or helpers. -All ten native tests pass. G06 subsequently passed all eight H1/H2 public branches in [40427815826/1](https://github.com/rustfs/rustfs/actions/runs/40427815826); an independent audit verified its ACK, physical, peer S3, overlap and 20 cleanup receipts, adding one canonical PASS. A09 and G01–G05 still need their complete source branches and clean finalization; I05's native subset does not supply its missing full timeout/stale-generation controller and live closure. +All ten native tests pass. G06 subsequently passed all eight H1/H2 public branches in [40427815826/1](https://github.com/rustfs/rustfs/actions/runs/40427815826); an independent audit verified its ACK, physical, peer S3, overlap and 20 cleanup receipts, adding one canonical PASS. A09 and G01/G03–G05 still need their complete source branches and clean finalization. G02's native/runtime prerequisites were admitted, but its first live run failed before completing the required matrix and is recorded below. I05's native subset does not supply its missing full timeout/stale-generation controller and live closure. + +### G02 retained live failure (2026-10-27) + +G02 `heal.race-versioned-put` run [40444207551/1](https://github.com/rustfs/rustfs/actions/runs/40444207551) +used RustFS candidate `bd0671ba781d8ebf509f4f03533228dfb44496f1`, +auto-testing `7094c16a5a5cf3b0f818c7be463f3bc0f8fd7645`, and the audited +candidate-bound native tools from [40311010126/1](https://github.com/rustfs/rustfs/actions/runs/40311010126). +The H1 `new-version-vs-historical-heal` branch failed at foreground ACK 000003: +PUT `objects/0256.bin` returned HTTP 503 `ServiceUnavailable` after 5.026 s, +with zero retries. The service log binds request +`2b5289c8-0faa-472d-b5c6-fe9db521659d` to `storage_error_code=Some(Lock)` and +records a contemporaneous `rename_data` operation pending for over 5 s on +`/data/rustfs4`. This supports a Heal/PUT lock-contention diagnosis; the log +does not correlate that rename operation to the request ID, so it is not a +complete call-level causal trace. This was a real failed PUT, not a recorder +false positive or successful ACK. + +The branch result is `FAIL/cleanup FAIL`; its last-ACK receipt records +`unresolvedMutation=true` and +`lastACKPhysicalAuthorityVerified=false`. Evidence-based recovery run +[40444795627](https://github.com/rustfs/rustfs/actions/runs/40444795627) +isolated the active namespace. It did not restore the scene, delete the +namespace or rewrite the verdict, and it claimed no canonical PASS. The +original failed scene remains retained and must not be replayed in place. + +The Auto Testing PR [#113](https://github.com/rustfs/auto-testing/pull/113) +contains the migrated G02 executor and remains Draft. RustFS PR +[#8455](https://github.com/rustfs/rustfs/pull/8455) completed its distributed +4-node/4-disk E2E in run [40631115072](https://github.com/rustfs/rustfs/actions/runs/40631115072) +on candidate `4454b029dd462e0f58f1b3acc0f4cd62137c24d6`: 92 tests passed and +873 were skipped. The PR includes Heal lock-wait observation and extensive +Heal regressions, but the E2E result does not bind those changes to the failed +PUT request or establish that the G02 lock contention is fixed. It is +supporting candidate evidence, not an immutable G02 package or case result. +Retest only after reviewing a concrete source correction, building and +auditing a package pinned to its exact candidate SHA with matching +tools/helper, and confirming the current Auto Testing fleet is safe for a +fresh run. A complete H1/H2 matrix, physical and peer S3 verification, and +clean run-owned finalization are required before G02 can add a canonical PASS. +Until then the 115/158 total is unchanged. ### Hosted runner inspection (2026-10-26) From 88fcc621cbaf42b3d8b4ae3c76426ff4f1c3d760 Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 28 Oct 2026 17:59:23 +0800 Subject: [PATCH 343/343] test(validation): complete source implementation migration --- .../workflows/validation-catalog-tests.yml | 7 +- docs/validation-heal-contract.md | 2 +- ...idation-migration-implementation-review.md | 123 + docs/validation-migration-map.md | 47 +- scripts/validation_heal/lock_observer.py | 7 +- scripts/validation_heal/lock_source.py | 225 ++ scripts/validation_heal/lock_successor.py | 2 + scripts/validation_heal/lock_timeout.py | 1 + .../validation_heal/lock_timeout_runner.py | 27 +- scripts/validation_heal/persistence.py | 87 +- scripts/validation_heal/run_case.sh | 13 +- scripts/validation_migration_review.py | 153 + test_validation_heal_lock_source.py | 184 + test_validation_heal_lock_timeout_runner.py | 63 +- test_validation_heal_persistence.py | 80 + test_validation_migration.py | 116 + validation-migration-review.json | 3223 +++++++++++++++++ 17 files changed, 4319 insertions(+), 41 deletions(-) create mode 100644 docs/validation-migration-implementation-review.md create mode 100644 scripts/validation_heal/lock_source.py create mode 100644 scripts/validation_migration_review.py create mode 100644 test_validation_heal_lock_source.py create mode 100644 validation-migration-review.json diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 1a5d2639..89f2ed9d 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -4,6 +4,8 @@ on: pull_request: paths: - validation-cases.json + - validation-migration-review.json + - scripts/validation_migration_review.py - scripts/validation_catalog.py - scripts/validation_report.py - scripts/validation_result.py @@ -75,6 +77,8 @@ on: branches: [main] paths: - validation-cases.json + - validation-migration-review.json + - scripts/validation_migration_review.py - scripts/validation_catalog.py - scripts/validation_report.py - scripts/validation_result.py @@ -155,6 +159,7 @@ jobs: with: persist-credentials: false - run: python3 scripts/validation_catalog.py check + - run: python3 -B scripts/validation_migration_review.py --require-complete - run: python3 -B -m unittest -v test_validation_migration.py - run: python3 -B -m unittest -v test_validation_ilm.py - run: python3 -B -m unittest -v test_validation_pool.py @@ -167,7 +172,7 @@ jobs: sudo apt-get install --no-install-recommends -y nftables fi - run: python3 -B -m unittest -v test_validation_heal.py test_validation_heal_recovery.py test_validation_heal_authorization.py test_validation_heal_full_restart.py test_validation_heal_mrf.py test_validation_heal_unformatted.py test_validation_heal_performance.py test_validation_heal_scopes.py test_validation_heal_multi_pool.py test_validation_heal_bucket_metadata.py test_validation_heal_native.py test_validation_heal_manager_native.py test_validation_heal_manager_fixture.py test_validation_heal_journal_fault.py test_validation_heal_checkpoint.py test_validation_heal_authority.py test_validation_heal_orphan.py test_validation_heal_options.py test_validation_heal_environment.py test_validation_heal_environment_gate.py test_validation_heal_queue_pressure.py test_validation_heal_nolock.py test_validation_heal_race.py test_validation_heal_write_quorum.py test_validation_heal_schema.py test_validation_heal_encrypted.py test_validation_heal_coordinator_crash.py test_validation_heal_flapping.py test_validation_heal_capacity.py test_validation_heal_network.py test_validation_oidc.py test_validation_webhooks.py - - run: python3 -B -m unittest -v test_validation_heal_timeout_retention.py test_validation_heal_race_retention.py test_validation_heal_retained_scene.py test_validation_heal_lock_timeout_runner.py test_validation_heal_lock_timeout.py test_validation_heal_lock_observer.py test_validation_heal_lock_successor.py test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py + - run: python3 -B -m unittest -v test_validation_heal_timeout_retention.py test_validation_heal_race_retention.py test_validation_heal_retained_scene.py test_validation_heal_lock_timeout_runner.py test_validation_heal_lock_source.py test_validation_heal_lock_timeout.py test_validation_heal_lock_observer.py test_validation_heal_lock_successor.py test_validation_heal_caller_abort.py test_validation_heal_status_keepalive.py test_validation_heal_mpu_race_runner.py test_validation_heal_race_io.py test_validation_heal_race_physical.py test_validation_heal_g03_source.py test_validation_heal_version_race_fixture.py test_validation_heal_version_race_physical.py test_validation_heal_version_race_adapter.py test_validation_heal_version_race_runner.py - run: python3 -B scripts/validation_heal/matrix.py check - run: python3 -B -m unittest -v test_validation_heal_outcome.py test_validation_heal_outcome_grace.py - run: python3 -B -m unittest -v test_validation_heal_persistence.py test_validation_heal_persistence_mrf.py diff --git a/docs/validation-heal-contract.md b/docs/validation-heal-contract.md index b7376bc7..63b216c7 100644 --- a/docs/validation-heal-contract.md +++ b/docs/validation-heal-contract.md @@ -6,7 +6,7 @@ 入口为 `scripts/validation_heal/matrix.py`、[逐例操作卡](validation-heal-cases.md) 和 [执行手册](validation-heal-operations.md)。运行环境是当前 auto-testing 的 Heal 节点；候选及 rc 使用当前 validation lane 的不可变包和实际二进制摘要。源仓库历史 pilot 结果不能计入本次通过数量。 -目前有 67 个自动执行器和 13 个默认关闭的环境场景入口；C11/F07/I05/J02 尚未接入调度。注册执行器不代表完整 canonical PASS，A09/G01–G06 的必需 native 分支仍被阻断。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。13 个环境场景保留源操作分支、未满足条件和显式启用入口；其完整 live 验证仍需资源适配器与源断言审核。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 +全部 84 个 Heal ID 已有调度路由，包括 67 个常规入口、13 个默认关闭的环境场景入口，以及 C11/F07/I05/J02 的四个诊断入口。测试实现完整性与产品 live 结果独立统计：C11/F07/I05/J02 的源分支已有执行流程；F07 组合相同候选的 manager 中断与目录 fsync 证据，I05 总控串行执行 H1/H2 超时和断连控制器，并独立绑定完整内容/物理代次及 owner 原生证明，见[实现审查](validation-migration-implementation-review.md)。四个诊断入口的 canonical 验收门禁继续关闭；注册入口或 native 子集通过均不能充作完整 PASS。A09/G01–G06 的精确 native 与候选准入按各自执行器核对，G06 已有完整独立 live 验收。产品 FAIL 不自动等于测试未迁移。分片执行器核对实际盘角色、停止服务后的文件故障、原始同角色分片摘要与逐块 HighwayHash。A03 通过原路径/token 比较 H2 协调者、独立 peer 和固定节点入口的任务身份、终态及完整 outcome；候选每响应生成的 startTime 不作为任务 epoch。A04 保留四个 DB 参数组合及两个 DV 分支，要求完整 dry_run_observed 集合且对象没有变化。A05 使用独立部署执行全部八个 H1/H2 分支，冻结实际 merge 配置、真实非零 running 窗口、精确冲突与 token 策略及旧/新任务责任，完成后在服务运行状态下证明所有 peer 退出和物理静默。A06/F06 证明取消后的物理静默、旧 token 隔离；F06 还保留取消任务原始持久化记录，重启后不得复活或覆盖新任务。A08 保留五种主体各自 start/status/stop 操作、真实过期签名、仅授权 admin:Heal 的临时用户及清理后凭据失效。B09 使用明确启用 MRF 的专用服务，记录节点离线时的降级 ACK，并先验逐盘补片再执行目标 S3 请求。J03 必须实际超过 1024 项保留窗口，截断的诊断清单仍需独立全量物理/S3 补证。13 个环境场景保留源操作分支、未满足条件和显式启用入口；其完整 live 验证仍需资源适配器与源断言审核。每项实际状态见 [迁移映射](validation-migration-map.md)；本地检查、操作卡和归档不能代替 live PASS。 B03/B04 replacement runner 先执行只读 mount admission preflight，再允许停止服务、下载/安装候选包或创建远端 run namespace；阻断结果明确记录 `execution_stage=pre-deployment`、候选/rc 来源和 `binary_sha256=null`，表示尚未观察到安装候选二进制。通过预检后，部署阶段仍会在 reservation 前再次读取同一组 mount/设备身份，避免预检与实际变更之间的环境漂移。 diff --git a/docs/validation-migration-implementation-review.md b/docs/validation-migration-implementation-review.md new file mode 100644 index 00000000..5c62603c --- /dev/null +++ b/docs/validation-migration-implementation-review.md @@ -0,0 +1,123 @@ +# 用例迁移实现审查 + +审查日期：2026-10-28。冻结源 catalog 提交为 +`524eb5b32c3236497fdd8a9271df0494fa3d9974`，SHA-256 为 +`463fdd02dee43e42fef4f3aacf7506709ce87af535e1a53d91c521bc2e361cad`。 +本次交付范围是 auto-testing 中的测试流程、原断言和证据采集。产品执行结果独立保留。 + +## 实现和执行结果 + +| 测试实现状态 | canonical 用例数 | 含义 | +| --- | ---: | --- | +| 已实现 | 145 | 各源分支已有执行路径、原断言及证据采集；其中 115 项已有完整 live 审计，30 项运行失败或尚未完整验收。 | +| 环境条件限制，默认关闭 | 13 | 操作流程、能力检查、原断言和未满足原因已保留；执行需要明确的资源及经过审核的资源适配器。 | +| 实现部分完成 | 0 | 已补齐 F07、I05 的测试接线和源流程总控。 | +| 合计 | 158 | 保留全部源 canonical ID。 | + +独立 live 状态仍为 115 项 accepted、22 项 retained-failure、4 项 pending、 +4 项 blocked、13 项 disabled。共 43 项尚无完整 live 验收，其中 13 项按约定默认关闭。 +这些结果跨多个不可变候选和历史批次，不构成某个候选通过全部 158 项的产品认证。 +22 项失败需要逐例归因，不能统一宣称为产品缺陷，也不能直接据此认定测试没有迁移。 + +`validation-migration-review.json` 列出全部 158 个 ID、实现状态、代码/测试引用、 +源分支、缺口及独立 live 状态。`reviewed_files` 固定引用文件摘要，变化后须重新审查。 +文件存在和本地测试通过不代替源语义审查或 live 审计。本文件不覆盖原始结果。 + +```bash +python3 -B scripts/validation_migration_review.py --require-complete +``` + +此门禁仅因测试实现缺口失败，产品 FAIL 不影响实现计数。13 项环境流程另行输出所需 +适配器清单。产品门禁 `validation_report.py` 仍要求源断言、清理、来源和完整 live PASS。 + +## F07：持久化临界点 + +冻结源 `docs/heal-cases.md#F07` / `scenarios/heal-recipes.json` 要求精确本 run 的 +checkpoint/MRF 故障、持久化责任保留、损坏记录保留，以及独立 native fsync/commit +窗口补证。入口 `persistence.py` 组合以下已存在的能力： + +| 能力 | 证明范围 | +| --- | --- | +| `persistence.py` / `persistence_mrf.py` | 五种 live 字节损坏、写失败及重扫分支，保留原责任、失败和恢复证据。 | +| manager checkpoint-page-order / checkpoint-stale-manager | 正常发布和 reload 次序；旧 manager 不能覆盖新 cursor。正常 reload 不冒称提交中断。 | +| manager mrf-migration-boundaries | 实际 AfterPayload、BeforeManifest、AfterManifest、AfterReadback 四个中断窗口及保留/重试断言。 | +| manager root-uncertain-publication | 实际终态写入后注入故障，保留 pending owner，重启隔离旧任务。 | +| `persistence_native.py` | 六个 ECStore CAS/MRF 目录 fsync 故障回归，覆盖更新、首次创建、删除和恢复锚点。 | + +checkpoint 通过 `EcstoreDiskAPI::compare_and_update_file` 落盘。manager 的发布次序、 +中断/重启责任与 ECStore CAS 落盘失败分别补证；没有为每个 fsync 发明额外测试要求。 +审查源码为允许的不可变候选 `072b57b37559fbe626aeda8abf61617fdcbdd926` 和 +`1aa9be87d18cf941e5020f19e8622c26ba4b10ac`；完整目录 fsync 组合只允许后者。 + +`collect_native_persistence` 在执行前核对 native manifest 与 fleet 候选，执行六项回归 +后独立验证并再次核对候选；`native_branch_result` 再核对 manager/native 同候选。 +成功和失败分别写入 `independent-native-persistence.json` 与 +`independent-native-persistence-failure.json`，原始 native 日志保留在 `native-persistence/`。 +缺少 artifact 时只允许原诊断子集，不能宣称完整 native coverage。 + +使用当前 auto-testing 环境时，由现有 workflow 传入以下目录： + +```text +VALIDATION_HEAL_MANAGER_DIRECTORY/ + tool-run.json + manager-tools/ + manager-evidence/ +VALIDATION_HEAL_NATIVE_DIRECTORY/ + tool-run.json + native-tools/ + native-evidence/ +``` + +伴随 workflow 在精确目录 fsync 候选上下载已有 native artifact，并将目录传给入口。 +同一候选的原生结果仍是独立 prerequisite；不代表 H1/H2 fleet 命中了内部窗口，不声称 +物理断电或每个 file fsync hook 已执行。F07 的完整独立 live 审计待完成，canonical 门禁保留。 + +## I05：源流程总控和代次证明 + +`lock_source.py` 复用 `lock_timeout_runner.py`、`lock_timeout.py`、`lock_successor.py` +及 `lock_observer.py`。默认 `source-plan` 顺序执行 timeout/H1、timeout/H2、 +queued-successor/H1、queued-successor/H2，每次使用独立 run ID、bucket 和证据目录。 +任何分支、证据链或清理失败立即停止后续部署，保留原请求、ACK、日志及停机现场。 + +部署前只读验证同一成功 hosted artifact 中的三个精确源码 native 子集： + +| 子集 | 证明范围 | +| --- | --- | +| I05 owner/unlink/stale | 历史 read-repair owner、取消 physical unlink owner 和原有 stale final rename 回归。旧 stale 测试只检查 ETag。 | +| G02 explicit-version append | 真实缺片重建在 final rename 屏障暂停；新版本 ACK 和物理 tail drain 后释放；检查类型化结果、全部存活 body、版本、dataDir 和逐盘 shard。 | +| G03 explicit-version delete | historical/current 删除与最终提交竞态，保留上述完整内容和物理代次断言。 | + +G02/G03 的较强证明来自已有原生测试，不给旧 ETag 测试补写能力。两组覆盖 4/2 与 +16/4，分别保留 6 与 12 条有序屏障里程碑。append 可能安全修复原选定版本并保留新版本， +不把所有类型化结果统一描述为 stale 拒绝。候选固定为 +`757b751482bb970bc321e92059690a7698554358`；三个 proof 的 package/source tree/ +Cargo.lock/二进制/工作流 attempt/helper/contract 身份必须一致。 + +原生前提可单独检查，命令不执行下载的二进制： + +```bash +python3 -B scripts/validation_heal/lock_source.py \ + --directory "$VALIDATION_HEAL_NATIVE_DIRECTORY" \ + --candidate candidate.json --testing-commit "$AUTO_TESTING_SHA" +``` + +fleet 控制器保留 DV 历史版本和 DL 单次并发请求/ACK ledger，继续核对所有 peer 的 +完整对象/版本、停机物理分片、锁事件和有界 FD 收尾。DL 是工作负载语义，不是新存储格式。 +新证据链关联请求写出、超时/排队、TCP 断开、原 guard drop、successor 获取和观察结束； +对原始 bytes 逐一固定 SHA-256，核对请求/lock owner/binding 和同一协调者时钟顺序。 +不会比较不同主机 monotonic 时钟，也不把本地 guard drop 当成分布式 owner 释放。 + +`source-plan-summary.json` 记录四个源分支的 fleet/native 证明范围； +`sourceProceduresComplete` 只有 H1/H2 四个实例完整且清理通过才为 true。单独选择 timeout +或 queued-successor 仍是诊断子集。全部控制器完成也不自动增加 canonical PASS；完整 +source-plan 的独立 live 审计仍待执行。 + +## 失败归因和后续验收 + +C11 的旧 metadata、checkpoint、MRF replay 和 native incarnation 均有执行路径； +J02 的混合 outcome、dry-run、deferred 和 unknown-negative 均有控制器。其历史产品 +结果和证据限制保留，尤其不能修改 J02 的 Failed/Unknown oracle 来获取全绿。 + +G01/G02 的失败 PUT 不重试，未知 mutation 不算成功。B03/B04 仍缺精确 mount/切换 +适配器，在部署前阻断。今后按产品缺陷、环境前提、证据缺失和测试实现问题逐项归因； +有明确测试实现问题才重开实现缺口。实现交付完成不代替候选包的后续回归与产品验收。 diff --git a/docs/validation-migration-map.md b/docs/validation-migration-map.md index 9ce13635..e92ecad9 100644 --- a/docs/validation-migration-map.md +++ b/docs/validation-migration-map.md @@ -7,7 +7,30 @@ and SHA-256 bind the snapshot to the source catalog. `reuse_candidates` lists co it is not a coverage claim. A case's run result remains `unverified` until a live run proves each source assertion on the required topology. -## Current acceptance and remaining work +## Current implementation scope (2026-10-28) + +The migration delivers test procedures and original oracles in auto-testing. +Candidate live failures are recorded independently from missing test code. +The explicit [implementation review](validation-migration-implementation-review.md) +and `validation-migration-review.json` record **145 implemented cases, +13 default-disabled environment procedures and zero partial implementations**. +The 13 procedures require reviewed resource adapters to execute. F07 composes +same-candidate manager interruption and ECStore directory-fsync evidence. I05 +runs timeout and queued-successor on H1/H2 in one source plan, with separate +same-artifact native owner and full-content/physical generation checks. All +30 implemented cases without full live acceptance retain their candidate +failures or pending evidence independently. Source-path review and local +contract tests do not certify every live branch. + +Run `python3 -B scripts/validation_migration_review.py --require-complete` for +the independent counts and implementation gate. It exits zero when the source +implementation has no recorded gaps; retained product FAIL does not change +that decision. `validation_report.py` continues to enforce original assertions, +cleanup and provenance for product acceptance. Unknown failure causes remain +unclassified. Detailed execution prerequisites and evidence scopes are in the +[implementation review](validation-migration-implementation-review.md). + +## Current live acceptance and retained evidence Count reconciliation (2026-10-23; updated 2026-10-28): the append-only ledger `canonical-migration-ledger-20261023-b08-39915022106.json` follows the @@ -17,12 +40,12 @@ record; it does not replace that retained failure or replay all 158 cases on one candidate. All 158 canonical IDs are retained. Independent complete live acceptance is -**115/158**. The current ledger has 21 retained live failures, five pending +**115/158**. The current review retains 22 live failures, four pending native/live race acceptances, four orchestration/evidence blockers and 13 -environment cases disabled by default. The active acceptance backlog remains -30 cases: G02 moved from pending race acceptance to retained live failure -after its first live run failed before completing the required matrix. -Counts are case-level canonical results across immutable candidates; +environment cases disabled by default. G01 and G02's actual failed PUTs are +included in retained failures; historical ledger bytes are unchanged. These +30 unaccepted active cases are a live acceptance backlog, not 30 missing test +implementations. Counts span immutable candidates and historical audits; diagnostic, native, isolation and partial results do not increment PASS. | Reconciled case | Complete live run | Required coverage | Original result SHA-256 | @@ -41,12 +64,12 @@ diagnostic, native, isolation and partial results do not increment PASS. | Acceptance state | Cases | Remaining requirement | | --- | ---: | --- | | Independently accepted complete live PASS | 115 | Preserve original provenance and complete results. | -| Retained live failures with existing executors | 21 | Diagnose each failure, fix the responsible product or controller path and verify the complete required matrix. | -| Native and live race acceptance pending | 5 | A09 and G01/G03–G05 require independently verified native groups, reviewed candidate-specific runtime admission and complete H1/H2 live evidence. | -| Orchestration or evidence blocked | 4 | C11, F07, I05 and J02 need their remaining native or full live evidence closure. | +| Retained live failures with existing executors | 22 | Preserve original results and attribute the cause; product failures do not imply missing test implementation. | +| Native and live race acceptance pending | 4 | A09 and G03–G05 await complete live acceptance; existing native/runtime paths retain their candidate gates. | +| Orchestration or evidence blocked | 4 | C11/F07/I05/J02 have source execution paths awaiting independent complete live acceptance; no implementation gaps remain in the current review. | | Environment gated, disabled by default | 13 | Control flows, oracles and unmet reasons are implemented; dedicated resource adapters and real execution remain conditional on the required resources. | -The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 30 unaccepted cases remain active migration work. C11/F07/J02 have explicit diagnostic routes, while canonical admission stays blocked until independent complete acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. +The current delivery keeps the 13 environment cases disabled with explicit reasons. The other 30 unaccepted cases retain their original live states independently of their completed implementations. C11/F07/I05/J02 have explicit execution routes, while canonical admission stays blocked until independent complete live acceptance. J02's real four-disk regression reproduced scheduler-level whole-batch retries overwriting an earlier repair receipt. RustFS commit `7951bb629821246868ee3761b321dd07c06db45c` retains exhausted Admin batch results; its 210 manager tests and five process-isolated receipt integrations pass. ### Native prerequisites and runtime admission (2026-10-26) @@ -54,7 +77,7 @@ Hosted-only tools [40086026324/1](https://github.com/rustfs/rustfs/actions/runs/ The registry exposes eight native prerequisite groups: A09, G01–G06 and part of I05. This is separate from runtime admission. The pinned helper admits A09, G05 and G06 on this candidate; it intentionally rejects G01–G04 on the same source before bootstrap because their runtime candidate contracts remain pinned to other reviewed sources. G01–G03 may use their reviewed version candidate `bd0671ba781d8ebf509f4f03533228dfb44496f1`, and G04 uses marker candidate `8380466f3a27c3c3ef4115084178a0ab99a78f32`, each with its own exact package and matching helper/tools. Alternatively, a separate source review and matching tools build must precede extending runtime admission. Do not infer admission from registry membership or mix artifacts from different candidates or helpers. -All ten native tests pass. G06 subsequently passed all eight H1/H2 public branches in [40427815826/1](https://github.com/rustfs/rustfs/actions/runs/40427815826); an independent audit verified its ACK, physical, peer S3, overlap and 20 cleanup receipts, adding one canonical PASS. A09 and G01/G03–G05 still need their complete source branches and clean finalization. G02's native/runtime prerequisites were admitted, but its first live run failed before completing the required matrix and is recorded below. I05's native subset does not supply its missing full timeout/stale-generation controller and live closure. +All ten native tests pass. G06 subsequently passed all eight H1/H2 public branches in [40427815826/1](https://github.com/rustfs/rustfs/actions/runs/40427815826); an independent audit verified its ACK, physical, peer S3, overlap and 20 cleanup receipts, adding one canonical PASS. A09 and G01/G03–G05 still need their complete source branches and clean finalization. G02's native/runtime prerequisites were admitted, but its first live run failed before completing the required matrix and is recorded below. I05 now composes the timeout/disconnect controllers and separate full-content/physical native generation checks through `lock_source.py`; complete independent source-plan live acceptance remains pending. ### G02 retained live failure (2026-10-27) @@ -568,7 +591,7 @@ and precise versioned tags, then faults an external data part, an entire xl.meta header, or a parity part. Exact physical/version references, complete S3 inventories, HEAD and ranges must agree after repair. This executor still requires live evidence. -Four Heal cases remain outside canonical dispatch: C11, F07, I05 and J02. All four have explicit diagnostic routes that preserve their blocking canonical verdicts; I05's H1/H2 timeout and queued-successor diagnostics have independently audited live results, exact-source native prerequisites and stopped-scene isolation. Exact server barriers and the complete source DL/DV branches remain required. The 13 environment-gated cases have disabled-by-default control flows and oracles in `environment_cases.py`; actual execution requires an assigned and reviewed auto-testing resource adapter, and canonical results remain review-gated. See [the environment gate reference](validation-heal-environment-gates.md) for each missing resource, capability and native barrier. +Four Heal cases retain a canonical acceptance gate: C11, F07, I05 and J02. All have dispatch routes. All four have explicit diagnostic routes that preserve their blocking canonical verdicts; I05's H1/H2 timeout and queued-successor diagnostics have independently audited live results, exact-source native prerequisites and stopped-scene isolation. I05 now composes both H1/H2 controllers with separate same-artifact native owner and full-content/physical generation checks. The native proof does not establish that the fleet request held a distributed lock at disconnect, and the composed source plan still requires independent complete live audit. The 13 environment-gated cases have disabled-by-default control flows and oracles in `environment.py`; actual execution requires an assigned and reviewed auto-testing resource adapter, and canonical results remain review-gated. See [the environment gate reference](validation-heal-environment-gates.md) for each missing resource, capability and native barrier. H4 needs at least 24 independent disk endpoints; H6 and native-fence branches retain their platform and source-bound evidence requirements. diff --git a/scripts/validation_heal/lock_observer.py b/scripts/validation_heal/lock_observer.py index 1e1fa11f..b3aec92d 100644 --- a/scripts/validation_heal/lock_observer.py +++ b/scripts/validation_heal/lock_observer.py @@ -13,13 +13,13 @@ try: from .caller_abort import disconnect_get from .fleet_node import PROCESS_FIELDS, runtime_config - from .heal_probe import MAX_BODY, Probe, canonical, require, save, validate_binding, validate_s3_request + from .heal_probe import CLOCK_DOMAIN, MAX_BODY, Probe, canonical, require, save, validate_binding, validate_s3_request from .healthy import Fleet from .scanner_repair import JournalWindow except ImportError: from caller_abort import disconnect_get from fleet_node import PROCESS_FIELDS, runtime_config - from heal_probe import MAX_BODY, Probe, canonical, require, save, validate_binding, validate_s3_request + from heal_probe import CLOCK_DOMAIN, MAX_BODY, Probe, canonical, require, save, validate_binding, validate_s3_request from healthy import Fleet from scanner_repair import JournalWindow @@ -249,7 +249,8 @@ def before_disconnect(sent): break require(time.monotonic() < deadline, 'NOT_OBSERVED: I05 verification did not return to its FD count baseline') time.sleep(.2) - result = {'observationComplete': True, 'baselineResources': baseline, + result = {'observationComplete': True, 'clockDomain': CLOCK_DOMAIN, + 'observationCompletedMonotonicNs': time.monotonic_ns(), 'baselineResources': baseline, 'admission': admitted, 'finalAttempt': observed, 'admittedWork': admitted_work, 'disconnectedWork': disconnected_work, diff --git a/scripts/validation_heal/lock_source.py b/scripts/validation_heal/lock_source.py new file mode 100644 index 00000000..961eeacb --- /dev/null +++ b/scripts/validation_heal/lock_source.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Compose I05 source procedures without converting diagnostics to canonical PASS.""" + +import copy +import hashlib +import json +from pathlib import Path + +try: + from .heal_probe import CLOCK_DOMAIN, canonical, require, save + from .race_native_evidence import verify as verify_native +except ImportError: + from heal_probe import CLOCK_DOMAIN, canonical, require, save + from race_native_evidence import verify as verify_native + + +CASE_ID = 'heal.lock-timeout-and-caller-abort' +GROUPS = { + CASE_ID: ('versioned-read-repair-lock-owner', 'cancelled-physical-unlink-owner', + 'stale-read-repair-final-rename'), + 'heal.race-versioned-put': ('versioned-read-repair-lock-owner', 'explicit-version-append-final-rename'), + 'heal.race-version-delete': ('explicit-version-delete-final-rename',), +} +IDENTITY = ('candidateCommit', 'candidatePackageSha256', 'candidateSourceTree', + 'candidateCargoLockSha256', 'binarySha256', 'runId', 'runAttempt', + 'driverCommit', 'testingCommit', 'selectedContractSha256') +LIVE_BRANCHES = ('timeout', 'queued-successor') + + +def verify_source_native(args, candidate): + """Reuse three reviewed subsets from one immutable successful artifact. + + The old stale test asserts ETag only. The append/delete subsets additionally + bind real reconstruction barriers, foreground ACK/tail drain, exact typed + outcomes, full surviving bodies and physical generations in 4/2 and 16/4. + Their native geometry does not claim a fleet request hit the same barrier. + """ + directory = getattr(args, 'native_directory', None) + require(directory is not None, 'I05 source-plan requires exact-source native evidence') + groups = {} + for case_id, labels in GROUPS.items(): + proof = verify_native(directory, candidate, args.testing_commit, case_id) + require(proof.get('caseId') == case_id and proof.get('requiredRegressionLabels') == list(labels) + and set(proof.get('observedRegressions', {})) == set(labels) + and proof.get('requiredNativeSubsetPassed') is True + and all(proof.get(key) is False for key in ( + 'wholeCasePassClaimed', 'fleetExecutionClaimed', 'canonicalCasePassClaimed')), + 'I05 source native proof lacks its exact reviewed subset or changes scope') + require(proof.get('candidateCommit') == candidate['source_sha'] + and proof.get('candidatePackageSha256') == candidate['package_sha256'] + and proof.get('testingCommit') == args.testing_commit, + 'I05 source native proof changed candidate or helper') + groups[case_id] = proof + first = groups[CASE_ID] + require(all(key in first and first[key] is not None for key in IDENTITY) + and all(type(proof.get(key)) is type(first[key]) and proof[key] == first[key] + for proof in groups.values() for key in IDENTITY), + 'I05 source native subsets mix candidates, binaries or workflow attempts') + return {**{key: first[key] for key in IDENTITY}, 'groups': groups, + 'nativeOwnerAndStaleFencesPassed': True, + 'staleContentAndPhysicalAuthorityScope': 'independent native append/delete in 4/2 and 16/4', + 'fleetExecutionClaimed': False, 'wholeCasePassClaimed': False, + 'canonicalCasePassClaimed': False} + + +def link_fleet_evidence(directory, branch, binding, diagnostic): + """Bind durable requests to coordinator timing and actual server identities. + + Controllers already verify original server logs, full peer inventories, + physical state and bounded resource drain. This joins their retained bytes; + it never compares a remote monotonic clock to the coordinator clock. + """ + require(branch in LIVE_BRANCHES, 'I05 source evidence has an unknown live branch') + directory = Path(directory) + hashes = {} + + def read(name): + path = directory / name + require(path.is_file() and not path.is_symlink() and path.stat().st_size < 16 * 1024**2, + 'I05 source chain lacks an original evidence file: ' + name) + raw = path.read_bytes() + hashes[name] = hashlib.sha256(raw).hexdigest() + return json.loads(raw) + + prefix = branch + '/' + scope = read(prefix + 'scope.json') + caller_prefix = prefix + 'caller/' + intent = read(caller_prefix + 'transport/request.json') + written = read(caller_prefix + 'transport/request-written.json') + closed = read(caller_prefix + 'transport/disconnected.json') + admission = read(caller_prefix + 'admission.json') + caller = read(caller_prefix + 'observation.json') + stored = read(prefix + 'diagnostic.json') + require(stored == diagnostic and diagnostic.get('diagnosticComplete') is True + and caller == diagnostic.get('callerObservation') and caller.get('admission') == admission + and caller.get('observationComplete') is True and caller.get('exactHistoricalContentPreserved') is True + and caller.get('callerResultUnknown') is True + and all(caller.get(key) is False for key in ( + 'heldAtDisconnectProven', 'distributedReleaseProven', 'canonicalCasePassClaimed')), + 'I05 source chain changed caller observation or invents ownership') + digest = hashlib.sha256(canonical(binding).encode()).hexdigest() + require(scope.get('runId') == intent.get('runId') == written.get('runId') == closed.get('runId') == binding['runId'] + and scope.get('bindingSha256') == intent.get('bindingSha256') == digest + and all(intent.get(key) == binding[key] for key in ( + 'candidateCommit', 'harnessCommit', 'candidatePackageSha256', 'candidateBinarySha256')) + and intent.get('params') == {'Bucket': binding['bucketPrefix'], 'Key': scope['entry']['key'], + 'VersionId': scope['entry']['versionId']} + and intent.get('retryAllowed') is False and intent.get('operation') == 'get_object' + and intent.get('endpoint') == binding['defaultEndpoint'] + and intent.get('endpointUrl') == binding['endpoints'][binding['defaultEndpoint']].rstrip('/'), + 'I05 source chain changed binding, candidate, run or exact historical request') + request_sha = hashes[caller_prefix + 'transport/request.json'] + require(all(row.get('requestSha256') == request_sha and row.get('callerMarker') == intent.get('callerMarker') + and row.get('requestTarget') == intent.get('requestTarget') + and row.get('requestInvocationStarted') is True and row.get('requestWriteCompleted') is True + and all(row.get(key) is False for key in ('responseObserved', 'acknowledged', 'retryAllowed', 'serverOwnershipProven')) + and row.get('resultUnknown') is True for row in (written, closed)) + and closed.get('connectionAttempts') == 1 and closed.get('clientSocketClosed') is True + and closed.get('shutdownCompleted') is True + and all(closed.get(key) == written.get(key) for key in ( + 'callStartedMonotonicNs', 'requestWrittenMonotonicNs', 'localAddress', 'peerAddress')), + 'I05 source chain lost its single unretried TCP request or unknown outcome') + holder, final = admission['attempt'], caller['finalAttempt'] + require(all(holder.get(key) == final.get(key) and holder.get(key) is not None for key in ( + 'request_id', 'lock_attempt_id', 'owner', 'namespace')) + and holder.get('observedAcquiredWithoutDrop') is True and final.get('diagnosticGuardDropped') is True, + 'I05 source chain lost original server request, owner or local guard drain') + edge_name = 'timeout.json' if branch == 'timeout' else 'queued.json' + edge = read(prefix + edge_name) + ack_prefix = prefix + ('contender/' if branch == 'timeout' else 'successor/') + request = read(ack_prefix + '000001-request.json') + ack = read(ack_prefix + '000001-ack.json') + require(ack.get('requestSha256') == hashes[ack_prefix + '000001-request.json'] + and request.get('runId') == ack.get('runId') == binding['runId'] + and request.get('params', {}).get('Bucket') == intent['params']['Bucket'] + and request.get('params', {}).get('Key') == intent['params']['Key'] + and request.get('operation') == 'put_object' and ack.get('operationCheck') == 'PASS' + and ack.get('observation', {}).get('response', {}).get('ResponseMetadata', {}).get('RequestId') == edge.get('requestId'), + 'I05 source chain lost the exact foreground request or original ACK') + times = [intent.get('preparedMonotonicNs'), written.get('callStartedMonotonicNs'), + written.get('requestWrittenMonotonicNs'), edge.get('observedMonotonicNs'), + closed.get('disconnectStartedMonotonicNs'), closed.get('clientClosedMonotonicNs'), + caller.get('observationCompletedMonotonicNs')] + timed = [intent, written, closed, edge, caller, request, ack, ack.get('observation', {})] + require(all(row.get('clockDomain') == CLOCK_DOMAIN for row in timed) + and all(type(value) is int and value > 0 for value in times) and times == sorted(times), + 'I05 source chain lost coordinator clock order around the actual TCP disconnect') + if branch == 'timeout': + require(edge.get('typedTimeoutBeyondBudgetObserved') is True and edge.get('writeLockAcquired') is False + and edge.get('readResponseBackpressureObservedAfterTimeout') is True + and diagnostic.get('timeoutContenderAcknowledged') is False, + 'I05 source chain has no typed contention beyond its registered timeout') + else: + completion = read(prefix + 'successor-complete.json') + require(edge.get('states') == ['acquiring'] + and all(completion.get(key) == edge.get(key) for key in ('requestId', 'lockAttemptId')) + and completion.get('states') == ['acquiring', 'acquired', 'guard_dropped'] + and completion.get('queuedBeforeDisconnectObserved') is True + and completion.get('originalDiagnosticDropBeforeAcquire') is True + and completion.get('heldAtDisconnectProven') is False + and completion.get('clockDomain') == CLOCK_DOMAIN + and type(completion.get('observedMonotonicNs')) is int + and times[-2] <= completion['observedMonotonicNs'] <= times[-1], + 'I05 source chain lacks the queued successor and ordered original guard release') + return {'sourceEvidenceComplete': True, 'branch': branch, 'runId': binding['runId'], + 'bindingSha256': digest, 'originalEvidenceSha256': hashes, + 'callerRequestId': holder['request_id'], 'callerLockAttemptId': holder['lock_attempt_id'], + 'foregroundRequestId': edge['requestId'], 'coordinatorClockDomain': CLOCK_DOMAIN, + 'coordinatorTimes': times, 'heldAtDisconnectProven': False, 'distributedReleaseProven': False, + 'nativeOwnerProofRequiredSeparately': True, 'canonicalCasePassClaimed': False} + + +def run_source_plan(args, candidate, inspector, run_topology, native_proof): + """Run real timeout/disconnect controllers serially, retaining the first dirty scene.""" + save(args.evidence / 'source-native-prerequisite.json', native_proof) + results = [] + remaining = [f'{branch}/{topology}' for branch in LIVE_BRANCHES for topology in args.topologies] + for branch in LIVE_BRANCHES: + child = copy.copy(args) + child.diagnostic_branch, child.source_plan = branch, True + child.evidence = args.evidence / 'source-plan' / branch + for topology in args.topologies: + result = run_topology(child, candidate, inspector, topology) + results.append(result) + if not (result.get('diagnosticComplete') is True and result.get('cleanup') == 'PASS' + and result.get('sourceEvidenceChain', {}).get('sourceEvidenceComplete') is True): + break + remaining.remove(f'{branch}/{topology}') + if remaining and remaining[0].startswith(branch + '/'): + break + passed = not remaining + summary = {'caseId': CASE_ID, 'diagnosticBranch': 'source-plan', 'diagnostics': results, + 'requestedTopologies': args.topologies, 'diagnosticComplete': passed, + 'sourceProceduresComplete': passed and args.topologies == ['H1', 'H2'], + 'sourceBranches': { + 'object-lock-timeout': {'scope': 'fleet', 'controller': 'timeout'}, + 'caller-disconnect': {'scope': 'fleet', 'controller': 'queued-successor'}, + 'stale-read-repair': {'scope': 'independent-native', + 'groups': ['heal.race-versioned-put', 'heal.race-version-delete']}, + 'native-lock-owner-fence': {'scope': 'independent-native', 'group': CASE_ID}}, + 'nativeProof': native_proof, 'remainingLiveInstances': remaining, + 'cleanup': 'PASS' if results and all(row['cleanup'] == 'PASS' for row in results) else 'FAIL', + 'complete': False, 'wholeCasePassClaimed': False, 'canonicalCasePassClaimed': False, + 'heldAtDisconnectProven': False, 'distributedReleaseProven': False, + 'remaining': ['independent complete live audit of the source plan']} + save(args.evidence / 'source-plan-summary.json', summary) + save(args.evidence / 'healthy-summary.json', summary) + return summary + + +def main(): + import argparse + from types import SimpleNamespace + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--directory', type=Path, required=True) + parser.add_argument('--candidate', type=Path, required=True) + parser.add_argument('--testing-commit', required=True) + args = parser.parse_args() + proof = verify_source_native(SimpleNamespace(native_directory=args.directory, + testing_commit=args.testing_commit), json.loads(args.candidate.read_text())) + print(json.dumps(proof, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/validation_heal/lock_successor.py b/scripts/validation_heal/lock_successor.py index 4d92eaff..e46ea06b 100644 --- a/scripts/validation_heal/lock_successor.py +++ b/scripts/validation_heal/lock_successor.py @@ -276,6 +276,7 @@ def queued(sent, admitted, window): if attempt is not None: require(attempt['states'] == ['acquiring'], 'I05 successor acquired before caller disconnect') + attempt.update(clockDomain=CLOCK_DOMAIN, observedMonotonicNs=time.monotonic_ns()) save(output / 'queued.json', attempt) return attempt time.sleep(.02) @@ -308,6 +309,7 @@ def complete(sent, admitted, window): after_ack = physical_content(after) target = physical_append(physical, after, bucket, baseline, added) save(output / 'successor-complete.json', {**attempt, 'added': added, + 'clockDomain': CLOCK_DOMAIN, 'observedMonotonicNs': time.monotonic_ns(), 'queuedBeforeDisconnectObserved': True, 'originalDiagnosticDropBeforeAcquire': True, 'heldAtDisconnectProven': False, 'canonicalCasePassClaimed': False}) return attempt diff --git a/scripts/validation_heal/lock_timeout.py b/scripts/validation_heal/lock_timeout.py index 5dff0a74..fea35333 100644 --- a/scripts/validation_heal/lock_timeout.py +++ b/scripts/validation_heal/lock_timeout.py @@ -279,6 +279,7 @@ def contend(sent, admitted, window): 'NOT_OBSERVED: I05 backpressure census exceeded its observation budget') proof['backpressureSocket'] = socket_state proof['readResponseBackpressureObservedAfterTimeout'] = True + proof.update(clockDomain=CLOCK_DOMAIN, observedMonotonicNs=time.monotonic_ns()) save(output / 'timeout.json', proof) return proof time.sleep(.1) diff --git a/scripts/validation_heal/lock_timeout_runner.py b/scripts/validation_heal/lock_timeout_runner.py index 2224d24c..5f6490f6 100644 --- a/scripts/validation_heal/lock_timeout_runner.py +++ b/scripts/validation_heal/lock_timeout_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Deploy and retain the I05 timeout diagnostic; never certify the whole case.""" +"""Run I05 source procedures and retain failures pending independent live audit.""" import argparse from datetime import datetime, timedelta, timezone @@ -15,6 +15,7 @@ import time try: + from . import lock_source from .heal_probe import Probe, compare_inventories, require, save, validate_binding from .heal_workload import Recorder, seed, verify_seed from .healthy import Fleet, physical_content, validate_topology @@ -26,6 +27,7 @@ from .shards import map_target, verify_cohort from .timeout_retention import seal_successor_scene except ImportError: + import lock_source from heal_probe import Probe, compare_inventories, require, save, validate_binding from heal_workload import Recorder, seed, verify_seed from healthy import Fleet, physical_content, validate_topology @@ -72,7 +74,9 @@ def preflight(args): require(nodes[:4] == list(NODES), 'I05 requires the current vm000-vm003 Heal node binding') inspector = verify_inspector(args.tool_directory, candidate) require(args.rc_bin.is_file(), 'I05 requires the pinned client binary') - if getattr(args, 'diagnostic_branch', 'timeout') == 'queued-successor': + if getattr(args, 'diagnostic_branch', 'timeout') == 'source-plan': + lock_source.verify_source_native(args, candidate) + elif getattr(args, 'diagnostic_branch', 'timeout') == 'queued-successor': verify_native_prerequisite(args, candidate) return candidate, inspector @@ -102,6 +106,8 @@ def run_topology(args, candidate, inspector, topology): directory = args.evidence / topology directory.mkdir(parents=True, exist_ok=False) run_id = f"gh-{os.environ['GITHUB_RUN_ID']}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}-{topology.lower()}" + if getattr(args, 'source_plan', False): + run_id += '-' + branch nodes = list(NODES[:1] if topology == 'H1' else NODES) bucket = 'rv-heal-' + hashlib.sha256(run_id.encode()).hexdigest()[:16] fleet = Fleet(nodes, directory, run_id, args.tool_directory, @@ -132,7 +138,8 @@ def error_info(error): fleet.deploy(args.binary_sha, bucket, access, secret) fleet.record('actual-topology', validate_topology(fleet.snapshot('deployed-topology'), topology)) recorder = Recorder(probe, directory / 'fixture') - # A bounded diagnostic fixture, not the full source DL/DV workload. + # DV history plus one concurrent, unretried DL request/ACK ledger. + # Large historical GETs make real timeout/disconnect contention observable. expected = seed(recorder, bucket, 'DV', 2, [MIN_BACKPRESSURE_BYTES, 1024]) target = next(row for row in expected if not row['deleteMarker'] and not row['isLatest'] and row['payload']['size'] >= MIN_BACKPRESSURE_BYTES) @@ -194,6 +201,9 @@ def error_info(error): require(physical_content(stopped) == physical_content( fleet.snapshot('I05-successor-stopped-after-integrity', quiescent=True)), 'I05 successor streaming verification changed physical content') + if getattr(args, 'source_plan', False): + result['sourceEvidenceChain'] = fleet.record('I05-source-evidence-chain', + lock_source.link_fleet_evidence(directory, branch, binding, diagnostic)) result['diagnosticComplete'] = True except (Exception, KeyboardInterrupt) as error: result['failure'] = error_info(error) @@ -232,7 +242,8 @@ def run(args): 'I05 deployment requires the installed binary digest') candidate, inspector = preflight(args) branch = getattr(args, 'diagnostic_branch', 'timeout') - native_proof = verify_native_prerequisite(args, candidate) if branch == 'queued-successor' else None + native_proof = (lock_source.verify_source_native(args, candidate) if branch == 'source-plan' else + verify_native_prerequisite(args, candidate) if branch == 'queued-successor' else None) # run_case.sh creates this parent and opens test.log before invoking us. # Reuse only those wrapper files; a prior topology or summary is a replay. require(not args.evidence.is_symlink(), 'I05 evidence directory must not be a symlink') @@ -240,6 +251,8 @@ def run(args): require(all(path.name in {'timeout-preflight.json', 'setup.log', 'test.log'} and path.is_file() and not path.is_symlink() for path in args.evidence.iterdir()), 'I05 requires fresh topology and summary evidence before deployment') + if branch == 'source-plan': + return lock_source.run_source_plan(args, candidate, inspector, run_topology, native_proof) if native_proof is not None: save(args.evidence / 'native-prerequisite.json', native_proof) results = [] @@ -277,8 +290,8 @@ def main(): parser.add_argument('--rc-bin', type=Path, required=True) parser.add_argument('--evidence', type=Path, required=not preflight_only) parser.add_argument('--topologies', nargs='+', choices=('H1', 'H2'), default=['H1', 'H2']) - parser.add_argument('--diagnostic-branch', choices=('timeout', 'queued-successor'), - default=os.environ.get('I05_DIAGNOSTIC_BRANCH', 'timeout')) + parser.add_argument('--diagnostic-branch', choices=('timeout', 'queued-successor', 'source-plan'), + default=os.environ.get('I05_DIAGNOSTIC_BRANCH', 'source-plan')) parser.add_argument('--native-directory', type=Path, default=os.environ.get('VALIDATION_HEAL_NATIVE_DIRECTORY')) args = parser.parse_args(argv) require(args.topologies in (['H1'], ['H2'], ['H1', 'H2']), 'I05 topology order must be H1 then H2') @@ -286,7 +299,7 @@ def main(): preflight(args) print(json.dumps({'preflightPassed': True, 'candidateCommit': CANDIDATE, 'diagnosticBranch': args.diagnostic_branch, - 'nativePrerequisitePassed': args.diagnostic_branch == 'queued-successor', + 'nativePrerequisitePassed': args.diagnostic_branch in ('queued-successor', 'source-plan'), 'diagnosticOnly': True, 'canonicalCasePassClaimed': False})) return 0 signal.signal(signal.SIGTERM, interrupted) diff --git a/scripts/validation_heal/persistence.py b/scripts/validation_heal/persistence.py index 60a6eb0f..6c7b83c4 100644 --- a/scripts/validation_heal/persistence.py +++ b/scripts/validation_heal/persistence.py @@ -28,6 +28,7 @@ from .scanner_repair import JournalWindow from .fleet_node import PROCESS_FIELDS from .persistence_mrf import run as run_mrf_checksum + from . import persistence_native from .versions import public_version_id, verify_proof, version_identity except ImportError: import checkpoint as checkpoints, journal_fault as journal @@ -43,6 +44,7 @@ from scanner_repair import JournalWindow from fleet_node import PROCESS_FIELDS from persistence_mrf import run as run_mrf_checksum + import persistence_native from versions import public_version_id, verify_proof, version_identity @@ -737,6 +739,63 @@ def allocate(): return result +def collect_native_persistence(args, candidate): + """Execute and re-verify the exact hosted ECStore subset when supplied. + + The six directory-fsync regressions are an independent native subset. They + share the candidate identity with the manager evidence, but they never + mutate the fleet and never certify the canonical F07 case. Missing native + artifacts keep the diagnostic prerequisite visible without fabricating a + PASS; a retained native failure is written as evidence for later triage. + """ + raw = getattr(args, "native_directory", None) or os.environ.get("VALIDATION_HEAL_NATIVE_DIRECTORY") + if not raw: + return None + root = Path(raw) + tools, evidence = root / "native-tools", root / "native-evidence" + out = args.evidence / "native-persistence" + args.evidence.mkdir(parents=True, exist_ok=True) + try: + manifest, _ = persistence_native.binding(tools, evidence) + require(manifest["candidateCommit"] == candidate["source_sha"], + "F07 native and fleet candidates differ") + persistence_native.run(tools, evidence, out, args.testing_commit) + proof = persistence_native.verify(tools, evidence, out, args.testing_commit) + require(proof.get("candidateCommit") == candidate["source_sha"], + "F07 verified native proof changed the selected candidate") + save(args.evidence / "independent-native-persistence.json", proof) + return proof + except (Exception, KeyboardInterrupt) as exc: + failure = {"complete": False, "canonicalCasePassClaimed": False, + "fleetPassClaimed": False, "nativeDirectory": str(root.resolve()), + "failure": {"type": type(exc).__name__, "message": str(exc)}} + save(args.evidence / "independent-native-persistence-failure.json", failure) + return failure + + +def native_branch_result(topology, fixture, manager_proof, persistence_proof): + complete = isinstance(persistence_proof, dict) and persistence_proof.get("complete") is True + attempted = persistence_proof is not None + if complete: + require(manager_proof.get("candidateCommit") == persistence_proof.get("candidateCommit") + and manager_proof.get("candidateCommit") is not None, + "F07 manager and directory-fsync proofs mix candidates") + result = {"requestedTopology": topology, "requestedFixture": fixture, "topology": None, + "fixture": None, "branch": "native-persist-fence", "verdict": "PREREQUISITE_PASS", + "cleanup": "PASS", "nativeProof": manager_proof, "nativePersistenceProof": persistence_proof, + "nativeExecutionAttempted": attempted, "fleetBranchExecuted": False, + "canonicalCasePassClaimed": False, + "evidenceScope": "exact-candidate checkpoint/MRF publication plus ECStore directory-fsync native subset", + "individualFsyncHooksClaimed": False, "physicalPowerLossClaimed": False, + "requiredNativeCoverageComplete": complete, + "requiredNativeCoverageScope": "source native-persist-fence: manager commit interruptions plus ECStore directory-fsync regressions", + "remainingNativeCoverage": None if complete else + "provide and pass the exact candidate-bound ECStore directory-fsync artifact"} + if attempted and not complete: + result["nativeFailureRetained"] = True + return result + + def main(): def interrupted(signum, frame): raise KeyboardInterrupt(f"F07 interrupted by signal {signum}") signal.signal(signal.SIGTERM, interrupted) @@ -744,6 +803,7 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"F07 interrupted by sig for name in ("candidate", "rc-bin", "evidence", "tool-directory"): parser.add_argument("--" + name, type=Path, required=True) parser.add_argument("--testing-commit", required=True) parser.add_argument("--binary-sha", required=True) + parser.add_argument("--native-directory", type=Path, default=os.environ.get("VALIDATION_HEAL_NATIVE_DIRECTORY")) parser.add_argument("--case-id", choices=(CASE_ID,), required=True) args = parser.parse_args() require(select(CASE_ID)[0]["variants"] == list(BRANCHES), "F07 recipe branches changed") @@ -752,6 +812,7 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"F07 interrupted by sig "F07 checkpoint failure contract requires an exact reviewed candidate with identical checkpoint sources") native = verify_manager(os.environ.get("VALIDATION_HEAL_MANAGER_DIRECTORY", ""), candidate, CASE_ID) save(args.evidence / "independent-native-persistence-prerequisites.json", native) + native_persistence = collect_native_persistence(args, candidate) require(re.fullmatch(r"[0-9a-f]{40}", args.testing_commit) and re.fullmatch(r"[0-9a-f]{64}", args.binary_sha) and re.fullmatch(r"[1-9][0-9]*", os.environ.get("GITHUB_RUN_ID", "")), "F07 requires immutable workflow and binary identities") nodes = os.environ.get("RUSTFS_NODES", "vm000 vm001 vm002 vm003").split() @@ -767,25 +828,29 @@ def interrupted(signum, frame): raise KeyboardInterrupt(f"F07 interrupted by sig elif branch == "mrf-checksum": result = run_mrf_checksum(args, topology, selected, fixture, candidate) elif branch == "native-persist-fence": - result = {"requestedTopology": topology, "requestedFixture": fixture, "topology": None, - "fixture": None, "branch": branch, "verdict": "PREREQUISITE_PASS", "cleanup": "PASS", - "nativeProof": native, "fleetBranchExecuted": False, "canonicalCasePassClaimed": False, - "evidenceScope": "exact-candidate checkpoint/MRF publication native prerequisite", - "individualFsyncHooksClaimed": False, "physicalPowerLossClaimed": False, - "requiredNativeCoverageComplete": False, - "remainingNativeCoverage": "review every required checkpoint/MRF fsync and commit cut against the source assertions"} + result = native_branch_result(topology, fixture, native, native_persistence) else: result = {"topology": topology, "fixture": fixture, "branch": branch, "verdict": "BLOCKED", "cleanup": "PASS", "reason": BLOCKERS[branch], "fleetBranchExecuted": False, "canonicalCasePassClaimed": False} results.append(result) if result["cleanup"] != "PASS": return 1 finally: + def branch_passed(row): + if row["cleanup"] != "PASS": + return False + if row["branch"] in BLOCKERS: + return row["verdict"] == "BLOCKED" + if row["branch"] == "native-persist-fence": + return row["verdict"] == "PREREQUISITE_PASS" and ( + not row.get("nativeExecutionAttempted") or row.get("requiredNativeCoverageComplete") is True) + return row["verdict"] == "PASS" save(args.evidence / "healthy-summary.json", {"caseId": CASE_ID, "results": results, "complete": False, "cleanup": "PASS" if results and all(row["cleanup"] == "PASS" for row in results) else "FAIL", - "diagnosticBranchesPassed": len(results) == 24 and all(row["cleanup"] == "PASS" and - row["verdict"] == ("BLOCKED" if row["branch"] in BLOCKERS else - "PREREQUISITE_PASS" if row["branch"] == "native-persist-fence" else "PASS") for row in results), - "canonicalCasePassClaimed": False, "reason": "full native cut coverage and independent live audit remain blocked"}) + "diagnosticBranchesPassed": len(results) == 24 and all(branch_passed(row) for row in results), + "nativePersistenceProof": native_persistence, "canonicalCasePassClaimed": False, + "reason": ("independent complete live audit remains pending" + if native_persistence and native_persistence.get("complete") is True else + "exact candidate-bound ECStore native evidence and independent complete live audit remain pending")}) return 1 diff --git a/scripts/validation_heal/run_case.sh b/scripts/validation_heal/run_case.sh index 2a4b1bf1..77dc86e3 100755 --- a/scripts/validation_heal/run_case.sh +++ b/scripts/validation_heal/run_case.sh @@ -9,6 +9,7 @@ driver=healthy.py environment_gate=0 replacement_admission_gate=0 i05_args=() +f07_args=() while [ "$#" -gt 0 ]; do case "$1" in --validation-case) case_id="$2"; shift 2 ;; @@ -98,15 +99,17 @@ if [ -n "${VALIDATION_HEAL_TOOL_DIRECTORY:-}" ]; then tool_args=(--tool-directory "${VALIDATION_HEAL_TOOL_DIRECTORY}") fi if [ "${case_id}" = heal.lock-timeout-and-caller-abort ]; then - diagnostic_branch="${I05_DIAGNOSTIC_BRANCH:-timeout}" + diagnostic_branch="${I05_DIAGNOSTIC_BRANCH:-source-plan}" case "${diagnostic_branch}" in - timeout|queued-successor) ;; + timeout|queued-successor|source-plan) ;; *) die "unsupported I05 diagnostic branch: ${diagnostic_branch}" ;; esac i05_args+=(--diagnostic-branch "${diagnostic_branch}") if [ -n "${VALIDATION_HEAL_NATIVE_DIRECTORY:-}" ]; then i05_args+=(--native-directory "${VALIDATION_HEAL_NATIVE_DIRECTORY}") fi +elif [ "${case_id}" = heal.checkpoint-and-mrf-corruption ] && [ -n "${VALIDATION_HEAL_NATIVE_DIRECTORY:-}" ]; then + f07_args+=(--native-directory "${VALIDATION_HEAL_NATIVE_DIRECTORY}") fi [ "${driver}" = healthy.py ] || [ "${#tool_args[@]}" -gt 0 ] || die "physical Heal cases require candidate-bound bitrot tools" evidence="${artifacts}/evidence/${case_id}" @@ -245,7 +248,7 @@ if [ "${bootstrap_rc}" -eq 0 ]; then done python3 -B "${ROOT}/scripts/validation_heal/${driver}" --candidate "${candidate}" \ --case-id "${case_id}" \ - "${tool_args[@]}" "${i05_args[@]}" \ + "${tool_args[@]}" "${i05_args[@]}" "${f07_args[@]}" \ --testing-commit "${testing_commit}" --binary-sha "${binary_sha}" --rc-bin "${rc_binary}" \ --evidence "${evidence}" > "${evidence}/test.log" 2>&1 || case_rc=$? cat "${evidence}/test.log" @@ -293,11 +296,11 @@ if [ "${case_id}" = heal.outcome-counters-and-error-truth ] && [ "${cleanup_stat fi if [ "${case_id}" = heal.lock-timeout-and-caller-abort ]; then # This route is an explicit diagnostic invocation, never canonical admission. - # Even an all-green timeout subset leaves the source's other branches open. + # Source-plan also retains independent native scope and awaits full live audit. if [ "${cleanup_status}" = PASS ] && [ "${case_rc}" -eq 0 ] && \ jq -e '.diagnosticComplete == true and .complete == false and .canonicalCasePassClaimed == false' \ "${evidence}/healthy-summary.json" >/dev/null 2>&1; then - verdict=BLOCKED; reason="I05 ${diagnostic_branch:-timeout} diagnostic completed; exact disconnect ownership, remaining native fences, stale-generation controller and complete source H1/H2 DL/DV acceptance remain required" + verdict=BLOCKED; reason="I05 ${diagnostic_branch:-source-plan} execution completed; independent complete source-plan live audit remains required" elif [ "${case_rc}" -eq 0 ]; then verdict=INFRA_ERROR; reason="I05 diagnostic summary or cleanup is incomplete; canonical PASS is prohibited" fi diff --git a/scripts/validation_migration_review.py b/scripts/validation_migration_review.py new file mode 100644 index 00000000..101018ce --- /dev/null +++ b/scripts/validation_migration_review.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Audit test implementation coverage independently of candidate live results.""" + +from __future__ import annotations + +import argparse +from collections import Counter +from datetime import date +import hashlib +import json +from pathlib import Path +import re +import sys + +try: + from .validation_catalog import validate_snapshot + from .validation_heal.execution_policy import ENVIRONMENT_GATED_CASES +except ImportError: + from validation_catalog import validate_snapshot + from validation_heal.execution_policy import ENVIRONMENT_GATED_CASES + +ROOT = Path(__file__).resolve().parent.parent +IMPLEMENTATION_STATES = {"implemented", "environment-gated", "partial"} +LIVE_STATES = {"accepted", "retained-failure", "pending", "blocked", "disabled"} +HEX40 = re.compile(r"[0-9a-f]{40}\Z") +HEX64 = re.compile(r"[0-9a-f]{64}\Z") + + +def require(condition: object, message: str) -> None: + if not condition: + raise ValueError(message) + + +def local_file(root: Path, name: str) -> Path: + require(isinstance(name, str) and bool(name), "missing implementation reference") + path = Path(name) + require(not path.is_absolute() and ".." not in path.parts, f"nonlocal implementation reference: {name}") + result = root / path + require(result.is_file() and not result.is_symlink() + and result.resolve().is_relative_to(root.resolve()), f"missing implementation reference: {name}") + return result + + +def validate_review(review: dict, catalog: dict, root: Path = ROOT) -> None: + """Validate an explicit review; never derive implementation state from FAIL.""" + require(isinstance(review, dict), "implementation review must be an object") + validate_snapshot(catalog) + require(isinstance(review.get("as_of"), str), "missing implementation review date") + require(date.fromisoformat(review["as_of"]).isoformat() == review["as_of"], "invalid implementation review date") + require(review.get("schema_version") == 1, "unsupported implementation review schema") + require(review.get("source_commit") == catalog["source_commit"] + and review.get("source_sha256") == catalog["source_sha256"], "implementation review uses another source catalog") + require(isinstance(review.get("reviewed_testing_commit"), str) + and HEX40.fullmatch(review["reviewed_testing_commit"]), "missing immutable review base") + files = review.get("reviewed_files") + require(isinstance(files, dict) and bool(files), "missing reviewed implementation files") + for name, digest in files.items(): + require(isinstance(digest, str) and HEX64.fullmatch(digest), f"invalid review digest: {name}") + require(hashlib.sha256(local_file(root, name).read_bytes()).hexdigest() == digest, + f"implementation changed since review: {name}") + recipes = json.loads((root / "scripts/validation_heal/recipes.json").read_text()) + variants = {case["id"]: set(case["variants"]) for case in recipes["cases"]} + catalog_cases = {case["id"]: case for module in catalog["modules"] for case in module["cases"]} + cases = review.get("cases") + require(isinstance(cases, list), "missing implementation case reviews") + seen = set() + for case in cases: + require(isinstance(case, dict), "implementation case review must be an object") + case_id = case.get("case_id") + require(isinstance(case_id, str) and case_id in catalog_cases, f"unknown implementation case: {case_id}") + require(case_id not in seen, f"duplicate implementation case: {case_id}") + seen.add(case_id) + state = case.get("implementation") + require(isinstance(state, str) and state in IMPLEMENTATION_STATES, f"invalid implementation state: {case_id}") + live = case.get("live", {}) + require(isinstance(live, dict) and isinstance(live.get("state"), str) and live["state"] in LIVE_STATES, f"invalid live state: {case_id}") + require(isinstance(case.get("basis"), str) and bool(case["basis"].strip()), f"missing review basis: {case_id}") + require(isinstance(case.get("references"), list) and bool(case["references"]), f"missing implementation references: {case_id}") + for name in case["references"]: + local_file(root, name) + require(name in files, f"unreviewed implementation reference: {case_id}: {name}") + require(isinstance(case.get("gaps"), list), f"missing explicit gap list: {case_id}") + if state == "partial": + require(bool(case["gaps"]) and all(isinstance(gap, str) and gap.strip() for gap in case["gaps"]), + f"partial implementation needs actionable gaps: {case_id}") + else: + require(not case["gaps"], f"implemented case still has implementation gaps: {case_id}") + if state == "environment-gated": + policy = ENVIRONMENT_GATED_CASES.get(case_id) + require(policy is not None and policy["disabledByDefault"] is True + and live["state"] == "disabled", f"invalid disabled environment procedure: {case_id}") + require(case.get("required_capabilities") == policy["requiredCapabilities"] + and case.get("reason_code") == policy["reasonCode"], f"environment requirements drifted: {case_id}") + else: + require(case_id not in ENVIRONMENT_GATED_CASES, f"environment case lost its gate: {case_id}") + if live["state"] == "accepted": + require(state == "implemented", f"accepted live case lacks complete implementation: {case_id}") + runs = live.get("runs") + require(isinstance(runs, list) and bool(runs) + and all(type(run) is int and run > 0 for run in runs), f"missing accepted run identity: {case_id}") + require(live.get("basis") == "independent-complete-live-audit", f"unsupported acceptance claim: {case_id}") + elif state != "environment-gated": + require(isinstance(live.get("finding"), str) and bool(live["finding"].strip()), f"missing failure scope: {case_id}") + source_variants = case.get("source_variants") + require(isinstance(source_variants, dict) and set(source_variants) == variants.get(case_id, {"catalog-assertions"}), + f"source branch coverage changed: {case_id}") + for branch, references in source_variants.items(): + require(isinstance(references, list) and (bool(references) or state == "partial"), + f"source branch has no implementation: {case_id}: {branch}") + require(all(name in case["references"] for name in references), f"unreviewed branch reference: {case_id}: {branch}") + require(seen == set(catalog_cases), f"missing implementation reviews: {sorted(set(catalog_cases) - seen)}") + + +def summarize(review: dict) -> dict: + cases = review["cases"] + implementation = Counter(case["implementation"] for case in cases) + live = Counter(case["live"]["state"] for case in cases) + return { + "as_of": review["as_of"], + "total": len(cases), + "implementation": {state: implementation[state] for state in sorted(IMPLEMENTATION_STATES)}, + "live": {state: live[state] for state in sorted(LIVE_STATES)}, + "remaining_implementation": [{"case_id": case["case_id"], "gaps": case["gaps"]} + for case in cases if case["implementation"] == "partial"], + "all_implementation_complete": implementation["partial"] == 0, + "environment_adapters_required": [case["case_id"] for case in cases + if case["implementation"] == "environment-gated"], + "all_cases_live_accepted": live["accepted"] == len(cases), + "candidate_pass_claimed": False, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--review", type=Path, default=ROOT / "validation-migration-review.json") + parser.add_argument("--catalog", type=Path, default=ROOT / "validation-cases.json") + parser.add_argument("--require-complete", action="store_true", + help="fail on implementation gaps; candidate FAIL never changes this decision") + args = parser.parse_args() + try: + review = json.loads(args.review.read_text()) + catalog = json.loads(args.catalog.read_text()) + validate_review(review, catalog) + summary = summarize(review) + print(json.dumps(summary, ensure_ascii=False, indent=2)) + return int(args.require_complete and not summary["all_implementation_complete"]) + except (OSError, ValueError) as exc: + print(f"implementation review invalid: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test_validation_heal_lock_source.py b/test_validation_heal_lock_source.py new file mode 100644 index 00000000..7be4ffee --- /dev/null +++ b/test_validation_heal_lock_source.py @@ -0,0 +1,184 @@ +"""I05 source orchestration and links to real caller transport/ACK evidence.""" + +import copy +import json +import subprocess +import sys +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from scripts.validation_heal import lock_source as source, lock_timeout_runner as runner +from scripts.validation_heal import race_native as native +import test_validation_heal_lock_timeout as timeout_tests +import test_validation_heal_lock_successor as successor_tests +import test_validation_heal_race_native as native_tests + + +class SourceNativeTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + native_tests.RaceNativeTests().fixture(self.root, source=runner.CANDIDATE) + self.args = SimpleNamespace(native_directory=self.root, testing_commit=native_tests.HELPER) + self.candidate = {'source_sha': runner.CANDIDATE, 'package_sha256': 'a' * 64} + + def test_real_readonly_verifier_composes_owner_and_full_stale_authority(self): + with patch.object(native.subprocess, 'run', side_effect=AssertionError('never execute downloaded binary')): + proof = source.verify_source_native(self.args, self.candidate) + self.assertEqual(set(proof['groups']), set(source.GROUPS)) + self.assertTrue(proof['nativeOwnerAndStaleFencesPassed']) + self.assertFalse(proof['fleetExecutionClaimed'] or proof['canonicalCasePassClaimed']) + for case, label, count in ( + ('heal.race-versioned-put', 'explicit-version-append-final-rename', 6), + ('heal.race-version-delete', 'explicit-version-delete-final-rename', 12)): + regression = proof['groups'][case]['observedRegressions'][label] + self.assertEqual(regression['witnessKind'], 'ordered-barrier-milestones') + self.assertEqual(len(regression['milestones']), count) + self.assertIn('full_bodies_verified', regression['milestones'][-1]) + + def test_workflow_cli_verifies_all_native_subsets_without_running_binary(self): + candidate = self.root / 'candidate.json' + candidate.write_text(json.dumps(self.candidate)) + result = subprocess.run([sys.executable, '-B', str(Path(source.__file__)), + '--directory', str(self.root), '--candidate', str(candidate), + '--testing-commit', self.args.testing_commit], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + proof = json.loads(result.stdout) + self.assertEqual(set(proof['groups']), set(source.GROUPS)) + self.assertTrue(proof['nativeOwnerAndStaleFencesPassed']) + self.assertFalse(proof['canonicalCasePassClaimed']) + + def test_mixed_workflow_attempt_binary_or_source_cannot_form_one_proof(self): + proofs = source.verify_source_native(self.args, self.candidate)['groups'] + for key, value in (('runId', 999), ('runAttempt', 99), ('binarySha256', 'f' * 64), + ('candidateSourceTree', 'f' * 40), ('selectedContractSha256', 'f' * 64)): + with self.subTest(key=key): + changed = copy.deepcopy(proofs) + changed['heal.race-version-delete'][key] = value + with patch.object(source, 'verify_native', side_effect=lambda d, c, h, case: changed[case]): + with self.assertRaisesRegex(ValueError, 'mix candidates'): + source.verify_source_native(self.args, self.candidate) + + def test_missing_stale_body_milestone_or_original_log_blocks_before_fleet(self): + log = self.root / 'native-evidence/explicit-version-append-final-rename.log' + log.write_text(log.read_text().replace('full_bodies_verified', 'etag_only')) + with self.assertRaisesRegex(ValueError, 'original native log bytes differ'): + source.verify_source_native(self.args, self.candidate) + + def test_missing_artifact_does_not_execute_a_partial_source_plan(self): + self.args.native_directory = None + with self.assertRaisesRegex(ValueError, 'requires exact-source native evidence'): + source.verify_source_native(self.args, self.candidate) + + +class SourceEvidenceTests(unittest.TestCase): + def scene(self, branch): + case = (timeout_tests.TimeoutFlowTests() if branch == 'timeout' else successor_tests.SuccessorFlowTests()) + case.setUp() + self.addCleanup(case.doCleanups) + diagnostic = case.invoke() + # Components execute inside a topology directory in the production runner. + parent = case.output.parent + desired = parent / branch + if case.output != desired: + case.output.rename(desired) + return parent, case.probe.binding, diagnostic + + def test_actual_tcp_timeout_and_queued_successor_bytes_form_separate_chains(self): + # Each test fixture supplies a real single-use HTTP connection; server + # lock observations are controlled test data, never live native proof. + for branch in source.LIVE_BRANCHES: + with self.subTest(branch=branch): + parent, binding, diagnostic = self.scene(branch) + chain = source.link_fleet_evidence(parent, branch, binding, diagnostic) + self.assertTrue(chain['sourceEvidenceComplete']) + self.assertFalse(chain['heldAtDisconnectProven'] or chain['distributedReleaseProven']) + self.assertTrue(chain['nativeOwnerProofRequiredSeparately']) + self.assertEqual(chain['coordinatorTimes'], sorted(chain['coordinatorTimes'])) + self.assertIn(branch + '/caller/transport/request.json', chain['originalEvidenceSha256']) + self.assertNotEqual(chain['callerRequestId'], chain['foregroundRequestId']) + # Prevent shared fixture patches from nesting across subtests. + self.doCleanups() + + def test_foreign_binding_and_changed_request_digest_are_rejected(self): + parent, binding, diagnostic = self.scene('timeout') + foreign = {**binding, 'candidateCommit': 'f' * 40} + with self.assertRaisesRegex(ValueError, 'changed binding'): + source.link_fleet_evidence(parent, 'timeout', foreign, diagnostic) + path = parent / 'timeout/caller/transport/request.json' + intent = json.loads(path.read_text()) + intent['preparedMonotonicNs'] += 1 + path.write_text(json.dumps(intent)) + with self.assertRaisesRegex(ValueError, 'single unretried TCP'): + source.link_fleet_evidence(parent, 'timeout', binding, diagnostic) + + def test_clock_inversion_and_invented_disconnect_ownership_are_rejected(self): + parent, binding, diagnostic = self.scene('queued-successor') + path = parent / 'queued-successor/queued.json' + original = path.read_text() + queued = json.loads(original) + queued['observedMonotonicNs'] = 1 + path.write_text(json.dumps(queued)) + with self.assertRaisesRegex(ValueError, 'clock order'): + source.link_fleet_evidence(parent, 'queued-successor', binding, diagnostic) + path.write_text(original) + path = parent / 'queued-successor/caller/observation.json' + caller = json.loads(path.read_text()) + caller['heldAtDisconnectProven'] = True + path.write_text(json.dumps(caller)) + with self.assertRaisesRegex(ValueError, 'invent.*ownership'): + source.link_fleet_evidence(parent, 'queued-successor', binding, diagnostic) + + +class SourcePlanTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.args = SimpleNamespace(evidence=Path(self.tmp.name), topologies=['H1', 'H2']) + self.calls = [] + self.failure = None + self.proof = {'nativeOwnerAndStaleFencesPassed': True} + + def topology(self, args, candidate, inspector, topology): + label = (args.diagnostic_branch, topology) + self.calls.append((label, args.evidence, args.source_plan)) + return {'branch': label[0], 'topology': topology, 'diagnosticComplete': label != self.failure, + 'cleanup': 'FAIL' if label == self.failure else 'PASS', + 'sourceEvidenceChain': {'sourceEvidenceComplete': label != self.failure}, + 'canonicalCasePassClaimed': False} + + def test_serial_source_plan_preserves_each_controller_and_native_scope(self): + summary = source.run_source_plan(self.args, {}, {}, self.topology, self.proof) + self.assertEqual([row[0] for row in self.calls], [ + ('timeout', 'H1'), ('timeout', 'H2'), ('queued-successor', 'H1'), ('queued-successor', 'H2')]) + self.assertTrue(all(row[2] for row in self.calls)) + self.assertNotEqual(self.calls[0][1], self.calls[-1][1]) + self.assertTrue(summary['sourceProceduresComplete'] and summary['diagnosticComplete']) + self.assertEqual(summary['remainingLiveInstances'], []) + self.assertFalse(summary['complete'] or summary['wholeCasePassClaimed'] or summary['canonicalCasePassClaimed']) + self.assertEqual(set(summary['sourceBranches']), { + 'object-lock-timeout', 'caller-disconnect', 'stale-read-repair', 'native-lock-owner-fence'}) + self.assertEqual(summary['sourceBranches']['stale-read-repair']['scope'], 'independent-native') + self.assertEqual(json.loads((self.args.evidence / 'healthy-summary.json').read_text()), summary) + + def test_product_or_cleanup_failure_retains_first_scene_and_never_runs_next(self): + self.failure = ('timeout', 'H2') + summary = source.run_source_plan(self.args, {}, {}, self.topology, self.proof) + self.assertEqual([row[0] for row in self.calls], [('timeout', 'H1'), ('timeout', 'H2')]) + self.assertFalse(summary['sourceProceduresComplete']) + self.assertEqual(summary['cleanup'], 'FAIL') + self.assertEqual(summary['remainingLiveInstances'], ['timeout/H2', 'queued-successor/H1', 'queued-successor/H2']) + + def test_h1_only_cannot_claim_h2_source_procedures_complete(self): + self.args.topologies = ['H1'] + summary = source.run_source_plan(self.args, {}, {}, self.topology, self.proof) + self.assertTrue(summary['diagnosticComplete']) + self.assertFalse(summary['sourceProceduresComplete']) + + +if __name__ == '__main__': + unittest.main() diff --git a/test_validation_heal_lock_timeout_runner.py b/test_validation_heal_lock_timeout_runner.py index 24092016..7a37acde 100644 --- a/test_validation_heal_lock_timeout_runner.py +++ b/test_validation_heal_lock_timeout_runner.py @@ -187,6 +187,67 @@ def test_preflight_checks_exact_source_helper_tools_and_workflow_before_remote_u with self.assertRaisesRegex(ValueError, 'exact reviewed'): runner.preflight(self.args) self.assertFalse(self.fleets or self.events) + def test_source_plan_runs_four_isolated_instances_through_real_recorder(self): + self.args.diagnostic_branch = 'source-plan' + proof = {'nativeOwnerAndStaleFencesPassed': True} + def link(directory, branch, binding, diagnostic): + self.assertEqual(directory, self.fleets[-1].evidence) + self.assertEqual(binding['runId'], self.fleets[-1].run_id) + self.assertTrue(list((directory / 'fixture').glob('*-ack.json'))) + self.events.append('source-chain') + return {'sourceEvidenceComplete': True} + with patch.object(runner, 'preflight', return_value=(self.candidate, self.inspector)), \ + patch.object(runner.lock_source, 'verify_source_native', return_value=proof), \ + patch.object(runner, 'verify_native_prerequisite', return_value=proof), \ + patch.object(runner, 'observe_successor', side_effect=self.successor), \ + patch.object(runner, 'map_target', side_effect=self.new_target), \ + patch.object(runner, 'verify_cohort', side_effect=self.integrity), \ + patch.object(runner.lock_source, 'link_fleet_evidence', side_effect=link) as chain: + result = runner.run(self.args) + self.assertTrue(result['sourceProceduresComplete']) + self.assertFalse(result['canonicalCasePassClaimed'] or result['wholeCasePassClaimed']) + self.assertEqual(chain.call_count, 4) + self.assertEqual([fleet.runtime_profile for fleet in self.fleets], + ['lock-timeout'] * 2 + ['lock-successor'] * 2) + self.assertEqual(len({fleet.run_id for fleet in self.fleets}), 4) + self.assertEqual({fleet.evidence.relative_to(self.args.evidence).as_posix() for fleet in self.fleets}, + {'source-plan/timeout/H1', 'source-plan/timeout/H2', + 'source-plan/queued-successor/H1', 'source-plan/queued-successor/H2'}) + bindings = [json.loads(next(fleet.evidence.glob('*-binding.json')).read_text()) for fleet in self.fleets] + self.assertEqual(len({binding['bucketPrefix'] for binding in bindings}), 4) + self.assertEqual([event for event in self.events if event in ('source-chain', 'cleanup')], + ['source-chain', 'cleanup'] * 4) + + def test_source_chain_failure_preserves_original_scene_before_cleanup(self): + self.args.diagnostic_branch = 'source-plan' + with patch.object(runner, 'preflight', return_value=(self.candidate, self.inspector)), \ + patch.object(runner.lock_source, 'verify_source_native', return_value={}), \ + patch.object(runner.lock_source, 'link_fleet_evidence', side_effect=ValueError('broken original ACK')): + result = runner.run(self.args) + self.assertFalse(result['sourceProceduresComplete']) + self.assertEqual(len(self.fleets), 1) + self.assertIn('broken original ACK', result['diagnostics'][0]['failure']['message']) + self.assertTrue(result['diagnostics'][0]['namespacesRetained']) + self.assertNotIn('cleanup', self.events) + self.assertIn('original-failure-scene', self.events) + self.assertTrue(list(self.fleets[0].evidence.glob('*-original-failure-logs.json'))) + + def test_source_plan_requires_native_before_deployment_and_evidence_creation(self): + self.args.diagnostic_branch = 'source-plan' + with patch.object(runner, 'preflight', return_value=(self.candidate, self.inspector)): + with self.assertRaisesRegex(ValueError, 'requires exact-source native evidence'): + runner.run(self.args) + self.assertFalse(self.fleets or self.events or self.args.evidence.exists()) + + def test_cli_defaults_to_complete_source_plan(self): + argv = self.cli() + start = argv.index('--diagnostic-branch') + del argv[start:start + 2] + with patch.dict(os.environ, {}, clear=True), patch.object(runner.sys, 'argv', argv), \ + patch.object(runner, 'run', return_value={'diagnosticComplete': True}) as run: + self.assertEqual(runner.main(), 0) + self.assertEqual(run.call_args.args[0].diagnostic_branch, 'source-plan') + def test_queued_successor_preflight_requires_exact_candidate_native_group(self): self.args.diagnostic_branch = 'queued-successor' self.args.native_directory = self.root / 'native' @@ -342,7 +403,7 @@ def cli(self): return ['lock_timeout_runner.py', '--candidate', str(self.candidate_file), '--case-id', runner.CASE_ID, '--testing-commit', self.args.testing_commit, '--binary-sha', self.args.binary_sha, '--tool-directory', str(self.args.tool_directory), '--rc-bin', str(self.rc), - '--evidence', str(self.args.evidence)] + '--evidence', str(self.args.evidence), '--diagnostic-branch', 'timeout'] def test_actual_cli_accepts_wrapper_directory_and_keeps_canonical_summary_blocked(self): self.args.evidence.mkdir() diff --git a/test_validation_heal_persistence.py b/test_validation_heal_persistence.py index e29c7a9a..740cdc90 100644 --- a/test_validation_heal_persistence.py +++ b/test_validation_heal_persistence.py @@ -219,6 +219,86 @@ def test_incomplete_branches_never_enter_fleet_mutation(self): with self.subTest(branch=branch), self.assertRaisesRegex(ValueError, "cannot mutate"): persistence.run_checkpoint_fault(None, "H1", ["node"], "DB", branch, {}) + def test_native_persistence_is_collected_and_failures_remain_independent(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + native_dir = root / "native" + evidence = root / "evidence" + args = SimpleNamespace(native_directory=native_dir, evidence=evidence, + testing_commit="a" * 40) + candidate = {"source_sha": "1aa9be87d18cf941e5020f19e8622c26ba4b10ac"} + manifest = {"candidateCommit": candidate["source_sha"]} + proof = {**manifest, "complete": True, "exactTests": 6, + "canonicalCasePassClaimed": False, "fleetPassClaimed": False} + with patch.object(persistence.persistence_native, "binding", return_value=(manifest, root / "binary")), \ + patch.object(persistence.persistence_native, "run") as run, \ + patch.object(persistence.persistence_native, "verify", return_value=proof) as verify: + result = persistence.collect_native_persistence(args, candidate) + run.assert_called_once_with(native_dir / "native-tools", native_dir / "native-evidence", + evidence / "native-persistence", "a" * 40) + verify.assert_called_once_with(native_dir / "native-tools", native_dir / "native-evidence", + evidence / "native-persistence", "a" * 40) + self.assertEqual(result, proof) + self.assertEqual(json.loads((evidence / "independent-native-persistence.json").read_text()), proof) + branch = persistence.native_branch_result("H1", "DB", {"candidateCommit": candidate["source_sha"]}, result) + self.assertTrue(branch["requiredNativeCoverageComplete"]) + self.assertFalse(branch["canonicalCasePassClaimed"]) + self.assertFalse(branch["fleetBranchExecuted"]) + + failed_evidence = root / "failed" + failed_args = SimpleNamespace(native_directory=native_dir, evidence=failed_evidence, + testing_commit="a" * 40) + with patch.object(persistence.persistence_native, "binding", return_value=(manifest, root / "binary")), \ + patch.object(persistence.persistence_native, "run", side_effect=RuntimeError("native failed")), \ + patch.object(persistence.persistence_native, "verify") as verify_failed: + failure = persistence.collect_native_persistence(failed_args, candidate) + verify_failed.assert_not_called() + self.assertFalse(failure["complete"]) + self.assertFalse(failure["canonicalCasePassClaimed"]) + retained = json.loads((failed_evidence / "independent-native-persistence-failure.json").read_text()) + self.assertEqual(retained["failure"]["message"], "native failed") + failed_branch = persistence.native_branch_result("H1", "DB", {"manager": True}, failure) + self.assertFalse(failed_branch["requiredNativeCoverageComplete"]) + self.assertTrue(failed_branch["nativeFailureRetained"]) + self.assertFalse(failed_branch["canonicalCasePassClaimed"]) + + def test_manager_and_directory_fsync_proofs_cannot_mix_candidates(self): + with self.assertRaisesRegex(ValueError, "mix candidates"): + persistence.native_branch_result("H1", "DB", {"candidateCommit": "a" * 40}, + {"candidateCommit": "b" * 40, "complete": True}) + + def test_native_evidence_from_a_different_candidate_never_executes(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + args = SimpleNamespace(native_directory=root / "native", evidence=root / "evidence", + testing_commit="a" * 40) + candidate = {"source_sha": "072b57b37559fbe626aeda8abf61617fdcbdd926"} + manifest = {"candidateCommit": "1aa9be87d18cf941e5020f19e8622c26ba4b10ac"} + with patch.object(persistence.persistence_native, "binding", return_value=(manifest, root / "binary")), \ + patch.object(persistence.persistence_native, "run") as run, \ + patch.object(persistence.persistence_native, "verify") as verify: + failure = persistence.collect_native_persistence(args, candidate) + run.assert_not_called() + verify.assert_not_called() + self.assertFalse(failure["complete"]) + self.assertIn("native and fleet candidates differ", failure["failure"]["message"]) + self.assertTrue((args.evidence / "independent-native-persistence-failure.json").is_file()) + + def test_native_proof_cannot_change_candidate_after_execution(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + args = SimpleNamespace(native_directory=root / "native", evidence=root / "evidence", + testing_commit="a" * 40) + candidate = {"source_sha": "1aa9be87d18cf941e5020f19e8622c26ba4b10ac"} + manifest = {"candidateCommit": candidate["source_sha"]} + with patch.object(persistence.persistence_native, "binding", return_value=(manifest, root / "binary")), \ + patch.object(persistence.persistence_native, "run"), \ + patch.object(persistence.persistence_native, "verify", return_value={"complete": True, "candidateCommit": "b" * 40}): + failure = persistence.collect_native_persistence(args, candidate) + self.assertFalse(failure["complete"]) + self.assertIn("changed the selected candidate", failure["failure"]["message"]) + self.assertFalse((args.evidence / "independent-native-persistence.json").exists()) + def test_native_branch_reuses_exact_evidence_without_claiming_fleet_or_full_case_pass(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp); candidate = root / "candidate.json" diff --git a/test_validation_migration.py b/test_validation_migration.py index 831109a8..d82c2798 100644 --- a/test_validation_migration.py +++ b/test_validation_migration.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import copy import json import subprocess import sys @@ -9,6 +10,7 @@ from scripts.validation_catalog import build_snapshot, validate_reuse_candidates, validate_snapshot from scripts.validation_report import validate_results +from scripts.validation_migration_review import summarize, validate_review ROOT = Path(__file__).resolve().parent @@ -253,5 +255,119 @@ def test_predeployment_blocked_result_does_not_claim_an_installed_binary(self): self.assertIsNone(b04_value["provenance"]["binary_sha256"]) +class ImplementationReviewTest(unittest.TestCase): + def setUp(self): + self.catalog = json.loads((ROOT / "validation-cases.json").read_text()) + self.review = json.loads((ROOT / "validation-migration-review.json").read_text()) + + def test_explicit_review_covers_the_entire_catalog(self): + validate_review(self.review, self.catalog) + summary = summarize(self.review) + self.assertEqual(summary["total"], 158) + self.assertEqual(sum(summary["implementation"].values()), 158) + self.assertEqual(sum(summary["live"].values()), 158) + self.assertFalse(summary["candidate_pass_claimed"]) + self.assertFalse(summary["all_cases_live_accepted"]) + + def test_retained_failure_is_independent_of_implementation_completion(self): + # Candidate failures must never silently become missing test code. + case = next(case for case in self.review["cases"] + if case["implementation"] == "implemented" + and case["live"]["state"] == "retained-failure") + before = summarize(self.review)["implementation"] + case["live"]["finding"] = "Candidate request returned a retained unexpected response; cause remains under review." + validate_review(self.review, self.catalog) + self.assertEqual(before, summarize(self.review)["implementation"]) + with tempfile.TemporaryDirectory() as directory: + result = {"schema_version": 1, "case_id": case["case_id"], + "source_status": next(source["source_status"] for module in self.catalog["modules"] + for source in module["cases"] if source["id"] == case["case_id"]), + "migration_status": "unverified", "profile": "cluster-1pool", + "verdict": "FAIL", "cleanup_status": "PASS", "reason": case["live"]["finding"]} + result_path = Path(directory) / "retained.json" + result_path.write_text(json.dumps(result)) + original = result_path.read_bytes() + _, errors = validate_results(self.catalog, Path(directory), gate_statuses=set(), + gate_cases={case["case_id"]}) + self.assertIn(f"{case['case_id']}: gate verdict is 'FAIL'", errors) + self.assertEqual(original, result_path.read_bytes()) + + def test_missing_or_duplicate_id_is_rejected(self): + for change, reason in (("missing", "missing implementation reviews"), + ("duplicate", "duplicate implementation case")): + with self.subTest(change=change): + review = copy.deepcopy(self.review) + if change == "missing": + review["cases"].pop() + else: + review["cases"].append(copy.deepcopy(review["cases"][0])) + with self.assertRaisesRegex(ValueError, reason): + validate_review(review, self.catalog) + + def test_source_or_code_drift_requires_another_review(self): + for change, reason in (("source", "another source catalog"), + ("code", "implementation changed since review"), + ("branch", "source branch coverage changed")): + with self.subTest(change=change): + review = copy.deepcopy(self.review) + if change == "source": + review["source_commit"] = "0" * 40 + elif change == "code": + review["reviewed_files"][next(iter(review["reviewed_files"]))] = "0" * 64 + else: + case = next(case for case in review["cases"] if "source_variants" in case) + case["source_variants"].pop(next(iter(case["source_variants"]))) + with self.assertRaisesRegex(ValueError, reason): + validate_review(review, self.catalog) + + def test_partial_implementation_must_name_a_concrete_gap(self): + case = self.review["cases"][0] + case.update(implementation="partial", gaps=[]) + with self.assertRaisesRegex(ValueError, "actionable gaps"): + validate_review(self.review, self.catalog) + + def test_default_disabled_environment_requirements_cannot_disappear(self): + case = next(case for case in self.review["cases"] if case["implementation"] == "environment-gated") + case["required_capabilities"] = [] + with self.assertRaisesRegex(ValueError, "environment requirements drifted"): + validate_review(self.review, self.catalog) + + def test_unreviewed_or_nonlocal_branch_references_are_rejected(self): + for name, reason in (("../outside.py", "nonlocal implementation reference"), + ("validation-migration-review.json", "unreviewed implementation reference")): + with self.subTest(name=name): + review = copy.deepcopy(self.review) + review["cases"][0]["references"] = [name] + with self.assertRaisesRegex(ValueError, reason): + validate_review(review, self.catalog) + + def test_invalid_record_types_raise_a_review_error(self): + for invalid in (None, [], {"implementation": []}, {"case_id": "unknown"}): + with self.subTest(invalid=invalid): + review = copy.deepcopy(self.review) + review["cases"][0] = invalid + with self.assertRaises(ValueError): + validate_review(review, self.catalog) + + def test_require_complete_checks_code_gaps_without_demanding_product_pass(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "review.json" + command = [sys.executable, "-B", str(ROOT / "scripts/validation_migration_review.py"), + "--review", str(path), "--require-complete"] + path.write_text(json.dumps(self.review)) + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertGreater(json.loads(result.stdout)["live"]["retained-failure"], 0) + self.assertFalse(json.loads(result.stdout)["candidate_pass_claimed"]) + # A real implementation gap must fail even with unchanged product results. + case = next(case for case in self.review["cases"] + if case["implementation"] == "implemented" and case["live"]["state"] == "retained-failure") + case.update(implementation="partial", gaps=["Missing source controller in this negative fixture."]) + path.write_text(json.dumps(self.review)) + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(json.loads(result.stdout)["remaining_implementation"][0]["case_id"], case["case_id"]) + + if __name__ == "__main__": unittest.main() diff --git a/validation-migration-review.json b/validation-migration-review.json new file mode 100644 index 00000000..e9ca52e9 --- /dev/null +++ b/validation-migration-review.json @@ -0,0 +1,3223 @@ +{ + "schema_version": 1, + "as_of": "2026-10-28", + "source_commit": "524eb5b32c3236497fdd8a9271df0494fa3d9974", + "source_sha256": "463fdd02dee43e42fef4f3aacf7506709ce87af535e1a53d91c521bc2e361cad", + "reviewed_testing_commit": "f0744d198769feecb2cb038ab96b507a1784d44d", + "scope": "Test implementation migration and candidate live acceptance are independent. This review does not rewrite original results, change migration_status semantics or relax the canonical gate. Environment procedures remain disabled by default under the accepted delivery scope.", + "evidence_basis": "20261026-canonical-migration-ledger-7094.json plus later G01/G02 attempts and source review on 2026-10-28; historical result bytes and audit reports remain in the separate evidence store.", + "reviewed_files": { + "scripts/run_validation_case.sh": "ac51328ff860074f285999236a07ec8d2274d56607dc7c6c6389747fd7a0c498", + "scripts/validation_heal/authority.py": "8f2225f184427674de1f45aff4c4976c812f9d70fd4bc927ce460726a4ff74a0", + "scripts/validation_heal/authority_runner.py": "d1dce07019a37318f9fbfc9c4d7383fa259d8d6da881ebacf083a67d04fc2545", + "scripts/validation_heal/bounded_volume.py": "56f57d2bef882af81a7bf05365fa8b5ae01f9fcdd515b3bc930e1f36cf98e944", + "scripts/validation_heal/bucket_metadata.py": "3fba015f0c3ca60b979b404d51c3071a61324ba1d1e9558be85b178b8f2d19fb", + "scripts/validation_heal/bucket_records.py": "2b46fc9923eb9d5f6d913688885580e270ac697c68467f2ce63d484fa3d7c2e1", + "scripts/validation_heal/caller_abort.py": "908f666cd689f4b71c3a6fac0b795b26d9058c1e20efd1e8f4858dc2359810af", + "scripts/validation_heal/capacity.py": "67a2b0836603cb7ddbb526a528c5f7896527cec406ba30c92afed4fc8188c990", + "scripts/validation_heal/coordinator_crash.py": "9188d624e2429900c168e8500b740a4b3222831dc465eb74eab9fb62c11ed235", + "scripts/validation_heal/environment.py": "93f13101553e6d3052147f1b4848cead44c201dc8447b4de5c898f7c1201865a", + "scripts/validation_heal/environment_gate.py": "b54ccb7a100d111289380759eb4b10c65a324280d3f50551031d87243205fcb4", + "scripts/validation_heal/execution_policy.py": "b5582404e09f19d70255f8da71cea0895c154afe4e1b3f5497c2d76361e53b6a", + "scripts/validation_heal/flapping.py": "58ea6a0d480fa3188975658421f9dd8a92a89109fbaa2f34329f668f71bd1096", + "scripts/validation_heal/generation_fault.py": "5a35483a3e4d21cbfede1e1262bb6bf911bb077f7f08eaeb94b34204d19775c1", + "scripts/validation_heal/incarnation.py": "8ea507c13a471fbc52e9666c26a3ab3f6e276337d4c75579885ba9f929f277c9", + "scripts/validation_heal/incarnation_native.py": "1cd9d069953d833e1f37455ded67719d52a2dedc9bd57d1295de6167443002c0", + "scripts/validation_heal/incarnation_replay.py": "e53de3d51f910a4f8467f6bcfaeb3496a7bc8b655acc0e7cd4a906ad3f1ba364", + "scripts/validation_heal/journal_fault.py": "cc171b8f53f6faf54ef4ebf559a8cdfcf6ec0c59ca3e290cb636b57a9c5e1a12", + "scripts/validation_heal/lock_observer.py": "9f513ec659e27338b05a23447ce63dfaecbb69eb6a53852f9eacd86b3cc956d3", + "scripts/validation_heal/lock_source.py": "02881928da918e20ca1f184dfac91518e92974d0fbb408a9c0adde274e47d889", + "scripts/validation_heal/lock_successor.py": "05ce5bbe93cad2fa2bec5c7b6ea8c6d956e40690dff58ac850c4aa47a4d6a683", + "scripts/validation_heal/lock_timeout.py": "e7e672955cfdbe30579476989125b31ccb997014fdaabd7867749742abd37b39", + "scripts/validation_heal/lock_timeout_runner.py": "0b546bb38471d0005b6d01a56844d633cae938d4196f82e3177f4345f0065304", + "scripts/validation_heal/manager_fixture.py": "d73018ed9261b9d0545c5ca6a8db51c5a225bc196e085950320c1a4ab51045e5", + "scripts/validation_heal/manager_native.py": "40d77e6bd18384520659e1cbe20ba58892f22929ef694c6ad39ccfd6d2af3740", + "scripts/validation_heal/markers.py": "4ddb66c0a0defd041acaccdfbf7db6f79c51633b76143cf3557200c080e016b5", + "scripts/validation_heal/mpu_race_runner.py": "22cdb841747882a6666988fa916b85aaae548ebb5314927373ef181b6347d25d", + "scripts/validation_heal/mrf_records.py": "0a262ba5e0ac61605c48bc8c582b7f7903a03d51585ec8c242d242240f43a7c9", + "scripts/validation_heal/mrf_replay.py": "95d4419232bf12799f179abd12d2073e7831b093aa3b2dd4ef386c12826da879", + "scripts/validation_heal/multi_pool.py": "82397e5d5562af01e3c5010c28def365368a669bef82aa3b1efd5f5633561c20", + "scripts/validation_heal/multi_pool_layout.py": "365febcf07fe05ccd05af92e17d20b744e6603271e42555271208dfb96121671", + "scripts/validation_heal/native_regressions.py": "96ed809536d3275e72bdbcc64aa534f65976da5d6e0e85b7056623a14c7e3702", + "scripts/validation_heal/network.py": "c969e3acf791a86dc1d425306af421339e358bef02200bd9639fa5c65e317c29", + "scripts/validation_heal/network_gate.py": "f4291a98095aee46bf1213d5e70a59b83858b213a484cc2c452e82c514c02d1a", + "scripts/validation_heal/network_load.py": "169a7b59c5cf12b939446f778735ad7edcb939817f23420725349487c2a45d04", + "scripts/validation_heal/node_outage.py": "cf10109d7df387550a985bee9d65e4af19b3c6618563b19e3418d48da7b23fab", + "scripts/validation_heal/nolock.py": "be1812cd4f03a433f808b468e59e26868047a765a70e2b1890a5d61cd5af264b", + "scripts/validation_heal/outcome.py": "173ea18f85f9f340bf8bb9d97924a74d01697488fe9bb52896cab5a4441ee0af", + "scripts/validation_heal/outcome_grace.py": "a4cf21cd442be6280194796b179ff8d756322d557c108a3d559d9b0e7ed52374", + "scripts/validation_heal/outcome_recovery.py": "e623c993afcfebdf392fd54f7d2e958e8835828b37865b6452373229bffb80ea", + "scripts/validation_heal/performance.py": "81f85a2ea0ecad3b8ddec701eba776b98b1ace5f1e303631bc53cf5c73bf5a57", + "scripts/validation_heal/persistence.py": "a1f3d86cf359bbcbb6be1f60f01f3c8bb4c69dfc74cf678a85c4f0170a73ccda", + "scripts/validation_heal/persistence_mrf.py": "018fcbde989d2a2c4f9f4c18ce1b066a6acff323ac5fbc1f4818a6e3b2502c06", + "scripts/validation_heal/persistence_native.py": "368dcc4b6171b4aba119694dc7b510faf460cc5a4a7d8e36d1a28f2b0583355a", + "scripts/validation_heal/race.py": "ebae41951dea116549ef7f3082a5e023151e9769e016641c19ebbd91028cc86c", + "scripts/validation_heal/race_io.py": "b45930a0327e18716a1953210888760717621bf229981d0bd8ed7b3c0d4f1322", + "scripts/validation_heal/race_native.py": "fca55264c42ebad30fc796085512e959fa827753b3f0f52e14c09738f21464f6", + "scripts/validation_heal/race_native_contracts.json": "cdd6228d9c99dc1c550d4044743d78c58e95c22f0bb8076bc9611b2fc18e01a4", + "scripts/validation_heal/race_native_evidence.py": "caa2b5c63a419e2c0c29c6d5e38b50ab1ca34df03de10ccbd2e30954158339f4", + "scripts/validation_heal/recipes.json": "9dbfc7336e9cac1b8b8b7f8b5889eeb117cf4bca1678774249f553ddc8270a03", + "scripts/validation_heal/replace_drive.py": "dd7e13a85fdaf5e92b2f744167c6a8e4af9307ca827a78462e11082a2ba2349c", + "scripts/validation_heal/replacement.py": "051161cf09769f327596e30088fa0630014ddf5fac039dcca01e0b613ab84b98", + "scripts/validation_heal/replacement_preflight.py": "5440f25f5d9951e92513953733d4975f3893c883b0d4403ee6280131e2ca20d0", + "scripts/validation_heal/run_case.sh": "c8ae24b39c554c52a0f85689617c46d1eeba10d4525c403c82da11c06ac180e6", + "scripts/validation_heal/schema.py": "75a92ddea811e744a405ff1fde1968c92c8e2424b48e063a6ea23d07de529efa", + "scripts/validation_heal/schema_faults.py": "80a3f88920e6312950ef09f044ead08afbb5b7839a2f1c4895eba57cdf821e5d", + "scripts/validation_heal/status.py": "3c9af875227afafb3b9be61bb2c5a818fed247916b52e0fd4619cd8153c62cc0", + "scripts/validation_heal/unformatted.py": "70d00af8ac1ef7e9bbd1d500c5c4de06b7995bd5219dc59527c824a56cb8f2c2", + "scripts/validation_heal/version_race_runner.py": "5dd244ae2aa816907d3711a6a7c9bf3dac76265c76967f7803437195894ff0e0", + "scripts/validation_heal/versions.py": "fa77397a58b0db4499114c0e30a766ef67f997d1c88f22fcfe02752820b5eea0", + "scripts/validation_pool/case_plan.py": "277777e040501c8a79e7104471f411b054d901afb920d18f3e222d3a0b7b03bc", + "scripts/validation_pool/pool_behavior.py": "203bc3a2b7e85fb95482dd33efc937cff9f1f7e23f8ac2aa0b9ad2f40aff4632", + "scripts/validation_pool/run_case.sh": "a1de6d972964ae4232bdc1c7c4a61cb9e4a444825c7c1ced5c74cbc72f6a5ebe", + "test_validation_heal.py": "6e4e25d2437a49db1af0b3020bb3a2509496b21cb3c27216a6b0322aea9bfbdc", + "test_validation_heal_authority.py": "a4334246aa9572be05a3d6881779df1e2495d9706d3a9289ba9437ac866a688a", + "test_validation_heal_bucket_metadata.py": "331bb5aa9bce132bd2d69b81262dae0eeffd76761139c9a382dd3884f400fb1c", + "test_validation_heal_caller_abort.py": "0d67e6f3b399c4c0392c768cf7bf9d7dd8e6166d911b942d7ed3826cb0aacfc5", + "test_validation_heal_capacity.py": "a07caec36bba2522b7ad48a86a953f8a702fc5219e0dd63d34ecb8b6f6f139e5", + "test_validation_heal_coordinator_crash.py": "10b8ff58c618e85b4cc97e4780cc8abe1b183f1be6f64a8a5649fe36d8f3a691", + "test_validation_heal_environment.py": "6c3a22ae0d091db396b8f9875eed9779ac35e570c90496dffa262d4cb09c34c3", + "test_validation_heal_environment_gate.py": "2ecbb7fe0fe2e723aa93160f609b2c63e97c6aa807a217f67170b169a9e3f9a1", + "test_validation_heal_flapping.py": "e310601f83d75dd65f6fa268484f40d56a7f8e8412fa9499f942d86bd53e0e3d", + "test_validation_heal_g02_source.py": "e817c99de1c7b213ca8b6c1277064ed3de2f438b4f82fa1ece14588c67870ff4", + "test_validation_heal_g03_source.py": "9cf62d56f13decdb5665d4da6cd34840c17ae6277d3e6ce829640b1bcc4b02f3", + "test_validation_heal_incarnation.py": "e9ade5528ded36ad9d04ec5b7bff623b9f68a9e646a9ab118ff56420c0408455", + "test_validation_heal_incarnation_native.py": "2cb9ed6763f45c3e6f124ff4053a7b183f35b3327153ac67818d65b74442e0b6", + "test_validation_heal_incarnation_replay.py": "89aa38d154152b699026c790e65093f1e28d9c4c2b6e8b6bf0f742074a39c3d3", + "test_validation_heal_journal_fault.py": "712ae93218b99cf4a793784ac860fe3a8c4fe1ab3e3577a2c73371a4de9a5d27", + "test_validation_heal_lock_observer.py": "d4b450b9f8251390c1128123129b45e7c3ad3190b0de4f4f6c2ae4175cf03f51", + "test_validation_heal_lock_source.py": "6038f1f59e878c6d6607d531733475949f5e8f788d8aefccd1258def6e40c513", + "test_validation_heal_lock_successor.py": "d31a684464619470c0de8b3213f08aaf055d7f56b220b7c4c2ee725d31026b38", + "test_validation_heal_lock_timeout.py": "59cd644449ed038a269ec52d735566da2e79a81e3d36a887ed68c289af0c7741", + "test_validation_heal_lock_timeout_runner.py": "e07c3ed6955a5d5c004bb648b2884c6867520716a8619a8302f85e2c30f0f819", + "test_validation_heal_manager_native.py": "a5635fae08b57f2a7fff8725549c580ba7fe48c15df8421889c80f519ae46da4", + "test_validation_heal_mpu_race_runner.py": "d0831311db460fe476831718c7268334eba529dd2978add1569f0eeeb77c3ade", + "test_validation_heal_mrf.py": "f9c9eb680e8e4686290baa58dae7a0e390ad4026b1714a19669ec83610c1be01", + "test_validation_heal_multi_pool.py": "d2bf816363548378bf63eae21e09bf4770874538a471615d8e1d450828d79352", + "test_validation_heal_native.py": "207b043f91fb4ce64e3a535a353c1c81b99ebf62964f68a7f66eb81d99c6c7ec", + "test_validation_heal_network.py": "b1ed0b1416f9b7b0a1954056df304d3ec667f2e7af935a86589e3563c1756acf", + "test_validation_heal_nolock.py": "66ca7212e9fdb86ffdb2cd3a6e9186a2f484d57edda287a900972cb9409a1771", + "test_validation_heal_outcome.py": "6744c6f0f79aba06e9ac1a97e51b69e5f6cef86a361ee9e2a2d201156fd05631", + "test_validation_heal_outcome_grace.py": "391f5ad225f80661562f59be9cb5150a857c0d258e3aa852e00cc07b1068cf4e", + "test_validation_heal_outcome_recovery.py": "46b21a03ad7fc656273d930178c1ebefb212bcc9faa125b3834c6f87a102305c", + "test_validation_heal_performance.py": "92e3e09fca551c0a8a1daef4d7abdaba11e13dd4c021a33e601ad1e0bad7d924", + "test_validation_heal_persistence.py": "d258b82634bfd19db7c4b1f6fb5dadcf33fafc595e3663a3a30a4720d0079f4f", + "test_validation_heal_persistence_mrf.py": "61dfa042c6b237e00b058b2660962411e201cda8908027c6900649da344b4cbe", + "test_validation_heal_race.py": "1f2097aff5ad08c4d2834bfd38e1eb1d1b373ecde8f7e6a3455b713114a07d3d", + "test_validation_heal_race_native.py": "850c9683ac918834d22b002ec2f5a8c0d7fd2b74266461e4912776dfceb0d93d", + "test_validation_heal_schema.py": "5df440df29e7085d48590665129f87b0791f9fa8c7f64c8c0432000bc2002606", + "test_validation_heal_unformatted.py": "58bc43f3f0905c8c4b7d0f8e656d25f0a17e197b5a2dfb66a88b6af9747883a7", + "test_validation_pool.py": "300adcef396533b8cbb54864ccd043b9f1b897558042039ad9b9aec60ed2aade" + }, + "cases": [ + { + "case_id": "pool.status", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38047001285 + ] + } + }, + { + "case_id": "pool.expand-and-rebalance", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38054208114 + ] + } + }, + { + "case_id": "pool.expansion-restart-persistence", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38054287896 + ] + } + }, + { + "case_id": "pool.rebalance-stop-resume", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110959836 + ] + } + }, + { + "case_id": "pool.rebalance-live-io", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38054696080 + ] + } + }, + { + "case_id": "pool.decommission", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38051227313 + ] + } + }, + { + "case_id": "pool.decommission-cancel-clear-resume", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38052304396 + ] + } + }, + { + "case_id": "pool.decommission-restart-recovery", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38053814812 + ] + } + }, + { + "case_id": "pool.decommission-versioned-integrity", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38054128863 + ] + } + }, + { + "case_id": "pool.decommission-multipart-integrity", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_pool/run_case.sh", + "scripts/validation_pool/pool_behavior.py", + "scripts/validation_pool/case_plan.py", + "test_validation_pool.py" + ], + "source_variants": { + "catalog-assertions": [ + "scripts/validation_pool/run_case.sh" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38053715867 + ], + "original_result_sha256": "ac83aa195a66592971b36cff1734aad8ebf5f6231cd1f1cfe1a0f720fd0bc786" + } + }, + { + "case_id": "pool.decommission-live-io", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38053444515 + ] + } + }, + { + "case_id": "pool.retired-config-removal", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38111198632 + ] + } + }, + { + "case_id": "pool.operation-guards", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38054540360 + ] + } + }, + { + "case_id": "heal.admin-healthy-noop", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38056636793 + ] + } + }, + { + "case_id": "heal.admin-scope-isolation", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38094094085 + ] + } + }, + { + "case_id": "heal.admin-token-cross-node-status", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38198122041 + ] + } + }, + { + "case_id": "heal.admin-dry-run", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38057908339 + ] + } + }, + { + "case_id": "heal.admin-overlap-and-force-start", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 39892106641 + ] + } + }, + { + "case_id": "heal.admin-stop-and-stale-token", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38099706345 + ] + } + }, + { + "case_id": "heal.admin-invalid-request", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38052868969, + 38053214984 + ] + } + }, + { + "case_id": "heal.admin-authorization", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38069648003 + ] + } + }, + { + "case_id": "heal.admin-nolock-safety", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/nolock.py", + "scripts/validation_heal/race_native_evidence.py", + "scripts/validation_heal/native_regressions.py", + "test_validation_heal_nolock.py", + "test_validation_heal_native.py" + ], + "source_variants": { + "overwrite-nolock": [ + "scripts/validation_heal/nolock.py" + ], + "delete-nolock": [ + "scripts/validation_heal/nolock.py" + ], + "native-lock-fence": [ + "scripts/validation_heal/nolock.py" + ] + }, + "gaps": [], + "live": { + "state": "pending", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation." + } + }, + { + "case_id": "heal.admin-option-contract", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 39197088183 + ] + } + }, + { + "case_id": "heal.disk-loss-and-repair", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 39899825839 + ] + } + }, + { + "case_id": "heal.disk-readonly-and-permission", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 39927051614 + ] + } + }, + { + "case_id": "heal.disk-replacement-formatted", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/versions.py", + "scripts/validation_heal/replace_drive.py", + "scripts/validation_heal/replacement_preflight.py", + "test_validation_heal.py", + "test_validation_heal_environment.py" + ], + "source_variants": { + "new-disk-data": [ + "scripts/validation_heal/versions.py" + ], + "new-disk-versioned": [ + "scripts/validation_heal/versions.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Environment admission: the current run-private endpoint is not an admitted exact mount and no reviewed mount-switch resource adapter is available. The source procedure and read-only predeployment refusal exist. Preserve old FAIL results; a new refused run is BLOCKED before fixture/deployment.", + "retained_runs": [ + 38084312163 + ], + "original_result_sha256": "be7474e0e78707362d32dd95e93aa090275afb3ce60dbe9b76dd0dbab6181916" + } + }, + { + "case_id": "heal.disk-replacement-unformatted", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/unformatted.py", + "scripts/validation_heal/replace_drive.py", + "scripts/validation_heal/replacement_preflight.py", + "test_validation_heal_unformatted.py", + "test_validation_heal_environment.py" + ], + "source_variants": { + "blank-unformatted": [ + "scripts/validation_heal/unformatted.py" + ], + "foreign-format": [ + "scripts/validation_heal/unformatted.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Environment admission: the current run-private endpoint is not an admitted exact mount and no reviewed mount-switch resource adapter is available. The source procedure and read-only predeployment refusal exist. Preserve old FAIL results; a new refused run is BLOCKED before fixture/deployment.", + "retained_runs": [ + 38087657754 + ], + "original_result_sha256": "ba4b7bab6407b7bbf1965ca8c54f5fbdbb705c79728f02375680b67a867c7e7f" + } + }, + { + "case_id": "heal.multiple-shard-loss", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 39843970314 + ] + } + }, + { + "case_id": "heal.read-quorum-boundary", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 39223475622 + ] + } + }, + { + "case_id": "heal.below-read-quorum", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38727279347 + ] + } + }, + { + "case_id": "heal.below-write-quorum", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 39915022106 + ] + } + }, + { + "case_id": "heal.node-outage-rejoin", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/node_outage.py", + "scripts/validation_heal/mrf_records.py", + "test_validation_heal.py" + ], + "source_variants": { + "one-node-stop": [ + "scripts/validation_heal/node_outage.py" + ], + "rejoin-with-new-versions": [ + "scripts/validation_heal/node_outage.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38202851374 + ], + "original_result_sha256": "f2f215cb5859a5b03db631537e8795c03a001d3b91e506ff678ae836b29836b4" + } + }, + { + "case_id": "heal.disk-and-node-flapping", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/flapping.py", + "scripts/validation_heal/replacement.py", + "scripts/validation_heal/mrf_records.py", + "test_validation_heal_flapping.py" + ], + "source_variants": { + "drive-flap-3": [ + "scripts/validation_heal/flapping.py" + ], + "node-flap-3": [ + "scripts/validation_heal/flapping.py" + ], + "interrupt-rebuild-again": [ + "scripts/validation_heal/flapping.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38104306522 + ], + "original_result_sha256": "3a41b150a39603eefce45f868588bdd2dca7b5136b1b13346803f968f22bd126" + } + }, + { + "case_id": "heal.missing-part-file", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38057373420 + ] + } + }, + { + "case_id": "heal.data-shard-bitrot", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 39281951263 + ] + } + }, + { + "case_id": "heal.parity-shard-bitrot", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 39286997859 + ] + } + }, + { + "case_id": "heal.truncated-and-zeroed-shard", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38203454300 + ] + } + }, + { + "case_id": "heal.missing-or-corrupt-xlmeta", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 39322217398 + ] + } + }, + { + "case_id": "heal.stale-version-rejoin", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38085554143 + ] + } + }, + { + "case_id": "heal.conflicting-subquorum-authority", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/authority_runner.py", + "scripts/validation_heal/authority.py", + "scripts/validation_heal/generation_fault.py", + "test_validation_heal_authority.py" + ], + "source_variants": { + "disjoint-version-quorums": [ + "scripts/validation_heal/authority_runner.py" + ], + "disjoint-incarnations": [ + "scripts/validation_heal/authority_runner.py" + ], + "native-authority-fence": [ + "scripts/validation_heal/authority_runner.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38119830051 + ], + "original_result_sha256": "ccc86e06eea4bf6146af9f2ed35ee2d89ec495ebf3f61b219a6f62f927909576" + } + }, + { + "case_id": "heal.metadata-payload-mismatch", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38235978152 + ] + } + }, + { + "case_id": "heal.foreign-format-and-unknown-schema", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/schema.py", + "scripts/validation_heal/schema_faults.py", + "test_validation_heal_schema.py" + ], + "source_variants": { + "foreign-disk": [ + "scripts/validation_heal/schema.py" + ], + "foreign-set": [ + "scripts/validation_heal/schema.py" + ], + "future-format": [ + "scripts/validation_heal/schema.py" + ], + "future-object-schema": [ + "scripts/validation_heal/schema.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38098069878 + ], + "original_result_sha256": "c831443b952fa70fe26e8315527895f49aa9089912f9fc21122ffc5467dcfd59" + } + }, + { + "case_id": "heal.dangling-and-orphan-safety", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38209676009 + ] + } + }, + { + "case_id": "heal.bucket-incarnation-recreation", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect. All source branches are present in the explicit diagnostic route; canonical admission remains independently blocked.", + "references": [ + "scripts/validation_heal/incarnation.py", + "scripts/validation_heal/incarnation_replay.py", + "scripts/validation_heal/incarnation_native.py", + "scripts/validation_heal/manager_native.py", + "test_validation_heal_incarnation.py", + "test_validation_heal_incarnation_replay.py", + "test_validation_heal_incarnation_native.py" + ], + "source_variants": { + "old-metadata-rejoin": [ + "scripts/validation_heal/incarnation.py" + ], + "old-checkpoint-replay": [ + "scripts/validation_heal/incarnation.py" + ], + "old-mrf-replay": [ + "scripts/validation_heal/incarnation.py" + ], + "native-incarnation-fence": [ + "scripts/validation_heal/incarnation.py" + ] + }, + "gaps": [], + "live": { + "state": "blocked", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation." + } + }, + { + "case_id": "heal.object-size-boundaries", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38067139720 + ] + } + }, + { + "case_id": "heal.completed-multipart-object", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38204339623 + ] + } + }, + { + "case_id": "heal.historical-versions", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/versions.py", + "test_validation_heal.py" + ], + "source_variants": { + "old-version-part": [ + "scripts/validation_heal/versions.py" + ], + "current-version-part": [ + "scripts/validation_heal/versions.py" + ], + "recursive-all-versions": [ + "scripts/validation_heal/versions.py" + ], + "multiple-version-targets": [ + "scripts/validation_heal/versions.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38059278988 + ], + "original_result_sha256": "9e7b278019acf80a56bcd395e2872ee6f227c5c0020c0b1be91246186441a9ee" + } + }, + { + "case_id": "heal.delete-marker-semantics", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/versions.py", + "scripts/validation_heal/markers.py", + "test_validation_heal.py" + ], + "source_variants": { + "current-marker": [ + "scripts/validation_heal/versions.py" + ], + "historical-marker": [ + "scripts/validation_heal/versions.py" + ], + "missing-marker-replica": [ + "scripts/validation_heal/versions.py" + ], + "stale-disk-marker": [ + "scripts/validation_heal/versions.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38204764944 + ], + "original_result_sha256": "33c0aa8fe51cc436a3e6fe9d8a948b570d06f336552500f1774ff0f84001a030" + } + }, + { + "case_id": "heal.versioning-suspended-null", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38061843290 + ] + } + }, + { + "case_id": "heal.object-metadata-and-tags", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38061589839 + ] + } + }, + { + "case_id": "heal.range-head-and-conditions", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38205235080 + ] + } + }, + { + "case_id": "heal.encrypted-objects", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38229589497 + ] + } + }, + { + "case_id": "heal.object-lock-retention", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38205854024 + ] + } + }, + { + "case_id": "heal.special-keys-and-pagination", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38207381816 + ] + } + }, + { + "case_id": "heal.bucket-metadata-integrity", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/bucket_metadata.py", + "scripts/validation_heal/bucket_records.py", + "test_validation_heal_bucket_metadata.py" + ], + "source_variants": { + "versioning-config": [ + "scripts/validation_heal/bucket_metadata.py" + ], + "bucket-policy-tags": [ + "scripts/validation_heal/bucket_metadata.py" + ], + "lock-config": [ + "scripts/validation_heal/bucket_metadata.py" + ], + "unknown-record-negative": [ + "scripts/validation_heal/bucket_metadata.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38096389824 + ], + "original_result_sha256": "92eb6b1c9dcfbfc5c2a6f8e7b896fed244e1d7dbd8b2cc510ac61cdd83bd2f31" + } + }, + { + "case_id": "heal.manual-normal-attribution", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38207956101 + ] + } + }, + { + "case_id": "heal.manual-deep-attribution", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38208251180 + ] + } + }, + { + "case_id": "heal.read-repair-attribution", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38237270640 + ] + } + }, + { + "case_id": "heal.scanner-cold-object-repair", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 39863505276 + ] + } + }, + { + "case_id": "heal.mrf-offline-write-replay", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/mrf_replay.py", + "scripts/validation_heal/mrf_records.py", + "test_validation_heal_mrf.py" + ], + "source_variants": { + "offline-put": [ + "scripts/validation_heal/mrf_replay.py" + ], + "offline-version-delete": [ + "scripts/validation_heal/mrf_replay.py" + ], + "offline-marker": [ + "scripts/validation_heal/mrf_replay.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38104630791 + ], + "original_result_sha256": "cda6c4dd785e025c16b1ed7b2b62df6168a31b8528e9f7f0fc89495d70818fed" + } + }, + { + "case_id": "heal.replacement-auto-rebuild", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/versions.py", + "scripts/validation_heal/replacement.py", + "test_validation_heal.py" + ], + "source_variants": { + "replacement-unversioned": [ + "scripts/validation_heal/versions.py" + ], + "replacement-versioned-marker": [ + "scripts/validation_heal/versions.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38085937937 + ], + "original_result_sha256": "4836929d6e21e88da89dcc6ae645ce3722554991d1fc01e53a9fbd0b03c6c167" + } + }, + { + "case_id": "heal.root-graceful-coordinator-restart", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38096950043 + ] + } + }, + { + "case_id": "heal.target-graceful-restart", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/versions.py", + "scripts/validation_heal/replacement.py", + "test_validation_heal.py" + ], + "source_variants": { + "target-partial-graceful": [ + "scripts/validation_heal/versions.py" + ], + "versioned-target-partial": [ + "scripts/validation_heal/versions.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38088240693 + ], + "original_result_sha256": "609a60b2de601fcb3a9bc703f5d0cce7bddfefc08b5c951fe3830a29698975d4" + } + }, + { + "case_id": "heal.target-process-crash", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/versions.py", + "scripts/validation_heal/replacement.py", + "test_validation_heal.py" + ], + "source_variants": { + "target-partial-sigkill": [ + "scripts/validation_heal/versions.py" + ], + "target-partial-sigkill-versioned": [ + "scripts/validation_heal/versions.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38088777413 + ], + "original_result_sha256": "404c026c77dc6b4eb87c132a316b700eea620659b8ff44c98648d0e30c5605c9" + } + }, + { + "case_id": "heal.coordinator-process-crash", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/coordinator_crash.py", + "scripts/validation_heal/replacement.py", + "test_validation_heal_coordinator_crash.py" + ], + "source_variants": { + "root-coordinator-kill": [ + "scripts/validation_heal/coordinator_crash.py" + ], + "mrf-coordinator-kill": [ + "scripts/validation_heal/coordinator_crash.py" + ], + "replacement-coordinator-kill": [ + "scripts/validation_heal/coordinator_crash.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38105011828 + ], + "original_result_sha256": "8100552121f4b50a621ebaa3012df968ec45b2e74038f49d53f34f085eed1fca" + } + }, + { + "case_id": "heal.full-cluster-restart", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38227684435 + ] + } + }, + { + "case_id": "heal.cancelled-task-restart", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38102031029 + ] + } + }, + { + "case_id": "heal.checkpoint-and-mrf-corruption", + "implementation": "implemented", + "basis": "The five source live corruption/write-failure/rescan branches retain their original byte, responsibility and recovery oracles. The native-persist-fence route now executes and independently verifies six ECStore directory-fsync regressions and composes them with the ten existing manager publication/restart regressions, with manifest-before-execution and proof-after-verification checks plus a same-candidate composition guard. Manager successful reload ordering, actual commit interruptions and ECStore CAS persistence failures retain their separate scopes; no exhaustive file-fsync or physical power-loss coverage is claimed. Missing immutable artifacts limit execution, independently of implementation completion.", + "references": [ + "scripts/validation_heal/persistence.py", + "scripts/validation_heal/persistence_mrf.py", + "scripts/validation_heal/journal_fault.py", + "scripts/validation_heal/manager_native.py", + "test_validation_heal_persistence.py", + "test_validation_heal_persistence_mrf.py", + "test_validation_heal_journal_fault.py", + "test_validation_heal_manager_native.py", + "scripts/validation_heal/persistence_native.py", + "scripts/validation_heal/manager_fixture.py", + "test_validation_heal_native.py" + ], + "source_variants": { + "checkpoint-truncate": [ + "scripts/validation_heal/persistence.py" + ], + "mrf-checksum": [ + "scripts/validation_heal/persistence.py" + ], + "future-schema": [ + "scripts/validation_heal/persistence.py" + ], + "stale-generation": [ + "scripts/validation_heal/persistence.py" + ], + "journal-write-failure": [ + "scripts/validation_heal/persistence.py" + ], + "native-persist-fence": [ + "scripts/validation_heal/persistence.py", + "scripts/validation_heal/manager_native.py", + "scripts/validation_heal/manager_fixture.py", + "scripts/validation_heal/persistence_native.py" + ] + }, + "gaps": [], + "live": { + "state": "blocked", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation." + } + }, + { + "case_id": "heal.race-unversioned-overwrite", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/race.py", + "scripts/validation_heal/race_native_evidence.py", + "scripts/validation_heal/race_io.py", + "test_validation_heal_race.py", + "test_validation_heal_race_native.py" + ], + "source_variants": { + "overwrite-vs-deep": [ + "scripts/validation_heal/race.py" + ], + "overwrite-vs-read-repair": [ + "scripts/validation_heal/race.py" + ], + "native-commit-cleanup": [ + "scripts/validation_heal/race.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Observed candidate PUT failure during overwrite-vs-deep. The Recorder retained ACK 000003 and refused to retry the ambiguous mutation. Recovery 40440199880 isolated the scene. Full live acceptance remains incomplete; this is not evidence of a missing overwrite test.", + "retained_runs": [ + 40439620403 + ] + } + }, + { + "case_id": "heal.race-versioned-put", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/race.py", + "scripts/validation_heal/version_race_runner.py", + "scripts/validation_heal/race_native_evidence.py", + "test_validation_heal_g02_source.py", + "test_validation_heal_race_native.py" + ], + "source_variants": { + "new-version-vs-historical-heal": [ + "scripts/validation_heal/race.py" + ], + "read-all-versions-during-heal": [ + "scripts/validation_heal/race.py" + ], + "native-version-fence": [ + "scripts/validation_heal/race.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Observed HTTP 503 ServiceUnavailable on the third versioned PUT, with server storage_error_code=Lock. The request is durably recorded with zero retries. Recovery 40444795627 isolated the scene. Exact root cause is not inferred from the status code alone.", + "retained_runs": [ + 40444207551 + ] + } + }, + { + "case_id": "heal.race-version-delete", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/race.py", + "scripts/validation_heal/version_race_runner.py", + "scripts/validation_heal/race_native_evidence.py", + "test_validation_heal_g03_source.py", + "test_validation_heal_race_native.py" + ], + "source_variants": { + "delete-old-version-vs-deep": [ + "scripts/validation_heal/race.py" + ], + "delete-current-version-vs-deep": [ + "scripts/validation_heal/race.py" + ], + "native-delete-fence": [ + "scripts/validation_heal/race.py" + ] + }, + "gaps": [], + "live": { + "state": "pending", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation." + } + }, + { + "case_id": "heal.race-markers-and-batch-delete", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/race.py", + "scripts/validation_heal/race_native_evidence.py", + "scripts/validation_heal/race_io.py", + "test_validation_heal_race.py", + "test_validation_heal_race_native.py" + ], + "source_variants": { + "create-marker": [ + "scripts/validation_heal/race.py" + ], + "delete-marker": [ + "scripts/validation_heal/race.py" + ], + "batch-version-delete": [ + "scripts/validation_heal/race.py" + ], + "unversioned-delete": [ + "scripts/validation_heal/race.py" + ], + "native-marker-fence": [ + "scripts/validation_heal/race.py" + ] + }, + "gaps": [], + "live": { + "state": "pending", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation." + } + }, + { + "case_id": "heal.race-multipart-complete-abort", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/race.py", + "scripts/validation_heal/mpu_race_runner.py", + "scripts/validation_heal/race_native_evidence.py", + "test_validation_heal_mpu_race_runner.py", + "test_validation_heal_race_native.py" + ], + "source_variants": { + "upload-continue": [ + "scripts/validation_heal/race.py" + ], + "complete": [ + "scripts/validation_heal/race.py" + ], + "abort": [ + "scripts/validation_heal/race.py" + ], + "native-mpu-fence": [ + "scripts/validation_heal/race.py" + ] + }, + "gaps": [], + "live": { + "state": "pending", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation." + } + }, + { + "case_id": "heal.race-copy-and-metadata-update", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 40427815826 + ] + } + }, + { + "case_id": "heal.race-ilm-expiry-and-transition", + "implementation": "environment-gated", + "basis": "Source procedure, controller contract, original assertions and capability refusal are implemented. This delivery keeps the case disabled by default; a supplied candidate-bound resource adapter is still needed for live execution.", + "references": [ + "scripts/validation_heal/environment_gate.py", + "scripts/validation_heal/environment.py", + "scripts/validation_heal/recipes.json", + "test_validation_heal_environment.py", + "test_validation_heal_environment_gate.py" + ], + "required_capabilities": [ + "dedicated_ilm_tier", + "real_expire_transition_restore_window", + "remote_version_mapping", + "isolated_rule_cleanup" + ], + "reason_code": "ENV_DEDICATED_ILM_TIER_WINDOW", + "gaps": [], + "live": { + "state": "disabled", + "finding": "需要隔离 ILM/tier 租户、真实 Expire/Transition/Restore 窗口和远端版本映射；共享 ILM 环境不能安全制造交叠窗口。" + } + }, + { + "case_id": "heal.race-replication-resync", + "implementation": "environment-gated", + "basis": "Source procedure, controller contract, original assertions and capability refusal are implemented. This delivery keeps the case disabled by default; a supplied candidate-bound resource adapter is still needed for live execution.", + "references": [ + "scripts/validation_heal/environment_gate.py", + "scripts/validation_heal/environment.py", + "scripts/validation_heal/recipes.json", + "test_validation_heal_environment.py", + "test_validation_heal_environment_gate.py" + ], + "required_capabilities": [ + "isolated_replication_pair", + "resync_control", + "cross_endpoint_version_mapping", + "replication_overlap_window" + ], + "reason_code": "ENV_ISOLATED_REPLICATION_PAIR", + "gaps": [], + "live": { + "state": "disabled", + "finding": "需要隔离的复制两端、resync 控制、跨端版本映射和 Heal/resync 实际交叠窗口；当前固定 fleet 没有独占复制对。" + } + }, + { + "case_id": "heal.slow-io-and-eio", + "implementation": "environment-gated", + "basis": "Source procedure, controller contract, original assertions and capability refusal are implemented. This delivery keeps the case disabled by default; a supplied candidate-bound resource adapter is still needed for live execution.", + "references": [ + "scripts/validation_heal/environment_gate.py", + "scripts/validation_heal/environment.py", + "scripts/validation_heal/recipes.json", + "test_validation_heal_environment.py", + "test_validation_heal_environment_gate.py" + ], + "required_capabilities": [ + "dedicated_faultfs_or_device", + "slow_io_short_read_eio", + "namespace_isolation", + "fault_layer_restore" + ], + "reason_code": "ENV_DEDICATED_FAULT_IO", + "gaps": [], + "live": { + "state": "disabled", + "finding": "需要专用 faultfs 或故障设备、可恢复的 slow/short-read/EIO 注入和 namespace 隔离；共享盘模拟会污染其他运行。" + } + }, + { + "case_id": "heal.enospc-inodes-and-quota", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/capacity.py", + "scripts/validation_heal/bounded_volume.py", + "test_validation_heal_capacity.py" + ], + "source_variants": { + "bounded-volume-enospc": [ + "scripts/validation_heal/capacity.py" + ], + "bounded-inode-exhaustion": [ + "scripts/validation_heal/capacity.py" + ], + "bucket-quota": [ + "scripts/validation_heal/capacity.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38106408335 + ], + "original_result_sha256": "f779fa1f4a24e2bc4ebbc3b07b447a2134e0fca4a44faedd5be42b461c7520df" + } + }, + { + "case_id": "heal.network-partition-and-lock-quorum", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/network.py", + "scripts/validation_heal/network_gate.py", + "scripts/validation_heal/network_load.py", + "test_validation_heal_network.py" + ], + "source_variants": { + "minority-partition": [ + "scripts/validation_heal/network.py" + ], + "read-boundary": [ + "scripts/validation_heal/network.py" + ], + "write-boundary": [ + "scripts/validation_heal/network.py" + ], + "lock-boundary": [ + "scripts/validation_heal/network.py" + ], + "rejoin": [ + "scripts/validation_heal/network.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38117058670 + ], + "original_result_sha256": "aac97fc7d2b328c628f9f42f2a6b426990ed20442015e1801ade7a18c3c5a61a" + } + }, + { + "case_id": "heal.queue-pressure-and-admission", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 39354651483 + ] + } + }, + { + "case_id": "heal.lock-timeout-and-caller-abort", + "implementation": "implemented", + "basis": "The source-plan entry serially executes real timeout and queued-successor controllers on H1/H2 with distinct namespaces, original request/ACK evidence chains, peer version/content and physical integrity checks, exact server lock identities and bounded FD finalization. Before deployment it composes one immutable successful artifact's I05 owner/unlink/stale, G02 explicit-version append and G03 explicit-version delete native subsets. The latter two provide real final-rename reconstruction barriers, foreground ACK/physical tail drain, exact typed outcomes and complete surviving body/version/dataDir/shard authority in 4/2 and 16/4. Coordinator clock-domain ordering binds actual TCP disconnect independently from native ownership proof; local guard release never claims distributed release. First failure retains the original scene and stops subsequent instances. Full independent source-plan live acceptance remains pending; product PASS is not an implementation prerequisite.", + "references": [ + "scripts/validation_heal/lock_timeout_runner.py", + "scripts/validation_heal/lock_timeout.py", + "scripts/validation_heal/caller_abort.py", + "scripts/validation_heal/lock_observer.py", + "scripts/validation_heal/lock_successor.py", + "scripts/validation_heal/native_regressions.py", + "test_validation_heal_lock_timeout_runner.py", + "test_validation_heal_lock_timeout.py", + "test_validation_heal_caller_abort.py", + "test_validation_heal_lock_observer.py", + "test_validation_heal_lock_successor.py", + "scripts/validation_heal/race_native.py", + "scripts/validation_heal/race_native_evidence.py", + "scripts/validation_heal/race_native_contracts.json", + "test_validation_heal_race_native.py", + "scripts/validation_heal/lock_source.py", + "test_validation_heal_lock_source.py" + ], + "source_variants": { + "object-lock-timeout": [ + "scripts/validation_heal/lock_source.py", + "scripts/validation_heal/lock_timeout_runner.py", + "scripts/validation_heal/lock_timeout.py" + ], + "caller-disconnect": [ + "scripts/validation_heal/lock_source.py", + "scripts/validation_heal/lock_timeout_runner.py", + "scripts/validation_heal/lock_successor.py", + "scripts/validation_heal/lock_observer.py" + ], + "stale-read-repair": [ + "scripts/validation_heal/lock_source.py", + "scripts/validation_heal/race_native.py", + "scripts/validation_heal/race_native_evidence.py", + "scripts/validation_heal/race_native_contracts.json" + ], + "native-lock-owner-fence": [ + "scripts/validation_heal/lock_source.py", + "scripts/validation_heal/race_native_contracts.json" + ] + }, + "gaps": [], + "live": { + "state": "blocked", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation." + } + }, + { + "case_id": "heal.kms-and-tier-outage", + "implementation": "environment-gated", + "basis": "Source procedure, controller contract, original assertions and capability refusal are implemented. This delivery keeps the case disabled by default; a supplied candidate-bound resource adapter is still needed for live execution.", + "references": [ + "scripts/validation_heal/environment_gate.py", + "scripts/validation_heal/environment.py", + "scripts/validation_heal/recipes.json", + "test_validation_heal_environment.py", + "test_validation_heal_environment_gate.py" + ], + "required_capabilities": [ + "dedicated_kms_tier_tenant", + "dependency_network_isolation", + "key_redaction", + "tier_version_mapping" + ], + "reason_code": "ENV_DEDICATED_KMS_TIER_TENANT", + "gaps": [], + "live": { + "state": "disabled", + "finding": "需要专用 KMS/tier 租户、依赖网络隔离和安全的 key/tier 版本证据；不能在共享依赖上全局断网。" + } + }, + { + "case_id": "heal.status-partial-disabled-unknown", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/status.py", + "test_validation_heal.py" + ], + "source_variants": { + "enabled-idle": [ + "scripts/validation_heal/status.py" + ], + "disabled": [ + "scripts/validation_heal/status.py" + ], + "uninitialized": [ + "scripts/validation_heal/status.py" + ], + "missing-peer": [ + "scripts/validation_heal/status.py" + ], + "recovered-complete": [ + "scripts/validation_heal/status.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "The startup listener returns 503 until runtime initialization, then status reports idle. The required externally visible 200/uninitialized source window was not observed. Preserve the original oracle and NOT_OBSERVED branch; implementation completion does not certify product compatibility.", + "retained_runs": [ + 38209400619, + 39929436131 + ], + "original_result_sha256": "817e04d80bf59d0ce369573b2f6500eb2ff0cc47662c830e3bb524e073d628fa" + } + }, + { + "case_id": "heal.outcome-counters-and-error-truth", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect. All source branches are present in the explicit diagnostic route; canonical admission remains independently blocked.", + "references": [ + "scripts/validation_heal/outcome.py", + "scripts/validation_heal/outcome_grace.py", + "scripts/validation_heal/outcome_recovery.py", + "test_validation_heal_outcome.py", + "test_validation_heal_outcome_grace.py", + "test_validation_heal_outcome_recovery.py" + ], + "source_variants": { + "mixed-healthy-repaired-failed": [ + "scripts/validation_heal/outcome.py" + ], + "dry-run": [ + "scripts/validation_heal/outcome.py" + ], + "deferred": [ + "scripts/validation_heal/outcome.py" + ], + "readonly-terminal-failure": [ + "scripts/validation_heal/outcome.py" + ], + "unknown-negative": [ + "scripts/validation_heal/outcome.py" + ] + }, + "gaps": [], + "live": { + "state": "blocked", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation." + } + }, + { + "case_id": "heal.incremental-cursor-and-truncation", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38089238767 + ] + } + }, + { + "case_id": "heal.repeat-idempotency", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38059567206 + ] + } + }, + { + "case_id": "heal.performance-abba", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/performance.py", + "test_validation_heal_performance.py" + ], + "source_variants": { + "abba-repeat-1": [ + "scripts/validation_heal/performance.py" + ], + "abba-repeat-2": [ + "scripts/validation_heal/performance.py" + ], + "abba-repeat-3": [ + "scripts/validation_heal/performance.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38195131282 + ], + "original_result_sha256": "f3c6ec359b4ac5e3d8dc3776ddd75c760303a672c7ffed1385f482d3c79c3ac8" + } + }, + { + "case_id": "scanner.usage-convergence", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38042200106 + ] + } + }, + { + "case_id": "heal.mixed-version-upgrade-rollback", + "implementation": "environment-gated", + "basis": "Source procedure, controller contract, original assertions and capability refusal are implemented. This delivery keeps the case disabled by default; a supplied candidate-bound resource adapter is still needed for live execution.", + "references": [ + "scripts/validation_heal/environment_gate.py", + "scripts/validation_heal/environment.py", + "scripts/validation_heal/recipes.json", + "test_validation_heal_environment.py", + "test_validation_heal_environment_gate.py" + ], + "required_capabilities": [ + "immutable_release_pair", + "mixed_version_peer_status", + "rolling_upgrade_rollback", + "rollback_compatibility_matrix" + ], + "reason_code": "ENV_IMMUTABLE_RELEASE_PAIR", + "gaps": [], + "live": { + "state": "disabled", + "finding": "需要两个已固定且相互兼容的 RustFS release、混合版本 peer 状态和经过审核的回滚兼容矩阵；当前 auto-testing Heal 节点没有这组不可变双版本资源。" + } + }, + { + "case_id": "heal.dedicated-host-forced-restart", + "implementation": "environment-gated", + "basis": "Source procedure, controller contract, original assertions and capability refusal are implemented. This delivery keeps the case disabled by default; a supplied candidate-bound resource adapter is still needed for live execution.", + "references": [ + "scripts/validation_heal/environment_gate.py", + "scripts/validation_heal/environment.py", + "scripts/validation_heal/recipes.json", + "test_validation_heal_environment.py", + "test_validation_heal_environment_gate.py" + ], + "required_capabilities": [ + "dedicated_h6_host", + "platform_power_cycle", + "out_of_band_recovery", + "boot_id_evidence" + ], + "reason_code": "ENV_DEDICATED_H6_HOST", + "gaps": [], + "live": { + "state": "disabled", + "finding": "需要 H6 专用主机、平台强制停启权限、带外恢复和 boot_id/平台事件证据；共享 Heal 节点不能证明真实电源循环。" + } + }, + { + "case_id": "heal.with-expansion-rebalance", + "implementation": "environment-gated", + "basis": "Source procedure, controller contract, original assertions and capability refusal are implemented. This delivery keeps the case disabled by default; a supplied candidate-bound resource adapter is still needed for live execution.", + "references": [ + "scripts/validation_heal/environment_gate.py", + "scripts/validation_heal/environment.py", + "scripts/validation_heal/recipes.json", + "test_validation_heal_environment.py", + "test_validation_heal_environment_gate.py" + ], + "required_capabilities": [ + "h5_pool_expansion", + "rebalance_task", + "per_object_pool_set_mapping", + "retired_scanner_evidence" + ], + "reason_code": "ENV_H5_EXPANSION_REBALANCE", + "gaps": [], + "live": { + "state": "disabled", + "finding": "需要 H5 pool 扩容、rebalance 和 Heal 同时活动的完整拓扑，并逐对象保存 pool/set 迁移证据；当前 auto-testing 未绑定扩容前后布局与专用操作窗口。" + } + }, + { + "case_id": "heal.with-rebalance-stop-resume", + "implementation": "environment-gated", + "basis": "Source procedure, controller contract, original assertions and capability refusal are implemented. This delivery keeps the case disabled by default; a supplied candidate-bound resource adapter is still needed for live execution.", + "references": [ + "scripts/validation_heal/environment_gate.py", + "scripts/validation_heal/environment.py", + "scripts/validation_heal/recipes.json", + "test_validation_heal_environment.py", + "test_validation_heal_environment_gate.py" + ], + "required_capabilities": [ + "h5_rebalance", + "rebalance_stop_resume", + "independent_task_identity", + "service_restart_control" + ], + "reason_code": "ENV_H5_REBALANCE_LIFECYCLE", + "gaps": [], + "live": { + "state": "disabled", + "finding": "需要可观测的 H5 rebalance、停止/恢复/服务重启控制及独立 task 身份；当前 auto-testing 未绑定这两类任务的独立身份和操作窗口。" + } + }, + { + "case_id": "heal.with-decommission", + "implementation": "environment-gated", + "basis": "Source procedure, controller contract, original assertions and capability refusal are implemented. This delivery keeps the case disabled by default; a supplied candidate-bound resource adapter is still needed for live execution.", + "references": [ + "scripts/validation_heal/environment_gate.py", + "scripts/validation_heal/environment.py", + "scripts/validation_heal/recipes.json", + "test_validation_heal_environment.py", + "test_validation_heal_environment_gate.py" + ], + "required_capabilities": [ + "h5_decommission", + "survivor_pool", + "decommission_write_guard", + "movement_ack_physical_proof" + ], + "reason_code": "ENV_H5_DECOMMISSION", + "gaps": [], + "live": { + "state": "disabled", + "finding": "需要 H5 pool decommission、survivor 目标、写入限制和移动 ACK/物理权威证据；当前 auto-testing 未绑定本例退役前后拓扑和恢复方案。" + } + }, + { + "case_id": "heal.with-decommission-cancel-clear", + "implementation": "environment-gated", + "basis": "Source procedure, controller contract, original assertions and capability refusal are implemented. This delivery keeps the case disabled by default; a supplied candidate-bound resource adapter is still needed for live execution.", + "references": [ + "scripts/validation_heal/environment_gate.py", + "scripts/validation_heal/environment.py", + "scripts/validation_heal/recipes.json", + "test_validation_heal_environment.py", + "test_validation_heal_environment_gate.py" + ], + "required_capabilities": [ + "h5_decommission", + "cancel_clear_resume", + "capacity_lock_retry_window", + "generation_bound_ack_ledger" + ], + "reason_code": "ENV_H5_DECOMMISSION_CANCEL", + "gaps": [], + "live": { + "state": "disabled", + "finding": "需要退役取消/清理/新代次恢复以及 #2423 所需容量锁重试窗口；普通取消结果不能替代这个临界点。" + } + }, + { + "case_id": "heal.after-retired-pool-removal", + "implementation": "environment-gated", + "basis": "Source procedure, controller contract, original assertions and capability refusal are implemented. This delivery keeps the case disabled by default; a supplied candidate-bound resource adapter is still needed for live execution.", + "references": [ + "scripts/validation_heal/environment_gate.py", + "scripts/validation_heal/environment.py", + "scripts/validation_heal/recipes.json", + "test_validation_heal_environment.py", + "test_validation_heal_environment_gate.py" + ], + "required_capabilities": [ + "h5_retired_pool_removal", + "source_offline_proof", + "survivor_authority", + "retired_pool_write_guard" + ], + "reason_code": "ENV_H5_RETIRED_POOL_REMOVAL", + "gaps": [], + "live": { + "state": "disabled", + "finding": "需要完整迁移后移除 retired pool、源保持离线和 survivor-only 访问；当前 auto-testing 未提供本例源持续离线和 survivor-only 的绑定证据。" + } + }, + { + "case_id": "heal.multi-pool-target-isolation", + "implementation": "implemented", + "basis": "Source branches have execution paths and strict original oracles in the referenced controller/helpers. Contract tests cover those paths; this source review is not a claim that every live branch passed or that an unattributed failure is a product defect.", + "references": [ + "scripts/validation_heal/multi_pool.py", + "scripts/validation_heal/multi_pool_layout.py", + "test_validation_heal_multi_pool.py" + ], + "source_variants": { + "selected-pool": [ + "scripts/validation_heal/multi_pool.py" + ], + "selected-set": [ + "scripts/validation_heal/multi_pool.py" + ], + "unselected-control-fault": [ + "scripts/validation_heal/multi_pool.py" + ] + }, + "gaps": [], + "live": { + "state": "retained-failure", + "finding": "Retained failure or incomplete live evidence. Product behavior, environment and controller responsibility must be attributed separately; a FAIL alone does not demonstrate a missing test implementation.", + "retained_runs": [ + 38119207583 + ], + "original_result_sha256": "e2c1a70ed32cf4db8815dfe3f397d82bde590489e8011fadf6d8ae6c2c90b967" + } + }, + { + "case_id": "heal.ec84-multi-set-restart", + "implementation": "environment-gated", + "basis": "Source procedure, controller contract, original assertions and capability refusal are implemented. This delivery keeps the case disabled by default; a supplied candidate-bound resource adapter is still needed for live execution.", + "references": [ + "scripts/validation_heal/environment_gate.py", + "scripts/validation_heal/environment.py", + "scripts/validation_heal/recipes.json", + "test_validation_heal_environment.py", + "test_validation_heal_environment_gate.py" + ], + "required_capabilities": [ + "h3_ec84_12_independent_disks", + "h4_ec84_24_independent_endpoints", + "per_set_identity", + "per_set_restart_crash_evidence" + ], + "reason_code": "ENV_EC84_MULTI_SET_TOPOLOGY", + "gaps": [], + "live": { + "state": "disabled", + "finding": "需要 H3 的 12 个独立盘和 H4 的至少 24 个独立 endpoint，并按 set 分别记录重启/崩溃窗口；当前 H4 资源不可用。" + } + }, + { + "case_id": "heal.long-soak", + "implementation": "environment-gated", + "basis": "Source procedure, controller contract, original assertions and capability refusal are implemented. This delivery keeps the case disabled by default; a supplied candidate-bound resource adapter is still needed for live execution.", + "references": [ + "scripts/validation_heal/environment_gate.py", + "scripts/validation_heal/environment.py", + "scripts/validation_heal/recipes.json", + "test_validation_heal_environment.py", + "test_validation_heal_environment_gate.py" + ], + "required_capabilities": [ + "dedicated_soak_lane", + "wall_clock_24h_72h", + "resource_budget_monitoring", + "scheduled_fault_recovery" + ], + "reason_code": "ENV_DEDICATED_LONG_SOAK_LANE", + "gaps": [], + "live": { + "state": "disabled", + "finding": "必须在独占 H2/H5 lane 真实运行 24 小时和 72 小时，包含限速、监控、到期回收和完整资源预算；短跑不能替代 soak 证据。" + } + }, + { + "case_id": "scanner.status-readiness", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38044125099 + ] + } + }, + { + "case_id": "ilm.configuration-roundtrip", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38044956285 + ] + } + }, + { + "case_id": "ilm.invalid-configuration", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38045047518 + ] + } + }, + { + "case_id": "ilm.current-expiration", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38045154019 + ] + } + }, + { + "case_id": "ilm.tag-filter-expiration", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38045412209 + ] + } + }, + { + "case_id": "ilm.disabled-rule", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38045496042 + ] + } + }, + { + "case_id": "ilm.rule-removal", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38045714246 + ] + } + }, + { + "case_id": "ilm.versioned-current-expiration", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38051148291 + ] + } + }, + { + "case_id": "ilm.noncurrent-expiration", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38050040606 + ] + } + }, + { + "case_id": "ilm.newer-noncurrent-retention", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38049965929 + ] + } + }, + { + "case_id": "ilm.delete-all-versions", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38049740183 + ] + } + }, + { + "case_id": "ilm.delete-marker-expiration", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38049817172 + ] + } + }, + { + "case_id": "ilm.multipart-expiration", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38049890130 + ] + } + }, + { + "case_id": "ilm.transition-and-restore", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38050720293 + ] + } + }, + { + "case_id": "ilm.restart-persistence", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38050434607 + ] + } + }, + { + "case_id": "bucket-replication.source-versioning-required", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38049256196 + ] + } + }, + { + "case_id": "bucket-replication.target-versioning-required", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38107763760 + ] + } + }, + { + "case_id": "bucket-replication.configuration-roundtrip", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38107813352 + ] + } + }, + { + "case_id": "bucket-replication.put-object", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38107862949 + ] + } + }, + { + "case_id": "bucket-replication.sync-put-object", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38107913366 + ] + } + }, + { + "case_id": "bucket-replication.prefix-filter", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38107971953 + ] + } + }, + { + "case_id": "bucket-replication.tag-filter", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38108042411 + ] + } + }, + { + "case_id": "bucket-replication.zero-byte-special-key", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38108111726 + ] + } + }, + { + "case_id": "bucket-replication.multipart-object", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109163901 + ] + } + }, + { + "case_id": "bucket-replication.copy-object", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109217115 + ] + } + }, + { + "case_id": "bucket-replication.delete-marker", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109278883 + ] + } + }, + { + "case_id": "bucket-replication.version-delete", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109339732 + ] + } + }, + { + "case_id": "bucket-replication.existing-object-enabled", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109405912 + ] + } + }, + { + "case_id": "bucket-replication.existing-object-disabled", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109466432 + ] + } + }, + { + "case_id": "bucket-replication.target-check", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109538351 + ] + } + }, + { + "case_id": "bucket-replication.status-observability", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109591289 + ] + } + }, + { + "case_id": "bucket-replication.outage-recovery", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109645139 + ] + } + }, + { + "case_id": "bucket-replication.source-restart-recovery", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109704025 + ] + } + }, + { + "case_id": "site-replication.unconfigured-info", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109764474 + ] + } + }, + { + "case_id": "site-replication.invalid-credentials", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109818973 + ] + } + }, + { + "case_id": "site-replication.add-and-info", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109868478 + ] + } + }, + { + "case_id": "site-replication.status-observability", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109917609 + ] + } + }, + { + "case_id": "site-replication.bucket-create", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38109969984 + ] + } + }, + { + "case_id": "site-replication.object-bidirectional", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110029711 + ] + } + }, + { + "case_id": "site-replication.versioning-metadata", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110091819 + ] + } + }, + { + "case_id": "site-replication.lifecycle-expiry-toggle", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110141316 + ] + } + }, + { + "case_id": "site-replication.bucket-policy-metadata", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110223862 + ] + } + }, + { + "case_id": "site-replication.object-tags", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110289545 + ] + } + }, + { + "case_id": "site-replication.delete-convergence", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110361943 + ] + } + }, + { + "case_id": "site-replication.iam-user-policy", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110422925 + ] + } + }, + { + "case_id": "site-replication.iam-group-policy", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110479224 + ] + } + }, + { + "case_id": "site-replication.service-account", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110530482 + ] + } + }, + { + "case_id": "site-replication.peer-edit", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110586469 + ] + } + }, + { + "case_id": "site-replication.resync", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110637512 + ] + } + }, + { + "case_id": "site-replication.outage-recovery", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110690366 + ] + } + }, + { + "case_id": "site-replication.restart-persistence", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110779315 + ] + } + }, + { + "case_id": "site-replication.remove-all", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110839532 + ] + } + }, + { + "case_id": "site-replication.service-account-rotation", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38110909474 + ] + } + }, + { + "case_id": "kms.sse-kms-round-trip", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38042359938 + ] + } + }, + { + "case_id": "tiering.transition-and-restore", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38042759074 + ] + } + }, + { + "case_id": "events.delivery", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38046470999 + ] + } + }, + { + "case_id": "audit.request-correlation", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38050138583 + ] + } + }, + { + "case_id": "iam.policy-enforcement", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38043548625 + ] + } + }, + { + "case_id": "oidc.login-and-session", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38051425083 + ] + } + }, + { + "case_id": "s3.basic-object-round-trip", + "implementation": "implemented", + "basis": "Independent complete live acceptance already exercised the source procedure, assertions, evidence and finalization; accepted runs retain their own candidate/helper provenance.", + "references": [ + "scripts/run_validation_case.sh" + ], + "gaps": [], + "live": { + "state": "accepted", + "basis": "independent-complete-live-audit", + "runs": [ + 38042083422 + ] + } + } + ], + "review_notes": "All source implementation gaps recorded in this review are closed. See docs/validation-migration-implementation-review.md for source semantics, exact candidate prerequisites and separate native/fleet proof scopes. Counts describe the accepted migration delivery scope; the 13 default-disabled environment procedures still require reviewed external resource adapters. Accepted live results remain historical independent audits across multiple candidates, never a single-candidate product certification. No historical FAIL is rewritten." +}